Move to a single queue to serialize requests within a tfm. When
testing using IPSec with a large number of network connections
the per cpu tfm queuing logic was not working properly.
Signed-off-by: Tom Lendacky <thomas.lendacky@xxxxxxx>
---
drivers/crypto/ccp/ccp-crypto-main.c | 164 ++++++++++------------------------
1 file changed, 48 insertions(+), 116 deletions(-)
diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c
index b3f22b0..010fded 100644
--- a/drivers/crypto/ccp/ccp-crypto-main.c
+++ b/drivers/crypto/ccp/ccp-crypto-main.c
@@ -38,23 +38,20 @@ MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value");
static LIST_HEAD(hash_algs);
static LIST_HEAD(cipher_algs);
-/* For any tfm, requests for that tfm on the same CPU must be returned
- * in the order received. With multiple queues available, the CCP can
- * process more than one cmd at a time. Therefore we must maintain
- * a cmd list to insure the proper ordering of requests on a given tfm/cpu
- * combination.
+/* For any tfm, requests for that tfm must be returned on the order
+ * received. With multiple queues available, the CCP can process more
+ * than one cmd at a time. Therefore we must maintain a cmd list to insure
+ * the proper ordering of requests on a given tfm.
*/
-struct ccp_crypto_cpu_queue {
+struct ccp_crypto_queue {
struct list_head cmds;
struct list_head *backlog;
unsigned int cmd_count;
};
-#define CCP_CRYPTO_MAX_QLEN 50
+#define CCP_CRYPTO_MAX_QLEN 100
-struct ccp_crypto_percpu_queue {
- struct ccp_crypto_cpu_queue __percpu *cpu_queue;
-};
-static struct ccp_crypto_percpu_queue req_queue;
+static struct ccp_crypto_queue req_queue;
+static spinlock_t req_queue_lock;
struct ccp_crypto_cmd {
struct list_head entry;
@@ -71,8 +68,6 @@ struct ccp_crypto_cmd {
/* Used for held command processing to determine state */
int ret;
-
- int cpu;
};
struct ccp_crypto_cpu {
@@ -91,25 +86,21 @@ static inline bool ccp_crypto_success(int err)
return true;
}
-/*
- * ccp_crypto_cmd_complete must be called while running on the appropriate
- * cpu and the caller must have done a get_cpu to disable preemption
- */
static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog)
{
- struct ccp_crypto_cpu_queue *cpu_queue;
struct ccp_crypto_cmd *held = NULL, *tmp;
+ unsigned long flags;
*backlog = NULL;
- cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
+ spin_lock_irqsave(&req_queue_lock, flags);
/* Held cmds will be after the current cmd in the queue so start
* searching for a cmd with a matching tfm for submission.
*/
tmp = crypto_cmd;
- list_for_each_entry_continue(tmp, &cpu_queue->cmds, entry) {
+ list_for_each_entry_continue(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm)
continue;
held = tmp;
@@ -120,47 +111,45 @@ static struct ccp_crypto_cmd *ccp_crypto_cmd_complete(
* Because cmds can be executed from any point in the cmd list
* special precautions have to be taken when handling the backlog.
*/
- if (cpu_queue->backlog != &cpu_queue->cmds) {
+ if (req_queue.backlog != &req_queue.cmds) {
/* Skip over this cmd if it is the next backlog cmd */
- if (cpu_queue->backlog == &crypto_cmd->entry)
- cpu_queue->backlog = crypto_cmd->entry.next;
+ if (req_queue.backlog == &crypto_cmd->entry)
+ req_queue.backlog = crypto_cmd->entry.next;
- *backlog = container_of(cpu_queue->backlog,
+ *backlog = container_of(req_queue.backlog,
struct ccp_crypto_cmd, entry);
- cpu_queue->backlog = cpu_queue->backlog->next;
+ req_queue.backlog = req_queue.backlog->next;
/* Skip over this cmd if it is now the next backlog cmd */
- if (cpu_queue->backlog == &crypto_cmd->entry)
- cpu_queue->backlog = crypto_cmd->entry.next;
+ if (req_queue.backlog == &crypto_cmd->entry)
+ req_queue.backlog = crypto_cmd->entry.next;
}
/* Remove the cmd entry from the list of cmds */
- cpu_queue->cmd_count--;
+ req_queue.cmd_count--;
list_del(&crypto_cmd->entry);
+ spin_unlock_irqrestore(&req_queue_lock, flags);
+
return held;
}
-static void ccp_crypto_complete_on_cpu(struct work_struct *work)
+static void ccp_crypto_complete(void *data, int err)
{
- struct ccp_crypto_cpu *cpu_work =
- container_of(work, struct ccp_crypto_cpu, work);
- struct ccp_crypto_cmd *crypto_cmd = cpu_work->crypto_cmd;
+ struct ccp_crypto_cmd *crypto_cmd = data;
struct ccp_crypto_cmd *held, *next, *backlog;
struct crypto_async_request *req = crypto_cmd->req;
struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm);
- int cpu, ret;
-
- cpu = get_cpu();
+ int ret;
- if (cpu_work->err == -EINPROGRESS) {
+ if (err == -EINPROGRESS) {
/* Only propogate the -EINPROGRESS if necessary */
if (crypto_cmd->ret == -EBUSY) {
crypto_cmd->ret = -EINPROGRESS;
req->complete(req, -EINPROGRESS);
}
- goto e_cpu;
+ return;
}
/* Operation has completed - update the queue before invoking
@@ -178,7 +167,7 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
req->complete(req, -EINPROGRESS);
/* Completion callbacks */
- ret = cpu_work->err;
+ ret = err;
if (ctx->complete)
ret = ctx->complete(req, ret);
req->complete(req, ret);
@@ -203,52 +192,28 @@ static void ccp_crypto_complete_on_cpu(struct work_struct *work)
}
kfree(crypto_cmd);
-
-e_cpu:
- put_cpu();
-
- complete(&cpu_work->completion);
-}
-
-static void ccp_crypto_complete(void *data, int err)
-{
- struct ccp_crypto_cmd *crypto_cmd = data;
- struct ccp_crypto_cpu cpu_work;
-
- INIT_WORK(&cpu_work.work, ccp_crypto_complete_on_cpu);
- init_completion(&cpu_work.completion);
- cpu_work.crypto_cmd = crypto_cmd;
- cpu_work.err = err;
-
- schedule_work_on(crypto_cmd->cpu, &cpu_work.work);
-
- /* Keep the completion call synchronous */
- wait_for_completion(&cpu_work.completion);
}
static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
{
- struct ccp_crypto_cpu_queue *cpu_queue;
struct ccp_crypto_cmd *active = NULL, *tmp;
- int cpu, ret;
-
- cpu = get_cpu();
- crypto_cmd->cpu = cpu;
+ unsigned long flags;
+ int ret;
- cpu_queue = this_cpu_ptr(req_queue.cpu_queue);
+ spin_lock_irqsave(&req_queue_lock, flags);
/* Check if the cmd can/should be queued */
- if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
+ if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY;
if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG))
- goto e_cpu;
+ goto e_lock;
}
/* Look for an entry with the same tfm. If there is a cmd
- * with the same tfm in the list for this cpu then the current
- * cmd cannot be submitted to the CCP yet.
+ * with the same tfm in the list then the current cmd cannot
+ * be submitted to the CCP yet.
*/
- list_for_each_entry(tmp, &cpu_queue->cmds, entry) {
+ list_for_each_entry(tmp, &req_queue.cmds, entry) {
if (crypto_cmd->tfm != tmp->tfm)
continue;
active = tmp;
@@ -259,21 +224,21 @@ static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd)
if (!active) {
ret = ccp_enqueue_cmd(crypto_cmd->cmd);
if (!ccp_crypto_success(ret))
- goto e_cpu;
+ goto e_lock;
}
- if (cpu_queue->cmd_count >= CCP_CRYPTO_MAX_QLEN) {
+ if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) {
ret = -EBUSY;
- if (cpu_queue->backlog == &cpu_queue->cmds)
- cpu_queue->backlog = &crypto_cmd->entry;
+ if (req_queue.backlog == &req_queue.cmds)
+ req_queue.backlog = &crypto_cmd->entry;
}
crypto_cmd->ret = ret;
- cpu_queue->cmd_count++;
- list_add_tail(&crypto_cmd->entry, &cpu_queue->cmds);
+ req_queue.cmd_count++;
+ list_add_tail(&crypto_cmd->entry, &req_queue.cmds);
-e_cpu:
- put_cpu();
+e_lock:
+ spin_unlock_irqrestore(&req_queue_lock, flags);
return ret;
}
@@ -387,50 +352,18 @@ static void ccp_unregister_algs(void)
}
}
-static int ccp_init_queues(void)
-{
- struct ccp_crypto_cpu_queue *cpu_queue;
- int cpu;
-
- req_queue.cpu_queue = alloc_percpu(struct ccp_crypto_cpu_queue);
- if (!req_queue.cpu_queue)
- return -ENOMEM;
-
- for_each_possible_cpu(cpu) {
- cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
- INIT_LIST_HEAD(&cpu_queue->cmds);
- cpu_queue->backlog = &cpu_queue->cmds;
- cpu_queue->cmd_count = 0;
- }
-
- return 0;
-}
-
-static void ccp_fini_queue(void)
-{
- struct ccp_crypto_cpu_queue *cpu_queue;
- int cpu;
-
- for_each_possible_cpu(cpu) {
- cpu_queue = per_cpu_ptr(req_queue.cpu_queue, cpu);
- BUG_ON(!list_empty(&cpu_queue->cmds));
- }
- free_percpu(req_queue.cpu_queue);
-}
-
static int ccp_crypto_init(void)
{
int ret;
- ret = ccp_init_queues();
- if (ret)
- return ret;
+ spin_lock_init(&req_queue_lock);
+ INIT_LIST_HEAD(&req_queue.cmds);
+ req_queue.backlog = &req_queue.cmds;
+ req_queue.cmd_count = 0;
ret = ccp_register_algs();
- if (ret) {
+ if (ret)
ccp_unregister_algs();
- ccp_fini_queue();
- }
return ret;
}
@@ -438,7 +371,6 @@ static int ccp_crypto_init(void)
static void ccp_crypto_exit(void)
{
ccp_unregister_algs();
- ccp_fini_queue();
}
module_init(ccp_crypto_init);
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
