This patch adds a notifier to the SHA1 multi-buffer algorithm
when CPU is giong idle, so it can take advantage of the available
CPU power to flush out any partially completed jobs. This
will eliminate possible extended latency in the multi-buffer
algorithm.
Signed-off-by: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx>
---
arch/x86/crypto/sha-mb/sha1_mb.c | 61 ++++++++++++++++++++++++++++++++++++++++
include/crypto/mcryptd.h | 1 +
2 files changed, 62 insertions(+)
diff --git a/arch/x86/crypto/sha-mb/sha1_mb.c b/arch/x86/crypto/sha-mb/sha1_mb.c
index 9c5feae..fd9b219 100644
--- a/arch/x86/crypto/sha-mb/sha1_mb.c
+++ b/arch/x86/crypto/sha-mb/sha1_mb.c
@@ -69,6 +69,7 @@
#include <asm/xcr.h>
#include <asm/xsave.h>
#include <linux/hardirq.h>
+#include <linux/sched.h>
#include <asm/fpu-internal.h>
#include "sha_mb_ctx.h"
@@ -820,6 +821,60 @@ static struct ahash_alg sha1_mb_async_alg = {
},
};
+void sha1_mb_force_flush(struct mcryptd_alg_cstate *cstate)
+{
+ struct mcryptd_hash_request_ctx *rctx;
+ struct sha1_hash_ctx *sha_ctx;
+
+ /* force flush uncompleted jobs in all data lanes before cpu becomes idle */
+ while (!list_empty(&cstate->work_list)) {
+ /* turn off flusher as we are flushing here */
+ if (cstate->flusher_engaged)
+ cstate->flusher_engaged = false;
+
+ kernel_fpu_begin();
+ sha_ctx = (struct sha1_hash_ctx *) sha1_ctx_mgr_flush(cstate->mgr);
+ kernel_fpu_end();
+ if (!sha_ctx) {
+ pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
+ break;
+ }
+ rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
+ sha_finish_walk(&rctx, cstate, true);
+ sha_complete_job(rctx, cstate, 0);
+ }
+
+ return;
+}
+
+void sha1_mb_earlyflush(struct work_struct *__work)
+{
+ struct mcryptd_alg_cstate *alg_cpu_state;
+
+ /* do not do early flush if other tasks are running */
+ if (nr_running_cpu(smp_processor_id()) > 1)
+ return;
+
+ alg_cpu_state = container_of(__work, struct mcryptd_alg_cstate, early_flush);
+ sha1_mb_force_flush(alg_cpu_state);
+}
+
+static int sha1_mb_idle_notifier(struct notifier_block *nb, unsigned long val,
+ void *data)
+{
+ struct mcryptd_alg_cstate *cstate =
+ this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
+
+ if (val == IDLE_START && cstate->flusher_engaged)
+ queue_work_on(smp_processor_id(), kcrypto_wq, &cstate->early_flush);
+
+ return 0;
+}
+
+static struct notifier_block sha1_mb_idle_nb = {
+ .notifier_call = sha1_mb_idle_notifier,
+};
+
unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
{
struct mcryptd_hash_request_ctx *rctx;
@@ -830,6 +885,9 @@ unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
cur_time = jiffies;
+ if (!cstate->flusher_engaged)
+ return 0;
+
while (!list_empty(&cstate->work_list)) {
rctx = list_entry(cstate->work_list.next,
struct mcryptd_hash_request_ctx, waiter);
@@ -889,6 +947,7 @@ static int __init sha1_mb_mod_init(void)
cpu_state->next_seq_num = 0;
cpu_state->flusher_engaged = false;
INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
+ INIT_WORK(&cpu_state->early_flush, sha1_mb_earlyflush);
cpu_state->cpu = cpu;
cpu_state->alg_state = &sha1_mb_alg_state;
cpu_state->mgr = (struct sha1_ctx_mgr *) kzalloc(sizeof(struct sha1_ctx_mgr), GFP_KERNEL);
@@ -907,6 +966,7 @@ static int __init sha1_mb_mod_init(void)
if (err)
goto err1;
+ idle_notifier_register(&sha1_mb_idle_nb);
return 0;
err1:
@@ -925,6 +985,7 @@ static void __exit sha1_mb_mod_fini(void)
int cpu;
struct mcryptd_alg_cstate *cpu_state;
+ idle_notifier_unregister(&sha1_mb_idle_nb);
crypto_unregister_ahash(&sha1_mb_async_alg);
crypto_unregister_shash(&sha1_mb_shash_alg);
for_each_possible_cpu(cpu) {
diff --git a/include/crypto/mcryptd.h b/include/crypto/mcryptd.h
index b2b9055..2ef1824 100644
--- a/include/crypto/mcryptd.h
+++ b/include/crypto/mcryptd.h
@@ -82,6 +82,7 @@ struct mcryptd_alg_cstate {
unsigned next_seq_num;
bool flusher_engaged;
struct delayed_work flush;
+ struct work_struct early_flush;
int cpu;
struct mcryptd_alg_state *alg_state;
void *mgr;
--
1.7.11.7
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
