On Mon, 2018-08-27 at 15:28 -0700, Tim Chen wrote: > On 08/22/2018 01:51 AM, Ard Biesheuvel wrote: > > As it turns out, the AVX2 multibuffer SHA routines are currently > > broken [0], in a way that would have likely been noticed if this > > code were in wide use. Since the code is too complicated to be > > maintained by anyone except the original authors, and since the > > performance benefits for real-world use cases are debatable to > > begin with, it is better to drop it entirely for the moment. > > > > [0] https://marc.info/?l=linux-crypto-vger&m=153476243825350&w=2 > > Sorry I was out of the loop for a while and haven't been following > the code too closely. > > Megha is maintaining the code now. Before we pull the code, > please give us a chance to fix it first. > > Thanks. > > Tim > Hi, I am working to find a fix for these corner cases. If possible, we would like to fix the issues instead of removing the code altogether. -Megha > > > > Suggested-by: Eric Biggers <ebiggers@xxxxxxxxxx> > > Cc: Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > Cc: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > Cc: Geert Uytterhoeven <geert@xxxxxxxxxxxxxx> > > Cc: Martin Schwidefsky <schwidefsky@xxxxxxxxxx> > > Cc: Heiko Carstens <heiko.carstens@xxxxxxxxxx> > > Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx> > > Cc: Ingo Molnar <mingo@xxxxxxxxxx> > > Signed-off-by: Ard Biesheuvel <ard.biesheuvel@xxxxxxxxxx> > > --- > > MAINTAINERS | 8 - > > arch/m68k/configs/amiga_defconfig | 1 - > > arch/m68k/configs/apollo_defconfig | 1 - > > arch/m68k/configs/atari_defconfig | 1 - > > arch/m68k/configs/bvme6000_defconfig | 1 - > > arch/m68k/configs/hp300_defconfig | 1 - > > arch/m68k/configs/mac_defconfig | 1 - > > arch/m68k/configs/multi_defconfig | 1 - > > arch/m68k/configs/mvme147_defconfig | 1 - > > arch/m68k/configs/mvme16x_defconfig | 1 - > > arch/m68k/configs/q40_defconfig | 1 - > > arch/m68k/configs/sun3_defconfig | 1 - > > arch/m68k/configs/sun3x_defconfig | 1 - > > arch/s390/configs/debug_defconfig | 1 - > > arch/s390/configs/performance_defconfig | 1 - > > arch/x86/crypto/Makefile | 3 - > > arch/x86/crypto/sha1-mb/Makefile | 14 - > > arch/x86/crypto/sha1-mb/sha1_mb.c | 1011 ---------------- > > arch/x86/crypto/sha1-mb/sha1_mb_ctx.h | 134 --- > > arch/x86/crypto/sha1-mb/sha1_mb_mgr.h | 110 -- > > .../crypto/sha1-mb/sha1_mb_mgr_datastruct.S | 287 ----- > > .../crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S | 304 ----- > > .../crypto/sha1-mb/sha1_mb_mgr_init_avx2.c | 64 - > > .../crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S | 209 ---- > > arch/x86/crypto/sha1-mb/sha1_x8_avx2.S | 492 -------- > > arch/x86/crypto/sha256-mb/Makefile | 14 - > > arch/x86/crypto/sha256-mb/sha256_mb.c | 1013 ---------------- > > arch/x86/crypto/sha256-mb/sha256_mb_ctx.h | 134 --- > > arch/x86/crypto/sha256-mb/sha256_mb_mgr.h | 108 -- > > .../sha256-mb/sha256_mb_mgr_datastruct.S | 304 ----- > > .../sha256-mb/sha256_mb_mgr_flush_avx2.S | 307 ----- > > .../sha256-mb/sha256_mb_mgr_init_avx2.c | 65 - > > .../sha256-mb/sha256_mb_mgr_submit_avx2.S | 214 ---- > > arch/x86/crypto/sha256-mb/sha256_x8_avx2.S | 598 ---------- > > arch/x86/crypto/sha512-mb/Makefile | 12 - > > arch/x86/crypto/sha512-mb/sha512_mb.c | 1047 ----------------- > > arch/x86/crypto/sha512-mb/sha512_mb_ctx.h | 128 -- > > arch/x86/crypto/sha512-mb/sha512_mb_mgr.h | 104 -- > > .../sha512-mb/sha512_mb_mgr_datastruct.S | 281 ----- > > .../sha512-mb/sha512_mb_mgr_flush_avx2.S | 297 ----- > > .../sha512-mb/sha512_mb_mgr_init_avx2.c | 69 -- > > .../sha512-mb/sha512_mb_mgr_submit_avx2.S | 224 ---- > > arch/x86/crypto/sha512-mb/sha512_x4_avx2.S | 531 --------- > > crypto/Kconfig | 62 - > > crypto/Makefile | 1 - > > crypto/mcryptd.c | 675 ----------- > > include/crypto/mcryptd.h | 114 -- > > 47 files changed, 8952 deletions(-) > > delete mode 100644 arch/x86/crypto/sha1-mb/Makefile > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb.c > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > > delete mode 100644 arch/x86/crypto/sha256-mb/Makefile > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb.c > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > > delete mode 100644 arch/x86/crypto/sha512-mb/Makefile > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb.c > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > > delete mode 100644 crypto/mcryptd.c > > delete mode 100644 include/crypto/mcryptd.h > > > > diff --git a/MAINTAINERS b/MAINTAINERS > > index 24b200d91b30..05747b8ac88e 100644 > > --- a/MAINTAINERS > > +++ b/MAINTAINERS > > @@ -7487,14 +7487,6 @@ S: Supported > > F: drivers/infiniband/hw/i40iw/ > > F: include/uapi/rdma/i40iw-abi.h > > > > -INTEL SHA MULTIBUFFER DRIVER > > -M: Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > -R: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > -L: linux-crypto@xxxxxxxxxxxxxxx > > -S: Supported > > -F: arch/x86/crypto/sha*-mb/ > > -F: crypto/mcryptd.c > > - > > INTEL TELEMETRY DRIVER > > M: Souvik Kumar Chakravarty <souvik.k.chakravarty@xxxxxxxxx> > > L: platform-driver-x86@xxxxxxxxxxxxxxx > > diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig > > index 1d5483f6e457..70b10d712624 100644 > > --- a/arch/m68k/configs/amiga_defconfig > > +++ b/arch/m68k/configs/amiga_defconfig > > @@ -621,7 +621,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig > > index 52a0af127951..211eec5859e8 100644 > > --- a/arch/m68k/configs/apollo_defconfig > > +++ b/arch/m68k/configs/apollo_defconfig > > @@ -578,7 +578,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig > > index b3103e51268a..0da45c6084f7 100644 > > --- a/arch/m68k/configs/atari_defconfig > > +++ b/arch/m68k/configs/atari_defconfig > > @@ -599,7 +599,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig > > index fb7d651a4cab..c09ae7219416 100644 > > --- a/arch/m68k/configs/bvme6000_defconfig > > +++ b/arch/m68k/configs/bvme6000_defconfig > > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig > > index 6b37f5537c39..8c4775b30748 100644 > > --- a/arch/m68k/configs/hp300_defconfig > > +++ b/arch/m68k/configs/hp300_defconfig > > @@ -580,7 +580,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig > > index c717bf879449..48ad520e2f2d 100644 > > --- a/arch/m68k/configs/mac_defconfig > > +++ b/arch/m68k/configs/mac_defconfig > > @@ -602,7 +602,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig > > index 226c994ce794..3a3cccb9f625 100644 > > --- a/arch/m68k/configs/multi_defconfig > > +++ b/arch/m68k/configs/multi_defconfig > > @@ -684,7 +684,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig > > index b383327fd77a..63dc311f94ff 100644 > > --- a/arch/m68k/configs/mvme147_defconfig > > +++ b/arch/m68k/configs/mvme147_defconfig > > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig > > index 9783d3deb9e9..1ae39d1f9bb5 100644 > > --- a/arch/m68k/configs/mvme16x_defconfig > > +++ b/arch/m68k/configs/mvme16x_defconfig > > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig > > index a35d10ee10cb..ba2f351811da 100644 > > --- a/arch/m68k/configs/q40_defconfig > > +++ b/arch/m68k/configs/q40_defconfig > > @@ -593,7 +593,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig > > index 573bf922d448..544b7475ff6a 100644 > > --- a/arch/m68k/configs/sun3_defconfig > > +++ b/arch/m68k/configs/sun3_defconfig > > @@ -571,7 +571,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig > > index efb27a7fcc55..149edafbb9f9 100644 > > --- a/arch/m68k/configs/sun3x_defconfig > > +++ b/arch/m68k/configs/sun3x_defconfig > > @@ -572,7 +572,6 @@ CONFIG_CRYPTO_ECDH=m > > CONFIG_CRYPTO_MANAGER=y > > CONFIG_CRYPTO_USER=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_AEGIS128=m > > diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig > > index 941d8cc6c9f5..259d1698ac50 100644 > > --- a/arch/s390/configs/debug_defconfig > > +++ b/arch/s390/configs/debug_defconfig > > @@ -668,7 +668,6 @@ CONFIG_CRYPTO_USER=m > > # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set > > CONFIG_CRYPTO_PCRYPT=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_LRW=m > > diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig > > index eb6f75f24208..37fd60c20e22 100644 > > --- a/arch/s390/configs/performance_defconfig > > +++ b/arch/s390/configs/performance_defconfig > > @@ -610,7 +610,6 @@ CONFIG_CRYPTO_USER=m > > # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set > > CONFIG_CRYPTO_PCRYPT=m > > CONFIG_CRYPTO_CRYPTD=m > > -CONFIG_CRYPTO_MCRYPTD=m > > CONFIG_CRYPTO_TEST=m > > CONFIG_CRYPTO_CHACHA20POLY1305=m > > CONFIG_CRYPTO_LRW=m > > diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile > > index a450ad573dcb..9edfa5469f9f 100644 > > --- a/arch/x86/crypto/Makefile > > +++ b/arch/x86/crypto/Makefile > > @@ -60,9 +60,6 @@ endif > > ifeq ($(avx2_supported),yes) > > obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o > > obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o > > - obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/ > > - obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/ > > - obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/ > > > > obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o > > endif > > diff --git a/arch/x86/crypto/sha1-mb/Makefile b/arch/x86/crypto/sha1-mb/Makefile > > deleted file mode 100644 > > index 815ded3ba90e..000000000000 > > --- a/arch/x86/crypto/sha1-mb/Makefile > > +++ /dev/null > > @@ -1,14 +0,0 @@ > > -# SPDX-License-Identifier: GPL-2.0 > > -# > > -# Arch-specific CryptoAPI modules. > > -# > > - > > -OBJECT_FILES_NON_STANDARD := y > > - > > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > > - $(comma)4)$(comma)%ymm2,yes,no) > > -ifeq ($(avx2_supported),yes) > > - obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb.o > > - sha1-mb-y := sha1_mb.o sha1_mb_mgr_flush_avx2.o \ > > - sha1_mb_mgr_init_avx2.o sha1_mb_mgr_submit_avx2.o sha1_x8_avx2.o > > -endif > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb.c b/arch/x86/crypto/sha1-mb/sha1_mb.c > > deleted file mode 100644 > > index b93805664c1d..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb.c > > +++ /dev/null > > @@ -1,1011 +0,0 @@ > > -/* > > - * Multi buffer SHA1 algorithm Glue Code > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > > - > > -#include <crypto/internal/hash.h> > > -#include <linux/init.h> > > -#include <linux/module.h> > > -#include <linux/mm.h> > > -#include <linux/cryptohash.h> > > -#include <linux/types.h> > > -#include <linux/list.h> > > -#include <crypto/scatterwalk.h> > > -#include <crypto/sha.h> > > -#include <crypto/mcryptd.h> > > -#include <crypto/crypto_wq.h> > > -#include <asm/byteorder.h> > > -#include <linux/hardirq.h> > > -#include <asm/fpu/api.h> > > -#include "sha1_mb_ctx.h" > > - > > -#define FLUSH_INTERVAL 1000 /* in usec */ > > - > > -static struct mcryptd_alg_state sha1_mb_alg_state; > > - > > -struct sha1_mb_ctx { > > - struct mcryptd_ahash *mcryptd_tfm; > > -}; > > - > > -static inline struct mcryptd_hash_request_ctx > > - *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) > > -{ > > - struct ahash_request *areq; > > - > > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > > -} > > - > > -static inline struct ahash_request > > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > > -{ > > - return container_of((void *) ctx, struct ahash_request, __ctx); > > -} > > - > > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > > - struct ahash_request *areq) > > -{ > > - rctx->flag = HASH_UPDATE; > > -} > > - > > -static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state); > > -static asmlinkage struct job_sha1* (*sha1_job_mgr_submit) > > - (struct sha1_mb_mgr *state, struct job_sha1 *job); > > -static asmlinkage struct job_sha1* (*sha1_job_mgr_flush) > > - (struct sha1_mb_mgr *state); > > -static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job) > > - (struct sha1_mb_mgr *state); > > - > > -static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2], > > - uint64_t total_len) > > -{ > > - uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1); > > - > > - memset(&padblock[i], 0, SHA1_BLOCK_SIZE); > > - padblock[i] = 0x80; > > - > > - i += ((SHA1_BLOCK_SIZE - 1) & > > - (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1))) > > - + 1 + SHA1_PADLENGTHFIELD_SIZE; > > - > > -#if SHA1_PADLENGTHFIELD_SIZE == 16 > > - *((uint64_t *) &padblock[i - 16]) = 0; > > -#endif > > - > > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > > - > > - /* Number of extra blocks to hash */ > > - return i >> SHA1_LOG2_BLOCK_SIZE; > > -} > > - > > -static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, > > - struct sha1_hash_ctx *ctx) > > -{ > > - while (ctx) { > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Clear PROCESSING bit */ > > - ctx->status = HASH_CTX_STS_COMPLETE; > > - return ctx; > > - } > > - > > - /* > > - * If the extra blocks are empty, begin hashing what remains > > - * in the user's buffer. > > - */ > > - if (ctx->partial_block_buffer_length == 0 && > > - ctx->incoming_buffer_length) { > > - > > - const void *buffer = ctx->incoming_buffer; > > - uint32_t len = ctx->incoming_buffer_length; > > - uint32_t copy_len; > > - > > - /* > > - * Only entire blocks can be hashed. > > - * Copy remainder to extra blocks buffer. > > - */ > > - copy_len = len & (SHA1_BLOCK_SIZE-1); > > - > > - if (copy_len) { > > - len -= copy_len; > > - memcpy(ctx->partial_block_buffer, > > - ((const char *) buffer + len), > > - copy_len); > > - ctx->partial_block_buffer_length = copy_len; > > - } > > - > > - ctx->incoming_buffer_length = 0; > > - > > - /* len should be a multiple of the block size now */ > > - assert((len % SHA1_BLOCK_SIZE) == 0); > > - > > - /* Set len to the number of blocks to be hashed */ > > - len >>= SHA1_LOG2_BLOCK_SIZE; > > - > > - if (len) { > > - > > - ctx->job.buffer = (uint8_t *) buffer; > > - ctx->job.len = len; > > - ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr, > > - &ctx->job); > > - continue; > > - } > > - } > > - > > - /* > > - * If the extra blocks are not empty, then we are > > - * either on the last block(s) or we need more > > - * user input before continuing. > > - */ > > - if (ctx->status & HASH_CTX_STS_LAST) { > > - > > - uint8_t *buf = ctx->partial_block_buffer; > > - uint32_t n_extra_blocks = > > - sha1_pad(buf, ctx->total_length); > > - > > - ctx->status = (HASH_CTX_STS_PROCESSING | > > - HASH_CTX_STS_COMPLETE); > > - ctx->job.buffer = buf; > > - ctx->job.len = (uint32_t) n_extra_blocks; > > - ctx = (struct sha1_hash_ctx *) > > - sha1_job_mgr_submit(&mgr->mgr, &ctx->job); > > - continue; > > - } > > - > > - ctx->status = HASH_CTX_STS_IDLE; > > - return ctx; > > - } > > - > > - return NULL; > > -} > > - > > -static struct sha1_hash_ctx > > - *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) > > -{ > > - /* > > - * If get_comp_job returns NULL, there are no jobs complete. > > - * If get_comp_job returns a job, verify that it is safe to return to > > - * the user. > > - * If it is not ready, resubmit the job to finish processing. > > - * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. > > - * Otherwise, all jobs currently being managed by the hash_ctx_mgr > > - * still need processing. > > - */ > > - struct sha1_hash_ctx *ctx; > > - > > - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr); > > - return sha1_ctx_mgr_resubmit(mgr, ctx); > > -} > > - > > -static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr) > > -{ > > - sha1_job_mgr_init(&mgr->mgr); > > -} > > - > > -static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, > > - struct sha1_hash_ctx *ctx, > > - const void *buffer, > > - uint32_t len, > > - int flags) > > -{ > > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > > - /* User should not pass anything other than UPDATE or LAST */ > > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > > - return ctx; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > > - /* Cannot submit to a currently processing job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > > - return ctx; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Cannot update a finished job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > > - return ctx; > > - } > > - > > - /* > > - * If we made it here, there were no errors during this call to > > - * submit > > - */ > > - ctx->error = HASH_CTX_ERROR_NONE; > > - > > - /* Store buffer ptr info from user */ > > - ctx->incoming_buffer = buffer; > > - ctx->incoming_buffer_length = len; > > - > > - /* > > - * Store the user's request flags and mark this ctx as currently > > - * being processed. > > - */ > > - ctx->status = (flags & HASH_LAST) ? > > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > > - HASH_CTX_STS_PROCESSING; > > - > > - /* Advance byte counter */ > > - ctx->total_length += len; > > - > > - /* > > - * If there is anything currently buffered in the extra blocks, > > - * append to it until it contains a whole block. > > - * Or if the user's buffer contains less than a whole block, > > - * append as much as possible to the extra block. > > - */ > > - if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) { > > - /* > > - * Compute how many bytes to copy from user buffer into > > - * extra block > > - */ > > - uint32_t copy_len = SHA1_BLOCK_SIZE - > > - ctx->partial_block_buffer_length; > > - if (len < copy_len) > > - copy_len = len; > > - > > - if (copy_len) { > > - /* Copy and update relevant pointers and counters */ > > - memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length], > > - buffer, copy_len); > > - > > - ctx->partial_block_buffer_length += copy_len; > > - ctx->incoming_buffer = (const void *) > > - ((const char *)buffer + copy_len); > > - ctx->incoming_buffer_length = len - copy_len; > > - } > > - > > - /* > > - * The extra block should never contain more than 1 block > > - * here > > - */ > > - assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE); > > - > > - /* > > - * If the extra block buffer contains exactly 1 block, it can > > - * be hashed. > > - */ > > - if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) { > > - ctx->partial_block_buffer_length = 0; > > - > > - ctx->job.buffer = ctx->partial_block_buffer; > > - ctx->job.len = 1; > > - ctx = (struct sha1_hash_ctx *) > > - sha1_job_mgr_submit(&mgr->mgr, &ctx->job); > > - } > > - } > > - > > - return sha1_ctx_mgr_resubmit(mgr, ctx); > > -} > > - > > -static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr) > > -{ > > - struct sha1_hash_ctx *ctx; > > - > > - while (1) { > > - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr); > > - > > - /* If flush returned 0, there are no more jobs in flight. */ > > - if (!ctx) > > - return NULL; > > - > > - /* > > - * If flush returned a job, resubmit the job to finish > > - * processing. > > - */ > > - ctx = sha1_ctx_mgr_resubmit(mgr, ctx); > > - > > - /* > > - * If sha1_ctx_mgr_resubmit returned a job, it is ready to be > > - * returned. Otherwise, all jobs currently being managed by the > > - * sha1_ctx_mgr still need processing. Loop. > > - */ > > - if (ctx) > > - return ctx; > > - } > > -} > > - > > -static int sha1_mb_init(struct ahash_request *areq) > > -{ > > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - hash_ctx_init(sctx); > > - sctx->job.result_digest[0] = SHA1_H0; > > - sctx->job.result_digest[1] = SHA1_H1; > > - sctx->job.result_digest[2] = SHA1_H2; > > - sctx->job.result_digest[3] = SHA1_H3; > > - sctx->job.result_digest[4] = SHA1_H4; > > - sctx->total_length = 0; > > - sctx->partial_block_buffer_length = 0; > > - sctx->status = HASH_CTX_STS_IDLE; > > - > > - return 0; > > -} > > - > > -static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > > -{ > > - int i; > > - struct sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > > - __be32 *dst = (__be32 *) rctx->out; > > - > > - for (i = 0; i < 5; ++i) > > - dst[i] = cpu_to_be32(sctx->job.result_digest[i]); > > - > > - return 0; > > -} > > - > > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > > - struct mcryptd_alg_cstate *cstate, bool flush) > > -{ > > - int flag = HASH_UPDATE; > > - int nbytes, err = 0; > > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > > - struct sha1_hash_ctx *sha_ctx; > > - > > - /* more work ? */ > > - while (!(rctx->flag & HASH_DONE)) { > > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > > - if (nbytes < 0) { > > - err = nbytes; > > - goto out; > > - } > > - /* check if the walk is done */ > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - if (rctx->flag & HASH_FINAL) > > - flag |= HASH_LAST; > > - > > - } > > - sha_ctx = (struct sha1_hash_ctx *) > > - ahash_request_ctx(&rctx->areq); > > - kernel_fpu_begin(); > > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, > > - rctx->walk.data, nbytes, flag); > > - if (!sha_ctx) { > > - if (flush) > > - sha_ctx = sha1_ctx_mgr_flush(cstate->mgr); > > - } > > - kernel_fpu_end(); > > - if (sha_ctx) > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - else { > > - rctx = NULL; > > - goto out; > > - } > > - } > > - > > - /* copy the results */ > > - if (rctx->flag & HASH_FINAL) > > - sha1_mb_set_results(rctx); > > - > > -out: > > - *ret_rctx = rctx; > > - return err; > > -} > > - > > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate, > > - int err) > > -{ > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha1_hash_ctx *sha_ctx; > > - struct mcryptd_hash_request_ctx *req_ctx; > > - int ret; > > - > > - /* remove from work list */ > > - spin_lock(&cstate->work_lock); > > - list_del(&rctx->waiter); > > - spin_unlock(&cstate->work_lock); > > - > > - if (irqs_disabled()) > > - rctx->complete(&req->base, err); > > - else { > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > - } > > - > > - /* check to see if there are other jobs that are done */ > > - sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); > > - while (sha_ctx) { > > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&req_ctx, cstate, false); > > - if (req_ctx) { > > - spin_lock(&cstate->work_lock); > > - list_del(&req_ctx->waiter); > > - spin_unlock(&cstate->work_lock); > > - > > - req = cast_mcryptd_ctx_to_req(req_ctx); > > - if (irqs_disabled()) > > - req_ctx->complete(&req->base, ret); > > - else { > > - local_bh_disable(); > > - req_ctx->complete(&req->base, ret); > > - local_bh_enable(); > > - } > > - } > > - sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); > > - } > > - > > - return 0; > > -} > > - > > -static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate) > > -{ > > - unsigned long next_flush; > > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > > - > > - /* initialize tag */ > > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > > - rctx->tag.seq_num = cstate->next_seq_num++; > > - next_flush = rctx->tag.arrival + delay; > > - rctx->tag.expire = next_flush; > > - > > - spin_lock(&cstate->work_lock); > > - list_add_tail(&rctx->waiter, &cstate->work_list); > > - spin_unlock(&cstate->work_lock); > > - > > - mcryptd_arm_flusher(cstate, delay); > > -} > > - > > -static int sha1_mb_update(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha1_hash_ctx *sha_ctx; > > - int ret = 0, nbytes; > > - > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) > > - rctx->flag |= HASH_DONE; > > - > > - /* submit */ > > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > > - sha1_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > > - nbytes, HASH_UPDATE); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha1_mb_finup(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha1_hash_ctx *sha_ctx; > > - int ret = 0, flag = HASH_UPDATE, nbytes; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - flag = HASH_LAST; > > - } > > - > > - /* submit */ > > - rctx->flag |= HASH_FINAL; > > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > > - sha1_mb_add_list(rctx, cstate); > > - > > - kernel_fpu_begin(); > > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > > - nbytes, flag); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha1_mb_final(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > > - > > - struct sha1_hash_ctx *sha_ctx; > > - int ret = 0; > > - u8 data; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - rctx->flag |= HASH_DONE | HASH_FINAL; > > - > > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > > - /* flag HASH_FINAL and 0 data size */ > > - sha1_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, > > - HASH_LAST); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha1_mb_export(struct ahash_request *areq, void *out) > > -{ > > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(out, sctx, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha1_mb_import(struct ahash_request *areq, const void *in) > > -{ > > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(sctx, in, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) > > -{ > > - struct mcryptd_ahash *mcryptd_tfm; > > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - struct mcryptd_hash_ctx *mctx; > > - > > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", > > - CRYPTO_ALG_INTERNAL, > > - CRYPTO_ALG_INTERNAL); > > - if (IS_ERR(mcryptd_tfm)) > > - return PTR_ERR(mcryptd_tfm); > > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > > - mctx->alg_state = &sha1_mb_alg_state; > > - ctx->mcryptd_tfm = mcryptd_tfm; > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > > - > > - return 0; > > -} > > - > > -static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm) > > -{ > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - sizeof(struct sha1_hash_ctx)); > > - > > - return 0; > > -} > > - > > -static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static struct ahash_alg sha1_mb_areq_alg = { > > - .init = sha1_mb_init, > > - .update = sha1_mb_update, > > - .final = sha1_mb_final, > > - .finup = sha1_mb_finup, > > - .export = sha1_mb_export, > > - .import = sha1_mb_import, > > - .halg = { > > - .digestsize = SHA1_DIGEST_SIZE, > > - .statesize = sizeof(struct sha1_hash_ctx), > > - .base = { > > - .cra_name = "__sha1-mb", > > - .cra_driver_name = "__intel_sha1-mb", > > - .cra_priority = 100, > > - /* > > - * use ASYNC flag as some buffers in multi-buffer > > - * algo may not have completed before hashing thread > > - * sleep > > - */ > > - .cra_flags = CRYPTO_ALG_ASYNC | > > - CRYPTO_ALG_INTERNAL, > > - .cra_blocksize = SHA1_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT > > - (sha1_mb_areq_alg.halg.base.cra_list), > > - .cra_init = sha1_mb_areq_init_tfm, > > - .cra_exit = sha1_mb_areq_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha1_hash_ctx), > > - } > > - } > > -}; > > - > > -static int sha1_mb_async_init(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_init(mcryptd_req); > > -} > > - > > -static int sha1_mb_async_update(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_update(mcryptd_req); > > -} > > - > > -static int sha1_mb_async_finup(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_finup(mcryptd_req); > > -} > > - > > -static int sha1_mb_async_final(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_final(mcryptd_req); > > -} > > - > > -static int sha1_mb_async_digest(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_digest(mcryptd_req); > > -} > > - > > -static int sha1_mb_async_export(struct ahash_request *req, void *out) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_export(mcryptd_req, out); > > -} > > - > > -static int sha1_mb_async_import(struct ahash_request *req, const void *in) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > > - struct mcryptd_hash_request_ctx *rctx; > > - struct ahash_request *areq; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - rctx = ahash_request_ctx(mcryptd_req); > > - areq = &rctx->areq; > > - > > - ahash_request_set_tfm(areq, child); > > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > > - rctx->complete, req); > > - > > - return crypto_ahash_import(mcryptd_req, in); > > -} > > - > > -static struct ahash_alg sha1_mb_async_alg = { > > - .init = sha1_mb_async_init, > > - .update = sha1_mb_async_update, > > - .final = sha1_mb_async_final, > > - .finup = sha1_mb_async_finup, > > - .digest = sha1_mb_async_digest, > > - .export = sha1_mb_async_export, > > - .import = sha1_mb_async_import, > > - .halg = { > > - .digestsize = SHA1_DIGEST_SIZE, > > - .statesize = sizeof(struct sha1_hash_ctx), > > - .base = { > > - .cra_name = "sha1", > > - .cra_driver_name = "sha1_mb", > > - /* > > - * Low priority, since with few concurrent hash requests > > - * this is extremely slow due to the flush delay. Users > > - * whose workloads would benefit from this can request > > - * it explicitly by driver name, or can increase its > > - * priority at runtime using NETLINK_CRYPTO. > > - */ > > - .cra_priority = 50, > > - .cra_flags = CRYPTO_ALG_ASYNC, > > - .cra_blocksize = SHA1_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list), > > - .cra_init = sha1_mb_async_init_tfm, > > - .cra_exit = sha1_mb_async_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha1_mb_ctx), > > - .cra_alignmask = 0, > > - }, > > - }, > > -}; > > - > > -static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) > > -{ > > - struct mcryptd_hash_request_ctx *rctx; > > - unsigned long cur_time; > > - unsigned long next_flush = 0; > > - struct sha1_hash_ctx *sha_ctx; > > - > > - > > - cur_time = jiffies; > > - > > - while (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - if (time_before(cur_time, rctx->tag.expire)) > > - break; > > - kernel_fpu_begin(); > > - sha_ctx = (struct sha1_hash_ctx *) > > - sha1_ctx_mgr_flush(cstate->mgr); > > - kernel_fpu_end(); > > - if (!sha_ctx) { > > - pr_err("sha1_mb error: nothing got flushed for non-empty list\n"); > > - break; > > - } > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - sha_finish_walk(&rctx, cstate, true); > > - sha_complete_job(rctx, cstate, 0); > > - } > > - > > - if (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - /* get the hash context and then flush time */ > > - next_flush = rctx->tag.expire; > > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > > - } > > - return next_flush; > > -} > > - > > -static int __init sha1_mb_mod_init(void) > > -{ > > - > > - int cpu; > > - int err; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - /* check for dependent cpu features */ > > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > > - !boot_cpu_has(X86_FEATURE_BMI2)) > > - return -ENODEV; > > - > > - /* initialize multibuffer structures */ > > - sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate); > > - > > - sha1_job_mgr_init = sha1_mb_mgr_init_avx2; > > - sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2; > > - sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2; > > - sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2; > > - > > - if (!sha1_mb_alg_state.alg_cstate) > > - return -ENOMEM; > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > > - cpu_state->next_flush = 0; > > - cpu_state->next_seq_num = 0; > > - cpu_state->flusher_engaged = false; > > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > > - cpu_state->cpu = cpu; > > - cpu_state->alg_state = &sha1_mb_alg_state; > > - cpu_state->mgr = kzalloc(sizeof(struct sha1_ctx_mgr), > > - GFP_KERNEL); > > - if (!cpu_state->mgr) > > - goto err2; > > - sha1_ctx_mgr_init(cpu_state->mgr); > > - INIT_LIST_HEAD(&cpu_state->work_list); > > - spin_lock_init(&cpu_state->work_lock); > > - } > > - sha1_mb_alg_state.flusher = &sha1_mb_flusher; > > - > > - err = crypto_register_ahash(&sha1_mb_areq_alg); > > - if (err) > > - goto err2; > > - err = crypto_register_ahash(&sha1_mb_async_alg); > > - if (err) > > - goto err1; > > - > > - > > - return 0; > > -err1: > > - crypto_unregister_ahash(&sha1_mb_areq_alg); > > -err2: > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha1_mb_alg_state.alg_cstate); > > - return -ENODEV; > > -} > > - > > -static void __exit sha1_mb_mod_fini(void) > > -{ > > - int cpu; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - crypto_unregister_ahash(&sha1_mb_async_alg); > > - crypto_unregister_ahash(&sha1_mb_areq_alg); > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha1_mb_alg_state.alg_cstate); > > -} > > - > > -module_init(sha1_mb_mod_init); > > -module_exit(sha1_mb_mod_fini); > > - > > -MODULE_LICENSE("GPL"); > > -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated"); > > - > > -MODULE_ALIAS_CRYPTO("sha1"); > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > > deleted file mode 100644 > > index 9454bd16f9f8..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > > +++ /dev/null > > @@ -1,134 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA context > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#ifndef _SHA_MB_CTX_INTERNAL_H > > -#define _SHA_MB_CTX_INTERNAL_H > > - > > -#include "sha1_mb_mgr.h" > > - > > -#define HASH_UPDATE 0x00 > > -#define HASH_LAST 0x01 > > -#define HASH_DONE 0x02 > > -#define HASH_FINAL 0x04 > > - > > -#define HASH_CTX_STS_IDLE 0x00 > > -#define HASH_CTX_STS_PROCESSING 0x01 > > -#define HASH_CTX_STS_LAST 0x02 > > -#define HASH_CTX_STS_COMPLETE 0x04 > > - > > -enum hash_ctx_error { > > - HASH_CTX_ERROR_NONE = 0, > > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > > - > > -#ifdef HASH_CTX_DEBUG > > - HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4, > > -#endif > > -}; > > - > > - > > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > > -#define hash_ctx_status(ctx) ((ctx)->status) > > -#define hash_ctx_error(ctx) ((ctx)->error) > > -#define hash_ctx_init(ctx) \ > > - do { \ > > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > > - } while (0) > > - > > - > > -/* Hash Constants and Typedefs */ > > -#define SHA1_DIGEST_LENGTH 5 > > -#define SHA1_LOG2_BLOCK_SIZE 6 > > - > > -#define SHA1_PADLENGTHFIELD_SIZE 8 > > - > > -#ifdef SHA_MB_DEBUG > > -#define assert(expr) \ > > -do { \ > > - if (unlikely(!(expr))) { \ > > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > > - #expr, __FILE__, __func__, __LINE__); \ > > - } \ > > -} while (0) > > -#else > > -#define assert(expr) do {} while (0) > > -#endif > > - > > -struct sha1_ctx_mgr { > > - struct sha1_mb_mgr mgr; > > -}; > > - > > -/* typedef struct sha1_ctx_mgr sha1_ctx_mgr; */ > > - > > -struct sha1_hash_ctx { > > - /* Must be at struct offset 0 */ > > - struct job_sha1 job; > > - /* status flag */ > > - int status; > > - /* error flag */ > > - int error; > > - > > - uint64_t total_length; > > - const void *incoming_buffer; > > - uint32_t incoming_buffer_length; > > - uint8_t partial_block_buffer[SHA1_BLOCK_SIZE * 2]; > > - uint32_t partial_block_buffer_length; > > - void *user_data; > > -}; > > - > > -#endif > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h b/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > > deleted file mode 100644 > > index 08ad1a9acfd7..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > > +++ /dev/null > > @@ -1,110 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA1 algorithm manager > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * James Guilford <james.guilford@xxxxxxxxx> > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > -#ifndef __SHA_MB_MGR_H > > -#define __SHA_MB_MGR_H > > - > > - > > -#include <linux/types.h> > > - > > -#define NUM_SHA1_DIGEST_WORDS 5 > > - > > -enum job_sts { STS_UNKNOWN = 0, > > - STS_BEING_PROCESSED = 1, > > - STS_COMPLETED = 2, > > - STS_INTERNAL_ERROR = 3, > > - STS_ERROR = 4 > > -}; > > - > > -struct job_sha1 { > > - u8 *buffer; > > - u32 len; > > - u32 result_digest[NUM_SHA1_DIGEST_WORDS] __aligned(32); > > - enum job_sts status; > > - void *user_data; > > -}; > > - > > -/* SHA1 out-of-order scheduler */ > > - > > -/* typedef uint32_t sha1_digest_array[5][8]; */ > > - > > -struct sha1_args_x8 { > > - uint32_t digest[5][8]; > > - uint8_t *data_ptr[8]; > > -}; > > - > > -struct sha1_lane_data { > > - struct job_sha1 *job_in_lane; > > -}; > > - > > -struct sha1_mb_mgr { > > - struct sha1_args_x8 args; > > - > > - uint32_t lens[8]; > > - > > - /* each byte is index (0...7) of unused lanes */ > > - uint64_t unused_lanes; > > - /* byte 4 is set to FF as a flag */ > > - struct sha1_lane_data ldata[8]; > > -}; > > - > > - > > -#define SHA1_MB_MGR_NUM_LANES_AVX2 8 > > - > > -void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state); > > -struct job_sha1 *sha1_mb_mgr_submit_avx2(struct sha1_mb_mgr *state, > > - struct job_sha1 *job); > > -struct job_sha1 *sha1_mb_mgr_flush_avx2(struct sha1_mb_mgr *state); > > -struct job_sha1 *sha1_mb_mgr_get_comp_job_avx2(struct sha1_mb_mgr *state); > > - > > -#endif > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > > deleted file mode 100644 > > index 86688c6e7a25..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > > +++ /dev/null > > @@ -1,287 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA1 algorithm data structure > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * James Guilford <james.guilford@xxxxxxxxx> > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -# Macros for defining data structures > > - > > -# Usage example > > - > > -#START_FIELDS # JOB_AES > > -### name size align > > -#FIELD _plaintext, 8, 8 # pointer to plaintext > > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > > -#FIELD _IV, 16, 8 # IV > > -#FIELD _keys, 8, 8 # pointer to keys > > -#FIELD _len, 4, 4 # length in bytes > > -#FIELD _status, 4, 4 # status enumeration > > -#FIELD _user_data, 8, 8 # pointer to user data > > -#UNION _union, size1, align1, \ > > -# size2, align2, \ > > -# size3, align3, \ > > -# ... > > -#END_FIELDS > > -#%assign _JOB_AES_size _FIELD_OFFSET > > -#%assign _JOB_AES_align _STRUCT_ALIGN > > - > > -######################################################################### > > - > > -# Alternate "struc-like" syntax: > > -# STRUCT job_aes2 > > -# RES_Q .plaintext, 1 > > -# RES_Q .ciphertext, 1 > > -# RES_DQ .IV, 1 > > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > > -# RES_U .union, size1, align1, \ > > -# size2, align2, \ > > -# ... > > -# ENDSTRUCT > > -# # Following only needed if nesting > > -# %assign job_aes2_size _FIELD_OFFSET > > -# %assign job_aes2_align _STRUCT_ALIGN > > -# > > -# RES_* macros take a name, a count and an optional alignment. > > -# The count in in terms of the base size of the macro, and the > > -# default alignment is the base size. > > -# The macros are: > > -# Macro Base size > > -# RES_B 1 > > -# RES_W 2 > > -# RES_D 4 > > -# RES_Q 8 > > -# RES_DQ 16 > > -# RES_Y 32 > > -# RES_Z 64 > > -# > > -# RES_U defines a union. It's arguments are a name and two or more > > -# pairs of "size, alignment" > > -# > > -# The two assigns are only needed if this structure is being nested > > -# within another. Even if the assigns are not done, one can still use > > -# STRUCT_NAME_size as the size of the structure. > > -# > > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > > -# > > -# The differences between this and using "struc" directly are that each > > -# type is implicitly aligned to its natural length (although this can be > > -# over-ridden with an explicit third parameter), and that the structure > > -# is padded at the end to its overall alignment. > > -# > > - > > -######################################################################### > > - > > -#ifndef _SHA1_MB_MGR_DATASTRUCT_ASM_ > > -#define _SHA1_MB_MGR_DATASTRUCT_ASM_ > > - > > -## START_FIELDS > > -.macro START_FIELDS > > - _FIELD_OFFSET = 0 > > - _STRUCT_ALIGN = 0 > > -.endm > > - > > -## FIELD name size align > > -.macro FIELD name size align > > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > > - \name = _FIELD_OFFSET > > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > > -.if (\align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = \align > > -.endif > > -.endm > > - > > -## END_FIELDS > > -.macro END_FIELDS > > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > > -.endm > > - > > -######################################################################## > > - > > -.macro STRUCT p1 > > -START_FIELDS > > -.struc \p1 > > -.endm > > - > > -.macro ENDSTRUCT > > - tmp = _FIELD_OFFSET > > - END_FIELDS > > - tmp = (_FIELD_OFFSET - %%tmp) > > -.if (tmp > 0) > > - .lcomm tmp > > -.endif > > -.endstruc > > -.endm > > - > > -## RES_int name size align > > -.macro RES_int p1 p2 p3 > > - name = \p1 > > - size = \p2 > > - align = .\p3 > > - > > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > > -.align align > > -.lcomm name size > > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > > -.if (align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = align > > -.endif > > -.endm > > - > > - > > - > > -# macro RES_B name, size [, align] > > -.macro RES_B _name, _size, _align=1 > > -RES_int _name _size _align > > -.endm > > - > > -# macro RES_W name, size [, align] > > -.macro RES_W _name, _size, _align=2 > > -RES_int _name 2*(_size) _align > > -.endm > > - > > -# macro RES_D name, size [, align] > > -.macro RES_D _name, _size, _align=4 > > -RES_int _name 4*(_size) _align > > -.endm > > - > > -# macro RES_Q name, size [, align] > > -.macro RES_Q _name, _size, _align=8 > > -RES_int _name 8*(_size) _align > > -.endm > > - > > -# macro RES_DQ name, size [, align] > > -.macro RES_DQ _name, _size, _align=16 > > -RES_int _name 16*(_size) _align > > -.endm > > - > > -# macro RES_Y name, size [, align] > > -.macro RES_Y _name, _size, _align=32 > > -RES_int _name 32*(_size) _align > > -.endm > > - > > -# macro RES_Z name, size [, align] > > -.macro RES_Z _name, _size, _align=64 > > -RES_int _name 64*(_size) _align > > -.endm > > - > > - > > -#endif > > - > > -######################################################################## > > -#### Define constants > > -######################################################################## > > - > > -######################################################################## > > -#### Define SHA1 Out Of Order Data Structures > > -######################################################################## > > - > > -START_FIELDS # LANE_DATA > > -### name size align > > -FIELD _job_in_lane, 8, 8 # pointer to job object > > -END_FIELDS > > - > > -_LANE_DATA_size = _FIELD_OFFSET > > -_LANE_DATA_align = _STRUCT_ALIGN > > - > > -######################################################################## > > - > > -START_FIELDS # SHA1_ARGS_X8 > > -### name size align > > -FIELD _digest, 4*5*8, 16 # transposed digest > > -FIELD _data_ptr, 8*8, 8 # array of pointers to data > > -END_FIELDS > > - > > -_SHA1_ARGS_X4_size = _FIELD_OFFSET > > -_SHA1_ARGS_X4_align = _STRUCT_ALIGN > > -_SHA1_ARGS_X8_size = _FIELD_OFFSET > > -_SHA1_ARGS_X8_align = _STRUCT_ALIGN > > - > > -######################################################################## > > - > > -START_FIELDS # MB_MGR > > -### name size align > > -FIELD _args, _SHA1_ARGS_X4_size, _SHA1_ARGS_X4_align > > -FIELD _lens, 4*8, 8 > > -FIELD _unused_lanes, 8, 8 > > -FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align > > -END_FIELDS > > - > > -_MB_MGR_size = _FIELD_OFFSET > > -_MB_MGR_align = _STRUCT_ALIGN > > - > > -_args_digest = _args + _digest > > -_args_data_ptr = _args + _data_ptr > > - > > - > > -######################################################################## > > -#### Define constants > > -######################################################################## > > - > > -#define STS_UNKNOWN 0 > > -#define STS_BEING_PROCESSED 1 > > -#define STS_COMPLETED 2 > > - > > -######################################################################## > > -#### Define JOB_SHA1 structure > > -######################################################################## > > - > > -START_FIELDS # JOB_SHA1 > > - > > -### name size align > > -FIELD _buffer, 8, 8 # pointer to buffer > > -FIELD _len, 4, 4 # length in bytes > > -FIELD _result_digest, 5*4, 32 # Digest (output) > > -FIELD _status, 4, 4 > > -FIELD _user_data, 8, 8 > > -END_FIELDS > > - > > -_JOB_SHA1_size = _FIELD_OFFSET > > -_JOB_SHA1_align = _STRUCT_ALIGN > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > > deleted file mode 100644 > > index 7cfba738f104..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > > +++ /dev/null > > @@ -1,304 +0,0 @@ > > -/* > > - * Flush routine for SHA1 multibuffer > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * James Guilford <james.guilford@xxxxxxxxx> > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha1_mb_mgr_datastruct.S" > > - > > - > > -.extern sha1_x8_avx2 > > - > > -# LINUX register definitions > > -#define arg1 %rdi > > -#define arg2 %rsi > > - > > -# Common definitions > > -#define state arg1 > > -#define job arg2 > > -#define len2 arg2 > > - > > -# idx must be a register not clobbered by sha1_x8_avx2 > > -#define idx %r8 > > -#define DWORD_idx %r8d > > - > > -#define unused_lanes %rbx > > -#define lane_data %rbx > > -#define tmp2 %rbx > > -#define tmp2_w %ebx > > - > > -#define job_rax %rax > > -#define tmp1 %rax > > -#define size_offset %rax > > -#define tmp %rax > > -#define start_offset %rax > > - > > -#define tmp3 %arg1 > > - > > -#define extra_blocks %arg2 > > -#define p %arg2 > > - > > -.macro LABEL prefix n > > -\prefix\n\(): > > -.endm > > - > > -.macro JNE_SKIP i > > -jne skip_\i > > -.endm > > - > > -.altmacro > > -.macro SET_OFFSET _offset > > -offset = \_offset > > -.endm > > -.noaltmacro > > - > > -# JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state) > > -# arg 1 : rcx : state > > -ENTRY(sha1_mb_mgr_flush_avx2) > > - FRAME_BEGIN > > - push %rbx > > - > > - # If bit (32+3) is set, then all lanes are empty > > - mov _unused_lanes(state), unused_lanes > > - bt $32+3, unused_lanes > > - jc return_null > > - > > - # find a lane with a non-null job > > - xor idx, idx > > - offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne one(%rip), idx > > - offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne two(%rip), idx > > - offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne three(%rip), idx > > - offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne four(%rip), idx > > - offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne five(%rip), idx > > - offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne six(%rip), idx > > - offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne seven(%rip), idx > > - > > - # copy idx to empty lanes > > -copy_lane_data: > > - offset = (_args + _data_ptr) > > - mov offset(state,idx,8), tmp > > - > > - I = 0 > > -.rep 8 > > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > -.altmacro > > - JNE_SKIP %I > > - offset = (_args + _data_ptr + 8*I) > > - mov tmp, offset(state) > > - offset = (_lens + 4*I) > > - movl $0xFFFFFFFF, offset(state) > > -LABEL skip_ %I > > - I = (I+1) > > -.noaltmacro > > -.endr > > - > > - # Find min length > > - vmovdqu _lens+0*16(state), %xmm0 > > - vmovdqu _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - mov idx, len2 > > - and $0xF, idx > > - shr $4, len2 > > - jz len_is_0 > > - > > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > > - vpshufd $0, %xmm2, %xmm2 > > - > > - vpsubd %xmm2, %xmm0, %xmm0 > > - vpsubd %xmm2, %xmm1, %xmm1 > > - > > - vmovdqu %xmm0, _lens+0*16(state) > > - vmovdqu %xmm1, _lens+1*16(state) > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha1_x8_avx2 > > - # state and idx are intact > > - > > - > > -len_is_0: > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens(state, idx, 4) > > - > > - vmovd _args_digest(state , idx, 4) , %xmm0 > > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > > - movl _args_digest+4*32(state, idx, 4), tmp2_w > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - offset = (_result_digest + 1*16) > > - mov tmp2_w, offset(job_rax) > > - > > -return: > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > -ENDPROC(sha1_mb_mgr_flush_avx2) > > - > > - > > -################################################################# > > - > > -.align 16 > > -ENTRY(sha1_mb_mgr_get_comp_job_avx2) > > - push %rbx > > - > > - ## if bit 32+3 is set, then all lanes are empty > > - mov _unused_lanes(state), unused_lanes > > - bt $(32+3), unused_lanes > > - jc .return_null > > - > > - # Find min length > > - vmovdqu _lens(state), %xmm0 > > - vmovdqu _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - test $~0xF, idx > > - jnz .return_null > > - > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens(state, idx, 4) > > - > > - vmovd _args_digest(state, idx, 4), %xmm0 > > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > > - movl _args_digest+4*32(state, idx, 4), tmp2_w > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - movl tmp2_w, _result_digest+1*16(job_rax) > > - > > - pop %rbx > > - > > - ret > > - > > -.return_null: > > - xor job_rax, job_rax > > - pop %rbx > > - ret > > -ENDPROC(sha1_mb_mgr_get_comp_job_avx2) > > - > > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > > -.align 16 > > -clear_low_nibble: > > -.octa 0x000000000000000000000000FFFFFFF0 > > - > > -.section .rodata.cst8, "aM", @progbits, 8 > > -.align 8 > > -one: > > -.quad 1 > > -two: > > -.quad 2 > > -three: > > -.quad 3 > > -four: > > -.quad 4 > > -five: > > -.quad 5 > > -six: > > -.quad 6 > > -seven: > > -.quad 7 > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > > deleted file mode 100644 > > index d2add0d35f43..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > > +++ /dev/null > > @@ -1,64 +0,0 @@ > > -/* > > - * Initialization code for multi buffer SHA1 algorithm for AVX2 > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include "sha1_mb_mgr.h" > > - > > -void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state) > > -{ > > - unsigned int j; > > - state->unused_lanes = 0xF76543210ULL; > > - for (j = 0; j < 8; j++) { > > - state->lens[j] = 0xFFFFFFFF; > > - state->ldata[j].job_in_lane = NULL; > > - } > > -} > > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > > deleted file mode 100644 > > index 7a93b1c0d69a..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > > +++ /dev/null > > @@ -1,209 +0,0 @@ > > -/* > > - * Buffer submit code for multi buffer SHA1 algorithm > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * James Guilford <james.guilford@xxxxxxxxx> > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha1_mb_mgr_datastruct.S" > > - > > - > > -.extern sha1_x8_avx > > - > > -# LINUX register definitions > > -arg1 = %rdi > > -arg2 = %rsi > > -size_offset = %rcx > > -tmp2 = %rcx > > -extra_blocks = %rdx > > - > > -# Common definitions > > -#define state arg1 > > -#define job %rsi > > -#define len2 arg2 > > -#define p2 arg2 > > - > > -# idx must be a register not clobberred by sha1_x8_avx2 > > -idx = %r8 > > -DWORD_idx = %r8d > > -last_len = %r8 > > - > > -p = %r11 > > -start_offset = %r11 > > - > > -unused_lanes = %rbx > > -BYTE_unused_lanes = %bl > > - > > -job_rax = %rax > > -len = %rax > > -DWORD_len = %eax > > - > > -lane = %r12 > > -tmp3 = %r12 > > - > > -tmp = %r9 > > -DWORD_tmp = %r9d > > - > > -lane_data = %r10 > > - > > -# JOB* submit_mb_mgr_submit_avx2(MB_MGR *state, job_sha1 *job) > > -# arg 1 : rcx : state > > -# arg 2 : rdx : job > > -ENTRY(sha1_mb_mgr_submit_avx2) > > - FRAME_BEGIN > > - push %rbx > > - push %r12 > > - > > - mov _unused_lanes(state), unused_lanes > > - mov unused_lanes, lane > > - and $0xF, lane > > - shr $4, unused_lanes > > - imul $_LANE_DATA_size, lane, lane_data > > - movl $STS_BEING_PROCESSED, _status(job) > > - lea _ldata(state, lane_data), lane_data > > - mov unused_lanes, _unused_lanes(state) > > - movl _len(job), DWORD_len > > - > > - mov job, _job_in_lane(lane_data) > > - shl $4, len > > - or lane, len > > - > > - movl DWORD_len, _lens(state , lane, 4) > > - > > - # Load digest words from result_digest > > - vmovdqu _result_digest(job), %xmm0 > > - mov _result_digest+1*16(job), DWORD_tmp > > - vmovd %xmm0, _args_digest(state, lane, 4) > > - vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4) > > - vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4) > > - vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4) > > - movl DWORD_tmp, _args_digest+4*32(state , lane, 4) > > - > > - mov _buffer(job), p > > - mov p, _args_data_ptr(state, lane, 8) > > - > > - cmp $0xF, unused_lanes > > - jne return_null > > - > > -start_loop: > > - # Find min length > > - vmovdqa _lens(state), %xmm0 > > - vmovdqa _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - mov idx, len2 > > - and $0xF, idx > > - shr $4, len2 > > - jz len_is_0 > > - > > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > > - vpshufd $0, %xmm2, %xmm2 > > - > > - vpsubd %xmm2, %xmm0, %xmm0 > > - vpsubd %xmm2, %xmm1, %xmm1 > > - > > - vmovdqa %xmm0, _lens + 0*16(state) > > - vmovdqa %xmm1, _lens + 1*16(state) > > - > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha1_x8_avx2 > > - > > - # state and idx are intact > > - > > -len_is_0: > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - mov _unused_lanes(state), unused_lanes > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens(state, idx, 4) > > - > > - vmovd _args_digest(state, idx, 4), %xmm0 > > - vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0 > > - movl _args_digest+4*32(state, idx, 4), DWORD_tmp > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - movl DWORD_tmp, _result_digest+1*16(job_rax) > > - > > -return: > > - pop %r12 > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > - > > -ENDPROC(sha1_mb_mgr_submit_avx2) > > - > > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > > -.align 16 > > -clear_low_nibble: > > - .octa 0x000000000000000000000000FFFFFFF0 > > diff --git a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > > deleted file mode 100644 > > index 20f77aa633de..000000000000 > > --- a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > > +++ /dev/null > > @@ -1,492 +0,0 @@ > > -/* > > - * Multi-buffer SHA1 algorithm hash compute routine > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * James Guilford <james.guilford@xxxxxxxxx> > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2014 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include "sha1_mb_mgr_datastruct.S" > > - > > -## code to compute oct SHA1 using SSE-256 > > -## outer calling routine takes care of save and restore of XMM registers > > - > > -## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15# ymm0-15 > > -## > > -## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15 > > -## Linux preserves: rdi rbp r8 > > -## > > -## clobbers ymm0-15 > > - > > - > > -# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1 > > -# "transpose" data in {r0...r7} using temps {t0...t1} > > -# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > > -# r4 = {e7 e6 e5 e4 e3 e2 e1 e0} > > -# r5 = {f7 f6 f5 f4 f3 f2 f1 f0} > > -# r6 = {g7 g6 g5 g4 g3 g2 g1 g0} > > -# r7 = {h7 h6 h5 h4 h3 h2 h1 h0} > > -# > > -# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > > -# r0 = {h0 g0 f0 e0 d0 c0 b0 a0} > > -# r1 = {h1 g1 f1 e1 d1 c1 b1 a1} > > -# r2 = {h2 g2 f2 e2 d2 c2 b2 a2} > > -# r3 = {h3 g3 f3 e3 d3 c3 b3 a3} > > -# r4 = {h4 g4 f4 e4 d4 c4 b4 a4} > > -# r5 = {h5 g5 f5 e5 d5 c5 b5 a5} > > -# r6 = {h6 g6 f6 e6 d6 c6 b6 a6} > > -# r7 = {h7 g7 f7 e7 d7 c7 b7 a7} > > -# > > - > > -.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1 > > - # process top half (r0..r3) {a...d} > > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > > - vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1} > > - vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2} > > - vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3} > > - vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0} > > - > > - # use r2 in place of t0 > > - # process bottom half (r4..r7) {e...h} > > - vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0} > > - vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2} > > - vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0} > > - vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2} > > - vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1} > > - vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2} > > - vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3} > > - vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0} > > - > > - vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6 > > - vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2 > > - vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5 > > - vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1 > > - vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7 > > - vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3 > > - vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4 > > - vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0 > > - > > -.endm > > -## > > -## Magic functions defined in FIPS 180-1 > > -## > > -# macro MAGIC_F0 F,B,C,D,T ## F = (D ^ (B & (C ^ D))) > > -.macro MAGIC_F0 regF regB regC regD regT > > - vpxor \regD, \regC, \regF > > - vpand \regB, \regF, \regF > > - vpxor \regD, \regF, \regF > > -.endm > > - > > -# macro MAGIC_F1 F,B,C,D,T ## F = (B ^ C ^ D) > > -.macro MAGIC_F1 regF regB regC regD regT > > - vpxor \regC, \regD, \regF > > - vpxor \regB, \regF, \regF > > -.endm > > - > > -# macro MAGIC_F2 F,B,C,D,T ## F = ((B & C) | (B & D) | (C & D)) > > -.macro MAGIC_F2 regF regB regC regD regT > > - vpor \regC, \regB, \regF > > - vpand \regC, \regB, \regT > > - vpand \regD, \regF, \regF > > - vpor \regT, \regF, \regF > > -.endm > > - > > -# macro MAGIC_F3 F,B,C,D,T ## F = (B ^ C ^ D) > > -.macro MAGIC_F3 regF regB regC regD regT > > - MAGIC_F1 \regF,\regB,\regC,\regD,\regT > > -.endm > > - > > -# PROLD reg, imm, tmp > > -.macro PROLD reg imm tmp > > - vpsrld $(32-\imm), \reg, \tmp > > - vpslld $\imm, \reg, \reg > > - vpor \tmp, \reg, \reg > > -.endm > > - > > -.macro PROLD_nd reg imm tmp src > > - vpsrld $(32-\imm), \src, \tmp > > - vpslld $\imm, \src, \reg > > - vpor \tmp, \reg, \reg > > -.endm > > - > > -.macro SHA1_STEP_00_15 regA regB regC regD regE regT regF memW immCNT MAGIC > > - vpaddd \immCNT, \regE, \regE > > - vpaddd \memW*32(%rsp), \regE, \regE > > - PROLD_nd \regT, 5, \regF, \regA > > - vpaddd \regT, \regE, \regE > > - \MAGIC \regF, \regB, \regC, \regD, \regT > > - PROLD \regB, 30, \regT > > - vpaddd \regF, \regE, \regE > > -.endm > > - > > -.macro SHA1_STEP_16_79 regA regB regC regD regE regT regF memW immCNT MAGIC > > - vpaddd \immCNT, \regE, \regE > > - offset = ((\memW - 14) & 15) * 32 > > - vmovdqu offset(%rsp), W14 > > - vpxor W14, W16, W16 > > - offset = ((\memW - 8) & 15) * 32 > > - vpxor offset(%rsp), W16, W16 > > - offset = ((\memW - 3) & 15) * 32 > > - vpxor offset(%rsp), W16, W16 > > - vpsrld $(32-1), W16, \regF > > - vpslld $1, W16, W16 > > - vpor W16, \regF, \regF > > - > > - ROTATE_W > > - > > - offset = ((\memW - 0) & 15) * 32 > > - vmovdqu \regF, offset(%rsp) > > - vpaddd \regF, \regE, \regE > > - PROLD_nd \regT, 5, \regF, \regA > > - vpaddd \regT, \regE, \regE > > - \MAGIC \regF,\regB,\regC,\regD,\regT ## FUN = MAGIC_Fi(B,C,D) > > - PROLD \regB,30, \regT > > - vpaddd \regF, \regE, \regE > > -.endm > > - > > -######################################################################## > > -######################################################################## > > -######################################################################## > > - > > -## FRAMESZ plus pushes must be an odd multiple of 8 > > -YMM_SAVE = (15-15)*32 > > -FRAMESZ = 32*16 + YMM_SAVE > > -_YMM = FRAMESZ - YMM_SAVE > > - > > -#define VMOVPS vmovups > > - > > -IDX = %rax > > -inp0 = %r9 > > -inp1 = %r10 > > -inp2 = %r11 > > -inp3 = %r12 > > -inp4 = %r13 > > -inp5 = %r14 > > -inp6 = %r15 > > -inp7 = %rcx > > -arg1 = %rdi > > -arg2 = %rsi > > -RSP_SAVE = %rdx > > - > > -# ymm0 A > > -# ymm1 B > > -# ymm2 C > > -# ymm3 D > > -# ymm4 E > > -# ymm5 F AA > > -# ymm6 T0 BB > > -# ymm7 T1 CC > > -# ymm8 T2 DD > > -# ymm9 T3 EE > > -# ymm10 T4 TMP > > -# ymm11 T5 FUN > > -# ymm12 T6 K > > -# ymm13 T7 W14 > > -# ymm14 T8 W15 > > -# ymm15 T9 W16 > > - > > - > > -A = %ymm0 > > -B = %ymm1 > > -C = %ymm2 > > -D = %ymm3 > > -E = %ymm4 > > -F = %ymm5 > > -T0 = %ymm6 > > -T1 = %ymm7 > > -T2 = %ymm8 > > -T3 = %ymm9 > > -T4 = %ymm10 > > -T5 = %ymm11 > > -T6 = %ymm12 > > -T7 = %ymm13 > > -T8 = %ymm14 > > -T9 = %ymm15 > > - > > -AA = %ymm5 > > -BB = %ymm6 > > -CC = %ymm7 > > -DD = %ymm8 > > -EE = %ymm9 > > -TMP = %ymm10 > > -FUN = %ymm11 > > -K = %ymm12 > > -W14 = %ymm13 > > -W15 = %ymm14 > > -W16 = %ymm15 > > - > > -.macro ROTATE_ARGS > > - TMP_ = E > > - E = D > > - D = C > > - C = B > > - B = A > > - A = TMP_ > > -.endm > > - > > -.macro ROTATE_W > > -TMP_ = W16 > > -W16 = W15 > > -W15 = W14 > > -W14 = TMP_ > > -.endm > > - > > -# 8 streams x 5 32bit words per digest x 4 bytes per word > > -#define DIGEST_SIZE (8*5*4) > > - > > -.align 32 > > - > > -# void sha1_x8_avx2(void **input_data, UINT128 *digest, UINT32 size) > > -# arg 1 : pointer to array[4] of pointer to input data > > -# arg 2 : size (in blocks) ;; assumed to be >= 1 > > -# > > -ENTRY(sha1_x8_avx2) > > - > > - # save callee-saved clobbered registers to comply with C function ABI > > - push %r12 > > - push %r13 > > - push %r14 > > - push %r15 > > - > > - #save rsp > > - mov %rsp, RSP_SAVE > > - sub $FRAMESZ, %rsp > > - > > - #align rsp to 32 Bytes > > - and $~0x1F, %rsp > > - > > - ## Initialize digests > > - vmovdqu 0*32(arg1), A > > - vmovdqu 1*32(arg1), B > > - vmovdqu 2*32(arg1), C > > - vmovdqu 3*32(arg1), D > > - vmovdqu 4*32(arg1), E > > - > > - ## transpose input onto stack > > - mov _data_ptr+0*8(arg1),inp0 > > - mov _data_ptr+1*8(arg1),inp1 > > - mov _data_ptr+2*8(arg1),inp2 > > - mov _data_ptr+3*8(arg1),inp3 > > - mov _data_ptr+4*8(arg1),inp4 > > - mov _data_ptr+5*8(arg1),inp5 > > - mov _data_ptr+6*8(arg1),inp6 > > - mov _data_ptr+7*8(arg1),inp7 > > - > > - xor IDX, IDX > > -lloop: > > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), F > > - I=0 > > -.rep 2 > > - VMOVPS (inp0, IDX), T0 > > - VMOVPS (inp1, IDX), T1 > > - VMOVPS (inp2, IDX), T2 > > - VMOVPS (inp3, IDX), T3 > > - VMOVPS (inp4, IDX), T4 > > - VMOVPS (inp5, IDX), T5 > > - VMOVPS (inp6, IDX), T6 > > - VMOVPS (inp7, IDX), T7 > > - > > - TRANSPOSE8 T0, T1, T2, T3, T4, T5, T6, T7, T8, T9 > > - vpshufb F, T0, T0 > > - vmovdqu T0, (I*8)*32(%rsp) > > - vpshufb F, T1, T1 > > - vmovdqu T1, (I*8+1)*32(%rsp) > > - vpshufb F, T2, T2 > > - vmovdqu T2, (I*8+2)*32(%rsp) > > - vpshufb F, T3, T3 > > - vmovdqu T3, (I*8+3)*32(%rsp) > > - vpshufb F, T4, T4 > > - vmovdqu T4, (I*8+4)*32(%rsp) > > - vpshufb F, T5, T5 > > - vmovdqu T5, (I*8+5)*32(%rsp) > > - vpshufb F, T6, T6 > > - vmovdqu T6, (I*8+6)*32(%rsp) > > - vpshufb F, T7, T7 > > - vmovdqu T7, (I*8+7)*32(%rsp) > > - add $32, IDX > > - I = (I+1) > > -.endr > > - # save old digests > > - vmovdqu A,AA > > - vmovdqu B,BB > > - vmovdqu C,CC > > - vmovdqu D,DD > > - vmovdqu E,EE > > - > > -## > > -## perform 0-79 steps > > -## > > - vmovdqu K00_19(%rip), K > > -## do rounds 0...15 > > - I = 0 > > -.rep 16 > > - SHA1_STEP_00_15 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0 > > - ROTATE_ARGS > > - I = (I+1) > > -.endr > > - > > -## do rounds 16...19 > > - vmovdqu ((16 - 16) & 15) * 32 (%rsp), W16 > > - vmovdqu ((16 - 15) & 15) * 32 (%rsp), W15 > > -.rep 4 > > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0 > > - ROTATE_ARGS > > - I = (I+1) > > -.endr > > - > > -## do rounds 20...39 > > - vmovdqu K20_39(%rip), K > > -.rep 20 > > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F1 > > - ROTATE_ARGS > > - I = (I+1) > > -.endr > > - > > -## do rounds 40...59 > > - vmovdqu K40_59(%rip), K > > -.rep 20 > > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F2 > > - ROTATE_ARGS > > - I = (I+1) > > -.endr > > - > > -## do rounds 60...79 > > - vmovdqu K60_79(%rip), K > > -.rep 20 > > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F3 > > - ROTATE_ARGS > > - I = (I+1) > > -.endr > > - > > - vpaddd AA,A,A > > - vpaddd BB,B,B > > - vpaddd CC,C,C > > - vpaddd DD,D,D > > - vpaddd EE,E,E > > - > > - sub $1, arg2 > > - jne lloop > > - > > - # write out digests > > - vmovdqu A, 0*32(arg1) > > - vmovdqu B, 1*32(arg1) > > - vmovdqu C, 2*32(arg1) > > - vmovdqu D, 3*32(arg1) > > - vmovdqu E, 4*32(arg1) > > - > > - # update input pointers > > - add IDX, inp0 > > - add IDX, inp1 > > - add IDX, inp2 > > - add IDX, inp3 > > - add IDX, inp4 > > - add IDX, inp5 > > - add IDX, inp6 > > - add IDX, inp7 > > - mov inp0, _data_ptr (arg1) > > - mov inp1, _data_ptr + 1*8(arg1) > > - mov inp2, _data_ptr + 2*8(arg1) > > - mov inp3, _data_ptr + 3*8(arg1) > > - mov inp4, _data_ptr + 4*8(arg1) > > - mov inp5, _data_ptr + 5*8(arg1) > > - mov inp6, _data_ptr + 6*8(arg1) > > - mov inp7, _data_ptr + 7*8(arg1) > > - > > - ################ > > - ## Postamble > > - > > - mov RSP_SAVE, %rsp > > - > > - # restore callee-saved clobbered registers > > - pop %r15 > > - pop %r14 > > - pop %r13 > > - pop %r12 > > - > > - ret > > -ENDPROC(sha1_x8_avx2) > > - > > - > > -.section .rodata.cst32.K00_19, "aM", @progbits, 32 > > -.align 32 > > -K00_19: > > -.octa 0x5A8279995A8279995A8279995A827999 > > -.octa 0x5A8279995A8279995A8279995A827999 > > - > > -.section .rodata.cst32.K20_39, "aM", @progbits, 32 > > -.align 32 > > -K20_39: > > -.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 > > -.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 > > - > > -.section .rodata.cst32.K40_59, "aM", @progbits, 32 > > -.align 32 > > -K40_59: > > -.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC > > -.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC > > - > > -.section .rodata.cst32.K60_79, "aM", @progbits, 32 > > -.align 32 > > -K60_79: > > -.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 > > -.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 > > - > > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > > -.align 32 > > -PSHUFFLE_BYTE_FLIP_MASK: > > -.octa 0x0c0d0e0f08090a0b0405060700010203 > > -.octa 0x0c0d0e0f08090a0b0405060700010203 > > diff --git a/arch/x86/crypto/sha256-mb/Makefile b/arch/x86/crypto/sha256-mb/Makefile > > deleted file mode 100644 > > index 53ad6e7db747..000000000000 > > --- a/arch/x86/crypto/sha256-mb/Makefile > > +++ /dev/null > > @@ -1,14 +0,0 @@ > > -# SPDX-License-Identifier: GPL-2.0 > > -# > > -# Arch-specific CryptoAPI modules. > > -# > > - > > -OBJECT_FILES_NON_STANDARD := y > > - > > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > > - $(comma)4)$(comma)%ymm2,yes,no) > > -ifeq ($(avx2_supported),yes) > > - obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb.o > > - sha256-mb-y := sha256_mb.o sha256_mb_mgr_flush_avx2.o \ > > - sha256_mb_mgr_init_avx2.o sha256_mb_mgr_submit_avx2.o sha256_x8_avx2.o > > -endif > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb.c b/arch/x86/crypto/sha256-mb/sha256_mb.c > > deleted file mode 100644 > > index 97c5fc43e115..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb.c > > +++ /dev/null > > @@ -1,1013 +0,0 @@ > > -/* > > - * Multi buffer SHA256 algorithm Glue Code > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > > - > > -#include <crypto/internal/hash.h> > > -#include <linux/init.h> > > -#include <linux/module.h> > > -#include <linux/mm.h> > > -#include <linux/cryptohash.h> > > -#include <linux/types.h> > > -#include <linux/list.h> > > -#include <crypto/scatterwalk.h> > > -#include <crypto/sha.h> > > -#include <crypto/mcryptd.h> > > -#include <crypto/crypto_wq.h> > > -#include <asm/byteorder.h> > > -#include <linux/hardirq.h> > > -#include <asm/fpu/api.h> > > -#include "sha256_mb_ctx.h" > > - > > -#define FLUSH_INTERVAL 1000 /* in usec */ > > - > > -static struct mcryptd_alg_state sha256_mb_alg_state; > > - > > -struct sha256_mb_ctx { > > - struct mcryptd_ahash *mcryptd_tfm; > > -}; > > - > > -static inline struct mcryptd_hash_request_ctx > > - *cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx *hash_ctx) > > -{ > > - struct ahash_request *areq; > > - > > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > > -} > > - > > -static inline struct ahash_request > > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > > -{ > > - return container_of((void *) ctx, struct ahash_request, __ctx); > > -} > > - > > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > > - struct ahash_request *areq) > > -{ > > - rctx->flag = HASH_UPDATE; > > -} > > - > > -static asmlinkage void (*sha256_job_mgr_init)(struct sha256_mb_mgr *state); > > -static asmlinkage struct job_sha256* (*sha256_job_mgr_submit) > > - (struct sha256_mb_mgr *state, struct job_sha256 *job); > > -static asmlinkage struct job_sha256* (*sha256_job_mgr_flush) > > - (struct sha256_mb_mgr *state); > > -static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job) > > - (struct sha256_mb_mgr *state); > > - > > -inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2], > > - uint64_t total_len) > > -{ > > - uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1); > > - > > - memset(&padblock[i], 0, SHA256_BLOCK_SIZE); > > - padblock[i] = 0x80; > > - > > - i += ((SHA256_BLOCK_SIZE - 1) & > > - (0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1))) > > - + 1 + SHA256_PADLENGTHFIELD_SIZE; > > - > > -#if SHA256_PADLENGTHFIELD_SIZE == 16 > > - *((uint64_t *) &padblock[i - 16]) = 0; > > -#endif > > - > > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > > - > > - /* Number of extra blocks to hash */ > > - return i >> SHA256_LOG2_BLOCK_SIZE; > > -} > > - > > -static struct sha256_hash_ctx > > - *sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr *mgr, > > - struct sha256_hash_ctx *ctx) > > -{ > > - while (ctx) { > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Clear PROCESSING bit */ > > - ctx->status = HASH_CTX_STS_COMPLETE; > > - return ctx; > > - } > > - > > - /* > > - * If the extra blocks are empty, begin hashing what remains > > - * in the user's buffer. > > - */ > > - if (ctx->partial_block_buffer_length == 0 && > > - ctx->incoming_buffer_length) { > > - > > - const void *buffer = ctx->incoming_buffer; > > - uint32_t len = ctx->incoming_buffer_length; > > - uint32_t copy_len; > > - > > - /* > > - * Only entire blocks can be hashed. > > - * Copy remainder to extra blocks buffer. > > - */ > > - copy_len = len & (SHA256_BLOCK_SIZE-1); > > - > > - if (copy_len) { > > - len -= copy_len; > > - memcpy(ctx->partial_block_buffer, > > - ((const char *) buffer + len), > > - copy_len); > > - ctx->partial_block_buffer_length = copy_len; > > - } > > - > > - ctx->incoming_buffer_length = 0; > > - > > - /* len should be a multiple of the block size now */ > > - assert((len % SHA256_BLOCK_SIZE) == 0); > > - > > - /* Set len to the number of blocks to be hashed */ > > - len >>= SHA256_LOG2_BLOCK_SIZE; > > - > > - if (len) { > > - > > - ctx->job.buffer = (uint8_t *) buffer; > > - ctx->job.len = len; > > - ctx = (struct sha256_hash_ctx *) > > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > > - continue; > > - } > > - } > > - > > - /* > > - * If the extra blocks are not empty, then we are > > - * either on the last block(s) or we need more > > - * user input before continuing. > > - */ > > - if (ctx->status & HASH_CTX_STS_LAST) { > > - > > - uint8_t *buf = ctx->partial_block_buffer; > > - uint32_t n_extra_blocks = > > - sha256_pad(buf, ctx->total_length); > > - > > - ctx->status = (HASH_CTX_STS_PROCESSING | > > - HASH_CTX_STS_COMPLETE); > > - ctx->job.buffer = buf; > > - ctx->job.len = (uint32_t) n_extra_blocks; > > - ctx = (struct sha256_hash_ctx *) > > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > > - continue; > > - } > > - > > - ctx->status = HASH_CTX_STS_IDLE; > > - return ctx; > > - } > > - > > - return NULL; > > -} > > - > > -static struct sha256_hash_ctx > > - *sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr *mgr) > > -{ > > - /* > > - * If get_comp_job returns NULL, there are no jobs complete. > > - * If get_comp_job returns a job, verify that it is safe to return to > > - * the user. If it is not ready, resubmit the job to finish processing. > > - * If sha256_ctx_mgr_resubmit returned a job, it is ready to be > > - * returned. Otherwise, all jobs currently being managed by the > > - * hash_ctx_mgr still need processing. > > - */ > > - struct sha256_hash_ctx *ctx; > > - > > - ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr); > > - return sha256_ctx_mgr_resubmit(mgr, ctx); > > -} > > - > > -static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr) > > -{ > > - sha256_job_mgr_init(&mgr->mgr); > > -} > > - > > -static struct sha256_hash_ctx *sha256_ctx_mgr_submit(struct sha256_ctx_mgr *mgr, > > - struct sha256_hash_ctx *ctx, > > - const void *buffer, > > - uint32_t len, > > - int flags) > > -{ > > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > > - /* User should not pass anything other than UPDATE or LAST */ > > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > > - return ctx; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > > - /* Cannot submit to a currently processing job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > > - return ctx; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Cannot update a finished job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > > - return ctx; > > - } > > - > > - /* If we made it here, there was no error during this call to submit */ > > - ctx->error = HASH_CTX_ERROR_NONE; > > - > > - /* Store buffer ptr info from user */ > > - ctx->incoming_buffer = buffer; > > - ctx->incoming_buffer_length = len; > > - > > - /* > > - * Store the user's request flags and mark this ctx as currently > > - * being processed. > > - */ > > - ctx->status = (flags & HASH_LAST) ? > > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > > - HASH_CTX_STS_PROCESSING; > > - > > - /* Advance byte counter */ > > - ctx->total_length += len; > > - > > - /* > > - * If there is anything currently buffered in the extra blocks, > > - * append to it until it contains a whole block. > > - * Or if the user's buffer contains less than a whole block, > > - * append as much as possible to the extra block. > > - */ > > - if (ctx->partial_block_buffer_length || len < SHA256_BLOCK_SIZE) { > > - /* > > - * Compute how many bytes to copy from user buffer into > > - * extra block > > - */ > > - uint32_t copy_len = SHA256_BLOCK_SIZE - > > - ctx->partial_block_buffer_length; > > - if (len < copy_len) > > - copy_len = len; > > - > > - if (copy_len) { > > - /* Copy and update relevant pointers and counters */ > > - memcpy( > > - &ctx->partial_block_buffer[ctx->partial_block_buffer_length], > > - buffer, copy_len); > > - > > - ctx->partial_block_buffer_length += copy_len; > > - ctx->incoming_buffer = (const void *) > > - ((const char *)buffer + copy_len); > > - ctx->incoming_buffer_length = len - copy_len; > > - } > > - > > - /* The extra block should never contain more than 1 block */ > > - assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE); > > - > > - /* > > - * If the extra block buffer contains exactly 1 block, > > - * it can be hashed. > > - */ > > - if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) { > > - ctx->partial_block_buffer_length = 0; > > - > > - ctx->job.buffer = ctx->partial_block_buffer; > > - ctx->job.len = 1; > > - ctx = (struct sha256_hash_ctx *) > > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > > - } > > - } > > - > > - return sha256_ctx_mgr_resubmit(mgr, ctx); > > -} > > - > > -static struct sha256_hash_ctx *sha256_ctx_mgr_flush(struct sha256_ctx_mgr *mgr) > > -{ > > - struct sha256_hash_ctx *ctx; > > - > > - while (1) { > > - ctx = (struct sha256_hash_ctx *) > > - sha256_job_mgr_flush(&mgr->mgr); > > - > > - /* If flush returned 0, there are no more jobs in flight. */ > > - if (!ctx) > > - return NULL; > > - > > - /* > > - * If flush returned a job, resubmit the job to finish > > - * processing. > > - */ > > - ctx = sha256_ctx_mgr_resubmit(mgr, ctx); > > - > > - /* > > - * If sha256_ctx_mgr_resubmit returned a job, it is ready to > > - * be returned. Otherwise, all jobs currently being managed by > > - * the sha256_ctx_mgr still need processing. Loop. > > - */ > > - if (ctx) > > - return ctx; > > - } > > -} > > - > > -static int sha256_mb_init(struct ahash_request *areq) > > -{ > > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - hash_ctx_init(sctx); > > - sctx->job.result_digest[0] = SHA256_H0; > > - sctx->job.result_digest[1] = SHA256_H1; > > - sctx->job.result_digest[2] = SHA256_H2; > > - sctx->job.result_digest[3] = SHA256_H3; > > - sctx->job.result_digest[4] = SHA256_H4; > > - sctx->job.result_digest[5] = SHA256_H5; > > - sctx->job.result_digest[6] = SHA256_H6; > > - sctx->job.result_digest[7] = SHA256_H7; > > - sctx->total_length = 0; > > - sctx->partial_block_buffer_length = 0; > > - sctx->status = HASH_CTX_STS_IDLE; > > - > > - return 0; > > -} > > - > > -static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > > -{ > > - int i; > > - struct sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > > - __be32 *dst = (__be32 *) rctx->out; > > - > > - for (i = 0; i < 8; ++i) > > - dst[i] = cpu_to_be32(sctx->job.result_digest[i]); > > - > > - return 0; > > -} > > - > > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > > - struct mcryptd_alg_cstate *cstate, bool flush) > > -{ > > - int flag = HASH_UPDATE; > > - int nbytes, err = 0; > > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > > - struct sha256_hash_ctx *sha_ctx; > > - > > - /* more work ? */ > > - while (!(rctx->flag & HASH_DONE)) { > > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > > - if (nbytes < 0) { > > - err = nbytes; > > - goto out; > > - } > > - /* check if the walk is done */ > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - if (rctx->flag & HASH_FINAL) > > - flag |= HASH_LAST; > > - > > - } > > - sha_ctx = (struct sha256_hash_ctx *) > > - ahash_request_ctx(&rctx->areq); > > - kernel_fpu_begin(); > > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, > > - rctx->walk.data, nbytes, flag); > > - if (!sha_ctx) { > > - if (flush) > > - sha_ctx = sha256_ctx_mgr_flush(cstate->mgr); > > - } > > - kernel_fpu_end(); > > - if (sha_ctx) > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - else { > > - rctx = NULL; > > - goto out; > > - } > > - } > > - > > - /* copy the results */ > > - if (rctx->flag & HASH_FINAL) > > - sha256_mb_set_results(rctx); > > - > > -out: > > - *ret_rctx = rctx; > > - return err; > > -} > > - > > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate, > > - int err) > > -{ > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha256_hash_ctx *sha_ctx; > > - struct mcryptd_hash_request_ctx *req_ctx; > > - int ret; > > - > > - /* remove from work list */ > > - spin_lock(&cstate->work_lock); > > - list_del(&rctx->waiter); > > - spin_unlock(&cstate->work_lock); > > - > > - if (irqs_disabled()) > > - rctx->complete(&req->base, err); > > - else { > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > - } > > - > > - /* check to see if there are other jobs that are done */ > > - sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); > > - while (sha_ctx) { > > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&req_ctx, cstate, false); > > - if (req_ctx) { > > - spin_lock(&cstate->work_lock); > > - list_del(&req_ctx->waiter); > > - spin_unlock(&cstate->work_lock); > > - > > - req = cast_mcryptd_ctx_to_req(req_ctx); > > - if (irqs_disabled()) > > - req_ctx->complete(&req->base, ret); > > - else { > > - local_bh_disable(); > > - req_ctx->complete(&req->base, ret); > > - local_bh_enable(); > > - } > > - } > > - sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); > > - } > > - > > - return 0; > > -} > > - > > -static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate) > > -{ > > - unsigned long next_flush; > > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > > - > > - /* initialize tag */ > > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > > - rctx->tag.seq_num = cstate->next_seq_num++; > > - next_flush = rctx->tag.arrival + delay; > > - rctx->tag.expire = next_flush; > > - > > - spin_lock(&cstate->work_lock); > > - list_add_tail(&rctx->waiter, &cstate->work_list); > > - spin_unlock(&cstate->work_lock); > > - > > - mcryptd_arm_flusher(cstate, delay); > > -} > > - > > -static int sha256_mb_update(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha256_hash_ctx *sha_ctx; > > - int ret = 0, nbytes; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) > > - rctx->flag |= HASH_DONE; > > - > > - /* submit */ > > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > > - sha256_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > > - nbytes, HASH_UPDATE); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha256_mb_finup(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha256_hash_ctx *sha_ctx; > > - int ret = 0, flag = HASH_UPDATE, nbytes; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - flag = HASH_LAST; > > - } > > - > > - /* submit */ > > - rctx->flag |= HASH_FINAL; > > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > > - sha256_mb_add_list(rctx, cstate); > > - > > - kernel_fpu_begin(); > > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > > - nbytes, flag); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha256_mb_final(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, > > - areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > > - > > - struct sha256_hash_ctx *sha_ctx; > > - int ret = 0; > > - u8 data; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - rctx->flag |= HASH_DONE | HASH_FINAL; > > - > > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > > - /* flag HASH_FINAL and 0 data size */ > > - sha256_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, > > - HASH_LAST); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha256_mb_export(struct ahash_request *areq, void *out) > > -{ > > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(out, sctx, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha256_mb_import(struct ahash_request *areq, const void *in) > > -{ > > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(sctx, in, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm) > > -{ > > - struct mcryptd_ahash *mcryptd_tfm; > > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - struct mcryptd_hash_ctx *mctx; > > - > > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb", > > - CRYPTO_ALG_INTERNAL, > > - CRYPTO_ALG_INTERNAL); > > - if (IS_ERR(mcryptd_tfm)) > > - return PTR_ERR(mcryptd_tfm); > > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > > - mctx->alg_state = &sha256_mb_alg_state; > > - ctx->mcryptd_tfm = mcryptd_tfm; > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > > - > > - return 0; > > -} > > - > > -static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm) > > -{ > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - sizeof(struct sha256_hash_ctx)); > > - > > - return 0; > > -} > > - > > -static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static struct ahash_alg sha256_mb_areq_alg = { > > - .init = sha256_mb_init, > > - .update = sha256_mb_update, > > - .final = sha256_mb_final, > > - .finup = sha256_mb_finup, > > - .export = sha256_mb_export, > > - .import = sha256_mb_import, > > - .halg = { > > - .digestsize = SHA256_DIGEST_SIZE, > > - .statesize = sizeof(struct sha256_hash_ctx), > > - .base = { > > - .cra_name = "__sha256-mb", > > - .cra_driver_name = "__intel_sha256-mb", > > - .cra_priority = 100, > > - /* > > - * use ASYNC flag as some buffers in multi-buffer > > - * algo may not have completed before hashing thread > > - * sleep > > - */ > > - .cra_flags = CRYPTO_ALG_ASYNC | > > - CRYPTO_ALG_INTERNAL, > > - .cra_blocksize = SHA256_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT > > - (sha256_mb_areq_alg.halg.base.cra_list), > > - .cra_init = sha256_mb_areq_init_tfm, > > - .cra_exit = sha256_mb_areq_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha256_hash_ctx), > > - } > > - } > > -}; > > - > > -static int sha256_mb_async_init(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_init(mcryptd_req); > > -} > > - > > -static int sha256_mb_async_update(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_update(mcryptd_req); > > -} > > - > > -static int sha256_mb_async_finup(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_finup(mcryptd_req); > > -} > > - > > -static int sha256_mb_async_final(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_final(mcryptd_req); > > -} > > - > > -static int sha256_mb_async_digest(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_digest(mcryptd_req); > > -} > > - > > -static int sha256_mb_async_export(struct ahash_request *req, void *out) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_export(mcryptd_req, out); > > -} > > - > > -static int sha256_mb_async_import(struct ahash_request *req, const void *in) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > > - struct mcryptd_hash_request_ctx *rctx; > > - struct ahash_request *areq; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - rctx = ahash_request_ctx(mcryptd_req); > > - areq = &rctx->areq; > > - > > - ahash_request_set_tfm(areq, child); > > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > > - rctx->complete, req); > > - > > - return crypto_ahash_import(mcryptd_req, in); > > -} > > - > > -static struct ahash_alg sha256_mb_async_alg = { > > - .init = sha256_mb_async_init, > > - .update = sha256_mb_async_update, > > - .final = sha256_mb_async_final, > > - .finup = sha256_mb_async_finup, > > - .export = sha256_mb_async_export, > > - .import = sha256_mb_async_import, > > - .digest = sha256_mb_async_digest, > > - .halg = { > > - .digestsize = SHA256_DIGEST_SIZE, > > - .statesize = sizeof(struct sha256_hash_ctx), > > - .base = { > > - .cra_name = "sha256", > > - .cra_driver_name = "sha256_mb", > > - /* > > - * Low priority, since with few concurrent hash requests > > - * this is extremely slow due to the flush delay. Users > > - * whose workloads would benefit from this can request > > - * it explicitly by driver name, or can increase its > > - * priority at runtime using NETLINK_CRYPTO. > > - */ > > - .cra_priority = 50, > > - .cra_flags = CRYPTO_ALG_ASYNC, > > - .cra_blocksize = SHA256_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT > > - (sha256_mb_async_alg.halg.base.cra_list), > > - .cra_init = sha256_mb_async_init_tfm, > > - .cra_exit = sha256_mb_async_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha256_mb_ctx), > > - .cra_alignmask = 0, > > - }, > > - }, > > -}; > > - > > -static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate) > > -{ > > - struct mcryptd_hash_request_ctx *rctx; > > - unsigned long cur_time; > > - unsigned long next_flush = 0; > > - struct sha256_hash_ctx *sha_ctx; > > - > > - > > - cur_time = jiffies; > > - > > - while (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - if (time_before(cur_time, rctx->tag.expire)) > > - break; > > - kernel_fpu_begin(); > > - sha_ctx = (struct sha256_hash_ctx *) > > - sha256_ctx_mgr_flush(cstate->mgr); > > - kernel_fpu_end(); > > - if (!sha_ctx) { > > - pr_err("sha256_mb error: nothing got" > > - " flushed for non-empty list\n"); > > - break; > > - } > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - sha_finish_walk(&rctx, cstate, true); > > - sha_complete_job(rctx, cstate, 0); > > - } > > - > > - if (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - /* get the hash context and then flush time */ > > - next_flush = rctx->tag.expire; > > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > > - } > > - return next_flush; > > -} > > - > > -static int __init sha256_mb_mod_init(void) > > -{ > > - > > - int cpu; > > - int err; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - /* check for dependent cpu features */ > > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > > - !boot_cpu_has(X86_FEATURE_BMI2)) > > - return -ENODEV; > > - > > - /* initialize multibuffer structures */ > > - sha256_mb_alg_state.alg_cstate = alloc_percpu > > - (struct mcryptd_alg_cstate); > > - > > - sha256_job_mgr_init = sha256_mb_mgr_init_avx2; > > - sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2; > > - sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2; > > - sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2; > > - > > - if (!sha256_mb_alg_state.alg_cstate) > > - return -ENOMEM; > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > > - cpu_state->next_flush = 0; > > - cpu_state->next_seq_num = 0; > > - cpu_state->flusher_engaged = false; > > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > > - cpu_state->cpu = cpu; > > - cpu_state->alg_state = &sha256_mb_alg_state; > > - cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr), > > - GFP_KERNEL); > > - if (!cpu_state->mgr) > > - goto err2; > > - sha256_ctx_mgr_init(cpu_state->mgr); > > - INIT_LIST_HEAD(&cpu_state->work_list); > > - spin_lock_init(&cpu_state->work_lock); > > - } > > - sha256_mb_alg_state.flusher = &sha256_mb_flusher; > > - > > - err = crypto_register_ahash(&sha256_mb_areq_alg); > > - if (err) > > - goto err2; > > - err = crypto_register_ahash(&sha256_mb_async_alg); > > - if (err) > > - goto err1; > > - > > - > > - return 0; > > -err1: > > - crypto_unregister_ahash(&sha256_mb_areq_alg); > > -err2: > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha256_mb_alg_state.alg_cstate); > > - return -ENODEV; > > -} > > - > > -static void __exit sha256_mb_mod_fini(void) > > -{ > > - int cpu; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - crypto_unregister_ahash(&sha256_mb_async_alg); > > - crypto_unregister_ahash(&sha256_mb_areq_alg); > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha256_mb_alg_state.alg_cstate); > > -} > > - > > -module_init(sha256_mb_mod_init); > > -module_exit(sha256_mb_mod_fini); > > - > > -MODULE_LICENSE("GPL"); > > -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated"); > > - > > -MODULE_ALIAS_CRYPTO("sha256"); > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > > deleted file mode 100644 > > index 7c432543dc7f..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > > +++ /dev/null > > @@ -1,134 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA256 context > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#ifndef _SHA_MB_CTX_INTERNAL_H > > -#define _SHA_MB_CTX_INTERNAL_H > > - > > -#include "sha256_mb_mgr.h" > > - > > -#define HASH_UPDATE 0x00 > > -#define HASH_LAST 0x01 > > -#define HASH_DONE 0x02 > > -#define HASH_FINAL 0x04 > > - > > -#define HASH_CTX_STS_IDLE 0x00 > > -#define HASH_CTX_STS_PROCESSING 0x01 > > -#define HASH_CTX_STS_LAST 0x02 > > -#define HASH_CTX_STS_COMPLETE 0x04 > > - > > -enum hash_ctx_error { > > - HASH_CTX_ERROR_NONE = 0, > > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > > - > > -#ifdef HASH_CTX_DEBUG > > - HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4, > > -#endif > > -}; > > - > > - > > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > > -#define hash_ctx_status(ctx) ((ctx)->status) > > -#define hash_ctx_error(ctx) ((ctx)->error) > > -#define hash_ctx_init(ctx) \ > > - do { \ > > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > > - } while (0) > > - > > - > > -/* Hash Constants and Typedefs */ > > -#define SHA256_DIGEST_LENGTH 8 > > -#define SHA256_LOG2_BLOCK_SIZE 6 > > - > > -#define SHA256_PADLENGTHFIELD_SIZE 8 > > - > > -#ifdef SHA_MB_DEBUG > > -#define assert(expr) \ > > -do { \ > > - if (unlikely(!(expr))) { \ > > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > > - #expr, __FILE__, __func__, __LINE__); \ > > - } \ > > -} while (0) > > -#else > > -#define assert(expr) do {} while (0) > > -#endif > > - > > -struct sha256_ctx_mgr { > > - struct sha256_mb_mgr mgr; > > -}; > > - > > -/* typedef struct sha256_ctx_mgr sha256_ctx_mgr; */ > > - > > -struct sha256_hash_ctx { > > - /* Must be at struct offset 0 */ > > - struct job_sha256 job; > > - /* status flag */ > > - int status; > > - /* error flag */ > > - int error; > > - > > - uint64_t total_length; > > - const void *incoming_buffer; > > - uint32_t incoming_buffer_length; > > - uint8_t partial_block_buffer[SHA256_BLOCK_SIZE * 2]; > > - uint32_t partial_block_buffer_length; > > - void *user_data; > > -}; > > - > > -#endif > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h b/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > > deleted file mode 100644 > > index b01ae408c56d..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > > +++ /dev/null > > @@ -1,108 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA256 algorithm manager > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > -#ifndef __SHA_MB_MGR_H > > -#define __SHA_MB_MGR_H > > - > > -#include <linux/types.h> > > - > > -#define NUM_SHA256_DIGEST_WORDS 8 > > - > > -enum job_sts { STS_UNKNOWN = 0, > > - STS_BEING_PROCESSED = 1, > > - STS_COMPLETED = 2, > > - STS_INTERNAL_ERROR = 3, > > - STS_ERROR = 4 > > -}; > > - > > -struct job_sha256 { > > - u8 *buffer; > > - u32 len; > > - u32 result_digest[NUM_SHA256_DIGEST_WORDS] __aligned(32); > > - enum job_sts status; > > - void *user_data; > > -}; > > - > > -/* SHA256 out-of-order scheduler */ > > - > > -/* typedef uint32_t sha8_digest_array[8][8]; */ > > - > > -struct sha256_args_x8 { > > - uint32_t digest[8][8]; > > - uint8_t *data_ptr[8]; > > -}; > > - > > -struct sha256_lane_data { > > - struct job_sha256 *job_in_lane; > > -}; > > - > > -struct sha256_mb_mgr { > > - struct sha256_args_x8 args; > > - > > - uint32_t lens[8]; > > - > > - /* each byte is index (0...7) of unused lanes */ > > - uint64_t unused_lanes; > > - /* byte 4 is set to FF as a flag */ > > - struct sha256_lane_data ldata[8]; > > -}; > > - > > - > > -#define SHA256_MB_MGR_NUM_LANES_AVX2 8 > > - > > -void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state); > > -struct job_sha256 *sha256_mb_mgr_submit_avx2(struct sha256_mb_mgr *state, > > - struct job_sha256 *job); > > -struct job_sha256 *sha256_mb_mgr_flush_avx2(struct sha256_mb_mgr *state); > > -struct job_sha256 *sha256_mb_mgr_get_comp_job_avx2(struct sha256_mb_mgr *state); > > - > > -#endif > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > > deleted file mode 100644 > > index 5c377bac21d0..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > > +++ /dev/null > > @@ -1,304 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA256 algorithm data structure > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -# Macros for defining data structures > > - > > -# Usage example > > - > > -#START_FIELDS # JOB_AES > > -### name size align > > -#FIELD _plaintext, 8, 8 # pointer to plaintext > > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > > -#FIELD _IV, 16, 8 # IV > > -#FIELD _keys, 8, 8 # pointer to keys > > -#FIELD _len, 4, 4 # length in bytes > > -#FIELD _status, 4, 4 # status enumeration > > -#FIELD _user_data, 8, 8 # pointer to user data > > -#UNION _union, size1, align1, \ > > -# size2, align2, \ > > -# size3, align3, \ > > -# ... > > -#END_FIELDS > > -#%assign _JOB_AES_size _FIELD_OFFSET > > -#%assign _JOB_AES_align _STRUCT_ALIGN > > - > > -######################################################################### > > - > > -# Alternate "struc-like" syntax: > > -# STRUCT job_aes2 > > -# RES_Q .plaintext, 1 > > -# RES_Q .ciphertext, 1 > > -# RES_DQ .IV, 1 > > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > > -# RES_U .union, size1, align1, \ > > -# size2, align2, \ > > -# ... > > -# ENDSTRUCT > > -# # Following only needed if nesting > > -# %assign job_aes2_size _FIELD_OFFSET > > -# %assign job_aes2_align _STRUCT_ALIGN > > -# > > -# RES_* macros take a name, a count and an optional alignment. > > -# The count in in terms of the base size of the macro, and the > > -# default alignment is the base size. > > -# The macros are: > > -# Macro Base size > > -# RES_B 1 > > -# RES_W 2 > > -# RES_D 4 > > -# RES_Q 8 > > -# RES_DQ 16 > > -# RES_Y 32 > > -# RES_Z 64 > > -# > > -# RES_U defines a union. It's arguments are a name and two or more > > -# pairs of "size, alignment" > > -# > > -# The two assigns are only needed if this structure is being nested > > -# within another. Even if the assigns are not done, one can still use > > -# STRUCT_NAME_size as the size of the structure. > > -# > > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > > -# > > -# The differences between this and using "struc" directly are that each > > -# type is implicitly aligned to its natural length (although this can be > > -# over-ridden with an explicit third parameter), and that the structure > > -# is padded at the end to its overall alignment. > > -# > > - > > -######################################################################### > > - > > -#ifndef _DATASTRUCT_ASM_ > > -#define _DATASTRUCT_ASM_ > > - > > -#define SZ8 8*SHA256_DIGEST_WORD_SIZE > > -#define ROUNDS 64*SZ8 > > -#define PTR_SZ 8 > > -#define SHA256_DIGEST_WORD_SIZE 4 > > -#define MAX_SHA256_LANES 8 > > -#define SHA256_DIGEST_WORDS 8 > > -#define SHA256_DIGEST_ROW_SIZE (MAX_SHA256_LANES * SHA256_DIGEST_WORD_SIZE) > > -#define SHA256_DIGEST_SIZE (SHA256_DIGEST_ROW_SIZE * SHA256_DIGEST_WORDS) > > -#define SHA256_BLK_SZ 64 > > - > > -# START_FIELDS > > -.macro START_FIELDS > > - _FIELD_OFFSET = 0 > > - _STRUCT_ALIGN = 0 > > -.endm > > - > > -# FIELD name size align > > -.macro FIELD name size align > > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > > - \name = _FIELD_OFFSET > > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > > -.if (\align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = \align > > -.endif > > -.endm > > - > > -# END_FIELDS > > -.macro END_FIELDS > > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > > -.endm > > - > > -######################################################################## > > - > > -.macro STRUCT p1 > > -START_FIELDS > > -.struc \p1 > > -.endm > > - > > -.macro ENDSTRUCT > > - tmp = _FIELD_OFFSET > > - END_FIELDS > > - tmp = (_FIELD_OFFSET - %%tmp) > > -.if (tmp > 0) > > - .lcomm tmp > > -.endif > > -.endstruc > > -.endm > > - > > -## RES_int name size align > > -.macro RES_int p1 p2 p3 > > - name = \p1 > > - size = \p2 > > - align = .\p3 > > - > > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > > -.align align > > -.lcomm name size > > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > > -.if (align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = align > > -.endif > > -.endm > > - > > -# macro RES_B name, size [, align] > > -.macro RES_B _name, _size, _align=1 > > -RES_int _name _size _align > > -.endm > > - > > -# macro RES_W name, size [, align] > > -.macro RES_W _name, _size, _align=2 > > -RES_int _name 2*(_size) _align > > -.endm > > - > > -# macro RES_D name, size [, align] > > -.macro RES_D _name, _size, _align=4 > > -RES_int _name 4*(_size) _align > > -.endm > > - > > -# macro RES_Q name, size [, align] > > -.macro RES_Q _name, _size, _align=8 > > -RES_int _name 8*(_size) _align > > -.endm > > - > > -# macro RES_DQ name, size [, align] > > -.macro RES_DQ _name, _size, _align=16 > > -RES_int _name 16*(_size) _align > > -.endm > > - > > -# macro RES_Y name, size [, align] > > -.macro RES_Y _name, _size, _align=32 > > -RES_int _name 32*(_size) _align > > -.endm > > - > > -# macro RES_Z name, size [, align] > > -.macro RES_Z _name, _size, _align=64 > > -RES_int _name 64*(_size) _align > > -.endm > > - > > -#endif > > - > > - > > -######################################################################## > > -#### Define SHA256 Out Of Order Data Structures > > -######################################################################## > > - > > -START_FIELDS # LANE_DATA > > -### name size align > > -FIELD _job_in_lane, 8, 8 # pointer to job object > > -END_FIELDS > > - > > - _LANE_DATA_size = _FIELD_OFFSET > > - _LANE_DATA_align = _STRUCT_ALIGN > > - > > -######################################################################## > > - > > -START_FIELDS # SHA256_ARGS_X4 > > -### name size align > > -FIELD _digest, 4*8*8, 4 # transposed digest > > -FIELD _data_ptr, 8*8, 8 # array of pointers to data > > -END_FIELDS > > - > > - _SHA256_ARGS_X4_size = _FIELD_OFFSET > > - _SHA256_ARGS_X4_align = _STRUCT_ALIGN > > - _SHA256_ARGS_X8_size = _FIELD_OFFSET > > - _SHA256_ARGS_X8_align = _STRUCT_ALIGN > > - > > -####################################################################### > > - > > -START_FIELDS # MB_MGR > > -### name size align > > -FIELD _args, _SHA256_ARGS_X4_size, _SHA256_ARGS_X4_align > > -FIELD _lens, 4*8, 8 > > -FIELD _unused_lanes, 8, 8 > > -FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align > > -END_FIELDS > > - > > - _MB_MGR_size = _FIELD_OFFSET > > - _MB_MGR_align = _STRUCT_ALIGN > > - > > -_args_digest = _args + _digest > > -_args_data_ptr = _args + _data_ptr > > - > > -####################################################################### > > - > > -START_FIELDS #STACK_FRAME > > -### name size align > > -FIELD _data, 16*SZ8, 1 # transposed digest > > -FIELD _digest, 8*SZ8, 1 # array of pointers to data > > -FIELD _ytmp, 4*SZ8, 1 > > -FIELD _rsp, 8, 1 > > -END_FIELDS > > - > > - _STACK_FRAME_size = _FIELD_OFFSET > > - _STACK_FRAME_align = _STRUCT_ALIGN > > - > > -####################################################################### > > - > > -######################################################################## > > -#### Define constants > > -######################################################################## > > - > > -#define STS_UNKNOWN 0 > > -#define STS_BEING_PROCESSED 1 > > -#define STS_COMPLETED 2 > > - > > -######################################################################## > > -#### Define JOB_SHA256 structure > > -######################################################################## > > - > > -START_FIELDS # JOB_SHA256 > > - > > -### name size align > > -FIELD _buffer, 8, 8 # pointer to buffer > > -FIELD _len, 8, 8 # length in bytes > > -FIELD _result_digest, 8*4, 32 # Digest (output) > > -FIELD _status, 4, 4 > > -FIELD _user_data, 8, 8 > > -END_FIELDS > > - > > - _JOB_SHA256_size = _FIELD_OFFSET > > - _JOB_SHA256_align = _STRUCT_ALIGN > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > > deleted file mode 100644 > > index d2364c55bbde..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > > +++ /dev/null > > @@ -1,307 +0,0 @@ > > -/* > > - * Flush routine for SHA256 multibuffer > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha256_mb_mgr_datastruct.S" > > - > > -.extern sha256_x8_avx2 > > - > > -#LINUX register definitions > > -#define arg1 %rdi > > -#define arg2 %rsi > > - > > -# Common register definitions > > -#define state arg1 > > -#define job arg2 > > -#define len2 arg2 > > - > > -# idx must be a register not clobberred by sha1_mult > > -#define idx %r8 > > -#define DWORD_idx %r8d > > - > > -#define unused_lanes %rbx > > -#define lane_data %rbx > > -#define tmp2 %rbx > > -#define tmp2_w %ebx > > - > > -#define job_rax %rax > > -#define tmp1 %rax > > -#define size_offset %rax > > -#define tmp %rax > > -#define start_offset %rax > > - > > -#define tmp3 %arg1 > > - > > -#define extra_blocks %arg2 > > -#define p %arg2 > > - > > -.macro LABEL prefix n > > -\prefix\n\(): > > -.endm > > - > > -.macro JNE_SKIP i > > -jne skip_\i > > -.endm > > - > > -.altmacro > > -.macro SET_OFFSET _offset > > -offset = \_offset > > -.endm > > -.noaltmacro > > - > > -# JOB_SHA256* sha256_mb_mgr_flush_avx2(MB_MGR *state) > > -# arg 1 : rcx : state > > -ENTRY(sha256_mb_mgr_flush_avx2) > > - FRAME_BEGIN > > - push %rbx > > - > > - # If bit (32+3) is set, then all lanes are empty > > - mov _unused_lanes(state), unused_lanes > > - bt $32+3, unused_lanes > > - jc return_null > > - > > - # find a lane with a non-null job > > - xor idx, idx > > - offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne one(%rip), idx > > - offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne two(%rip), idx > > - offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne three(%rip), idx > > - offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne four(%rip), idx > > - offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne five(%rip), idx > > - offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne six(%rip), idx > > - offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne seven(%rip), idx > > - > > - # copy idx to empty lanes > > -copy_lane_data: > > - offset = (_args + _data_ptr) > > - mov offset(state,idx,8), tmp > > - > > - I = 0 > > -.rep 8 > > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > -.altmacro > > - JNE_SKIP %I > > - offset = (_args + _data_ptr + 8*I) > > - mov tmp, offset(state) > > - offset = (_lens + 4*I) > > - movl $0xFFFFFFFF, offset(state) > > -LABEL skip_ %I > > - I = (I+1) > > -.noaltmacro > > -.endr > > - > > - # Find min length > > - vmovdqu _lens+0*16(state), %xmm0 > > - vmovdqu _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - mov idx, len2 > > - and $0xF, idx > > - shr $4, len2 > > - jz len_is_0 > > - > > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > > - vpshufd $0, %xmm2, %xmm2 > > - > > - vpsubd %xmm2, %xmm0, %xmm0 > > - vpsubd %xmm2, %xmm1, %xmm1 > > - > > - vmovdqu %xmm0, _lens+0*16(state) > > - vmovdqu %xmm1, _lens+1*16(state) > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha256_x8_avx2 > > - # state and idx are intact > > - > > -len_is_0: > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - > > - mov unused_lanes, _unused_lanes(state) > > - movl $0xFFFFFFFF, _lens(state,idx,4) > > - > > - vmovd _args_digest(state , idx, 4) , %xmm0 > > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > > - vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 > > - vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 > > - vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - offset = (_result_digest + 1*16) > > - vmovdqu %xmm1, offset(job_rax) > > - > > -return: > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > -ENDPROC(sha256_mb_mgr_flush_avx2) > > - > > -############################################################################## > > - > > -.align 16 > > -ENTRY(sha256_mb_mgr_get_comp_job_avx2) > > - push %rbx > > - > > - ## if bit 32+3 is set, then all lanes are empty > > - mov _unused_lanes(state), unused_lanes > > - bt $(32+3), unused_lanes > > - jc .return_null > > - > > - # Find min length > > - vmovdqu _lens(state), %xmm0 > > - vmovdqu _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - test $~0xF, idx > > - jnz .return_null > > - > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens(state, idx, 4) > > - > > - vmovd _args_digest(state, idx, 4), %xmm0 > > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > > - vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 > > - vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 > > - vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - offset = (_result_digest + 1*16) > > - vmovdqu %xmm1, offset(job_rax) > > - > > - pop %rbx > > - > > - ret > > - > > -.return_null: > > - xor job_rax, job_rax > > - pop %rbx > > - ret > > -ENDPROC(sha256_mb_mgr_get_comp_job_avx2) > > - > > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > > -.align 16 > > -clear_low_nibble: > > -.octa 0x000000000000000000000000FFFFFFF0 > > - > > -.section .rodata.cst8, "aM", @progbits, 8 > > -.align 8 > > -one: > > -.quad 1 > > -two: > > -.quad 2 > > -three: > > -.quad 3 > > -four: > > -.quad 4 > > -five: > > -.quad 5 > > -six: > > -.quad 6 > > -seven: > > -.quad 7 > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > > deleted file mode 100644 > > index b0c498371e67..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > > +++ /dev/null > > @@ -1,65 +0,0 @@ > > -/* > > - * Initialization code for multi buffer SHA256 algorithm for AVX2 > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include "sha256_mb_mgr.h" > > - > > -void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state) > > -{ > > - unsigned int j; > > - > > - state->unused_lanes = 0xF76543210ULL; > > - for (j = 0; j < 8; j++) { > > - state->lens[j] = 0xFFFFFFFF; > > - state->ldata[j].job_in_lane = NULL; > > - } > > -} > > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > > deleted file mode 100644 > > index b36ae7454084..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > > +++ /dev/null > > @@ -1,214 +0,0 @@ > > -/* > > - * Buffer submit code for multi buffer SHA256 algorithm > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha256_mb_mgr_datastruct.S" > > - > > -.extern sha256_x8_avx2 > > - > > -# LINUX register definitions > > -arg1 = %rdi > > -arg2 = %rsi > > -size_offset = %rcx > > -tmp2 = %rcx > > -extra_blocks = %rdx > > - > > -# Common definitions > > -#define state arg1 > > -#define job %rsi > > -#define len2 arg2 > > -#define p2 arg2 > > - > > -# idx must be a register not clobberred by sha1_x8_avx2 > > -idx = %r8 > > -DWORD_idx = %r8d > > -last_len = %r8 > > - > > -p = %r11 > > -start_offset = %r11 > > - > > -unused_lanes = %rbx > > -BYTE_unused_lanes = %bl > > - > > -job_rax = %rax > > -len = %rax > > -DWORD_len = %eax > > - > > -lane = %r12 > > -tmp3 = %r12 > > - > > -tmp = %r9 > > -DWORD_tmp = %r9d > > - > > -lane_data = %r10 > > - > > -# JOB* sha256_mb_mgr_submit_avx2(MB_MGR *state, JOB_SHA256 *job) > > -# arg 1 : rcx : state > > -# arg 2 : rdx : job > > -ENTRY(sha256_mb_mgr_submit_avx2) > > - FRAME_BEGIN > > - push %rbx > > - push %r12 > > - > > - mov _unused_lanes(state), unused_lanes > > - mov unused_lanes, lane > > - and $0xF, lane > > - shr $4, unused_lanes > > - imul $_LANE_DATA_size, lane, lane_data > > - movl $STS_BEING_PROCESSED, _status(job) > > - lea _ldata(state, lane_data), lane_data > > - mov unused_lanes, _unused_lanes(state) > > - movl _len(job), DWORD_len > > - > > - mov job, _job_in_lane(lane_data) > > - shl $4, len > > - or lane, len > > - > > - movl DWORD_len, _lens(state , lane, 4) > > - > > - # Load digest words from result_digest > > - vmovdqu _result_digest(job), %xmm0 > > - vmovdqu _result_digest+1*16(job), %xmm1 > > - vmovd %xmm0, _args_digest(state, lane, 4) > > - vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4) > > - vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4) > > - vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4) > > - vmovd %xmm1, _args_digest+4*32(state , lane, 4) > > - > > - vpextrd $1, %xmm1, _args_digest+5*32(state , lane, 4) > > - vpextrd $2, %xmm1, _args_digest+6*32(state , lane, 4) > > - vpextrd $3, %xmm1, _args_digest+7*32(state , lane, 4) > > - > > - mov _buffer(job), p > > - mov p, _args_data_ptr(state, lane, 8) > > - > > - cmp $0xF, unused_lanes > > - jne return_null > > - > > -start_loop: > > - # Find min length > > - vmovdqa _lens(state), %xmm0 > > - vmovdqa _lens+1*16(state), %xmm1 > > - > > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > > - > > - vmovd %xmm2, DWORD_idx > > - mov idx, len2 > > - and $0xF, idx > > - shr $4, len2 > > - jz len_is_0 > > - > > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > > - vpshufd $0, %xmm2, %xmm2 > > - > > - vpsubd %xmm2, %xmm0, %xmm0 > > - vpsubd %xmm2, %xmm1, %xmm1 > > - > > - vmovdqa %xmm0, _lens + 0*16(state) > > - vmovdqa %xmm1, _lens + 1*16(state) > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha256_x8_avx2 > > - > > - # state and idx are intact > > - > > -len_is_0: > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - mov _unused_lanes(state), unused_lanes > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - shl $4, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens(state,idx,4) > > - > > - vmovd _args_digest(state, idx, 4), %xmm0 > > - vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0 > > - vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0 > > - vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0 > > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > > - > > - vpinsrd $1, _args_digest+5*32(state , idx, 4), %xmm1, %xmm1 > > - vpinsrd $2, _args_digest+6*32(state , idx, 4), %xmm1, %xmm1 > > - vpinsrd $3, _args_digest+7*32(state , idx, 4), %xmm1, %xmm1 > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > > - > > -return: > > - pop %r12 > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > - > > -ENDPROC(sha256_mb_mgr_submit_avx2) > > - > > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > > -.align 16 > > -clear_low_nibble: > > - .octa 0x000000000000000000000000FFFFFFF0 > > diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > > deleted file mode 100644 > > index 1687c80c5995..000000000000 > > --- a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > > +++ /dev/null > > @@ -1,598 +0,0 @@ > > -/* > > - * Multi-buffer SHA256 algorithm hash compute routine > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include "sha256_mb_mgr_datastruct.S" > > - > > -## code to compute oct SHA256 using SSE-256 > > -## outer calling routine takes care of save and restore of XMM registers > > -## Logic designed/laid out by JDG > > - > > -## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; %ymm0-15 > > -## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15 > > -## Linux preserves: rdi rbp r8 > > -## > > -## clobbers %ymm0-15 > > - > > -arg1 = %rdi > > -arg2 = %rsi > > -reg3 = %rcx > > -reg4 = %rdx > > - > > -# Common definitions > > -STATE = arg1 > > -INP_SIZE = arg2 > > - > > -IDX = %rax > > -ROUND = %rbx > > -TBL = reg3 > > - > > -inp0 = %r9 > > -inp1 = %r10 > > -inp2 = %r11 > > -inp3 = %r12 > > -inp4 = %r13 > > -inp5 = %r14 > > -inp6 = %r15 > > -inp7 = reg4 > > - > > -a = %ymm0 > > -b = %ymm1 > > -c = %ymm2 > > -d = %ymm3 > > -e = %ymm4 > > -f = %ymm5 > > -g = %ymm6 > > -h = %ymm7 > > - > > -T1 = %ymm8 > > - > > -a0 = %ymm12 > > -a1 = %ymm13 > > -a2 = %ymm14 > > -TMP = %ymm15 > > -TMP0 = %ymm6 > > -TMP1 = %ymm7 > > - > > -TT0 = %ymm8 > > -TT1 = %ymm9 > > -TT2 = %ymm10 > > -TT3 = %ymm11 > > -TT4 = %ymm12 > > -TT5 = %ymm13 > > -TT6 = %ymm14 > > -TT7 = %ymm15 > > - > > -# Define stack usage > > - > > -# Assume stack aligned to 32 bytes before call > > -# Therefore FRAMESZ mod 32 must be 32-8 = 24 > > - > > -#define FRAMESZ 0x388 > > - > > -#define VMOVPS vmovups > > - > > -# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1 > > -# "transpose" data in {r0...r7} using temps {t0...t1} > > -# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > > -# r4 = {e7 e6 e5 e4 e3 e2 e1 e0} > > -# r5 = {f7 f6 f5 f4 f3 f2 f1 f0} > > -# r6 = {g7 g6 g5 g4 g3 g2 g1 g0} > > -# r7 = {h7 h6 h5 h4 h3 h2 h1 h0} > > -# > > -# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > > -# r0 = {h0 g0 f0 e0 d0 c0 b0 a0} > > -# r1 = {h1 g1 f1 e1 d1 c1 b1 a1} > > -# r2 = {h2 g2 f2 e2 d2 c2 b2 a2} > > -# r3 = {h3 g3 f3 e3 d3 c3 b3 a3} > > -# r4 = {h4 g4 f4 e4 d4 c4 b4 a4} > > -# r5 = {h5 g5 f5 e5 d5 c5 b5 a5} > > -# r6 = {h6 g6 f6 e6 d6 c6 b6 a6} > > -# r7 = {h7 g7 f7 e7 d7 c7 b7 a7} > > -# > > - > > -.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1 > > - # process top half (r0..r3) {a...d} > > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > > - vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1} > > - vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2} > > - vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3} > > - vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0} > > - > > - # use r2 in place of t0 > > - # process bottom half (r4..r7) {e...h} > > - vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0} > > - vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2} > > - vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0} > > - vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2} > > - vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1} > > - vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2} > > - vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3} > > - vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0} > > - > > - vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6 > > - vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2 > > - vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5 > > - vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1 > > - vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7 > > - vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3 > > - vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4 > > - vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0 > > - > > -.endm > > - > > -.macro ROTATE_ARGS > > -TMP_ = h > > -h = g > > -g = f > > -f = e > > -e = d > > -d = c > > -c = b > > -b = a > > -a = TMP_ > > -.endm > > - > > -.macro _PRORD reg imm tmp > > - vpslld $(32-\imm),\reg,\tmp > > - vpsrld $\imm,\reg, \reg > > - vpor \tmp,\reg, \reg > > -.endm > > - > > -# PRORD_nd reg, imm, tmp, src > > -.macro _PRORD_nd reg imm tmp src > > - vpslld $(32-\imm), \src, \tmp > > - vpsrld $\imm, \src, \reg > > - vpor \tmp, \reg, \reg > > -.endm > > - > > -# PRORD dst/src, amt > > -.macro PRORD reg imm > > - _PRORD \reg,\imm,TMP > > -.endm > > - > > -# PRORD_nd dst, src, amt > > -.macro PRORD_nd reg tmp imm > > - _PRORD_nd \reg, \imm, TMP, \tmp > > -.endm > > - > > -# arguments passed implicitly in preprocessor symbols i, a...h > > -.macro ROUND_00_15 _T1 i > > - PRORD_nd a0,e,5 # sig1: a0 = (e >> 5) > > - > > - vpxor g, f, a2 # ch: a2 = f^g > > - vpand e,a2, a2 # ch: a2 = (f^g)&e > > - vpxor g, a2, a2 # a2 = ch > > - > > - PRORD_nd a1,e,25 # sig1: a1 = (e >> 25) > > - > > - vmovdqu \_T1,(SZ8*(\i & 0xf))(%rsp) > > - vpaddd (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K > > - vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) > > - PRORD a0, 6 # sig1: a0 = (e >> 6) ^ (e >> 11) > > - vpaddd a2, h, h # h = h + ch > > - PRORD_nd a2,a,11 # sig0: a2 = (a >> 11) > > - vpaddd \_T1,h, h # h = h + ch + W + K > > - vpxor a1, a0, a0 # a0 = sigma1 > > - PRORD_nd a1,a,22 # sig0: a1 = (a >> 22) > > - vpxor c, a, \_T1 # maj: T1 = a^c > > - add $SZ8, ROUND # ROUND++ > > - vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b > > - vpaddd a0, h, h > > - vpaddd h, d, d > > - vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) > > - PRORD a2,2 # sig0: a2 = (a >> 2) ^ (a >> 13) > > - vpxor a1, a2, a2 # a2 = sig0 > > - vpand c, a, a1 # maj: a1 = a&c > > - vpor \_T1, a1, a1 # a1 = maj > > - vpaddd a1, h, h # h = h + ch + W + K + maj > > - vpaddd a2, h, h # h = h + ch + W + K + maj + sigma0 > > - ROTATE_ARGS > > -.endm > > - > > -# arguments passed implicitly in preprocessor symbols i, a...h > > -.macro ROUND_16_XX _T1 i > > - vmovdqu (SZ8*((\i-15)&0xf))(%rsp), \_T1 > > - vmovdqu (SZ8*((\i-2)&0xf))(%rsp), a1 > > - vmovdqu \_T1, a0 > > - PRORD \_T1,11 > > - vmovdqu a1, a2 > > - PRORD a1,2 > > - vpxor a0, \_T1, \_T1 > > - PRORD \_T1, 7 > > - vpxor a2, a1, a1 > > - PRORD a1, 17 > > - vpsrld $3, a0, a0 > > - vpxor a0, \_T1, \_T1 > > - vpsrld $10, a2, a2 > > - vpxor a2, a1, a1 > > - vpaddd (SZ8*((\i-16)&0xf))(%rsp), \_T1, \_T1 > > - vpaddd (SZ8*((\i-7)&0xf))(%rsp), a1, a1 > > - vpaddd a1, \_T1, \_T1 > > - > > - ROUND_00_15 \_T1,\i > > -.endm > > - > > -# SHA256_ARGS: > > -# UINT128 digest[8]; // transposed digests > > -# UINT8 *data_ptr[4]; > > - > > -# void sha256_x8_avx2(SHA256_ARGS *args, UINT64 bytes); > > -# arg 1 : STATE : pointer to array of pointers to input data > > -# arg 2 : INP_SIZE : size of input in blocks > > - # general registers preserved in outer calling routine > > - # outer calling routine saves all the XMM registers > > - # save rsp, allocate 32-byte aligned for local variables > > -ENTRY(sha256_x8_avx2) > > - > > - # save callee-saved clobbered registers to comply with C function ABI > > - push %r12 > > - push %r13 > > - push %r14 > > - push %r15 > > - > > - mov %rsp, IDX > > - sub $FRAMESZ, %rsp > > - and $~0x1F, %rsp > > - mov IDX, _rsp(%rsp) > > - > > - # Load the pre-transposed incoming digest. > > - vmovdqu 0*SHA256_DIGEST_ROW_SIZE(STATE),a > > - vmovdqu 1*SHA256_DIGEST_ROW_SIZE(STATE),b > > - vmovdqu 2*SHA256_DIGEST_ROW_SIZE(STATE),c > > - vmovdqu 3*SHA256_DIGEST_ROW_SIZE(STATE),d > > - vmovdqu 4*SHA256_DIGEST_ROW_SIZE(STATE),e > > - vmovdqu 5*SHA256_DIGEST_ROW_SIZE(STATE),f > > - vmovdqu 6*SHA256_DIGEST_ROW_SIZE(STATE),g > > - vmovdqu 7*SHA256_DIGEST_ROW_SIZE(STATE),h > > - > > - lea K256_8(%rip),TBL > > - > > - # load the address of each of the 4 message lanes > > - # getting ready to transpose input onto stack > > - mov _args_data_ptr+0*PTR_SZ(STATE),inp0 > > - mov _args_data_ptr+1*PTR_SZ(STATE),inp1 > > - mov _args_data_ptr+2*PTR_SZ(STATE),inp2 > > - mov _args_data_ptr+3*PTR_SZ(STATE),inp3 > > - mov _args_data_ptr+4*PTR_SZ(STATE),inp4 > > - mov _args_data_ptr+5*PTR_SZ(STATE),inp5 > > - mov _args_data_ptr+6*PTR_SZ(STATE),inp6 > > - mov _args_data_ptr+7*PTR_SZ(STATE),inp7 > > - > > - xor IDX, IDX > > -lloop: > > - xor ROUND, ROUND > > - > > - # save old digest > > - vmovdqu a, _digest(%rsp) > > - vmovdqu b, _digest+1*SZ8(%rsp) > > - vmovdqu c, _digest+2*SZ8(%rsp) > > - vmovdqu d, _digest+3*SZ8(%rsp) > > - vmovdqu e, _digest+4*SZ8(%rsp) > > - vmovdqu f, _digest+5*SZ8(%rsp) > > - vmovdqu g, _digest+6*SZ8(%rsp) > > - vmovdqu h, _digest+7*SZ8(%rsp) > > - i = 0 > > -.rep 2 > > - VMOVPS i*32(inp0, IDX), TT0 > > - VMOVPS i*32(inp1, IDX), TT1 > > - VMOVPS i*32(inp2, IDX), TT2 > > - VMOVPS i*32(inp3, IDX), TT3 > > - VMOVPS i*32(inp4, IDX), TT4 > > - VMOVPS i*32(inp5, IDX), TT5 > > - VMOVPS i*32(inp6, IDX), TT6 > > - VMOVPS i*32(inp7, IDX), TT7 > > - vmovdqu g, _ytmp(%rsp) > > - vmovdqu h, _ytmp+1*SZ8(%rsp) > > - TRANSPOSE8 TT0, TT1, TT2, TT3, TT4, TT5, TT6, TT7, TMP0, TMP1 > > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP1 > > - vmovdqu _ytmp(%rsp), g > > - vpshufb TMP1, TT0, TT0 > > - vpshufb TMP1, TT1, TT1 > > - vpshufb TMP1, TT2, TT2 > > - vpshufb TMP1, TT3, TT3 > > - vpshufb TMP1, TT4, TT4 > > - vpshufb TMP1, TT5, TT5 > > - vpshufb TMP1, TT6, TT6 > > - vpshufb TMP1, TT7, TT7 > > - vmovdqu _ytmp+1*SZ8(%rsp), h > > - vmovdqu TT4, _ytmp(%rsp) > > - vmovdqu TT5, _ytmp+1*SZ8(%rsp) > > - vmovdqu TT6, _ytmp+2*SZ8(%rsp) > > - vmovdqu TT7, _ytmp+3*SZ8(%rsp) > > - ROUND_00_15 TT0,(i*8+0) > > - vmovdqu _ytmp(%rsp), TT0 > > - ROUND_00_15 TT1,(i*8+1) > > - vmovdqu _ytmp+1*SZ8(%rsp), TT1 > > - ROUND_00_15 TT2,(i*8+2) > > - vmovdqu _ytmp+2*SZ8(%rsp), TT2 > > - ROUND_00_15 TT3,(i*8+3) > > - vmovdqu _ytmp+3*SZ8(%rsp), TT3 > > - ROUND_00_15 TT0,(i*8+4) > > - ROUND_00_15 TT1,(i*8+5) > > - ROUND_00_15 TT2,(i*8+6) > > - ROUND_00_15 TT3,(i*8+7) > > - i = (i+1) > > -.endr > > - add $64, IDX > > - i = (i*8) > > - > > - jmp Lrounds_16_xx > > -.align 16 > > -Lrounds_16_xx: > > -.rep 16 > > - ROUND_16_XX T1, i > > - i = (i+1) > > -.endr > > - > > - cmp $ROUNDS,ROUND > > - jb Lrounds_16_xx > > - > > - # add old digest > > - vpaddd _digest+0*SZ8(%rsp), a, a > > - vpaddd _digest+1*SZ8(%rsp), b, b > > - vpaddd _digest+2*SZ8(%rsp), c, c > > - vpaddd _digest+3*SZ8(%rsp), d, d > > - vpaddd _digest+4*SZ8(%rsp), e, e > > - vpaddd _digest+5*SZ8(%rsp), f, f > > - vpaddd _digest+6*SZ8(%rsp), g, g > > - vpaddd _digest+7*SZ8(%rsp), h, h > > - > > - sub $1, INP_SIZE # unit is blocks > > - jne lloop > > - > > - # write back to memory (state object) the transposed digest > > - vmovdqu a, 0*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu b, 1*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu c, 2*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu d, 3*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu e, 4*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu f, 5*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu g, 6*SHA256_DIGEST_ROW_SIZE(STATE) > > - vmovdqu h, 7*SHA256_DIGEST_ROW_SIZE(STATE) > > - > > - # update input pointers > > - add IDX, inp0 > > - mov inp0, _args_data_ptr+0*8(STATE) > > - add IDX, inp1 > > - mov inp1, _args_data_ptr+1*8(STATE) > > - add IDX, inp2 > > - mov inp2, _args_data_ptr+2*8(STATE) > > - add IDX, inp3 > > - mov inp3, _args_data_ptr+3*8(STATE) > > - add IDX, inp4 > > - mov inp4, _args_data_ptr+4*8(STATE) > > - add IDX, inp5 > > - mov inp5, _args_data_ptr+5*8(STATE) > > - add IDX, inp6 > > - mov inp6, _args_data_ptr+6*8(STATE) > > - add IDX, inp7 > > - mov inp7, _args_data_ptr+7*8(STATE) > > - > > - # Postamble > > - mov _rsp(%rsp), %rsp > > - > > - # restore callee-saved clobbered registers > > - pop %r15 > > - pop %r14 > > - pop %r13 > > - pop %r12 > > - > > - ret > > -ENDPROC(sha256_x8_avx2) > > - > > -.section .rodata.K256_8, "a", @progbits > > -.align 64 > > -K256_8: > > - .octa 0x428a2f98428a2f98428a2f98428a2f98 > > - .octa 0x428a2f98428a2f98428a2f98428a2f98 > > - .octa 0x71374491713744917137449171374491 > > - .octa 0x71374491713744917137449171374491 > > - .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf > > - .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf > > - .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 > > - .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 > > - .octa 0x3956c25b3956c25b3956c25b3956c25b > > - .octa 0x3956c25b3956c25b3956c25b3956c25b > > - .octa 0x59f111f159f111f159f111f159f111f1 > > - .octa 0x59f111f159f111f159f111f159f111f1 > > - .octa 0x923f82a4923f82a4923f82a4923f82a4 > > - .octa 0x923f82a4923f82a4923f82a4923f82a4 > > - .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 > > - .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 > > - .octa 0xd807aa98d807aa98d807aa98d807aa98 > > - .octa 0xd807aa98d807aa98d807aa98d807aa98 > > - .octa 0x12835b0112835b0112835b0112835b01 > > - .octa 0x12835b0112835b0112835b0112835b01 > > - .octa 0x243185be243185be243185be243185be > > - .octa 0x243185be243185be243185be243185be > > - .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 > > - .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 > > - .octa 0x72be5d7472be5d7472be5d7472be5d74 > > - .octa 0x72be5d7472be5d7472be5d7472be5d74 > > - .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe > > - .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe > > - .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 > > - .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 > > - .octa 0xc19bf174c19bf174c19bf174c19bf174 > > - .octa 0xc19bf174c19bf174c19bf174c19bf174 > > - .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 > > - .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 > > - .octa 0xefbe4786efbe4786efbe4786efbe4786 > > - .octa 0xefbe4786efbe4786efbe4786efbe4786 > > - .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 > > - .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 > > - .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc > > - .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc > > - .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f > > - .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f > > - .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa > > - .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa > > - .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc > > - .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc > > - .octa 0x76f988da76f988da76f988da76f988da > > - .octa 0x76f988da76f988da76f988da76f988da > > - .octa 0x983e5152983e5152983e5152983e5152 > > - .octa 0x983e5152983e5152983e5152983e5152 > > - .octa 0xa831c66da831c66da831c66da831c66d > > - .octa 0xa831c66da831c66da831c66da831c66d > > - .octa 0xb00327c8b00327c8b00327c8b00327c8 > > - .octa 0xb00327c8b00327c8b00327c8b00327c8 > > - .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 > > - .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 > > - .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 > > - .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 > > - .octa 0xd5a79147d5a79147d5a79147d5a79147 > > - .octa 0xd5a79147d5a79147d5a79147d5a79147 > > - .octa 0x06ca635106ca635106ca635106ca6351 > > - .octa 0x06ca635106ca635106ca635106ca6351 > > - .octa 0x14292967142929671429296714292967 > > - .octa 0x14292967142929671429296714292967 > > - .octa 0x27b70a8527b70a8527b70a8527b70a85 > > - .octa 0x27b70a8527b70a8527b70a8527b70a85 > > - .octa 0x2e1b21382e1b21382e1b21382e1b2138 > > - .octa 0x2e1b21382e1b21382e1b21382e1b2138 > > - .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc > > - .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc > > - .octa 0x53380d1353380d1353380d1353380d13 > > - .octa 0x53380d1353380d1353380d1353380d13 > > - .octa 0x650a7354650a7354650a7354650a7354 > > - .octa 0x650a7354650a7354650a7354650a7354 > > - .octa 0x766a0abb766a0abb766a0abb766a0abb > > - .octa 0x766a0abb766a0abb766a0abb766a0abb > > - .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e > > - .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e > > - .octa 0x92722c8592722c8592722c8592722c85 > > - .octa 0x92722c8592722c8592722c8592722c85 > > - .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 > > - .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 > > - .octa 0xa81a664ba81a664ba81a664ba81a664b > > - .octa 0xa81a664ba81a664ba81a664ba81a664b > > - .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 > > - .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 > > - .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 > > - .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 > > - .octa 0xd192e819d192e819d192e819d192e819 > > - .octa 0xd192e819d192e819d192e819d192e819 > > - .octa 0xd6990624d6990624d6990624d6990624 > > - .octa 0xd6990624d6990624d6990624d6990624 > > - .octa 0xf40e3585f40e3585f40e3585f40e3585 > > - .octa 0xf40e3585f40e3585f40e3585f40e3585 > > - .octa 0x106aa070106aa070106aa070106aa070 > > - .octa 0x106aa070106aa070106aa070106aa070 > > - .octa 0x19a4c11619a4c11619a4c11619a4c116 > > - .octa 0x19a4c11619a4c11619a4c11619a4c116 > > - .octa 0x1e376c081e376c081e376c081e376c08 > > - .octa 0x1e376c081e376c081e376c081e376c08 > > - .octa 0x2748774c2748774c2748774c2748774c > > - .octa 0x2748774c2748774c2748774c2748774c > > - .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 > > - .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 > > - .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 > > - .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 > > - .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a > > - .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a > > - .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f > > - .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f > > - .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 > > - .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 > > - .octa 0x748f82ee748f82ee748f82ee748f82ee > > - .octa 0x748f82ee748f82ee748f82ee748f82ee > > - .octa 0x78a5636f78a5636f78a5636f78a5636f > > - .octa 0x78a5636f78a5636f78a5636f78a5636f > > - .octa 0x84c8781484c8781484c8781484c87814 > > - .octa 0x84c8781484c8781484c8781484c87814 > > - .octa 0x8cc702088cc702088cc702088cc70208 > > - .octa 0x8cc702088cc702088cc702088cc70208 > > - .octa 0x90befffa90befffa90befffa90befffa > > - .octa 0x90befffa90befffa90befffa90befffa > > - .octa 0xa4506ceba4506ceba4506ceba4506ceb > > - .octa 0xa4506ceba4506ceba4506ceba4506ceb > > - .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 > > - .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 > > - .octa 0xc67178f2c67178f2c67178f2c67178f2 > > - .octa 0xc67178f2c67178f2c67178f2c67178f2 > > - > > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > > -.align 32 > > -PSHUFFLE_BYTE_FLIP_MASK: > > -.octa 0x0c0d0e0f08090a0b0405060700010203 > > -.octa 0x0c0d0e0f08090a0b0405060700010203 > > - > > -.section .rodata.cst256.K256, "aM", @progbits, 256 > > -.align 64 > > -.global K256 > > -K256: > > - .int 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 > > - .int 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 > > - .int 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 > > - .int 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 > > - .int 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc > > - .int 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da > > - .int 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 > > - .int 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 > > - .int 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 > > - .int 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 > > - .int 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 > > - .int 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 > > - .int 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 > > - .int 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 > > - .int 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 > > - .int 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 > > diff --git a/arch/x86/crypto/sha512-mb/Makefile b/arch/x86/crypto/sha512-mb/Makefile > > deleted file mode 100644 > > index 90f1ef69152e..000000000000 > > --- a/arch/x86/crypto/sha512-mb/Makefile > > +++ /dev/null > > @@ -1,12 +0,0 @@ > > -# SPDX-License-Identifier: GPL-2.0 > > -# > > -# Arch-specific CryptoAPI modules. > > -# > > - > > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > > - $(comma)4)$(comma)%ymm2,yes,no) > > -ifeq ($(avx2_supported),yes) > > - obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb.o > > - sha512-mb-y := sha512_mb.o sha512_mb_mgr_flush_avx2.o \ > > - sha512_mb_mgr_init_avx2.o sha512_mb_mgr_submit_avx2.o sha512_x4_avx2.o > > -endif > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c > > deleted file mode 100644 > > index 26b85678012d..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb.c > > +++ /dev/null > > @@ -1,1047 +0,0 @@ > > -/* > > - * Multi buffer SHA512 algorithm Glue Code > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > > - > > -#include <crypto/internal/hash.h> > > -#include <linux/init.h> > > -#include <linux/module.h> > > -#include <linux/mm.h> > > -#include <linux/cryptohash.h> > > -#include <linux/types.h> > > -#include <linux/list.h> > > -#include <crypto/scatterwalk.h> > > -#include <crypto/sha.h> > > -#include <crypto/mcryptd.h> > > -#include <crypto/crypto_wq.h> > > -#include <asm/byteorder.h> > > -#include <linux/hardirq.h> > > -#include <asm/fpu/api.h> > > -#include "sha512_mb_ctx.h" > > - > > -#define FLUSH_INTERVAL 1000 /* in usec */ > > - > > -static struct mcryptd_alg_state sha512_mb_alg_state; > > - > > -struct sha512_mb_ctx { > > - struct mcryptd_ahash *mcryptd_tfm; > > -}; > > - > > -static inline struct mcryptd_hash_request_ctx > > - *cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx *hash_ctx) > > -{ > > - struct ahash_request *areq; > > - > > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > > -} > > - > > -static inline struct ahash_request > > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > > -{ > > - return container_of((void *) ctx, struct ahash_request, __ctx); > > -} > > - > > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > > - struct ahash_request *areq) > > -{ > > - rctx->flag = HASH_UPDATE; > > -} > > - > > -static asmlinkage void (*sha512_job_mgr_init)(struct sha512_mb_mgr *state); > > -static asmlinkage struct job_sha512* (*sha512_job_mgr_submit) > > - (struct sha512_mb_mgr *state, > > - struct job_sha512 *job); > > -static asmlinkage struct job_sha512* (*sha512_job_mgr_flush) > > - (struct sha512_mb_mgr *state); > > -static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job) > > - (struct sha512_mb_mgr *state); > > - > > -inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2], > > - uint64_t total_len) > > -{ > > - uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1); > > - > > - memset(&padblock[i], 0, SHA512_BLOCK_SIZE); > > - padblock[i] = 0x80; > > - > > - i += ((SHA512_BLOCK_SIZE - 1) & > > - (0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1))) > > - + 1 + SHA512_PADLENGTHFIELD_SIZE; > > - > > -#if SHA512_PADLENGTHFIELD_SIZE == 16 > > - *((uint64_t *) &padblock[i - 16]) = 0; > > -#endif > > - > > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > > - > > - /* Number of extra blocks to hash */ > > - return i >> SHA512_LOG2_BLOCK_SIZE; > > -} > > - > > -static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit > > - (struct sha512_ctx_mgr *mgr, struct sha512_hash_ctx *ctx) > > -{ > > - while (ctx) { > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Clear PROCESSING bit */ > > - ctx->status = HASH_CTX_STS_COMPLETE; > > - return ctx; > > - } > > - > > - /* > > - * If the extra blocks are empty, begin hashing what remains > > - * in the user's buffer. > > - */ > > - if (ctx->partial_block_buffer_length == 0 && > > - ctx->incoming_buffer_length) { > > - > > - const void *buffer = ctx->incoming_buffer; > > - uint32_t len = ctx->incoming_buffer_length; > > - uint32_t copy_len; > > - > > - /* > > - * Only entire blocks can be hashed. > > - * Copy remainder to extra blocks buffer. > > - */ > > - copy_len = len & (SHA512_BLOCK_SIZE-1); > > - > > - if (copy_len) { > > - len -= copy_len; > > - memcpy(ctx->partial_block_buffer, > > - ((const char *) buffer + len), > > - copy_len); > > - ctx->partial_block_buffer_length = copy_len; > > - } > > - > > - ctx->incoming_buffer_length = 0; > > - > > - /* len should be a multiple of the block size now */ > > - assert((len % SHA512_BLOCK_SIZE) == 0); > > - > > - /* Set len to the number of blocks to be hashed */ > > - len >>= SHA512_LOG2_BLOCK_SIZE; > > - > > - if (len) { > > - > > - ctx->job.buffer = (uint8_t *) buffer; > > - ctx->job.len = len; > > - ctx = (struct sha512_hash_ctx *) > > - sha512_job_mgr_submit(&mgr->mgr, > > - &ctx->job); > > - continue; > > - } > > - } > > - > > - /* > > - * If the extra blocks are not empty, then we are > > - * either on the last block(s) or we need more > > - * user input before continuing. > > - */ > > - if (ctx->status & HASH_CTX_STS_LAST) { > > - > > - uint8_t *buf = ctx->partial_block_buffer; > > - uint32_t n_extra_blocks = > > - sha512_pad(buf, ctx->total_length); > > - > > - ctx->status = (HASH_CTX_STS_PROCESSING | > > - HASH_CTX_STS_COMPLETE); > > - ctx->job.buffer = buf; > > - ctx->job.len = (uint32_t) n_extra_blocks; > > - ctx = (struct sha512_hash_ctx *) > > - sha512_job_mgr_submit(&mgr->mgr, &ctx->job); > > - continue; > > - } > > - > > - if (ctx) > > - ctx->status = HASH_CTX_STS_IDLE; > > - return ctx; > > - } > > - > > - return NULL; > > -} > > - > > -static struct sha512_hash_ctx > > - *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate) > > -{ > > - /* > > - * If get_comp_job returns NULL, there are no jobs complete. > > - * If get_comp_job returns a job, verify that it is safe to return to > > - * the user. > > - * If it is not ready, resubmit the job to finish processing. > > - * If sha512_ctx_mgr_resubmit returned a job, it is ready to be > > - * returned. > > - * Otherwise, all jobs currently being managed by the hash_ctx_mgr > > - * still need processing. > > - */ > > - struct sha512_ctx_mgr *mgr; > > - struct sha512_hash_ctx *ctx; > > - unsigned long flags; > > - > > - mgr = cstate->mgr; > > - spin_lock_irqsave(&cstate->work_lock, flags); > > - ctx = (struct sha512_hash_ctx *) > > - sha512_job_mgr_get_comp_job(&mgr->mgr); > > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > > - spin_unlock_irqrestore(&cstate->work_lock, flags); > > - return ctx; > > -} > > - > > -static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr) > > -{ > > - sha512_job_mgr_init(&mgr->mgr); > > -} > > - > > -static struct sha512_hash_ctx > > - *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate, > > - struct sha512_hash_ctx *ctx, > > - const void *buffer, > > - uint32_t len, > > - int flags) > > -{ > > - struct sha512_ctx_mgr *mgr; > > - unsigned long irqflags; > > - > > - mgr = cstate->mgr; > > - spin_lock_irqsave(&cstate->work_lock, irqflags); > > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > > - /* User should not pass anything other than UPDATE or LAST */ > > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > > - goto unlock; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > > - /* Cannot submit to a currently processing job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > > - goto unlock; > > - } > > - > > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > > - /* Cannot update a finished job. */ > > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > > - goto unlock; > > - } > > - > > - /* > > - * If we made it here, there were no errors during this call to > > - * submit > > - */ > > - ctx->error = HASH_CTX_ERROR_NONE; > > - > > - /* Store buffer ptr info from user */ > > - ctx->incoming_buffer = buffer; > > - ctx->incoming_buffer_length = len; > > - > > - /* > > - * Store the user's request flags and mark this ctx as currently being > > - * processed. > > - */ > > - ctx->status = (flags & HASH_LAST) ? > > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > > - HASH_CTX_STS_PROCESSING; > > - > > - /* Advance byte counter */ > > - ctx->total_length += len; > > - > > - /* > > - * If there is anything currently buffered in the extra blocks, > > - * append to it until it contains a whole block. > > - * Or if the user's buffer contains less than a whole block, > > - * append as much as possible to the extra block. > > - */ > > - if (ctx->partial_block_buffer_length || len < SHA512_BLOCK_SIZE) { > > - /* Compute how many bytes to copy from user buffer into extra > > - * block > > - */ > > - uint32_t copy_len = SHA512_BLOCK_SIZE - > > - ctx->partial_block_buffer_length; > > - if (len < copy_len) > > - copy_len = len; > > - > > - if (copy_len) { > > - /* Copy and update relevant pointers and counters */ > > - memcpy > > - (&ctx->partial_block_buffer[ctx->partial_block_buffer_length], > > - buffer, copy_len); > > - > > - ctx->partial_block_buffer_length += copy_len; > > - ctx->incoming_buffer = (const void *) > > - ((const char *)buffer + copy_len); > > - ctx->incoming_buffer_length = len - copy_len; > > - } > > - > > - /* The extra block should never contain more than 1 block > > - * here > > - */ > > - assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE); > > - > > - /* If the extra block buffer contains exactly 1 block, it can > > - * be hashed. > > - */ > > - if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) { > > - ctx->partial_block_buffer_length = 0; > > - > > - ctx->job.buffer = ctx->partial_block_buffer; > > - ctx->job.len = 1; > > - ctx = (struct sha512_hash_ctx *) > > - sha512_job_mgr_submit(&mgr->mgr, &ctx->job); > > - } > > - } > > - > > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > > -unlock: > > - spin_unlock_irqrestore(&cstate->work_lock, irqflags); > > - return ctx; > > -} > > - > > -static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate) > > -{ > > - struct sha512_ctx_mgr *mgr; > > - struct sha512_hash_ctx *ctx; > > - unsigned long flags; > > - > > - mgr = cstate->mgr; > > - spin_lock_irqsave(&cstate->work_lock, flags); > > - while (1) { > > - ctx = (struct sha512_hash_ctx *) > > - sha512_job_mgr_flush(&mgr->mgr); > > - > > - /* If flush returned 0, there are no more jobs in flight. */ > > - if (!ctx) > > - break; > > - > > - /* > > - * If flush returned a job, resubmit the job to finish > > - * processing. > > - */ > > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > > - > > - /* > > - * If sha512_ctx_mgr_resubmit returned a job, it is ready to > > - * be returned. Otherwise, all jobs currently being managed by > > - * the sha512_ctx_mgr still need processing. Loop. > > - */ > > - if (ctx) > > - break; > > - } > > - spin_unlock_irqrestore(&cstate->work_lock, flags); > > - return ctx; > > -} > > - > > -static int sha512_mb_init(struct ahash_request *areq) > > -{ > > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - hash_ctx_init(sctx); > > - sctx->job.result_digest[0] = SHA512_H0; > > - sctx->job.result_digest[1] = SHA512_H1; > > - sctx->job.result_digest[2] = SHA512_H2; > > - sctx->job.result_digest[3] = SHA512_H3; > > - sctx->job.result_digest[4] = SHA512_H4; > > - sctx->job.result_digest[5] = SHA512_H5; > > - sctx->job.result_digest[6] = SHA512_H6; > > - sctx->job.result_digest[7] = SHA512_H7; > > - sctx->total_length = 0; > > - sctx->partial_block_buffer_length = 0; > > - sctx->status = HASH_CTX_STS_IDLE; > > - > > - return 0; > > -} > > - > > -static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > > -{ > > - int i; > > - struct sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > > - __be64 *dst = (__be64 *) rctx->out; > > - > > - for (i = 0; i < 8; ++i) > > - dst[i] = cpu_to_be64(sctx->job.result_digest[i]); > > - > > - return 0; > > -} > > - > > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > > - struct mcryptd_alg_cstate *cstate, bool flush) > > -{ > > - int flag = HASH_UPDATE; > > - int nbytes, err = 0; > > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > > - struct sha512_hash_ctx *sha_ctx; > > - > > - /* more work ? */ > > - while (!(rctx->flag & HASH_DONE)) { > > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > > - if (nbytes < 0) { > > - err = nbytes; > > - goto out; > > - } > > - /* check if the walk is done */ > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - if (rctx->flag & HASH_FINAL) > > - flag |= HASH_LAST; > > - > > - } > > - sha_ctx = (struct sha512_hash_ctx *) > > - ahash_request_ctx(&rctx->areq); > > - kernel_fpu_begin(); > > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, > > - rctx->walk.data, nbytes, flag); > > - if (!sha_ctx) { > > - if (flush) > > - sha_ctx = sha512_ctx_mgr_flush(cstate); > > - } > > - kernel_fpu_end(); > > - if (sha_ctx) > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - else { > > - rctx = NULL; > > - goto out; > > - } > > - } > > - > > - /* copy the results */ > > - if (rctx->flag & HASH_FINAL) > > - sha512_mb_set_results(rctx); > > - > > -out: > > - *ret_rctx = rctx; > > - return err; > > -} > > - > > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate, > > - int err) > > -{ > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha512_hash_ctx *sha_ctx; > > - struct mcryptd_hash_request_ctx *req_ctx; > > - int ret; > > - unsigned long flags; > > - > > - /* remove from work list */ > > - spin_lock_irqsave(&cstate->work_lock, flags); > > - list_del(&rctx->waiter); > > - spin_unlock_irqrestore(&cstate->work_lock, flags); > > - > > - if (irqs_disabled()) > > - rctx->complete(&req->base, err); > > - else { > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > - } > > - > > - /* check to see if there are other jobs that are done */ > > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate); > > - while (sha_ctx) { > > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&req_ctx, cstate, false); > > - if (req_ctx) { > > - spin_lock_irqsave(&cstate->work_lock, flags); > > - list_del(&req_ctx->waiter); > > - spin_unlock_irqrestore(&cstate->work_lock, flags); > > - > > - req = cast_mcryptd_ctx_to_req(req_ctx); > > - if (irqs_disabled()) > > - req_ctx->complete(&req->base, ret); > > - else { > > - local_bh_disable(); > > - req_ctx->complete(&req->base, ret); > > - local_bh_enable(); > > - } > > - } > > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate); > > - } > > - > > - return 0; > > -} > > - > > -static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > > - struct mcryptd_alg_cstate *cstate) > > -{ > > - unsigned long next_flush; > > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > > - unsigned long flags; > > - > > - /* initialize tag */ > > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > > - rctx->tag.seq_num = cstate->next_seq_num++; > > - next_flush = rctx->tag.arrival + delay; > > - rctx->tag.expire = next_flush; > > - > > - spin_lock_irqsave(&cstate->work_lock, flags); > > - list_add_tail(&rctx->waiter, &cstate->work_list); > > - spin_unlock_irqrestore(&cstate->work_lock, flags); > > - > > - mcryptd_arm_flusher(cstate, delay); > > -} > > - > > -static int sha512_mb_update(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, > > - areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha512_hash_ctx *sha_ctx; > > - int ret = 0, nbytes; > > - > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) > > - rctx->flag |= HASH_DONE; > > - > > - /* submit */ > > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > > - sha512_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data, > > - nbytes, HASH_UPDATE); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha512_mb_finup(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, > > - areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > > - > > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > > - struct sha512_hash_ctx *sha_ctx; > > - int ret = 0, flag = HASH_UPDATE, nbytes; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > > - > > - if (nbytes < 0) { > > - ret = nbytes; > > - goto done; > > - } > > - > > - if (crypto_ahash_walk_last(&rctx->walk)) { > > - rctx->flag |= HASH_DONE; > > - flag = HASH_LAST; > > - } > > - > > - /* submit */ > > - rctx->flag |= HASH_FINAL; > > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > > - sha512_mb_add_list(rctx, cstate); > > - > > - kernel_fpu_begin(); > > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data, > > - nbytes, flag); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha512_mb_final(struct ahash_request *areq) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = > > - container_of(areq, struct mcryptd_hash_request_ctx, > > - areq); > > - struct mcryptd_alg_cstate *cstate = > > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > > - > > - struct sha512_hash_ctx *sha_ctx; > > - int ret = 0; > > - u8 data; > > - > > - /* sanity check */ > > - if (rctx->tag.cpu != smp_processor_id()) { > > - pr_err("mcryptd error: cpu clash\n"); > > - goto done; > > - } > > - > > - /* need to init context */ > > - req_ctx_init(rctx, areq); > > - > > - rctx->flag |= HASH_DONE | HASH_FINAL; > > - > > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > > - /* flag HASH_FINAL and 0 data size */ > > - sha512_mb_add_list(rctx, cstate); > > - kernel_fpu_begin(); > > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST); > > - kernel_fpu_end(); > > - > > - /* check if anything is returned */ > > - if (!sha_ctx) > > - return -EINPROGRESS; > > - > > - if (sha_ctx->error) { > > - ret = sha_ctx->error; > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - goto done; > > - } > > - > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - ret = sha_finish_walk(&rctx, cstate, false); > > - if (!rctx) > > - return -EINPROGRESS; > > -done: > > - sha_complete_job(rctx, cstate, ret); > > - return ret; > > -} > > - > > -static int sha512_mb_export(struct ahash_request *areq, void *out) > > -{ > > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(out, sctx, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha512_mb_import(struct ahash_request *areq, const void *in) > > -{ > > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > > - > > - memcpy(sctx, in, sizeof(*sctx)); > > - > > - return 0; > > -} > > - > > -static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm) > > -{ > > - struct mcryptd_ahash *mcryptd_tfm; > > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - struct mcryptd_hash_ctx *mctx; > > - > > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb", > > - CRYPTO_ALG_INTERNAL, > > - CRYPTO_ALG_INTERNAL); > > - if (IS_ERR(mcryptd_tfm)) > > - return PTR_ERR(mcryptd_tfm); > > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > > - mctx->alg_state = &sha512_mb_alg_state; > > - ctx->mcryptd_tfm = mcryptd_tfm; > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > > - > > - return 0; > > -} > > - > > -static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm) > > -{ > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct ahash_request) + > > - sizeof(struct sha512_hash_ctx)); > > - > > - return 0; > > -} > > - > > -static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - mcryptd_free_ahash(ctx->mcryptd_tfm); > > -} > > - > > -static struct ahash_alg sha512_mb_areq_alg = { > > - .init = sha512_mb_init, > > - .update = sha512_mb_update, > > - .final = sha512_mb_final, > > - .finup = sha512_mb_finup, > > - .export = sha512_mb_export, > > - .import = sha512_mb_import, > > - .halg = { > > - .digestsize = SHA512_DIGEST_SIZE, > > - .statesize = sizeof(struct sha512_hash_ctx), > > - .base = { > > - .cra_name = "__sha512-mb", > > - .cra_driver_name = "__intel_sha512-mb", > > - .cra_priority = 100, > > - /* > > - * use ASYNC flag as some buffers in multi-buffer > > - * algo may not have completed before hashing thread > > - * sleep > > - */ > > - .cra_flags = CRYPTO_ALG_ASYNC | > > - CRYPTO_ALG_INTERNAL, > > - .cra_blocksize = SHA512_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT > > - (sha512_mb_areq_alg.halg.base.cra_list), > > - .cra_init = sha512_mb_areq_init_tfm, > > - .cra_exit = sha512_mb_areq_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha512_hash_ctx), > > - } > > - } > > -}; > > - > > -static int sha512_mb_async_init(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_init(mcryptd_req); > > -} > > - > > -static int sha512_mb_async_update(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_update(mcryptd_req); > > -} > > - > > -static int sha512_mb_async_finup(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_finup(mcryptd_req); > > -} > > - > > -static int sha512_mb_async_final(struct ahash_request *req) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_final(mcryptd_req); > > -} > > - > > -static int sha512_mb_async_digest(struct ahash_request *req) > > -{ > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_digest(mcryptd_req); > > -} > > - > > -static int sha512_mb_async_export(struct ahash_request *req, void *out) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - return crypto_ahash_export(mcryptd_req, out); > > -} > > - > > -static int sha512_mb_async_import(struct ahash_request *req, const void *in) > > -{ > > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > > - struct mcryptd_hash_request_ctx *rctx; > > - struct ahash_request *areq; > > - > > - memcpy(mcryptd_req, req, sizeof(*req)); > > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > > - rctx = ahash_request_ctx(mcryptd_req); > > - > > - areq = &rctx->areq; > > - > > - ahash_request_set_tfm(areq, child); > > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > > - rctx->complete, req); > > - > > - return crypto_ahash_import(mcryptd_req, in); > > -} > > - > > -static struct ahash_alg sha512_mb_async_alg = { > > - .init = sha512_mb_async_init, > > - .update = sha512_mb_async_update, > > - .final = sha512_mb_async_final, > > - .finup = sha512_mb_async_finup, > > - .digest = sha512_mb_async_digest, > > - .export = sha512_mb_async_export, > > - .import = sha512_mb_async_import, > > - .halg = { > > - .digestsize = SHA512_DIGEST_SIZE, > > - .statesize = sizeof(struct sha512_hash_ctx), > > - .base = { > > - .cra_name = "sha512", > > - .cra_driver_name = "sha512_mb", > > - /* > > - * Low priority, since with few concurrent hash requests > > - * this is extremely slow due to the flush delay. Users > > - * whose workloads would benefit from this can request > > - * it explicitly by driver name, or can increase its > > - * priority at runtime using NETLINK_CRYPTO. > > - */ > > - .cra_priority = 50, > > - .cra_flags = CRYPTO_ALG_ASYNC, > > - .cra_blocksize = SHA512_BLOCK_SIZE, > > - .cra_module = THIS_MODULE, > > - .cra_list = LIST_HEAD_INIT > > - (sha512_mb_async_alg.halg.base.cra_list), > > - .cra_init = sha512_mb_async_init_tfm, > > - .cra_exit = sha512_mb_async_exit_tfm, > > - .cra_ctxsize = sizeof(struct sha512_mb_ctx), > > - .cra_alignmask = 0, > > - }, > > - }, > > -}; > > - > > -static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate) > > -{ > > - struct mcryptd_hash_request_ctx *rctx; > > - unsigned long cur_time; > > - unsigned long next_flush = 0; > > - struct sha512_hash_ctx *sha_ctx; > > - > > - > > - cur_time = jiffies; > > - > > - while (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - if time_before(cur_time, rctx->tag.expire) > > - break; > > - kernel_fpu_begin(); > > - sha_ctx = (struct sha512_hash_ctx *) > > - sha512_ctx_mgr_flush(cstate); > > - kernel_fpu_end(); > > - if (!sha_ctx) { > > - pr_err("sha512_mb error: nothing got flushed for" > > - " non-empty list\n"); > > - break; > > - } > > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > > - sha_finish_walk(&rctx, cstate, true); > > - sha_complete_job(rctx, cstate, 0); > > - } > > - > > - if (!list_empty(&cstate->work_list)) { > > - rctx = list_entry(cstate->work_list.next, > > - struct mcryptd_hash_request_ctx, waiter); > > - /* get the hash context and then flush time */ > > - next_flush = rctx->tag.expire; > > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > > - } > > - return next_flush; > > -} > > - > > -static int __init sha512_mb_mod_init(void) > > -{ > > - > > - int cpu; > > - int err; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - /* check for dependent cpu features */ > > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > > - !boot_cpu_has(X86_FEATURE_BMI2)) > > - return -ENODEV; > > - > > - /* initialize multibuffer structures */ > > - sha512_mb_alg_state.alg_cstate = > > - alloc_percpu(struct mcryptd_alg_cstate); > > - > > - sha512_job_mgr_init = sha512_mb_mgr_init_avx2; > > - sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2; > > - sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2; > > - sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2; > > - > > - if (!sha512_mb_alg_state.alg_cstate) > > - return -ENOMEM; > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > > - cpu_state->next_flush = 0; > > - cpu_state->next_seq_num = 0; > > - cpu_state->flusher_engaged = false; > > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > > - cpu_state->cpu = cpu; > > - cpu_state->alg_state = &sha512_mb_alg_state; > > - cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr), > > - GFP_KERNEL); > > - if (!cpu_state->mgr) > > - goto err2; > > - sha512_ctx_mgr_init(cpu_state->mgr); > > - INIT_LIST_HEAD(&cpu_state->work_list); > > - spin_lock_init(&cpu_state->work_lock); > > - } > > - sha512_mb_alg_state.flusher = &sha512_mb_flusher; > > - > > - err = crypto_register_ahash(&sha512_mb_areq_alg); > > - if (err) > > - goto err2; > > - err = crypto_register_ahash(&sha512_mb_async_alg); > > - if (err) > > - goto err1; > > - > > - > > - return 0; > > -err1: > > - crypto_unregister_ahash(&sha512_mb_areq_alg); > > -err2: > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha512_mb_alg_state.alg_cstate); > > - return -ENODEV; > > -} > > - > > -static void __exit sha512_mb_mod_fini(void) > > -{ > > - int cpu; > > - struct mcryptd_alg_cstate *cpu_state; > > - > > - crypto_unregister_ahash(&sha512_mb_async_alg); > > - crypto_unregister_ahash(&sha512_mb_areq_alg); > > - for_each_possible_cpu(cpu) { > > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > > - kfree(cpu_state->mgr); > > - } > > - free_percpu(sha512_mb_alg_state.alg_cstate); > > -} > > - > > -module_init(sha512_mb_mod_init); > > -module_exit(sha512_mb_mod_fini); > > - > > -MODULE_LICENSE("GPL"); > > -MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated"); > > - > > -MODULE_ALIAS("sha512"); > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > > deleted file mode 100644 > > index e5c465bd821e..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > > +++ /dev/null > > @@ -1,128 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA512 context > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#ifndef _SHA_MB_CTX_INTERNAL_H > > -#define _SHA_MB_CTX_INTERNAL_H > > - > > -#include "sha512_mb_mgr.h" > > - > > -#define HASH_UPDATE 0x00 > > -#define HASH_LAST 0x01 > > -#define HASH_DONE 0x02 > > -#define HASH_FINAL 0x04 > > - > > -#define HASH_CTX_STS_IDLE 0x00 > > -#define HASH_CTX_STS_PROCESSING 0x01 > > -#define HASH_CTX_STS_LAST 0x02 > > -#define HASH_CTX_STS_COMPLETE 0x04 > > - > > -enum hash_ctx_error { > > - HASH_CTX_ERROR_NONE = 0, > > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > > -}; > > - > > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > > -#define hash_ctx_status(ctx) ((ctx)->status) > > -#define hash_ctx_error(ctx) ((ctx)->error) > > -#define hash_ctx_init(ctx) \ > > - do { \ > > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > > - } while (0) > > - > > -/* Hash Constants and Typedefs */ > > -#define SHA512_DIGEST_LENGTH 8 > > -#define SHA512_LOG2_BLOCK_SIZE 7 > > - > > -#define SHA512_PADLENGTHFIELD_SIZE 16 > > - > > -#ifdef SHA_MB_DEBUG > > -#define assert(expr) \ > > -do { \ > > - if (unlikely(!(expr))) { \ > > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > > - #expr, __FILE__, __func__, __LINE__); \ > > - } \ > > -} while (0) > > -#else > > -#define assert(expr) do {} while (0) > > -#endif > > - > > -struct sha512_ctx_mgr { > > - struct sha512_mb_mgr mgr; > > -}; > > - > > -/* typedef struct sha512_ctx_mgr sha512_ctx_mgr; */ > > - > > -struct sha512_hash_ctx { > > - /* Must be at struct offset 0 */ > > - struct job_sha512 job; > > - /* status flag */ > > - int status; > > - /* error flag */ > > - int error; > > - > > - uint64_t total_length; > > - const void *incoming_buffer; > > - uint32_t incoming_buffer_length; > > - uint8_t partial_block_buffer[SHA512_BLOCK_SIZE * 2]; > > - uint32_t partial_block_buffer_length; > > - void *user_data; > > -}; > > - > > -#endif > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h b/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > > deleted file mode 100644 > > index 178f17eef382..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > > +++ /dev/null > > @@ -1,104 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA512 algorithm manager > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#ifndef __SHA_MB_MGR_H > > -#define __SHA_MB_MGR_H > > - > > -#include <linux/types.h> > > - > > -#define NUM_SHA512_DIGEST_WORDS 8 > > - > > -enum job_sts {STS_UNKNOWN = 0, > > - STS_BEING_PROCESSED = 1, > > - STS_COMPLETED = 2, > > - STS_INTERNAL_ERROR = 3, > > - STS_ERROR = 4 > > -}; > > - > > -struct job_sha512 { > > - u8 *buffer; > > - u64 len; > > - u64 result_digest[NUM_SHA512_DIGEST_WORDS] __aligned(32); > > - enum job_sts status; > > - void *user_data; > > -}; > > - > > -struct sha512_args_x4 { > > - uint64_t digest[8][4]; > > - uint8_t *data_ptr[4]; > > -}; > > - > > -struct sha512_lane_data { > > - struct job_sha512 *job_in_lane; > > -}; > > - > > -struct sha512_mb_mgr { > > - struct sha512_args_x4 args; > > - > > - uint64_t lens[4]; > > - > > - /* each byte is index (0...7) of unused lanes */ > > - uint64_t unused_lanes; > > - /* byte 4 is set to FF as a flag */ > > - struct sha512_lane_data ldata[4]; > > -}; > > - > > -#define SHA512_MB_MGR_NUM_LANES_AVX2 4 > > - > > -void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state); > > -struct job_sha512 *sha512_mb_mgr_submit_avx2(struct sha512_mb_mgr *state, > > - struct job_sha512 *job); > > -struct job_sha512 *sha512_mb_mgr_flush_avx2(struct sha512_mb_mgr *state); > > -struct job_sha512 *sha512_mb_mgr_get_comp_job_avx2(struct sha512_mb_mgr *state); > > - > > -#endif > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > > deleted file mode 100644 > > index cf2636d4c9ba..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > > +++ /dev/null > > @@ -1,281 +0,0 @@ > > -/* > > - * Header file for multi buffer SHA256 algorithm data structure > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -# Macros for defining data structures > > - > > -# Usage example > > - > > -#START_FIELDS # JOB_AES > > -### name size align > > -#FIELD _plaintext, 8, 8 # pointer to plaintext > > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > > -#FIELD _IV, 16, 8 # IV > > -#FIELD _keys, 8, 8 # pointer to keys > > -#FIELD _len, 4, 4 # length in bytes > > -#FIELD _status, 4, 4 # status enumeration > > -#FIELD _user_data, 8, 8 # pointer to user data > > -#UNION _union, size1, align1, \ > > -# size2, align2, \ > > -# size3, align3, \ > > -# ... > > -#END_FIELDS > > -#%assign _JOB_AES_size _FIELD_OFFSET > > -#%assign _JOB_AES_align _STRUCT_ALIGN > > - > > -######################################################################### > > - > > -# Alternate "struc-like" syntax: > > -# STRUCT job_aes2 > > -# RES_Q .plaintext, 1 > > -# RES_Q .ciphertext, 1 > > -# RES_DQ .IV, 1 > > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > > -# RES_U .union, size1, align1, \ > > -# size2, align2, \ > > -# ... > > -# ENDSTRUCT > > -# # Following only needed if nesting > > -# %assign job_aes2_size _FIELD_OFFSET > > -# %assign job_aes2_align _STRUCT_ALIGN > > -# > > -# RES_* macros take a name, a count and an optional alignment. > > -# The count in in terms of the base size of the macro, and the > > -# default alignment is the base size. > > -# The macros are: > > -# Macro Base size > > -# RES_B 1 > > -# RES_W 2 > > -# RES_D 4 > > -# RES_Q 8 > > -# RES_DQ 16 > > -# RES_Y 32 > > -# RES_Z 64 > > -# > > -# RES_U defines a union. It's arguments are a name and two or more > > -# pairs of "size, alignment" > > -# > > -# The two assigns are only needed if this structure is being nested > > -# within another. Even if the assigns are not done, one can still use > > -# STRUCT_NAME_size as the size of the structure. > > -# > > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > > -# > > -# The differences between this and using "struc" directly are that each > > -# type is implicitly aligned to its natural length (although this can be > > -# over-ridden with an explicit third parameter), and that the structure > > -# is padded at the end to its overall alignment. > > -# > > - > > -######################################################################### > > - > > -#ifndef _DATASTRUCT_ASM_ > > -#define _DATASTRUCT_ASM_ > > - > > -#define PTR_SZ 8 > > -#define SHA512_DIGEST_WORD_SIZE 8 > > -#define SHA512_MB_MGR_NUM_LANES_AVX2 4 > > -#define NUM_SHA512_DIGEST_WORDS 8 > > -#define SZ4 4*SHA512_DIGEST_WORD_SIZE > > -#define ROUNDS 80*SZ4 > > -#define SHA512_DIGEST_ROW_SIZE (SHA512_MB_MGR_NUM_LANES_AVX2 * 8) > > - > > -# START_FIELDS > > -.macro START_FIELDS > > - _FIELD_OFFSET = 0 > > - _STRUCT_ALIGN = 0 > > -.endm > > - > > -# FIELD name size align > > -.macro FIELD name size align > > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > > - \name = _FIELD_OFFSET > > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > > -.if (\align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = \align > > -.endif > > -.endm > > - > > -# END_FIELDS > > -.macro END_FIELDS > > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > > -.endm > > - > > -.macro STRUCT p1 > > -START_FIELDS > > -.struc \p1 > > -.endm > > - > > -.macro ENDSTRUCT > > - tmp = _FIELD_OFFSET > > - END_FIELDS > > - tmp = (_FIELD_OFFSET - ##tmp) > > -.if (tmp > 0) > > - .lcomm tmp > > -.endm > > - > > -## RES_int name size align > > -.macro RES_int p1 p2 p3 > > - name = \p1 > > - size = \p2 > > - align = .\p3 > > - > > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > > -.align align > > -.lcomm name size > > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > > -.if (align > _STRUCT_ALIGN) > > - _STRUCT_ALIGN = align > > -.endif > > -.endm > > - > > -# macro RES_B name, size [, align] > > -.macro RES_B _name, _size, _align=1 > > -RES_int _name _size _align > > -.endm > > - > > -# macro RES_W name, size [, align] > > -.macro RES_W _name, _size, _align=2 > > -RES_int _name 2*(_size) _align > > -.endm > > - > > -# macro RES_D name, size [, align] > > -.macro RES_D _name, _size, _align=4 > > -RES_int _name 4*(_size) _align > > -.endm > > - > > -# macro RES_Q name, size [, align] > > -.macro RES_Q _name, _size, _align=8 > > -RES_int _name 8*(_size) _align > > -.endm > > - > > -# macro RES_DQ name, size [, align] > > -.macro RES_DQ _name, _size, _align=16 > > -RES_int _name 16*(_size) _align > > -.endm > > - > > -# macro RES_Y name, size [, align] > > -.macro RES_Y _name, _size, _align=32 > > -RES_int _name 32*(_size) _align > > -.endm > > - > > -# macro RES_Z name, size [, align] > > -.macro RES_Z _name, _size, _align=64 > > -RES_int _name 64*(_size) _align > > -.endm > > - > > -#endif > > - > > -################################################################### > > -### Define SHA512 Out Of Order Data Structures > > -################################################################### > > - > > -START_FIELDS # LANE_DATA > > -### name size align > > -FIELD _job_in_lane, 8, 8 # pointer to job object > > -END_FIELDS > > - > > - _LANE_DATA_size = _FIELD_OFFSET > > - _LANE_DATA_align = _STRUCT_ALIGN > > - > > -#################################################################### > > - > > -START_FIELDS # SHA512_ARGS_X4 > > -### name size align > > -FIELD _digest, 8*8*4, 4 # transposed digest > > -FIELD _data_ptr, 8*4, 8 # array of pointers to data > > -END_FIELDS > > - > > - _SHA512_ARGS_X4_size = _FIELD_OFFSET > > - _SHA512_ARGS_X4_align = _STRUCT_ALIGN > > - > > -##################################################################### > > - > > -START_FIELDS # MB_MGR > > -### name size align > > -FIELD _args, _SHA512_ARGS_X4_size, _SHA512_ARGS_X4_align > > -FIELD _lens, 8*4, 8 > > -FIELD _unused_lanes, 8, 8 > > -FIELD _ldata, _LANE_DATA_size*4, _LANE_DATA_align > > -END_FIELDS > > - > > - _MB_MGR_size = _FIELD_OFFSET > > - _MB_MGR_align = _STRUCT_ALIGN > > - > > -_args_digest = _args + _digest > > -_args_data_ptr = _args + _data_ptr > > - > > -####################################################################### > > - > > -####################################################################### > > -#### Define constants > > -####################################################################### > > - > > -#define STS_UNKNOWN 0 > > -#define STS_BEING_PROCESSED 1 > > -#define STS_COMPLETED 2 > > - > > -####################################################################### > > -#### Define JOB_SHA512 structure > > -####################################################################### > > - > > -START_FIELDS # JOB_SHA512 > > -### name size align > > -FIELD _buffer, 8, 8 # pointer to buffer > > -FIELD _len, 8, 8 # length in bytes > > -FIELD _result_digest, 8*8, 32 # Digest (output) > > -FIELD _status, 4, 4 > > -FIELD _user_data, 8, 8 > > -END_FIELDS > > - > > - _JOB_SHA512_size = _FIELD_OFFSET > > - _JOB_SHA512_align = _STRUCT_ALIGN > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > > deleted file mode 100644 > > index 7c629caebc05..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > > +++ /dev/null > > @@ -1,297 +0,0 @@ > > -/* > > - * Flush routine for SHA512 multibuffer > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha512_mb_mgr_datastruct.S" > > - > > -.extern sha512_x4_avx2 > > - > > -# LINUX register definitions > > -#define arg1 %rdi > > -#define arg2 %rsi > > - > > -# idx needs to be other than arg1, arg2, rbx, r12 > > -#define idx %rdx > > - > > -# Common definitions > > -#define state arg1 > > -#define job arg2 > > -#define len2 arg2 > > - > > -#define unused_lanes %rbx > > -#define lane_data %rbx > > -#define tmp2 %rbx > > - > > -#define job_rax %rax > > -#define tmp1 %rax > > -#define size_offset %rax > > -#define tmp %rax > > -#define start_offset %rax > > - > > -#define tmp3 arg1 > > - > > -#define extra_blocks arg2 > > -#define p arg2 > > - > > -#define tmp4 %r8 > > -#define lens0 %r8 > > - > > -#define lens1 %r9 > > -#define lens2 %r10 > > -#define lens3 %r11 > > - > > -.macro LABEL prefix n > > -\prefix\n\(): > > -.endm > > - > > -.macro JNE_SKIP i > > -jne skip_\i > > -.endm > > - > > -.altmacro > > -.macro SET_OFFSET _offset > > -offset = \_offset > > -.endm > > -.noaltmacro > > - > > -# JOB* sha512_mb_mgr_flush_avx2(MB_MGR *state) > > -# arg 1 : rcx : state > > -ENTRY(sha512_mb_mgr_flush_avx2) > > - FRAME_BEGIN > > - push %rbx > > - > > - # If bit (32+3) is set, then all lanes are empty > > - mov _unused_lanes(state), unused_lanes > > - bt $32+7, unused_lanes > > - jc return_null > > - > > - # find a lane with a non-null job > > - xor idx, idx > > - offset = (_ldata + 1*_LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne one(%rip), idx > > - offset = (_ldata + 2*_LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne two(%rip), idx > > - offset = (_ldata + 3*_LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > - cmovne three(%rip), idx > > - > > - # copy idx to empty lanes > > -copy_lane_data: > > - offset = (_args + _data_ptr) > > - mov offset(state,idx,8), tmp > > - > > - I = 0 > > -.rep 4 > > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > > - cmpq $0, offset(state) > > -.altmacro > > - JNE_SKIP %I > > - offset = (_args + _data_ptr + 8*I) > > - mov tmp, offset(state) > > - offset = (_lens + 8*I +4) > > - movl $0xFFFFFFFF, offset(state) > > -LABEL skip_ %I > > - I = (I+1) > > -.noaltmacro > > -.endr > > - > > - # Find min length > > - mov _lens + 0*8(state),lens0 > > - mov lens0,idx > > - mov _lens + 1*8(state),lens1 > > - cmp idx,lens1 > > - cmovb lens1,idx > > - mov _lens + 2*8(state),lens2 > > - cmp idx,lens2 > > - cmovb lens2,idx > > - mov _lens + 3*8(state),lens3 > > - cmp idx,lens3 > > - cmovb lens3,idx > > - mov idx,len2 > > - and $0xF,idx > > - and $~0xFF,len2 > > - jz len_is_0 > > - > > - sub len2, lens0 > > - sub len2, lens1 > > - sub len2, lens2 > > - sub len2, lens3 > > - shr $32,len2 > > - mov lens0, _lens + 0*8(state) > > - mov lens1, _lens + 1*8(state) > > - mov lens2, _lens + 2*8(state) > > - mov lens3, _lens + 3*8(state) > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha512_x4_avx2 > > - # state and idx are intact > > - > > -len_is_0: > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $8, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens+4(state, idx, 8) > > - > > - vmovq _args_digest+0*32(state, idx, 8), %xmm0 > > - vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 > > - vmovq _args_digest+2*32(state, idx, 8), %xmm1 > > - vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 > > - vmovq _args_digest+4*32(state, idx, 8), %xmm2 > > - vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 > > - vmovq _args_digest+6*32(state, idx, 8), %xmm3 > > - vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 > > - > > - vmovdqu %xmm0, _result_digest(job_rax) > > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > > - vmovdqu %xmm2, _result_digest+2*16(job_rax) > > - vmovdqu %xmm3, _result_digest+3*16(job_rax) > > - > > -return: > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > -ENDPROC(sha512_mb_mgr_flush_avx2) > > -.align 16 > > - > > -ENTRY(sha512_mb_mgr_get_comp_job_avx2) > > - push %rbx > > - > > - mov _unused_lanes(state), unused_lanes > > - bt $(32+7), unused_lanes > > - jc .return_null > > - > > - # Find min length > > - mov _lens(state),lens0 > > - mov lens0,idx > > - mov _lens+1*8(state),lens1 > > - cmp idx,lens1 > > - cmovb lens1,idx > > - mov _lens+2*8(state),lens2 > > - cmp idx,lens2 > > - cmovb lens2,idx > > - mov _lens+3*8(state),lens3 > > - cmp idx,lens3 > > - cmovb lens3,idx > > - test $~0xF,idx > > - jnz .return_null > > - and $0xF,idx > > - > > - #process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - mov _unused_lanes(state), unused_lanes > > - shl $8, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF, _lens+4(state, idx, 8) > > - > > - vmovq _args_digest(state, idx, 8), %xmm0 > > - vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 > > - vmovq _args_digest+2*32(state, idx, 8), %xmm1 > > - vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 > > - vmovq _args_digest+4*32(state, idx, 8), %xmm2 > > - vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 > > - vmovq _args_digest+6*32(state, idx, 8), %xmm3 > > - vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 > > - > > - vmovdqu %xmm0, _result_digest+0*16(job_rax) > > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > > - vmovdqu %xmm2, _result_digest+2*16(job_rax) > > - vmovdqu %xmm3, _result_digest+3*16(job_rax) > > - > > - pop %rbx > > - > > - ret > > - > > -.return_null: > > - xor job_rax, job_rax > > - pop %rbx > > - ret > > -ENDPROC(sha512_mb_mgr_get_comp_job_avx2) > > - > > -.section .rodata.cst8.one, "aM", @progbits, 8 > > -.align 8 > > -one: > > -.quad 1 > > - > > -.section .rodata.cst8.two, "aM", @progbits, 8 > > -.align 8 > > -two: > > -.quad 2 > > - > > -.section .rodata.cst8.three, "aM", @progbits, 8 > > -.align 8 > > -three: > > -.quad 3 > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > > deleted file mode 100644 > > index d08805032f01..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > > +++ /dev/null > > @@ -1,69 +0,0 @@ > > -/* > > - * Initialization code for multi buffer SHA256 algorithm for AVX2 > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include "sha512_mb_mgr.h" > > - > > -void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state) > > -{ > > - unsigned int j; > > - > > - /* initially all lanes are unused */ > > - state->lens[0] = 0xFFFFFFFF00000000; > > - state->lens[1] = 0xFFFFFFFF00000001; > > - state->lens[2] = 0xFFFFFFFF00000002; > > - state->lens[3] = 0xFFFFFFFF00000003; > > - > > - state->unused_lanes = 0xFF03020100; > > - for (j = 0; j < 4; j++) > > - state->ldata[j].job_in_lane = NULL; > > -} > > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > > deleted file mode 100644 > > index 4ba709ba78e5..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > > +++ /dev/null > > @@ -1,224 +0,0 @@ > > -/* > > - * Buffer submit code for multi buffer SHA512 algorithm > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -#include <linux/linkage.h> > > -#include <asm/frame.h> > > -#include "sha512_mb_mgr_datastruct.S" > > - > > -.extern sha512_x4_avx2 > > - > > -#define arg1 %rdi > > -#define arg2 %rsi > > - > > -#define idx %rdx > > -#define last_len %rdx > > - > > -#define size_offset %rcx > > -#define tmp2 %rcx > > - > > -# Common definitions > > -#define state arg1 > > -#define job arg2 > > -#define len2 arg2 > > -#define p2 arg2 > > - > > -#define p %r11 > > -#define start_offset %r11 > > - > > -#define unused_lanes %rbx > > - > > -#define job_rax %rax > > -#define len %rax > > - > > -#define lane %r12 > > -#define tmp3 %r12 > > -#define lens3 %r12 > > - > > -#define extra_blocks %r8 > > -#define lens0 %r8 > > - > > -#define tmp %r9 > > -#define lens1 %r9 > > - > > -#define lane_data %r10 > > -#define lens2 %r10 > > - > > -#define DWORD_len %eax > > - > > -# JOB* sha512_mb_mgr_submit_avx2(MB_MGR *state, JOB *job) > > -# arg 1 : rcx : state > > -# arg 2 : rdx : job > > -ENTRY(sha512_mb_mgr_submit_avx2) > > - FRAME_BEGIN > > - push %rbx > > - push %r12 > > - > > - mov _unused_lanes(state), unused_lanes > > - movzb %bl,lane > > - shr $8, unused_lanes > > - imul $_LANE_DATA_size, lane,lane_data > > - movl $STS_BEING_PROCESSED, _status(job) > > - lea _ldata(state, lane_data), lane_data > > - mov unused_lanes, _unused_lanes(state) > > - movl _len(job), DWORD_len > > - > > - mov job, _job_in_lane(lane_data) > > - movl DWORD_len,_lens+4(state , lane, 8) > > - > > - # Load digest words from result_digest > > - vmovdqu _result_digest+0*16(job), %xmm0 > > - vmovdqu _result_digest+1*16(job), %xmm1 > > - vmovdqu _result_digest+2*16(job), %xmm2 > > - vmovdqu _result_digest+3*16(job), %xmm3 > > - > > - vmovq %xmm0, _args_digest(state, lane, 8) > > - vpextrq $1, %xmm0, _args_digest+1*32(state , lane, 8) > > - vmovq %xmm1, _args_digest+2*32(state , lane, 8) > > - vpextrq $1, %xmm1, _args_digest+3*32(state , lane, 8) > > - vmovq %xmm2, _args_digest+4*32(state , lane, 8) > > - vpextrq $1, %xmm2, _args_digest+5*32(state , lane, 8) > > - vmovq %xmm3, _args_digest+6*32(state , lane, 8) > > - vpextrq $1, %xmm3, _args_digest+7*32(state , lane, 8) > > - > > - mov _buffer(job), p > > - mov p, _args_data_ptr(state, lane, 8) > > - > > - cmp $0xFF, unused_lanes > > - jne return_null > > - > > -start_loop: > > - > > - # Find min length > > - mov _lens+0*8(state),lens0 > > - mov lens0,idx > > - mov _lens+1*8(state),lens1 > > - cmp idx,lens1 > > - cmovb lens1, idx > > - mov _lens+2*8(state),lens2 > > - cmp idx,lens2 > > - cmovb lens2,idx > > - mov _lens+3*8(state),lens3 > > - cmp idx,lens3 > > - cmovb lens3,idx > > - mov idx,len2 > > - and $0xF,idx > > - and $~0xFF,len2 > > - jz len_is_0 > > - > > - sub len2,lens0 > > - sub len2,lens1 > > - sub len2,lens2 > > - sub len2,lens3 > > - shr $32,len2 > > - mov lens0, _lens + 0*8(state) > > - mov lens1, _lens + 1*8(state) > > - mov lens2, _lens + 2*8(state) > > - mov lens3, _lens + 3*8(state) > > - > > - # "state" and "args" are the same address, arg1 > > - # len is arg2 > > - call sha512_x4_avx2 > > - # state and idx are intact > > - > > -len_is_0: > > - > > - # process completed job "idx" > > - imul $_LANE_DATA_size, idx, lane_data > > - lea _ldata(state, lane_data), lane_data > > - > > - mov _job_in_lane(lane_data), job_rax > > - mov _unused_lanes(state), unused_lanes > > - movq $0, _job_in_lane(lane_data) > > - movl $STS_COMPLETED, _status(job_rax) > > - shl $8, unused_lanes > > - or idx, unused_lanes > > - mov unused_lanes, _unused_lanes(state) > > - > > - movl $0xFFFFFFFF,_lens+4(state,idx,8) > > - vmovq _args_digest+0*32(state , idx, 8), %xmm0 > > - vpinsrq $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0 > > - vmovq _args_digest+2*32(state , idx, 8), %xmm1 > > - vpinsrq $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1 > > - vmovq _args_digest+4*32(state , idx, 8), %xmm2 > > - vpinsrq $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2 > > - vmovq _args_digest+6*32(state , idx, 8), %xmm3 > > - vpinsrq $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3 > > - > > - vmovdqu %xmm0, _result_digest + 0*16(job_rax) > > - vmovdqu %xmm1, _result_digest + 1*16(job_rax) > > - vmovdqu %xmm2, _result_digest + 2*16(job_rax) > > - vmovdqu %xmm3, _result_digest + 3*16(job_rax) > > - > > -return: > > - pop %r12 > > - pop %rbx > > - FRAME_END > > - ret > > - > > -return_null: > > - xor job_rax, job_rax > > - jmp return > > -ENDPROC(sha512_mb_mgr_submit_avx2) > > - > > -/* UNUSED? > > -.section .rodata.cst16, "aM", @progbits, 16 > > -.align 16 > > -H0: .int 0x6a09e667 > > -H1: .int 0xbb67ae85 > > -H2: .int 0x3c6ef372 > > -H3: .int 0xa54ff53a > > -H4: .int 0x510e527f > > -H5: .int 0x9b05688c > > -H6: .int 0x1f83d9ab > > -H7: .int 0x5be0cd19 > > -*/ > > diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > > deleted file mode 100644 > > index e22e907643a6..000000000000 > > --- a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > > +++ /dev/null > > @@ -1,531 +0,0 @@ > > -/* > > - * Multi-buffer SHA512 algorithm hash compute routine > > - * > > - * This file is provided under a dual BSD/GPLv2 license. When using or > > - * redistributing this file, you may do so under either license. > > - * > > - * GPL LICENSE SUMMARY > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * This program is free software; you can redistribute it and/or modify > > - * it under the terms of version 2 of the GNU General Public License as > > - * published by the Free Software Foundation. > > - * > > - * This program is distributed in the hope that it will be useful, but > > - * WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * General Public License for more details. > > - * > > - * Contact Information: > > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > > - * > > - * BSD LICENSE > > - * > > - * Copyright(c) 2016 Intel Corporation. > > - * > > - * Redistribution and use in source and binary forms, with or without > > - * modification, are permitted provided that the following conditions > > - * are met: > > - * > > - * * Redistributions of source code must retain the above copyright > > - * notice, this list of conditions and the following disclaimer. > > - * * Redistributions in binary form must reproduce the above copyright > > - * notice, this list of conditions and the following disclaimer in > > - * the documentation and/or other materials provided with the > > - * distribution. > > - * * Neither the name of Intel Corporation nor the names of its > > - * contributors may be used to endorse or promote products derived > > - * from this software without specific prior written permission. > > - * > > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > > - */ > > - > > -# code to compute quad SHA512 using AVX2 > > -# use YMMs to tackle the larger digest size > > -# outer calling routine takes care of save and restore of XMM registers > > -# Logic designed/laid out by JDG > > - > > -# Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; ymm0-15 > > -# Stack must be aligned to 32 bytes before call > > -# Linux clobbers: rax rbx rcx rsi r8 r9 r10 r11 r12 > > -# Linux preserves: rcx rdx rdi rbp r13 r14 r15 > > -# clobbers ymm0-15 > > - > > -#include <linux/linkage.h> > > -#include "sha512_mb_mgr_datastruct.S" > > - > > -arg1 = %rdi > > -arg2 = %rsi > > - > > -# Common definitions > > -STATE = arg1 > > -INP_SIZE = arg2 > > - > > -IDX = %rax > > -ROUND = %rbx > > -TBL = %r8 > > - > > -inp0 = %r9 > > -inp1 = %r10 > > -inp2 = %r11 > > -inp3 = %r12 > > - > > -a = %ymm0 > > -b = %ymm1 > > -c = %ymm2 > > -d = %ymm3 > > -e = %ymm4 > > -f = %ymm5 > > -g = %ymm6 > > -h = %ymm7 > > - > > -a0 = %ymm8 > > -a1 = %ymm9 > > -a2 = %ymm10 > > - > > -TT0 = %ymm14 > > -TT1 = %ymm13 > > -TT2 = %ymm12 > > -TT3 = %ymm11 > > -TT4 = %ymm10 > > -TT5 = %ymm9 > > - > > -T1 = %ymm14 > > -TMP = %ymm15 > > - > > -# Define stack usage > > -STACK_SPACE1 = SZ4*16 + NUM_SHA512_DIGEST_WORDS*SZ4 + 24 > > - > > -#define VMOVPD vmovupd > > -_digest = SZ4*16 > > - > > -# transpose r0, r1, r2, r3, t0, t1 > > -# "transpose" data in {r0..r3} using temps {t0..t3} > > -# Input looks like: {r0 r1 r2 r3} > > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > > -# > > -# output looks like: {t0 r1 r0 r3} > > -# t0 = {d1 d0 c1 c0 b1 b0 a1 a0} > > -# r1 = {d3 d2 c3 c2 b3 b2 a3 a2} > > -# r0 = {d5 d4 c5 c4 b5 b4 a5 a4} > > -# r3 = {d7 d6 c7 c6 b7 b6 a7 a6} > > - > > -.macro TRANSPOSE r0 r1 r2 r3 t0 t1 > > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > > - > > - vperm2f128 $0x20, \r2, \r0, \r1 # h6...a6 > > - vperm2f128 $0x31, \r2, \r0, \r3 # h2...a2 > > - vperm2f128 $0x31, \t1, \t0, \r0 # h5...a5 > > - vperm2f128 $0x20, \t1, \t0, \t0 # h1...a1 > > -.endm > > - > > -.macro ROTATE_ARGS > > -TMP_ = h > > -h = g > > -g = f > > -f = e > > -e = d > > -d = c > > -c = b > > -b = a > > -a = TMP_ > > -.endm > > - > > -# PRORQ reg, imm, tmp > > -# packed-rotate-right-double > > -# does a rotate by doing two shifts and an or > > -.macro _PRORQ reg imm tmp > > - vpsllq $(64-\imm),\reg,\tmp > > - vpsrlq $\imm,\reg, \reg > > - vpor \tmp,\reg, \reg > > -.endm > > - > > -# non-destructive > > -# PRORQ_nd reg, imm, tmp, src > > -.macro _PRORQ_nd reg imm tmp src > > - vpsllq $(64-\imm), \src, \tmp > > - vpsrlq $\imm, \src, \reg > > - vpor \tmp, \reg, \reg > > -.endm > > - > > -# PRORQ dst/src, amt > > -.macro PRORQ reg imm > > - _PRORQ \reg, \imm, TMP > > -.endm > > - > > -# PRORQ_nd dst, src, amt > > -.macro PRORQ_nd reg tmp imm > > - _PRORQ_nd \reg, \imm, TMP, \tmp > > -.endm > > - > > -#; arguments passed implicitly in preprocessor symbols i, a...h > > -.macro ROUND_00_15 _T1 i > > - PRORQ_nd a0, e, (18-14) # sig1: a0 = (e >> 4) > > - > > - vpxor g, f, a2 # ch: a2 = f^g > > - vpand e,a2, a2 # ch: a2 = (f^g)&e > > - vpxor g, a2, a2 # a2 = ch > > - > > - PRORQ_nd a1,e,41 # sig1: a1 = (e >> 25) > > - > > - offset = SZ4*(\i & 0xf) > > - vmovdqu \_T1,offset(%rsp) > > - vpaddq (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K > > - vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) > > - PRORQ a0, 14 # sig1: a0 = (e >> 6) ^ (e >> 11) > > - vpaddq a2, h, h # h = h + ch > > - PRORQ_nd a2,a,6 # sig0: a2 = (a >> 11) > > - vpaddq \_T1,h, h # h = h + ch + W + K > > - vpxor a1, a0, a0 # a0 = sigma1 > > - vmovdqu a,\_T1 > > - PRORQ_nd a1,a,39 # sig0: a1 = (a >> 22) > > - vpxor c, \_T1, \_T1 # maj: T1 = a^c > > - add $SZ4, ROUND # ROUND++ > > - vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b > > - vpaddq a0, h, h > > - vpaddq h, d, d > > - vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) > > - PRORQ a2,28 # sig0: a2 = (a >> 2) ^ (a >> 13) > > - vpxor a1, a2, a2 # a2 = sig0 > > - vpand c, a, a1 # maj: a1 = a&c > > - vpor \_T1, a1, a1 # a1 = maj > > - vpaddq a1, h, h # h = h + ch + W + K + maj > > - vpaddq a2, h, h # h = h + ch + W + K + maj + sigma0 > > - ROTATE_ARGS > > -.endm > > - > > - > > -#; arguments passed implicitly in preprocessor symbols i, a...h > > -.macro ROUND_16_XX _T1 i > > - vmovdqu SZ4*((\i-15)&0xf)(%rsp), \_T1 > > - vmovdqu SZ4*((\i-2)&0xf)(%rsp), a1 > > - vmovdqu \_T1, a0 > > - PRORQ \_T1,7 > > - vmovdqu a1, a2 > > - PRORQ a1,42 > > - vpxor a0, \_T1, \_T1 > > - PRORQ \_T1, 1 > > - vpxor a2, a1, a1 > > - PRORQ a1, 19 > > - vpsrlq $7, a0, a0 > > - vpxor a0, \_T1, \_T1 > > - vpsrlq $6, a2, a2 > > - vpxor a2, a1, a1 > > - vpaddq SZ4*((\i-16)&0xf)(%rsp), \_T1, \_T1 > > - vpaddq SZ4*((\i-7)&0xf)(%rsp), a1, a1 > > - vpaddq a1, \_T1, \_T1 > > - > > - ROUND_00_15 \_T1,\i > > -.endm > > - > > - > > -# void sha512_x4_avx2(void *STATE, const int INP_SIZE) > > -# arg 1 : STATE : pointer to input data > > -# arg 2 : INP_SIZE : size of data in blocks (assumed >= 1) > > -ENTRY(sha512_x4_avx2) > > - # general registers preserved in outer calling routine > > - # outer calling routine saves all the XMM registers > > - # save callee-saved clobbered registers to comply with C function ABI > > - push %r12 > > - push %r13 > > - push %r14 > > - push %r15 > > - > > - sub $STACK_SPACE1, %rsp > > - > > - # Load the pre-transposed incoming digest. > > - vmovdqu 0*SHA512_DIGEST_ROW_SIZE(STATE),a > > - vmovdqu 1*SHA512_DIGEST_ROW_SIZE(STATE),b > > - vmovdqu 2*SHA512_DIGEST_ROW_SIZE(STATE),c > > - vmovdqu 3*SHA512_DIGEST_ROW_SIZE(STATE),d > > - vmovdqu 4*SHA512_DIGEST_ROW_SIZE(STATE),e > > - vmovdqu 5*SHA512_DIGEST_ROW_SIZE(STATE),f > > - vmovdqu 6*SHA512_DIGEST_ROW_SIZE(STATE),g > > - vmovdqu 7*SHA512_DIGEST_ROW_SIZE(STATE),h > > - > > - lea K512_4(%rip),TBL > > - > > - # load the address of each of the 4 message lanes > > - # getting ready to transpose input onto stack > > - mov _data_ptr+0*PTR_SZ(STATE),inp0 > > - mov _data_ptr+1*PTR_SZ(STATE),inp1 > > - mov _data_ptr+2*PTR_SZ(STATE),inp2 > > - mov _data_ptr+3*PTR_SZ(STATE),inp3 > > - > > - xor IDX, IDX > > -lloop: > > - xor ROUND, ROUND > > - > > - # save old digest > > - vmovdqu a, _digest(%rsp) > > - vmovdqu b, _digest+1*SZ4(%rsp) > > - vmovdqu c, _digest+2*SZ4(%rsp) > > - vmovdqu d, _digest+3*SZ4(%rsp) > > - vmovdqu e, _digest+4*SZ4(%rsp) > > - vmovdqu f, _digest+5*SZ4(%rsp) > > - vmovdqu g, _digest+6*SZ4(%rsp) > > - vmovdqu h, _digest+7*SZ4(%rsp) > > - i = 0 > > -.rep 4 > > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP > > - VMOVPD i*32(inp0, IDX), TT2 > > - VMOVPD i*32(inp1, IDX), TT1 > > - VMOVPD i*32(inp2, IDX), TT4 > > - VMOVPD i*32(inp3, IDX), TT3 > > - TRANSPOSE TT2, TT1, TT4, TT3, TT0, TT5 > > - vpshufb TMP, TT0, TT0 > > - vpshufb TMP, TT1, TT1 > > - vpshufb TMP, TT2, TT2 > > - vpshufb TMP, TT3, TT3 > > - ROUND_00_15 TT0,(i*4+0) > > - ROUND_00_15 TT1,(i*4+1) > > - ROUND_00_15 TT2,(i*4+2) > > - ROUND_00_15 TT3,(i*4+3) > > - i = (i+1) > > -.endr > > - add $128, IDX > > - > > - i = (i*4) > > - > > - jmp Lrounds_16_xx > > -.align 16 > > -Lrounds_16_xx: > > -.rep 16 > > - ROUND_16_XX T1, i > > - i = (i+1) > > -.endr > > - cmp $0xa00,ROUND > > - jb Lrounds_16_xx > > - > > - # add old digest > > - vpaddq _digest(%rsp), a, a > > - vpaddq _digest+1*SZ4(%rsp), b, b > > - vpaddq _digest+2*SZ4(%rsp), c, c > > - vpaddq _digest+3*SZ4(%rsp), d, d > > - vpaddq _digest+4*SZ4(%rsp), e, e > > - vpaddq _digest+5*SZ4(%rsp), f, f > > - vpaddq _digest+6*SZ4(%rsp), g, g > > - vpaddq _digest+7*SZ4(%rsp), h, h > > - > > - sub $1, INP_SIZE # unit is blocks > > - jne lloop > > - > > - # write back to memory (state object) the transposed digest > > - vmovdqu a, 0*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu b, 1*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu c, 2*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu d, 3*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu e, 4*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu f, 5*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu g, 6*SHA512_DIGEST_ROW_SIZE(STATE) > > - vmovdqu h, 7*SHA512_DIGEST_ROW_SIZE(STATE) > > - > > - # update input data pointers > > - add IDX, inp0 > > - mov inp0, _data_ptr+0*PTR_SZ(STATE) > > - add IDX, inp1 > > - mov inp1, _data_ptr+1*PTR_SZ(STATE) > > - add IDX, inp2 > > - mov inp2, _data_ptr+2*PTR_SZ(STATE) > > - add IDX, inp3 > > - mov inp3, _data_ptr+3*PTR_SZ(STATE) > > - > > - #;;;;;;;;;;;;;;; > > - #; Postamble > > - add $STACK_SPACE1, %rsp > > - # restore callee-saved clobbered registers > > - > > - pop %r15 > > - pop %r14 > > - pop %r13 > > - pop %r12 > > - > > - # outer calling routine restores XMM and other GP registers > > - ret > > -ENDPROC(sha512_x4_avx2) > > - > > -.section .rodata.K512_4, "a", @progbits > > -.align 64 > > -K512_4: > > - .octa 0x428a2f98d728ae22428a2f98d728ae22,\ > > - 0x428a2f98d728ae22428a2f98d728ae22 > > - .octa 0x7137449123ef65cd7137449123ef65cd,\ > > - 0x7137449123ef65cd7137449123ef65cd > > - .octa 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f,\ > > - 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f > > - .octa 0xe9b5dba58189dbbce9b5dba58189dbbc,\ > > - 0xe9b5dba58189dbbce9b5dba58189dbbc > > - .octa 0x3956c25bf348b5383956c25bf348b538,\ > > - 0x3956c25bf348b5383956c25bf348b538 > > - .octa 0x59f111f1b605d01959f111f1b605d019,\ > > - 0x59f111f1b605d01959f111f1b605d019 > > - .octa 0x923f82a4af194f9b923f82a4af194f9b,\ > > - 0x923f82a4af194f9b923f82a4af194f9b > > - .octa 0xab1c5ed5da6d8118ab1c5ed5da6d8118,\ > > - 0xab1c5ed5da6d8118ab1c5ed5da6d8118 > > - .octa 0xd807aa98a3030242d807aa98a3030242,\ > > - 0xd807aa98a3030242d807aa98a3030242 > > - .octa 0x12835b0145706fbe12835b0145706fbe,\ > > - 0x12835b0145706fbe12835b0145706fbe > > - .octa 0x243185be4ee4b28c243185be4ee4b28c,\ > > - 0x243185be4ee4b28c243185be4ee4b28c > > - .octa 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2,\ > > - 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2 > > - .octa 0x72be5d74f27b896f72be5d74f27b896f,\ > > - 0x72be5d74f27b896f72be5d74f27b896f > > - .octa 0x80deb1fe3b1696b180deb1fe3b1696b1,\ > > - 0x80deb1fe3b1696b180deb1fe3b1696b1 > > - .octa 0x9bdc06a725c712359bdc06a725c71235,\ > > - 0x9bdc06a725c712359bdc06a725c71235 > > - .octa 0xc19bf174cf692694c19bf174cf692694,\ > > - 0xc19bf174cf692694c19bf174cf692694 > > - .octa 0xe49b69c19ef14ad2e49b69c19ef14ad2,\ > > - 0xe49b69c19ef14ad2e49b69c19ef14ad2 > > - .octa 0xefbe4786384f25e3efbe4786384f25e3,\ > > - 0xefbe4786384f25e3efbe4786384f25e3 > > - .octa 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5,\ > > - 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5 > > - .octa 0x240ca1cc77ac9c65240ca1cc77ac9c65,\ > > - 0x240ca1cc77ac9c65240ca1cc77ac9c65 > > - .octa 0x2de92c6f592b02752de92c6f592b0275,\ > > - 0x2de92c6f592b02752de92c6f592b0275 > > - .octa 0x4a7484aa6ea6e4834a7484aa6ea6e483,\ > > - 0x4a7484aa6ea6e4834a7484aa6ea6e483 > > - .octa 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4,\ > > - 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4 > > - .octa 0x76f988da831153b576f988da831153b5,\ > > - 0x76f988da831153b576f988da831153b5 > > - .octa 0x983e5152ee66dfab983e5152ee66dfab,\ > > - 0x983e5152ee66dfab983e5152ee66dfab > > - .octa 0xa831c66d2db43210a831c66d2db43210,\ > > - 0xa831c66d2db43210a831c66d2db43210 > > - .octa 0xb00327c898fb213fb00327c898fb213f,\ > > - 0xb00327c898fb213fb00327c898fb213f > > - .octa 0xbf597fc7beef0ee4bf597fc7beef0ee4,\ > > - 0xbf597fc7beef0ee4bf597fc7beef0ee4 > > - .octa 0xc6e00bf33da88fc2c6e00bf33da88fc2,\ > > - 0xc6e00bf33da88fc2c6e00bf33da88fc2 > > - .octa 0xd5a79147930aa725d5a79147930aa725,\ > > - 0xd5a79147930aa725d5a79147930aa725 > > - .octa 0x06ca6351e003826f06ca6351e003826f,\ > > - 0x06ca6351e003826f06ca6351e003826f > > - .octa 0x142929670a0e6e70142929670a0e6e70,\ > > - 0x142929670a0e6e70142929670a0e6e70 > > - .octa 0x27b70a8546d22ffc27b70a8546d22ffc,\ > > - 0x27b70a8546d22ffc27b70a8546d22ffc > > - .octa 0x2e1b21385c26c9262e1b21385c26c926,\ > > - 0x2e1b21385c26c9262e1b21385c26c926 > > - .octa 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed,\ > > - 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed > > - .octa 0x53380d139d95b3df53380d139d95b3df,\ > > - 0x53380d139d95b3df53380d139d95b3df > > - .octa 0x650a73548baf63de650a73548baf63de,\ > > - 0x650a73548baf63de650a73548baf63de > > - .octa 0x766a0abb3c77b2a8766a0abb3c77b2a8,\ > > - 0x766a0abb3c77b2a8766a0abb3c77b2a8 > > - .octa 0x81c2c92e47edaee681c2c92e47edaee6,\ > > - 0x81c2c92e47edaee681c2c92e47edaee6 > > - .octa 0x92722c851482353b92722c851482353b,\ > > - 0x92722c851482353b92722c851482353b > > - .octa 0xa2bfe8a14cf10364a2bfe8a14cf10364,\ > > - 0xa2bfe8a14cf10364a2bfe8a14cf10364 > > - .octa 0xa81a664bbc423001a81a664bbc423001,\ > > - 0xa81a664bbc423001a81a664bbc423001 > > - .octa 0xc24b8b70d0f89791c24b8b70d0f89791,\ > > - 0xc24b8b70d0f89791c24b8b70d0f89791 > > - .octa 0xc76c51a30654be30c76c51a30654be30,\ > > - 0xc76c51a30654be30c76c51a30654be30 > > - .octa 0xd192e819d6ef5218d192e819d6ef5218,\ > > - 0xd192e819d6ef5218d192e819d6ef5218 > > - .octa 0xd69906245565a910d69906245565a910,\ > > - 0xd69906245565a910d69906245565a910 > > - .octa 0xf40e35855771202af40e35855771202a,\ > > - 0xf40e35855771202af40e35855771202a > > - .octa 0x106aa07032bbd1b8106aa07032bbd1b8,\ > > - 0x106aa07032bbd1b8106aa07032bbd1b8 > > - .octa 0x19a4c116b8d2d0c819a4c116b8d2d0c8,\ > > - 0x19a4c116b8d2d0c819a4c116b8d2d0c8 > > - .octa 0x1e376c085141ab531e376c085141ab53,\ > > - 0x1e376c085141ab531e376c085141ab53 > > - .octa 0x2748774cdf8eeb992748774cdf8eeb99,\ > > - 0x2748774cdf8eeb992748774cdf8eeb99 > > - .octa 0x34b0bcb5e19b48a834b0bcb5e19b48a8,\ > > - 0x34b0bcb5e19b48a834b0bcb5e19b48a8 > > - .octa 0x391c0cb3c5c95a63391c0cb3c5c95a63,\ > > - 0x391c0cb3c5c95a63391c0cb3c5c95a63 > > - .octa 0x4ed8aa4ae3418acb4ed8aa4ae3418acb,\ > > - 0x4ed8aa4ae3418acb4ed8aa4ae3418acb > > - .octa 0x5b9cca4f7763e3735b9cca4f7763e373,\ > > - 0x5b9cca4f7763e3735b9cca4f7763e373 > > - .octa 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3,\ > > - 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3 > > - .octa 0x748f82ee5defb2fc748f82ee5defb2fc,\ > > - 0x748f82ee5defb2fc748f82ee5defb2fc > > - .octa 0x78a5636f43172f6078a5636f43172f60,\ > > - 0x78a5636f43172f6078a5636f43172f60 > > - .octa 0x84c87814a1f0ab7284c87814a1f0ab72,\ > > - 0x84c87814a1f0ab7284c87814a1f0ab72 > > - .octa 0x8cc702081a6439ec8cc702081a6439ec,\ > > - 0x8cc702081a6439ec8cc702081a6439ec > > - .octa 0x90befffa23631e2890befffa23631e28,\ > > - 0x90befffa23631e2890befffa23631e28 > > - .octa 0xa4506cebde82bde9a4506cebde82bde9,\ > > - 0xa4506cebde82bde9a4506cebde82bde9 > > - .octa 0xbef9a3f7b2c67915bef9a3f7b2c67915,\ > > - 0xbef9a3f7b2c67915bef9a3f7b2c67915 > > - .octa 0xc67178f2e372532bc67178f2e372532b,\ > > - 0xc67178f2e372532bc67178f2e372532b > > - .octa 0xca273eceea26619cca273eceea26619c,\ > > - 0xca273eceea26619cca273eceea26619c > > - .octa 0xd186b8c721c0c207d186b8c721c0c207,\ > > - 0xd186b8c721c0c207d186b8c721c0c207 > > - .octa 0xeada7dd6cde0eb1eeada7dd6cde0eb1e,\ > > - 0xeada7dd6cde0eb1eeada7dd6cde0eb1e > > - .octa 0xf57d4f7fee6ed178f57d4f7fee6ed178,\ > > - 0xf57d4f7fee6ed178f57d4f7fee6ed178 > > - .octa 0x06f067aa72176fba06f067aa72176fba,\ > > - 0x06f067aa72176fba06f067aa72176fba > > - .octa 0x0a637dc5a2c898a60a637dc5a2c898a6,\ > > - 0x0a637dc5a2c898a60a637dc5a2c898a6 > > - .octa 0x113f9804bef90dae113f9804bef90dae,\ > > - 0x113f9804bef90dae113f9804bef90dae > > - .octa 0x1b710b35131c471b1b710b35131c471b,\ > > - 0x1b710b35131c471b1b710b35131c471b > > - .octa 0x28db77f523047d8428db77f523047d84,\ > > - 0x28db77f523047d8428db77f523047d84 > > - .octa 0x32caab7b40c7249332caab7b40c72493,\ > > - 0x32caab7b40c7249332caab7b40c72493 > > - .octa 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc,\ > > - 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc > > - .octa 0x431d67c49c100d4c431d67c49c100d4c,\ > > - 0x431d67c49c100d4c431d67c49c100d4c > > - .octa 0x4cc5d4becb3e42b64cc5d4becb3e42b6,\ > > - 0x4cc5d4becb3e42b64cc5d4becb3e42b6 > > - .octa 0x597f299cfc657e2a597f299cfc657e2a,\ > > - 0x597f299cfc657e2a597f299cfc657e2a > > - .octa 0x5fcb6fab3ad6faec5fcb6fab3ad6faec,\ > > - 0x5fcb6fab3ad6faec5fcb6fab3ad6faec > > - .octa 0x6c44198c4a4758176c44198c4a475817,\ > > - 0x6c44198c4a4758176c44198c4a475817 > > - > > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > > -.align 32 > > -PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607 > > - .octa 0x18191a1b1c1d1e1f1011121314151617 > > diff --git a/crypto/Kconfig b/crypto/Kconfig > > index f3e40ac56d93..4ee600bdefdb 100644 > > --- a/crypto/Kconfig > > +++ b/crypto/Kconfig > > @@ -213,20 +213,6 @@ config CRYPTO_CRYPTD > > converts an arbitrary synchronous software crypto algorithm > > into an asynchronous algorithm that executes in a kernel thread. > > > > -config CRYPTO_MCRYPTD > > - tristate "Software async multi-buffer crypto daemon" > > - select CRYPTO_BLKCIPHER > > - select CRYPTO_HASH > > - select CRYPTO_MANAGER > > - select CRYPTO_WORKQUEUE > > - help > > - This is a generic software asynchronous crypto daemon that > > - provides the kernel thread to assist multi-buffer crypto > > - algorithms for submitting jobs and flushing jobs in multi-buffer > > - crypto algorithms. Multi-buffer crypto algorithms are executed > > - in the context of this kernel thread and drivers can post > > - their crypto request asynchronously to be processed by this daemon. > > - > > config CRYPTO_AUTHENC > > tristate "Authenc support" > > select CRYPTO_AEAD > > @@ -848,54 +834,6 @@ config CRYPTO_SHA1_PPC_SPE > > SHA-1 secure hash standard (DFIPS 180-4) implemented > > using powerpc SPE SIMD instruction set. > > > > -config CRYPTO_SHA1_MB > > - tristate "SHA1 digest algorithm (x86_64 Multi-Buffer, Experimental)" > > - depends on X86 && 64BIT > > - select CRYPTO_SHA1 > > - select CRYPTO_HASH > > - select CRYPTO_MCRYPTD > > - help > > - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > > - using multi-buffer technique. This algorithm computes on > > - multiple data lanes concurrently with SIMD instructions for > > - better throughput. It should not be enabled by default but > > - used when there is significant amount of work to keep the keep > > - the data lanes filled to get performance benefit. If the data > > - lanes remain unfilled, a flush operation will be initiated to > > - process the crypto jobs, adding a slight latency. > > - > > -config CRYPTO_SHA256_MB > > - tristate "SHA256 digest algorithm (x86_64 Multi-Buffer, Experimental)" > > - depends on X86 && 64BIT > > - select CRYPTO_SHA256 > > - select CRYPTO_HASH > > - select CRYPTO_MCRYPTD > > - help > > - SHA-256 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > > - using multi-buffer technique. This algorithm computes on > > - multiple data lanes concurrently with SIMD instructions for > > - better throughput. It should not be enabled by default but > > - used when there is significant amount of work to keep the keep > > - the data lanes filled to get performance benefit. If the data > > - lanes remain unfilled, a flush operation will be initiated to > > - process the crypto jobs, adding a slight latency. > > - > > -config CRYPTO_SHA512_MB > > - tristate "SHA512 digest algorithm (x86_64 Multi-Buffer, Experimental)" > > - depends on X86 && 64BIT > > - select CRYPTO_SHA512 > > - select CRYPTO_HASH > > - select CRYPTO_MCRYPTD > > - help > > - SHA-512 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > > - using multi-buffer technique. This algorithm computes on > > - multiple data lanes concurrently with SIMD instructions for > > - better throughput. It should not be enabled by default but > > - used when there is significant amount of work to keep the keep > > - the data lanes filled to get performance benefit. If the data > > - lanes remain unfilled, a flush operation will be initiated to > > - process the crypto jobs, adding a slight latency. > > - > > config CRYPTO_SHA256 > > tristate "SHA224 and SHA256 digest algorithm" > > select CRYPTO_HASH > > diff --git a/crypto/Makefile b/crypto/Makefile > > index 6d1d40eeb964..80e3da755cbf 100644 > > --- a/crypto/Makefile > > +++ b/crypto/Makefile > > @@ -93,7 +93,6 @@ obj-$(CONFIG_CRYPTO_MORUS640) += morus640.o > > obj-$(CONFIG_CRYPTO_MORUS1280) += morus1280.o > > obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o > > obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o > > -obj-$(CONFIG_CRYPTO_MCRYPTD) += mcryptd.o > > obj-$(CONFIG_CRYPTO_DES) += des_generic.o > > obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o > > obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o > > diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c > > deleted file mode 100644 > > index f14152147ce8..000000000000 > > --- a/crypto/mcryptd.c > > +++ /dev/null > > @@ -1,675 +0,0 @@ > > -/* > > - * Software multibuffer async crypto daemon. > > - * > > - * Copyright (c) 2014 Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * Adapted from crypto daemon. > > - * > > - * This program is free software; you can redistribute it and/or modify it > > - * under the terms of the GNU General Public License as published by the Free > > - * Software Foundation; either version 2 of the License, or (at your option) > > - * any later version. > > - * > > - */ > > - > > -#include <crypto/algapi.h> > > -#include <crypto/internal/hash.h> > > -#include <crypto/internal/aead.h> > > -#include <crypto/mcryptd.h> > > -#include <crypto/crypto_wq.h> > > -#include <linux/err.h> > > -#include <linux/init.h> > > -#include <linux/kernel.h> > > -#include <linux/list.h> > > -#include <linux/module.h> > > -#include <linux/scatterlist.h> > > -#include <linux/sched.h> > > -#include <linux/sched/stat.h> > > -#include <linux/slab.h> > > - > > -#define MCRYPTD_MAX_CPU_QLEN 100 > > -#define MCRYPTD_BATCH 9 > > - > > -static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, > > - unsigned int tail); > > - > > -struct mcryptd_flush_list { > > - struct list_head list; > > - struct mutex lock; > > -}; > > - > > -static struct mcryptd_flush_list __percpu *mcryptd_flist; > > - > > -struct hashd_instance_ctx { > > - struct crypto_ahash_spawn spawn; > > - struct mcryptd_queue *queue; > > -}; > > - > > -static void mcryptd_queue_worker(struct work_struct *work); > > - > > -void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay) > > -{ > > - struct mcryptd_flush_list *flist; > > - > > - if (!cstate->flusher_engaged) { > > - /* put the flusher on the flush list */ > > - flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); > > - mutex_lock(&flist->lock); > > - list_add_tail(&cstate->flush_list, &flist->list); > > - cstate->flusher_engaged = true; > > - cstate->next_flush = jiffies + delay; > > - queue_delayed_work_on(smp_processor_id(), kcrypto_wq, > > - &cstate->flush, delay); > > - mutex_unlock(&flist->lock); > > - } > > -} > > -EXPORT_SYMBOL(mcryptd_arm_flusher); > > - > > -static int mcryptd_init_queue(struct mcryptd_queue *queue, > > - unsigned int max_cpu_qlen) > > -{ > > - int cpu; > > - struct mcryptd_cpu_queue *cpu_queue; > > - > > - queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue); > > - pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue); > > - if (!queue->cpu_queue) > > - return -ENOMEM; > > - for_each_possible_cpu(cpu) { > > - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); > > - pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue); > > - crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); > > - INIT_WORK(&cpu_queue->work, mcryptd_queue_worker); > > - spin_lock_init(&cpu_queue->q_lock); > > - } > > - return 0; > > -} > > - > > -static void mcryptd_fini_queue(struct mcryptd_queue *queue) > > -{ > > - int cpu; > > - struct mcryptd_cpu_queue *cpu_queue; > > - > > - for_each_possible_cpu(cpu) { > > - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); > > - BUG_ON(cpu_queue->queue.qlen); > > - } > > - free_percpu(queue->cpu_queue); > > -} > > - > > -static int mcryptd_enqueue_request(struct mcryptd_queue *queue, > > - struct crypto_async_request *request, > > - struct mcryptd_hash_request_ctx *rctx) > > -{ > > - int cpu, err; > > - struct mcryptd_cpu_queue *cpu_queue; > > - > > - cpu_queue = raw_cpu_ptr(queue->cpu_queue); > > - spin_lock(&cpu_queue->q_lock); > > - cpu = smp_processor_id(); > > - rctx->tag.cpu = smp_processor_id(); > > - > > - err = crypto_enqueue_request(&cpu_queue->queue, request); > > - pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n", > > - cpu, cpu_queue, request); > > - spin_unlock(&cpu_queue->q_lock); > > - queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); > > - > > - return err; > > -} > > - > > -/* > > - * Try to opportunisticlly flush the partially completed jobs if > > - * crypto daemon is the only task running. > > - */ > > -static void mcryptd_opportunistic_flush(void) > > -{ > > - struct mcryptd_flush_list *flist; > > - struct mcryptd_alg_cstate *cstate; > > - > > - flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); > > - while (single_task_running()) { > > - mutex_lock(&flist->lock); > > - cstate = list_first_entry_or_null(&flist->list, > > - struct mcryptd_alg_cstate, flush_list); > > - if (!cstate || !cstate->flusher_engaged) { > > - mutex_unlock(&flist->lock); > > - return; > > - } > > - list_del(&cstate->flush_list); > > - cstate->flusher_engaged = false; > > - mutex_unlock(&flist->lock); > > - cstate->alg_state->flusher(cstate); > > - } > > -} > > - > > -/* > > - * Called in workqueue context, do one real cryption work (via > > - * req->complete) and reschedule itself if there are more work to > > - * do. > > - */ > > -static void mcryptd_queue_worker(struct work_struct *work) > > -{ > > - struct mcryptd_cpu_queue *cpu_queue; > > - struct crypto_async_request *req, *backlog; > > - int i; > > - > > - /* > > - * Need to loop through more than once for multi-buffer to > > - * be effective. > > - */ > > - > > - cpu_queue = container_of(work, struct mcryptd_cpu_queue, work); > > - i = 0; > > - while (i < MCRYPTD_BATCH || single_task_running()) { > > - > > - spin_lock_bh(&cpu_queue->q_lock); > > - backlog = crypto_get_backlog(&cpu_queue->queue); > > - req = crypto_dequeue_request(&cpu_queue->queue); > > - spin_unlock_bh(&cpu_queue->q_lock); > > - > > - if (!req) { > > - mcryptd_opportunistic_flush(); > > - return; > > - } > > - > > - if (backlog) > > - backlog->complete(backlog, -EINPROGRESS); > > - req->complete(req, 0); > > - if (!cpu_queue->queue.qlen) > > - return; > > - ++i; > > - } > > - if (cpu_queue->queue.qlen) > > - queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work); > > -} > > - > > -void mcryptd_flusher(struct work_struct *__work) > > -{ > > - struct mcryptd_alg_cstate *alg_cpu_state; > > - struct mcryptd_alg_state *alg_state; > > - struct mcryptd_flush_list *flist; > > - int cpu; > > - > > - cpu = smp_processor_id(); > > - alg_cpu_state = container_of(to_delayed_work(__work), > > - struct mcryptd_alg_cstate, flush); > > - alg_state = alg_cpu_state->alg_state; > > - if (alg_cpu_state->cpu != cpu) > > - pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n", > > - cpu, alg_cpu_state->cpu); > > - > > - if (alg_cpu_state->flusher_engaged) { > > - flist = per_cpu_ptr(mcryptd_flist, cpu); > > - mutex_lock(&flist->lock); > > - list_del(&alg_cpu_state->flush_list); > > - alg_cpu_state->flusher_engaged = false; > > - mutex_unlock(&flist->lock); > > - alg_state->flusher(alg_cpu_state); > > - } > > -} > > -EXPORT_SYMBOL_GPL(mcryptd_flusher); > > - > > -static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm) > > -{ > > - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); > > - struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst); > > - > > - return ictx->queue; > > -} > > - > > -static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, > > - unsigned int tail) > > -{ > > - char *p; > > - struct crypto_instance *inst; > > - int err; > > - > > - p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); > > - if (!p) > > - return ERR_PTR(-ENOMEM); > > - > > - inst = (void *)(p + head); > > - > > - err = -ENAMETOOLONG; > > - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, > > - "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) > > - goto out_free_inst; > > - > > - memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); > > - > > - inst->alg.cra_priority = alg->cra_priority + 50; > > - inst->alg.cra_blocksize = alg->cra_blocksize; > > - inst->alg.cra_alignmask = alg->cra_alignmask; > > - > > -out: > > - return p; > > - > > -out_free_inst: > > - kfree(p); > > - p = ERR_PTR(err); > > - goto out; > > -} > > - > > -static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type, > > - u32 *mask) > > -{ > > - struct crypto_attr_type *algt; > > - > > - algt = crypto_get_attr_type(tb); > > - if (IS_ERR(algt)) > > - return false; > > - > > - *type |= algt->type & CRYPTO_ALG_INTERNAL; > > - *mask |= algt->mask & CRYPTO_ALG_INTERNAL; > > - > > - if (*type & *mask & CRYPTO_ALG_INTERNAL) > > - return true; > > - else > > - return false; > > -} > > - > > -static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) > > -{ > > - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); > > - struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); > > - struct crypto_ahash_spawn *spawn = &ictx->spawn; > > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); > > - struct crypto_ahash *hash; > > - > > - hash = crypto_spawn_ahash(spawn); > > - if (IS_ERR(hash)) > > - return PTR_ERR(hash); > > - > > - ctx->child = hash; > > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > > - sizeof(struct mcryptd_hash_request_ctx) + > > - crypto_ahash_reqsize(hash)); > > - return 0; > > -} > > - > > -static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) > > -{ > > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); > > - > > - crypto_free_ahash(ctx->child); > > -} > > - > > -static int mcryptd_hash_setkey(struct crypto_ahash *parent, > > - const u8 *key, unsigned int keylen) > > -{ > > - struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); > > - struct crypto_ahash *child = ctx->child; > > - int err; > > - > > - crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); > > - crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) & > > - CRYPTO_TFM_REQ_MASK); > > - err = crypto_ahash_setkey(child, key, keylen); > > - crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) & > > - CRYPTO_TFM_RES_MASK); > > - return err; > > -} > > - > > -static int mcryptd_hash_enqueue(struct ahash_request *req, > > - crypto_completion_t complete) > > -{ > > - int ret; > > - > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > > - struct mcryptd_queue *queue = > > - mcryptd_get_queue(crypto_ahash_tfm(tfm)); > > - > > - rctx->complete = req->base.complete; > > - req->base.complete = complete; > > - > > - ret = mcryptd_enqueue_request(queue, &req->base, rctx); > > - > > - return ret; > > -} > > - > > -static void mcryptd_hash_init(struct crypto_async_request *req_async, int err) > > -{ > > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); > > - struct crypto_ahash *child = ctx->child; > > - struct ahash_request *req = ahash_request_cast(req_async); > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - struct ahash_request *desc = &rctx->areq; > > - > > - if (unlikely(err == -EINPROGRESS)) > > - goto out; > > - > > - ahash_request_set_tfm(desc, child); > > - ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, > > - rctx->complete, req_async); > > - > > - rctx->out = req->result; > > - err = crypto_ahash_init(desc); > > - > > -out: > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > -} > > - > > -static int mcryptd_hash_init_enqueue(struct ahash_request *req) > > -{ > > - return mcryptd_hash_enqueue(req, mcryptd_hash_init); > > -} > > - > > -static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) > > -{ > > - struct ahash_request *req = ahash_request_cast(req_async); > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - > > - if (unlikely(err == -EINPROGRESS)) > > - goto out; > > - > > - rctx->out = req->result; > > - err = crypto_ahash_update(&rctx->areq); > > - if (err) { > > - req->base.complete = rctx->complete; > > - goto out; > > - } > > - > > - return; > > -out: > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > -} > > - > > -static int mcryptd_hash_update_enqueue(struct ahash_request *req) > > -{ > > - return mcryptd_hash_enqueue(req, mcryptd_hash_update); > > -} > > - > > -static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) > > -{ > > - struct ahash_request *req = ahash_request_cast(req_async); > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - > > - if (unlikely(err == -EINPROGRESS)) > > - goto out; > > - > > - rctx->out = req->result; > > - err = crypto_ahash_final(&rctx->areq); > > - if (err) { > > - req->base.complete = rctx->complete; > > - goto out; > > - } > > - > > - return; > > -out: > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > -} > > - > > -static int mcryptd_hash_final_enqueue(struct ahash_request *req) > > -{ > > - return mcryptd_hash_enqueue(req, mcryptd_hash_final); > > -} > > - > > -static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) > > -{ > > - struct ahash_request *req = ahash_request_cast(req_async); > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - > > - if (unlikely(err == -EINPROGRESS)) > > - goto out; > > - rctx->out = req->result; > > - err = crypto_ahash_finup(&rctx->areq); > > - > > - if (err) { > > - req->base.complete = rctx->complete; > > - goto out; > > - } > > - > > - return; > > -out: > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > -} > > - > > -static int mcryptd_hash_finup_enqueue(struct ahash_request *req) > > -{ > > - return mcryptd_hash_enqueue(req, mcryptd_hash_finup); > > -} > > - > > -static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err) > > -{ > > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); > > - struct crypto_ahash *child = ctx->child; > > - struct ahash_request *req = ahash_request_cast(req_async); > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - struct ahash_request *desc = &rctx->areq; > > - > > - if (unlikely(err == -EINPROGRESS)) > > - goto out; > > - > > - ahash_request_set_tfm(desc, child); > > - ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, > > - rctx->complete, req_async); > > - > > - rctx->out = req->result; > > - err = crypto_ahash_init(desc) ?: crypto_ahash_finup(desc); > > - > > -out: > > - local_bh_disable(); > > - rctx->complete(&req->base, err); > > - local_bh_enable(); > > -} > > - > > -static int mcryptd_hash_digest_enqueue(struct ahash_request *req) > > -{ > > - return mcryptd_hash_enqueue(req, mcryptd_hash_digest); > > -} > > - > > -static int mcryptd_hash_export(struct ahash_request *req, void *out) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - > > - return crypto_ahash_export(&rctx->areq, out); > > -} > > - > > -static int mcryptd_hash_import(struct ahash_request *req, const void *in) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - > > - return crypto_ahash_import(&rctx->areq, in); > > -} > > - > > -static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, > > - struct mcryptd_queue *queue) > > -{ > > - struct hashd_instance_ctx *ctx; > > - struct ahash_instance *inst; > > - struct hash_alg_common *halg; > > - struct crypto_alg *alg; > > - u32 type = 0; > > - u32 mask = 0; > > - int err; > > - > > - if (!mcryptd_check_internal(tb, &type, &mask)) > > - return -EINVAL; > > - > > - halg = ahash_attr_alg(tb[1], type, mask); > > - if (IS_ERR(halg)) > > - return PTR_ERR(halg); > > - > > - alg = &halg->base; > > - pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name); > > - inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(), > > - sizeof(*ctx)); > > - err = PTR_ERR(inst); > > - if (IS_ERR(inst)) > > - goto out_put_alg; > > - > > - ctx = ahash_instance_ctx(inst); > > - ctx->queue = queue; > > - > > - err = crypto_init_ahash_spawn(&ctx->spawn, halg, > > - ahash_crypto_instance(inst)); > > - if (err) > > - goto out_free_inst; > > - > > - inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC | > > - (alg->cra_flags & (CRYPTO_ALG_INTERNAL | > > - CRYPTO_ALG_OPTIONAL_KEY)); > > - > > - inst->alg.halg.digestsize = halg->digestsize; > > - inst->alg.halg.statesize = halg->statesize; > > - inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx); > > - > > - inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm; > > - inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm; > > - > > - inst->alg.init = mcryptd_hash_init_enqueue; > > - inst->alg.update = mcryptd_hash_update_enqueue; > > - inst->alg.final = mcryptd_hash_final_enqueue; > > - inst->alg.finup = mcryptd_hash_finup_enqueue; > > - inst->alg.export = mcryptd_hash_export; > > - inst->alg.import = mcryptd_hash_import; > > - if (crypto_hash_alg_has_setkey(halg)) > > - inst->alg.setkey = mcryptd_hash_setkey; > > - inst->alg.digest = mcryptd_hash_digest_enqueue; > > - > > - err = ahash_register_instance(tmpl, inst); > > - if (err) { > > - crypto_drop_ahash(&ctx->spawn); > > -out_free_inst: > > - kfree(inst); > > - } > > - > > -out_put_alg: > > - crypto_mod_put(alg); > > - return err; > > -} > > - > > -static struct mcryptd_queue mqueue; > > - > > -static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb) > > -{ > > - struct crypto_attr_type *algt; > > - > > - algt = crypto_get_attr_type(tb); > > - if (IS_ERR(algt)) > > - return PTR_ERR(algt); > > - > > - switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { > > - case CRYPTO_ALG_TYPE_DIGEST: > > - return mcryptd_create_hash(tmpl, tb, &mqueue); > > - break; > > - } > > - > > - return -EINVAL; > > -} > > - > > -static void mcryptd_free(struct crypto_instance *inst) > > -{ > > - struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst); > > - struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); > > - > > - switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { > > - case CRYPTO_ALG_TYPE_AHASH: > > - crypto_drop_ahash(&hctx->spawn); > > - kfree(ahash_instance(inst)); > > - return; > > - default: > > - crypto_drop_spawn(&ctx->spawn); > > - kfree(inst); > > - } > > -} > > - > > -static struct crypto_template mcryptd_tmpl = { > > - .name = "mcryptd", > > - .create = mcryptd_create, > > - .free = mcryptd_free, > > - .module = THIS_MODULE, > > -}; > > - > > -struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, > > - u32 type, u32 mask) > > -{ > > - char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME]; > > - struct crypto_ahash *tfm; > > - > > - if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME, > > - "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) > > - return ERR_PTR(-EINVAL); > > - tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask); > > - if (IS_ERR(tfm)) > > - return ERR_CAST(tfm); > > - if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { > > - crypto_free_ahash(tfm); > > - return ERR_PTR(-EINVAL); > > - } > > - > > - return __mcryptd_ahash_cast(tfm); > > -} > > -EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash); > > - > > -struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) > > -{ > > - struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); > > - > > - return ctx->child; > > -} > > -EXPORT_SYMBOL_GPL(mcryptd_ahash_child); > > - > > -struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req) > > -{ > > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > > - return &rctx->areq; > > -} > > -EXPORT_SYMBOL_GPL(mcryptd_ahash_desc); > > - > > -void mcryptd_free_ahash(struct mcryptd_ahash *tfm) > > -{ > > - crypto_free_ahash(&tfm->base); > > -} > > -EXPORT_SYMBOL_GPL(mcryptd_free_ahash); > > - > > -static int __init mcryptd_init(void) > > -{ > > - int err, cpu; > > - struct mcryptd_flush_list *flist; > > - > > - mcryptd_flist = alloc_percpu(struct mcryptd_flush_list); > > - for_each_possible_cpu(cpu) { > > - flist = per_cpu_ptr(mcryptd_flist, cpu); > > - INIT_LIST_HEAD(&flist->list); > > - mutex_init(&flist->lock); > > - } > > - > > - err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN); > > - if (err) { > > - free_percpu(mcryptd_flist); > > - return err; > > - } > > - > > - err = crypto_register_template(&mcryptd_tmpl); > > - if (err) { > > - mcryptd_fini_queue(&mqueue); > > - free_percpu(mcryptd_flist); > > - } > > - > > - return err; > > -} > > - > > -static void __exit mcryptd_exit(void) > > -{ > > - mcryptd_fini_queue(&mqueue); > > - crypto_unregister_template(&mcryptd_tmpl); > > - free_percpu(mcryptd_flist); > > -} > > - > > -subsys_initcall(mcryptd_init); > > -module_exit(mcryptd_exit); > > - > > -MODULE_LICENSE("GPL"); > > -MODULE_DESCRIPTION("Software async multibuffer crypto daemon"); > > -MODULE_ALIAS_CRYPTO("mcryptd"); > > diff --git a/include/crypto/mcryptd.h b/include/crypto/mcryptd.h > > deleted file mode 100644 > > index b67404fc4b34..000000000000 > > --- a/include/crypto/mcryptd.h > > +++ /dev/null > > @@ -1,114 +0,0 @@ > > -/* SPDX-License-Identifier: GPL-2.0 */ > > -/* > > - * Software async multibuffer crypto daemon headers > > - * > > - * Author: > > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > > - * > > - * Copyright (c) 2014, Intel Corporation. > > - */ > > - > > -#ifndef _CRYPTO_MCRYPT_H > > -#define _CRYPTO_MCRYPT_H > > - > > -#include <linux/crypto.h> > > -#include <linux/kernel.h> > > -#include <crypto/hash.h> > > - > > -struct mcryptd_ahash { > > - struct crypto_ahash base; > > -}; > > - > > -static inline struct mcryptd_ahash *__mcryptd_ahash_cast( > > - struct crypto_ahash *tfm) > > -{ > > - return (struct mcryptd_ahash *)tfm; > > -} > > - > > -struct mcryptd_cpu_queue { > > - struct crypto_queue queue; > > - spinlock_t q_lock; > > - struct work_struct work; > > -}; > > - > > -struct mcryptd_queue { > > - struct mcryptd_cpu_queue __percpu *cpu_queue; > > -}; > > - > > -struct mcryptd_instance_ctx { > > - struct crypto_spawn spawn; > > - struct mcryptd_queue *queue; > > -}; > > - > > -struct mcryptd_hash_ctx { > > - struct crypto_ahash *child; > > - struct mcryptd_alg_state *alg_state; > > -}; > > - > > -struct mcryptd_tag { > > - /* seq number of request */ > > - unsigned seq_num; > > - /* arrival time of request */ > > - unsigned long arrival; > > - unsigned long expire; > > - int cpu; > > -}; > > - > > -struct mcryptd_hash_request_ctx { > > - struct list_head waiter; > > - crypto_completion_t complete; > > - struct mcryptd_tag tag; > > - struct crypto_hash_walk walk; > > - u8 *out; > > - int flag; > > - struct ahash_request areq; > > -}; > > - > > -struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, > > - u32 type, u32 mask); > > -struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm); > > -struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req); > > -void mcryptd_free_ahash(struct mcryptd_ahash *tfm); > > -void mcryptd_flusher(struct work_struct *work); > > - > > -enum mcryptd_req_type { > > - MCRYPTD_NONE, > > - MCRYPTD_UPDATE, > > - MCRYPTD_FINUP, > > - MCRYPTD_DIGEST, > > - MCRYPTD_FINAL > > -}; > > - > > -struct mcryptd_alg_cstate { > > - unsigned long next_flush; > > - unsigned next_seq_num; > > - bool flusher_engaged; > > - struct delayed_work flush; > > - int cpu; > > - struct mcryptd_alg_state *alg_state; > > - void *mgr; > > - spinlock_t work_lock; > > - struct list_head work_list; > > - struct list_head flush_list; > > -}; > > - > > -struct mcryptd_alg_state { > > - struct mcryptd_alg_cstate __percpu *alg_cstate; > > - unsigned long (*flusher)(struct mcryptd_alg_cstate *cstate); > > -}; > > - > > -/* return delay in jiffies from current time */ > > -static inline unsigned long get_delay(unsigned long t) > > -{ > > - long delay; > > - > > - delay = (long) t - (long) jiffies; > > - if (delay <= 0) > > - return 0; > > - else > > - return (unsigned long) delay; > > -} > > - > > -void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay); > > - > > -#endif > > >
