On 08/22/2018 01:51 AM, Ard Biesheuvel wrote: > As it turns out, the AVX2 multibuffer SHA routines are currently > broken [0], in a way that would have likely been noticed if this > code were in wide use. Since the code is too complicated to be > maintained by anyone except the original authors, and since the > performance benefits for real-world use cases are debatable to > begin with, it is better to drop it entirely for the moment. > > [0] https://marc.info/?l=linux-crypto-vger&m=153476243825350&w=2 Sorry I was out of the loop for a while and haven't been following the code too closely. Megha is maintaining the code now. Before we pull the code, please give us a chance to fix it first. Thanks. Tim > > Suggested-by: Eric Biggers <ebiggers@xxxxxxxxxx> > Cc: Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > Cc: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > Cc: Geert Uytterhoeven <geert@xxxxxxxxxxxxxx> > Cc: Martin Schwidefsky <schwidefsky@xxxxxxxxxx> > Cc: Heiko Carstens <heiko.carstens@xxxxxxxxxx> > Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx> > Cc: Ingo Molnar <mingo@xxxxxxxxxx> > Signed-off-by: Ard Biesheuvel <ard.biesheuvel@xxxxxxxxxx> > --- > MAINTAINERS | 8 - > arch/m68k/configs/amiga_defconfig | 1 - > arch/m68k/configs/apollo_defconfig | 1 - > arch/m68k/configs/atari_defconfig | 1 - > arch/m68k/configs/bvme6000_defconfig | 1 - > arch/m68k/configs/hp300_defconfig | 1 - > arch/m68k/configs/mac_defconfig | 1 - > arch/m68k/configs/multi_defconfig | 1 - > arch/m68k/configs/mvme147_defconfig | 1 - > arch/m68k/configs/mvme16x_defconfig | 1 - > arch/m68k/configs/q40_defconfig | 1 - > arch/m68k/configs/sun3_defconfig | 1 - > arch/m68k/configs/sun3x_defconfig | 1 - > arch/s390/configs/debug_defconfig | 1 - > arch/s390/configs/performance_defconfig | 1 - > arch/x86/crypto/Makefile | 3 - > arch/x86/crypto/sha1-mb/Makefile | 14 - > arch/x86/crypto/sha1-mb/sha1_mb.c | 1011 ---------------- > arch/x86/crypto/sha1-mb/sha1_mb_ctx.h | 134 --- > arch/x86/crypto/sha1-mb/sha1_mb_mgr.h | 110 -- > .../crypto/sha1-mb/sha1_mb_mgr_datastruct.S | 287 ----- > .../crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S | 304 ----- > .../crypto/sha1-mb/sha1_mb_mgr_init_avx2.c | 64 - > .../crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S | 209 ---- > arch/x86/crypto/sha1-mb/sha1_x8_avx2.S | 492 -------- > arch/x86/crypto/sha256-mb/Makefile | 14 - > arch/x86/crypto/sha256-mb/sha256_mb.c | 1013 ---------------- > arch/x86/crypto/sha256-mb/sha256_mb_ctx.h | 134 --- > arch/x86/crypto/sha256-mb/sha256_mb_mgr.h | 108 -- > .../sha256-mb/sha256_mb_mgr_datastruct.S | 304 ----- > .../sha256-mb/sha256_mb_mgr_flush_avx2.S | 307 ----- > .../sha256-mb/sha256_mb_mgr_init_avx2.c | 65 - > .../sha256-mb/sha256_mb_mgr_submit_avx2.S | 214 ---- > arch/x86/crypto/sha256-mb/sha256_x8_avx2.S | 598 ---------- > arch/x86/crypto/sha512-mb/Makefile | 12 - > arch/x86/crypto/sha512-mb/sha512_mb.c | 1047 ----------------- > arch/x86/crypto/sha512-mb/sha512_mb_ctx.h | 128 -- > arch/x86/crypto/sha512-mb/sha512_mb_mgr.h | 104 -- > .../sha512-mb/sha512_mb_mgr_datastruct.S | 281 ----- > .../sha512-mb/sha512_mb_mgr_flush_avx2.S | 297 ----- > .../sha512-mb/sha512_mb_mgr_init_avx2.c | 69 -- > .../sha512-mb/sha512_mb_mgr_submit_avx2.S | 224 ---- > arch/x86/crypto/sha512-mb/sha512_x4_avx2.S | 531 --------- > crypto/Kconfig | 62 - > crypto/Makefile | 1 - > crypto/mcryptd.c | 675 ----------- > include/crypto/mcryptd.h | 114 -- > 47 files changed, 8952 deletions(-) > delete mode 100644 arch/x86/crypto/sha1-mb/Makefile > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb.c > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > delete mode 100644 arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > delete mode 100644 arch/x86/crypto/sha256-mb/Makefile > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb.c > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > delete mode 100644 arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > delete mode 100644 arch/x86/crypto/sha512-mb/Makefile > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb.c > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > delete mode 100644 arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > delete mode 100644 crypto/mcryptd.c > delete mode 100644 include/crypto/mcryptd.h > > diff --git a/MAINTAINERS b/MAINTAINERS > index 24b200d91b30..05747b8ac88e 100644 > --- a/MAINTAINERS > +++ b/MAINTAINERS > @@ -7487,14 +7487,6 @@ S: Supported > F: drivers/infiniband/hw/i40iw/ > F: include/uapi/rdma/i40iw-abi.h > > -INTEL SHA MULTIBUFFER DRIVER > -M: Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > -R: Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > -L: linux-crypto@xxxxxxxxxxxxxxx > -S: Supported > -F: arch/x86/crypto/sha*-mb/ > -F: crypto/mcryptd.c > - > INTEL TELEMETRY DRIVER > M: Souvik Kumar Chakravarty <souvik.k.chakravarty@xxxxxxxxx> > L: platform-driver-x86@xxxxxxxxxxxxxxx > diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig > index 1d5483f6e457..70b10d712624 100644 > --- a/arch/m68k/configs/amiga_defconfig > +++ b/arch/m68k/configs/amiga_defconfig > @@ -621,7 +621,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig > index 52a0af127951..211eec5859e8 100644 > --- a/arch/m68k/configs/apollo_defconfig > +++ b/arch/m68k/configs/apollo_defconfig > @@ -578,7 +578,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig > index b3103e51268a..0da45c6084f7 100644 > --- a/arch/m68k/configs/atari_defconfig > +++ b/arch/m68k/configs/atari_defconfig > @@ -599,7 +599,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig > index fb7d651a4cab..c09ae7219416 100644 > --- a/arch/m68k/configs/bvme6000_defconfig > +++ b/arch/m68k/configs/bvme6000_defconfig > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig > index 6b37f5537c39..8c4775b30748 100644 > --- a/arch/m68k/configs/hp300_defconfig > +++ b/arch/m68k/configs/hp300_defconfig > @@ -580,7 +580,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig > index c717bf879449..48ad520e2f2d 100644 > --- a/arch/m68k/configs/mac_defconfig > +++ b/arch/m68k/configs/mac_defconfig > @@ -602,7 +602,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig > index 226c994ce794..3a3cccb9f625 100644 > --- a/arch/m68k/configs/multi_defconfig > +++ b/arch/m68k/configs/multi_defconfig > @@ -684,7 +684,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig > index b383327fd77a..63dc311f94ff 100644 > --- a/arch/m68k/configs/mvme147_defconfig > +++ b/arch/m68k/configs/mvme147_defconfig > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig > index 9783d3deb9e9..1ae39d1f9bb5 100644 > --- a/arch/m68k/configs/mvme16x_defconfig > +++ b/arch/m68k/configs/mvme16x_defconfig > @@ -570,7 +570,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig > index a35d10ee10cb..ba2f351811da 100644 > --- a/arch/m68k/configs/q40_defconfig > +++ b/arch/m68k/configs/q40_defconfig > @@ -593,7 +593,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig > index 573bf922d448..544b7475ff6a 100644 > --- a/arch/m68k/configs/sun3_defconfig > +++ b/arch/m68k/configs/sun3_defconfig > @@ -571,7 +571,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig > index efb27a7fcc55..149edafbb9f9 100644 > --- a/arch/m68k/configs/sun3x_defconfig > +++ b/arch/m68k/configs/sun3x_defconfig > @@ -572,7 +572,6 @@ CONFIG_CRYPTO_ECDH=m > CONFIG_CRYPTO_MANAGER=y > CONFIG_CRYPTO_USER=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_AEGIS128=m > diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig > index 941d8cc6c9f5..259d1698ac50 100644 > --- a/arch/s390/configs/debug_defconfig > +++ b/arch/s390/configs/debug_defconfig > @@ -668,7 +668,6 @@ CONFIG_CRYPTO_USER=m > # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set > CONFIG_CRYPTO_PCRYPT=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_LRW=m > diff --git a/arch/s390/configs/performance_defconfig b/arch/s390/configs/performance_defconfig > index eb6f75f24208..37fd60c20e22 100644 > --- a/arch/s390/configs/performance_defconfig > +++ b/arch/s390/configs/performance_defconfig > @@ -610,7 +610,6 @@ CONFIG_CRYPTO_USER=m > # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set > CONFIG_CRYPTO_PCRYPT=m > CONFIG_CRYPTO_CRYPTD=m > -CONFIG_CRYPTO_MCRYPTD=m > CONFIG_CRYPTO_TEST=m > CONFIG_CRYPTO_CHACHA20POLY1305=m > CONFIG_CRYPTO_LRW=m > diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile > index a450ad573dcb..9edfa5469f9f 100644 > --- a/arch/x86/crypto/Makefile > +++ b/arch/x86/crypto/Makefile > @@ -60,9 +60,6 @@ endif > ifeq ($(avx2_supported),yes) > obj-$(CONFIG_CRYPTO_CAMELLIA_AESNI_AVX2_X86_64) += camellia-aesni-avx2.o > obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o > - obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb/ > - obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb/ > - obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb/ > > obj-$(CONFIG_CRYPTO_MORUS1280_AVX2) += morus1280-avx2.o > endif > diff --git a/arch/x86/crypto/sha1-mb/Makefile b/arch/x86/crypto/sha1-mb/Makefile > deleted file mode 100644 > index 815ded3ba90e..000000000000 > --- a/arch/x86/crypto/sha1-mb/Makefile > +++ /dev/null > @@ -1,14 +0,0 @@ > -# SPDX-License-Identifier: GPL-2.0 > -# > -# Arch-specific CryptoAPI modules. > -# > - > -OBJECT_FILES_NON_STANDARD := y > - > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > - $(comma)4)$(comma)%ymm2,yes,no) > -ifeq ($(avx2_supported),yes) > - obj-$(CONFIG_CRYPTO_SHA1_MB) += sha1-mb.o > - sha1-mb-y := sha1_mb.o sha1_mb_mgr_flush_avx2.o \ > - sha1_mb_mgr_init_avx2.o sha1_mb_mgr_submit_avx2.o sha1_x8_avx2.o > -endif > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb.c b/arch/x86/crypto/sha1-mb/sha1_mb.c > deleted file mode 100644 > index b93805664c1d..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb.c > +++ /dev/null > @@ -1,1011 +0,0 @@ > -/* > - * Multi buffer SHA1 algorithm Glue Code > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > - > -#include <crypto/internal/hash.h> > -#include <linux/init.h> > -#include <linux/module.h> > -#include <linux/mm.h> > -#include <linux/cryptohash.h> > -#include <linux/types.h> > -#include <linux/list.h> > -#include <crypto/scatterwalk.h> > -#include <crypto/sha.h> > -#include <crypto/mcryptd.h> > -#include <crypto/crypto_wq.h> > -#include <asm/byteorder.h> > -#include <linux/hardirq.h> > -#include <asm/fpu/api.h> > -#include "sha1_mb_ctx.h" > - > -#define FLUSH_INTERVAL 1000 /* in usec */ > - > -static struct mcryptd_alg_state sha1_mb_alg_state; > - > -struct sha1_mb_ctx { > - struct mcryptd_ahash *mcryptd_tfm; > -}; > - > -static inline struct mcryptd_hash_request_ctx > - *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx) > -{ > - struct ahash_request *areq; > - > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > -} > - > -static inline struct ahash_request > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > -{ > - return container_of((void *) ctx, struct ahash_request, __ctx); > -} > - > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > - struct ahash_request *areq) > -{ > - rctx->flag = HASH_UPDATE; > -} > - > -static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state); > -static asmlinkage struct job_sha1* (*sha1_job_mgr_submit) > - (struct sha1_mb_mgr *state, struct job_sha1 *job); > -static asmlinkage struct job_sha1* (*sha1_job_mgr_flush) > - (struct sha1_mb_mgr *state); > -static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job) > - (struct sha1_mb_mgr *state); > - > -static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2], > - uint64_t total_len) > -{ > - uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1); > - > - memset(&padblock[i], 0, SHA1_BLOCK_SIZE); > - padblock[i] = 0x80; > - > - i += ((SHA1_BLOCK_SIZE - 1) & > - (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1))) > - + 1 + SHA1_PADLENGTHFIELD_SIZE; > - > -#if SHA1_PADLENGTHFIELD_SIZE == 16 > - *((uint64_t *) &padblock[i - 16]) = 0; > -#endif > - > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > - > - /* Number of extra blocks to hash */ > - return i >> SHA1_LOG2_BLOCK_SIZE; > -} > - > -static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr, > - struct sha1_hash_ctx *ctx) > -{ > - while (ctx) { > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Clear PROCESSING bit */ > - ctx->status = HASH_CTX_STS_COMPLETE; > - return ctx; > - } > - > - /* > - * If the extra blocks are empty, begin hashing what remains > - * in the user's buffer. > - */ > - if (ctx->partial_block_buffer_length == 0 && > - ctx->incoming_buffer_length) { > - > - const void *buffer = ctx->incoming_buffer; > - uint32_t len = ctx->incoming_buffer_length; > - uint32_t copy_len; > - > - /* > - * Only entire blocks can be hashed. > - * Copy remainder to extra blocks buffer. > - */ > - copy_len = len & (SHA1_BLOCK_SIZE-1); > - > - if (copy_len) { > - len -= copy_len; > - memcpy(ctx->partial_block_buffer, > - ((const char *) buffer + len), > - copy_len); > - ctx->partial_block_buffer_length = copy_len; > - } > - > - ctx->incoming_buffer_length = 0; > - > - /* len should be a multiple of the block size now */ > - assert((len % SHA1_BLOCK_SIZE) == 0); > - > - /* Set len to the number of blocks to be hashed */ > - len >>= SHA1_LOG2_BLOCK_SIZE; > - > - if (len) { > - > - ctx->job.buffer = (uint8_t *) buffer; > - ctx->job.len = len; > - ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr, > - &ctx->job); > - continue; > - } > - } > - > - /* > - * If the extra blocks are not empty, then we are > - * either on the last block(s) or we need more > - * user input before continuing. > - */ > - if (ctx->status & HASH_CTX_STS_LAST) { > - > - uint8_t *buf = ctx->partial_block_buffer; > - uint32_t n_extra_blocks = > - sha1_pad(buf, ctx->total_length); > - > - ctx->status = (HASH_CTX_STS_PROCESSING | > - HASH_CTX_STS_COMPLETE); > - ctx->job.buffer = buf; > - ctx->job.len = (uint32_t) n_extra_blocks; > - ctx = (struct sha1_hash_ctx *) > - sha1_job_mgr_submit(&mgr->mgr, &ctx->job); > - continue; > - } > - > - ctx->status = HASH_CTX_STS_IDLE; > - return ctx; > - } > - > - return NULL; > -} > - > -static struct sha1_hash_ctx > - *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr) > -{ > - /* > - * If get_comp_job returns NULL, there are no jobs complete. > - * If get_comp_job returns a job, verify that it is safe to return to > - * the user. > - * If it is not ready, resubmit the job to finish processing. > - * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned. > - * Otherwise, all jobs currently being managed by the hash_ctx_mgr > - * still need processing. > - */ > - struct sha1_hash_ctx *ctx; > - > - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr); > - return sha1_ctx_mgr_resubmit(mgr, ctx); > -} > - > -static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr) > -{ > - sha1_job_mgr_init(&mgr->mgr); > -} > - > -static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr, > - struct sha1_hash_ctx *ctx, > - const void *buffer, > - uint32_t len, > - int flags) > -{ > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > - /* User should not pass anything other than UPDATE or LAST */ > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > - return ctx; > - } > - > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > - /* Cannot submit to a currently processing job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > - return ctx; > - } > - > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Cannot update a finished job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > - return ctx; > - } > - > - /* > - * If we made it here, there were no errors during this call to > - * submit > - */ > - ctx->error = HASH_CTX_ERROR_NONE; > - > - /* Store buffer ptr info from user */ > - ctx->incoming_buffer = buffer; > - ctx->incoming_buffer_length = len; > - > - /* > - * Store the user's request flags and mark this ctx as currently > - * being processed. > - */ > - ctx->status = (flags & HASH_LAST) ? > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > - HASH_CTX_STS_PROCESSING; > - > - /* Advance byte counter */ > - ctx->total_length += len; > - > - /* > - * If there is anything currently buffered in the extra blocks, > - * append to it until it contains a whole block. > - * Or if the user's buffer contains less than a whole block, > - * append as much as possible to the extra block. > - */ > - if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) { > - /* > - * Compute how many bytes to copy from user buffer into > - * extra block > - */ > - uint32_t copy_len = SHA1_BLOCK_SIZE - > - ctx->partial_block_buffer_length; > - if (len < copy_len) > - copy_len = len; > - > - if (copy_len) { > - /* Copy and update relevant pointers and counters */ > - memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length], > - buffer, copy_len); > - > - ctx->partial_block_buffer_length += copy_len; > - ctx->incoming_buffer = (const void *) > - ((const char *)buffer + copy_len); > - ctx->incoming_buffer_length = len - copy_len; > - } > - > - /* > - * The extra block should never contain more than 1 block > - * here > - */ > - assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE); > - > - /* > - * If the extra block buffer contains exactly 1 block, it can > - * be hashed. > - */ > - if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) { > - ctx->partial_block_buffer_length = 0; > - > - ctx->job.buffer = ctx->partial_block_buffer; > - ctx->job.len = 1; > - ctx = (struct sha1_hash_ctx *) > - sha1_job_mgr_submit(&mgr->mgr, &ctx->job); > - } > - } > - > - return sha1_ctx_mgr_resubmit(mgr, ctx); > -} > - > -static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr) > -{ > - struct sha1_hash_ctx *ctx; > - > - while (1) { > - ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr); > - > - /* If flush returned 0, there are no more jobs in flight. */ > - if (!ctx) > - return NULL; > - > - /* > - * If flush returned a job, resubmit the job to finish > - * processing. > - */ > - ctx = sha1_ctx_mgr_resubmit(mgr, ctx); > - > - /* > - * If sha1_ctx_mgr_resubmit returned a job, it is ready to be > - * returned. Otherwise, all jobs currently being managed by the > - * sha1_ctx_mgr still need processing. Loop. > - */ > - if (ctx) > - return ctx; > - } > -} > - > -static int sha1_mb_init(struct ahash_request *areq) > -{ > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > - > - hash_ctx_init(sctx); > - sctx->job.result_digest[0] = SHA1_H0; > - sctx->job.result_digest[1] = SHA1_H1; > - sctx->job.result_digest[2] = SHA1_H2; > - sctx->job.result_digest[3] = SHA1_H3; > - sctx->job.result_digest[4] = SHA1_H4; > - sctx->total_length = 0; > - sctx->partial_block_buffer_length = 0; > - sctx->status = HASH_CTX_STS_IDLE; > - > - return 0; > -} > - > -static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > -{ > - int i; > - struct sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > - __be32 *dst = (__be32 *) rctx->out; > - > - for (i = 0; i < 5; ++i) > - dst[i] = cpu_to_be32(sctx->job.result_digest[i]); > - > - return 0; > -} > - > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > - struct mcryptd_alg_cstate *cstate, bool flush) > -{ > - int flag = HASH_UPDATE; > - int nbytes, err = 0; > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > - struct sha1_hash_ctx *sha_ctx; > - > - /* more work ? */ > - while (!(rctx->flag & HASH_DONE)) { > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > - if (nbytes < 0) { > - err = nbytes; > - goto out; > - } > - /* check if the walk is done */ > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - if (rctx->flag & HASH_FINAL) > - flag |= HASH_LAST; > - > - } > - sha_ctx = (struct sha1_hash_ctx *) > - ahash_request_ctx(&rctx->areq); > - kernel_fpu_begin(); > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, > - rctx->walk.data, nbytes, flag); > - if (!sha_ctx) { > - if (flush) > - sha_ctx = sha1_ctx_mgr_flush(cstate->mgr); > - } > - kernel_fpu_end(); > - if (sha_ctx) > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - else { > - rctx = NULL; > - goto out; > - } > - } > - > - /* copy the results */ > - if (rctx->flag & HASH_FINAL) > - sha1_mb_set_results(rctx); > - > -out: > - *ret_rctx = rctx; > - return err; > -} > - > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate, > - int err) > -{ > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha1_hash_ctx *sha_ctx; > - struct mcryptd_hash_request_ctx *req_ctx; > - int ret; > - > - /* remove from work list */ > - spin_lock(&cstate->work_lock); > - list_del(&rctx->waiter); > - spin_unlock(&cstate->work_lock); > - > - if (irqs_disabled()) > - rctx->complete(&req->base, err); > - else { > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > - } > - > - /* check to see if there are other jobs that are done */ > - sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); > - while (sha_ctx) { > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&req_ctx, cstate, false); > - if (req_ctx) { > - spin_lock(&cstate->work_lock); > - list_del(&req_ctx->waiter); > - spin_unlock(&cstate->work_lock); > - > - req = cast_mcryptd_ctx_to_req(req_ctx); > - if (irqs_disabled()) > - req_ctx->complete(&req->base, ret); > - else { > - local_bh_disable(); > - req_ctx->complete(&req->base, ret); > - local_bh_enable(); > - } > - } > - sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr); > - } > - > - return 0; > -} > - > -static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate) > -{ > - unsigned long next_flush; > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > - > - /* initialize tag */ > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > - rctx->tag.seq_num = cstate->next_seq_num++; > - next_flush = rctx->tag.arrival + delay; > - rctx->tag.expire = next_flush; > - > - spin_lock(&cstate->work_lock); > - list_add_tail(&rctx->waiter, &cstate->work_list); > - spin_unlock(&cstate->work_lock); > - > - mcryptd_arm_flusher(cstate, delay); > -} > - > -static int sha1_mb_update(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha1_hash_ctx *sha_ctx; > - int ret = 0, nbytes; > - > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) > - rctx->flag |= HASH_DONE; > - > - /* submit */ > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > - sha1_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > - nbytes, HASH_UPDATE); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha1_mb_finup(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha1_hash_ctx *sha_ctx; > - int ret = 0, flag = HASH_UPDATE, nbytes; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - flag = HASH_LAST; > - } > - > - /* submit */ > - rctx->flag |= HASH_FINAL; > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > - sha1_mb_add_list(rctx, cstate); > - > - kernel_fpu_begin(); > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > - nbytes, flag); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha1_mb_final(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha1_mb_alg_state.alg_cstate); > - > - struct sha1_hash_ctx *sha_ctx; > - int ret = 0; > - u8 data; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - rctx->flag |= HASH_DONE | HASH_FINAL; > - > - sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq); > - /* flag HASH_FINAL and 0 data size */ > - sha1_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, > - HASH_LAST); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha1_mb_export(struct ahash_request *areq, void *out) > -{ > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(out, sctx, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha1_mb_import(struct ahash_request *areq, const void *in) > -{ > - struct sha1_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(sctx, in, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm) > -{ > - struct mcryptd_ahash *mcryptd_tfm; > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - struct mcryptd_hash_ctx *mctx; > - > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb", > - CRYPTO_ALG_INTERNAL, > - CRYPTO_ALG_INTERNAL); > - if (IS_ERR(mcryptd_tfm)) > - return PTR_ERR(mcryptd_tfm); > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > - mctx->alg_state = &sha1_mb_alg_state; > - ctx->mcryptd_tfm = mcryptd_tfm; > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > - > - return 0; > -} > - > -static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm) > -{ > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - sizeof(struct sha1_hash_ctx)); > - > - return 0; > -} > - > -static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static struct ahash_alg sha1_mb_areq_alg = { > - .init = sha1_mb_init, > - .update = sha1_mb_update, > - .final = sha1_mb_final, > - .finup = sha1_mb_finup, > - .export = sha1_mb_export, > - .import = sha1_mb_import, > - .halg = { > - .digestsize = SHA1_DIGEST_SIZE, > - .statesize = sizeof(struct sha1_hash_ctx), > - .base = { > - .cra_name = "__sha1-mb", > - .cra_driver_name = "__intel_sha1-mb", > - .cra_priority = 100, > - /* > - * use ASYNC flag as some buffers in multi-buffer > - * algo may not have completed before hashing thread > - * sleep > - */ > - .cra_flags = CRYPTO_ALG_ASYNC | > - CRYPTO_ALG_INTERNAL, > - .cra_blocksize = SHA1_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT > - (sha1_mb_areq_alg.halg.base.cra_list), > - .cra_init = sha1_mb_areq_init_tfm, > - .cra_exit = sha1_mb_areq_exit_tfm, > - .cra_ctxsize = sizeof(struct sha1_hash_ctx), > - } > - } > -}; > - > -static int sha1_mb_async_init(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_init(mcryptd_req); > -} > - > -static int sha1_mb_async_update(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_update(mcryptd_req); > -} > - > -static int sha1_mb_async_finup(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_finup(mcryptd_req); > -} > - > -static int sha1_mb_async_final(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_final(mcryptd_req); > -} > - > -static int sha1_mb_async_digest(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_digest(mcryptd_req); > -} > - > -static int sha1_mb_async_export(struct ahash_request *req, void *out) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_export(mcryptd_req, out); > -} > - > -static int sha1_mb_async_import(struct ahash_request *req, const void *in) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > - struct mcryptd_hash_request_ctx *rctx; > - struct ahash_request *areq; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - rctx = ahash_request_ctx(mcryptd_req); > - areq = &rctx->areq; > - > - ahash_request_set_tfm(areq, child); > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > - rctx->complete, req); > - > - return crypto_ahash_import(mcryptd_req, in); > -} > - > -static struct ahash_alg sha1_mb_async_alg = { > - .init = sha1_mb_async_init, > - .update = sha1_mb_async_update, > - .final = sha1_mb_async_final, > - .finup = sha1_mb_async_finup, > - .digest = sha1_mb_async_digest, > - .export = sha1_mb_async_export, > - .import = sha1_mb_async_import, > - .halg = { > - .digestsize = SHA1_DIGEST_SIZE, > - .statesize = sizeof(struct sha1_hash_ctx), > - .base = { > - .cra_name = "sha1", > - .cra_driver_name = "sha1_mb", > - /* > - * Low priority, since with few concurrent hash requests > - * this is extremely slow due to the flush delay. Users > - * whose workloads would benefit from this can request > - * it explicitly by driver name, or can increase its > - * priority at runtime using NETLINK_CRYPTO. > - */ > - .cra_priority = 50, > - .cra_flags = CRYPTO_ALG_ASYNC, > - .cra_blocksize = SHA1_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list), > - .cra_init = sha1_mb_async_init_tfm, > - .cra_exit = sha1_mb_async_exit_tfm, > - .cra_ctxsize = sizeof(struct sha1_mb_ctx), > - .cra_alignmask = 0, > - }, > - }, > -}; > - > -static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate) > -{ > - struct mcryptd_hash_request_ctx *rctx; > - unsigned long cur_time; > - unsigned long next_flush = 0; > - struct sha1_hash_ctx *sha_ctx; > - > - > - cur_time = jiffies; > - > - while (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - if (time_before(cur_time, rctx->tag.expire)) > - break; > - kernel_fpu_begin(); > - sha_ctx = (struct sha1_hash_ctx *) > - sha1_ctx_mgr_flush(cstate->mgr); > - kernel_fpu_end(); > - if (!sha_ctx) { > - pr_err("sha1_mb error: nothing got flushed for non-empty list\n"); > - break; > - } > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - sha_finish_walk(&rctx, cstate, true); > - sha_complete_job(rctx, cstate, 0); > - } > - > - if (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - /* get the hash context and then flush time */ > - next_flush = rctx->tag.expire; > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > - } > - return next_flush; > -} > - > -static int __init sha1_mb_mod_init(void) > -{ > - > - int cpu; > - int err; > - struct mcryptd_alg_cstate *cpu_state; > - > - /* check for dependent cpu features */ > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > - !boot_cpu_has(X86_FEATURE_BMI2)) > - return -ENODEV; > - > - /* initialize multibuffer structures */ > - sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate); > - > - sha1_job_mgr_init = sha1_mb_mgr_init_avx2; > - sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2; > - sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2; > - sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2; > - > - if (!sha1_mb_alg_state.alg_cstate) > - return -ENOMEM; > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > - cpu_state->next_flush = 0; > - cpu_state->next_seq_num = 0; > - cpu_state->flusher_engaged = false; > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > - cpu_state->cpu = cpu; > - cpu_state->alg_state = &sha1_mb_alg_state; > - cpu_state->mgr = kzalloc(sizeof(struct sha1_ctx_mgr), > - GFP_KERNEL); > - if (!cpu_state->mgr) > - goto err2; > - sha1_ctx_mgr_init(cpu_state->mgr); > - INIT_LIST_HEAD(&cpu_state->work_list); > - spin_lock_init(&cpu_state->work_lock); > - } > - sha1_mb_alg_state.flusher = &sha1_mb_flusher; > - > - err = crypto_register_ahash(&sha1_mb_areq_alg); > - if (err) > - goto err2; > - err = crypto_register_ahash(&sha1_mb_async_alg); > - if (err) > - goto err1; > - > - > - return 0; > -err1: > - crypto_unregister_ahash(&sha1_mb_areq_alg); > -err2: > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha1_mb_alg_state.alg_cstate); > - return -ENODEV; > -} > - > -static void __exit sha1_mb_mod_fini(void) > -{ > - int cpu; > - struct mcryptd_alg_cstate *cpu_state; > - > - crypto_unregister_ahash(&sha1_mb_async_alg); > - crypto_unregister_ahash(&sha1_mb_areq_alg); > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha1_mb_alg_state.alg_cstate); > -} > - > -module_init(sha1_mb_mod_init); > -module_exit(sha1_mb_mod_fini); > - > -MODULE_LICENSE("GPL"); > -MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated"); > - > -MODULE_ALIAS_CRYPTO("sha1"); > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h b/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > deleted file mode 100644 > index 9454bd16f9f8..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_ctx.h > +++ /dev/null > @@ -1,134 +0,0 @@ > -/* > - * Header file for multi buffer SHA context > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#ifndef _SHA_MB_CTX_INTERNAL_H > -#define _SHA_MB_CTX_INTERNAL_H > - > -#include "sha1_mb_mgr.h" > - > -#define HASH_UPDATE 0x00 > -#define HASH_LAST 0x01 > -#define HASH_DONE 0x02 > -#define HASH_FINAL 0x04 > - > -#define HASH_CTX_STS_IDLE 0x00 > -#define HASH_CTX_STS_PROCESSING 0x01 > -#define HASH_CTX_STS_LAST 0x02 > -#define HASH_CTX_STS_COMPLETE 0x04 > - > -enum hash_ctx_error { > - HASH_CTX_ERROR_NONE = 0, > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > - > -#ifdef HASH_CTX_DEBUG > - HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4, > -#endif > -}; > - > - > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > -#define hash_ctx_status(ctx) ((ctx)->status) > -#define hash_ctx_error(ctx) ((ctx)->error) > -#define hash_ctx_init(ctx) \ > - do { \ > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > - } while (0) > - > - > -/* Hash Constants and Typedefs */ > -#define SHA1_DIGEST_LENGTH 5 > -#define SHA1_LOG2_BLOCK_SIZE 6 > - > -#define SHA1_PADLENGTHFIELD_SIZE 8 > - > -#ifdef SHA_MB_DEBUG > -#define assert(expr) \ > -do { \ > - if (unlikely(!(expr))) { \ > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > - #expr, __FILE__, __func__, __LINE__); \ > - } \ > -} while (0) > -#else > -#define assert(expr) do {} while (0) > -#endif > - > -struct sha1_ctx_mgr { > - struct sha1_mb_mgr mgr; > -}; > - > -/* typedef struct sha1_ctx_mgr sha1_ctx_mgr; */ > - > -struct sha1_hash_ctx { > - /* Must be at struct offset 0 */ > - struct job_sha1 job; > - /* status flag */ > - int status; > - /* error flag */ > - int error; > - > - uint64_t total_length; > - const void *incoming_buffer; > - uint32_t incoming_buffer_length; > - uint8_t partial_block_buffer[SHA1_BLOCK_SIZE * 2]; > - uint32_t partial_block_buffer_length; > - void *user_data; > -}; > - > -#endif > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h b/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > deleted file mode 100644 > index 08ad1a9acfd7..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr.h > +++ /dev/null > @@ -1,110 +0,0 @@ > -/* > - * Header file for multi buffer SHA1 algorithm manager > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * James Guilford <james.guilford@xxxxxxxxx> > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > -#ifndef __SHA_MB_MGR_H > -#define __SHA_MB_MGR_H > - > - > -#include <linux/types.h> > - > -#define NUM_SHA1_DIGEST_WORDS 5 > - > -enum job_sts { STS_UNKNOWN = 0, > - STS_BEING_PROCESSED = 1, > - STS_COMPLETED = 2, > - STS_INTERNAL_ERROR = 3, > - STS_ERROR = 4 > -}; > - > -struct job_sha1 { > - u8 *buffer; > - u32 len; > - u32 result_digest[NUM_SHA1_DIGEST_WORDS] __aligned(32); > - enum job_sts status; > - void *user_data; > -}; > - > -/* SHA1 out-of-order scheduler */ > - > -/* typedef uint32_t sha1_digest_array[5][8]; */ > - > -struct sha1_args_x8 { > - uint32_t digest[5][8]; > - uint8_t *data_ptr[8]; > -}; > - > -struct sha1_lane_data { > - struct job_sha1 *job_in_lane; > -}; > - > -struct sha1_mb_mgr { > - struct sha1_args_x8 args; > - > - uint32_t lens[8]; > - > - /* each byte is index (0...7) of unused lanes */ > - uint64_t unused_lanes; > - /* byte 4 is set to FF as a flag */ > - struct sha1_lane_data ldata[8]; > -}; > - > - > -#define SHA1_MB_MGR_NUM_LANES_AVX2 8 > - > -void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state); > -struct job_sha1 *sha1_mb_mgr_submit_avx2(struct sha1_mb_mgr *state, > - struct job_sha1 *job); > -struct job_sha1 *sha1_mb_mgr_flush_avx2(struct sha1_mb_mgr *state); > -struct job_sha1 *sha1_mb_mgr_get_comp_job_avx2(struct sha1_mb_mgr *state); > - > -#endif > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > deleted file mode 100644 > index 86688c6e7a25..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_datastruct.S > +++ /dev/null > @@ -1,287 +0,0 @@ > -/* > - * Header file for multi buffer SHA1 algorithm data structure > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * James Guilford <james.guilford@xxxxxxxxx> > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -# Macros for defining data structures > - > -# Usage example > - > -#START_FIELDS # JOB_AES > -### name size align > -#FIELD _plaintext, 8, 8 # pointer to plaintext > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > -#FIELD _IV, 16, 8 # IV > -#FIELD _keys, 8, 8 # pointer to keys > -#FIELD _len, 4, 4 # length in bytes > -#FIELD _status, 4, 4 # status enumeration > -#FIELD _user_data, 8, 8 # pointer to user data > -#UNION _union, size1, align1, \ > -# size2, align2, \ > -# size3, align3, \ > -# ... > -#END_FIELDS > -#%assign _JOB_AES_size _FIELD_OFFSET > -#%assign _JOB_AES_align _STRUCT_ALIGN > - > -######################################################################### > - > -# Alternate "struc-like" syntax: > -# STRUCT job_aes2 > -# RES_Q .plaintext, 1 > -# RES_Q .ciphertext, 1 > -# RES_DQ .IV, 1 > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > -# RES_U .union, size1, align1, \ > -# size2, align2, \ > -# ... > -# ENDSTRUCT > -# # Following only needed if nesting > -# %assign job_aes2_size _FIELD_OFFSET > -# %assign job_aes2_align _STRUCT_ALIGN > -# > -# RES_* macros take a name, a count and an optional alignment. > -# The count in in terms of the base size of the macro, and the > -# default alignment is the base size. > -# The macros are: > -# Macro Base size > -# RES_B 1 > -# RES_W 2 > -# RES_D 4 > -# RES_Q 8 > -# RES_DQ 16 > -# RES_Y 32 > -# RES_Z 64 > -# > -# RES_U defines a union. It's arguments are a name and two or more > -# pairs of "size, alignment" > -# > -# The two assigns are only needed if this structure is being nested > -# within another. Even if the assigns are not done, one can still use > -# STRUCT_NAME_size as the size of the structure. > -# > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > -# > -# The differences between this and using "struc" directly are that each > -# type is implicitly aligned to its natural length (although this can be > -# over-ridden with an explicit third parameter), and that the structure > -# is padded at the end to its overall alignment. > -# > - > -######################################################################### > - > -#ifndef _SHA1_MB_MGR_DATASTRUCT_ASM_ > -#define _SHA1_MB_MGR_DATASTRUCT_ASM_ > - > -## START_FIELDS > -.macro START_FIELDS > - _FIELD_OFFSET = 0 > - _STRUCT_ALIGN = 0 > -.endm > - > -## FIELD name size align > -.macro FIELD name size align > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > - \name = _FIELD_OFFSET > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > -.if (\align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = \align > -.endif > -.endm > - > -## END_FIELDS > -.macro END_FIELDS > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > -.endm > - > -######################################################################## > - > -.macro STRUCT p1 > -START_FIELDS > -.struc \p1 > -.endm > - > -.macro ENDSTRUCT > - tmp = _FIELD_OFFSET > - END_FIELDS > - tmp = (_FIELD_OFFSET - %%tmp) > -.if (tmp > 0) > - .lcomm tmp > -.endif > -.endstruc > -.endm > - > -## RES_int name size align > -.macro RES_int p1 p2 p3 > - name = \p1 > - size = \p2 > - align = .\p3 > - > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > -.align align > -.lcomm name size > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > -.if (align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = align > -.endif > -.endm > - > - > - > -# macro RES_B name, size [, align] > -.macro RES_B _name, _size, _align=1 > -RES_int _name _size _align > -.endm > - > -# macro RES_W name, size [, align] > -.macro RES_W _name, _size, _align=2 > -RES_int _name 2*(_size) _align > -.endm > - > -# macro RES_D name, size [, align] > -.macro RES_D _name, _size, _align=4 > -RES_int _name 4*(_size) _align > -.endm > - > -# macro RES_Q name, size [, align] > -.macro RES_Q _name, _size, _align=8 > -RES_int _name 8*(_size) _align > -.endm > - > -# macro RES_DQ name, size [, align] > -.macro RES_DQ _name, _size, _align=16 > -RES_int _name 16*(_size) _align > -.endm > - > -# macro RES_Y name, size [, align] > -.macro RES_Y _name, _size, _align=32 > -RES_int _name 32*(_size) _align > -.endm > - > -# macro RES_Z name, size [, align] > -.macro RES_Z _name, _size, _align=64 > -RES_int _name 64*(_size) _align > -.endm > - > - > -#endif > - > -######################################################################## > -#### Define constants > -######################################################################## > - > -######################################################################## > -#### Define SHA1 Out Of Order Data Structures > -######################################################################## > - > -START_FIELDS # LANE_DATA > -### name size align > -FIELD _job_in_lane, 8, 8 # pointer to job object > -END_FIELDS > - > -_LANE_DATA_size = _FIELD_OFFSET > -_LANE_DATA_align = _STRUCT_ALIGN > - > -######################################################################## > - > -START_FIELDS # SHA1_ARGS_X8 > -### name size align > -FIELD _digest, 4*5*8, 16 # transposed digest > -FIELD _data_ptr, 8*8, 8 # array of pointers to data > -END_FIELDS > - > -_SHA1_ARGS_X4_size = _FIELD_OFFSET > -_SHA1_ARGS_X4_align = _STRUCT_ALIGN > -_SHA1_ARGS_X8_size = _FIELD_OFFSET > -_SHA1_ARGS_X8_align = _STRUCT_ALIGN > - > -######################################################################## > - > -START_FIELDS # MB_MGR > -### name size align > -FIELD _args, _SHA1_ARGS_X4_size, _SHA1_ARGS_X4_align > -FIELD _lens, 4*8, 8 > -FIELD _unused_lanes, 8, 8 > -FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align > -END_FIELDS > - > -_MB_MGR_size = _FIELD_OFFSET > -_MB_MGR_align = _STRUCT_ALIGN > - > -_args_digest = _args + _digest > -_args_data_ptr = _args + _data_ptr > - > - > -######################################################################## > -#### Define constants > -######################################################################## > - > -#define STS_UNKNOWN 0 > -#define STS_BEING_PROCESSED 1 > -#define STS_COMPLETED 2 > - > -######################################################################## > -#### Define JOB_SHA1 structure > -######################################################################## > - > -START_FIELDS # JOB_SHA1 > - > -### name size align > -FIELD _buffer, 8, 8 # pointer to buffer > -FIELD _len, 4, 4 # length in bytes > -FIELD _result_digest, 5*4, 32 # Digest (output) > -FIELD _status, 4, 4 > -FIELD _user_data, 8, 8 > -END_FIELDS > - > -_JOB_SHA1_size = _FIELD_OFFSET > -_JOB_SHA1_align = _STRUCT_ALIGN > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > deleted file mode 100644 > index 7cfba738f104..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_flush_avx2.S > +++ /dev/null > @@ -1,304 +0,0 @@ > -/* > - * Flush routine for SHA1 multibuffer > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * James Guilford <james.guilford@xxxxxxxxx> > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha1_mb_mgr_datastruct.S" > - > - > -.extern sha1_x8_avx2 > - > -# LINUX register definitions > -#define arg1 %rdi > -#define arg2 %rsi > - > -# Common definitions > -#define state arg1 > -#define job arg2 > -#define len2 arg2 > - > -# idx must be a register not clobbered by sha1_x8_avx2 > -#define idx %r8 > -#define DWORD_idx %r8d > - > -#define unused_lanes %rbx > -#define lane_data %rbx > -#define tmp2 %rbx > -#define tmp2_w %ebx > - > -#define job_rax %rax > -#define tmp1 %rax > -#define size_offset %rax > -#define tmp %rax > -#define start_offset %rax > - > -#define tmp3 %arg1 > - > -#define extra_blocks %arg2 > -#define p %arg2 > - > -.macro LABEL prefix n > -\prefix\n\(): > -.endm > - > -.macro JNE_SKIP i > -jne skip_\i > -.endm > - > -.altmacro > -.macro SET_OFFSET _offset > -offset = \_offset > -.endm > -.noaltmacro > - > -# JOB* sha1_mb_mgr_flush_avx2(MB_MGR *state) > -# arg 1 : rcx : state > -ENTRY(sha1_mb_mgr_flush_avx2) > - FRAME_BEGIN > - push %rbx > - > - # If bit (32+3) is set, then all lanes are empty > - mov _unused_lanes(state), unused_lanes > - bt $32+3, unused_lanes > - jc return_null > - > - # find a lane with a non-null job > - xor idx, idx > - offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne one(%rip), idx > - offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne two(%rip), idx > - offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne three(%rip), idx > - offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne four(%rip), idx > - offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne five(%rip), idx > - offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne six(%rip), idx > - offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne seven(%rip), idx > - > - # copy idx to empty lanes > -copy_lane_data: > - offset = (_args + _data_ptr) > - mov offset(state,idx,8), tmp > - > - I = 0 > -.rep 8 > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > -.altmacro > - JNE_SKIP %I > - offset = (_args + _data_ptr + 8*I) > - mov tmp, offset(state) > - offset = (_lens + 4*I) > - movl $0xFFFFFFFF, offset(state) > -LABEL skip_ %I > - I = (I+1) > -.noaltmacro > -.endr > - > - # Find min length > - vmovdqu _lens+0*16(state), %xmm0 > - vmovdqu _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > - > - vmovd %xmm2, DWORD_idx > - mov idx, len2 > - and $0xF, idx > - shr $4, len2 > - jz len_is_0 > - > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > - vpshufd $0, %xmm2, %xmm2 > - > - vpsubd %xmm2, %xmm0, %xmm0 > - vpsubd %xmm2, %xmm1, %xmm1 > - > - vmovdqu %xmm0, _lens+0*16(state) > - vmovdqu %xmm1, _lens+1*16(state) > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha1_x8_avx2 > - # state and idx are intact > - > - > -len_is_0: > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $4, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens(state, idx, 4) > - > - vmovd _args_digest(state , idx, 4) , %xmm0 > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > - movl _args_digest+4*32(state, idx, 4), tmp2_w > - > - vmovdqu %xmm0, _result_digest(job_rax) > - offset = (_result_digest + 1*16) > - mov tmp2_w, offset(job_rax) > - > -return: > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > -ENDPROC(sha1_mb_mgr_flush_avx2) > - > - > -################################################################# > - > -.align 16 > -ENTRY(sha1_mb_mgr_get_comp_job_avx2) > - push %rbx > - > - ## if bit 32+3 is set, then all lanes are empty > - mov _unused_lanes(state), unused_lanes > - bt $(32+3), unused_lanes > - jc .return_null > - > - # Find min length > - vmovdqu _lens(state), %xmm0 > - vmovdqu _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > - > - vmovd %xmm2, DWORD_idx > - test $~0xF, idx > - jnz .return_null > - > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $4, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens(state, idx, 4) > - > - vmovd _args_digest(state, idx, 4), %xmm0 > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > - movl _args_digest+4*32(state, idx, 4), tmp2_w > - > - vmovdqu %xmm0, _result_digest(job_rax) > - movl tmp2_w, _result_digest+1*16(job_rax) > - > - pop %rbx > - > - ret > - > -.return_null: > - xor job_rax, job_rax > - pop %rbx > - ret > -ENDPROC(sha1_mb_mgr_get_comp_job_avx2) > - > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > -.align 16 > -clear_low_nibble: > -.octa 0x000000000000000000000000FFFFFFF0 > - > -.section .rodata.cst8, "aM", @progbits, 8 > -.align 8 > -one: > -.quad 1 > -two: > -.quad 2 > -three: > -.quad 3 > -four: > -.quad 4 > -five: > -.quad 5 > -six: > -.quad 6 > -seven: > -.quad 7 > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > deleted file mode 100644 > index d2add0d35f43..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_init_avx2.c > +++ /dev/null > @@ -1,64 +0,0 @@ > -/* > - * Initialization code for multi buffer SHA1 algorithm for AVX2 > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include "sha1_mb_mgr.h" > - > -void sha1_mb_mgr_init_avx2(struct sha1_mb_mgr *state) > -{ > - unsigned int j; > - state->unused_lanes = 0xF76543210ULL; > - for (j = 0; j < 8; j++) { > - state->lens[j] = 0xFFFFFFFF; > - state->ldata[j].job_in_lane = NULL; > - } > -} > diff --git a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > deleted file mode 100644 > index 7a93b1c0d69a..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_mb_mgr_submit_avx2.S > +++ /dev/null > @@ -1,209 +0,0 @@ > -/* > - * Buffer submit code for multi buffer SHA1 algorithm > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * James Guilford <james.guilford@xxxxxxxxx> > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha1_mb_mgr_datastruct.S" > - > - > -.extern sha1_x8_avx > - > -# LINUX register definitions > -arg1 = %rdi > -arg2 = %rsi > -size_offset = %rcx > -tmp2 = %rcx > -extra_blocks = %rdx > - > -# Common definitions > -#define state arg1 > -#define job %rsi > -#define len2 arg2 > -#define p2 arg2 > - > -# idx must be a register not clobberred by sha1_x8_avx2 > -idx = %r8 > -DWORD_idx = %r8d > -last_len = %r8 > - > -p = %r11 > -start_offset = %r11 > - > -unused_lanes = %rbx > -BYTE_unused_lanes = %bl > - > -job_rax = %rax > -len = %rax > -DWORD_len = %eax > - > -lane = %r12 > -tmp3 = %r12 > - > -tmp = %r9 > -DWORD_tmp = %r9d > - > -lane_data = %r10 > - > -# JOB* submit_mb_mgr_submit_avx2(MB_MGR *state, job_sha1 *job) > -# arg 1 : rcx : state > -# arg 2 : rdx : job > -ENTRY(sha1_mb_mgr_submit_avx2) > - FRAME_BEGIN > - push %rbx > - push %r12 > - > - mov _unused_lanes(state), unused_lanes > - mov unused_lanes, lane > - and $0xF, lane > - shr $4, unused_lanes > - imul $_LANE_DATA_size, lane, lane_data > - movl $STS_BEING_PROCESSED, _status(job) > - lea _ldata(state, lane_data), lane_data > - mov unused_lanes, _unused_lanes(state) > - movl _len(job), DWORD_len > - > - mov job, _job_in_lane(lane_data) > - shl $4, len > - or lane, len > - > - movl DWORD_len, _lens(state , lane, 4) > - > - # Load digest words from result_digest > - vmovdqu _result_digest(job), %xmm0 > - mov _result_digest+1*16(job), DWORD_tmp > - vmovd %xmm0, _args_digest(state, lane, 4) > - vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4) > - vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4) > - vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4) > - movl DWORD_tmp, _args_digest+4*32(state , lane, 4) > - > - mov _buffer(job), p > - mov p, _args_data_ptr(state, lane, 8) > - > - cmp $0xF, unused_lanes > - jne return_null > - > -start_loop: > - # Find min length > - vmovdqa _lens(state), %xmm0 > - vmovdqa _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min value in low dword > - > - vmovd %xmm2, DWORD_idx > - mov idx, len2 > - and $0xF, idx > - shr $4, len2 > - jz len_is_0 > - > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > - vpshufd $0, %xmm2, %xmm2 > - > - vpsubd %xmm2, %xmm0, %xmm0 > - vpsubd %xmm2, %xmm1, %xmm1 > - > - vmovdqa %xmm0, _lens + 0*16(state) > - vmovdqa %xmm1, _lens + 1*16(state) > - > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha1_x8_avx2 > - > - # state and idx are intact > - > -len_is_0: > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - mov _unused_lanes(state), unused_lanes > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - shl $4, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens(state, idx, 4) > - > - vmovd _args_digest(state, idx, 4), %xmm0 > - vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0 > - movl _args_digest+4*32(state, idx, 4), DWORD_tmp > - > - vmovdqu %xmm0, _result_digest(job_rax) > - movl DWORD_tmp, _result_digest+1*16(job_rax) > - > -return: > - pop %r12 > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > - > -ENDPROC(sha1_mb_mgr_submit_avx2) > - > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > -.align 16 > -clear_low_nibble: > - .octa 0x000000000000000000000000FFFFFFF0 > diff --git a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S b/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > deleted file mode 100644 > index 20f77aa633de..000000000000 > --- a/arch/x86/crypto/sha1-mb/sha1_x8_avx2.S > +++ /dev/null > @@ -1,492 +0,0 @@ > -/* > - * Multi-buffer SHA1 algorithm hash compute routine > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * James Guilford <james.guilford@xxxxxxxxx> > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2014 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include "sha1_mb_mgr_datastruct.S" > - > -## code to compute oct SHA1 using SSE-256 > -## outer calling routine takes care of save and restore of XMM registers > - > -## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15# ymm0-15 > -## > -## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15 > -## Linux preserves: rdi rbp r8 > -## > -## clobbers ymm0-15 > - > - > -# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1 > -# "transpose" data in {r0...r7} using temps {t0...t1} > -# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > -# r4 = {e7 e6 e5 e4 e3 e2 e1 e0} > -# r5 = {f7 f6 f5 f4 f3 f2 f1 f0} > -# r6 = {g7 g6 g5 g4 g3 g2 g1 g0} > -# r7 = {h7 h6 h5 h4 h3 h2 h1 h0} > -# > -# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > -# r0 = {h0 g0 f0 e0 d0 c0 b0 a0} > -# r1 = {h1 g1 f1 e1 d1 c1 b1 a1} > -# r2 = {h2 g2 f2 e2 d2 c2 b2 a2} > -# r3 = {h3 g3 f3 e3 d3 c3 b3 a3} > -# r4 = {h4 g4 f4 e4 d4 c4 b4 a4} > -# r5 = {h5 g5 f5 e5 d5 c5 b5 a5} > -# r6 = {h6 g6 f6 e6 d6 c6 b6 a6} > -# r7 = {h7 g7 f7 e7 d7 c7 b7 a7} > -# > - > -.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1 > - # process top half (r0..r3) {a...d} > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > - vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1} > - vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2} > - vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3} > - vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0} > - > - # use r2 in place of t0 > - # process bottom half (r4..r7) {e...h} > - vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0} > - vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2} > - vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0} > - vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2} > - vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1} > - vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2} > - vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3} > - vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0} > - > - vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6 > - vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2 > - vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5 > - vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1 > - vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7 > - vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3 > - vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4 > - vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0 > - > -.endm > -## > -## Magic functions defined in FIPS 180-1 > -## > -# macro MAGIC_F0 F,B,C,D,T ## F = (D ^ (B & (C ^ D))) > -.macro MAGIC_F0 regF regB regC regD regT > - vpxor \regD, \regC, \regF > - vpand \regB, \regF, \regF > - vpxor \regD, \regF, \regF > -.endm > - > -# macro MAGIC_F1 F,B,C,D,T ## F = (B ^ C ^ D) > -.macro MAGIC_F1 regF regB regC regD regT > - vpxor \regC, \regD, \regF > - vpxor \regB, \regF, \regF > -.endm > - > -# macro MAGIC_F2 F,B,C,D,T ## F = ((B & C) | (B & D) | (C & D)) > -.macro MAGIC_F2 regF regB regC regD regT > - vpor \regC, \regB, \regF > - vpand \regC, \regB, \regT > - vpand \regD, \regF, \regF > - vpor \regT, \regF, \regF > -.endm > - > -# macro MAGIC_F3 F,B,C,D,T ## F = (B ^ C ^ D) > -.macro MAGIC_F3 regF regB regC regD regT > - MAGIC_F1 \regF,\regB,\regC,\regD,\regT > -.endm > - > -# PROLD reg, imm, tmp > -.macro PROLD reg imm tmp > - vpsrld $(32-\imm), \reg, \tmp > - vpslld $\imm, \reg, \reg > - vpor \tmp, \reg, \reg > -.endm > - > -.macro PROLD_nd reg imm tmp src > - vpsrld $(32-\imm), \src, \tmp > - vpslld $\imm, \src, \reg > - vpor \tmp, \reg, \reg > -.endm > - > -.macro SHA1_STEP_00_15 regA regB regC regD regE regT regF memW immCNT MAGIC > - vpaddd \immCNT, \regE, \regE > - vpaddd \memW*32(%rsp), \regE, \regE > - PROLD_nd \regT, 5, \regF, \regA > - vpaddd \regT, \regE, \regE > - \MAGIC \regF, \regB, \regC, \regD, \regT > - PROLD \regB, 30, \regT > - vpaddd \regF, \regE, \regE > -.endm > - > -.macro SHA1_STEP_16_79 regA regB regC regD regE regT regF memW immCNT MAGIC > - vpaddd \immCNT, \regE, \regE > - offset = ((\memW - 14) & 15) * 32 > - vmovdqu offset(%rsp), W14 > - vpxor W14, W16, W16 > - offset = ((\memW - 8) & 15) * 32 > - vpxor offset(%rsp), W16, W16 > - offset = ((\memW - 3) & 15) * 32 > - vpxor offset(%rsp), W16, W16 > - vpsrld $(32-1), W16, \regF > - vpslld $1, W16, W16 > - vpor W16, \regF, \regF > - > - ROTATE_W > - > - offset = ((\memW - 0) & 15) * 32 > - vmovdqu \regF, offset(%rsp) > - vpaddd \regF, \regE, \regE > - PROLD_nd \regT, 5, \regF, \regA > - vpaddd \regT, \regE, \regE > - \MAGIC \regF,\regB,\regC,\regD,\regT ## FUN = MAGIC_Fi(B,C,D) > - PROLD \regB,30, \regT > - vpaddd \regF, \regE, \regE > -.endm > - > -######################################################################## > -######################################################################## > -######################################################################## > - > -## FRAMESZ plus pushes must be an odd multiple of 8 > -YMM_SAVE = (15-15)*32 > -FRAMESZ = 32*16 + YMM_SAVE > -_YMM = FRAMESZ - YMM_SAVE > - > -#define VMOVPS vmovups > - > -IDX = %rax > -inp0 = %r9 > -inp1 = %r10 > -inp2 = %r11 > -inp3 = %r12 > -inp4 = %r13 > -inp5 = %r14 > -inp6 = %r15 > -inp7 = %rcx > -arg1 = %rdi > -arg2 = %rsi > -RSP_SAVE = %rdx > - > -# ymm0 A > -# ymm1 B > -# ymm2 C > -# ymm3 D > -# ymm4 E > -# ymm5 F AA > -# ymm6 T0 BB > -# ymm7 T1 CC > -# ymm8 T2 DD > -# ymm9 T3 EE > -# ymm10 T4 TMP > -# ymm11 T5 FUN > -# ymm12 T6 K > -# ymm13 T7 W14 > -# ymm14 T8 W15 > -# ymm15 T9 W16 > - > - > -A = %ymm0 > -B = %ymm1 > -C = %ymm2 > -D = %ymm3 > -E = %ymm4 > -F = %ymm5 > -T0 = %ymm6 > -T1 = %ymm7 > -T2 = %ymm8 > -T3 = %ymm9 > -T4 = %ymm10 > -T5 = %ymm11 > -T6 = %ymm12 > -T7 = %ymm13 > -T8 = %ymm14 > -T9 = %ymm15 > - > -AA = %ymm5 > -BB = %ymm6 > -CC = %ymm7 > -DD = %ymm8 > -EE = %ymm9 > -TMP = %ymm10 > -FUN = %ymm11 > -K = %ymm12 > -W14 = %ymm13 > -W15 = %ymm14 > -W16 = %ymm15 > - > -.macro ROTATE_ARGS > - TMP_ = E > - E = D > - D = C > - C = B > - B = A > - A = TMP_ > -.endm > - > -.macro ROTATE_W > -TMP_ = W16 > -W16 = W15 > -W15 = W14 > -W14 = TMP_ > -.endm > - > -# 8 streams x 5 32bit words per digest x 4 bytes per word > -#define DIGEST_SIZE (8*5*4) > - > -.align 32 > - > -# void sha1_x8_avx2(void **input_data, UINT128 *digest, UINT32 size) > -# arg 1 : pointer to array[4] of pointer to input data > -# arg 2 : size (in blocks) ;; assumed to be >= 1 > -# > -ENTRY(sha1_x8_avx2) > - > - # save callee-saved clobbered registers to comply with C function ABI > - push %r12 > - push %r13 > - push %r14 > - push %r15 > - > - #save rsp > - mov %rsp, RSP_SAVE > - sub $FRAMESZ, %rsp > - > - #align rsp to 32 Bytes > - and $~0x1F, %rsp > - > - ## Initialize digests > - vmovdqu 0*32(arg1), A > - vmovdqu 1*32(arg1), B > - vmovdqu 2*32(arg1), C > - vmovdqu 3*32(arg1), D > - vmovdqu 4*32(arg1), E > - > - ## transpose input onto stack > - mov _data_ptr+0*8(arg1),inp0 > - mov _data_ptr+1*8(arg1),inp1 > - mov _data_ptr+2*8(arg1),inp2 > - mov _data_ptr+3*8(arg1),inp3 > - mov _data_ptr+4*8(arg1),inp4 > - mov _data_ptr+5*8(arg1),inp5 > - mov _data_ptr+6*8(arg1),inp6 > - mov _data_ptr+7*8(arg1),inp7 > - > - xor IDX, IDX > -lloop: > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), F > - I=0 > -.rep 2 > - VMOVPS (inp0, IDX), T0 > - VMOVPS (inp1, IDX), T1 > - VMOVPS (inp2, IDX), T2 > - VMOVPS (inp3, IDX), T3 > - VMOVPS (inp4, IDX), T4 > - VMOVPS (inp5, IDX), T5 > - VMOVPS (inp6, IDX), T6 > - VMOVPS (inp7, IDX), T7 > - > - TRANSPOSE8 T0, T1, T2, T3, T4, T5, T6, T7, T8, T9 > - vpshufb F, T0, T0 > - vmovdqu T0, (I*8)*32(%rsp) > - vpshufb F, T1, T1 > - vmovdqu T1, (I*8+1)*32(%rsp) > - vpshufb F, T2, T2 > - vmovdqu T2, (I*8+2)*32(%rsp) > - vpshufb F, T3, T3 > - vmovdqu T3, (I*8+3)*32(%rsp) > - vpshufb F, T4, T4 > - vmovdqu T4, (I*8+4)*32(%rsp) > - vpshufb F, T5, T5 > - vmovdqu T5, (I*8+5)*32(%rsp) > - vpshufb F, T6, T6 > - vmovdqu T6, (I*8+6)*32(%rsp) > - vpshufb F, T7, T7 > - vmovdqu T7, (I*8+7)*32(%rsp) > - add $32, IDX > - I = (I+1) > -.endr > - # save old digests > - vmovdqu A,AA > - vmovdqu B,BB > - vmovdqu C,CC > - vmovdqu D,DD > - vmovdqu E,EE > - > -## > -## perform 0-79 steps > -## > - vmovdqu K00_19(%rip), K > -## do rounds 0...15 > - I = 0 > -.rep 16 > - SHA1_STEP_00_15 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0 > - ROTATE_ARGS > - I = (I+1) > -.endr > - > -## do rounds 16...19 > - vmovdqu ((16 - 16) & 15) * 32 (%rsp), W16 > - vmovdqu ((16 - 15) & 15) * 32 (%rsp), W15 > -.rep 4 > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F0 > - ROTATE_ARGS > - I = (I+1) > -.endr > - > -## do rounds 20...39 > - vmovdqu K20_39(%rip), K > -.rep 20 > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F1 > - ROTATE_ARGS > - I = (I+1) > -.endr > - > -## do rounds 40...59 > - vmovdqu K40_59(%rip), K > -.rep 20 > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F2 > - ROTATE_ARGS > - I = (I+1) > -.endr > - > -## do rounds 60...79 > - vmovdqu K60_79(%rip), K > -.rep 20 > - SHA1_STEP_16_79 A,B,C,D,E, TMP,FUN, I, K, MAGIC_F3 > - ROTATE_ARGS > - I = (I+1) > -.endr > - > - vpaddd AA,A,A > - vpaddd BB,B,B > - vpaddd CC,C,C > - vpaddd DD,D,D > - vpaddd EE,E,E > - > - sub $1, arg2 > - jne lloop > - > - # write out digests > - vmovdqu A, 0*32(arg1) > - vmovdqu B, 1*32(arg1) > - vmovdqu C, 2*32(arg1) > - vmovdqu D, 3*32(arg1) > - vmovdqu E, 4*32(arg1) > - > - # update input pointers > - add IDX, inp0 > - add IDX, inp1 > - add IDX, inp2 > - add IDX, inp3 > - add IDX, inp4 > - add IDX, inp5 > - add IDX, inp6 > - add IDX, inp7 > - mov inp0, _data_ptr (arg1) > - mov inp1, _data_ptr + 1*8(arg1) > - mov inp2, _data_ptr + 2*8(arg1) > - mov inp3, _data_ptr + 3*8(arg1) > - mov inp4, _data_ptr + 4*8(arg1) > - mov inp5, _data_ptr + 5*8(arg1) > - mov inp6, _data_ptr + 6*8(arg1) > - mov inp7, _data_ptr + 7*8(arg1) > - > - ################ > - ## Postamble > - > - mov RSP_SAVE, %rsp > - > - # restore callee-saved clobbered registers > - pop %r15 > - pop %r14 > - pop %r13 > - pop %r12 > - > - ret > -ENDPROC(sha1_x8_avx2) > - > - > -.section .rodata.cst32.K00_19, "aM", @progbits, 32 > -.align 32 > -K00_19: > -.octa 0x5A8279995A8279995A8279995A827999 > -.octa 0x5A8279995A8279995A8279995A827999 > - > -.section .rodata.cst32.K20_39, "aM", @progbits, 32 > -.align 32 > -K20_39: > -.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 > -.octa 0x6ED9EBA16ED9EBA16ED9EBA16ED9EBA1 > - > -.section .rodata.cst32.K40_59, "aM", @progbits, 32 > -.align 32 > -K40_59: > -.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC > -.octa 0x8F1BBCDC8F1BBCDC8F1BBCDC8F1BBCDC > - > -.section .rodata.cst32.K60_79, "aM", @progbits, 32 > -.align 32 > -K60_79: > -.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 > -.octa 0xCA62C1D6CA62C1D6CA62C1D6CA62C1D6 > - > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > -.align 32 > -PSHUFFLE_BYTE_FLIP_MASK: > -.octa 0x0c0d0e0f08090a0b0405060700010203 > -.octa 0x0c0d0e0f08090a0b0405060700010203 > diff --git a/arch/x86/crypto/sha256-mb/Makefile b/arch/x86/crypto/sha256-mb/Makefile > deleted file mode 100644 > index 53ad6e7db747..000000000000 > --- a/arch/x86/crypto/sha256-mb/Makefile > +++ /dev/null > @@ -1,14 +0,0 @@ > -# SPDX-License-Identifier: GPL-2.0 > -# > -# Arch-specific CryptoAPI modules. > -# > - > -OBJECT_FILES_NON_STANDARD := y > - > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > - $(comma)4)$(comma)%ymm2,yes,no) > -ifeq ($(avx2_supported),yes) > - obj-$(CONFIG_CRYPTO_SHA256_MB) += sha256-mb.o > - sha256-mb-y := sha256_mb.o sha256_mb_mgr_flush_avx2.o \ > - sha256_mb_mgr_init_avx2.o sha256_mb_mgr_submit_avx2.o sha256_x8_avx2.o > -endif > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb.c b/arch/x86/crypto/sha256-mb/sha256_mb.c > deleted file mode 100644 > index 97c5fc43e115..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb.c > +++ /dev/null > @@ -1,1013 +0,0 @@ > -/* > - * Multi buffer SHA256 algorithm Glue Code > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > - > -#include <crypto/internal/hash.h> > -#include <linux/init.h> > -#include <linux/module.h> > -#include <linux/mm.h> > -#include <linux/cryptohash.h> > -#include <linux/types.h> > -#include <linux/list.h> > -#include <crypto/scatterwalk.h> > -#include <crypto/sha.h> > -#include <crypto/mcryptd.h> > -#include <crypto/crypto_wq.h> > -#include <asm/byteorder.h> > -#include <linux/hardirq.h> > -#include <asm/fpu/api.h> > -#include "sha256_mb_ctx.h" > - > -#define FLUSH_INTERVAL 1000 /* in usec */ > - > -static struct mcryptd_alg_state sha256_mb_alg_state; > - > -struct sha256_mb_ctx { > - struct mcryptd_ahash *mcryptd_tfm; > -}; > - > -static inline struct mcryptd_hash_request_ctx > - *cast_hash_to_mcryptd_ctx(struct sha256_hash_ctx *hash_ctx) > -{ > - struct ahash_request *areq; > - > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > -} > - > -static inline struct ahash_request > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > -{ > - return container_of((void *) ctx, struct ahash_request, __ctx); > -} > - > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > - struct ahash_request *areq) > -{ > - rctx->flag = HASH_UPDATE; > -} > - > -static asmlinkage void (*sha256_job_mgr_init)(struct sha256_mb_mgr *state); > -static asmlinkage struct job_sha256* (*sha256_job_mgr_submit) > - (struct sha256_mb_mgr *state, struct job_sha256 *job); > -static asmlinkage struct job_sha256* (*sha256_job_mgr_flush) > - (struct sha256_mb_mgr *state); > -static asmlinkage struct job_sha256* (*sha256_job_mgr_get_comp_job) > - (struct sha256_mb_mgr *state); > - > -inline uint32_t sha256_pad(uint8_t padblock[SHA256_BLOCK_SIZE * 2], > - uint64_t total_len) > -{ > - uint32_t i = total_len & (SHA256_BLOCK_SIZE - 1); > - > - memset(&padblock[i], 0, SHA256_BLOCK_SIZE); > - padblock[i] = 0x80; > - > - i += ((SHA256_BLOCK_SIZE - 1) & > - (0 - (total_len + SHA256_PADLENGTHFIELD_SIZE + 1))) > - + 1 + SHA256_PADLENGTHFIELD_SIZE; > - > -#if SHA256_PADLENGTHFIELD_SIZE == 16 > - *((uint64_t *) &padblock[i - 16]) = 0; > -#endif > - > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > - > - /* Number of extra blocks to hash */ > - return i >> SHA256_LOG2_BLOCK_SIZE; > -} > - > -static struct sha256_hash_ctx > - *sha256_ctx_mgr_resubmit(struct sha256_ctx_mgr *mgr, > - struct sha256_hash_ctx *ctx) > -{ > - while (ctx) { > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Clear PROCESSING bit */ > - ctx->status = HASH_CTX_STS_COMPLETE; > - return ctx; > - } > - > - /* > - * If the extra blocks are empty, begin hashing what remains > - * in the user's buffer. > - */ > - if (ctx->partial_block_buffer_length == 0 && > - ctx->incoming_buffer_length) { > - > - const void *buffer = ctx->incoming_buffer; > - uint32_t len = ctx->incoming_buffer_length; > - uint32_t copy_len; > - > - /* > - * Only entire blocks can be hashed. > - * Copy remainder to extra blocks buffer. > - */ > - copy_len = len & (SHA256_BLOCK_SIZE-1); > - > - if (copy_len) { > - len -= copy_len; > - memcpy(ctx->partial_block_buffer, > - ((const char *) buffer + len), > - copy_len); > - ctx->partial_block_buffer_length = copy_len; > - } > - > - ctx->incoming_buffer_length = 0; > - > - /* len should be a multiple of the block size now */ > - assert((len % SHA256_BLOCK_SIZE) == 0); > - > - /* Set len to the number of blocks to be hashed */ > - len >>= SHA256_LOG2_BLOCK_SIZE; > - > - if (len) { > - > - ctx->job.buffer = (uint8_t *) buffer; > - ctx->job.len = len; > - ctx = (struct sha256_hash_ctx *) > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > - continue; > - } > - } > - > - /* > - * If the extra blocks are not empty, then we are > - * either on the last block(s) or we need more > - * user input before continuing. > - */ > - if (ctx->status & HASH_CTX_STS_LAST) { > - > - uint8_t *buf = ctx->partial_block_buffer; > - uint32_t n_extra_blocks = > - sha256_pad(buf, ctx->total_length); > - > - ctx->status = (HASH_CTX_STS_PROCESSING | > - HASH_CTX_STS_COMPLETE); > - ctx->job.buffer = buf; > - ctx->job.len = (uint32_t) n_extra_blocks; > - ctx = (struct sha256_hash_ctx *) > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > - continue; > - } > - > - ctx->status = HASH_CTX_STS_IDLE; > - return ctx; > - } > - > - return NULL; > -} > - > -static struct sha256_hash_ctx > - *sha256_ctx_mgr_get_comp_ctx(struct sha256_ctx_mgr *mgr) > -{ > - /* > - * If get_comp_job returns NULL, there are no jobs complete. > - * If get_comp_job returns a job, verify that it is safe to return to > - * the user. If it is not ready, resubmit the job to finish processing. > - * If sha256_ctx_mgr_resubmit returned a job, it is ready to be > - * returned. Otherwise, all jobs currently being managed by the > - * hash_ctx_mgr still need processing. > - */ > - struct sha256_hash_ctx *ctx; > - > - ctx = (struct sha256_hash_ctx *) sha256_job_mgr_get_comp_job(&mgr->mgr); > - return sha256_ctx_mgr_resubmit(mgr, ctx); > -} > - > -static void sha256_ctx_mgr_init(struct sha256_ctx_mgr *mgr) > -{ > - sha256_job_mgr_init(&mgr->mgr); > -} > - > -static struct sha256_hash_ctx *sha256_ctx_mgr_submit(struct sha256_ctx_mgr *mgr, > - struct sha256_hash_ctx *ctx, > - const void *buffer, > - uint32_t len, > - int flags) > -{ > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > - /* User should not pass anything other than UPDATE or LAST */ > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > - return ctx; > - } > - > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > - /* Cannot submit to a currently processing job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > - return ctx; > - } > - > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Cannot update a finished job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > - return ctx; > - } > - > - /* If we made it here, there was no error during this call to submit */ > - ctx->error = HASH_CTX_ERROR_NONE; > - > - /* Store buffer ptr info from user */ > - ctx->incoming_buffer = buffer; > - ctx->incoming_buffer_length = len; > - > - /* > - * Store the user's request flags and mark this ctx as currently > - * being processed. > - */ > - ctx->status = (flags & HASH_LAST) ? > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > - HASH_CTX_STS_PROCESSING; > - > - /* Advance byte counter */ > - ctx->total_length += len; > - > - /* > - * If there is anything currently buffered in the extra blocks, > - * append to it until it contains a whole block. > - * Or if the user's buffer contains less than a whole block, > - * append as much as possible to the extra block. > - */ > - if (ctx->partial_block_buffer_length || len < SHA256_BLOCK_SIZE) { > - /* > - * Compute how many bytes to copy from user buffer into > - * extra block > - */ > - uint32_t copy_len = SHA256_BLOCK_SIZE - > - ctx->partial_block_buffer_length; > - if (len < copy_len) > - copy_len = len; > - > - if (copy_len) { > - /* Copy and update relevant pointers and counters */ > - memcpy( > - &ctx->partial_block_buffer[ctx->partial_block_buffer_length], > - buffer, copy_len); > - > - ctx->partial_block_buffer_length += copy_len; > - ctx->incoming_buffer = (const void *) > - ((const char *)buffer + copy_len); > - ctx->incoming_buffer_length = len - copy_len; > - } > - > - /* The extra block should never contain more than 1 block */ > - assert(ctx->partial_block_buffer_length <= SHA256_BLOCK_SIZE); > - > - /* > - * If the extra block buffer contains exactly 1 block, > - * it can be hashed. > - */ > - if (ctx->partial_block_buffer_length >= SHA256_BLOCK_SIZE) { > - ctx->partial_block_buffer_length = 0; > - > - ctx->job.buffer = ctx->partial_block_buffer; > - ctx->job.len = 1; > - ctx = (struct sha256_hash_ctx *) > - sha256_job_mgr_submit(&mgr->mgr, &ctx->job); > - } > - } > - > - return sha256_ctx_mgr_resubmit(mgr, ctx); > -} > - > -static struct sha256_hash_ctx *sha256_ctx_mgr_flush(struct sha256_ctx_mgr *mgr) > -{ > - struct sha256_hash_ctx *ctx; > - > - while (1) { > - ctx = (struct sha256_hash_ctx *) > - sha256_job_mgr_flush(&mgr->mgr); > - > - /* If flush returned 0, there are no more jobs in flight. */ > - if (!ctx) > - return NULL; > - > - /* > - * If flush returned a job, resubmit the job to finish > - * processing. > - */ > - ctx = sha256_ctx_mgr_resubmit(mgr, ctx); > - > - /* > - * If sha256_ctx_mgr_resubmit returned a job, it is ready to > - * be returned. Otherwise, all jobs currently being managed by > - * the sha256_ctx_mgr still need processing. Loop. > - */ > - if (ctx) > - return ctx; > - } > -} > - > -static int sha256_mb_init(struct ahash_request *areq) > -{ > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > - > - hash_ctx_init(sctx); > - sctx->job.result_digest[0] = SHA256_H0; > - sctx->job.result_digest[1] = SHA256_H1; > - sctx->job.result_digest[2] = SHA256_H2; > - sctx->job.result_digest[3] = SHA256_H3; > - sctx->job.result_digest[4] = SHA256_H4; > - sctx->job.result_digest[5] = SHA256_H5; > - sctx->job.result_digest[6] = SHA256_H6; > - sctx->job.result_digest[7] = SHA256_H7; > - sctx->total_length = 0; > - sctx->partial_block_buffer_length = 0; > - sctx->status = HASH_CTX_STS_IDLE; > - > - return 0; > -} > - > -static int sha256_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > -{ > - int i; > - struct sha256_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > - __be32 *dst = (__be32 *) rctx->out; > - > - for (i = 0; i < 8; ++i) > - dst[i] = cpu_to_be32(sctx->job.result_digest[i]); > - > - return 0; > -} > - > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > - struct mcryptd_alg_cstate *cstate, bool flush) > -{ > - int flag = HASH_UPDATE; > - int nbytes, err = 0; > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > - struct sha256_hash_ctx *sha_ctx; > - > - /* more work ? */ > - while (!(rctx->flag & HASH_DONE)) { > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > - if (nbytes < 0) { > - err = nbytes; > - goto out; > - } > - /* check if the walk is done */ > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - if (rctx->flag & HASH_FINAL) > - flag |= HASH_LAST; > - > - } > - sha_ctx = (struct sha256_hash_ctx *) > - ahash_request_ctx(&rctx->areq); > - kernel_fpu_begin(); > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, > - rctx->walk.data, nbytes, flag); > - if (!sha_ctx) { > - if (flush) > - sha_ctx = sha256_ctx_mgr_flush(cstate->mgr); > - } > - kernel_fpu_end(); > - if (sha_ctx) > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - else { > - rctx = NULL; > - goto out; > - } > - } > - > - /* copy the results */ > - if (rctx->flag & HASH_FINAL) > - sha256_mb_set_results(rctx); > - > -out: > - *ret_rctx = rctx; > - return err; > -} > - > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate, > - int err) > -{ > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha256_hash_ctx *sha_ctx; > - struct mcryptd_hash_request_ctx *req_ctx; > - int ret; > - > - /* remove from work list */ > - spin_lock(&cstate->work_lock); > - list_del(&rctx->waiter); > - spin_unlock(&cstate->work_lock); > - > - if (irqs_disabled()) > - rctx->complete(&req->base, err); > - else { > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > - } > - > - /* check to see if there are other jobs that are done */ > - sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); > - while (sha_ctx) { > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&req_ctx, cstate, false); > - if (req_ctx) { > - spin_lock(&cstate->work_lock); > - list_del(&req_ctx->waiter); > - spin_unlock(&cstate->work_lock); > - > - req = cast_mcryptd_ctx_to_req(req_ctx); > - if (irqs_disabled()) > - req_ctx->complete(&req->base, ret); > - else { > - local_bh_disable(); > - req_ctx->complete(&req->base, ret); > - local_bh_enable(); > - } > - } > - sha_ctx = sha256_ctx_mgr_get_comp_ctx(cstate->mgr); > - } > - > - return 0; > -} > - > -static void sha256_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate) > -{ > - unsigned long next_flush; > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > - > - /* initialize tag */ > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > - rctx->tag.seq_num = cstate->next_seq_num++; > - next_flush = rctx->tag.arrival + delay; > - rctx->tag.expire = next_flush; > - > - spin_lock(&cstate->work_lock); > - list_add_tail(&rctx->waiter, &cstate->work_list); > - spin_unlock(&cstate->work_lock); > - > - mcryptd_arm_flusher(cstate, delay); > -} > - > -static int sha256_mb_update(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha256_hash_ctx *sha_ctx; > - int ret = 0, nbytes; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) > - rctx->flag |= HASH_DONE; > - > - /* submit */ > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > - sha256_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > - nbytes, HASH_UPDATE); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha256_mb_finup(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha256_hash_ctx *sha_ctx; > - int ret = 0, flag = HASH_UPDATE, nbytes; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - flag = HASH_LAST; > - } > - > - /* submit */ > - rctx->flag |= HASH_FINAL; > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > - sha256_mb_add_list(rctx, cstate); > - > - kernel_fpu_begin(); > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data, > - nbytes, flag); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha256_mb_final(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, > - areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha256_mb_alg_state.alg_cstate); > - > - struct sha256_hash_ctx *sha_ctx; > - int ret = 0; > - u8 data; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - rctx->flag |= HASH_DONE | HASH_FINAL; > - > - sha_ctx = (struct sha256_hash_ctx *) ahash_request_ctx(areq); > - /* flag HASH_FINAL and 0 data size */ > - sha256_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha256_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0, > - HASH_LAST); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha256_mb_export(struct ahash_request *areq, void *out) > -{ > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(out, sctx, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha256_mb_import(struct ahash_request *areq, const void *in) > -{ > - struct sha256_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(sctx, in, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha256_mb_async_init_tfm(struct crypto_tfm *tfm) > -{ > - struct mcryptd_ahash *mcryptd_tfm; > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - struct mcryptd_hash_ctx *mctx; > - > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha256-mb", > - CRYPTO_ALG_INTERNAL, > - CRYPTO_ALG_INTERNAL); > - if (IS_ERR(mcryptd_tfm)) > - return PTR_ERR(mcryptd_tfm); > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > - mctx->alg_state = &sha256_mb_alg_state; > - ctx->mcryptd_tfm = mcryptd_tfm; > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > - > - return 0; > -} > - > -static void sha256_mb_async_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static int sha256_mb_areq_init_tfm(struct crypto_tfm *tfm) > -{ > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - sizeof(struct sha256_hash_ctx)); > - > - return 0; > -} > - > -static void sha256_mb_areq_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha256_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static struct ahash_alg sha256_mb_areq_alg = { > - .init = sha256_mb_init, > - .update = sha256_mb_update, > - .final = sha256_mb_final, > - .finup = sha256_mb_finup, > - .export = sha256_mb_export, > - .import = sha256_mb_import, > - .halg = { > - .digestsize = SHA256_DIGEST_SIZE, > - .statesize = sizeof(struct sha256_hash_ctx), > - .base = { > - .cra_name = "__sha256-mb", > - .cra_driver_name = "__intel_sha256-mb", > - .cra_priority = 100, > - /* > - * use ASYNC flag as some buffers in multi-buffer > - * algo may not have completed before hashing thread > - * sleep > - */ > - .cra_flags = CRYPTO_ALG_ASYNC | > - CRYPTO_ALG_INTERNAL, > - .cra_blocksize = SHA256_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT > - (sha256_mb_areq_alg.halg.base.cra_list), > - .cra_init = sha256_mb_areq_init_tfm, > - .cra_exit = sha256_mb_areq_exit_tfm, > - .cra_ctxsize = sizeof(struct sha256_hash_ctx), > - } > - } > -}; > - > -static int sha256_mb_async_init(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_init(mcryptd_req); > -} > - > -static int sha256_mb_async_update(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_update(mcryptd_req); > -} > - > -static int sha256_mb_async_finup(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_finup(mcryptd_req); > -} > - > -static int sha256_mb_async_final(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_final(mcryptd_req); > -} > - > -static int sha256_mb_async_digest(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_digest(mcryptd_req); > -} > - > -static int sha256_mb_async_export(struct ahash_request *req, void *out) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_export(mcryptd_req, out); > -} > - > -static int sha256_mb_async_import(struct ahash_request *req, const void *in) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha256_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > - struct mcryptd_hash_request_ctx *rctx; > - struct ahash_request *areq; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - rctx = ahash_request_ctx(mcryptd_req); > - areq = &rctx->areq; > - > - ahash_request_set_tfm(areq, child); > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > - rctx->complete, req); > - > - return crypto_ahash_import(mcryptd_req, in); > -} > - > -static struct ahash_alg sha256_mb_async_alg = { > - .init = sha256_mb_async_init, > - .update = sha256_mb_async_update, > - .final = sha256_mb_async_final, > - .finup = sha256_mb_async_finup, > - .export = sha256_mb_async_export, > - .import = sha256_mb_async_import, > - .digest = sha256_mb_async_digest, > - .halg = { > - .digestsize = SHA256_DIGEST_SIZE, > - .statesize = sizeof(struct sha256_hash_ctx), > - .base = { > - .cra_name = "sha256", > - .cra_driver_name = "sha256_mb", > - /* > - * Low priority, since with few concurrent hash requests > - * this is extremely slow due to the flush delay. Users > - * whose workloads would benefit from this can request > - * it explicitly by driver name, or can increase its > - * priority at runtime using NETLINK_CRYPTO. > - */ > - .cra_priority = 50, > - .cra_flags = CRYPTO_ALG_ASYNC, > - .cra_blocksize = SHA256_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT > - (sha256_mb_async_alg.halg.base.cra_list), > - .cra_init = sha256_mb_async_init_tfm, > - .cra_exit = sha256_mb_async_exit_tfm, > - .cra_ctxsize = sizeof(struct sha256_mb_ctx), > - .cra_alignmask = 0, > - }, > - }, > -}; > - > -static unsigned long sha256_mb_flusher(struct mcryptd_alg_cstate *cstate) > -{ > - struct mcryptd_hash_request_ctx *rctx; > - unsigned long cur_time; > - unsigned long next_flush = 0; > - struct sha256_hash_ctx *sha_ctx; > - > - > - cur_time = jiffies; > - > - while (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - if (time_before(cur_time, rctx->tag.expire)) > - break; > - kernel_fpu_begin(); > - sha_ctx = (struct sha256_hash_ctx *) > - sha256_ctx_mgr_flush(cstate->mgr); > - kernel_fpu_end(); > - if (!sha_ctx) { > - pr_err("sha256_mb error: nothing got" > - " flushed for non-empty list\n"); > - break; > - } > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - sha_finish_walk(&rctx, cstate, true); > - sha_complete_job(rctx, cstate, 0); > - } > - > - if (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - /* get the hash context and then flush time */ > - next_flush = rctx->tag.expire; > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > - } > - return next_flush; > -} > - > -static int __init sha256_mb_mod_init(void) > -{ > - > - int cpu; > - int err; > - struct mcryptd_alg_cstate *cpu_state; > - > - /* check for dependent cpu features */ > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > - !boot_cpu_has(X86_FEATURE_BMI2)) > - return -ENODEV; > - > - /* initialize multibuffer structures */ > - sha256_mb_alg_state.alg_cstate = alloc_percpu > - (struct mcryptd_alg_cstate); > - > - sha256_job_mgr_init = sha256_mb_mgr_init_avx2; > - sha256_job_mgr_submit = sha256_mb_mgr_submit_avx2; > - sha256_job_mgr_flush = sha256_mb_mgr_flush_avx2; > - sha256_job_mgr_get_comp_job = sha256_mb_mgr_get_comp_job_avx2; > - > - if (!sha256_mb_alg_state.alg_cstate) > - return -ENOMEM; > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > - cpu_state->next_flush = 0; > - cpu_state->next_seq_num = 0; > - cpu_state->flusher_engaged = false; > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > - cpu_state->cpu = cpu; > - cpu_state->alg_state = &sha256_mb_alg_state; > - cpu_state->mgr = kzalloc(sizeof(struct sha256_ctx_mgr), > - GFP_KERNEL); > - if (!cpu_state->mgr) > - goto err2; > - sha256_ctx_mgr_init(cpu_state->mgr); > - INIT_LIST_HEAD(&cpu_state->work_list); > - spin_lock_init(&cpu_state->work_lock); > - } > - sha256_mb_alg_state.flusher = &sha256_mb_flusher; > - > - err = crypto_register_ahash(&sha256_mb_areq_alg); > - if (err) > - goto err2; > - err = crypto_register_ahash(&sha256_mb_async_alg); > - if (err) > - goto err1; > - > - > - return 0; > -err1: > - crypto_unregister_ahash(&sha256_mb_areq_alg); > -err2: > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha256_mb_alg_state.alg_cstate); > - return -ENODEV; > -} > - > -static void __exit sha256_mb_mod_fini(void) > -{ > - int cpu; > - struct mcryptd_alg_cstate *cpu_state; > - > - crypto_unregister_ahash(&sha256_mb_async_alg); > - crypto_unregister_ahash(&sha256_mb_areq_alg); > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha256_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha256_mb_alg_state.alg_cstate); > -} > - > -module_init(sha256_mb_mod_init); > -module_exit(sha256_mb_mod_fini); > - > -MODULE_LICENSE("GPL"); > -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, multi buffer accelerated"); > - > -MODULE_ALIAS_CRYPTO("sha256"); > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h b/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > deleted file mode 100644 > index 7c432543dc7f..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_ctx.h > +++ /dev/null > @@ -1,134 +0,0 @@ > -/* > - * Header file for multi buffer SHA256 context > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#ifndef _SHA_MB_CTX_INTERNAL_H > -#define _SHA_MB_CTX_INTERNAL_H > - > -#include "sha256_mb_mgr.h" > - > -#define HASH_UPDATE 0x00 > -#define HASH_LAST 0x01 > -#define HASH_DONE 0x02 > -#define HASH_FINAL 0x04 > - > -#define HASH_CTX_STS_IDLE 0x00 > -#define HASH_CTX_STS_PROCESSING 0x01 > -#define HASH_CTX_STS_LAST 0x02 > -#define HASH_CTX_STS_COMPLETE 0x04 > - > -enum hash_ctx_error { > - HASH_CTX_ERROR_NONE = 0, > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > - > -#ifdef HASH_CTX_DEBUG > - HASH_CTX_ERROR_DEBUG_DIGEST_MISMATCH = -4, > -#endif > -}; > - > - > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > -#define hash_ctx_status(ctx) ((ctx)->status) > -#define hash_ctx_error(ctx) ((ctx)->error) > -#define hash_ctx_init(ctx) \ > - do { \ > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > - } while (0) > - > - > -/* Hash Constants and Typedefs */ > -#define SHA256_DIGEST_LENGTH 8 > -#define SHA256_LOG2_BLOCK_SIZE 6 > - > -#define SHA256_PADLENGTHFIELD_SIZE 8 > - > -#ifdef SHA_MB_DEBUG > -#define assert(expr) \ > -do { \ > - if (unlikely(!(expr))) { \ > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > - #expr, __FILE__, __func__, __LINE__); \ > - } \ > -} while (0) > -#else > -#define assert(expr) do {} while (0) > -#endif > - > -struct sha256_ctx_mgr { > - struct sha256_mb_mgr mgr; > -}; > - > -/* typedef struct sha256_ctx_mgr sha256_ctx_mgr; */ > - > -struct sha256_hash_ctx { > - /* Must be at struct offset 0 */ > - struct job_sha256 job; > - /* status flag */ > - int status; > - /* error flag */ > - int error; > - > - uint64_t total_length; > - const void *incoming_buffer; > - uint32_t incoming_buffer_length; > - uint8_t partial_block_buffer[SHA256_BLOCK_SIZE * 2]; > - uint32_t partial_block_buffer_length; > - void *user_data; > -}; > - > -#endif > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h b/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > deleted file mode 100644 > index b01ae408c56d..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr.h > +++ /dev/null > @@ -1,108 +0,0 @@ > -/* > - * Header file for multi buffer SHA256 algorithm manager > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > -#ifndef __SHA_MB_MGR_H > -#define __SHA_MB_MGR_H > - > -#include <linux/types.h> > - > -#define NUM_SHA256_DIGEST_WORDS 8 > - > -enum job_sts { STS_UNKNOWN = 0, > - STS_BEING_PROCESSED = 1, > - STS_COMPLETED = 2, > - STS_INTERNAL_ERROR = 3, > - STS_ERROR = 4 > -}; > - > -struct job_sha256 { > - u8 *buffer; > - u32 len; > - u32 result_digest[NUM_SHA256_DIGEST_WORDS] __aligned(32); > - enum job_sts status; > - void *user_data; > -}; > - > -/* SHA256 out-of-order scheduler */ > - > -/* typedef uint32_t sha8_digest_array[8][8]; */ > - > -struct sha256_args_x8 { > - uint32_t digest[8][8]; > - uint8_t *data_ptr[8]; > -}; > - > -struct sha256_lane_data { > - struct job_sha256 *job_in_lane; > -}; > - > -struct sha256_mb_mgr { > - struct sha256_args_x8 args; > - > - uint32_t lens[8]; > - > - /* each byte is index (0...7) of unused lanes */ > - uint64_t unused_lanes; > - /* byte 4 is set to FF as a flag */ > - struct sha256_lane_data ldata[8]; > -}; > - > - > -#define SHA256_MB_MGR_NUM_LANES_AVX2 8 > - > -void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state); > -struct job_sha256 *sha256_mb_mgr_submit_avx2(struct sha256_mb_mgr *state, > - struct job_sha256 *job); > -struct job_sha256 *sha256_mb_mgr_flush_avx2(struct sha256_mb_mgr *state); > -struct job_sha256 *sha256_mb_mgr_get_comp_job_avx2(struct sha256_mb_mgr *state); > - > -#endif > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > deleted file mode 100644 > index 5c377bac21d0..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_datastruct.S > +++ /dev/null > @@ -1,304 +0,0 @@ > -/* > - * Header file for multi buffer SHA256 algorithm data structure > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -# Macros for defining data structures > - > -# Usage example > - > -#START_FIELDS # JOB_AES > -### name size align > -#FIELD _plaintext, 8, 8 # pointer to plaintext > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > -#FIELD _IV, 16, 8 # IV > -#FIELD _keys, 8, 8 # pointer to keys > -#FIELD _len, 4, 4 # length in bytes > -#FIELD _status, 4, 4 # status enumeration > -#FIELD _user_data, 8, 8 # pointer to user data > -#UNION _union, size1, align1, \ > -# size2, align2, \ > -# size3, align3, \ > -# ... > -#END_FIELDS > -#%assign _JOB_AES_size _FIELD_OFFSET > -#%assign _JOB_AES_align _STRUCT_ALIGN > - > -######################################################################### > - > -# Alternate "struc-like" syntax: > -# STRUCT job_aes2 > -# RES_Q .plaintext, 1 > -# RES_Q .ciphertext, 1 > -# RES_DQ .IV, 1 > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > -# RES_U .union, size1, align1, \ > -# size2, align2, \ > -# ... > -# ENDSTRUCT > -# # Following only needed if nesting > -# %assign job_aes2_size _FIELD_OFFSET > -# %assign job_aes2_align _STRUCT_ALIGN > -# > -# RES_* macros take a name, a count and an optional alignment. > -# The count in in terms of the base size of the macro, and the > -# default alignment is the base size. > -# The macros are: > -# Macro Base size > -# RES_B 1 > -# RES_W 2 > -# RES_D 4 > -# RES_Q 8 > -# RES_DQ 16 > -# RES_Y 32 > -# RES_Z 64 > -# > -# RES_U defines a union. It's arguments are a name and two or more > -# pairs of "size, alignment" > -# > -# The two assigns are only needed if this structure is being nested > -# within another. Even if the assigns are not done, one can still use > -# STRUCT_NAME_size as the size of the structure. > -# > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > -# > -# The differences between this and using "struc" directly are that each > -# type is implicitly aligned to its natural length (although this can be > -# over-ridden with an explicit third parameter), and that the structure > -# is padded at the end to its overall alignment. > -# > - > -######################################################################### > - > -#ifndef _DATASTRUCT_ASM_ > -#define _DATASTRUCT_ASM_ > - > -#define SZ8 8*SHA256_DIGEST_WORD_SIZE > -#define ROUNDS 64*SZ8 > -#define PTR_SZ 8 > -#define SHA256_DIGEST_WORD_SIZE 4 > -#define MAX_SHA256_LANES 8 > -#define SHA256_DIGEST_WORDS 8 > -#define SHA256_DIGEST_ROW_SIZE (MAX_SHA256_LANES * SHA256_DIGEST_WORD_SIZE) > -#define SHA256_DIGEST_SIZE (SHA256_DIGEST_ROW_SIZE * SHA256_DIGEST_WORDS) > -#define SHA256_BLK_SZ 64 > - > -# START_FIELDS > -.macro START_FIELDS > - _FIELD_OFFSET = 0 > - _STRUCT_ALIGN = 0 > -.endm > - > -# FIELD name size align > -.macro FIELD name size align > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > - \name = _FIELD_OFFSET > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > -.if (\align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = \align > -.endif > -.endm > - > -# END_FIELDS > -.macro END_FIELDS > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > -.endm > - > -######################################################################## > - > -.macro STRUCT p1 > -START_FIELDS > -.struc \p1 > -.endm > - > -.macro ENDSTRUCT > - tmp = _FIELD_OFFSET > - END_FIELDS > - tmp = (_FIELD_OFFSET - %%tmp) > -.if (tmp > 0) > - .lcomm tmp > -.endif > -.endstruc > -.endm > - > -## RES_int name size align > -.macro RES_int p1 p2 p3 > - name = \p1 > - size = \p2 > - align = .\p3 > - > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > -.align align > -.lcomm name size > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > -.if (align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = align > -.endif > -.endm > - > -# macro RES_B name, size [, align] > -.macro RES_B _name, _size, _align=1 > -RES_int _name _size _align > -.endm > - > -# macro RES_W name, size [, align] > -.macro RES_W _name, _size, _align=2 > -RES_int _name 2*(_size) _align > -.endm > - > -# macro RES_D name, size [, align] > -.macro RES_D _name, _size, _align=4 > -RES_int _name 4*(_size) _align > -.endm > - > -# macro RES_Q name, size [, align] > -.macro RES_Q _name, _size, _align=8 > -RES_int _name 8*(_size) _align > -.endm > - > -# macro RES_DQ name, size [, align] > -.macro RES_DQ _name, _size, _align=16 > -RES_int _name 16*(_size) _align > -.endm > - > -# macro RES_Y name, size [, align] > -.macro RES_Y _name, _size, _align=32 > -RES_int _name 32*(_size) _align > -.endm > - > -# macro RES_Z name, size [, align] > -.macro RES_Z _name, _size, _align=64 > -RES_int _name 64*(_size) _align > -.endm > - > -#endif > - > - > -######################################################################## > -#### Define SHA256 Out Of Order Data Structures > -######################################################################## > - > -START_FIELDS # LANE_DATA > -### name size align > -FIELD _job_in_lane, 8, 8 # pointer to job object > -END_FIELDS > - > - _LANE_DATA_size = _FIELD_OFFSET > - _LANE_DATA_align = _STRUCT_ALIGN > - > -######################################################################## > - > -START_FIELDS # SHA256_ARGS_X4 > -### name size align > -FIELD _digest, 4*8*8, 4 # transposed digest > -FIELD _data_ptr, 8*8, 8 # array of pointers to data > -END_FIELDS > - > - _SHA256_ARGS_X4_size = _FIELD_OFFSET > - _SHA256_ARGS_X4_align = _STRUCT_ALIGN > - _SHA256_ARGS_X8_size = _FIELD_OFFSET > - _SHA256_ARGS_X8_align = _STRUCT_ALIGN > - > -####################################################################### > - > -START_FIELDS # MB_MGR > -### name size align > -FIELD _args, _SHA256_ARGS_X4_size, _SHA256_ARGS_X4_align > -FIELD _lens, 4*8, 8 > -FIELD _unused_lanes, 8, 8 > -FIELD _ldata, _LANE_DATA_size*8, _LANE_DATA_align > -END_FIELDS > - > - _MB_MGR_size = _FIELD_OFFSET > - _MB_MGR_align = _STRUCT_ALIGN > - > -_args_digest = _args + _digest > -_args_data_ptr = _args + _data_ptr > - > -####################################################################### > - > -START_FIELDS #STACK_FRAME > -### name size align > -FIELD _data, 16*SZ8, 1 # transposed digest > -FIELD _digest, 8*SZ8, 1 # array of pointers to data > -FIELD _ytmp, 4*SZ8, 1 > -FIELD _rsp, 8, 1 > -END_FIELDS > - > - _STACK_FRAME_size = _FIELD_OFFSET > - _STACK_FRAME_align = _STRUCT_ALIGN > - > -####################################################################### > - > -######################################################################## > -#### Define constants > -######################################################################## > - > -#define STS_UNKNOWN 0 > -#define STS_BEING_PROCESSED 1 > -#define STS_COMPLETED 2 > - > -######################################################################## > -#### Define JOB_SHA256 structure > -######################################################################## > - > -START_FIELDS # JOB_SHA256 > - > -### name size align > -FIELD _buffer, 8, 8 # pointer to buffer > -FIELD _len, 8, 8 # length in bytes > -FIELD _result_digest, 8*4, 32 # Digest (output) > -FIELD _status, 4, 4 > -FIELD _user_data, 8, 8 > -END_FIELDS > - > - _JOB_SHA256_size = _FIELD_OFFSET > - _JOB_SHA256_align = _STRUCT_ALIGN > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > deleted file mode 100644 > index d2364c55bbde..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_flush_avx2.S > +++ /dev/null > @@ -1,307 +0,0 @@ > -/* > - * Flush routine for SHA256 multibuffer > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha256_mb_mgr_datastruct.S" > - > -.extern sha256_x8_avx2 > - > -#LINUX register definitions > -#define arg1 %rdi > -#define arg2 %rsi > - > -# Common register definitions > -#define state arg1 > -#define job arg2 > -#define len2 arg2 > - > -# idx must be a register not clobberred by sha1_mult > -#define idx %r8 > -#define DWORD_idx %r8d > - > -#define unused_lanes %rbx > -#define lane_data %rbx > -#define tmp2 %rbx > -#define tmp2_w %ebx > - > -#define job_rax %rax > -#define tmp1 %rax > -#define size_offset %rax > -#define tmp %rax > -#define start_offset %rax > - > -#define tmp3 %arg1 > - > -#define extra_blocks %arg2 > -#define p %arg2 > - > -.macro LABEL prefix n > -\prefix\n\(): > -.endm > - > -.macro JNE_SKIP i > -jne skip_\i > -.endm > - > -.altmacro > -.macro SET_OFFSET _offset > -offset = \_offset > -.endm > -.noaltmacro > - > -# JOB_SHA256* sha256_mb_mgr_flush_avx2(MB_MGR *state) > -# arg 1 : rcx : state > -ENTRY(sha256_mb_mgr_flush_avx2) > - FRAME_BEGIN > - push %rbx > - > - # If bit (32+3) is set, then all lanes are empty > - mov _unused_lanes(state), unused_lanes > - bt $32+3, unused_lanes > - jc return_null > - > - # find a lane with a non-null job > - xor idx, idx > - offset = (_ldata + 1 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne one(%rip), idx > - offset = (_ldata + 2 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne two(%rip), idx > - offset = (_ldata + 3 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne three(%rip), idx > - offset = (_ldata + 4 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne four(%rip), idx > - offset = (_ldata + 5 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne five(%rip), idx > - offset = (_ldata + 6 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne six(%rip), idx > - offset = (_ldata + 7 * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne seven(%rip), idx > - > - # copy idx to empty lanes > -copy_lane_data: > - offset = (_args + _data_ptr) > - mov offset(state,idx,8), tmp > - > - I = 0 > -.rep 8 > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > -.altmacro > - JNE_SKIP %I > - offset = (_args + _data_ptr + 8*I) > - mov tmp, offset(state) > - offset = (_lens + 4*I) > - movl $0xFFFFFFFF, offset(state) > -LABEL skip_ %I > - I = (I+1) > -.noaltmacro > -.endr > - > - # Find min length > - vmovdqu _lens+0*16(state), %xmm0 > - vmovdqu _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > - > - vmovd %xmm2, DWORD_idx > - mov idx, len2 > - and $0xF, idx > - shr $4, len2 > - jz len_is_0 > - > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > - vpshufd $0, %xmm2, %xmm2 > - > - vpsubd %xmm2, %xmm0, %xmm0 > - vpsubd %xmm2, %xmm1, %xmm1 > - > - vmovdqu %xmm0, _lens+0*16(state) > - vmovdqu %xmm1, _lens+1*16(state) > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha256_x8_avx2 > - # state and idx are intact > - > -len_is_0: > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $4, unused_lanes > - or idx, unused_lanes > - > - mov unused_lanes, _unused_lanes(state) > - movl $0xFFFFFFFF, _lens(state,idx,4) > - > - vmovd _args_digest(state , idx, 4) , %xmm0 > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > - vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 > - vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 > - vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 > - > - vmovdqu %xmm0, _result_digest(job_rax) > - offset = (_result_digest + 1*16) > - vmovdqu %xmm1, offset(job_rax) > - > -return: > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > -ENDPROC(sha256_mb_mgr_flush_avx2) > - > -############################################################################## > - > -.align 16 > -ENTRY(sha256_mb_mgr_get_comp_job_avx2) > - push %rbx > - > - ## if bit 32+3 is set, then all lanes are empty > - mov _unused_lanes(state), unused_lanes > - bt $(32+3), unused_lanes > - jc .return_null > - > - # Find min length > - vmovdqu _lens(state), %xmm0 > - vmovdqu _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > - > - vmovd %xmm2, DWORD_idx > - test $~0xF, idx > - jnz .return_null > - > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $4, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens(state, idx, 4) > - > - vmovd _args_digest(state, idx, 4), %xmm0 > - vpinsrd $1, _args_digest+1*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state, idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state, idx, 4), %xmm0, %xmm0 > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > - vpinsrd $1, _args_digest+5*32(state, idx, 4), %xmm1, %xmm1 > - vpinsrd $2, _args_digest+6*32(state, idx, 4), %xmm1, %xmm1 > - vpinsrd $3, _args_digest+7*32(state, idx, 4), %xmm1, %xmm1 > - > - vmovdqu %xmm0, _result_digest(job_rax) > - offset = (_result_digest + 1*16) > - vmovdqu %xmm1, offset(job_rax) > - > - pop %rbx > - > - ret > - > -.return_null: > - xor job_rax, job_rax > - pop %rbx > - ret > -ENDPROC(sha256_mb_mgr_get_comp_job_avx2) > - > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > -.align 16 > -clear_low_nibble: > -.octa 0x000000000000000000000000FFFFFFF0 > - > -.section .rodata.cst8, "aM", @progbits, 8 > -.align 8 > -one: > -.quad 1 > -two: > -.quad 2 > -three: > -.quad 3 > -four: > -.quad 4 > -five: > -.quad 5 > -six: > -.quad 6 > -seven: > -.quad 7 > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > deleted file mode 100644 > index b0c498371e67..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_init_avx2.c > +++ /dev/null > @@ -1,65 +0,0 @@ > -/* > - * Initialization code for multi buffer SHA256 algorithm for AVX2 > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include "sha256_mb_mgr.h" > - > -void sha256_mb_mgr_init_avx2(struct sha256_mb_mgr *state) > -{ > - unsigned int j; > - > - state->unused_lanes = 0xF76543210ULL; > - for (j = 0; j < 8; j++) { > - state->lens[j] = 0xFFFFFFFF; > - state->ldata[j].job_in_lane = NULL; > - } > -} > diff --git a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > deleted file mode 100644 > index b36ae7454084..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_mb_mgr_submit_avx2.S > +++ /dev/null > @@ -1,214 +0,0 @@ > -/* > - * Buffer submit code for multi buffer SHA256 algorithm > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha256_mb_mgr_datastruct.S" > - > -.extern sha256_x8_avx2 > - > -# LINUX register definitions > -arg1 = %rdi > -arg2 = %rsi > -size_offset = %rcx > -tmp2 = %rcx > -extra_blocks = %rdx > - > -# Common definitions > -#define state arg1 > -#define job %rsi > -#define len2 arg2 > -#define p2 arg2 > - > -# idx must be a register not clobberred by sha1_x8_avx2 > -idx = %r8 > -DWORD_idx = %r8d > -last_len = %r8 > - > -p = %r11 > -start_offset = %r11 > - > -unused_lanes = %rbx > -BYTE_unused_lanes = %bl > - > -job_rax = %rax > -len = %rax > -DWORD_len = %eax > - > -lane = %r12 > -tmp3 = %r12 > - > -tmp = %r9 > -DWORD_tmp = %r9d > - > -lane_data = %r10 > - > -# JOB* sha256_mb_mgr_submit_avx2(MB_MGR *state, JOB_SHA256 *job) > -# arg 1 : rcx : state > -# arg 2 : rdx : job > -ENTRY(sha256_mb_mgr_submit_avx2) > - FRAME_BEGIN > - push %rbx > - push %r12 > - > - mov _unused_lanes(state), unused_lanes > - mov unused_lanes, lane > - and $0xF, lane > - shr $4, unused_lanes > - imul $_LANE_DATA_size, lane, lane_data > - movl $STS_BEING_PROCESSED, _status(job) > - lea _ldata(state, lane_data), lane_data > - mov unused_lanes, _unused_lanes(state) > - movl _len(job), DWORD_len > - > - mov job, _job_in_lane(lane_data) > - shl $4, len > - or lane, len > - > - movl DWORD_len, _lens(state , lane, 4) > - > - # Load digest words from result_digest > - vmovdqu _result_digest(job), %xmm0 > - vmovdqu _result_digest+1*16(job), %xmm1 > - vmovd %xmm0, _args_digest(state, lane, 4) > - vpextrd $1, %xmm0, _args_digest+1*32(state , lane, 4) > - vpextrd $2, %xmm0, _args_digest+2*32(state , lane, 4) > - vpextrd $3, %xmm0, _args_digest+3*32(state , lane, 4) > - vmovd %xmm1, _args_digest+4*32(state , lane, 4) > - > - vpextrd $1, %xmm1, _args_digest+5*32(state , lane, 4) > - vpextrd $2, %xmm1, _args_digest+6*32(state , lane, 4) > - vpextrd $3, %xmm1, _args_digest+7*32(state , lane, 4) > - > - mov _buffer(job), p > - mov p, _args_data_ptr(state, lane, 8) > - > - cmp $0xF, unused_lanes > - jne return_null > - > -start_loop: > - # Find min length > - vmovdqa _lens(state), %xmm0 > - vmovdqa _lens+1*16(state), %xmm1 > - > - vpminud %xmm1, %xmm0, %xmm2 # xmm2 has {D,C,B,A} > - vpalignr $8, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,D,C} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has {x,x,E,F} > - vpalignr $4, %xmm2, %xmm3, %xmm3 # xmm3 has {x,x,x,E} > - vpminud %xmm3, %xmm2, %xmm2 # xmm2 has min val in low dword > - > - vmovd %xmm2, DWORD_idx > - mov idx, len2 > - and $0xF, idx > - shr $4, len2 > - jz len_is_0 > - > - vpand clear_low_nibble(%rip), %xmm2, %xmm2 > - vpshufd $0, %xmm2, %xmm2 > - > - vpsubd %xmm2, %xmm0, %xmm0 > - vpsubd %xmm2, %xmm1, %xmm1 > - > - vmovdqa %xmm0, _lens + 0*16(state) > - vmovdqa %xmm1, _lens + 1*16(state) > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha256_x8_avx2 > - > - # state and idx are intact > - > -len_is_0: > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - mov _unused_lanes(state), unused_lanes > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - shl $4, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens(state,idx,4) > - > - vmovd _args_digest(state, idx, 4), %xmm0 > - vpinsrd $1, _args_digest+1*32(state , idx, 4), %xmm0, %xmm0 > - vpinsrd $2, _args_digest+2*32(state , idx, 4), %xmm0, %xmm0 > - vpinsrd $3, _args_digest+3*32(state , idx, 4), %xmm0, %xmm0 > - vmovd _args_digest+4*32(state, idx, 4), %xmm1 > - > - vpinsrd $1, _args_digest+5*32(state , idx, 4), %xmm1, %xmm1 > - vpinsrd $2, _args_digest+6*32(state , idx, 4), %xmm1, %xmm1 > - vpinsrd $3, _args_digest+7*32(state , idx, 4), %xmm1, %xmm1 > - > - vmovdqu %xmm0, _result_digest(job_rax) > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > - > -return: > - pop %r12 > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > - > -ENDPROC(sha256_mb_mgr_submit_avx2) > - > -.section .rodata.cst16.clear_low_nibble, "aM", @progbits, 16 > -.align 16 > -clear_low_nibble: > - .octa 0x000000000000000000000000FFFFFFF0 > diff --git a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S b/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > deleted file mode 100644 > index 1687c80c5995..000000000000 > --- a/arch/x86/crypto/sha256-mb/sha256_x8_avx2.S > +++ /dev/null > @@ -1,598 +0,0 @@ > -/* > - * Multi-buffer SHA256 algorithm hash compute routine > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include "sha256_mb_mgr_datastruct.S" > - > -## code to compute oct SHA256 using SSE-256 > -## outer calling routine takes care of save and restore of XMM registers > -## Logic designed/laid out by JDG > - > -## Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; %ymm0-15 > -## Linux clobbers: rax rbx rcx rdx rsi r9 r10 r11 r12 r13 r14 r15 > -## Linux preserves: rdi rbp r8 > -## > -## clobbers %ymm0-15 > - > -arg1 = %rdi > -arg2 = %rsi > -reg3 = %rcx > -reg4 = %rdx > - > -# Common definitions > -STATE = arg1 > -INP_SIZE = arg2 > - > -IDX = %rax > -ROUND = %rbx > -TBL = reg3 > - > -inp0 = %r9 > -inp1 = %r10 > -inp2 = %r11 > -inp3 = %r12 > -inp4 = %r13 > -inp5 = %r14 > -inp6 = %r15 > -inp7 = reg4 > - > -a = %ymm0 > -b = %ymm1 > -c = %ymm2 > -d = %ymm3 > -e = %ymm4 > -f = %ymm5 > -g = %ymm6 > -h = %ymm7 > - > -T1 = %ymm8 > - > -a0 = %ymm12 > -a1 = %ymm13 > -a2 = %ymm14 > -TMP = %ymm15 > -TMP0 = %ymm6 > -TMP1 = %ymm7 > - > -TT0 = %ymm8 > -TT1 = %ymm9 > -TT2 = %ymm10 > -TT3 = %ymm11 > -TT4 = %ymm12 > -TT5 = %ymm13 > -TT6 = %ymm14 > -TT7 = %ymm15 > - > -# Define stack usage > - > -# Assume stack aligned to 32 bytes before call > -# Therefore FRAMESZ mod 32 must be 32-8 = 24 > - > -#define FRAMESZ 0x388 > - > -#define VMOVPS vmovups > - > -# TRANSPOSE8 r0, r1, r2, r3, r4, r5, r6, r7, t0, t1 > -# "transpose" data in {r0...r7} using temps {t0...t1} > -# Input looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > -# r4 = {e7 e6 e5 e4 e3 e2 e1 e0} > -# r5 = {f7 f6 f5 f4 f3 f2 f1 f0} > -# r6 = {g7 g6 g5 g4 g3 g2 g1 g0} > -# r7 = {h7 h6 h5 h4 h3 h2 h1 h0} > -# > -# Output looks like: {r0 r1 r2 r3 r4 r5 r6 r7} > -# r0 = {h0 g0 f0 e0 d0 c0 b0 a0} > -# r1 = {h1 g1 f1 e1 d1 c1 b1 a1} > -# r2 = {h2 g2 f2 e2 d2 c2 b2 a2} > -# r3 = {h3 g3 f3 e3 d3 c3 b3 a3} > -# r4 = {h4 g4 f4 e4 d4 c4 b4 a4} > -# r5 = {h5 g5 f5 e5 d5 c5 b5 a5} > -# r6 = {h6 g6 f6 e6 d6 c6 b6 a6} > -# r7 = {h7 g7 f7 e7 d7 c7 b7 a7} > -# > - > -.macro TRANSPOSE8 r0 r1 r2 r3 r4 r5 r6 r7 t0 t1 > - # process top half (r0..r3) {a...d} > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > - vshufps $0xDD, \t1, \t0, \r3 # r3 = {d5 c5 b5 a5 d1 c1 b1 a1} > - vshufps $0x88, \r2, \r0, \r1 # r1 = {d6 c6 b6 a6 d2 c2 b2 a2} > - vshufps $0xDD, \r2, \r0, \r0 # r0 = {d7 c7 b7 a7 d3 c3 b3 a3} > - vshufps $0x88, \t1, \t0, \t0 # t0 = {d4 c4 b4 a4 d0 c0 b0 a0} > - > - # use r2 in place of t0 > - # process bottom half (r4..r7) {e...h} > - vshufps $0x44, \r5, \r4, \r2 # r2 = {f5 f4 e5 e4 f1 f0 e1 e0} > - vshufps $0xEE, \r5, \r4, \r4 # r4 = {f7 f6 e7 e6 f3 f2 e3 e2} > - vshufps $0x44, \r7, \r6, \t1 # t1 = {h5 h4 g5 g4 h1 h0 g1 g0} > - vshufps $0xEE, \r7, \r6, \r6 # r6 = {h7 h6 g7 g6 h3 h2 g3 g2} > - vshufps $0xDD, \t1, \r2, \r7 # r7 = {h5 g5 f5 e5 h1 g1 f1 e1} > - vshufps $0x88, \r6, \r4, \r5 # r5 = {h6 g6 f6 e6 h2 g2 f2 e2} > - vshufps $0xDD, \r6, \r4, \r4 # r4 = {h7 g7 f7 e7 h3 g3 f3 e3} > - vshufps $0x88, \t1, \r2, \t1 # t1 = {h4 g4 f4 e4 h0 g0 f0 e0} > - > - vperm2f128 $0x13, \r1, \r5, \r6 # h6...a6 > - vperm2f128 $0x02, \r1, \r5, \r2 # h2...a2 > - vperm2f128 $0x13, \r3, \r7, \r5 # h5...a5 > - vperm2f128 $0x02, \r3, \r7, \r1 # h1...a1 > - vperm2f128 $0x13, \r0, \r4, \r7 # h7...a7 > - vperm2f128 $0x02, \r0, \r4, \r3 # h3...a3 > - vperm2f128 $0x13, \t0, \t1, \r4 # h4...a4 > - vperm2f128 $0x02, \t0, \t1, \r0 # h0...a0 > - > -.endm > - > -.macro ROTATE_ARGS > -TMP_ = h > -h = g > -g = f > -f = e > -e = d > -d = c > -c = b > -b = a > -a = TMP_ > -.endm > - > -.macro _PRORD reg imm tmp > - vpslld $(32-\imm),\reg,\tmp > - vpsrld $\imm,\reg, \reg > - vpor \tmp,\reg, \reg > -.endm > - > -# PRORD_nd reg, imm, tmp, src > -.macro _PRORD_nd reg imm tmp src > - vpslld $(32-\imm), \src, \tmp > - vpsrld $\imm, \src, \reg > - vpor \tmp, \reg, \reg > -.endm > - > -# PRORD dst/src, amt > -.macro PRORD reg imm > - _PRORD \reg,\imm,TMP > -.endm > - > -# PRORD_nd dst, src, amt > -.macro PRORD_nd reg tmp imm > - _PRORD_nd \reg, \imm, TMP, \tmp > -.endm > - > -# arguments passed implicitly in preprocessor symbols i, a...h > -.macro ROUND_00_15 _T1 i > - PRORD_nd a0,e,5 # sig1: a0 = (e >> 5) > - > - vpxor g, f, a2 # ch: a2 = f^g > - vpand e,a2, a2 # ch: a2 = (f^g)&e > - vpxor g, a2, a2 # a2 = ch > - > - PRORD_nd a1,e,25 # sig1: a1 = (e >> 25) > - > - vmovdqu \_T1,(SZ8*(\i & 0xf))(%rsp) > - vpaddd (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K > - vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) > - PRORD a0, 6 # sig1: a0 = (e >> 6) ^ (e >> 11) > - vpaddd a2, h, h # h = h + ch > - PRORD_nd a2,a,11 # sig0: a2 = (a >> 11) > - vpaddd \_T1,h, h # h = h + ch + W + K > - vpxor a1, a0, a0 # a0 = sigma1 > - PRORD_nd a1,a,22 # sig0: a1 = (a >> 22) > - vpxor c, a, \_T1 # maj: T1 = a^c > - add $SZ8, ROUND # ROUND++ > - vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b > - vpaddd a0, h, h > - vpaddd h, d, d > - vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) > - PRORD a2,2 # sig0: a2 = (a >> 2) ^ (a >> 13) > - vpxor a1, a2, a2 # a2 = sig0 > - vpand c, a, a1 # maj: a1 = a&c > - vpor \_T1, a1, a1 # a1 = maj > - vpaddd a1, h, h # h = h + ch + W + K + maj > - vpaddd a2, h, h # h = h + ch + W + K + maj + sigma0 > - ROTATE_ARGS > -.endm > - > -# arguments passed implicitly in preprocessor symbols i, a...h > -.macro ROUND_16_XX _T1 i > - vmovdqu (SZ8*((\i-15)&0xf))(%rsp), \_T1 > - vmovdqu (SZ8*((\i-2)&0xf))(%rsp), a1 > - vmovdqu \_T1, a0 > - PRORD \_T1,11 > - vmovdqu a1, a2 > - PRORD a1,2 > - vpxor a0, \_T1, \_T1 > - PRORD \_T1, 7 > - vpxor a2, a1, a1 > - PRORD a1, 17 > - vpsrld $3, a0, a0 > - vpxor a0, \_T1, \_T1 > - vpsrld $10, a2, a2 > - vpxor a2, a1, a1 > - vpaddd (SZ8*((\i-16)&0xf))(%rsp), \_T1, \_T1 > - vpaddd (SZ8*((\i-7)&0xf))(%rsp), a1, a1 > - vpaddd a1, \_T1, \_T1 > - > - ROUND_00_15 \_T1,\i > -.endm > - > -# SHA256_ARGS: > -# UINT128 digest[8]; // transposed digests > -# UINT8 *data_ptr[4]; > - > -# void sha256_x8_avx2(SHA256_ARGS *args, UINT64 bytes); > -# arg 1 : STATE : pointer to array of pointers to input data > -# arg 2 : INP_SIZE : size of input in blocks > - # general registers preserved in outer calling routine > - # outer calling routine saves all the XMM registers > - # save rsp, allocate 32-byte aligned for local variables > -ENTRY(sha256_x8_avx2) > - > - # save callee-saved clobbered registers to comply with C function ABI > - push %r12 > - push %r13 > - push %r14 > - push %r15 > - > - mov %rsp, IDX > - sub $FRAMESZ, %rsp > - and $~0x1F, %rsp > - mov IDX, _rsp(%rsp) > - > - # Load the pre-transposed incoming digest. > - vmovdqu 0*SHA256_DIGEST_ROW_SIZE(STATE),a > - vmovdqu 1*SHA256_DIGEST_ROW_SIZE(STATE),b > - vmovdqu 2*SHA256_DIGEST_ROW_SIZE(STATE),c > - vmovdqu 3*SHA256_DIGEST_ROW_SIZE(STATE),d > - vmovdqu 4*SHA256_DIGEST_ROW_SIZE(STATE),e > - vmovdqu 5*SHA256_DIGEST_ROW_SIZE(STATE),f > - vmovdqu 6*SHA256_DIGEST_ROW_SIZE(STATE),g > - vmovdqu 7*SHA256_DIGEST_ROW_SIZE(STATE),h > - > - lea K256_8(%rip),TBL > - > - # load the address of each of the 4 message lanes > - # getting ready to transpose input onto stack > - mov _args_data_ptr+0*PTR_SZ(STATE),inp0 > - mov _args_data_ptr+1*PTR_SZ(STATE),inp1 > - mov _args_data_ptr+2*PTR_SZ(STATE),inp2 > - mov _args_data_ptr+3*PTR_SZ(STATE),inp3 > - mov _args_data_ptr+4*PTR_SZ(STATE),inp4 > - mov _args_data_ptr+5*PTR_SZ(STATE),inp5 > - mov _args_data_ptr+6*PTR_SZ(STATE),inp6 > - mov _args_data_ptr+7*PTR_SZ(STATE),inp7 > - > - xor IDX, IDX > -lloop: > - xor ROUND, ROUND > - > - # save old digest > - vmovdqu a, _digest(%rsp) > - vmovdqu b, _digest+1*SZ8(%rsp) > - vmovdqu c, _digest+2*SZ8(%rsp) > - vmovdqu d, _digest+3*SZ8(%rsp) > - vmovdqu e, _digest+4*SZ8(%rsp) > - vmovdqu f, _digest+5*SZ8(%rsp) > - vmovdqu g, _digest+6*SZ8(%rsp) > - vmovdqu h, _digest+7*SZ8(%rsp) > - i = 0 > -.rep 2 > - VMOVPS i*32(inp0, IDX), TT0 > - VMOVPS i*32(inp1, IDX), TT1 > - VMOVPS i*32(inp2, IDX), TT2 > - VMOVPS i*32(inp3, IDX), TT3 > - VMOVPS i*32(inp4, IDX), TT4 > - VMOVPS i*32(inp5, IDX), TT5 > - VMOVPS i*32(inp6, IDX), TT6 > - VMOVPS i*32(inp7, IDX), TT7 > - vmovdqu g, _ytmp(%rsp) > - vmovdqu h, _ytmp+1*SZ8(%rsp) > - TRANSPOSE8 TT0, TT1, TT2, TT3, TT4, TT5, TT6, TT7, TMP0, TMP1 > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP1 > - vmovdqu _ytmp(%rsp), g > - vpshufb TMP1, TT0, TT0 > - vpshufb TMP1, TT1, TT1 > - vpshufb TMP1, TT2, TT2 > - vpshufb TMP1, TT3, TT3 > - vpshufb TMP1, TT4, TT4 > - vpshufb TMP1, TT5, TT5 > - vpshufb TMP1, TT6, TT6 > - vpshufb TMP1, TT7, TT7 > - vmovdqu _ytmp+1*SZ8(%rsp), h > - vmovdqu TT4, _ytmp(%rsp) > - vmovdqu TT5, _ytmp+1*SZ8(%rsp) > - vmovdqu TT6, _ytmp+2*SZ8(%rsp) > - vmovdqu TT7, _ytmp+3*SZ8(%rsp) > - ROUND_00_15 TT0,(i*8+0) > - vmovdqu _ytmp(%rsp), TT0 > - ROUND_00_15 TT1,(i*8+1) > - vmovdqu _ytmp+1*SZ8(%rsp), TT1 > - ROUND_00_15 TT2,(i*8+2) > - vmovdqu _ytmp+2*SZ8(%rsp), TT2 > - ROUND_00_15 TT3,(i*8+3) > - vmovdqu _ytmp+3*SZ8(%rsp), TT3 > - ROUND_00_15 TT0,(i*8+4) > - ROUND_00_15 TT1,(i*8+5) > - ROUND_00_15 TT2,(i*8+6) > - ROUND_00_15 TT3,(i*8+7) > - i = (i+1) > -.endr > - add $64, IDX > - i = (i*8) > - > - jmp Lrounds_16_xx > -.align 16 > -Lrounds_16_xx: > -.rep 16 > - ROUND_16_XX T1, i > - i = (i+1) > -.endr > - > - cmp $ROUNDS,ROUND > - jb Lrounds_16_xx > - > - # add old digest > - vpaddd _digest+0*SZ8(%rsp), a, a > - vpaddd _digest+1*SZ8(%rsp), b, b > - vpaddd _digest+2*SZ8(%rsp), c, c > - vpaddd _digest+3*SZ8(%rsp), d, d > - vpaddd _digest+4*SZ8(%rsp), e, e > - vpaddd _digest+5*SZ8(%rsp), f, f > - vpaddd _digest+6*SZ8(%rsp), g, g > - vpaddd _digest+7*SZ8(%rsp), h, h > - > - sub $1, INP_SIZE # unit is blocks > - jne lloop > - > - # write back to memory (state object) the transposed digest > - vmovdqu a, 0*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu b, 1*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu c, 2*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu d, 3*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu e, 4*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu f, 5*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu g, 6*SHA256_DIGEST_ROW_SIZE(STATE) > - vmovdqu h, 7*SHA256_DIGEST_ROW_SIZE(STATE) > - > - # update input pointers > - add IDX, inp0 > - mov inp0, _args_data_ptr+0*8(STATE) > - add IDX, inp1 > - mov inp1, _args_data_ptr+1*8(STATE) > - add IDX, inp2 > - mov inp2, _args_data_ptr+2*8(STATE) > - add IDX, inp3 > - mov inp3, _args_data_ptr+3*8(STATE) > - add IDX, inp4 > - mov inp4, _args_data_ptr+4*8(STATE) > - add IDX, inp5 > - mov inp5, _args_data_ptr+5*8(STATE) > - add IDX, inp6 > - mov inp6, _args_data_ptr+6*8(STATE) > - add IDX, inp7 > - mov inp7, _args_data_ptr+7*8(STATE) > - > - # Postamble > - mov _rsp(%rsp), %rsp > - > - # restore callee-saved clobbered registers > - pop %r15 > - pop %r14 > - pop %r13 > - pop %r12 > - > - ret > -ENDPROC(sha256_x8_avx2) > - > -.section .rodata.K256_8, "a", @progbits > -.align 64 > -K256_8: > - .octa 0x428a2f98428a2f98428a2f98428a2f98 > - .octa 0x428a2f98428a2f98428a2f98428a2f98 > - .octa 0x71374491713744917137449171374491 > - .octa 0x71374491713744917137449171374491 > - .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf > - .octa 0xb5c0fbcfb5c0fbcfb5c0fbcfb5c0fbcf > - .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 > - .octa 0xe9b5dba5e9b5dba5e9b5dba5e9b5dba5 > - .octa 0x3956c25b3956c25b3956c25b3956c25b > - .octa 0x3956c25b3956c25b3956c25b3956c25b > - .octa 0x59f111f159f111f159f111f159f111f1 > - .octa 0x59f111f159f111f159f111f159f111f1 > - .octa 0x923f82a4923f82a4923f82a4923f82a4 > - .octa 0x923f82a4923f82a4923f82a4923f82a4 > - .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 > - .octa 0xab1c5ed5ab1c5ed5ab1c5ed5ab1c5ed5 > - .octa 0xd807aa98d807aa98d807aa98d807aa98 > - .octa 0xd807aa98d807aa98d807aa98d807aa98 > - .octa 0x12835b0112835b0112835b0112835b01 > - .octa 0x12835b0112835b0112835b0112835b01 > - .octa 0x243185be243185be243185be243185be > - .octa 0x243185be243185be243185be243185be > - .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 > - .octa 0x550c7dc3550c7dc3550c7dc3550c7dc3 > - .octa 0x72be5d7472be5d7472be5d7472be5d74 > - .octa 0x72be5d7472be5d7472be5d7472be5d74 > - .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe > - .octa 0x80deb1fe80deb1fe80deb1fe80deb1fe > - .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 > - .octa 0x9bdc06a79bdc06a79bdc06a79bdc06a7 > - .octa 0xc19bf174c19bf174c19bf174c19bf174 > - .octa 0xc19bf174c19bf174c19bf174c19bf174 > - .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 > - .octa 0xe49b69c1e49b69c1e49b69c1e49b69c1 > - .octa 0xefbe4786efbe4786efbe4786efbe4786 > - .octa 0xefbe4786efbe4786efbe4786efbe4786 > - .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 > - .octa 0x0fc19dc60fc19dc60fc19dc60fc19dc6 > - .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc > - .octa 0x240ca1cc240ca1cc240ca1cc240ca1cc > - .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f > - .octa 0x2de92c6f2de92c6f2de92c6f2de92c6f > - .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa > - .octa 0x4a7484aa4a7484aa4a7484aa4a7484aa > - .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc > - .octa 0x5cb0a9dc5cb0a9dc5cb0a9dc5cb0a9dc > - .octa 0x76f988da76f988da76f988da76f988da > - .octa 0x76f988da76f988da76f988da76f988da > - .octa 0x983e5152983e5152983e5152983e5152 > - .octa 0x983e5152983e5152983e5152983e5152 > - .octa 0xa831c66da831c66da831c66da831c66d > - .octa 0xa831c66da831c66da831c66da831c66d > - .octa 0xb00327c8b00327c8b00327c8b00327c8 > - .octa 0xb00327c8b00327c8b00327c8b00327c8 > - .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 > - .octa 0xbf597fc7bf597fc7bf597fc7bf597fc7 > - .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 > - .octa 0xc6e00bf3c6e00bf3c6e00bf3c6e00bf3 > - .octa 0xd5a79147d5a79147d5a79147d5a79147 > - .octa 0xd5a79147d5a79147d5a79147d5a79147 > - .octa 0x06ca635106ca635106ca635106ca6351 > - .octa 0x06ca635106ca635106ca635106ca6351 > - .octa 0x14292967142929671429296714292967 > - .octa 0x14292967142929671429296714292967 > - .octa 0x27b70a8527b70a8527b70a8527b70a85 > - .octa 0x27b70a8527b70a8527b70a8527b70a85 > - .octa 0x2e1b21382e1b21382e1b21382e1b2138 > - .octa 0x2e1b21382e1b21382e1b21382e1b2138 > - .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc > - .octa 0x4d2c6dfc4d2c6dfc4d2c6dfc4d2c6dfc > - .octa 0x53380d1353380d1353380d1353380d13 > - .octa 0x53380d1353380d1353380d1353380d13 > - .octa 0x650a7354650a7354650a7354650a7354 > - .octa 0x650a7354650a7354650a7354650a7354 > - .octa 0x766a0abb766a0abb766a0abb766a0abb > - .octa 0x766a0abb766a0abb766a0abb766a0abb > - .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e > - .octa 0x81c2c92e81c2c92e81c2c92e81c2c92e > - .octa 0x92722c8592722c8592722c8592722c85 > - .octa 0x92722c8592722c8592722c8592722c85 > - .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 > - .octa 0xa2bfe8a1a2bfe8a1a2bfe8a1a2bfe8a1 > - .octa 0xa81a664ba81a664ba81a664ba81a664b > - .octa 0xa81a664ba81a664ba81a664ba81a664b > - .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 > - .octa 0xc24b8b70c24b8b70c24b8b70c24b8b70 > - .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 > - .octa 0xc76c51a3c76c51a3c76c51a3c76c51a3 > - .octa 0xd192e819d192e819d192e819d192e819 > - .octa 0xd192e819d192e819d192e819d192e819 > - .octa 0xd6990624d6990624d6990624d6990624 > - .octa 0xd6990624d6990624d6990624d6990624 > - .octa 0xf40e3585f40e3585f40e3585f40e3585 > - .octa 0xf40e3585f40e3585f40e3585f40e3585 > - .octa 0x106aa070106aa070106aa070106aa070 > - .octa 0x106aa070106aa070106aa070106aa070 > - .octa 0x19a4c11619a4c11619a4c11619a4c116 > - .octa 0x19a4c11619a4c11619a4c11619a4c116 > - .octa 0x1e376c081e376c081e376c081e376c08 > - .octa 0x1e376c081e376c081e376c081e376c08 > - .octa 0x2748774c2748774c2748774c2748774c > - .octa 0x2748774c2748774c2748774c2748774c > - .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 > - .octa 0x34b0bcb534b0bcb534b0bcb534b0bcb5 > - .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 > - .octa 0x391c0cb3391c0cb3391c0cb3391c0cb3 > - .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a > - .octa 0x4ed8aa4a4ed8aa4a4ed8aa4a4ed8aa4a > - .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f > - .octa 0x5b9cca4f5b9cca4f5b9cca4f5b9cca4f > - .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 > - .octa 0x682e6ff3682e6ff3682e6ff3682e6ff3 > - .octa 0x748f82ee748f82ee748f82ee748f82ee > - .octa 0x748f82ee748f82ee748f82ee748f82ee > - .octa 0x78a5636f78a5636f78a5636f78a5636f > - .octa 0x78a5636f78a5636f78a5636f78a5636f > - .octa 0x84c8781484c8781484c8781484c87814 > - .octa 0x84c8781484c8781484c8781484c87814 > - .octa 0x8cc702088cc702088cc702088cc70208 > - .octa 0x8cc702088cc702088cc702088cc70208 > - .octa 0x90befffa90befffa90befffa90befffa > - .octa 0x90befffa90befffa90befffa90befffa > - .octa 0xa4506ceba4506ceba4506ceba4506ceb > - .octa 0xa4506ceba4506ceba4506ceba4506ceb > - .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 > - .octa 0xbef9a3f7bef9a3f7bef9a3f7bef9a3f7 > - .octa 0xc67178f2c67178f2c67178f2c67178f2 > - .octa 0xc67178f2c67178f2c67178f2c67178f2 > - > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > -.align 32 > -PSHUFFLE_BYTE_FLIP_MASK: > -.octa 0x0c0d0e0f08090a0b0405060700010203 > -.octa 0x0c0d0e0f08090a0b0405060700010203 > - > -.section .rodata.cst256.K256, "aM", @progbits, 256 > -.align 64 > -.global K256 > -K256: > - .int 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 > - .int 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 > - .int 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 > - .int 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 > - .int 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc > - .int 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da > - .int 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 > - .int 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 > - .int 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 > - .int 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 > - .int 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 > - .int 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 > - .int 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 > - .int 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 > - .int 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 > - .int 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 > diff --git a/arch/x86/crypto/sha512-mb/Makefile b/arch/x86/crypto/sha512-mb/Makefile > deleted file mode 100644 > index 90f1ef69152e..000000000000 > --- a/arch/x86/crypto/sha512-mb/Makefile > +++ /dev/null > @@ -1,12 +0,0 @@ > -# SPDX-License-Identifier: GPL-2.0 > -# > -# Arch-specific CryptoAPI modules. > -# > - > -avx2_supported := $(call as-instr,vpgatherdd %ymm0$(comma)(%eax$(comma)%ymm1\ > - $(comma)4)$(comma)%ymm2,yes,no) > -ifeq ($(avx2_supported),yes) > - obj-$(CONFIG_CRYPTO_SHA512_MB) += sha512-mb.o > - sha512-mb-y := sha512_mb.o sha512_mb_mgr_flush_avx2.o \ > - sha512_mb_mgr_init_avx2.o sha512_mb_mgr_submit_avx2.o sha512_x4_avx2.o > -endif > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb.c b/arch/x86/crypto/sha512-mb/sha512_mb.c > deleted file mode 100644 > index 26b85678012d..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb.c > +++ /dev/null > @@ -1,1047 +0,0 @@ > -/* > - * Multi buffer SHA512 algorithm Glue Code > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt > - > -#include <crypto/internal/hash.h> > -#include <linux/init.h> > -#include <linux/module.h> > -#include <linux/mm.h> > -#include <linux/cryptohash.h> > -#include <linux/types.h> > -#include <linux/list.h> > -#include <crypto/scatterwalk.h> > -#include <crypto/sha.h> > -#include <crypto/mcryptd.h> > -#include <crypto/crypto_wq.h> > -#include <asm/byteorder.h> > -#include <linux/hardirq.h> > -#include <asm/fpu/api.h> > -#include "sha512_mb_ctx.h" > - > -#define FLUSH_INTERVAL 1000 /* in usec */ > - > -static struct mcryptd_alg_state sha512_mb_alg_state; > - > -struct sha512_mb_ctx { > - struct mcryptd_ahash *mcryptd_tfm; > -}; > - > -static inline struct mcryptd_hash_request_ctx > - *cast_hash_to_mcryptd_ctx(struct sha512_hash_ctx *hash_ctx) > -{ > - struct ahash_request *areq; > - > - areq = container_of((void *) hash_ctx, struct ahash_request, __ctx); > - return container_of(areq, struct mcryptd_hash_request_ctx, areq); > -} > - > -static inline struct ahash_request > - *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx) > -{ > - return container_of((void *) ctx, struct ahash_request, __ctx); > -} > - > -static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx, > - struct ahash_request *areq) > -{ > - rctx->flag = HASH_UPDATE; > -} > - > -static asmlinkage void (*sha512_job_mgr_init)(struct sha512_mb_mgr *state); > -static asmlinkage struct job_sha512* (*sha512_job_mgr_submit) > - (struct sha512_mb_mgr *state, > - struct job_sha512 *job); > -static asmlinkage struct job_sha512* (*sha512_job_mgr_flush) > - (struct sha512_mb_mgr *state); > -static asmlinkage struct job_sha512* (*sha512_job_mgr_get_comp_job) > - (struct sha512_mb_mgr *state); > - > -inline uint32_t sha512_pad(uint8_t padblock[SHA512_BLOCK_SIZE * 2], > - uint64_t total_len) > -{ > - uint32_t i = total_len & (SHA512_BLOCK_SIZE - 1); > - > - memset(&padblock[i], 0, SHA512_BLOCK_SIZE); > - padblock[i] = 0x80; > - > - i += ((SHA512_BLOCK_SIZE - 1) & > - (0 - (total_len + SHA512_PADLENGTHFIELD_SIZE + 1))) > - + 1 + SHA512_PADLENGTHFIELD_SIZE; > - > -#if SHA512_PADLENGTHFIELD_SIZE == 16 > - *((uint64_t *) &padblock[i - 16]) = 0; > -#endif > - > - *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3); > - > - /* Number of extra blocks to hash */ > - return i >> SHA512_LOG2_BLOCK_SIZE; > -} > - > -static struct sha512_hash_ctx *sha512_ctx_mgr_resubmit > - (struct sha512_ctx_mgr *mgr, struct sha512_hash_ctx *ctx) > -{ > - while (ctx) { > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Clear PROCESSING bit */ > - ctx->status = HASH_CTX_STS_COMPLETE; > - return ctx; > - } > - > - /* > - * If the extra blocks are empty, begin hashing what remains > - * in the user's buffer. > - */ > - if (ctx->partial_block_buffer_length == 0 && > - ctx->incoming_buffer_length) { > - > - const void *buffer = ctx->incoming_buffer; > - uint32_t len = ctx->incoming_buffer_length; > - uint32_t copy_len; > - > - /* > - * Only entire blocks can be hashed. > - * Copy remainder to extra blocks buffer. > - */ > - copy_len = len & (SHA512_BLOCK_SIZE-1); > - > - if (copy_len) { > - len -= copy_len; > - memcpy(ctx->partial_block_buffer, > - ((const char *) buffer + len), > - copy_len); > - ctx->partial_block_buffer_length = copy_len; > - } > - > - ctx->incoming_buffer_length = 0; > - > - /* len should be a multiple of the block size now */ > - assert((len % SHA512_BLOCK_SIZE) == 0); > - > - /* Set len to the number of blocks to be hashed */ > - len >>= SHA512_LOG2_BLOCK_SIZE; > - > - if (len) { > - > - ctx->job.buffer = (uint8_t *) buffer; > - ctx->job.len = len; > - ctx = (struct sha512_hash_ctx *) > - sha512_job_mgr_submit(&mgr->mgr, > - &ctx->job); > - continue; > - } > - } > - > - /* > - * If the extra blocks are not empty, then we are > - * either on the last block(s) or we need more > - * user input before continuing. > - */ > - if (ctx->status & HASH_CTX_STS_LAST) { > - > - uint8_t *buf = ctx->partial_block_buffer; > - uint32_t n_extra_blocks = > - sha512_pad(buf, ctx->total_length); > - > - ctx->status = (HASH_CTX_STS_PROCESSING | > - HASH_CTX_STS_COMPLETE); > - ctx->job.buffer = buf; > - ctx->job.len = (uint32_t) n_extra_blocks; > - ctx = (struct sha512_hash_ctx *) > - sha512_job_mgr_submit(&mgr->mgr, &ctx->job); > - continue; > - } > - > - if (ctx) > - ctx->status = HASH_CTX_STS_IDLE; > - return ctx; > - } > - > - return NULL; > -} > - > -static struct sha512_hash_ctx > - *sha512_ctx_mgr_get_comp_ctx(struct mcryptd_alg_cstate *cstate) > -{ > - /* > - * If get_comp_job returns NULL, there are no jobs complete. > - * If get_comp_job returns a job, verify that it is safe to return to > - * the user. > - * If it is not ready, resubmit the job to finish processing. > - * If sha512_ctx_mgr_resubmit returned a job, it is ready to be > - * returned. > - * Otherwise, all jobs currently being managed by the hash_ctx_mgr > - * still need processing. > - */ > - struct sha512_ctx_mgr *mgr; > - struct sha512_hash_ctx *ctx; > - unsigned long flags; > - > - mgr = cstate->mgr; > - spin_lock_irqsave(&cstate->work_lock, flags); > - ctx = (struct sha512_hash_ctx *) > - sha512_job_mgr_get_comp_job(&mgr->mgr); > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > - spin_unlock_irqrestore(&cstate->work_lock, flags); > - return ctx; > -} > - > -static void sha512_ctx_mgr_init(struct sha512_ctx_mgr *mgr) > -{ > - sha512_job_mgr_init(&mgr->mgr); > -} > - > -static struct sha512_hash_ctx > - *sha512_ctx_mgr_submit(struct mcryptd_alg_cstate *cstate, > - struct sha512_hash_ctx *ctx, > - const void *buffer, > - uint32_t len, > - int flags) > -{ > - struct sha512_ctx_mgr *mgr; > - unsigned long irqflags; > - > - mgr = cstate->mgr; > - spin_lock_irqsave(&cstate->work_lock, irqflags); > - if (flags & ~(HASH_UPDATE | HASH_LAST)) { > - /* User should not pass anything other than UPDATE or LAST */ > - ctx->error = HASH_CTX_ERROR_INVALID_FLAGS; > - goto unlock; > - } > - > - if (ctx->status & HASH_CTX_STS_PROCESSING) { > - /* Cannot submit to a currently processing job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING; > - goto unlock; > - } > - > - if (ctx->status & HASH_CTX_STS_COMPLETE) { > - /* Cannot update a finished job. */ > - ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED; > - goto unlock; > - } > - > - /* > - * If we made it here, there were no errors during this call to > - * submit > - */ > - ctx->error = HASH_CTX_ERROR_NONE; > - > - /* Store buffer ptr info from user */ > - ctx->incoming_buffer = buffer; > - ctx->incoming_buffer_length = len; > - > - /* > - * Store the user's request flags and mark this ctx as currently being > - * processed. > - */ > - ctx->status = (flags & HASH_LAST) ? > - (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) : > - HASH_CTX_STS_PROCESSING; > - > - /* Advance byte counter */ > - ctx->total_length += len; > - > - /* > - * If there is anything currently buffered in the extra blocks, > - * append to it until it contains a whole block. > - * Or if the user's buffer contains less than a whole block, > - * append as much as possible to the extra block. > - */ > - if (ctx->partial_block_buffer_length || len < SHA512_BLOCK_SIZE) { > - /* Compute how many bytes to copy from user buffer into extra > - * block > - */ > - uint32_t copy_len = SHA512_BLOCK_SIZE - > - ctx->partial_block_buffer_length; > - if (len < copy_len) > - copy_len = len; > - > - if (copy_len) { > - /* Copy and update relevant pointers and counters */ > - memcpy > - (&ctx->partial_block_buffer[ctx->partial_block_buffer_length], > - buffer, copy_len); > - > - ctx->partial_block_buffer_length += copy_len; > - ctx->incoming_buffer = (const void *) > - ((const char *)buffer + copy_len); > - ctx->incoming_buffer_length = len - copy_len; > - } > - > - /* The extra block should never contain more than 1 block > - * here > - */ > - assert(ctx->partial_block_buffer_length <= SHA512_BLOCK_SIZE); > - > - /* If the extra block buffer contains exactly 1 block, it can > - * be hashed. > - */ > - if (ctx->partial_block_buffer_length >= SHA512_BLOCK_SIZE) { > - ctx->partial_block_buffer_length = 0; > - > - ctx->job.buffer = ctx->partial_block_buffer; > - ctx->job.len = 1; > - ctx = (struct sha512_hash_ctx *) > - sha512_job_mgr_submit(&mgr->mgr, &ctx->job); > - } > - } > - > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > -unlock: > - spin_unlock_irqrestore(&cstate->work_lock, irqflags); > - return ctx; > -} > - > -static struct sha512_hash_ctx *sha512_ctx_mgr_flush(struct mcryptd_alg_cstate *cstate) > -{ > - struct sha512_ctx_mgr *mgr; > - struct sha512_hash_ctx *ctx; > - unsigned long flags; > - > - mgr = cstate->mgr; > - spin_lock_irqsave(&cstate->work_lock, flags); > - while (1) { > - ctx = (struct sha512_hash_ctx *) > - sha512_job_mgr_flush(&mgr->mgr); > - > - /* If flush returned 0, there are no more jobs in flight. */ > - if (!ctx) > - break; > - > - /* > - * If flush returned a job, resubmit the job to finish > - * processing. > - */ > - ctx = sha512_ctx_mgr_resubmit(mgr, ctx); > - > - /* > - * If sha512_ctx_mgr_resubmit returned a job, it is ready to > - * be returned. Otherwise, all jobs currently being managed by > - * the sha512_ctx_mgr still need processing. Loop. > - */ > - if (ctx) > - break; > - } > - spin_unlock_irqrestore(&cstate->work_lock, flags); > - return ctx; > -} > - > -static int sha512_mb_init(struct ahash_request *areq) > -{ > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > - > - hash_ctx_init(sctx); > - sctx->job.result_digest[0] = SHA512_H0; > - sctx->job.result_digest[1] = SHA512_H1; > - sctx->job.result_digest[2] = SHA512_H2; > - sctx->job.result_digest[3] = SHA512_H3; > - sctx->job.result_digest[4] = SHA512_H4; > - sctx->job.result_digest[5] = SHA512_H5; > - sctx->job.result_digest[6] = SHA512_H6; > - sctx->job.result_digest[7] = SHA512_H7; > - sctx->total_length = 0; > - sctx->partial_block_buffer_length = 0; > - sctx->status = HASH_CTX_STS_IDLE; > - > - return 0; > -} > - > -static int sha512_mb_set_results(struct mcryptd_hash_request_ctx *rctx) > -{ > - int i; > - struct sha512_hash_ctx *sctx = ahash_request_ctx(&rctx->areq); > - __be64 *dst = (__be64 *) rctx->out; > - > - for (i = 0; i < 8; ++i) > - dst[i] = cpu_to_be64(sctx->job.result_digest[i]); > - > - return 0; > -} > - > -static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx, > - struct mcryptd_alg_cstate *cstate, bool flush) > -{ > - int flag = HASH_UPDATE; > - int nbytes, err = 0; > - struct mcryptd_hash_request_ctx *rctx = *ret_rctx; > - struct sha512_hash_ctx *sha_ctx; > - > - /* more work ? */ > - while (!(rctx->flag & HASH_DONE)) { > - nbytes = crypto_ahash_walk_done(&rctx->walk, 0); > - if (nbytes < 0) { > - err = nbytes; > - goto out; > - } > - /* check if the walk is done */ > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - if (rctx->flag & HASH_FINAL) > - flag |= HASH_LAST; > - > - } > - sha_ctx = (struct sha512_hash_ctx *) > - ahash_request_ctx(&rctx->areq); > - kernel_fpu_begin(); > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, > - rctx->walk.data, nbytes, flag); > - if (!sha_ctx) { > - if (flush) > - sha_ctx = sha512_ctx_mgr_flush(cstate); > - } > - kernel_fpu_end(); > - if (sha_ctx) > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - else { > - rctx = NULL; > - goto out; > - } > - } > - > - /* copy the results */ > - if (rctx->flag & HASH_FINAL) > - sha512_mb_set_results(rctx); > - > -out: > - *ret_rctx = rctx; > - return err; > -} > - > -static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate, > - int err) > -{ > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha512_hash_ctx *sha_ctx; > - struct mcryptd_hash_request_ctx *req_ctx; > - int ret; > - unsigned long flags; > - > - /* remove from work list */ > - spin_lock_irqsave(&cstate->work_lock, flags); > - list_del(&rctx->waiter); > - spin_unlock_irqrestore(&cstate->work_lock, flags); > - > - if (irqs_disabled()) > - rctx->complete(&req->base, err); > - else { > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > - } > - > - /* check to see if there are other jobs that are done */ > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate); > - while (sha_ctx) { > - req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&req_ctx, cstate, false); > - if (req_ctx) { > - spin_lock_irqsave(&cstate->work_lock, flags); > - list_del(&req_ctx->waiter); > - spin_unlock_irqrestore(&cstate->work_lock, flags); > - > - req = cast_mcryptd_ctx_to_req(req_ctx); > - if (irqs_disabled()) > - req_ctx->complete(&req->base, ret); > - else { > - local_bh_disable(); > - req_ctx->complete(&req->base, ret); > - local_bh_enable(); > - } > - } > - sha_ctx = sha512_ctx_mgr_get_comp_ctx(cstate); > - } > - > - return 0; > -} > - > -static void sha512_mb_add_list(struct mcryptd_hash_request_ctx *rctx, > - struct mcryptd_alg_cstate *cstate) > -{ > - unsigned long next_flush; > - unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL); > - unsigned long flags; > - > - /* initialize tag */ > - rctx->tag.arrival = jiffies; /* tag the arrival time */ > - rctx->tag.seq_num = cstate->next_seq_num++; > - next_flush = rctx->tag.arrival + delay; > - rctx->tag.expire = next_flush; > - > - spin_lock_irqsave(&cstate->work_lock, flags); > - list_add_tail(&rctx->waiter, &cstate->work_list); > - spin_unlock_irqrestore(&cstate->work_lock, flags); > - > - mcryptd_arm_flusher(cstate, delay); > -} > - > -static int sha512_mb_update(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, > - areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha512_hash_ctx *sha_ctx; > - int ret = 0, nbytes; > - > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) > - rctx->flag |= HASH_DONE; > - > - /* submit */ > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > - sha512_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data, > - nbytes, HASH_UPDATE); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha512_mb_finup(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, > - areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > - > - struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx); > - struct sha512_hash_ctx *sha_ctx; > - int ret = 0, flag = HASH_UPDATE, nbytes; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - nbytes = crypto_ahash_walk_first(req, &rctx->walk); > - > - if (nbytes < 0) { > - ret = nbytes; > - goto done; > - } > - > - if (crypto_ahash_walk_last(&rctx->walk)) { > - rctx->flag |= HASH_DONE; > - flag = HASH_LAST; > - } > - > - /* submit */ > - rctx->flag |= HASH_FINAL; > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > - sha512_mb_add_list(rctx, cstate); > - > - kernel_fpu_begin(); > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, rctx->walk.data, > - nbytes, flag); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha512_mb_final(struct ahash_request *areq) > -{ > - struct mcryptd_hash_request_ctx *rctx = > - container_of(areq, struct mcryptd_hash_request_ctx, > - areq); > - struct mcryptd_alg_cstate *cstate = > - this_cpu_ptr(sha512_mb_alg_state.alg_cstate); > - > - struct sha512_hash_ctx *sha_ctx; > - int ret = 0; > - u8 data; > - > - /* sanity check */ > - if (rctx->tag.cpu != smp_processor_id()) { > - pr_err("mcryptd error: cpu clash\n"); > - goto done; > - } > - > - /* need to init context */ > - req_ctx_init(rctx, areq); > - > - rctx->flag |= HASH_DONE | HASH_FINAL; > - > - sha_ctx = (struct sha512_hash_ctx *) ahash_request_ctx(areq); > - /* flag HASH_FINAL and 0 data size */ > - sha512_mb_add_list(rctx, cstate); > - kernel_fpu_begin(); > - sha_ctx = sha512_ctx_mgr_submit(cstate, sha_ctx, &data, 0, HASH_LAST); > - kernel_fpu_end(); > - > - /* check if anything is returned */ > - if (!sha_ctx) > - return -EINPROGRESS; > - > - if (sha_ctx->error) { > - ret = sha_ctx->error; > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - goto done; > - } > - > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - ret = sha_finish_walk(&rctx, cstate, false); > - if (!rctx) > - return -EINPROGRESS; > -done: > - sha_complete_job(rctx, cstate, ret); > - return ret; > -} > - > -static int sha512_mb_export(struct ahash_request *areq, void *out) > -{ > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(out, sctx, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha512_mb_import(struct ahash_request *areq, const void *in) > -{ > - struct sha512_hash_ctx *sctx = ahash_request_ctx(areq); > - > - memcpy(sctx, in, sizeof(*sctx)); > - > - return 0; > -} > - > -static int sha512_mb_async_init_tfm(struct crypto_tfm *tfm) > -{ > - struct mcryptd_ahash *mcryptd_tfm; > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - struct mcryptd_hash_ctx *mctx; > - > - mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha512-mb", > - CRYPTO_ALG_INTERNAL, > - CRYPTO_ALG_INTERNAL); > - if (IS_ERR(mcryptd_tfm)) > - return PTR_ERR(mcryptd_tfm); > - mctx = crypto_ahash_ctx(&mcryptd_tfm->base); > - mctx->alg_state = &sha512_mb_alg_state; > - ctx->mcryptd_tfm = mcryptd_tfm; > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - crypto_ahash_reqsize(&mcryptd_tfm->base)); > - > - return 0; > -} > - > -static void sha512_mb_async_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static int sha512_mb_areq_init_tfm(struct crypto_tfm *tfm) > -{ > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct ahash_request) + > - sizeof(struct sha512_hash_ctx)); > - > - return 0; > -} > - > -static void sha512_mb_areq_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct sha512_mb_ctx *ctx = crypto_tfm_ctx(tfm); > - > - mcryptd_free_ahash(ctx->mcryptd_tfm); > -} > - > -static struct ahash_alg sha512_mb_areq_alg = { > - .init = sha512_mb_init, > - .update = sha512_mb_update, > - .final = sha512_mb_final, > - .finup = sha512_mb_finup, > - .export = sha512_mb_export, > - .import = sha512_mb_import, > - .halg = { > - .digestsize = SHA512_DIGEST_SIZE, > - .statesize = sizeof(struct sha512_hash_ctx), > - .base = { > - .cra_name = "__sha512-mb", > - .cra_driver_name = "__intel_sha512-mb", > - .cra_priority = 100, > - /* > - * use ASYNC flag as some buffers in multi-buffer > - * algo may not have completed before hashing thread > - * sleep > - */ > - .cra_flags = CRYPTO_ALG_ASYNC | > - CRYPTO_ALG_INTERNAL, > - .cra_blocksize = SHA512_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT > - (sha512_mb_areq_alg.halg.base.cra_list), > - .cra_init = sha512_mb_areq_init_tfm, > - .cra_exit = sha512_mb_areq_exit_tfm, > - .cra_ctxsize = sizeof(struct sha512_hash_ctx), > - } > - } > -}; > - > -static int sha512_mb_async_init(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_init(mcryptd_req); > -} > - > -static int sha512_mb_async_update(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_update(mcryptd_req); > -} > - > -static int sha512_mb_async_finup(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_finup(mcryptd_req); > -} > - > -static int sha512_mb_async_final(struct ahash_request *req) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_final(mcryptd_req); > -} > - > -static int sha512_mb_async_digest(struct ahash_request *req) > -{ > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_digest(mcryptd_req); > -} > - > -static int sha512_mb_async_export(struct ahash_request *req, void *out) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - return crypto_ahash_export(mcryptd_req, out); > -} > - > -static int sha512_mb_async_import(struct ahash_request *req, const void *in) > -{ > - struct ahash_request *mcryptd_req = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct sha512_mb_ctx *ctx = crypto_ahash_ctx(tfm); > - struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm; > - struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm); > - struct mcryptd_hash_request_ctx *rctx; > - struct ahash_request *areq; > - > - memcpy(mcryptd_req, req, sizeof(*req)); > - ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base); > - rctx = ahash_request_ctx(mcryptd_req); > - > - areq = &rctx->areq; > - > - ahash_request_set_tfm(areq, child); > - ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP, > - rctx->complete, req); > - > - return crypto_ahash_import(mcryptd_req, in); > -} > - > -static struct ahash_alg sha512_mb_async_alg = { > - .init = sha512_mb_async_init, > - .update = sha512_mb_async_update, > - .final = sha512_mb_async_final, > - .finup = sha512_mb_async_finup, > - .digest = sha512_mb_async_digest, > - .export = sha512_mb_async_export, > - .import = sha512_mb_async_import, > - .halg = { > - .digestsize = SHA512_DIGEST_SIZE, > - .statesize = sizeof(struct sha512_hash_ctx), > - .base = { > - .cra_name = "sha512", > - .cra_driver_name = "sha512_mb", > - /* > - * Low priority, since with few concurrent hash requests > - * this is extremely slow due to the flush delay. Users > - * whose workloads would benefit from this can request > - * it explicitly by driver name, or can increase its > - * priority at runtime using NETLINK_CRYPTO. > - */ > - .cra_priority = 50, > - .cra_flags = CRYPTO_ALG_ASYNC, > - .cra_blocksize = SHA512_BLOCK_SIZE, > - .cra_module = THIS_MODULE, > - .cra_list = LIST_HEAD_INIT > - (sha512_mb_async_alg.halg.base.cra_list), > - .cra_init = sha512_mb_async_init_tfm, > - .cra_exit = sha512_mb_async_exit_tfm, > - .cra_ctxsize = sizeof(struct sha512_mb_ctx), > - .cra_alignmask = 0, > - }, > - }, > -}; > - > -static unsigned long sha512_mb_flusher(struct mcryptd_alg_cstate *cstate) > -{ > - struct mcryptd_hash_request_ctx *rctx; > - unsigned long cur_time; > - unsigned long next_flush = 0; > - struct sha512_hash_ctx *sha_ctx; > - > - > - cur_time = jiffies; > - > - while (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - if time_before(cur_time, rctx->tag.expire) > - break; > - kernel_fpu_begin(); > - sha_ctx = (struct sha512_hash_ctx *) > - sha512_ctx_mgr_flush(cstate); > - kernel_fpu_end(); > - if (!sha_ctx) { > - pr_err("sha512_mb error: nothing got flushed for" > - " non-empty list\n"); > - break; > - } > - rctx = cast_hash_to_mcryptd_ctx(sha_ctx); > - sha_finish_walk(&rctx, cstate, true); > - sha_complete_job(rctx, cstate, 0); > - } > - > - if (!list_empty(&cstate->work_list)) { > - rctx = list_entry(cstate->work_list.next, > - struct mcryptd_hash_request_ctx, waiter); > - /* get the hash context and then flush time */ > - next_flush = rctx->tag.expire; > - mcryptd_arm_flusher(cstate, get_delay(next_flush)); > - } > - return next_flush; > -} > - > -static int __init sha512_mb_mod_init(void) > -{ > - > - int cpu; > - int err; > - struct mcryptd_alg_cstate *cpu_state; > - > - /* check for dependent cpu features */ > - if (!boot_cpu_has(X86_FEATURE_AVX2) || > - !boot_cpu_has(X86_FEATURE_BMI2)) > - return -ENODEV; > - > - /* initialize multibuffer structures */ > - sha512_mb_alg_state.alg_cstate = > - alloc_percpu(struct mcryptd_alg_cstate); > - > - sha512_job_mgr_init = sha512_mb_mgr_init_avx2; > - sha512_job_mgr_submit = sha512_mb_mgr_submit_avx2; > - sha512_job_mgr_flush = sha512_mb_mgr_flush_avx2; > - sha512_job_mgr_get_comp_job = sha512_mb_mgr_get_comp_job_avx2; > - > - if (!sha512_mb_alg_state.alg_cstate) > - return -ENOMEM; > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > - cpu_state->next_flush = 0; > - cpu_state->next_seq_num = 0; > - cpu_state->flusher_engaged = false; > - INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher); > - cpu_state->cpu = cpu; > - cpu_state->alg_state = &sha512_mb_alg_state; > - cpu_state->mgr = kzalloc(sizeof(struct sha512_ctx_mgr), > - GFP_KERNEL); > - if (!cpu_state->mgr) > - goto err2; > - sha512_ctx_mgr_init(cpu_state->mgr); > - INIT_LIST_HEAD(&cpu_state->work_list); > - spin_lock_init(&cpu_state->work_lock); > - } > - sha512_mb_alg_state.flusher = &sha512_mb_flusher; > - > - err = crypto_register_ahash(&sha512_mb_areq_alg); > - if (err) > - goto err2; > - err = crypto_register_ahash(&sha512_mb_async_alg); > - if (err) > - goto err1; > - > - > - return 0; > -err1: > - crypto_unregister_ahash(&sha512_mb_areq_alg); > -err2: > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha512_mb_alg_state.alg_cstate); > - return -ENODEV; > -} > - > -static void __exit sha512_mb_mod_fini(void) > -{ > - int cpu; > - struct mcryptd_alg_cstate *cpu_state; > - > - crypto_unregister_ahash(&sha512_mb_async_alg); > - crypto_unregister_ahash(&sha512_mb_areq_alg); > - for_each_possible_cpu(cpu) { > - cpu_state = per_cpu_ptr(sha512_mb_alg_state.alg_cstate, cpu); > - kfree(cpu_state->mgr); > - } > - free_percpu(sha512_mb_alg_state.alg_cstate); > -} > - > -module_init(sha512_mb_mod_init); > -module_exit(sha512_mb_mod_fini); > - > -MODULE_LICENSE("GPL"); > -MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, multi buffer accelerated"); > - > -MODULE_ALIAS("sha512"); > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h b/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > deleted file mode 100644 > index e5c465bd821e..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_ctx.h > +++ /dev/null > @@ -1,128 +0,0 @@ > -/* > - * Header file for multi buffer SHA512 context > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#ifndef _SHA_MB_CTX_INTERNAL_H > -#define _SHA_MB_CTX_INTERNAL_H > - > -#include "sha512_mb_mgr.h" > - > -#define HASH_UPDATE 0x00 > -#define HASH_LAST 0x01 > -#define HASH_DONE 0x02 > -#define HASH_FINAL 0x04 > - > -#define HASH_CTX_STS_IDLE 0x00 > -#define HASH_CTX_STS_PROCESSING 0x01 > -#define HASH_CTX_STS_LAST 0x02 > -#define HASH_CTX_STS_COMPLETE 0x04 > - > -enum hash_ctx_error { > - HASH_CTX_ERROR_NONE = 0, > - HASH_CTX_ERROR_INVALID_FLAGS = -1, > - HASH_CTX_ERROR_ALREADY_PROCESSING = -2, > - HASH_CTX_ERROR_ALREADY_COMPLETED = -3, > -}; > - > -#define hash_ctx_user_data(ctx) ((ctx)->user_data) > -#define hash_ctx_digest(ctx) ((ctx)->job.result_digest) > -#define hash_ctx_processing(ctx) ((ctx)->status & HASH_CTX_STS_PROCESSING) > -#define hash_ctx_complete(ctx) ((ctx)->status == HASH_CTX_STS_COMPLETE) > -#define hash_ctx_status(ctx) ((ctx)->status) > -#define hash_ctx_error(ctx) ((ctx)->error) > -#define hash_ctx_init(ctx) \ > - do { \ > - (ctx)->error = HASH_CTX_ERROR_NONE; \ > - (ctx)->status = HASH_CTX_STS_COMPLETE; \ > - } while (0) > - > -/* Hash Constants and Typedefs */ > -#define SHA512_DIGEST_LENGTH 8 > -#define SHA512_LOG2_BLOCK_SIZE 7 > - > -#define SHA512_PADLENGTHFIELD_SIZE 16 > - > -#ifdef SHA_MB_DEBUG > -#define assert(expr) \ > -do { \ > - if (unlikely(!(expr))) { \ > - printk(KERN_ERR "Assertion failed! %s,%s,%s,line=%d\n", \ > - #expr, __FILE__, __func__, __LINE__); \ > - } \ > -} while (0) > -#else > -#define assert(expr) do {} while (0) > -#endif > - > -struct sha512_ctx_mgr { > - struct sha512_mb_mgr mgr; > -}; > - > -/* typedef struct sha512_ctx_mgr sha512_ctx_mgr; */ > - > -struct sha512_hash_ctx { > - /* Must be at struct offset 0 */ > - struct job_sha512 job; > - /* status flag */ > - int status; > - /* error flag */ > - int error; > - > - uint64_t total_length; > - const void *incoming_buffer; > - uint32_t incoming_buffer_length; > - uint8_t partial_block_buffer[SHA512_BLOCK_SIZE * 2]; > - uint32_t partial_block_buffer_length; > - void *user_data; > -}; > - > -#endif > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h b/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > deleted file mode 100644 > index 178f17eef382..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr.h > +++ /dev/null > @@ -1,104 +0,0 @@ > -/* > - * Header file for multi buffer SHA512 algorithm manager > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#ifndef __SHA_MB_MGR_H > -#define __SHA_MB_MGR_H > - > -#include <linux/types.h> > - > -#define NUM_SHA512_DIGEST_WORDS 8 > - > -enum job_sts {STS_UNKNOWN = 0, > - STS_BEING_PROCESSED = 1, > - STS_COMPLETED = 2, > - STS_INTERNAL_ERROR = 3, > - STS_ERROR = 4 > -}; > - > -struct job_sha512 { > - u8 *buffer; > - u64 len; > - u64 result_digest[NUM_SHA512_DIGEST_WORDS] __aligned(32); > - enum job_sts status; > - void *user_data; > -}; > - > -struct sha512_args_x4 { > - uint64_t digest[8][4]; > - uint8_t *data_ptr[4]; > -}; > - > -struct sha512_lane_data { > - struct job_sha512 *job_in_lane; > -}; > - > -struct sha512_mb_mgr { > - struct sha512_args_x4 args; > - > - uint64_t lens[4]; > - > - /* each byte is index (0...7) of unused lanes */ > - uint64_t unused_lanes; > - /* byte 4 is set to FF as a flag */ > - struct sha512_lane_data ldata[4]; > -}; > - > -#define SHA512_MB_MGR_NUM_LANES_AVX2 4 > - > -void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state); > -struct job_sha512 *sha512_mb_mgr_submit_avx2(struct sha512_mb_mgr *state, > - struct job_sha512 *job); > -struct job_sha512 *sha512_mb_mgr_flush_avx2(struct sha512_mb_mgr *state); > -struct job_sha512 *sha512_mb_mgr_get_comp_job_avx2(struct sha512_mb_mgr *state); > - > -#endif > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > deleted file mode 100644 > index cf2636d4c9ba..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_datastruct.S > +++ /dev/null > @@ -1,281 +0,0 @@ > -/* > - * Header file for multi buffer SHA256 algorithm data structure > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -# Macros for defining data structures > - > -# Usage example > - > -#START_FIELDS # JOB_AES > -### name size align > -#FIELD _plaintext, 8, 8 # pointer to plaintext > -#FIELD _ciphertext, 8, 8 # pointer to ciphertext > -#FIELD _IV, 16, 8 # IV > -#FIELD _keys, 8, 8 # pointer to keys > -#FIELD _len, 4, 4 # length in bytes > -#FIELD _status, 4, 4 # status enumeration > -#FIELD _user_data, 8, 8 # pointer to user data > -#UNION _union, size1, align1, \ > -# size2, align2, \ > -# size3, align3, \ > -# ... > -#END_FIELDS > -#%assign _JOB_AES_size _FIELD_OFFSET > -#%assign _JOB_AES_align _STRUCT_ALIGN > - > -######################################################################### > - > -# Alternate "struc-like" syntax: > -# STRUCT job_aes2 > -# RES_Q .plaintext, 1 > -# RES_Q .ciphertext, 1 > -# RES_DQ .IV, 1 > -# RES_B .nested, _JOB_AES_SIZE, _JOB_AES_ALIGN > -# RES_U .union, size1, align1, \ > -# size2, align2, \ > -# ... > -# ENDSTRUCT > -# # Following only needed if nesting > -# %assign job_aes2_size _FIELD_OFFSET > -# %assign job_aes2_align _STRUCT_ALIGN > -# > -# RES_* macros take a name, a count and an optional alignment. > -# The count in in terms of the base size of the macro, and the > -# default alignment is the base size. > -# The macros are: > -# Macro Base size > -# RES_B 1 > -# RES_W 2 > -# RES_D 4 > -# RES_Q 8 > -# RES_DQ 16 > -# RES_Y 32 > -# RES_Z 64 > -# > -# RES_U defines a union. It's arguments are a name and two or more > -# pairs of "size, alignment" > -# > -# The two assigns are only needed if this structure is being nested > -# within another. Even if the assigns are not done, one can still use > -# STRUCT_NAME_size as the size of the structure. > -# > -# Note that for nesting, you still need to assign to STRUCT_NAME_size. > -# > -# The differences between this and using "struc" directly are that each > -# type is implicitly aligned to its natural length (although this can be > -# over-ridden with an explicit third parameter), and that the structure > -# is padded at the end to its overall alignment. > -# > - > -######################################################################### > - > -#ifndef _DATASTRUCT_ASM_ > -#define _DATASTRUCT_ASM_ > - > -#define PTR_SZ 8 > -#define SHA512_DIGEST_WORD_SIZE 8 > -#define SHA512_MB_MGR_NUM_LANES_AVX2 4 > -#define NUM_SHA512_DIGEST_WORDS 8 > -#define SZ4 4*SHA512_DIGEST_WORD_SIZE > -#define ROUNDS 80*SZ4 > -#define SHA512_DIGEST_ROW_SIZE (SHA512_MB_MGR_NUM_LANES_AVX2 * 8) > - > -# START_FIELDS > -.macro START_FIELDS > - _FIELD_OFFSET = 0 > - _STRUCT_ALIGN = 0 > -.endm > - > -# FIELD name size align > -.macro FIELD name size align > - _FIELD_OFFSET = (_FIELD_OFFSET + (\align) - 1) & (~ ((\align)-1)) > - \name = _FIELD_OFFSET > - _FIELD_OFFSET = _FIELD_OFFSET + (\size) > -.if (\align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = \align > -.endif > -.endm > - > -# END_FIELDS > -.macro END_FIELDS > - _FIELD_OFFSET = (_FIELD_OFFSET + _STRUCT_ALIGN-1) & (~ (_STRUCT_ALIGN-1)) > -.endm > - > -.macro STRUCT p1 > -START_FIELDS > -.struc \p1 > -.endm > - > -.macro ENDSTRUCT > - tmp = _FIELD_OFFSET > - END_FIELDS > - tmp = (_FIELD_OFFSET - ##tmp) > -.if (tmp > 0) > - .lcomm tmp > -.endm > - > -## RES_int name size align > -.macro RES_int p1 p2 p3 > - name = \p1 > - size = \p2 > - align = .\p3 > - > - _FIELD_OFFSET = (_FIELD_OFFSET + (align) - 1) & (~ ((align)-1)) > -.align align > -.lcomm name size > - _FIELD_OFFSET = _FIELD_OFFSET + (size) > -.if (align > _STRUCT_ALIGN) > - _STRUCT_ALIGN = align > -.endif > -.endm > - > -# macro RES_B name, size [, align] > -.macro RES_B _name, _size, _align=1 > -RES_int _name _size _align > -.endm > - > -# macro RES_W name, size [, align] > -.macro RES_W _name, _size, _align=2 > -RES_int _name 2*(_size) _align > -.endm > - > -# macro RES_D name, size [, align] > -.macro RES_D _name, _size, _align=4 > -RES_int _name 4*(_size) _align > -.endm > - > -# macro RES_Q name, size [, align] > -.macro RES_Q _name, _size, _align=8 > -RES_int _name 8*(_size) _align > -.endm > - > -# macro RES_DQ name, size [, align] > -.macro RES_DQ _name, _size, _align=16 > -RES_int _name 16*(_size) _align > -.endm > - > -# macro RES_Y name, size [, align] > -.macro RES_Y _name, _size, _align=32 > -RES_int _name 32*(_size) _align > -.endm > - > -# macro RES_Z name, size [, align] > -.macro RES_Z _name, _size, _align=64 > -RES_int _name 64*(_size) _align > -.endm > - > -#endif > - > -################################################################### > -### Define SHA512 Out Of Order Data Structures > -################################################################### > - > -START_FIELDS # LANE_DATA > -### name size align > -FIELD _job_in_lane, 8, 8 # pointer to job object > -END_FIELDS > - > - _LANE_DATA_size = _FIELD_OFFSET > - _LANE_DATA_align = _STRUCT_ALIGN > - > -#################################################################### > - > -START_FIELDS # SHA512_ARGS_X4 > -### name size align > -FIELD _digest, 8*8*4, 4 # transposed digest > -FIELD _data_ptr, 8*4, 8 # array of pointers to data > -END_FIELDS > - > - _SHA512_ARGS_X4_size = _FIELD_OFFSET > - _SHA512_ARGS_X4_align = _STRUCT_ALIGN > - > -##################################################################### > - > -START_FIELDS # MB_MGR > -### name size align > -FIELD _args, _SHA512_ARGS_X4_size, _SHA512_ARGS_X4_align > -FIELD _lens, 8*4, 8 > -FIELD _unused_lanes, 8, 8 > -FIELD _ldata, _LANE_DATA_size*4, _LANE_DATA_align > -END_FIELDS > - > - _MB_MGR_size = _FIELD_OFFSET > - _MB_MGR_align = _STRUCT_ALIGN > - > -_args_digest = _args + _digest > -_args_data_ptr = _args + _data_ptr > - > -####################################################################### > - > -####################################################################### > -#### Define constants > -####################################################################### > - > -#define STS_UNKNOWN 0 > -#define STS_BEING_PROCESSED 1 > -#define STS_COMPLETED 2 > - > -####################################################################### > -#### Define JOB_SHA512 structure > -####################################################################### > - > -START_FIELDS # JOB_SHA512 > -### name size align > -FIELD _buffer, 8, 8 # pointer to buffer > -FIELD _len, 8, 8 # length in bytes > -FIELD _result_digest, 8*8, 32 # Digest (output) > -FIELD _status, 4, 4 > -FIELD _user_data, 8, 8 > -END_FIELDS > - > - _JOB_SHA512_size = _FIELD_OFFSET > - _JOB_SHA512_align = _STRUCT_ALIGN > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > deleted file mode 100644 > index 7c629caebc05..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_flush_avx2.S > +++ /dev/null > @@ -1,297 +0,0 @@ > -/* > - * Flush routine for SHA512 multibuffer > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha512_mb_mgr_datastruct.S" > - > -.extern sha512_x4_avx2 > - > -# LINUX register definitions > -#define arg1 %rdi > -#define arg2 %rsi > - > -# idx needs to be other than arg1, arg2, rbx, r12 > -#define idx %rdx > - > -# Common definitions > -#define state arg1 > -#define job arg2 > -#define len2 arg2 > - > -#define unused_lanes %rbx > -#define lane_data %rbx > -#define tmp2 %rbx > - > -#define job_rax %rax > -#define tmp1 %rax > -#define size_offset %rax > -#define tmp %rax > -#define start_offset %rax > - > -#define tmp3 arg1 > - > -#define extra_blocks arg2 > -#define p arg2 > - > -#define tmp4 %r8 > -#define lens0 %r8 > - > -#define lens1 %r9 > -#define lens2 %r10 > -#define lens3 %r11 > - > -.macro LABEL prefix n > -\prefix\n\(): > -.endm > - > -.macro JNE_SKIP i > -jne skip_\i > -.endm > - > -.altmacro > -.macro SET_OFFSET _offset > -offset = \_offset > -.endm > -.noaltmacro > - > -# JOB* sha512_mb_mgr_flush_avx2(MB_MGR *state) > -# arg 1 : rcx : state > -ENTRY(sha512_mb_mgr_flush_avx2) > - FRAME_BEGIN > - push %rbx > - > - # If bit (32+3) is set, then all lanes are empty > - mov _unused_lanes(state), unused_lanes > - bt $32+7, unused_lanes > - jc return_null > - > - # find a lane with a non-null job > - xor idx, idx > - offset = (_ldata + 1*_LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne one(%rip), idx > - offset = (_ldata + 2*_LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne two(%rip), idx > - offset = (_ldata + 3*_LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > - cmovne three(%rip), idx > - > - # copy idx to empty lanes > -copy_lane_data: > - offset = (_args + _data_ptr) > - mov offset(state,idx,8), tmp > - > - I = 0 > -.rep 4 > - offset = (_ldata + I * _LANE_DATA_size + _job_in_lane) > - cmpq $0, offset(state) > -.altmacro > - JNE_SKIP %I > - offset = (_args + _data_ptr + 8*I) > - mov tmp, offset(state) > - offset = (_lens + 8*I +4) > - movl $0xFFFFFFFF, offset(state) > -LABEL skip_ %I > - I = (I+1) > -.noaltmacro > -.endr > - > - # Find min length > - mov _lens + 0*8(state),lens0 > - mov lens0,idx > - mov _lens + 1*8(state),lens1 > - cmp idx,lens1 > - cmovb lens1,idx > - mov _lens + 2*8(state),lens2 > - cmp idx,lens2 > - cmovb lens2,idx > - mov _lens + 3*8(state),lens3 > - cmp idx,lens3 > - cmovb lens3,idx > - mov idx,len2 > - and $0xF,idx > - and $~0xFF,len2 > - jz len_is_0 > - > - sub len2, lens0 > - sub len2, lens1 > - sub len2, lens2 > - sub len2, lens3 > - shr $32,len2 > - mov lens0, _lens + 0*8(state) > - mov lens1, _lens + 1*8(state) > - mov lens2, _lens + 2*8(state) > - mov lens3, _lens + 3*8(state) > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha512_x4_avx2 > - # state and idx are intact > - > -len_is_0: > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $8, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens+4(state, idx, 8) > - > - vmovq _args_digest+0*32(state, idx, 8), %xmm0 > - vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 > - vmovq _args_digest+2*32(state, idx, 8), %xmm1 > - vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 > - vmovq _args_digest+4*32(state, idx, 8), %xmm2 > - vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 > - vmovq _args_digest+6*32(state, idx, 8), %xmm3 > - vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 > - > - vmovdqu %xmm0, _result_digest(job_rax) > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > - vmovdqu %xmm2, _result_digest+2*16(job_rax) > - vmovdqu %xmm3, _result_digest+3*16(job_rax) > - > -return: > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > -ENDPROC(sha512_mb_mgr_flush_avx2) > -.align 16 > - > -ENTRY(sha512_mb_mgr_get_comp_job_avx2) > - push %rbx > - > - mov _unused_lanes(state), unused_lanes > - bt $(32+7), unused_lanes > - jc .return_null > - > - # Find min length > - mov _lens(state),lens0 > - mov lens0,idx > - mov _lens+1*8(state),lens1 > - cmp idx,lens1 > - cmovb lens1,idx > - mov _lens+2*8(state),lens2 > - cmp idx,lens2 > - cmovb lens2,idx > - mov _lens+3*8(state),lens3 > - cmp idx,lens3 > - cmovb lens3,idx > - test $~0xF,idx > - jnz .return_null > - and $0xF,idx > - > - #process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - mov _unused_lanes(state), unused_lanes > - shl $8, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF, _lens+4(state, idx, 8) > - > - vmovq _args_digest(state, idx, 8), %xmm0 > - vpinsrq $1, _args_digest+1*32(state, idx, 8), %xmm0, %xmm0 > - vmovq _args_digest+2*32(state, idx, 8), %xmm1 > - vpinsrq $1, _args_digest+3*32(state, idx, 8), %xmm1, %xmm1 > - vmovq _args_digest+4*32(state, idx, 8), %xmm2 > - vpinsrq $1, _args_digest+5*32(state, idx, 8), %xmm2, %xmm2 > - vmovq _args_digest+6*32(state, idx, 8), %xmm3 > - vpinsrq $1, _args_digest+7*32(state, idx, 8), %xmm3, %xmm3 > - > - vmovdqu %xmm0, _result_digest+0*16(job_rax) > - vmovdqu %xmm1, _result_digest+1*16(job_rax) > - vmovdqu %xmm2, _result_digest+2*16(job_rax) > - vmovdqu %xmm3, _result_digest+3*16(job_rax) > - > - pop %rbx > - > - ret > - > -.return_null: > - xor job_rax, job_rax > - pop %rbx > - ret > -ENDPROC(sha512_mb_mgr_get_comp_job_avx2) > - > -.section .rodata.cst8.one, "aM", @progbits, 8 > -.align 8 > -one: > -.quad 1 > - > -.section .rodata.cst8.two, "aM", @progbits, 8 > -.align 8 > -two: > -.quad 2 > - > -.section .rodata.cst8.three, "aM", @progbits, 8 > -.align 8 > -three: > -.quad 3 > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > deleted file mode 100644 > index d08805032f01..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_init_avx2.c > +++ /dev/null > @@ -1,69 +0,0 @@ > -/* > - * Initialization code for multi buffer SHA256 algorithm for AVX2 > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include "sha512_mb_mgr.h" > - > -void sha512_mb_mgr_init_avx2(struct sha512_mb_mgr *state) > -{ > - unsigned int j; > - > - /* initially all lanes are unused */ > - state->lens[0] = 0xFFFFFFFF00000000; > - state->lens[1] = 0xFFFFFFFF00000001; > - state->lens[2] = 0xFFFFFFFF00000002; > - state->lens[3] = 0xFFFFFFFF00000003; > - > - state->unused_lanes = 0xFF03020100; > - for (j = 0; j < 4; j++) > - state->ldata[j].job_in_lane = NULL; > -} > diff --git a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S b/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > deleted file mode 100644 > index 4ba709ba78e5..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_mb_mgr_submit_avx2.S > +++ /dev/null > @@ -1,224 +0,0 @@ > -/* > - * Buffer submit code for multi buffer SHA512 algorithm > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -#include <linux/linkage.h> > -#include <asm/frame.h> > -#include "sha512_mb_mgr_datastruct.S" > - > -.extern sha512_x4_avx2 > - > -#define arg1 %rdi > -#define arg2 %rsi > - > -#define idx %rdx > -#define last_len %rdx > - > -#define size_offset %rcx > -#define tmp2 %rcx > - > -# Common definitions > -#define state arg1 > -#define job arg2 > -#define len2 arg2 > -#define p2 arg2 > - > -#define p %r11 > -#define start_offset %r11 > - > -#define unused_lanes %rbx > - > -#define job_rax %rax > -#define len %rax > - > -#define lane %r12 > -#define tmp3 %r12 > -#define lens3 %r12 > - > -#define extra_blocks %r8 > -#define lens0 %r8 > - > -#define tmp %r9 > -#define lens1 %r9 > - > -#define lane_data %r10 > -#define lens2 %r10 > - > -#define DWORD_len %eax > - > -# JOB* sha512_mb_mgr_submit_avx2(MB_MGR *state, JOB *job) > -# arg 1 : rcx : state > -# arg 2 : rdx : job > -ENTRY(sha512_mb_mgr_submit_avx2) > - FRAME_BEGIN > - push %rbx > - push %r12 > - > - mov _unused_lanes(state), unused_lanes > - movzb %bl,lane > - shr $8, unused_lanes > - imul $_LANE_DATA_size, lane,lane_data > - movl $STS_BEING_PROCESSED, _status(job) > - lea _ldata(state, lane_data), lane_data > - mov unused_lanes, _unused_lanes(state) > - movl _len(job), DWORD_len > - > - mov job, _job_in_lane(lane_data) > - movl DWORD_len,_lens+4(state , lane, 8) > - > - # Load digest words from result_digest > - vmovdqu _result_digest+0*16(job), %xmm0 > - vmovdqu _result_digest+1*16(job), %xmm1 > - vmovdqu _result_digest+2*16(job), %xmm2 > - vmovdqu _result_digest+3*16(job), %xmm3 > - > - vmovq %xmm0, _args_digest(state, lane, 8) > - vpextrq $1, %xmm0, _args_digest+1*32(state , lane, 8) > - vmovq %xmm1, _args_digest+2*32(state , lane, 8) > - vpextrq $1, %xmm1, _args_digest+3*32(state , lane, 8) > - vmovq %xmm2, _args_digest+4*32(state , lane, 8) > - vpextrq $1, %xmm2, _args_digest+5*32(state , lane, 8) > - vmovq %xmm3, _args_digest+6*32(state , lane, 8) > - vpextrq $1, %xmm3, _args_digest+7*32(state , lane, 8) > - > - mov _buffer(job), p > - mov p, _args_data_ptr(state, lane, 8) > - > - cmp $0xFF, unused_lanes > - jne return_null > - > -start_loop: > - > - # Find min length > - mov _lens+0*8(state),lens0 > - mov lens0,idx > - mov _lens+1*8(state),lens1 > - cmp idx,lens1 > - cmovb lens1, idx > - mov _lens+2*8(state),lens2 > - cmp idx,lens2 > - cmovb lens2,idx > - mov _lens+3*8(state),lens3 > - cmp idx,lens3 > - cmovb lens3,idx > - mov idx,len2 > - and $0xF,idx > - and $~0xFF,len2 > - jz len_is_0 > - > - sub len2,lens0 > - sub len2,lens1 > - sub len2,lens2 > - sub len2,lens3 > - shr $32,len2 > - mov lens0, _lens + 0*8(state) > - mov lens1, _lens + 1*8(state) > - mov lens2, _lens + 2*8(state) > - mov lens3, _lens + 3*8(state) > - > - # "state" and "args" are the same address, arg1 > - # len is arg2 > - call sha512_x4_avx2 > - # state and idx are intact > - > -len_is_0: > - > - # process completed job "idx" > - imul $_LANE_DATA_size, idx, lane_data > - lea _ldata(state, lane_data), lane_data > - > - mov _job_in_lane(lane_data), job_rax > - mov _unused_lanes(state), unused_lanes > - movq $0, _job_in_lane(lane_data) > - movl $STS_COMPLETED, _status(job_rax) > - shl $8, unused_lanes > - or idx, unused_lanes > - mov unused_lanes, _unused_lanes(state) > - > - movl $0xFFFFFFFF,_lens+4(state,idx,8) > - vmovq _args_digest+0*32(state , idx, 8), %xmm0 > - vpinsrq $1, _args_digest+1*32(state , idx, 8), %xmm0, %xmm0 > - vmovq _args_digest+2*32(state , idx, 8), %xmm1 > - vpinsrq $1, _args_digest+3*32(state , idx, 8), %xmm1, %xmm1 > - vmovq _args_digest+4*32(state , idx, 8), %xmm2 > - vpinsrq $1, _args_digest+5*32(state , idx, 8), %xmm2, %xmm2 > - vmovq _args_digest+6*32(state , idx, 8), %xmm3 > - vpinsrq $1, _args_digest+7*32(state , idx, 8), %xmm3, %xmm3 > - > - vmovdqu %xmm0, _result_digest + 0*16(job_rax) > - vmovdqu %xmm1, _result_digest + 1*16(job_rax) > - vmovdqu %xmm2, _result_digest + 2*16(job_rax) > - vmovdqu %xmm3, _result_digest + 3*16(job_rax) > - > -return: > - pop %r12 > - pop %rbx > - FRAME_END > - ret > - > -return_null: > - xor job_rax, job_rax > - jmp return > -ENDPROC(sha512_mb_mgr_submit_avx2) > - > -/* UNUSED? > -.section .rodata.cst16, "aM", @progbits, 16 > -.align 16 > -H0: .int 0x6a09e667 > -H1: .int 0xbb67ae85 > -H2: .int 0x3c6ef372 > -H3: .int 0xa54ff53a > -H4: .int 0x510e527f > -H5: .int 0x9b05688c > -H6: .int 0x1f83d9ab > -H7: .int 0x5be0cd19 > -*/ > diff --git a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S b/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > deleted file mode 100644 > index e22e907643a6..000000000000 > --- a/arch/x86/crypto/sha512-mb/sha512_x4_avx2.S > +++ /dev/null > @@ -1,531 +0,0 @@ > -/* > - * Multi-buffer SHA512 algorithm hash compute routine > - * > - * This file is provided under a dual BSD/GPLv2 license. When using or > - * redistributing this file, you may do so under either license. > - * > - * GPL LICENSE SUMMARY > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * This program is free software; you can redistribute it and/or modify > - * it under the terms of version 2 of the GNU General Public License as > - * published by the Free Software Foundation. > - * > - * This program is distributed in the hope that it will be useful, but > - * WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * General Public License for more details. > - * > - * Contact Information: > - * Megha Dey <megha.dey@xxxxxxxxxxxxxxx> > - * > - * BSD LICENSE > - * > - * Copyright(c) 2016 Intel Corporation. > - * > - * Redistribution and use in source and binary forms, with or without > - * modification, are permitted provided that the following conditions > - * are met: > - * > - * * Redistributions of source code must retain the above copyright > - * notice, this list of conditions and the following disclaimer. > - * * Redistributions in binary form must reproduce the above copyright > - * notice, this list of conditions and the following disclaimer in > - * the documentation and/or other materials provided with the > - * distribution. > - * * Neither the name of Intel Corporation nor the names of its > - * contributors may be used to endorse or promote products derived > - * from this software without specific prior written permission. > - * > - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS > - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT > - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR > - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT > - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, > - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT > - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, > - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY > - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT > - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE > - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. > - */ > - > -# code to compute quad SHA512 using AVX2 > -# use YMMs to tackle the larger digest size > -# outer calling routine takes care of save and restore of XMM registers > -# Logic designed/laid out by JDG > - > -# Function clobbers: rax, rcx, rdx, rbx, rsi, rdi, r9-r15; ymm0-15 > -# Stack must be aligned to 32 bytes before call > -# Linux clobbers: rax rbx rcx rsi r8 r9 r10 r11 r12 > -# Linux preserves: rcx rdx rdi rbp r13 r14 r15 > -# clobbers ymm0-15 > - > -#include <linux/linkage.h> > -#include "sha512_mb_mgr_datastruct.S" > - > -arg1 = %rdi > -arg2 = %rsi > - > -# Common definitions > -STATE = arg1 > -INP_SIZE = arg2 > - > -IDX = %rax > -ROUND = %rbx > -TBL = %r8 > - > -inp0 = %r9 > -inp1 = %r10 > -inp2 = %r11 > -inp3 = %r12 > - > -a = %ymm0 > -b = %ymm1 > -c = %ymm2 > -d = %ymm3 > -e = %ymm4 > -f = %ymm5 > -g = %ymm6 > -h = %ymm7 > - > -a0 = %ymm8 > -a1 = %ymm9 > -a2 = %ymm10 > - > -TT0 = %ymm14 > -TT1 = %ymm13 > -TT2 = %ymm12 > -TT3 = %ymm11 > -TT4 = %ymm10 > -TT5 = %ymm9 > - > -T1 = %ymm14 > -TMP = %ymm15 > - > -# Define stack usage > -STACK_SPACE1 = SZ4*16 + NUM_SHA512_DIGEST_WORDS*SZ4 + 24 > - > -#define VMOVPD vmovupd > -_digest = SZ4*16 > - > -# transpose r0, r1, r2, r3, t0, t1 > -# "transpose" data in {r0..r3} using temps {t0..t3} > -# Input looks like: {r0 r1 r2 r3} > -# r0 = {a7 a6 a5 a4 a3 a2 a1 a0} > -# r1 = {b7 b6 b5 b4 b3 b2 b1 b0} > -# r2 = {c7 c6 c5 c4 c3 c2 c1 c0} > -# r3 = {d7 d6 d5 d4 d3 d2 d1 d0} > -# > -# output looks like: {t0 r1 r0 r3} > -# t0 = {d1 d0 c1 c0 b1 b0 a1 a0} > -# r1 = {d3 d2 c3 c2 b3 b2 a3 a2} > -# r0 = {d5 d4 c5 c4 b5 b4 a5 a4} > -# r3 = {d7 d6 c7 c6 b7 b6 a7 a6} > - > -.macro TRANSPOSE r0 r1 r2 r3 t0 t1 > - vshufps $0x44, \r1, \r0, \t0 # t0 = {b5 b4 a5 a4 b1 b0 a1 a0} > - vshufps $0xEE, \r1, \r0, \r0 # r0 = {b7 b6 a7 a6 b3 b2 a3 a2} > - vshufps $0x44, \r3, \r2, \t1 # t1 = {d5 d4 c5 c4 d1 d0 c1 c0} > - vshufps $0xEE, \r3, \r2, \r2 # r2 = {d7 d6 c7 c6 d3 d2 c3 c2} > - > - vperm2f128 $0x20, \r2, \r0, \r1 # h6...a6 > - vperm2f128 $0x31, \r2, \r0, \r3 # h2...a2 > - vperm2f128 $0x31, \t1, \t0, \r0 # h5...a5 > - vperm2f128 $0x20, \t1, \t0, \t0 # h1...a1 > -.endm > - > -.macro ROTATE_ARGS > -TMP_ = h > -h = g > -g = f > -f = e > -e = d > -d = c > -c = b > -b = a > -a = TMP_ > -.endm > - > -# PRORQ reg, imm, tmp > -# packed-rotate-right-double > -# does a rotate by doing two shifts and an or > -.macro _PRORQ reg imm tmp > - vpsllq $(64-\imm),\reg,\tmp > - vpsrlq $\imm,\reg, \reg > - vpor \tmp,\reg, \reg > -.endm > - > -# non-destructive > -# PRORQ_nd reg, imm, tmp, src > -.macro _PRORQ_nd reg imm tmp src > - vpsllq $(64-\imm), \src, \tmp > - vpsrlq $\imm, \src, \reg > - vpor \tmp, \reg, \reg > -.endm > - > -# PRORQ dst/src, amt > -.macro PRORQ reg imm > - _PRORQ \reg, \imm, TMP > -.endm > - > -# PRORQ_nd dst, src, amt > -.macro PRORQ_nd reg tmp imm > - _PRORQ_nd \reg, \imm, TMP, \tmp > -.endm > - > -#; arguments passed implicitly in preprocessor symbols i, a...h > -.macro ROUND_00_15 _T1 i > - PRORQ_nd a0, e, (18-14) # sig1: a0 = (e >> 4) > - > - vpxor g, f, a2 # ch: a2 = f^g > - vpand e,a2, a2 # ch: a2 = (f^g)&e > - vpxor g, a2, a2 # a2 = ch > - > - PRORQ_nd a1,e,41 # sig1: a1 = (e >> 25) > - > - offset = SZ4*(\i & 0xf) > - vmovdqu \_T1,offset(%rsp) > - vpaddq (TBL,ROUND,1), \_T1, \_T1 # T1 = W + K > - vpxor e,a0, a0 # sig1: a0 = e ^ (e >> 5) > - PRORQ a0, 14 # sig1: a0 = (e >> 6) ^ (e >> 11) > - vpaddq a2, h, h # h = h + ch > - PRORQ_nd a2,a,6 # sig0: a2 = (a >> 11) > - vpaddq \_T1,h, h # h = h + ch + W + K > - vpxor a1, a0, a0 # a0 = sigma1 > - vmovdqu a,\_T1 > - PRORQ_nd a1,a,39 # sig0: a1 = (a >> 22) > - vpxor c, \_T1, \_T1 # maj: T1 = a^c > - add $SZ4, ROUND # ROUND++ > - vpand b, \_T1, \_T1 # maj: T1 = (a^c)&b > - vpaddq a0, h, h > - vpaddq h, d, d > - vpxor a, a2, a2 # sig0: a2 = a ^ (a >> 11) > - PRORQ a2,28 # sig0: a2 = (a >> 2) ^ (a >> 13) > - vpxor a1, a2, a2 # a2 = sig0 > - vpand c, a, a1 # maj: a1 = a&c > - vpor \_T1, a1, a1 # a1 = maj > - vpaddq a1, h, h # h = h + ch + W + K + maj > - vpaddq a2, h, h # h = h + ch + W + K + maj + sigma0 > - ROTATE_ARGS > -.endm > - > - > -#; arguments passed implicitly in preprocessor symbols i, a...h > -.macro ROUND_16_XX _T1 i > - vmovdqu SZ4*((\i-15)&0xf)(%rsp), \_T1 > - vmovdqu SZ4*((\i-2)&0xf)(%rsp), a1 > - vmovdqu \_T1, a0 > - PRORQ \_T1,7 > - vmovdqu a1, a2 > - PRORQ a1,42 > - vpxor a0, \_T1, \_T1 > - PRORQ \_T1, 1 > - vpxor a2, a1, a1 > - PRORQ a1, 19 > - vpsrlq $7, a0, a0 > - vpxor a0, \_T1, \_T1 > - vpsrlq $6, a2, a2 > - vpxor a2, a1, a1 > - vpaddq SZ4*((\i-16)&0xf)(%rsp), \_T1, \_T1 > - vpaddq SZ4*((\i-7)&0xf)(%rsp), a1, a1 > - vpaddq a1, \_T1, \_T1 > - > - ROUND_00_15 \_T1,\i > -.endm > - > - > -# void sha512_x4_avx2(void *STATE, const int INP_SIZE) > -# arg 1 : STATE : pointer to input data > -# arg 2 : INP_SIZE : size of data in blocks (assumed >= 1) > -ENTRY(sha512_x4_avx2) > - # general registers preserved in outer calling routine > - # outer calling routine saves all the XMM registers > - # save callee-saved clobbered registers to comply with C function ABI > - push %r12 > - push %r13 > - push %r14 > - push %r15 > - > - sub $STACK_SPACE1, %rsp > - > - # Load the pre-transposed incoming digest. > - vmovdqu 0*SHA512_DIGEST_ROW_SIZE(STATE),a > - vmovdqu 1*SHA512_DIGEST_ROW_SIZE(STATE),b > - vmovdqu 2*SHA512_DIGEST_ROW_SIZE(STATE),c > - vmovdqu 3*SHA512_DIGEST_ROW_SIZE(STATE),d > - vmovdqu 4*SHA512_DIGEST_ROW_SIZE(STATE),e > - vmovdqu 5*SHA512_DIGEST_ROW_SIZE(STATE),f > - vmovdqu 6*SHA512_DIGEST_ROW_SIZE(STATE),g > - vmovdqu 7*SHA512_DIGEST_ROW_SIZE(STATE),h > - > - lea K512_4(%rip),TBL > - > - # load the address of each of the 4 message lanes > - # getting ready to transpose input onto stack > - mov _data_ptr+0*PTR_SZ(STATE),inp0 > - mov _data_ptr+1*PTR_SZ(STATE),inp1 > - mov _data_ptr+2*PTR_SZ(STATE),inp2 > - mov _data_ptr+3*PTR_SZ(STATE),inp3 > - > - xor IDX, IDX > -lloop: > - xor ROUND, ROUND > - > - # save old digest > - vmovdqu a, _digest(%rsp) > - vmovdqu b, _digest+1*SZ4(%rsp) > - vmovdqu c, _digest+2*SZ4(%rsp) > - vmovdqu d, _digest+3*SZ4(%rsp) > - vmovdqu e, _digest+4*SZ4(%rsp) > - vmovdqu f, _digest+5*SZ4(%rsp) > - vmovdqu g, _digest+6*SZ4(%rsp) > - vmovdqu h, _digest+7*SZ4(%rsp) > - i = 0 > -.rep 4 > - vmovdqu PSHUFFLE_BYTE_FLIP_MASK(%rip), TMP > - VMOVPD i*32(inp0, IDX), TT2 > - VMOVPD i*32(inp1, IDX), TT1 > - VMOVPD i*32(inp2, IDX), TT4 > - VMOVPD i*32(inp3, IDX), TT3 > - TRANSPOSE TT2, TT1, TT4, TT3, TT0, TT5 > - vpshufb TMP, TT0, TT0 > - vpshufb TMP, TT1, TT1 > - vpshufb TMP, TT2, TT2 > - vpshufb TMP, TT3, TT3 > - ROUND_00_15 TT0,(i*4+0) > - ROUND_00_15 TT1,(i*4+1) > - ROUND_00_15 TT2,(i*4+2) > - ROUND_00_15 TT3,(i*4+3) > - i = (i+1) > -.endr > - add $128, IDX > - > - i = (i*4) > - > - jmp Lrounds_16_xx > -.align 16 > -Lrounds_16_xx: > -.rep 16 > - ROUND_16_XX T1, i > - i = (i+1) > -.endr > - cmp $0xa00,ROUND > - jb Lrounds_16_xx > - > - # add old digest > - vpaddq _digest(%rsp), a, a > - vpaddq _digest+1*SZ4(%rsp), b, b > - vpaddq _digest+2*SZ4(%rsp), c, c > - vpaddq _digest+3*SZ4(%rsp), d, d > - vpaddq _digest+4*SZ4(%rsp), e, e > - vpaddq _digest+5*SZ4(%rsp), f, f > - vpaddq _digest+6*SZ4(%rsp), g, g > - vpaddq _digest+7*SZ4(%rsp), h, h > - > - sub $1, INP_SIZE # unit is blocks > - jne lloop > - > - # write back to memory (state object) the transposed digest > - vmovdqu a, 0*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu b, 1*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu c, 2*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu d, 3*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu e, 4*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu f, 5*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu g, 6*SHA512_DIGEST_ROW_SIZE(STATE) > - vmovdqu h, 7*SHA512_DIGEST_ROW_SIZE(STATE) > - > - # update input data pointers > - add IDX, inp0 > - mov inp0, _data_ptr+0*PTR_SZ(STATE) > - add IDX, inp1 > - mov inp1, _data_ptr+1*PTR_SZ(STATE) > - add IDX, inp2 > - mov inp2, _data_ptr+2*PTR_SZ(STATE) > - add IDX, inp3 > - mov inp3, _data_ptr+3*PTR_SZ(STATE) > - > - #;;;;;;;;;;;;;;; > - #; Postamble > - add $STACK_SPACE1, %rsp > - # restore callee-saved clobbered registers > - > - pop %r15 > - pop %r14 > - pop %r13 > - pop %r12 > - > - # outer calling routine restores XMM and other GP registers > - ret > -ENDPROC(sha512_x4_avx2) > - > -.section .rodata.K512_4, "a", @progbits > -.align 64 > -K512_4: > - .octa 0x428a2f98d728ae22428a2f98d728ae22,\ > - 0x428a2f98d728ae22428a2f98d728ae22 > - .octa 0x7137449123ef65cd7137449123ef65cd,\ > - 0x7137449123ef65cd7137449123ef65cd > - .octa 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f,\ > - 0xb5c0fbcfec4d3b2fb5c0fbcfec4d3b2f > - .octa 0xe9b5dba58189dbbce9b5dba58189dbbc,\ > - 0xe9b5dba58189dbbce9b5dba58189dbbc > - .octa 0x3956c25bf348b5383956c25bf348b538,\ > - 0x3956c25bf348b5383956c25bf348b538 > - .octa 0x59f111f1b605d01959f111f1b605d019,\ > - 0x59f111f1b605d01959f111f1b605d019 > - .octa 0x923f82a4af194f9b923f82a4af194f9b,\ > - 0x923f82a4af194f9b923f82a4af194f9b > - .octa 0xab1c5ed5da6d8118ab1c5ed5da6d8118,\ > - 0xab1c5ed5da6d8118ab1c5ed5da6d8118 > - .octa 0xd807aa98a3030242d807aa98a3030242,\ > - 0xd807aa98a3030242d807aa98a3030242 > - .octa 0x12835b0145706fbe12835b0145706fbe,\ > - 0x12835b0145706fbe12835b0145706fbe > - .octa 0x243185be4ee4b28c243185be4ee4b28c,\ > - 0x243185be4ee4b28c243185be4ee4b28c > - .octa 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2,\ > - 0x550c7dc3d5ffb4e2550c7dc3d5ffb4e2 > - .octa 0x72be5d74f27b896f72be5d74f27b896f,\ > - 0x72be5d74f27b896f72be5d74f27b896f > - .octa 0x80deb1fe3b1696b180deb1fe3b1696b1,\ > - 0x80deb1fe3b1696b180deb1fe3b1696b1 > - .octa 0x9bdc06a725c712359bdc06a725c71235,\ > - 0x9bdc06a725c712359bdc06a725c71235 > - .octa 0xc19bf174cf692694c19bf174cf692694,\ > - 0xc19bf174cf692694c19bf174cf692694 > - .octa 0xe49b69c19ef14ad2e49b69c19ef14ad2,\ > - 0xe49b69c19ef14ad2e49b69c19ef14ad2 > - .octa 0xefbe4786384f25e3efbe4786384f25e3,\ > - 0xefbe4786384f25e3efbe4786384f25e3 > - .octa 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5,\ > - 0x0fc19dc68b8cd5b50fc19dc68b8cd5b5 > - .octa 0x240ca1cc77ac9c65240ca1cc77ac9c65,\ > - 0x240ca1cc77ac9c65240ca1cc77ac9c65 > - .octa 0x2de92c6f592b02752de92c6f592b0275,\ > - 0x2de92c6f592b02752de92c6f592b0275 > - .octa 0x4a7484aa6ea6e4834a7484aa6ea6e483,\ > - 0x4a7484aa6ea6e4834a7484aa6ea6e483 > - .octa 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4,\ > - 0x5cb0a9dcbd41fbd45cb0a9dcbd41fbd4 > - .octa 0x76f988da831153b576f988da831153b5,\ > - 0x76f988da831153b576f988da831153b5 > - .octa 0x983e5152ee66dfab983e5152ee66dfab,\ > - 0x983e5152ee66dfab983e5152ee66dfab > - .octa 0xa831c66d2db43210a831c66d2db43210,\ > - 0xa831c66d2db43210a831c66d2db43210 > - .octa 0xb00327c898fb213fb00327c898fb213f,\ > - 0xb00327c898fb213fb00327c898fb213f > - .octa 0xbf597fc7beef0ee4bf597fc7beef0ee4,\ > - 0xbf597fc7beef0ee4bf597fc7beef0ee4 > - .octa 0xc6e00bf33da88fc2c6e00bf33da88fc2,\ > - 0xc6e00bf33da88fc2c6e00bf33da88fc2 > - .octa 0xd5a79147930aa725d5a79147930aa725,\ > - 0xd5a79147930aa725d5a79147930aa725 > - .octa 0x06ca6351e003826f06ca6351e003826f,\ > - 0x06ca6351e003826f06ca6351e003826f > - .octa 0x142929670a0e6e70142929670a0e6e70,\ > - 0x142929670a0e6e70142929670a0e6e70 > - .octa 0x27b70a8546d22ffc27b70a8546d22ffc,\ > - 0x27b70a8546d22ffc27b70a8546d22ffc > - .octa 0x2e1b21385c26c9262e1b21385c26c926,\ > - 0x2e1b21385c26c9262e1b21385c26c926 > - .octa 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed,\ > - 0x4d2c6dfc5ac42aed4d2c6dfc5ac42aed > - .octa 0x53380d139d95b3df53380d139d95b3df,\ > - 0x53380d139d95b3df53380d139d95b3df > - .octa 0x650a73548baf63de650a73548baf63de,\ > - 0x650a73548baf63de650a73548baf63de > - .octa 0x766a0abb3c77b2a8766a0abb3c77b2a8,\ > - 0x766a0abb3c77b2a8766a0abb3c77b2a8 > - .octa 0x81c2c92e47edaee681c2c92e47edaee6,\ > - 0x81c2c92e47edaee681c2c92e47edaee6 > - .octa 0x92722c851482353b92722c851482353b,\ > - 0x92722c851482353b92722c851482353b > - .octa 0xa2bfe8a14cf10364a2bfe8a14cf10364,\ > - 0xa2bfe8a14cf10364a2bfe8a14cf10364 > - .octa 0xa81a664bbc423001a81a664bbc423001,\ > - 0xa81a664bbc423001a81a664bbc423001 > - .octa 0xc24b8b70d0f89791c24b8b70d0f89791,\ > - 0xc24b8b70d0f89791c24b8b70d0f89791 > - .octa 0xc76c51a30654be30c76c51a30654be30,\ > - 0xc76c51a30654be30c76c51a30654be30 > - .octa 0xd192e819d6ef5218d192e819d6ef5218,\ > - 0xd192e819d6ef5218d192e819d6ef5218 > - .octa 0xd69906245565a910d69906245565a910,\ > - 0xd69906245565a910d69906245565a910 > - .octa 0xf40e35855771202af40e35855771202a,\ > - 0xf40e35855771202af40e35855771202a > - .octa 0x106aa07032bbd1b8106aa07032bbd1b8,\ > - 0x106aa07032bbd1b8106aa07032bbd1b8 > - .octa 0x19a4c116b8d2d0c819a4c116b8d2d0c8,\ > - 0x19a4c116b8d2d0c819a4c116b8d2d0c8 > - .octa 0x1e376c085141ab531e376c085141ab53,\ > - 0x1e376c085141ab531e376c085141ab53 > - .octa 0x2748774cdf8eeb992748774cdf8eeb99,\ > - 0x2748774cdf8eeb992748774cdf8eeb99 > - .octa 0x34b0bcb5e19b48a834b0bcb5e19b48a8,\ > - 0x34b0bcb5e19b48a834b0bcb5e19b48a8 > - .octa 0x391c0cb3c5c95a63391c0cb3c5c95a63,\ > - 0x391c0cb3c5c95a63391c0cb3c5c95a63 > - .octa 0x4ed8aa4ae3418acb4ed8aa4ae3418acb,\ > - 0x4ed8aa4ae3418acb4ed8aa4ae3418acb > - .octa 0x5b9cca4f7763e3735b9cca4f7763e373,\ > - 0x5b9cca4f7763e3735b9cca4f7763e373 > - .octa 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3,\ > - 0x682e6ff3d6b2b8a3682e6ff3d6b2b8a3 > - .octa 0x748f82ee5defb2fc748f82ee5defb2fc,\ > - 0x748f82ee5defb2fc748f82ee5defb2fc > - .octa 0x78a5636f43172f6078a5636f43172f60,\ > - 0x78a5636f43172f6078a5636f43172f60 > - .octa 0x84c87814a1f0ab7284c87814a1f0ab72,\ > - 0x84c87814a1f0ab7284c87814a1f0ab72 > - .octa 0x8cc702081a6439ec8cc702081a6439ec,\ > - 0x8cc702081a6439ec8cc702081a6439ec > - .octa 0x90befffa23631e2890befffa23631e28,\ > - 0x90befffa23631e2890befffa23631e28 > - .octa 0xa4506cebde82bde9a4506cebde82bde9,\ > - 0xa4506cebde82bde9a4506cebde82bde9 > - .octa 0xbef9a3f7b2c67915bef9a3f7b2c67915,\ > - 0xbef9a3f7b2c67915bef9a3f7b2c67915 > - .octa 0xc67178f2e372532bc67178f2e372532b,\ > - 0xc67178f2e372532bc67178f2e372532b > - .octa 0xca273eceea26619cca273eceea26619c,\ > - 0xca273eceea26619cca273eceea26619c > - .octa 0xd186b8c721c0c207d186b8c721c0c207,\ > - 0xd186b8c721c0c207d186b8c721c0c207 > - .octa 0xeada7dd6cde0eb1eeada7dd6cde0eb1e,\ > - 0xeada7dd6cde0eb1eeada7dd6cde0eb1e > - .octa 0xf57d4f7fee6ed178f57d4f7fee6ed178,\ > - 0xf57d4f7fee6ed178f57d4f7fee6ed178 > - .octa 0x06f067aa72176fba06f067aa72176fba,\ > - 0x06f067aa72176fba06f067aa72176fba > - .octa 0x0a637dc5a2c898a60a637dc5a2c898a6,\ > - 0x0a637dc5a2c898a60a637dc5a2c898a6 > - .octa 0x113f9804bef90dae113f9804bef90dae,\ > - 0x113f9804bef90dae113f9804bef90dae > - .octa 0x1b710b35131c471b1b710b35131c471b,\ > - 0x1b710b35131c471b1b710b35131c471b > - .octa 0x28db77f523047d8428db77f523047d84,\ > - 0x28db77f523047d8428db77f523047d84 > - .octa 0x32caab7b40c7249332caab7b40c72493,\ > - 0x32caab7b40c7249332caab7b40c72493 > - .octa 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc,\ > - 0x3c9ebe0a15c9bebc3c9ebe0a15c9bebc > - .octa 0x431d67c49c100d4c431d67c49c100d4c,\ > - 0x431d67c49c100d4c431d67c49c100d4c > - .octa 0x4cc5d4becb3e42b64cc5d4becb3e42b6,\ > - 0x4cc5d4becb3e42b64cc5d4becb3e42b6 > - .octa 0x597f299cfc657e2a597f299cfc657e2a,\ > - 0x597f299cfc657e2a597f299cfc657e2a > - .octa 0x5fcb6fab3ad6faec5fcb6fab3ad6faec,\ > - 0x5fcb6fab3ad6faec5fcb6fab3ad6faec > - .octa 0x6c44198c4a4758176c44198c4a475817,\ > - 0x6c44198c4a4758176c44198c4a475817 > - > -.section .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32 > -.align 32 > -PSHUFFLE_BYTE_FLIP_MASK: .octa 0x08090a0b0c0d0e0f0001020304050607 > - .octa 0x18191a1b1c1d1e1f1011121314151617 > diff --git a/crypto/Kconfig b/crypto/Kconfig > index f3e40ac56d93..4ee600bdefdb 100644 > --- a/crypto/Kconfig > +++ b/crypto/Kconfig > @@ -213,20 +213,6 @@ config CRYPTO_CRYPTD > converts an arbitrary synchronous software crypto algorithm > into an asynchronous algorithm that executes in a kernel thread. > > -config CRYPTO_MCRYPTD > - tristate "Software async multi-buffer crypto daemon" > - select CRYPTO_BLKCIPHER > - select CRYPTO_HASH > - select CRYPTO_MANAGER > - select CRYPTO_WORKQUEUE > - help > - This is a generic software asynchronous crypto daemon that > - provides the kernel thread to assist multi-buffer crypto > - algorithms for submitting jobs and flushing jobs in multi-buffer > - crypto algorithms. Multi-buffer crypto algorithms are executed > - in the context of this kernel thread and drivers can post > - their crypto request asynchronously to be processed by this daemon. > - > config CRYPTO_AUTHENC > tristate "Authenc support" > select CRYPTO_AEAD > @@ -848,54 +834,6 @@ config CRYPTO_SHA1_PPC_SPE > SHA-1 secure hash standard (DFIPS 180-4) implemented > using powerpc SPE SIMD instruction set. > > -config CRYPTO_SHA1_MB > - tristate "SHA1 digest algorithm (x86_64 Multi-Buffer, Experimental)" > - depends on X86 && 64BIT > - select CRYPTO_SHA1 > - select CRYPTO_HASH > - select CRYPTO_MCRYPTD > - help > - SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > - using multi-buffer technique. This algorithm computes on > - multiple data lanes concurrently with SIMD instructions for > - better throughput. It should not be enabled by default but > - used when there is significant amount of work to keep the keep > - the data lanes filled to get performance benefit. If the data > - lanes remain unfilled, a flush operation will be initiated to > - process the crypto jobs, adding a slight latency. > - > -config CRYPTO_SHA256_MB > - tristate "SHA256 digest algorithm (x86_64 Multi-Buffer, Experimental)" > - depends on X86 && 64BIT > - select CRYPTO_SHA256 > - select CRYPTO_HASH > - select CRYPTO_MCRYPTD > - help > - SHA-256 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > - using multi-buffer technique. This algorithm computes on > - multiple data lanes concurrently with SIMD instructions for > - better throughput. It should not be enabled by default but > - used when there is significant amount of work to keep the keep > - the data lanes filled to get performance benefit. If the data > - lanes remain unfilled, a flush operation will be initiated to > - process the crypto jobs, adding a slight latency. > - > -config CRYPTO_SHA512_MB > - tristate "SHA512 digest algorithm (x86_64 Multi-Buffer, Experimental)" > - depends on X86 && 64BIT > - select CRYPTO_SHA512 > - select CRYPTO_HASH > - select CRYPTO_MCRYPTD > - help > - SHA-512 secure hash standard (FIPS 180-1/DFIPS 180-2) implemented > - using multi-buffer technique. This algorithm computes on > - multiple data lanes concurrently with SIMD instructions for > - better throughput. It should not be enabled by default but > - used when there is significant amount of work to keep the keep > - the data lanes filled to get performance benefit. If the data > - lanes remain unfilled, a flush operation will be initiated to > - process the crypto jobs, adding a slight latency. > - > config CRYPTO_SHA256 > tristate "SHA224 and SHA256 digest algorithm" > select CRYPTO_HASH > diff --git a/crypto/Makefile b/crypto/Makefile > index 6d1d40eeb964..80e3da755cbf 100644 > --- a/crypto/Makefile > +++ b/crypto/Makefile > @@ -93,7 +93,6 @@ obj-$(CONFIG_CRYPTO_MORUS640) += morus640.o > obj-$(CONFIG_CRYPTO_MORUS1280) += morus1280.o > obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o > obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o > -obj-$(CONFIG_CRYPTO_MCRYPTD) += mcryptd.o > obj-$(CONFIG_CRYPTO_DES) += des_generic.o > obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o > obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish_generic.o > diff --git a/crypto/mcryptd.c b/crypto/mcryptd.c > deleted file mode 100644 > index f14152147ce8..000000000000 > --- a/crypto/mcryptd.c > +++ /dev/null > @@ -1,675 +0,0 @@ > -/* > - * Software multibuffer async crypto daemon. > - * > - * Copyright (c) 2014 Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * Adapted from crypto daemon. > - * > - * This program is free software; you can redistribute it and/or modify it > - * under the terms of the GNU General Public License as published by the Free > - * Software Foundation; either version 2 of the License, or (at your option) > - * any later version. > - * > - */ > - > -#include <crypto/algapi.h> > -#include <crypto/internal/hash.h> > -#include <crypto/internal/aead.h> > -#include <crypto/mcryptd.h> > -#include <crypto/crypto_wq.h> > -#include <linux/err.h> > -#include <linux/init.h> > -#include <linux/kernel.h> > -#include <linux/list.h> > -#include <linux/module.h> > -#include <linux/scatterlist.h> > -#include <linux/sched.h> > -#include <linux/sched/stat.h> > -#include <linux/slab.h> > - > -#define MCRYPTD_MAX_CPU_QLEN 100 > -#define MCRYPTD_BATCH 9 > - > -static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, > - unsigned int tail); > - > -struct mcryptd_flush_list { > - struct list_head list; > - struct mutex lock; > -}; > - > -static struct mcryptd_flush_list __percpu *mcryptd_flist; > - > -struct hashd_instance_ctx { > - struct crypto_ahash_spawn spawn; > - struct mcryptd_queue *queue; > -}; > - > -static void mcryptd_queue_worker(struct work_struct *work); > - > -void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay) > -{ > - struct mcryptd_flush_list *flist; > - > - if (!cstate->flusher_engaged) { > - /* put the flusher on the flush list */ > - flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); > - mutex_lock(&flist->lock); > - list_add_tail(&cstate->flush_list, &flist->list); > - cstate->flusher_engaged = true; > - cstate->next_flush = jiffies + delay; > - queue_delayed_work_on(smp_processor_id(), kcrypto_wq, > - &cstate->flush, delay); > - mutex_unlock(&flist->lock); > - } > -} > -EXPORT_SYMBOL(mcryptd_arm_flusher); > - > -static int mcryptd_init_queue(struct mcryptd_queue *queue, > - unsigned int max_cpu_qlen) > -{ > - int cpu; > - struct mcryptd_cpu_queue *cpu_queue; > - > - queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue); > - pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue); > - if (!queue->cpu_queue) > - return -ENOMEM; > - for_each_possible_cpu(cpu) { > - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); > - pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue); > - crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); > - INIT_WORK(&cpu_queue->work, mcryptd_queue_worker); > - spin_lock_init(&cpu_queue->q_lock); > - } > - return 0; > -} > - > -static void mcryptd_fini_queue(struct mcryptd_queue *queue) > -{ > - int cpu; > - struct mcryptd_cpu_queue *cpu_queue; > - > - for_each_possible_cpu(cpu) { > - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); > - BUG_ON(cpu_queue->queue.qlen); > - } > - free_percpu(queue->cpu_queue); > -} > - > -static int mcryptd_enqueue_request(struct mcryptd_queue *queue, > - struct crypto_async_request *request, > - struct mcryptd_hash_request_ctx *rctx) > -{ > - int cpu, err; > - struct mcryptd_cpu_queue *cpu_queue; > - > - cpu_queue = raw_cpu_ptr(queue->cpu_queue); > - spin_lock(&cpu_queue->q_lock); > - cpu = smp_processor_id(); > - rctx->tag.cpu = smp_processor_id(); > - > - err = crypto_enqueue_request(&cpu_queue->queue, request); > - pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n", > - cpu, cpu_queue, request); > - spin_unlock(&cpu_queue->q_lock); > - queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); > - > - return err; > -} > - > -/* > - * Try to opportunisticlly flush the partially completed jobs if > - * crypto daemon is the only task running. > - */ > -static void mcryptd_opportunistic_flush(void) > -{ > - struct mcryptd_flush_list *flist; > - struct mcryptd_alg_cstate *cstate; > - > - flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); > - while (single_task_running()) { > - mutex_lock(&flist->lock); > - cstate = list_first_entry_or_null(&flist->list, > - struct mcryptd_alg_cstate, flush_list); > - if (!cstate || !cstate->flusher_engaged) { > - mutex_unlock(&flist->lock); > - return; > - } > - list_del(&cstate->flush_list); > - cstate->flusher_engaged = false; > - mutex_unlock(&flist->lock); > - cstate->alg_state->flusher(cstate); > - } > -} > - > -/* > - * Called in workqueue context, do one real cryption work (via > - * req->complete) and reschedule itself if there are more work to > - * do. > - */ > -static void mcryptd_queue_worker(struct work_struct *work) > -{ > - struct mcryptd_cpu_queue *cpu_queue; > - struct crypto_async_request *req, *backlog; > - int i; > - > - /* > - * Need to loop through more than once for multi-buffer to > - * be effective. > - */ > - > - cpu_queue = container_of(work, struct mcryptd_cpu_queue, work); > - i = 0; > - while (i < MCRYPTD_BATCH || single_task_running()) { > - > - spin_lock_bh(&cpu_queue->q_lock); > - backlog = crypto_get_backlog(&cpu_queue->queue); > - req = crypto_dequeue_request(&cpu_queue->queue); > - spin_unlock_bh(&cpu_queue->q_lock); > - > - if (!req) { > - mcryptd_opportunistic_flush(); > - return; > - } > - > - if (backlog) > - backlog->complete(backlog, -EINPROGRESS); > - req->complete(req, 0); > - if (!cpu_queue->queue.qlen) > - return; > - ++i; > - } > - if (cpu_queue->queue.qlen) > - queue_work_on(smp_processor_id(), kcrypto_wq, &cpu_queue->work); > -} > - > -void mcryptd_flusher(struct work_struct *__work) > -{ > - struct mcryptd_alg_cstate *alg_cpu_state; > - struct mcryptd_alg_state *alg_state; > - struct mcryptd_flush_list *flist; > - int cpu; > - > - cpu = smp_processor_id(); > - alg_cpu_state = container_of(to_delayed_work(__work), > - struct mcryptd_alg_cstate, flush); > - alg_state = alg_cpu_state->alg_state; > - if (alg_cpu_state->cpu != cpu) > - pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n", > - cpu, alg_cpu_state->cpu); > - > - if (alg_cpu_state->flusher_engaged) { > - flist = per_cpu_ptr(mcryptd_flist, cpu); > - mutex_lock(&flist->lock); > - list_del(&alg_cpu_state->flush_list); > - alg_cpu_state->flusher_engaged = false; > - mutex_unlock(&flist->lock); > - alg_state->flusher(alg_cpu_state); > - } > -} > -EXPORT_SYMBOL_GPL(mcryptd_flusher); > - > -static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm) > -{ > - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); > - struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst); > - > - return ictx->queue; > -} > - > -static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, > - unsigned int tail) > -{ > - char *p; > - struct crypto_instance *inst; > - int err; > - > - p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); > - if (!p) > - return ERR_PTR(-ENOMEM); > - > - inst = (void *)(p + head); > - > - err = -ENAMETOOLONG; > - if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, > - "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) > - goto out_free_inst; > - > - memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); > - > - inst->alg.cra_priority = alg->cra_priority + 50; > - inst->alg.cra_blocksize = alg->cra_blocksize; > - inst->alg.cra_alignmask = alg->cra_alignmask; > - > -out: > - return p; > - > -out_free_inst: > - kfree(p); > - p = ERR_PTR(err); > - goto out; > -} > - > -static inline bool mcryptd_check_internal(struct rtattr **tb, u32 *type, > - u32 *mask) > -{ > - struct crypto_attr_type *algt; > - > - algt = crypto_get_attr_type(tb); > - if (IS_ERR(algt)) > - return false; > - > - *type |= algt->type & CRYPTO_ALG_INTERNAL; > - *mask |= algt->mask & CRYPTO_ALG_INTERNAL; > - > - if (*type & *mask & CRYPTO_ALG_INTERNAL) > - return true; > - else > - return false; > -} > - > -static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) > -{ > - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); > - struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); > - struct crypto_ahash_spawn *spawn = &ictx->spawn; > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); > - struct crypto_ahash *hash; > - > - hash = crypto_spawn_ahash(spawn); > - if (IS_ERR(hash)) > - return PTR_ERR(hash); > - > - ctx->child = hash; > - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), > - sizeof(struct mcryptd_hash_request_ctx) + > - crypto_ahash_reqsize(hash)); > - return 0; > -} > - > -static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) > -{ > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); > - > - crypto_free_ahash(ctx->child); > -} > - > -static int mcryptd_hash_setkey(struct crypto_ahash *parent, > - const u8 *key, unsigned int keylen) > -{ > - struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); > - struct crypto_ahash *child = ctx->child; > - int err; > - > - crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); > - crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) & > - CRYPTO_TFM_REQ_MASK); > - err = crypto_ahash_setkey(child, key, keylen); > - crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) & > - CRYPTO_TFM_RES_MASK); > - return err; > -} > - > -static int mcryptd_hash_enqueue(struct ahash_request *req, > - crypto_completion_t complete) > -{ > - int ret; > - > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); > - struct mcryptd_queue *queue = > - mcryptd_get_queue(crypto_ahash_tfm(tfm)); > - > - rctx->complete = req->base.complete; > - req->base.complete = complete; > - > - ret = mcryptd_enqueue_request(queue, &req->base, rctx); > - > - return ret; > -} > - > -static void mcryptd_hash_init(struct crypto_async_request *req_async, int err) > -{ > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); > - struct crypto_ahash *child = ctx->child; > - struct ahash_request *req = ahash_request_cast(req_async); > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - struct ahash_request *desc = &rctx->areq; > - > - if (unlikely(err == -EINPROGRESS)) > - goto out; > - > - ahash_request_set_tfm(desc, child); > - ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, > - rctx->complete, req_async); > - > - rctx->out = req->result; > - err = crypto_ahash_init(desc); > - > -out: > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > -} > - > -static int mcryptd_hash_init_enqueue(struct ahash_request *req) > -{ > - return mcryptd_hash_enqueue(req, mcryptd_hash_init); > -} > - > -static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) > -{ > - struct ahash_request *req = ahash_request_cast(req_async); > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - > - if (unlikely(err == -EINPROGRESS)) > - goto out; > - > - rctx->out = req->result; > - err = crypto_ahash_update(&rctx->areq); > - if (err) { > - req->base.complete = rctx->complete; > - goto out; > - } > - > - return; > -out: > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > -} > - > -static int mcryptd_hash_update_enqueue(struct ahash_request *req) > -{ > - return mcryptd_hash_enqueue(req, mcryptd_hash_update); > -} > - > -static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) > -{ > - struct ahash_request *req = ahash_request_cast(req_async); > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - > - if (unlikely(err == -EINPROGRESS)) > - goto out; > - > - rctx->out = req->result; > - err = crypto_ahash_final(&rctx->areq); > - if (err) { > - req->base.complete = rctx->complete; > - goto out; > - } > - > - return; > -out: > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > -} > - > -static int mcryptd_hash_final_enqueue(struct ahash_request *req) > -{ > - return mcryptd_hash_enqueue(req, mcryptd_hash_final); > -} > - > -static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) > -{ > - struct ahash_request *req = ahash_request_cast(req_async); > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - > - if (unlikely(err == -EINPROGRESS)) > - goto out; > - rctx->out = req->result; > - err = crypto_ahash_finup(&rctx->areq); > - > - if (err) { > - req->base.complete = rctx->complete; > - goto out; > - } > - > - return; > -out: > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > -} > - > -static int mcryptd_hash_finup_enqueue(struct ahash_request *req) > -{ > - return mcryptd_hash_enqueue(req, mcryptd_hash_finup); > -} > - > -static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err) > -{ > - struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); > - struct crypto_ahash *child = ctx->child; > - struct ahash_request *req = ahash_request_cast(req_async); > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - struct ahash_request *desc = &rctx->areq; > - > - if (unlikely(err == -EINPROGRESS)) > - goto out; > - > - ahash_request_set_tfm(desc, child); > - ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, > - rctx->complete, req_async); > - > - rctx->out = req->result; > - err = crypto_ahash_init(desc) ?: crypto_ahash_finup(desc); > - > -out: > - local_bh_disable(); > - rctx->complete(&req->base, err); > - local_bh_enable(); > -} > - > -static int mcryptd_hash_digest_enqueue(struct ahash_request *req) > -{ > - return mcryptd_hash_enqueue(req, mcryptd_hash_digest); > -} > - > -static int mcryptd_hash_export(struct ahash_request *req, void *out) > -{ > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - > - return crypto_ahash_export(&rctx->areq, out); > -} > - > -static int mcryptd_hash_import(struct ahash_request *req, const void *in) > -{ > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - > - return crypto_ahash_import(&rctx->areq, in); > -} > - > -static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, > - struct mcryptd_queue *queue) > -{ > - struct hashd_instance_ctx *ctx; > - struct ahash_instance *inst; > - struct hash_alg_common *halg; > - struct crypto_alg *alg; > - u32 type = 0; > - u32 mask = 0; > - int err; > - > - if (!mcryptd_check_internal(tb, &type, &mask)) > - return -EINVAL; > - > - halg = ahash_attr_alg(tb[1], type, mask); > - if (IS_ERR(halg)) > - return PTR_ERR(halg); > - > - alg = &halg->base; > - pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name); > - inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(), > - sizeof(*ctx)); > - err = PTR_ERR(inst); > - if (IS_ERR(inst)) > - goto out_put_alg; > - > - ctx = ahash_instance_ctx(inst); > - ctx->queue = queue; > - > - err = crypto_init_ahash_spawn(&ctx->spawn, halg, > - ahash_crypto_instance(inst)); > - if (err) > - goto out_free_inst; > - > - inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC | > - (alg->cra_flags & (CRYPTO_ALG_INTERNAL | > - CRYPTO_ALG_OPTIONAL_KEY)); > - > - inst->alg.halg.digestsize = halg->digestsize; > - inst->alg.halg.statesize = halg->statesize; > - inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx); > - > - inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm; > - inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm; > - > - inst->alg.init = mcryptd_hash_init_enqueue; > - inst->alg.update = mcryptd_hash_update_enqueue; > - inst->alg.final = mcryptd_hash_final_enqueue; > - inst->alg.finup = mcryptd_hash_finup_enqueue; > - inst->alg.export = mcryptd_hash_export; > - inst->alg.import = mcryptd_hash_import; > - if (crypto_hash_alg_has_setkey(halg)) > - inst->alg.setkey = mcryptd_hash_setkey; > - inst->alg.digest = mcryptd_hash_digest_enqueue; > - > - err = ahash_register_instance(tmpl, inst); > - if (err) { > - crypto_drop_ahash(&ctx->spawn); > -out_free_inst: > - kfree(inst); > - } > - > -out_put_alg: > - crypto_mod_put(alg); > - return err; > -} > - > -static struct mcryptd_queue mqueue; > - > -static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb) > -{ > - struct crypto_attr_type *algt; > - > - algt = crypto_get_attr_type(tb); > - if (IS_ERR(algt)) > - return PTR_ERR(algt); > - > - switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { > - case CRYPTO_ALG_TYPE_DIGEST: > - return mcryptd_create_hash(tmpl, tb, &mqueue); > - break; > - } > - > - return -EINVAL; > -} > - > -static void mcryptd_free(struct crypto_instance *inst) > -{ > - struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst); > - struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); > - > - switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { > - case CRYPTO_ALG_TYPE_AHASH: > - crypto_drop_ahash(&hctx->spawn); > - kfree(ahash_instance(inst)); > - return; > - default: > - crypto_drop_spawn(&ctx->spawn); > - kfree(inst); > - } > -} > - > -static struct crypto_template mcryptd_tmpl = { > - .name = "mcryptd", > - .create = mcryptd_create, > - .free = mcryptd_free, > - .module = THIS_MODULE, > -}; > - > -struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, > - u32 type, u32 mask) > -{ > - char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME]; > - struct crypto_ahash *tfm; > - > - if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME, > - "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) > - return ERR_PTR(-EINVAL); > - tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask); > - if (IS_ERR(tfm)) > - return ERR_CAST(tfm); > - if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { > - crypto_free_ahash(tfm); > - return ERR_PTR(-EINVAL); > - } > - > - return __mcryptd_ahash_cast(tfm); > -} > -EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash); > - > -struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) > -{ > - struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); > - > - return ctx->child; > -} > -EXPORT_SYMBOL_GPL(mcryptd_ahash_child); > - > -struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req) > -{ > - struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); > - return &rctx->areq; > -} > -EXPORT_SYMBOL_GPL(mcryptd_ahash_desc); > - > -void mcryptd_free_ahash(struct mcryptd_ahash *tfm) > -{ > - crypto_free_ahash(&tfm->base); > -} > -EXPORT_SYMBOL_GPL(mcryptd_free_ahash); > - > -static int __init mcryptd_init(void) > -{ > - int err, cpu; > - struct mcryptd_flush_list *flist; > - > - mcryptd_flist = alloc_percpu(struct mcryptd_flush_list); > - for_each_possible_cpu(cpu) { > - flist = per_cpu_ptr(mcryptd_flist, cpu); > - INIT_LIST_HEAD(&flist->list); > - mutex_init(&flist->lock); > - } > - > - err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN); > - if (err) { > - free_percpu(mcryptd_flist); > - return err; > - } > - > - err = crypto_register_template(&mcryptd_tmpl); > - if (err) { > - mcryptd_fini_queue(&mqueue); > - free_percpu(mcryptd_flist); > - } > - > - return err; > -} > - > -static void __exit mcryptd_exit(void) > -{ > - mcryptd_fini_queue(&mqueue); > - crypto_unregister_template(&mcryptd_tmpl); > - free_percpu(mcryptd_flist); > -} > - > -subsys_initcall(mcryptd_init); > -module_exit(mcryptd_exit); > - > -MODULE_LICENSE("GPL"); > -MODULE_DESCRIPTION("Software async multibuffer crypto daemon"); > -MODULE_ALIAS_CRYPTO("mcryptd"); > diff --git a/include/crypto/mcryptd.h b/include/crypto/mcryptd.h > deleted file mode 100644 > index b67404fc4b34..000000000000 > --- a/include/crypto/mcryptd.h > +++ /dev/null > @@ -1,114 +0,0 @@ > -/* SPDX-License-Identifier: GPL-2.0 */ > -/* > - * Software async multibuffer crypto daemon headers > - * > - * Author: > - * Tim Chen <tim.c.chen@xxxxxxxxxxxxxxx> > - * > - * Copyright (c) 2014, Intel Corporation. > - */ > - > -#ifndef _CRYPTO_MCRYPT_H > -#define _CRYPTO_MCRYPT_H > - > -#include <linux/crypto.h> > -#include <linux/kernel.h> > -#include <crypto/hash.h> > - > -struct mcryptd_ahash { > - struct crypto_ahash base; > -}; > - > -static inline struct mcryptd_ahash *__mcryptd_ahash_cast( > - struct crypto_ahash *tfm) > -{ > - return (struct mcryptd_ahash *)tfm; > -} > - > -struct mcryptd_cpu_queue { > - struct crypto_queue queue; > - spinlock_t q_lock; > - struct work_struct work; > -}; > - > -struct mcryptd_queue { > - struct mcryptd_cpu_queue __percpu *cpu_queue; > -}; > - > -struct mcryptd_instance_ctx { > - struct crypto_spawn spawn; > - struct mcryptd_queue *queue; > -}; > - > -struct mcryptd_hash_ctx { > - struct crypto_ahash *child; > - struct mcryptd_alg_state *alg_state; > -}; > - > -struct mcryptd_tag { > - /* seq number of request */ > - unsigned seq_num; > - /* arrival time of request */ > - unsigned long arrival; > - unsigned long expire; > - int cpu; > -}; > - > -struct mcryptd_hash_request_ctx { > - struct list_head waiter; > - crypto_completion_t complete; > - struct mcryptd_tag tag; > - struct crypto_hash_walk walk; > - u8 *out; > - int flag; > - struct ahash_request areq; > -}; > - > -struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, > - u32 type, u32 mask); > -struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm); > -struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req); > -void mcryptd_free_ahash(struct mcryptd_ahash *tfm); > -void mcryptd_flusher(struct work_struct *work); > - > -enum mcryptd_req_type { > - MCRYPTD_NONE, > - MCRYPTD_UPDATE, > - MCRYPTD_FINUP, > - MCRYPTD_DIGEST, > - MCRYPTD_FINAL > -}; > - > -struct mcryptd_alg_cstate { > - unsigned long next_flush; > - unsigned next_seq_num; > - bool flusher_engaged; > - struct delayed_work flush; > - int cpu; > - struct mcryptd_alg_state *alg_state; > - void *mgr; > - spinlock_t work_lock; > - struct list_head work_list; > - struct list_head flush_list; > -}; > - > -struct mcryptd_alg_state { > - struct mcryptd_alg_cstate __percpu *alg_cstate; > - unsigned long (*flusher)(struct mcryptd_alg_cstate *cstate); > -}; > - > -/* return delay in jiffies from current time */ > -static inline unsigned long get_delay(unsigned long t) > -{ > - long delay; > - > - delay = (long) t - (long) jiffies; > - if (delay <= 0) > - return 0; > - else > - return (unsigned long) delay; > -} > - > -void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay); > - > -#endif >
