Introduce the "by16" implementation of the AES CTR mode using AVX512
optimizations. "by16" means that 16 independent blocks (each block
being 128 bits) can be ciphered simultaneously as opposed to the
current 8 blocks.
The glue code in AESNI module overrides the existing "by8" CTR mode
encryption/decryption routines with the "by16" ones when the following
criteria are met:
At compile time:
1. CONFIG_CRYPTO_AVX512 is enabled
2. toolchain(assembler) supports VAES instructions
At runtime:
1. VAES and AVX512VL features are supported on platform (currently
only Icelake)
2. If compiled as built-in or loadable module, aesni_intel.use_avx512
is set at boot time.
The functions aes_ctr_enc_128_avx512_by16(), aes_ctr_enc_192_avx512_by16()
and aes_ctr_enc_256_avx512_by16() are adapted from Intel Optimized IPSEC
Cryptographic library.
On a Icelake desktop, with turbo disabled and all CPUs running at maximum
frequency, the "by16" CTR mode optimization shows better performance
across data & key sizes as measured by tcrypt.
The average performance improvement of the "by16" version over the "by8"
version is as follows:
For all key sizes(128/192/256 bits),
data sizes < 128 bytes/block, negligible improvement(~3% loss)
data sizes > 128 bytes/block, there is an average improvement of
48% for both encryption and decryption.
A typical run of tcrypt with AES CTR mode encryption/decryption of the
"by8" and "by16" optimization on a Icelake desktop shows the following
results:
--------------------------------------------------------------
| key | bytes | cycles/op (lower is better)| percentage |
| length | per | encryption | decryption | loss/gain |
| (bits) | block |-------------------------------------------|
| | | by8 | by16 | by8 | by16 | enc | dec |
|------------------------------------------------------------|
| 128 | 16 | 156 | 168 | 164 | 168 | -7.7 | -2.5 |
| 128 | 64 | 180 | 190 | 157 | 146 | -5.6 | 7.1 |
| 128 | 256 | 248 | 158 | 251 | 161 | 36.3 | 35.9 |
| 128 | 1024 | 633 | 316 | 642 | 319 | 50.1 | 50.4 |
| 128 | 1472 | 853 | 411 | 877 | 407 | 51.9 | 53.6 |
| 128 | 8192 | 4463 | 1959 | 4447 | 1940 | 56.2 | 56.4 |
| 192 | 16 | 136 | 145 | 149 | 166 | -6.7 | -11.5 |
| 192 | 64 | 159 | 154 | 157 | 160 | 3.2 | -2 |
| 192 | 256 | 268 | 172 | 274 | 177 | 35.9 | 35.5 |
| 192 | 1024 | 710 | 358 | 720 | 355 | 49.6 | 50.7 |
| 192 | 1472 | 989 | 468 | 983 | 469 | 52.7 | 52.3 |
| 192 | 8192 | 6326 | 3551 | 6301 | 3567 | 43.9 | 43.4 |
| 256 | 16 | 153 | 165 | 139 | 156 | -7.9 | -12.3 |
| 256 | 64 | 158 | 152 | 174 | 161 | 3.8 | 7.5 |
| 256 | 256 | 283 | 176 | 287 | 202 | 37.9 | 29.7 |
| 256 | 1024 | 797 | 393 | 807 | 395 | 50.7 | 51.1 |
| 256 | 1472 | 1108 | 534 | 1107 | 527 | 51.9 | 52.4 |
| 256 | 8192 | 5763 | 2616 | 5773 | 2617 | 54.7 | 54.7 |
--------------------------------------------------------------
This work was inspired by the AES CTR mode optimization published
in Intel Optimized IPSEC Cryptographic library.
https://github.com/intel/intel-ipsec-mb/blob/master/lib/avx512/cntr_vaes_avx512.asm
Co-developed-by: Tomasz Kantecki <tomasz.kantecki@xxxxxxxxx>
Signed-off-by: Tomasz Kantecki <tomasz.kantecki@xxxxxxxxx>
Signed-off-by: Megha Dey <megha.dey@xxxxxxxxx>
---
arch/x86/crypto/Makefile | 1 +
arch/x86/crypto/aes_avx512_common.S | 341 ++++++++++
arch/x86/crypto/aes_ctrby16_avx512-x86_64.S | 955 ++++++++++++++++++++++++++++
arch/x86/crypto/aesni-intel_glue.c | 42 +-
arch/x86/include/asm/disabled-features.h | 8 +-
crypto/Kconfig | 12 +
6 files changed, 1357 insertions(+), 2 deletions(-)
create mode 100644 arch/x86/crypto/aes_avx512_common.S
create mode 100644 arch/x86/crypto/aes_ctrby16_avx512-x86_64.S
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 662c16d..3f48460 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -51,6 +51,7 @@ chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o
+aesni-intel-$(CONFIG_CRYPTO_AES_CTR_AVX512) += aes_ctrby16_avx512-x86_64.o
obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ssse3_glue.o
diff --git a/arch/x86/crypto/aes_avx512_common.S b/arch/x86/crypto/aes_avx512_common.S
new file mode 100644
index 0000000..61c012b
--- /dev/null
+++ b/arch/x86/crypto/aes_avx512_common.S
@@ -0,0 +1,341 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright © 2021 Intel Corporation.
+ *
+ * Collection of macros which are shared between AVX512 versions of
+ * AESNI CTR and GCM algorithms.
+ */
+
+#include <linux/linkage.h>
+#include <asm/inst.h>
+
+/*
+ * Generic macro to produce code that executes OPCODE instruction
+ * on selected number of AES blocks (16 bytes long ) between 0 and 16.
+ * All three operands of the instruction come from registers.
+ */
+#define ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUM_BLOCKS, OPCODE, DST0, DST1, DST2, DST3, SRC1_0, SRC1_1, SRC1_2, SRC1_3, SRC2_0, SRC2_1, SRC2_2, SRC2_3) \
+.set blocks_left,NUM_BLOCKS; \
+.if NUM_BLOCKS < 4; \
+ .if blocks_left == 1; \
+ OPCODE XWORD(SRC2_0), XWORD(SRC1_0), XWORD(DST0); \
+ .elseif blocks_left == 2; \
+ OPCODE YWORD(SRC2_0), YWORD(SRC1_0), YWORD(DST0); \
+ .elseif blocks_left == 3; \
+ OPCODE SRC2_0, SRC1_0, DST0; \
+ .endif; \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8; \
+ OPCODE SRC2_0, SRC1_0, DST0; \
+ .set blocks_left, blocks_left - 4; \
+ .if blocks_left == 1; \
+ OPCODE XWORD(SRC2_1), XWORD(SRC1_1), XWORD(DST1); \
+ .elseif blocks_left == 2; \
+ OPCODE YWORD(SRC2_1), YWORD(SRC1_1), YWORD(DST1); \
+ .elseif blocks_left == 3; \
+ OPCODE SRC2_1, SRC1_1, DST1; \
+ .endif; \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12; \
+ OPCODE SRC2_0, SRC1_0, DST0; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_1, SRC1_1, DST1; \
+ .set blocks_left, blocks_left - 4; \
+ .if blocks_left == 1; \
+ OPCODE XWORD(SRC2_2), XWORD(SRC1_2), XWORD(DST2); \
+ .elseif blocks_left == 2; \
+ OPCODE YWORD(SRC2_2), YWORD(SRC1_2), YWORD(DST2); \
+ .elseif blocks_left == 3; \
+ OPCODE SRC2_2, SRC1_2, DST2; \
+ .endif; \
+.elseif NUM_BLOCKS >= 12 && NUM_BLOCKS < 16; \
+ OPCODE SRC2_0, SRC1_0, DST0; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_1, SRC1_1, DST1; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_2, SRC1_2, DST2; \
+ .set blocks_left, blocks_left - 4; \
+ .if blocks_left == 1; \
+ OPCODE XWORD(SRC2_3), XWORD(SRC1_3), XWORD(DST3); \
+ .elseif blocks_left == 2; \
+ OPCODE YWORD(SRC2_3), YWORD(SRC1_3), YWORD(DST3); \
+ .elseif blocks_left == 3; \
+ OPCODE SRC2_3, SRC1_3, DST3; \
+ .endif; \
+.else; \
+ OPCODE SRC2_0, SRC1_0, DST0; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_1, SRC1_1, DST1; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_2, SRC1_2, DST2; \
+ .set blocks_left, blocks_left - 4; \
+ OPCODE SRC2_3, SRC1_3, DST3; \
+ .set blocks_left, blocks_left - 4; \
+.endif;
+
+/*
+ * Handles AES encryption rounds. It handles special cases: the last and
+ * first rounds. Optionally, it performs XOR with data after the last AES
+ * round. Uses NROUNDS parameter to check what needs to be done for the
+ * current round.
+ */
+#define ZMM_AESENC_ROUND_BLOCKS_0_16(L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, ROUND, D0_3, D4_7, D8_11, D12_15, NUMBL, NROUNDS) \
+.if ROUND < 1; \
+ ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vpxorq, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY) \
+.endif; \
+.if (ROUND >= 1) && (ROUND <= NROUNDS); \
+ ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vaesenc, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY) \
+.endif; \
+.if ROUND > NROUNDS; \
+ ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vaesenclast, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY) \
+ .ifnc D0_3, no_data; \
+ .ifnc D4_7, no_data; \
+ .ifnc D8_11, no_data; \
+ .ifnc D12_15, no_data; \
+ ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vpxorq, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, D0_3, D4_7, D8_11, D12_15) \
+ .endif; \
+ .endif; \
+ .endif; \
+ .endif; \
+.endif;
+
+/*
+ * Loads specified number of AES blocks into ZMM registers using mask register
+ * for the last loaded register (xmm, ymm or zmm). Loads take place at 1 byte
+ * granularity.
+ */
+#define ZMM_LOAD_MASKED_BLOCKS_0_16(NUM_BLOCKS, INP, DATA_OFFSET, DST0, DST1, DST2, DST3, MASK) \
+.set src_offset,0; \
+.set blocks_left, NUM_BLOCKS; \
+.if NUM_BLOCKS <= 4; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST0){MASK}{z}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST0){MASK}{z}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0{MASK}{z}; \
+ .endif; \
+.elseif NUM_BLOCKS > 4 && NUM_BLOCKS <= 8; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST1){MASK}{z}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST1){MASK}{z}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1{MASK}{z}; \
+ .endif; \
+.elseif NUM_BLOCKS > 8 && NUM_BLOCKS <= 12; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST2){MASK}{z}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST2){MASK}{z}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST2{MASK}{z}; \
+ .endif; \
+.else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST2; \
+ .set blocks_left, blocks_left - 4; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST3){MASK}{z}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST3){MASK}{z}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST3{MASK}{z}; \
+ .endif; \
+.endif;
+
+/*
+ * Stores specified number of AES blocks from ZMM registers with mask register
+ * for the last loaded register (xmm, ymm or zmm). Stores take place at 1 byte
+ * granularity.
+ */
+#define ZMM_STORE_MASKED_BLOCKS_0_16(NUM_BLOCKS, OUTP, DATA_OFFSET, SRC0, SRC1, SRC2, SRC3, MASK) \
+.set blocks_left, NUM_BLOCKS; \
+.set dst_offset, 0; \
+.if NUM_BLOCKS <= 4; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC0), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .endif; \
+.elseif NUM_BLOCKS > 4 && NUM_BLOCKS <=8; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC1), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 SRC1, dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .endif; \
+.elseif NUM_BLOCKS > 8 && NUM_BLOCKS <= 12; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC1, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC2), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 SRC2, dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .endif; \
+.else; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC1, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC2, dst_offset(OUTP, DATA_OFFSET); \
+ .set blocks_left, blocks_left - 4; \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC3), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .elseif (blocks_left == 3 || blocks_left == 4); \
+ vmovdqu8 SRC3, dst_offset(OUTP, DATA_OFFSET){MASK}; \
+ .endif; \
+.endif;
+
+/* Loads specified number of AES blocks into ZMM registers */
+#define ZMM_LOAD_BLOCKS_0_16(NUM_BLOCKS, INP, DATA_OFFSET, DST0, DST1, DST2, DST3, FLAGS) \
+.set src_offset, 0; \
+.set blocks_left, NUM_BLOCKS % 4; \
+.if NUM_BLOCKS < 4; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST0); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST0); \
+ .elseif blocks_left == 3; \
+ .ifc FLAGS, load_4_instead_of_3; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST0); \
+ vinserti64x2 $2, src_offset + 32(INP, DATA_OFFSET), DST0, DST0; \
+ .endif; \
+ .endif; \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST1); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST1); \
+ .elseif blocks_left == 3; \
+ .ifc FLAGS, load_4_instead_of_3; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1; \
+ .else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST1); \
+ vinserti64x2 $2, src_offset + 32(INP, DATA_OFFSET), DST1, DST1; \
+ .endif; \
+ .endif; \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST2); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST2); \
+ .elseif blocks_left == 3; \
+ .ifc FLAGS, load_4_instead_of_3; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST2; \
+ .else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST2); \
+ vinserti64x2 $2, src_offset + 32(INP, DATA_OFFSET), DST2, DST2; \
+ .endif; \
+ .endif; \
+.else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST0; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST1; \
+ .set src_offset, src_offset + 64; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST2; \
+ .set src_offset, src_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), XWORD(DST3); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST3); \
+ .elseif blocks_left == 3; \
+ .ifc FLAGS, load_4_instead_of_3; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), DST3; \
+ .else; \
+ vmovdqu8 src_offset(INP, DATA_OFFSET), YWORD(DST3); \
+ vinserti64x2 $2, src_offset + 32(INP, DATA_OFFSET), DST3, DST3; \
+ .endif; \
+ .endif; \
+.endif;
+
+/* Stores specified number of AES blocks from ZMM registers */
+#define ZMM_STORE_BLOCKS_0_16(NUM_BLOCKS, OUTP, DATA_OFFSET, SRC0, SRC1, SRC2, SRC3) \
+.set dst_offset, 0; \
+.set blocks_left, NUM_BLOCKS % 4; \
+.if NUM_BLOCKS < 4; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC0), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 3; \
+ vmovdqu8 YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET); \
+ vextracti32x4 $2, SRC0, dst_offset + 32(OUTP, DATA_OFFSET); \
+ .endif; \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC1), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 3; \
+ vmovdqu8 YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET); \
+ vextracti32x4 $2, SRC1, dst_offset + 32(OUTP, DATA_OFFSET); \
+ .endif; \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC1, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC2), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 3; \
+ vmovdqu8 YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET); \
+ vextracti32x4 $2, SRC2, dst_offset + 32(OUTP, DATA_OFFSET); \
+ .endif; \
+.else; \
+ vmovdqu8 SRC0, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC1, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ vmovdqu8 SRC2, dst_offset(OUTP, DATA_OFFSET); \
+ .set dst_offset, dst_offset + 64; \
+ .if blocks_left == 1; \
+ vmovdqu8 XWORD(SRC3), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 2; \
+ vmovdqu8 YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET); \
+ .elseif blocks_left == 3; \
+ vmovdqu8 YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET); \
+ vextracti32x4 $2, SRC3, dst_offset + 32(OUTP, DATA_OFFSET); \
+ .endif; \
+.endif;
diff --git a/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S b/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S
new file mode 100644
index 0000000..787ea47
--- /dev/null
+++ b/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S
@@ -0,0 +1,955 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright © 2021 Intel Corporation.
+ *
+ * Implement AES CTR mode by16 optimization with VAES instructions. (x86_64)
+ *
+ * This is AES128/192/256 CTR mode optimization implementation. It requires
+ * the support of Intel(R) AVX512VL and VAES instructions.
+ */
+
+#include "aes_avx512_common.S"
+
+#define ZKEY0 %zmm17
+#define ZKEY1 %zmm18
+#define ZKEY2 %zmm19
+#define ZKEY3 %zmm20
+#define ZKEY4 %zmm21
+#define ZKEY5 %zmm22
+#define ZKEY6 %zmm23
+#define ZKEY7 %zmm24
+#define ZKEY8 %zmm25
+#define ZKEY9 %zmm26
+#define ZKEY10 %zmm27
+#define ZKEY11 %zmm28
+#define ZKEY12 %zmm29
+#define ZKEY13 %zmm30
+#define ZKEY14 %zmm31
+
+#define TMP0 %r10
+#define TMP1 %r11
+#define TMP2 %r12
+#define TMP3 %rax
+#define DATA_OFFSET %r13
+#define RBITS %r14
+#define MASKREG %k1
+#define SHUFREG %zmm13
+#define ADD8REG %zmm14
+
+#define CTR_BLOCKx %xmm0
+#define CTR_BLOCK_1_4 %zmm1
+#define CTR_BLOCK_5_8 %zmm2
+#define CTR_BLOCK_9_12 %zmm3
+#define CTR_BLOCK_13_16 %zmm4
+
+#define ZTMP0 %zmm5
+#define ZTMP1 %zmm6
+#define ZTMP2 %zmm7
+#define ZTMP3 %zmm8
+#define ZTMP4 %zmm9
+#define ZTMP5 %zmm10
+#define ZTMP6 %zmm11
+#define ZTMP7 %zmm12
+
+#define XTMP %xmm15
+
+#define zmm31y ymm31
+#define zmm30y ymm30
+#define zmm29y ymm29
+#define zmm28y ymm28
+#define zmm27y ymm27
+#define zmm26y ymm26
+#define zmm25y ymm25
+#define zmm24y ymm24
+#define zmm23y ymm23
+#define zmm22y ymm22
+#define zmm21y ymm21
+#define zmm20y ymm20
+#define zmm19y ymm19
+#define zmm18y ymm18
+#define zmm17y ymm17
+#define zmm13y ymm13
+#define zmm12y ymm12
+#define zmm11y ymm11
+#define zmm10y ymm10
+#define zmm9y ymm9
+#define zmm8y ymm8
+#define zmm7y ymm7
+#define zmm6y ymm6
+#define zmm5y ymm5
+#define zmm1y ymm1
+#define zmm0y ymm0
+
+#define zmm31x xmm31
+#define zmm30x xmm30
+#define zmm29x xmm29
+#define zmm28x xmm28
+#define zmm27x xmm27
+#define zmm26x xmm26
+#define zmm25x xmm25
+#define zmm24x xmm24
+#define zmm23x xmm23
+#define zmm22x xmm22
+#define zmm21x xmm21
+#define zmm20x xmm20
+#define zmm19x xmm19
+#define zmm18x xmm18
+#define zmm17x xmm17
+#define zmm13x xmm13
+#define zmm12x xmm12
+#define zmm11x xmm11
+#define zmm10x xmm10
+#define zmm9x xmm9
+#define zmm8x xmm8
+#define zmm7x xmm7
+#define zmm6x xmm6
+#define zmm5x xmm5
+#define zmm4x xmm4
+#define zmm3x xmm3
+#define zmm2x xmm2
+#define zmm1x xmm1
+#define zmm0x xmm0
+
+#define xmm0y ymm0
+#define xmm0z zmm0
+#define xmm15x xmm15
+#define xmm0x xmm0
+
+#define stringify(reg,y) reg##y
+#define str(reg,y) stringify(reg,y)
+#define concat(reg,y) str(reg,y)
+
+#define YWORD(reg) concat(reg, y)
+#define XWORD(reg) concat(reg, x)
+#define ZWORD(reg) concat(reg, z)
+
+#define arg1 %rdi
+#define arg2 %rsi
+#define arg3 %rdx
+#define arg4 %rcx
+#define arg5 %r8
+#define arg6 %r9
+
+#define STACK_FRAME_SIZE_CTR (5*8) /* space for 5 GP registers */
+
+.data
+.align 16
+ONE:
+.octa 0x00000000000000000000000000000001
+
+.align 16
+ddq_low_msk:
+.octa 0x0000000000000000FFFFFFFFFFFFFFFF
+
+.align 16
+ddq_high_add_1:
+.octa 0x00000000000000010000000000000000
+
+.align 16
+ddq_add_1:
+.octa 0x00000000000000000000000000000001
+
+.align 64
+SHUF_MASK:
+.octa 0x000102030405060708090A0B0C0D0E0F
+.octa 0x000102030405060708090A0B0C0D0E0F
+.octa 0x000102030405060708090A0B0C0D0E0F
+.octa 0x000102030405060708090A0B0C0D0E0F
+
+.align 64
+byte64_len_to_mask_table:
+.octa 0x00000000000000010000000000000000
+.octa 0x00000000000000070000000000000003
+.octa 0x000000000000001f000000000000000f
+.octa 0x000000000000007f000000000000003f
+.octa 0x00000000000001ff00000000000000ff
+.octa 0x00000000000007ff00000000000003ff
+.octa 0x0000000000001fff0000000000000fff
+.octa 0x0000000000007fff0000000000003fff
+.octa 0x000000000001ffff000000000000ffff
+.octa 0x000000000007ffff000000000003ffff
+.octa 0x00000000001fffff00000000000fffff
+.octa 0x00000000007fffff00000000003fffff
+.octa 0x0000000001ffffff0000000000ffffff
+.octa 0x0000000007ffffff0000000003ffffff
+.octa 0x000000001fffffff000000000fffffff
+.octa 0x000000007fffffff000000003fffffff
+.octa 0x00000001ffffffff00000000ffffffff
+.octa 0x00000007ffffffff00000003ffffffff
+.octa 0x0000001fffffffff0000000fffffffff
+.octa 0x0000007fffffffff0000003fffffffff
+.octa 0x000001ffffffffff000000ffffffffff
+.octa 0x000007ffffffffff000003ffffffffff
+.octa 0x00001fffffffffff00000fffffffffff
+.octa 0x00007fffffffffff00003fffffffffff
+.octa 0x0001ffffffffffff0000ffffffffffff
+.octa 0x0007ffffffffffff0003ffffffffffff
+.octa 0x001fffffffffffff000fffffffffffff
+.octa 0x007fffffffffffff003fffffffffffff
+.octa 0x01ffffffffffffff00ffffffffffffff
+.octa 0x07ffffffffffffff03ffffffffffffff
+.octa 0x1fffffffffffffff0fffffffffffffff
+.octa 0x7fffffffffffffff3fffffffffffffff
+.octa 0xffffffffffffffff
+
+.align 64
+ddq_add_12_15:
+.octa 0x0000000000000000000000000000000c
+.octa 0x0000000000000000000000000000000d
+.octa 0x0000000000000000000000000000000e
+.octa 0x0000000000000000000000000000000f
+
+.align 64
+ddq_add_8_11:
+.octa 0x00000000000000000000000000000008
+.octa 0x00000000000000000000000000000009
+.octa 0x0000000000000000000000000000000a
+.octa 0x0000000000000000000000000000000b
+
+.align 64
+ddq_add_4_7:
+.octa 0x00000000000000000000000000000004
+.octa 0x00000000000000000000000000000005
+.octa 0x00000000000000000000000000000006
+.octa 0x00000000000000000000000000000007
+
+.align 64
+ddq_add_0_3:
+.octa 0x00000000000000000000000000000000
+.octa 0x00000000000000000000000000000001
+.octa 0x00000000000000000000000000000002
+.octa 0x00000000000000000000000000000003
+
+.align 64
+ddq_add_13_16:
+.octa 0x0000000000000000000000000000000d
+.octa 0x0000000000000000000000000000000e
+.octa 0x0000000000000000000000000000000f
+.octa 0x00000000000000000000000000000010
+
+.align 64
+ddq_add_9_12:
+.octa 0x00000000000000000000000000000009
+.octa 0x0000000000000000000000000000000a
+.octa 0x0000000000000000000000000000000b
+.octa 0x0000000000000000000000000000000c
+
+.align 64
+ddq_add_5_8:
+.octa 0x00000000000000000000000000000005
+.octa 0x00000000000000000000000000000006
+.octa 0x00000000000000000000000000000007
+.octa 0x00000000000000000000000000000008
+
+.align 64
+ddq_add_1_4:
+.octa 0x00000000000000000000000000000001
+.octa 0x00000000000000000000000000000002
+.octa 0x00000000000000000000000000000003
+.octa 0x00000000000000000000000000000004
+
+.align 64
+ddq_add_16:
+.octa 0x00000000000000000000000000000010
+.octa 0x00000000000000000000000000000010
+.octa 0x00000000000000000000000000000010
+.octa 0x00000000000000000000000000000010
+
+.text
+/* Save register content for the caller */
+#define FUNC_SAVE_CTR() \
+ mov %rsp, %rax; \
+ sub $STACK_FRAME_SIZE_CTR, %rsp; \
+ and $~63, %rsp; \
+ mov %r12, (%rsp); \
+ mov %r13, 0x8(%rsp); \
+ mov %rax, 0x18(%rsp);
+
+/* Restore register content for the caller */
+#define FUNC_RESTORE_CTR() \
+ vzeroupper; \
+ mov (%rsp), %r12; \
+ mov 0x8(%rsp), %r13; \
+ mov 0x18(%rsp), %rsp;
+
+/* Update next counter value. */
+#define GET_NEXT_COUNTER(CTR, XTMP) \
+ vpaddq ddq_add_1(%rip), XWORD(CTR), XWORD(XTMP); \
+ vptest ddq_low_msk(%rip), XWORD(XTMP); \
+ /* If next counter overflows, add 1 to upper 64 bits \
+ * ensure 128 bit addition \
+ */ \
+ jnz 80f; \
+ vpaddq ddq_high_add_1(%rip), XWORD(XTMP), XWORD(XTMP); \
+80:
+
+/*
+ * CTR(128 bits) needs to be incremented. Since there is no 128-bit add
+ * instruction, we need to increment 64-bit (least significant) and if an
+ * overflow is detected, increment the most significant 64-bits.
+ */
+#define INCR_CNTR_128(CTR, ZT, XTMP, num) \
+ GET_NEXT_COUNTER(CTR, XTMP) \
+ vinserti64x2 $num, XTMP, ZT, ZT; \
+ vmovdqa64 XTMP, XWORD(CTR);
+
+/* Increment 4, 128 bit counters stored in a ZMM register */
+#define INCR_4_CNTRS(CTR, ZT) \
+ vmovdqa64 XWORD(CTR), XWORD(ZT); \
+ INCR_CNTR_128(CTR, ZT, XTMP, 1) \
+ INCR_CNTR_128(CTR, ZT, XTMP, 2) \
+ INCR_CNTR_128(CTR, ZT, XTMP, 3) \
+ vextracti32x4 $3, ZT, XWORD(CTR);
+
+/* Increment 1 to 16 counters (1 to 4 ZMM registers) based on number of blocks */
+#define INCR_CNTR_NUM_BLOCKS(CNTR, ZTMP0, ZTMP1, ZTMP2, ZTMP3, NUM) \
+.if NUM > 0; \
+ INCR_4_CNTRS(CNTR, ZTMP0) \
+.endif; \
+.if NUM > 1; \
+ GET_NEXT_COUNTER(CNTR, CNTR) \
+ INCR_4_CNTRS(CNTR, ZTMP1) \
+.endif; \
+.if NUM > 2; \
+ GET_NEXT_COUNTER(CNTR, CNTR) \
+ INCR_4_CNTRS(CNTR, ZTMP2) \
+.endif; \
+.if NUM > 3; \
+ GET_NEXT_COUNTER(CNTR, CNTR) \
+ INCR_4_CNTRS(CNTR, ZTMP3) \
+.endif;
+
+/* Update counters based on number of blocks */
+#define UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, num_initial_blocks) \
+ vshufi64x2 $0, ZWORD(CTR), ZWORD(CTR), ZWORD(CTR); \
+ vmovq XWORD(CTR), TMP3; \
+.if num_initial_blocks <=4; \
+ cmp $3, TMP3; \
+ jb 77f; \
+ INCR_CNTR_NUM_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, 1) \
+ jmp 78f; \
+.elseif num_initial_blocks <=8; \
+ cmp $7, TMP3; \
+ jb 77f; \
+ INCR_CNTR_NUM_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, 2) \
+ jmp 78f; \
+.elseif num_initial_blocks <=12; \
+ cmp $11, TMP3; \
+ jb 77f; \
+ INCR_CNTR_NUM_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, 3) \
+ jmp 78f; \
+.else; \
+ cmp $15, TMP3; \
+ jb 77f; \
+ INCR_CNTR_NUM_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, 4) \
+ jmp 78f; \
+.endif; \
+77:; \
+.if num_initial_blocks >= 1; \
+ vpaddq ddq_add_0_3(%rip), ZWORD(CTR), ZT1; \
+.endif; \
+.if num_initial_blocks > 4; \
+ vpaddq ddq_add_4_7(%rip), ZWORD(CTR), ZT2; \
+.endif; \
+.if num_initial_blocks > 8; \
+ vpaddq ddq_add_8_11(%rip), ZWORD(CTR), ZT3; \
+.endif; \
+.if num_initial_blocks > 12; \
+ vpaddq ddq_add_12_15(%rip), ZWORD(CTR), ZT4; \
+.endif; \
+78:;
+
+/* Prepares the AES counter blocks */
+#define PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_blocks) \
+.if num_blocks == 1; \
+ vmovdqa64 XWORD(CTR), XWORD(ZT1); \
+.elseif num_blocks == 2; \
+ vshufi64x2 $0, YWORD(CTR), YWORD(CTR), YWORD(ZT1); \
+ vmovq XWORD(CTR), TMP3; \
+ cmp $~1, TMP3; \
+ jb 50f; \
+ vpaddq ddq_add_1(%rip), XWORD(CTR), XTMP; \
+ vptest ddq_low_msk(%rip), XTMP; \
+ jnz 64f; \
+ vpaddq ddq_high_add_1(%rip), XTMP, XTMP; \
+64:; \
+ vinserti64x2 $1, XTMP, YWORD(ZT1) , YWORD(ZT1); \
+ jmp 55f; \
+50:; \
+ vpaddq ddq_add_0_3(%rip), YWORD(ZT1), YWORD(ZT1); \
+.else; \
+ UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, num_blocks) \
+.endif; \
+55:;
+
+/* Extract and Shuffle the updated counters for AES rounds */
+#define EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_initial_blocks) \
+.if num_initial_blocks == 1; \
+ vpshufb XWORD(SHUFREG), CTR, XWORD(ZT1); \
+.elseif num_initial_blocks == 2; \
+ vextracti32x4 $1, YWORD(ZT1), CTR; \
+ vpshufb YWORD(SHUFREG), YWORD(ZT1), YWORD(ZT1); \
+.elseif num_initial_blocks <= 4; \
+ vextracti32x4 $(num_initial_blocks - 1), ZT1, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+.elseif num_initial_blocks == 5; \
+ vmovdqa64 XWORD(ZT2), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb XWORD(SHUFREG), XWORD(ZT2), XWORD(ZT2); \
+.elseif num_initial_blocks == 6; \
+ vextracti32x4 $1, YWORD(ZT2), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb YWORD(SHUFREG), YWORD(ZT2), YWORD(ZT2); \
+.elseif num_initial_blocks == 7; \
+ vextracti32x4 $2, ZT2, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+.elseif num_initial_blocks == 8; \
+ vextracti32x4 $3, ZT2, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+.elseif num_initial_blocks == 9; \
+ vmovdqa64 XWORD(ZT3), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb XWORD(SHUFREG), XWORD(ZT3), XWORD(ZT3); \
+.elseif num_initial_blocks == 10; \
+ vextracti32x4 $1, YWORD(ZT3), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb YWORD(SHUFREG), YWORD(ZT3), YWORD(ZT3); \
+.elseif num_initial_blocks == 11; \
+ vextracti32x4 $2, ZT3, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb SHUFREG, ZT3, ZT3; \
+.elseif num_initial_blocks == 12; \
+ vextracti32x4 $3, ZT3, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb SHUFREG, ZT3, ZT3; \
+.elseif num_initial_blocks == 13; \
+ vmovdqa64 XWORD(ZT4), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb SHUFREG, ZT3, ZT3; \
+ vpshufb XWORD(SHUFREG), XWORD(ZT4), XWORD(ZT4); \
+.elseif num_initial_blocks == 14; \
+ vextracti32x4 $1, YWORD(ZT4), CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb SHUFREG, ZT3, ZT3; \
+ vpshufb YWORD(SHUFREG), YWORD(ZT4), YWORD(ZT4); \
+.elseif num_initial_blocks == 15; \
+ vextracti32x4 $2, ZT4, CTR; \
+ vpshufb SHUFREG, ZT1, ZT1; \
+ vpshufb SHUFREG, ZT2, ZT2; \
+ vpshufb SHUFREG, ZT3, ZT3; \
+ vpshufb SHUFREG, ZT4, ZT4; \
+.endif;
+
+/* AES rounds and XOR with plain/cipher text */
+#define AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY_0, ZKEY_1, ZKEY_2, ZKEY_3, ZKEY_4, ZKEY_5, ZKEY_6, ZKEY_7, ZKEY_8, ZKEY_9, ZKEY_10, ZKEY_11, ZKEY_12, ZKEY_13, ZKEY_14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY0, 0, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY1, 1, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY2, 2, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY3, 3, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY4, 4, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY5, 5, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY6, 6, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY7, 7, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY8, 8, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY9, 9, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY10, 10, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+.if NROUNDS == 9; \
+ jmp 29f; \
+.else; \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY11, 11, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY12, 12, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+.if NROUNDS == 11; \
+ jmp 29f; \
+.else; \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY13, 13, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY14, 14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+.endif; \
+.endif; \
+29:;
+
+/*
+ * This macro may look similar to INITIAL_BLOCKS but its usage is
+ * different. It is not meant to cipher counter blocks for the main
+ * by16 loop. Just ciphers amount of blocks. Used for small packets
+ * (<256 bytes).
+ */
+#define INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LEN, num_blks, CTR, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ /* get load/store mask */ \
+ lea byte64_len_to_mask_table(%rip), IA0; \
+ mov LEN, IA1; \
+.if num_blks > 12; \
+ sub $192, IA1; \
+.elseif num_blks > 8; \
+ sub $128, IA1; \
+.elseif num_blks > 4; \
+ sub $64, IA1; \
+.endif; \
+ kmovq (IA0, IA1, 8), MASKREG; \
+ ZMM_LOAD_MASKED_BLOCKS_0_16(num_blks, PLAIN_CYPH_IN, 1, ZT5, ZT6, ZT7, ZT8, MASKREG) \
+ PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_blks) \
+ EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_blks) \
+ AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, num_blks, NROUNDS) \
+ /* write cipher/plain text back to output */ \
+ ZMM_STORE_MASKED_BLOCKS_0_16(num_blks, CYPH_PLAIN_OUT, 1, ZT1, ZT2, ZT3, ZT4, MASKREG)
+
+/* This macro is used to "warm-up" pipeline for ENCRYPT_16_PARALLEL macro
+ * code. It is called only for data lengths 256 and above. The flow is as
+ * follows:
+ * - encrypt the initial num_initial_blocks blocks (can be 0)
+ * - encrypt the next 16 blocks
+ */
+#define INITIAL_BLOCKS(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, DATA_OFFSET, num_initial_blocks, CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+.if num_initial_blocks > 0; \
+ /* load plain/cipher text */ \
+ ZMM_LOAD_BLOCKS_0_16(num_initial_blocks, PLAIN_CYPH_IN, 1, ZT5, ZT6, ZT7, ZT8, load_4_instead_of_3) \
+ PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_initial_blocks) \
+ EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_initial_blocks) \
+ AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS) \
+ /* write cipher/plain text back to output */ \
+ ZMM_STORE_BLOCKS_0_16(num_initial_blocks, CYPH_PLAIN_OUT, 1, ZT1, ZT2, ZT3, ZT4) \
+ /* adjust data offset and length */ \
+ sub $(num_initial_blocks * 16), LENGTH; \
+ add $(num_initial_blocks * 16), DATA_OFFSET; \
+.endif; \
+\
+ /* - cipher of num_initial_blocks is done \
+ * - prepare counter blocks for the next 16 blocks (ZT5-ZT8) \
+ * - shuffle the blocks for AES \
+ * - encrypt the next 16 blocks \
+ */ \
+\
+ /* get text load/store mask (assume full mask by default) */ \
+ mov $~0, IA0; \
+.if num_initial_blocks > 0; \
+ /* This macro is executed for length 256 and up, zero length \
+ * is checked in CNTR_ENC_DEC. We know there is a partial block \
+ * if: LENGTH - 16*num_initial_blocks < 256 \
+ */ \
+ cmp $256, LENGTH; \
+ jge 56f; \
+ mov %rcx, IA1; \
+ mov $256, %ecx; \
+ sub LENGTH, %rcx; \
+ shr %cl, IA0; \
+ mov IA1, %rcx; \
+56:; \
+.endif; \
+ kmovq IA0, MASKREG; \
+ /* load plain or cipher text */ \
+ vmovdqu8 (PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT5; \
+ vmovdqu8 64(PLAIN_CYPH_IN, DATA_OFFSET), ZT6; \
+ vmovdqu8 128(PLAIN_CYPH_IN, DATA_OFFSET), ZT7; \
+ vmovdqu8 192(PLAIN_CYPH_IN, DATA_OFFSET), ZT8{MASKREG}{z}; \
+\
+ /* prepare next counter blocks */ \
+ vshufi64x2 $0, ZWORD(CTR), ZWORD(CTR), ZWORD(CTR); \
+.if num_initial_blocks > 0; \
+ vmovq XWORD(CTR), TMP3; \
+ cmp $~16, TMP3; \
+ jb 58f; \
+ vpaddq ddq_add_1(%rip), XWORD(CTR), XWORD(CTR); \
+ vptest ddq_low_msk(%rip), XWORD(CTR); \
+ jnz 57f; \
+ vpaddq ddq_high_add_1(%rip), XWORD(CTR), XWORD(CTR); \
+57:; \
+ INCR_CNTR_NUM_BLOCKS(CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, 4) \
+ jmp 60f; \
+58:; \
+ vpaddq ddq_add_1_4(%rip), ZWORD(CTR), CTR_1_4; \
+ vpaddq ddq_add_5_8(%rip), ZWORD(CTR), CTR_5_8; \
+ vpaddq ddq_add_9_12(%rip), ZWORD(CTR), CTR_9_12; \
+ vpaddq ddq_add_13_16(%rip), ZWORD(CTR), CTR_13_16; \
+.else; \
+ vmovq XWORD(CTR), TMP3; \
+ cmp $~15, TMP3; \
+ jb 59f; \
+ INCR_CNTR_NUM_BLOCKS(CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, 4) \
+ jmp 60f; \
+59:; \
+ vpaddq ddq_add_0_3(%rip), ZWORD(CTR), CTR_1_4; \
+ vpaddq ddq_add_4_7(%rip), ZWORD(CTR), CTR_5_8; \
+ vpaddq ddq_add_8_11(%rip), ZWORD(CTR), CTR_9_12; \
+ vpaddq ddq_add_12_15(%rip), ZWORD(CTR), CTR_13_16; \
+.endif; \
+60:; \
+ vextracti32x4 $3, CTR_13_16, XWORD(CTR); \
+ vpshufb SHUFREG, CTR_1_4, ZT1; \
+ vpshufb SHUFREG, CTR_5_8, ZT2; \
+ vpshufb SHUFREG, CTR_9_12, ZT3; \
+ vpshufb SHUFREG, CTR_13_16, ZT4; \
+\
+ AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, 16, NROUNDS) \
+\
+ /* write cipher/plain text back to output */ \
+ vmovdqu8 ZT1, (CYPH_PLAIN_OUT, DATA_OFFSET); \
+ vmovdqu8 ZT2, 64(CYPH_PLAIN_OUT, DATA_OFFSET, 1); \
+ vmovdqu8 ZT3, 128(CYPH_PLAIN_OUT, DATA_OFFSET, 1); \
+ vmovdqu8 ZT4, 192(CYPH_PLAIN_OUT, DATA_OFFSET, 1){MASKREG}; \
+\
+ /* adjust offset and length */ \
+ add $256, DATA_OFFSET; \
+ sub $256, LENGTH; \
+
+/*
+ * This macro ciphers payloads shorter than 256 bytes. The number of blocks in
+ * the message comes as an argument. Depending on the number of blocks, an
+ * optimized variant of INITIAL_BLOCKS_SMALL is invoked
+ */
+#define CNTR_ENC_DEC_SMALL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, NUM_BLOCKS, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ cmp $8, NUM_BLOCKS; \
+ je 38f; \
+ jl 48f; \
+ /* Initial blocks 9-16 */ \
+ cmp $12, NUM_BLOCKS; \
+ je 42f; \
+ jl 49f; \
+ /* Initial blocks 13-16 */ \
+ cmp $16, NUM_BLOCKS; \
+ je 46f; \
+ cmp $15, NUM_BLOCKS; \
+ je 45f; \
+ cmp $14, NUM_BLOCKS; \
+ je 44f; \
+ cmp $13, NUM_BLOCKS; \
+ je 43f; \
+49:; \
+ cmp $11, NUM_BLOCKS; \
+ je 41f; \
+ cmp $10, NUM_BLOCKS; \
+ je 40f; \
+ cmp $9, NUM_BLOCKS; \
+ je 39f; \
+48:; \
+ cmp $4, NUM_BLOCKS; \
+ je 34f; \
+ jl 47f; \
+ /* Initial blocks 5-7 */ \
+ cmp $7, NUM_BLOCKS; \
+ je 37f; \
+ cmp $6, NUM_BLOCKS; \
+ je 36f; \
+ cmp $5, NUM_BLOCKS; \
+ je 35f; \
+47:; \
+ cmp $3, NUM_BLOCKS; \
+ je 33f; \
+ cmp $2, NUM_BLOCKS; \
+ je 32f; \
+ jmp 31f; \
+46:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 16, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+45:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 15, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+44:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 14, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+43:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 13, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+42:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 12, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+41:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 11, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+40:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 10, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+39:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 9, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+38:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 8, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+37:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 7, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+36:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 6, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+35:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 5, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+34:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 4, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+33:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 3, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+32:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 2, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 30f; \
+31:; \
+ INITIAL_BLOCKS_SMALL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 1, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS) \
+30:;
+
+/*
+ * This is the main CNTR macro. It operates on single stream and encrypts 16
+ * blocks at a time
+ */
+#define ENCRYPT_16_PARALLEL(CYPH_PLAIN_OUT, PLAIN_CYPH_IN, DATA_OFFSET, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, IA0, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, SHUFREG, ADD8REG, NROUNDS, CTR) \
+ /* load the text data */ \
+ vmovdqu8 (PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT5; \
+ vmovdqu8 64(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT6; \
+ vmovdqu8 128(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT7; \
+ vmovdqu8 192(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT8; \
+ /* Populate counter blocks */ \
+ vmovq XWORD(CTR_1_4), IA0; \
+ cmp $~0x1F, IA0; \
+ jb 26f; \
+ vpaddq ddq_add_1(%rip), CTR, CTR; \
+ vptest ddq_low_msk(%rip), CTR; \
+ jnz 27f; \
+ vpaddq ddq_high_add_1(%rip), CTR, CTR; \
+27:; \
+ INCR_CNTR_NUM_BLOCKS(CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, 4) \
+ jmp 28f; \
+26:; \
+ vpaddq ADD8REG, CTR_1_4, CTR_1_4; \
+ vpaddq ADD8REG, CTR_5_8, CTR_5_8; \
+ vpaddq ADD8REG, CTR_9_12, CTR_9_12; \
+ vpaddq ADD8REG, CTR_13_16, CTR_13_16; \
+28:; \
+ vpshufb SHUFREG, CTR_1_4, ZT1; \
+ vpshufb SHUFREG, CTR_5_8, ZT2; \
+ vpshufb SHUFREG, CTR_9_12, ZT3; \
+ vpshufb SHUFREG, CTR_13_16, ZT4; \
+ AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, 16, NROUNDS) \
+ /*store the text data */ \
+ vmovdqu8 ZT1, (CYPH_PLAIN_OUT, DATA_OFFSET); \
+ vmovdqu8 ZT2, 64(CYPH_PLAIN_OUT, DATA_OFFSET, 1); \
+ vmovdqu8 ZT3, 128(CYPH_PLAIN_OUT, DATA_OFFSET, 1); \
+ vmovdqu8 ZT4, 192(CYPH_PLAIN_OUT, DATA_OFFSET, 1); \
+ vextracti32x4 $3, CTR_BLOCK_13_16, CTR;
+
+/*
+ * CNTR_ENC_DEC Encodes/Decodes given data. Requires the input data be
+ * at least 1 byte long because of READ_SMALL_INPUT_DATA.
+ */
+#define CNTR_ENC_DEC(KEYS, DST, SRC, LEN, IV, NROUNDS) \
+ or LEN, LEN; \
+ je 22f; \
+/* \
+ * Macro flow: \
+ * - calculate the number of 16byte blocks in the message \
+ * - process (number of 16byte blocks) mod 16 \
+ * - process 16x16 byte blocks at a time until all are done \
+ */ \
+ xor DATA_OFFSET, DATA_OFFSET; \
+/* Prepare round keys */ \
+ vbroadcastf64x2 16*0(KEYS), ZKEY0; \
+ vbroadcastf64x2 16*1(KEYS), ZKEY1; \
+ vbroadcastf64x2 16*2(KEYS), ZKEY2; \
+ vbroadcastf64x2 16*3(KEYS), ZKEY3; \
+ vbroadcastf64x2 16*4(KEYS), ZKEY4; \
+ vbroadcastf64x2 16*5(KEYS), ZKEY5; \
+ vbroadcastf64x2 16*6(KEYS), ZKEY6; \
+ vbroadcastf64x2 16*7(KEYS), ZKEY7; \
+ vbroadcastf64x2 16*8(KEYS), ZKEY8; \
+ vbroadcastf64x2 16*9(KEYS), ZKEY9; \
+ vbroadcastf64x2 16*10(KEYS), ZKEY10; \
+.if NROUNDS == 9; \
+ jmp 23f; \
+.else; \
+ vbroadcastf64x2 16*11(KEYS), ZKEY11; \
+ vbroadcastf64x2 16*12(KEYS), ZKEY12; \
+.if NROUNDS == 11; \
+ jmp 23f; \
+.else; \
+ vbroadcastf64x2 16*13(KEYS), ZKEY13; \
+ vbroadcastf64x2 16*14(KEYS), ZKEY14; \
+.endif; \
+.endif; \
+23:; \
+ /* Set mask to read 16 IV bytes */ \
+ vmovdqu8 (IV), CTR_BLOCKx; \
+ vmovdqa64 SHUF_MASK(%rip), SHUFREG; \
+ /* Store IV as counter in LE format */ \
+ vpshufb XWORD(SHUFREG), CTR_BLOCKx, CTR_BLOCKx; \
+ /* Determine how many blocks to process in INITIAL */ \
+ mov LEN, TMP1; \
+ shr $4, TMP1; \
+ and $0xf, TMP1; \
+ /* Less than 256B will be handled by the small message \
+ * code, which can process up to 16x blocks (16 bytes) \
+ */ \
+ cmp $256, LEN; \
+ jge 20f; \
+ CNTR_ENC_DEC_SMALL(KEYS, DST, SRC, LEN, TMP1, CTR_BLOCKx, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP2, MASKREG, SHUFREG, NROUNDS) \
+ jmp 22f; \
+20:; \
+ and $0xf, TMP1; \
+ je 16f; \
+ cmp $15, TMP1; \
+ je 15f; \
+ cmp $14, TMP1; \
+ je 14f; \
+ cmp $13, TMP1; \
+ je 13f; \
+ cmp $12, TMP1; \
+ je 12f; \
+ cmp $11, TMP1; \
+ je 11f; \
+ cmp $10, TMP1; \
+ je 10f; \
+ cmp $9, TMP1; \
+ je 9f; \
+ cmp $8, TMP1; \
+ je 8f; \
+ cmp $7, TMP1; \
+ je 7f; \
+ cmp $6, TMP1; \
+ je 6f; \
+ cmp $5, TMP1; \
+ je 5f; \
+ cmp $4, TMP1; \
+ je 4f; \
+ cmp $3, TMP1; \
+ je 3f; \
+ cmp $2, TMP1; \
+ je 2f; \
+ jmp 1f; \
+\
+ and $0xf, TMP1; \
+ je 16f; \
+ cmp $8, TMP1; \
+ je 8f; \
+ jl 18f; \
+ /* Initial blocks 9-15 */ \
+ cmp $12, TMP1; \
+ je 12f; \
+ jl 19f; \
+ /* Initial blocks 13-15 */ \
+ cmp $15, TMP1; \
+ je 15f; \
+ cmp $14, TMP1; \
+ je 14f; \
+ cmp $13, TMP1; \
+ je 13f; \
+19:; \
+ cmp $11, TMP1; \
+ je 11f; \
+ cmp $10, TMP1; \
+ je 10f; \
+ cmp $9, TMP1; \
+ je 9f; \
+18:; \
+ cmp $4, TMP1; \
+ je 4f; \
+ jl 17f; \
+ /* Initial blocks 5-7 */ \
+ cmp $7, TMP1; \
+ je 7f; \
+ cmp $6, TMP1; \
+ je 6f; \
+ cmp $5, TMP1; \
+ je 5f; \
+17:; \
+ cmp $3, TMP1; \
+ je 3f; \
+ cmp $2, TMP1; \
+ je 2f; \
+ jmp 1f; \
+15:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 15, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+14:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 14, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+13:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 13, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+12:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 12, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+11:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 11, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+10:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 10, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+9:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 9, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+8:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 8, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+7:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 7, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+6:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 6, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+5:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 5, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+4:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 4, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+3:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 3, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+2:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 2, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+1:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 1, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+ jmp 21f; \
+16:; \
+ INITIAL_BLOCKS(DST, SRC, LEN, DATA_OFFSET, 0, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS) \
+21:; \
+ or LEN, LEN; \
+ je 22f; \
+ vmovdqa64 ddq_add_16(%rip), ADD8REG; \
+25:; \
+ ENCRYPT_16_PARALLEL(DST, SRC, DATA_OFFSET, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, TMP0, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, SHUFREG, ADD8REG, NROUNDS, CTR_BLOCKx) \
+ add $256, DATA_OFFSET; \
+ sub $256, LEN; \
+ cmp $256, LEN; \
+ jge 25b; \
+22:; \
+ /* Update IV for next round */ \
+ vpaddq ONE(%rip), CTR_BLOCKx, CTR_BLOCKx; \
+ vptest ddq_low_msk(%rip), CTR_BLOCKx; \
+ jnz 24f; \
+ vpaddq ddq_high_add_1(%rip), CTR_BLOCKx, CTR_BLOCKx;; \
+24:; \
+ vpshufb SHUF_MASK(%rip), CTR_BLOCKx, CTR_BLOCKx;; \
+ vmovdqu CTR_BLOCKx, (IV);
+
+#define AES_CNTR_ENC_AVX512_BY16(keys, out, in, len, iv, NROUNDS) \
+ FUNC_SAVE_CTR() \
+ /* call the aes main loop */ \
+ CNTR_ENC_DEC(keys, out, in, len, iv, NROUNDS) \
+ FUNC_RESTORE_CTR() \
+ ret;
+
+/* Routine to do AES128/192/256 CTR enc/decrypt "by16"
+ * void aes_ctr_enc_128_avx512_by16/ aes_ctr_enc_192_avx512_by16/
+ * aes_ctr_enc_256_avx512_by16/
+ * (void *keys,
+ * u8 *out,
+ * const u8 *in,
+ * unsigned int num_bytes,
+ * u8 *iv);
+ */
+SYM_FUNC_START(aes_ctr_enc_128_avx512_by16)
+ AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 9)
+SYM_FUNC_END(aes_ctr_enc_128_avx512_by16)
+
+SYM_FUNC_START(aes_ctr_enc_192_avx512_by16)
+ AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 11)
+SYM_FUNC_END(aes_ctr_enc_192_avx512_by16)
+
+SYM_FUNC_START(aes_ctr_enc_256_avx512_by16)
+ AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 13)
+SYM_FUNC_END(aes_ctr_enc_256_avx512_by16)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index e2aae07..edf0056 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -45,6 +45,10 @@
#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
+static bool use_avx512;
+module_param(use_avx512, bool, 0644);
+MODULE_PARM_DESC(use_avx512, "Use AVX512 optimized algorithm, if available");
+
/* This data is stored at the end of the crypto_tfm struct.
* It's a type of per "session" data storage location.
* This needs to be 16 byte aligned.
@@ -135,6 +139,20 @@ asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
void *keys, u8 *out, unsigned int num_bytes);
asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
void *keys, u8 *out, unsigned int num_bytes);
+
+asmlinkage void aes_ctr_enc_128_avx512_by16(void *keys, u8 *out,
+ const u8 *in,
+ unsigned int num_bytes,
+ u8 *iv);
+asmlinkage void aes_ctr_enc_192_avx512_by16(void *keys, u8 *out,
+ const u8 *in,
+ unsigned int num_bytes,
+ u8 *iv);
+asmlinkage void aes_ctr_enc_256_avx512_by16(void *keys, u8 *out,
+ const u8 *in,
+ unsigned int num_bytes,
+ u8 *iv);
+
/*
* asmlinkage void aesni_gcm_init_avx_gen2()
* gcm_data *my_ctx_data, context data
@@ -493,6 +511,23 @@ static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
}
+static void aesni_ctr_enc_avx512_tfm(struct crypto_aes_ctx *ctx, u8 *out,
+ const u8 *in, unsigned int len, u8 *iv)
+{
+ /*
+ * Based on key length, override with the by16 version
+ * of ctr mode encryption/decryption for improved performance.
+ * aes_set_key_common() ensures that key length is one of
+ * {128,192,256}
+ */
+ if (ctx->key_length == AES_KEYSIZE_128)
+ aes_ctr_enc_128_avx512_by16((void *)ctx, out, in, len, iv);
+ else if (ctx->key_length == AES_KEYSIZE_192)
+ aes_ctr_enc_192_avx512_by16((void *)ctx, out, in, len, iv);
+ else
+ aes_ctr_enc_256_avx512_by16((void *)ctx, out, in, len, iv);
+}
+
static int ctr_crypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -1152,7 +1187,12 @@ static int __init aesni_init(void)
} else {
pr_info("SSE version of gcm_enc/dec engaged.\n");
}
- if (boot_cpu_has(X86_FEATURE_AVX)) {
+ if (use_avx512 && IS_ENABLED(CONFIG_CRYPTO_AES_CTR_AVX512) &&
+ cpu_feature_enabled(X86_FEATURE_VAES)) {
+ /* Ctr mode performance optimization using AVX512 */
+ static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx512_tfm);
+ pr_info("AES CTR mode by16 optimization enabled\n");
+ } else if (boot_cpu_has(X86_FEATURE_AVX)) {
/* optimize performance of ctr mode encryption transform */
static_call_update(aesni_ctr_enc_tfm, aesni_ctr_enc_avx_tfm);
pr_info("AES CTR mode by8 optimization enabled\n");
diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index e6e3b9f..a4f5dd9 100644
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -62,6 +62,12 @@
# define DISABLE_VPCLMULQDQ (1 << (X86_FEATURE_VPCLMULQDQ & 31))
#endif
+#if defined(CONFIG_AS_VAES_AVX512)
+# define DISABLE_VAES 0
+#else
+# define DISABLE_VAES (1 << (X86_FEATURE_VAES & 31))
+#endif
+
#ifdef CONFIG_IOMMU_SUPPORT
# define DISABLE_ENQCMD 0
#else
@@ -94,7 +100,7 @@
#define DISABLED_MASK14 0
#define DISABLED_MASK15 0
#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
- DISABLE_ENQCMD|DISABLE_VPCLMULQDQ)
+ DISABLE_ENQCMD|DISABLE_VPCLMULQDQ|DISABLE_VAES)
#define DISABLED_MASK17 0
#define DISABLED_MASK18 0
#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 3059b07..5c15471 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -635,6 +635,18 @@ config CRYPTO_CRCT10DIF_AVX512
depends on CRYPTO_CRCT10DIF_PCLMUL
depends on AS_VPCLMULQDQ
+# We default CRYPTO_AES_CTR_AVX512 to Y but depend on CRYPTO_AVX512 in
+# order to have a singular option (CRYPTO_AVX512) select multiple algorithms
+# when supported. Specifically, if the platform and/or toolset does not
+# support VPLMULQDQ. Then this algorithm should not be supported as part of
+# the set that CRYPTO_AVX512 selects.
+config CRYPTO_AES_CTR_AVX512
+ bool
+ default y
+ depends on CRYPTO_AVX512
+ depends on CRYPTO_AES_NI_INTEL
+ depends on AS_VAES_AVX512
+
config CRYPTO_CRC32C_SPARC64
tristate "CRC32c CRC algorithm (SPARC64)"
depends on SPARC64
--
2.7.4
