[RFC V1 5/7] crypto: aesni - AES CTR x86_64 "by16" AVX512 optimization

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Introduce the "by16" implementation of the AES CTR mode using AVX512
optimizations. "by16" means that 16 independent blocks (each block
being 128 bits) can be ciphered simultaneously as opposed to the
current 8 blocks.

The glue code in AESNI module overrides the existing "by8" CTR mode
encryption/decryption routines with the "by16" ones when the following
criteria are met:
At compile time:
1. CONFIG_CRYPTO_AVX512 is enabled
2. toolchain(assembler) supports VAES instructions
At runtime:
1. VAES and AVX512VL features are supported on platform (currently
   only Icelake)
2. aesni_intel.use_avx512 module parameter is set at boot time. For this
   algorithm, switching from AVX512 optimized version is not possible once
   set at boot time because of how the code is structured today.(Can be
   changed later if required)

The functions aes_ctr_enc_128_avx512_by16(), aes_ctr_enc_192_avx512_by16()
and aes_ctr_enc_256_avx512_by16() are adapted from Intel Optimized IPSEC
Cryptographic library.

On a Icelake desktop, with turbo disabled and all CPUs running at maximum
frequency, the "by16" CTR mode optimization shows better performance
across data & key sizes as measured by tcrypt.

The average performance improvement of the "by16" version over the "by8"
version is as follows:
For all key sizes(128/192/256 bits),
        data sizes < 128 bytes/block, negligible improvement(~3% loss)
        data sizes > 128 bytes/block, there is an average improvement of
48% for both encryption and decryption.

A typical run of tcrypt with AES CTR mode encryption/decryption of the
"by8" and "by16" optimization on a Icelake desktop shows the following
results:

--------------------------------------------------------------
|  key   | bytes | cycles/op (lower is better)| percentage   |
| length |  per  |  encryption  |  decryption |  loss/gain   |
| (bits) | block |-------------------------------------------|
|        |       | by8  | by16  | by8  | by16 |  enc | dec   |
|------------------------------------------------------------|
|  128   |  16   | 156  | 168   | 164  | 168  | -7.7 |  -2.5 |
|  128   |  64   | 180  | 190   | 157  | 146  | -5.6 |   7.1 |
|  128   |  256  | 248  | 158   | 251  | 161  | 36.3 |  35.9 |
|  128   |  1024 | 633  | 316   | 642  | 319  | 50.1 |  50.4 |
|  128   |  1472 | 853  | 411   | 877  | 407  | 51.9 |  53.6 |
|  128   |  8192 | 4463 | 1959  | 4447 | 1940 | 56.2 |  56.4 |
|  192   |  16   | 136  | 145   | 149  | 166  | -6.7 | -11.5 |
|  192   |  64   | 159  | 154   | 157  | 160  |  3.2 |  -2   |
|  192   |  256  | 268  | 172   | 274  | 177  | 35.9 |  35.5 |
|  192   |  1024 | 710  | 358   | 720  | 355  | 49.6 |  50.7 |
|  192   |  1472 | 989  | 468   | 983  | 469  | 52.7 |  52.3 |
|  192   |  8192 | 6326 | 3551  | 6301 | 3567 | 43.9 |  43.4 |
|  256   |  16   | 153  | 165   | 139  | 156  | -7.9 | -12.3 |
|  256   |  64   | 158  | 152   | 174  | 161  |  3.8 |   7.5 |
|  256   |  256  | 283  | 176   | 287  | 202  | 37.9 |  29.7 |
|  256   |  1024 | 797  | 393   | 807  | 395  | 50.7 |  51.1 |
|  256   |  1472 | 1108 | 534   | 1107 | 527  | 51.9 |  52.4 |
|  256   |  8192 | 5763 | 2616  | 5773 | 2617 | 54.7 |  54.7 |
--------------------------------------------------------------

This work was inspired by the AES CTR mode optimization published
in Intel Optimized IPSEC Cryptographic library.
https://github.com/intel/intel-ipsec-mb/blob/master/lib/avx512/cntr_vaes_avx512.asm

Co-developed-by: Tomasz Kantecki <tomasz.kantecki@xxxxxxxxx>
Signed-off-by: Tomasz Kantecki <tomasz.kantecki@xxxxxxxxx>
Signed-off-by: Megha Dey <megha.dey@xxxxxxxxx>
---
 arch/x86/crypto/Makefile                    |   1 +
 arch/x86/crypto/aes_ctrby16_avx512-x86_64.S | 856 ++++++++++++++++++++++++++++
 arch/x86/crypto/aesni-intel_glue.c          |  57 +-
 arch/x86/crypto/avx512_vaes_common.S        | 422 ++++++++++++++
 arch/x86/include/asm/disabled-features.h    |   8 +-
 crypto/Kconfig                              |  12 +
 6 files changed, 1354 insertions(+), 2 deletions(-)
 create mode 100644 arch/x86/crypto/aes_ctrby16_avx512-x86_64.S

diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index 0a86cfb..5fd9b35 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -53,6 +53,7 @@ chacha-x86_64-$(CONFIG_AS_AVX512) += chacha-avx512vl-x86_64.o
 obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
 aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o
 aesni-intel-$(CONFIG_64BIT) += aesni-intel_avx-x86_64.o aes_ctrby8_avx-x86_64.o
+aesni-intel-$(CONFIG_CRYPTO_AES_CTR_AVX512) += aes_ctrby16_avx512-x86_64.o
 
 obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o
 sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ssse3_glue.o
diff --git a/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S b/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S
new file mode 100644
index 0000000..7ccfcde
--- /dev/null
+++ b/arch/x86/crypto/aes_ctrby16_avx512-x86_64.S
@@ -0,0 +1,856 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright © 2020 Intel Corporation.
+ *
+ * Implement AES CTR mode by16 optimization with VAES instructions. (x86_64)
+ *
+ * This is AES128/192/256 CTR mode optimization implementation. It requires
+ * the support of Intel(R) AVX512VL and VAES instructions.
+ */
+
+#include "avx512_vaes_common.S"
+
+#define ZKEY0	%zmm17
+#define ZKEY1	%zmm18
+#define ZKEY2	%zmm19
+#define ZKEY3	%zmm20
+#define ZKEY4	%zmm21
+#define ZKEY5	%zmm22
+#define ZKEY6	%zmm23
+#define ZKEY7	%zmm24
+#define ZKEY8	%zmm25
+#define ZKEY9	%zmm26
+#define ZKEY10	%zmm27
+#define ZKEY11	%zmm28
+#define ZKEY12	%zmm29
+#define ZKEY13	%zmm30
+#define ZKEY14	%zmm31
+
+#define TMP0		%r10
+#define TMP1		%r11
+#define TMP2		%r12
+#define	TMP3		%rax
+#define DATA_OFFSET	%r13
+#define RBITS		%r14
+#define MASKREG		%k1
+#define SHUFREG		%zmm13
+#define ADD8REG		%zmm14
+
+#define CTR_BLOCKx	%xmm0
+#define CTR_BLOCK_1_4	%zmm1
+#define CTR_BLOCK_5_8	%zmm2
+#define CTR_BLOCK_9_12	%zmm3
+#define CTR_BLOCK_13_16	%zmm4
+
+#define ZTMP0		%zmm5
+#define ZTMP1		%zmm6
+#define ZTMP2		%zmm7
+#define ZTMP3		%zmm8
+#define ZTMP4		%zmm9
+#define ZTMP5		%zmm10
+#define ZTMP6		%zmm11
+#define ZTMP7		%zmm12
+
+#define	XTMP		%xmm15
+
+#define STACK_FRAME_SIZE_CTR	(5*8)	/* space for 5 GP registers */
+
+.text
+
+/* Save register content for the caller */
+#define FUNC_SAVE_CTR()				\
+	mov	%rsp, %rax;			\
+	sub	$STACK_FRAME_SIZE_CTR, %rsp;	\
+	and	$~63, %rsp;			\
+	mov	%r12, (%rsp);			\
+	mov	%r13, 0x8(%rsp);		\
+	mov	%rax, 0x18(%rsp);
+
+/* Restore register content for the caller */
+#define FUNC_RESTORE_CTR()			\
+	vzeroupper;			\
+	mov	(%rsp), %r12;		\
+	mov	0x8(%rsp), %r13;	\
+	mov	0x18(%rsp), %rsp;
+
+/*
+ * Maintain the bits from the output text when writing out the output blocks,
+ * in case there are some bits that do not require encryption
+ */
+#define PRESERVE_BITS(RBITS, LENGTH, CYPH_PLAIN_OUT, ZIN_OUT, ZTMP0, ZTMP1, ZTMP2, IA0, IA1, blocks_to_skip, FULL_PARTIAL, MASKREG, DATA_OFFSET, NUM_ARGS)	\
+/* offset = number of sets of 4 blocks to skip */			\
+.set offset, (((blocks_to_skip) / 4) * 64);				\
+\
+/* num_left_blocks = blocks in the last set, range 1-4 blocks	*/	\
+.set num_left_blocks,(((blocks_to_skip) & 3) + 1);			\
+\
+.if NUM_ARGS == 13;							\
+	/* Load output to get last partial byte */			\
+	.ifc FULL_PARTIAL, partial;					\
+		vmovdqu8	offset(CYPH_PLAIN_OUT, DATA_OFFSET), ZTMP0{MASKREG};	\
+	.else;								\
+		vmovdqu8	offset(CYPH_PLAIN_OUT, DATA_OFFSET), ZTMP0;		\
+	.endif;								\
+.else;									\
+	/* Load o/p to get last partial byte (up to the last 4 blocks) */\
+	ZMM_LOAD_MASKED_BLOCKS_0_16(num_left_blocks, CYPH_PLAIN_OUT, offset, ZTMP0, no_zmm, no_zmm, no_zmm, MASKREG)	\
+.endif;									\
+\
+	/* Save rcx in temporary GP register */				\
+	mov	%rcx, IA0;						\
+	mov	$0xff, DWORD(IA1);					\
+	mov	BYTE(RBITS), %cl;					\
+	/* e.g. 3 remaining bits=> mask = 00011111 */			\
+	shr	%cl, DWORD(IA1);					\
+	mov	IA0, %rcx;						\
+\
+	vmovq	IA1, XWORD(ZTMP1);					\
+\
+	/*
+	 * Get number of full bytes in last block.			\
+	 * Subtracting the bytes in the blocks to skip to the length of	\
+	 * whole set of blocks gives us the number of bytes in the last	\
+	 * block, but the last block has a partial byte at the end, so	\
+	 * an extra byte needs to be subtracted.			\
+	 */								\
+	mov	LENGTH, IA1;						\
+	sub	$(blocks_to_skip * 16 + 1), IA1;			\
+	lea	shift_tab_16 + 16(%rip), IA0;				\
+	sub	IA1, IA0;						\
+	vmovdqu (IA0), XWORD(ZTMP2);					\
+	vpshufb XWORD(ZTMP2), XWORD(ZTMP1), XWORD(ZTMP1);		\
+.if num_left_blocks == 4;						\
+	vshufi64x2	$0x15, ZTMP1, ZTMP1, ZTMP1;			\
+.elseif num_left_blocks == 3;						\
+	vshufi64x2	$0x45, ZTMP1, ZTMP1, ZTMP1;			\
+.elseif num_left_blocks == 2;						\
+	vshufi64x2	$0x51, ZTMP1, ZTMP1, ZTMP1;			\
+.endif;	/* No need to shift if there is only one block */		\
+\
+	/*								\
+	 * At this point, ZTMP1 contains a mask with all 0s, but with	\
+	 * some 1s in the partial byte.					\
+	 * First, clear last bits (not to be ciphered) of last output	\
+	 * block. ZIN_OUT = ZIN_OUT AND NOT ZTMP1 (0x50 = andA!C)	\
+	 */								\
+	vpternlogq	$0x50, ZTMP1, ZTMP1, ZIN_OUT;			\
+\
+	/*								\
+	 * Then, set these last bits to the bits coming from output.	\
+	 * ZIN_OUT = ZIN_OUT OR (ZTMP0 AND ZTMP1) (0xF8 = orAandBC)	\
+	 */								\
+	vpternlogq	$0xF8, ZTMP1, ZTMP0, ZIN_OUT;
+
+/*
+ * CTR(128 bits) needs to be incremented. Since there is no 128-bit add
+ * instruction, we need to increment 64-bit (least significant) and if an
+ * overflow is detected, increment the most significant 64-bits.
+ */
+#define INCR_CNTR_128(CTR, ZT, const, num)			\
+	vpaddq	const(%rip), XWORD(CTR), XTMP;			\
+	vptest	ddq_low_msk(%rip), XTMP;			\
+	jnz	64f;						\
+	vpaddq	ddq_high_add_1(%rip), XTMP, XTMP;		\
+	vpaddq	ddq_high_add_1(%rip), XWORD(CTR), XWORD(CTR);	\
+64:;	\
+	vinserti64x2	$num, XTMP, ZT, ZT;
+
+/* Increment 4, 128 bit counters stored in a ZMM register */
+#define INCR_CNTR_4_128(CTR, ZT)		\
+	vmovdqa64	XWORD(CTR), XWORD(ZT);	\
+	vshufi64x2	$0, ZT, ZT, ZT;		\
+	INCR_CNTR_128(CTR, ZT, ddq_add_1, 1)	\
+	INCR_CNTR_128(CTR, ZT, ddq_add_2, 2)	\
+	INCR_CNTR_128(CTR, ZT, ddq_add_3, 3)	\
+	vextracti32x4	$3, ZT, XWORD(CTR);
+
+#define up_count(CTR, NUM_blocks, num, ZTMP)		\
+.if NUM_blocks == num;					\
+	jmp	76f;					\
+.endif;							\
+.if NUM_blocks > num;					\
+	vpaddq	ddq_add_1(%rip), XWORD(CTR), XWORD(CTR);\
+	INCR_CNTR_4_128(CTR, ZTMP)			\
+.endif;							\
+76:;
+
+/* Increment 1 to 16 counters (1 to 4 ZMM registers based on number of blocks */
+#define INCR_CNTR_NUM_BLOCKS(CNTR, ZTMP0, ZTMP1, ZTMP2, ZTMP3, NUM)	\
+	INCR_CNTR_4_128(CNTR, ZTMP0)	\
+	up_count(CNTR, NUM, 1, ZTMP1)	\
+	up_count(CNTR, NUM, 2, ZTMP2)	\
+	up_count(CNTR, NUM, 3, ZTMP3)
+
+#define UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, of_num, num)	\
+	vshufi64x2	$0, ZWORD(CTR), ZWORD(CTR), ZWORD(CTR); \
+	vmovq		XWORD(CTR), TMP3;			\
+	cmp		$~of_num, TMP3;				\
+	jb		77f;					\
+	INCR_CNTR_NUM_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num)	\
+	jmp		78f;					\
+77:;								\
+	vpaddd	ddq_add_0_3(%rip), ZWORD(CTR), ZT1;		\
+.if (num > 1);							\
+	vpaddd	ddq_add_4_7(%rip), ZWORD(CTR), ZT2;		\
+.endif;								\
+.if (num > 2);							\
+	vpaddd	ddq_add_8_11(%rip), ZWORD(CTR), ZT3;		\
+.endif;								\
+.if (num > 3);							\
+	vpaddd	ddq_add_12_15(%rip), ZWORD(CTR), ZT4;		\
+.endif;								\
+78:;
+
+/* Prepares the AES counter blocka */
+#define PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_initial_blocks)	\
+.if num_initial_blocks == 1;					\
+	vmovdqa64	XWORD(CTR), XWORD(ZT1);			\
+.elseif num_initial_blocks == 2;				\
+	vshufi64x2	$0, YWORD(CTR), YWORD(CTR), YWORD(ZT1); \
+	vmovq		XWORD(CTR), TMP3;			\
+	cmp		$~1, TMP3;				\
+	jb		50f;					\
+	vmovdqa64	XWORD(CTR), XWORD(ZT1);			\
+	vshufi64x2	$0, YWORD(ZT1), YWORD(ZT1), YWORD(ZT1); \
+	INCR_CNTR_128(CTR, ZT1, ddq_add_1, 1)			\
+	vextracti32x4	$1, YWORD(ZT1), XWORD(CTR);		\
+	jmp		55f;					\
+50:;								\
+	vpaddd	ddq_add_0_3(%rip), YWORD(ZT1), YWORD(ZT1);	\
+.elseif num_initial_blocks <= 4;				\
+	UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, 3, 1)		\
+.elseif num_initial_blocks <= 8;				\
+	UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, 7, 2)		\
+.elseif num_initial_blocks <= 12;				\
+	UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, 11, 3)		\
+.else;								\
+	UPDATE_COUNTERS(CTR, ZT1, ZT2, ZT3, ZT4, 15, 4)		\
+.endif;								\
+55:;
+
+/* Extract and Shuffle the updated counters for AES rounds */
+#define	EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_initial_blocks)	\
+.if num_initial_blocks == 1;					\
+	vpshufb		XWORD(SHUFREG), CTR, XWORD(ZT1);	\
+.elseif num_initial_blocks == 2;				\
+	vextracti32x4	$1, YWORD(ZT1), CTR;			\
+	vpshufb		YWORD(SHUFREG), YWORD(ZT1), YWORD(ZT1); \
+.elseif num_initial_blocks <= 4;				\
+	vextracti32x4	$(num_initial_blocks - 1), ZT1, CTR;	\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+.elseif num_initial_blocks == 5;				\
+	vmovdqa64	XWORD(ZT2), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		XWORD(SHUFREG), XWORD(ZT2), XWORD(ZT2); \
+.elseif num_initial_blocks == 6;				\
+	vextracti32x4	$1, YWORD(ZT2), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		YWORD(SHUFREG), YWORD(ZT2), YWORD(ZT2); \
+.elseif num_initial_blocks == 7;				\
+	vextracti32x4	$2, ZT2, CTR;				\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+.elseif num_initial_blocks == 8;				\
+	vextracti32x4	$3, ZT2, CTR;				\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+.elseif num_initial_blocks == 9;				\
+	vmovdqa64	XWORD(ZT3), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		XWORD(SHUFREG), XWORD(ZT3), XWORD(ZT3); \
+.elseif num_initial_blocks == 10;				\
+	vextracti32x4	$1, YWORD(ZT3), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		YWORD(SHUFREG), YWORD(ZT3), YWORD(ZT3); \
+.elseif num_initial_blocks == 11;				\
+	vextracti32x4	$2, ZT3, CTR;				\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		SHUFREG, ZT3, ZT3;			\
+.elseif num_initial_blocks == 12;				\
+	vextracti32x4	$3, ZT3, CTR;				\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		SHUFREG, ZT3, ZT3;			\
+.elseif num_initial_blocks == 13;				\
+	vmovdqa64	XWORD(ZT4), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		SHUFREG, ZT3, ZT3;			\
+	vpshufb		XWORD(SHUFREG), XWORD(ZT4), XWORD(ZT4);	\
+.elseif num_initial_blocks == 14;				\
+	vextracti32x4	$1, YWORD(ZT4), CTR;			\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		SHUFREG, ZT3, ZT3;			\
+	vpshufb		YWORD(SHUFREG), YWORD(ZT4), YWORD(ZT4);	\
+.elseif num_initial_blocks == 15;				\
+	vextracti32x4	$2, ZT4, CTR;				\
+	vpshufb		SHUFREG, ZT1, ZT1;			\
+	vpshufb		SHUFREG, ZT2, ZT2;			\
+	vpshufb		SHUFREG, ZT3, ZT3;			\
+	vpshufb		SHUFREG, ZT4, ZT4;			\
+.endif;
+
+/* AES rounds and XOR with plain/cipher text */
+#define AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY_0, ZKEY_1, ZKEY_2, ZKEY_3, ZKEY_4, ZKEY_5, ZKEY_6, ZKEY_7, ZKEY_8, ZKEY_9, ZKEY_10, ZKEY_11, ZKEY_12, ZKEY_13, ZKEY_14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY0, 0, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY1, 1, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY2, 2, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY3, 3, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY4, 4, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY5, 5, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY6, 6, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY7, 7, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY8, 8, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY9, 9, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY10, 10, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+.if NROUNDS == 9;	\
+	jmp 28f;	\
+.else;			\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY11, 11, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY12, 12, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+.if NROUNDS == 11;	\
+	jmp 28f;	\
+.else;			\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY13, 13, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	ZMM_AESENC_ROUND_BLOCKS_0_16(ZT1, ZT2, ZT3, ZT4, ZKEY14, 14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+.endif;			\
+.endif;			\
+28:;
+
+/* IV is updated to current counter + 1 and returned to the upper glue layers */
+#define UPDATE_IV(CTR)				\
+	vpaddd	ONE(%rip), CTR, CTR;		\
+	vptest	ddq_low_msk(%rip), CTR;		\
+	jnz 27f;				\
+	vpaddq	ddq_high_add_1(%rip), CTR, CTR;	\
+27:;						\
+	vpshufb SHUF_MASK(%rip), CTR, CTR;	\
+	vmovdqu CTR, (arg5);
+
+/*
+ * Macro with support for a partial final block. It may look similar to
+ * INITIAL_BLOCKS but its usage is different. It is not meant to cipher
+ * counter blocks for the main by16 loop. Just ciphers amount of blocks.
+ * Used for small packets (<256 bytes). num_initial_blocks is expected
+ * to include the partial final block in the count.
+ */
+#define INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, num_initial_blocks, CTR, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	/* get load/store mask */				\
+	lea	byte64_len_to_mask_table(%rip), IA0;		\
+	mov	LENGTH, IA1;					\
+.if num_initial_blocks > 12;					\
+	sub	 $192, IA1;					\
+.elseif num_initial_blocks > 8;					\
+	sub	$128, IA1;					\
+.elseif num_initial_blocks > 4;					\
+	sub	$64, IA1;					\
+.endif;								\
+	kmovq	(IA0, IA1, 8), MASKREG;				\
+\
+	ZMM_LOAD_MASKED_BLOCKS_0_16(num_initial_blocks, PLAIN_CYPH_IN, 1, ZT5, ZT6, ZT7, ZT8, MASKREG)	\
+	PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_initial_blocks)	\
+	EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_initial_blocks)	\
+	AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)     \
+	/* write cipher/plain text back to output */		\
+	ZMM_STORE_MASKED_BLOCKS_0_16(num_initial_blocks, CYPH_PLAIN_OUT, 1, ZT1, ZT2, ZT3, ZT4, MASKREG)	\
+	UPDATE_IV(XWORD(CTR))
+
+/* This macro is used to "warm-up" pipeline for ENCRYPT_16_PARALLEL macro
+ * code. It is called only for data lengths 256 and above. The flow is as
+ * follows:
+ * - encrypt the initial num_initial_blocks blocks (can be 0)
+ * - encrypt the next 16 blocks
+ * - the last 16th block can be partial (lengths between 257 and 367)
+ * - partial block ciphering is handled within this macro
+ */
+#define INITIAL_BLOCKS(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, DATA_OFFSET, num_initial_blocks, CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+.if num_initial_blocks > 0;						\
+	/* load plain/cipher text */					\
+	ZMM_LOAD_BLOCKS_0_16(num_initial_blocks, PLAIN_CYPH_IN, 1, ZT5, ZT6, ZT7, ZT8, load_4_instead_of_3)	\
+	PREPARE_COUNTER_BLOCKS(CTR, ZT1, ZT2, ZT3, ZT4, num_initial_blocks)	\
+	EXTRACT_CNTR_VAL(ZT1, ZT2, ZT3, ZT4, SHUFREG, CTR, num_initial_blocks)	\
+	 AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, num_initial_blocks, NROUNDS)	\
+	/* write cipher/plain text back to output */			\
+	ZMM_STORE_BLOCKS_0_16(num_initial_blocks, CYPH_PLAIN_OUT, 1, ZT1, ZT2, ZT3, ZT4)	\
+	/* adjust data offset and length */				\
+	sub	$(num_initial_blocks * 16), LENGTH;			\
+	add	$(num_initial_blocks * 16), DATA_OFFSET;		\
+.endif;									\
+\
+	/* - cipher of num_initial_blocks is done			\
+	 * - prepare counter blocks for the next 16 blocks (ZT5-ZT8)	\
+	 * - shuffle the blocks for AES					\
+	 * - encrypt the next 16 blocks					\
+	 */								\
+\
+	 /* get text load/store mask (assume full mask by default) */	\
+	 mov	 $~0, IA0;						\
+.if num_initial_blocks > 0;						\
+	/* This macro is executed for length 256 and up, zero length	\
+	 * is checked in CNTR_ENC_DEC. We know there is a partial block	\
+	 * if: LENGTH - 16*num_initial_blocks < 256			\
+	 */								\
+	cmp	$256, LENGTH;						\
+	jge	56f;							\
+	mov	%rcx, IA1;						\
+	mov	$256, %ecx;						\
+	sub	LENGTH, %rcx;						\
+	shr	%cl, IA0;						\
+	mov	IA1, %rcx;						\
+56:;									\
+.endif;									\
+	kmovq	IA0, MASKREG;						\
+	/* load plain or cipher text */					\
+	vmovdqu8	(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT5;		\
+	vmovdqu8	64(PLAIN_CYPH_IN, DATA_OFFSET), ZT6;		\
+	vmovdqu8	128(PLAIN_CYPH_IN, DATA_OFFSET), ZT7;		\
+	vmovdqu8	192(PLAIN_CYPH_IN, DATA_OFFSET), ZT8{MASKREG}{z};	\
+\
+	/* prepare next counter blocks */				\
+	vshufi64x2	$0, ZWORD(CTR), ZWORD(CTR), ZWORD(CTR);		\
+.if num_initial_blocks > 0;						\
+	vmovq	XWORD(CTR), TMP3;					\
+	cmp	$~16, TMP3;						\
+	jb	58f;							\
+	vpaddq	ddq_add_1(%rip), XWORD(CTR), XWORD(CTR);		\
+	vptest	ddq_low_msk(%rip), XWORD(CTR);				\
+	jnz 57f;							\
+	vpaddq	ddq_high_add_1(%rip), XWORD(CTR), XWORD(CTR);		\
+57:;	\
+	INCR_CNTR_NUM_BLOCKS(CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, 4)	\
+	jmp 60f;							\
+58:;									\
+	vpaddd	      ddq_add_1_4(%rip), ZWORD(CTR), CTR_1_4;		\
+	vpaddd	      ddq_add_5_8(%rip), ZWORD(CTR), CTR_5_8;		\
+	vpaddd	      ddq_add_9_12(%rip), ZWORD(CTR), CTR_9_12;		\
+	vpaddd	      ddq_add_13_16(%rip), ZWORD(CTR), CTR_13_16;	\
+.else;									\
+	vmovq	XWORD(CTR), TMP3;					\
+	cmp	$~15, TMP3;						\
+	jb	59f;							\
+	INCR_CNTR_NUM_BLOCKS(CTR, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, 4)	\
+	jmp 60f;							\
+59:;									\
+	vpaddd	      ddq_add_0_3(%rip), ZWORD(CTR), CTR_1_4;		\
+	vpaddd	      ddq_add_4_7(%rip), ZWORD(CTR), CTR_5_8;		\
+	vpaddd	      ddq_add_8_11(%rip), ZWORD(CTR), CTR_9_12;		\
+	vpaddd	      ddq_add_12_15(%rip), ZWORD(CTR), CTR_13_16;	\
+.endif;									\
+60:;									\
+\
+	vpshufb		SHUFREG, CTR_1_4, ZT1;				\
+	vpshufb		SHUFREG, CTR_5_8, ZT2;				\
+	vpshufb		SHUFREG, CTR_9_12, ZT3;				\
+	vpshufb		SHUFREG, CTR_13_16, ZT4;			\
+\
+	AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, 16, NROUNDS)	\
+\
+	/* write cipher/plain text back to output */			\
+	vmovdqu8	ZT1, (CYPH_PLAIN_OUT, DATA_OFFSET);		\
+	vmovdqu8	ZT2, 64(CYPH_PLAIN_OUT, DATA_OFFSET, 1);	\
+	vmovdqu8	ZT3, 128(CYPH_PLAIN_OUT, DATA_OFFSET, 1);	\
+	vmovdqu8	ZT4, 192(CYPH_PLAIN_OUT, DATA_OFFSET, 1){MASKREG};	\
+\
+	vextracti32x4 $3, CTR_13_16, XWORD(CTR);			\
+	UPDATE_IV(CTR)							\
+\
+	/* check if there is partial block */				\
+	cmp	$256, LENGTH;						\
+	jl	61f;							\
+	/* adjust offset and length */					\
+	add	$256, DATA_OFFSET;					\
+	sub	$256, LENGTH;						\
+	jmp	62f;							\
+61:;									\
+	/* zero the length (all encryption is complete) */		\
+	xor	LENGTH, LENGTH;						\
+62:;
+
+/*
+ * This macro ciphers payloads shorter than 256 bytes. The number of blocks in
+ * the message comes as an argument. Depending on the number of blocks, an
+ * optimized variant of INITIAL_BLOCKS_PARTIAL is invoked
+ */
+#define CNTR_ENC_DEC_SMALL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, NUM_BLOCKS, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)  \
+	cmp	$8, NUM_BLOCKS;		\
+	je	38f;			\
+	jl	48f;			\
+	/* Initial blocks 9-16 */	\
+	cmp	$12, NUM_BLOCKS;	\
+	je	42f;			\
+	jl	49f;			\
+	/* Initial blocks 13-16 */	\
+	cmp	$16, NUM_BLOCKS;	\
+	je	46f;			\
+	cmp	$15, NUM_BLOCKS;	\
+	je	45f;			\
+	cmp	$14, NUM_BLOCKS;	\
+	je	44f;			\
+	cmp	$13, NUM_BLOCKS;	\
+	je	43f;			\
+49:;					\
+	cmp	$11, NUM_BLOCKS;	\
+	je	41f;			\
+	cmp	$10, NUM_BLOCKS;	\
+	je	40f;			\
+	cmp	$9, NUM_BLOCKS;		\
+	je	39f;			\
+48:;					\
+	cmp	$4, NUM_BLOCKS;		\
+	je	34f;			\
+	jl	47f;			\
+	/* Initial blocks 5-7 */	\
+	cmp	$7, NUM_BLOCKS;		\
+	je	37f;			\
+	cmp	$6, NUM_BLOCKS;		\
+	je	36f;			\
+	cmp	$5, NUM_BLOCKS;		\
+	je	35f;			\
+47:;					\
+	cmp	$3, NUM_BLOCKS;		\
+	je	33f;			\
+	cmp	$2, NUM_BLOCKS;		\
+	je	32f;			\
+	jmp	31f;			\
+46:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 16, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+45:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 15, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+44:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 14, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+43:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 13, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+42:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 12, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+41:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 11, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+40:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 10, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)      \
+	jmp	30f;	\
+39:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 9, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+38:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 8, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+37:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 7, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+36:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 6, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+35:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 5, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+34:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 4, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+33:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 3, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+32:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 2, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+	jmp	30f;	\
+31:;	\
+	INITIAL_BLOCKS_PARTIAL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, LENGTH, 1, CTR, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, ZTMP8, IA0, IA1, MASKREG, SHUFREG, NROUNDS, RBITS)	    \
+30:;
+
+/*
+ * This is the main CNTR macro. It operates on single stream and encrypts 16
+ * blocks at a time
+ */
+#define ENCRYPT_16_PARALLEL(KEY, CYPH_PLAIN_OUT, PLAIN_CYPH_IN, DATA_OFFSET, CTR_1_4, CTR_5_8, CTR_9_12, CTR_13_16, FULL_PARTIAL, IA0, IA1, LENGTH, ZT1, ZT2, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, MASKREG, SHUFREG, ADD8REG, NROUNDS, RBITS)	\
+	/* load/store mask (partial case) and load the text data */	\
+.ifc FULL_PARTIAL, full;	\
+	vmovdqu8	(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT5;		\
+	vmovdqu8	64(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT6;		\
+	vmovdqu8	128(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT7;	\
+	vmovdqu8	192(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT8;	\
+.else;									\
+	lea		byte64_len_to_mask_table(%rip), IA0;		\
+	mov		LENGTH, IA1;					\
+	sub		$192, IA1;					\
+	kmovq		(IA0, IA1, 8), MASKREG;				\
+	vmovdqu8	(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT5;		\
+	vmovdqu8	64(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT6;		\
+	vmovdqu8	128(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT7;	\
+	vmovdqu8	192(PLAIN_CYPH_IN, DATA_OFFSET, 1), ZT8{MASKREG}{z};	\
+.endif;									\
+	/*								\
+	 * populate counter blocks					\
+	 * CTR is shuffled outside the scope of this macro		\
+	 * it has to be kept in unshuffled form				\
+	 */								\
+	vpaddd		ADD8REG, CTR_1_4, CTR_1_4;			\
+	vpaddd		ADD8REG, CTR_5_8, CTR_5_8;			\
+	vpaddd		ADD8REG, CTR_9_12, CTR_9_12;			\
+	vpaddd		ADD8REG, CTR_13_16, CTR_13_16;			\
+	vpshufb		SHUFREG, CTR_1_4, ZT1;				\
+	vpshufb		SHUFREG, CTR_5_8, ZT2;				\
+	vpshufb		SHUFREG, CTR_9_12, ZT3;				\
+	vpshufb		SHUFREG, CTR_13_16, ZT4;			\
+	AES_XOR_ROUNDS(ZT1, ZT2, ZT3, ZT4, ZKEY0, ZKEY1, ZKEY2, ZKEY3, ZKEY4, ZKEY5, ZKEY6, ZKEY7, ZKEY8, ZKEY9, ZKEY10, ZKEY11, ZKEY12, ZKEY13, ZKEY14, ZT5, ZT6, ZT7, ZT8, 16, NROUNDS)     \
+	/*store the text data */					\
+	vmovdqu8	ZT1, (CYPH_PLAIN_OUT, DATA_OFFSET);		\
+	vmovdqu8	ZT2, 64(CYPH_PLAIN_OUT, DATA_OFFSET, 1);	\
+	vmovdqu8	ZT3, 128(CYPH_PLAIN_OUT, DATA_OFFSET, 1);	\
+.ifc FULL_PARTIAL, full;						\
+	vmovdqu8	ZT4, 192(CYPH_PLAIN_OUT, DATA_OFFSET, 1);	\
+.else;									\
+	vmovdqu8	ZT4, 192(CYPH_PLAIN_OUT, DATA_OFFSET, 1){MASKREG};	\
+.endif;
+
+/*
+ * CNTR_ENC_DEC Encodes/Decodes given data. Requires the input data be
+ * at least 1 byte long because of READ_SMALL_INPUT_DATA.
+ */
+#define CNTR_ENC_DEC(KEYS, DST, SRC, LEN, IV, NROUNDS)		\
+	or	LEN, LEN;					\
+	je	22f;						\
+/*								\
+ * Macro flow:							\
+ * - calculate the number of 16byte blocks in the message	\
+ * - process (number of 16byte blocks) mod 16			\
+ * - process 16x16 byte blocks at a time until all are done	\
+ */								\
+	xor	DATA_OFFSET, DATA_OFFSET;			\
+/* Prepare round keys */					\
+	vbroadcastf64x2 16*0(KEYS), ZKEY0;			\
+	vbroadcastf64x2 16*1(KEYS), ZKEY1;			\
+	vbroadcastf64x2 16*2(KEYS), ZKEY2;			\
+	vbroadcastf64x2 16*3(KEYS), ZKEY3;			\
+	vbroadcastf64x2 16*4(KEYS), ZKEY4;			\
+	vbroadcastf64x2 16*5(KEYS), ZKEY5;			\
+	vbroadcastf64x2 16*6(KEYS), ZKEY6;			\
+	vbroadcastf64x2 16*7(KEYS), ZKEY7;			\
+	vbroadcastf64x2 16*8(KEYS), ZKEY8;			\
+	vbroadcastf64x2 16*9(KEYS), ZKEY9;			\
+	vbroadcastf64x2 16*10(KEYS), ZKEY10;			\
+.if NROUNDS == 9;						\
+	jmp 23f;						\
+.else;								\
+	vbroadcastf64x2 16*11(KEYS), ZKEY11;			\
+	vbroadcastf64x2 16*12(KEYS), ZKEY12;			\
+.if NROUNDS == 11;						\
+	jmp 23f;						\
+.else;								\
+	vbroadcastf64x2 16*13(KEYS), ZKEY13;			\
+	vbroadcastf64x2 16*14(KEYS), ZKEY14;			\
+.endif;								\
+.endif;								\
+23:;								\
+	mov	$16, TMP2;					\
+	/* Set mask to read 16 IV bytes */			\
+	mov	mask_16_bytes(%rip), TMP0;			\
+	kmovq	TMP0, MASKREG;					\
+	vmovdqu8	(IV), CTR_BLOCKx{MASKREG};		\
+	vmovdqa64	SHUF_MASK(%rip), SHUFREG;		\
+	/* store IV as counter in LE format */			\
+	vpshufb XWORD(SHUFREG), CTR_BLOCKx, CTR_BLOCKx;		\
+	/* Determine how many blocks to process in INITIAL */	\
+	mov	LEN, TMP1;					\
+	shr	$4, TMP1;					\
+	and	$0xf, TMP1;					\
+	/*							\
+	 * Process one additional block in INITIAL_ macros if	\
+	 * there is a partial block.				\
+	 */							\
+	mov	LEN, TMP0;					\
+	and	$0xf, TMP0;					\
+	add	$0xf, TMP0;					\
+	shr	$4, TMP0;					\
+	add	TMP0, TMP1;					\
+	/* IA1 can be in the range from 0 to 16 */		\
+\
+	/* Less than 256B will be handled by the small message	\
+	 * code, which can process up to 16x blocks (16 bytes)	\
+	 */							\
+	cmp	$256, LEN;					\
+	jge	20f;						\
+	CNTR_ENC_DEC_SMALL(KEYS, DST, SRC, LEN, TMP1, CTR_BLOCKx, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP2, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	22f;						\
+20:;								\
+	and	$0xf, TMP1;					\
+	je	16f;						\
+	cmp	$15, TMP1;					\
+	je	15f;						\
+	cmp	$14, TMP1;					\
+	je	14f;						\
+	cmp	$13, TMP1;					\
+	je	13f;						\
+	cmp	$12, TMP1;					\
+	je	12f;						\
+	cmp	$11, TMP1;					\
+	je	11f;						\
+	cmp	$10, TMP1;					\
+	je	10f;						\
+	cmp	$9, TMP1;					\
+	je	9f;						\
+	cmp	$8, TMP1;					\
+	je	8f;						\
+	cmp	$7, TMP1;					\
+	je	7f;						\
+	cmp	$6, TMP1;					\
+	je	6f;						\
+	cmp	$5, TMP1;					\
+	je	5f;						\
+	cmp	$4, TMP1;					\
+	je	4f;						\
+	cmp	$3, TMP1;					\
+	je	3f;						\
+	cmp	$2, TMP1;					\
+	je	2f;						\
+	jmp	1f;						\
+\
+	and	$0xf, TMP1;					\
+	je	16f;						\
+	cmp	$8, TMP1;					\
+	je	8f;						\
+	jl	18f;						\
+	/* Initial blocks 9-15 */				\
+	cmp	$12, TMP1;					\
+	je	12f;						\
+	jl	19f;						\
+	/* Initial blocks 13-15 */				\
+	cmp	$15, TMP1;					\
+	je	15f;						\
+	cmp	$14, TMP1;					\
+	je	14f;						\
+	cmp	$13, TMP1;					\
+	je	13f;						\
+19:;								\
+	cmp	$11, TMP1;					\
+	je	11f;						\
+	cmp	$10, TMP1;					\
+	je	10f;						\
+	cmp	$9, TMP1;					\
+	je	9f;						\
+18:;								\
+	cmp	$4, TMP1;					\
+	je	4f;						\
+	jl	17f;						\
+	/* Initial blocks 5-7 */				\
+	cmp	$7, TMP1;					\
+	je	7f;						\
+	cmp	$6, TMP1;					\
+	je	6f;						\
+	cmp	$5, TMP1;					\
+	je	5f;						\
+17:;								\
+	cmp	$3, TMP1;					\
+	je	3f;						\
+	cmp	$2, TMP1;					\
+	je	2f;						\
+	jmp	1f;						\
+15:;								\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 15, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+14:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 14, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+13:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 13, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+12:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 12, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+11:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 11, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+10:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 10, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+9:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 9, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+8:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 8, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+7:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 7, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+6:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 6, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+5:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 5, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+4:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 4, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+3:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 3, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+2:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 2, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+1:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 1, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+	jmp	21f;	\
+16:;	\
+	INITIAL_BLOCKS(KEYS, DST, SRC, LEN, DATA_OFFSET, 0, CTR_BLOCKx, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, TMP0, TMP1, MASKREG, SHUFREG, NROUNDS, RBITS)	\
+21:;	\
+	or		LEN, LEN;				\
+	je		22f;					\
+	vmovdqa64	ddq_add_16(%rip), ADD8REG;		\
+	/* Process 15 full blocks plus a partial block */	\
+	cmp		$256, LEN;				\
+	jl		24f;					\
+25:;								\
+	ENCRYPT_16_PARALLEL(KEYS, DST, SRC, DATA_OFFSET, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, full, TMP0, TMP1, LEN, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, MASKREG, SHUFREG, ADD8REG, NROUNDS, RBITS)		\
+	add	$256, DATA_OFFSET;				\
+	sub	$256, LEN;					\
+	vextracti32x4 $3, CTR_BLOCK_13_16, CTR_BLOCKx;		\
+	UPDATE_IV(CTR_BLOCKx)					\
+	cmp	$256, LEN;					\
+	jge	25b;						\
+26:;								\
+	/*							\
+	 * Test to see if we need a by 16 with partial block.	\
+	 * At this point bytes remaining should be either zero	\
+	 * or between 241-255.					\
+	 */							\
+	or	LEN, LEN;					\
+	je	22f;						\
+24:;								\
+	ENCRYPT_16_PARALLEL(KEYS, DST, SRC, DATA_OFFSET, CTR_BLOCK_1_4, CTR_BLOCK_5_8, CTR_BLOCK_9_12, CTR_BLOCK_13_16, partial, TMP0, TMP1, LEN, ZTMP0, ZTMP1, ZTMP2, ZTMP3, ZTMP4, ZTMP5, ZTMP6, ZTMP7, MASKREG, SHUFREG, ADD8REG, NROUNDS, RBITS)	\
+	vextracti32x4 $3, CTR_BLOCK_13_16, CTR_BLOCKx;		\
+	UPDATE_IV(CTR_BLOCKx)					\
+22:;
+
+#define AES_CNTR_ENC_AVX512_BY16(keys, out, in, len, iv, NROUNDS)	\
+	FUNC_SAVE_CTR()							\
+	/* call the aes main loop */					\
+	CNTR_ENC_DEC(keys, out, in, len, iv, NROUNDS)			\
+	FUNC_RESTORE_CTR()							\
+	ret;
+
+/* Routine to do AES128/192/256 CTR enc/decrypt "by16"
+ * void aes_ctr_enc_128_avx512_by16/ aes_ctr_enc_192_avx512_by16/
+ *	aes_ctr_enc_256_avx512_by16/
+ *		(void *keys,
+ *		 u8 *out,
+ *		 const u8 *in,
+ *		 unsigned int num_bytes,
+ *		 u8 *iv);
+ */
+SYM_FUNC_START(aes_ctr_enc_128_avx512_by16)
+	AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 9)
+SYM_FUNC_END(aes_ctr_enc_128_avx512_by16)
+
+SYM_FUNC_START(aes_ctr_enc_192_avx512_by16)
+	AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 11)
+SYM_FUNC_END(aes_ctr_enc_192_avx512_by16)
+
+SYM_FUNC_START(aes_ctr_enc_256_avx512_by16)
+	AES_CNTR_ENC_AVX512_BY16(arg1, arg2, arg3, arg4, arg5, 13)
+SYM_FUNC_END(aes_ctr_enc_256_avx512_by16)
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index ad8a718..f45059e 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -46,6 +46,10 @@
 #define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
 #define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
 
+static bool use_avx512;
+module_param(use_avx512, bool, 0644);
+MODULE_PARM_DESC(use_avx512, "Use AVX512 optimized algorithm, if available");
+
 /* This data is stored at the end of the crypto_tfm struct.
  * It's a type of per "session" data storage location.
  * This needs to be 16 byte aligned.
@@ -191,6 +195,35 @@ asmlinkage void aes_ctr_enc_192_avx_by8(const u8 *in, u8 *iv,
 		void *keys, u8 *out, unsigned int num_bytes);
 asmlinkage void aes_ctr_enc_256_avx_by8(const u8 *in, u8 *iv,
 		void *keys, u8 *out, unsigned int num_bytes);
+
+#ifdef CONFIG_CRYPTO_AES_CTR_AVX512
+asmlinkage void aes_ctr_enc_128_avx512_by16(void *keys, u8 *out,
+					    const u8 *in,
+					    unsigned int num_bytes,
+					    u8 *iv);
+asmlinkage void aes_ctr_enc_192_avx512_by16(void *keys, u8 *out,
+					    const u8 *in,
+					    unsigned int num_bytes,
+					    u8 *iv);
+asmlinkage void aes_ctr_enc_256_avx512_by16(void *keys, u8 *out,
+					    const u8 *in,
+					    unsigned int num_bytes,
+					    u8 *iv);
+#else
+static inline void aes_ctr_enc_128_avx512_by16(void *keys, u8 *out,
+					       const u8 *in,
+					       unsigned int num_bytes,
+					       u8 *iv) {}
+static inline void aes_ctr_enc_192_avx512_by16(void *keys, u8 *out,
+					       const u8 *in,
+					       unsigned int num_bytes,
+					       u8 *iv) {}
+static inline void aes_ctr_enc_256_avx512_by16(void *keys, u8 *out,
+					       const u8 *in,
+					       unsigned int num_bytes,
+					       u8 *iv) {}
+#endif
+
 /*
  * asmlinkage void aesni_gcm_init_avx_gen2()
  * gcm_data *my_ctx_data, context data
@@ -487,6 +520,23 @@ static void aesni_ctr_enc_avx_tfm(struct crypto_aes_ctx *ctx, u8 *out,
 		aes_ctr_enc_256_avx_by8(in, iv, (void *)ctx, out, len);
 }
 
+static void aesni_ctr_enc_avx512_tfm(struct crypto_aes_ctx *ctx, u8 *out,
+				     const u8 *in, unsigned int len, u8 *iv)
+{
+	/*
+	 * based on key length, override with the by16 version
+	 * of ctr mode encryption/decryption for improved performance.
+	 * aes_set_key_common() ensures that key length is one of
+	 * {128,192,256}
+	 */
+	if (ctx->key_length == AES_KEYSIZE_128)
+		aes_ctr_enc_128_avx512_by16((void *)ctx, out, in, len, iv);
+	else if (ctx->key_length == AES_KEYSIZE_192)
+		aes_ctr_enc_192_avx512_by16((void *)ctx, out, in, len, iv);
+	else
+		aes_ctr_enc_256_avx512_by16((void *)ctx, out, in, len, iv);
+}
+
 static int ctr_crypt(struct skcipher_request *req)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -1076,7 +1126,12 @@ static int __init aesni_init(void)
 		aesni_gcm_tfm = &aesni_gcm_tfm_sse;
 	}
 	aesni_ctr_enc_tfm = aesni_ctr_enc;
-	if (boot_cpu_has(X86_FEATURE_AVX)) {
+	if (use_avx512 && IS_ENABLED(CONFIG_CRYPTO_AES_CTR_AVX512) &&
+	    cpu_feature_enabled(X86_FEATURE_VAES)) {
+		/* Ctr mode performance optimization using AVX512 */
+		aesni_ctr_enc_tfm = aesni_ctr_enc_avx512_tfm;
+		pr_info("AES CTR mode by16 optimization enabled\n");
+	} else if (boot_cpu_has(X86_FEATURE_AVX)) {
 		/* optimize performance of ctr mode encryption transform */
 		aesni_ctr_enc_tfm = aesni_ctr_enc_avx_tfm;
 		pr_info("AES CTR mode by8 optimization enabled\n");
diff --git a/arch/x86/crypto/avx512_vaes_common.S b/arch/x86/crypto/avx512_vaes_common.S
index f3ee898..a499e22 100644
--- a/arch/x86/crypto/avx512_vaes_common.S
+++ b/arch/x86/crypto/avx512_vaes_common.S
@@ -348,6 +348,96 @@ POLY2:
 byteswap_const:
 .octa	0x000102030405060708090A0B0C0D0E0F
 
+.align 16
+ddq_low_msk:
+.octa  0x0000000000000000FFFFFFFFFFFFFFFF
+
+.align 16
+ddq_high_add_1:
+.octa  0x00000000000000010000000000000000
+
+.align 16
+ddq_add_1:
+.octa  0x00000000000000000000000000000001
+
+.align 16
+ddq_add_2:
+.octa  0x00000000000000000000000000000002
+
+.align 16
+ddq_add_3:
+.octa  0x00000000000000000000000000000003
+
+.align 16
+ddq_add_4:
+.octa  0x00000000000000000000000000000004
+
+.align 64
+ddq_add_12_15:
+.octa  0x0000000000000000000000000000000c
+.octa  0x0000000000000000000000000000000d
+.octa  0x0000000000000000000000000000000e
+.octa  0x0000000000000000000000000000000f
+
+.align 64
+ddq_add_8_11:
+.octa  0x00000000000000000000000000000008
+.octa  0x00000000000000000000000000000009
+.octa  0x0000000000000000000000000000000a
+.octa  0x0000000000000000000000000000000b
+
+.align 64
+ddq_add_4_7:
+.octa  0x00000000000000000000000000000004
+.octa  0x00000000000000000000000000000005
+.octa  0x00000000000000000000000000000006
+.octa  0x00000000000000000000000000000007
+
+.align 64
+ddq_add_0_3:
+.octa  0x00000000000000000000000000000000
+.octa  0x00000000000000000000000000000001
+.octa  0x00000000000000000000000000000002
+.octa  0x00000000000000000000000000000003
+
+.align 64
+ddq_add_13_16:
+.octa  0x0000000000000000000000000000000d
+.octa  0x0000000000000000000000000000000e
+.octa  0x0000000000000000000000000000000f
+.octa  0x00000000000000000000000000000010
+
+.align 64
+ddq_add_9_12:
+.octa  0x00000000000000000000000000000009
+.octa  0x0000000000000000000000000000000a
+.octa  0x0000000000000000000000000000000b
+.octa  0x0000000000000000000000000000000c
+
+.align 64
+ddq_add_5_8:
+.octa  0x00000000000000000000000000000005
+.octa  0x00000000000000000000000000000006
+.octa  0x00000000000000000000000000000007
+.octa  0x00000000000000000000000000000008
+
+.align 64
+ddq_add_1_4:
+.octa  0x00000000000000000000000000000001
+.octa  0x00000000000000000000000000000002
+.octa  0x00000000000000000000000000000003
+.octa  0x00000000000000000000000000000004
+
+.align 64
+ddq_add_16:
+.octa  0x00000000000000000000000000000010
+.octa  0x00000000000000000000000000000010
+.octa  0x00000000000000000000000000000010
+.octa  0x00000000000000000000000000000010
+
+mask_16_bytes:
+.octa  0x000000000000ffff
+
 .text
 
 /* Save register content for the caller */
@@ -1209,3 +1299,335 @@ byteswap_const:
 	vpshufb XWORD(ZT13), XWORD(ZT1), XWORD(ZT1); \
 	GHASH_1_TO_16(GDATA_KEY, OFFSET, ZWORD(AAD_HASH), ZT0, ZT3, ZT4, ZT5, ZT6, ZT7, ZT8, ZT9, ZT10, ZWORD(AAD_HASH), ZT1, no_zmm, no_zmm, no_zmm, 1, 1, single_call, NULL, NULL, NULL, NULL, NULL, NULL) \
 0:;
+
+/*
+ * Generic macro to produce code that executes OPCODE instruction
+ * on selected number of AES blocks (16 bytes long ) between 0 and 16.
+ * All three operands of the instruction come from registers.
+ */
+#define ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUM_BLOCKS, OPCODE, DST0, DST1, DST2, DST3, SRC1_0, SRC1_1, SRC1_2, SRC1_3, SRC2_0, SRC2_1, SRC2_2, SRC2_3) \
+.set blocks_left,NUM_BLOCKS;					       \
+.if NUM_BLOCKS < 4;						       \
+       .if blocks_left == 1;					       \
+	       OPCODE	     XWORD(SRC2_0), XWORD(SRC1_0), XWORD(DST0);\
+       .elseif blocks_left == 2;				       \
+	       OPCODE	     YWORD(SRC2_0), YWORD(SRC1_0), YWORD(DST0);\
+       .elseif blocks_left == 3;				       \
+	       OPCODE	     SRC2_0, SRC1_0, DST0;		       \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8;			       \
+       OPCODE  SRC2_0, SRC1_0, DST0;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       .if blocks_left == 1;					       \
+	       OPCODE	     XWORD(SRC2_1), XWORD(SRC1_1), XWORD(DST1);\
+       .elseif blocks_left == 2;				       \
+	       OPCODE	     YWORD(SRC2_1), YWORD(SRC1_1), YWORD(DST1);\
+       .elseif blocks_left == 3;				       \
+	       OPCODE	     SRC2_1, SRC1_1, DST1;		       \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12;			       \
+       OPCODE  SRC2_0, SRC1_0, DST0;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_1, SRC1_1, DST1;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       .if blocks_left == 1;					       \
+	       OPCODE	     XWORD(SRC2_2), XWORD(SRC1_2), XWORD(DST2);\
+       .elseif blocks_left == 2;				       \
+	       OPCODE	     YWORD(SRC2_2), YWORD(SRC1_2), YWORD(DST2);\
+       .elseif blocks_left == 3;				       \
+	       OPCODE	     SRC2_2, SRC1_2, DST2;		       \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 12 && NUM_BLOCKS < 16;			       \
+       OPCODE  SRC2_0, SRC1_0, DST0;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_1, SRC1_1, DST1;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_2, SRC1_2, DST2;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       .if blocks_left == 1;					       \
+	       OPCODE	     XWORD(SRC2_3), XWORD(SRC1_3), XWORD(DST3);\
+       .elseif blocks_left == 2;				       \
+	       OPCODE	     YWORD(SRC2_3), YWORD(SRC1_3), YWORD(DST3);\
+       .elseif blocks_left == 3;				       \
+	       OPCODE	     SRC2_3, SRC1_3, DST3;		       \
+       .endif;							       \
+.else;								       \
+       OPCODE  SRC2_0, SRC1_0, DST0;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_1, SRC1_1, DST1;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_2, SRC1_2, DST2;				       \
+       .set blocks_left, blocks_left - 4;			       \
+       OPCODE  SRC2_3, SRC1_3, DST3;				       \
+       .set blocks_left, blocks_left - 4;			       \
+.endif;
+
+/*
+ * Handles AES encryption rounds. It handles special cases: the last and
+ * first rounds. Optionally, it performs XOR with data after the last AES
+ * round. Uses NROUNDS parameter to check what needs to be done for the
+ * current round.
+ */
+#define ZMM_AESENC_ROUND_BLOCKS_0_16(L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, ROUND, D0_3, D4_7, D8_11, D12_15, NUMBL, NROUNDS) \
+.if ROUND < 1;			       \
+       ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vpxorq, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY)       \
+.endif;					       \
+.if (ROUND >= 1) && (ROUND <= NROUNDS);        \
+       ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vaesenc, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY)      \
+.endif;					       \
+.if ROUND > NROUNDS;		       \
+       ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vaesenclast, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, KEY, KEY, KEY, KEY)  \
+       .ifnc D0_3, no_data;	       \
+       .ifnc D4_7, no_data;	       \
+       .ifnc D8_11, no_data;	       \
+       .ifnc D12_15, no_data;	       \
+	       ZMM_OPCODE3_DSTR_SRC1R_SRC2R_BLOCKS_0_16(NUMBL, vpxorq, L0B0_3, L0B4_7, L0B8_11, L0B12_15, L0B0_3, L0B4_7, L0B8_11, L0B12_15, D0_3, D4_7, D8_11, D12_15)        \
+       .endif;			       \
+       .endif;			       \
+       .endif;			       \
+       .endif;			       \
+.endif;
+
+/*
+ * Loads specified number of AES blocks into ZMM registers using mask register
+ * for the last loaded register (xmm, ymm or zmm). Loads take place at 1 byte
+ * granularity.
+ */
+#define ZMM_LOAD_MASKED_BLOCKS_0_16(NUM_BLOCKS, INP, DATA_OFFSET, DST0, DST1, DST2, DST3, MASK)        \
+.set src_offset,0;						       \
+.set blocks_left, NUM_BLOCKS;					       \
+.if NUM_BLOCKS <= 4;						       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST0){MASK}{z};     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST0){MASK}{z};     \
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0{MASK}{z};    \
+       .endif;							       \
+.elseif NUM_BLOCKS > 4 && NUM_BLOCKS <= 8;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST1){MASK}{z};     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST1){MASK}{z};     \
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST1{MASK}{z};    \
+       .endif;							       \
+.elseif NUM_BLOCKS > 8 && NUM_BLOCKS <= 12;			       \
+	vmovdqu8	src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+	vmovdqu8	src_offset(INP, DATA_OFFSET), DST1;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST2){MASK}{z};     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST2){MASK}{z};     \
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST2{MASK}{z};    \
+       .endif;							       \
+.else;								       \
+	vmovdqu8	src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+	vmovdqu8	src_offset(INP, DATA_OFFSET), DST1;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST2;	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST3){MASK}{z};     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST3){MASK}{z};     \
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST3{MASK}{z};    \
+       .endif;							       \
+.endif;
+
+/*
+ * Stores specified number of AES blocks from ZMM registers with mask register
+ * for the last loaded register (xmm, ymm or zmm). Stores take place at 1 byte
+ * granularity.
+ */
+#define ZMM_STORE_MASKED_BLOCKS_0_16(NUM_BLOCKS, OUTP, DATA_OFFSET, SRC0, SRC1, SRC2, SRC3, MASK) \
+.set blocks_left, NUM_BLOCKS;					       \
+.set dst_offset, 0;						       \
+.if NUM_BLOCKS <= 4;						       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC0), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET){MASK};      \
+       .endif;							       \
+.elseif NUM_BLOCKS > 4 && NUM_BLOCKS <=8;			       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC1), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        SRC1, dst_offset(OUTP, DATA_OFFSET){MASK};      \
+       .endif;							       \
+.elseif NUM_BLOCKS > 8 && NUM_BLOCKS <= 12;			       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC1, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC2), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        SRC2, dst_offset(OUTP, DATA_OFFSET){MASK};      \
+       .endif;							       \
+.else;								       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC1, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC2, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set blocks_left, blocks_left - 4;			       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC3), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET){MASK};\
+       .elseif (blocks_left == 3 || blocks_left == 4);		       \
+	       vmovdqu8        SRC3, dst_offset(OUTP, DATA_OFFSET){MASK};      \
+       .endif;							       \
+.endif;
+
+/* Loads specified number of AES blocks into ZMM registers */
+#define ZMM_LOAD_BLOCKS_0_16(NUM_BLOCKS, INP, DATA_OFFSET, DST0, DST1, DST2, DST3, FLAGS) \
+.set src_offset, 0;						       \
+.set blocks_left, NUM_BLOCKS % 4;				       \
+.if NUM_BLOCKS < 4;						       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST0);      \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST0);      \
+       .elseif blocks_left == 3;				       \
+	       .ifc FLAGS, load_4_instead_of_3;			       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0;     \
+	       .else;						       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST0);      \
+	       vinserti64x2    $2, src_offset + 32(INP, DATA_OFFSET), DST0, DST0;      \
+	       .endif;						       \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST1);      \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST1);      \
+       .elseif blocks_left == 3;				       \
+	       .ifc FLAGS, load_4_instead_of_3;			       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST1;     \
+	       .else;						       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST1);      \
+	       vinserti64x2    $2, src_offset + 32(INP, DATA_OFFSET), DST1, DST1;      \
+	       .endif;						       \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set src_offset, src_offset + 64;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST1;	       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST2);      \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST2);      \
+       .elseif blocks_left == 3;				       \
+	       .ifc FLAGS, load_4_instead_of_3;			       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST2;     \
+	       .else;						       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST2);      \
+	       vinserti64x2    $2, src_offset + 32(INP, DATA_OFFSET), DST2, DST2; \
+	       .endif;						       \
+       .endif;							       \
+.else;								       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST0;	       \
+       .set src_offset, src_offset + 64;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST1;	       \
+       .set src_offset, src_offset + 64;			       \
+       vmovdqu8        src_offset(INP, DATA_OFFSET), DST2;	       \
+       .set src_offset, src_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), XWORD(DST3);      \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST3);      \
+       .elseif blocks_left == 3;				       \
+	       .ifc FLAGS, load_4_instead_of_3;			       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), DST3;     \
+	       .else;						       \
+	       vmovdqu8        src_offset(INP, DATA_OFFSET), YWORD(DST3);      \
+	       vinserti64x2    $2, src_offset + 32(INP, DATA_OFFSET), DST3, DST3; \
+	       .endif;						       \
+       .endif;							       \
+.endif;
+
+/* Stores specified number of AES blocks from ZMM registers */
+#define ZMM_STORE_BLOCKS_0_16(NUM_BLOCKS, OUTP, DATA_OFFSET, SRC0, SRC1, SRC2, SRC3)   \
+.set dst_offset, 0;						       \
+.set blocks_left, NUM_BLOCKS % 4;				       \
+.if NUM_BLOCKS < 4;						       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC0), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 3;				       \
+	       vmovdqu8        YWORD(SRC0), dst_offset(OUTP, DATA_OFFSET);     \
+	       vextracti32x4   $2, SRC0, dst_offset + 32(OUTP, DATA_OFFSET);   \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 4 && NUM_BLOCKS < 8;			       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC1), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 3;				       \
+	       vmovdqu8        YWORD(SRC1), dst_offset(OUTP, DATA_OFFSET);     \
+	       vextracti32x4   $2, SRC1, dst_offset + 32(OUTP, DATA_OFFSET);   \
+       .endif;							       \
+.elseif NUM_BLOCKS >= 8 && NUM_BLOCKS < 12;			       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC1, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC2), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 3;				       \
+	       vmovdqu8        YWORD(SRC2), dst_offset(OUTP, DATA_OFFSET);     \
+	       vextracti32x4   $2, SRC2, dst_offset + 32(OUTP, DATA_OFFSET);   \
+       .endif;							       \
+.else;								       \
+       vmovdqu8        SRC0, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC1, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       vmovdqu8        SRC2, dst_offset(OUTP, DATA_OFFSET);	       \
+       .set dst_offset, dst_offset + 64;			       \
+       .if blocks_left == 1;					       \
+	       vmovdqu8        XWORD(SRC3), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 2;				       \
+	       vmovdqu8        YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET);     \
+       .elseif blocks_left == 3;				       \
+	       vmovdqu8        YWORD(SRC3), dst_offset(OUTP, DATA_OFFSET);     \
+	       vextracti32x4   $2, SRC3, dst_offset + 32(OUTP, DATA_OFFSET);   \
+       .endif;							       \
+.endif;
diff --git a/arch/x86/include/asm/disabled-features.h b/arch/x86/include/asm/disabled-features.h
index 1192dea..251c652 100644
--- a/arch/x86/include/asm/disabled-features.h
+++ b/arch/x86/include/asm/disabled-features.h
@@ -62,6 +62,12 @@
 # define DISABLE_VPCLMULQDQ	(1 << (X86_FEATURE_VPCLMULQDQ & 31))
 #endif
 
+#if defined(CONFIG_AS_VAES_AVX512)
+# define DISABLE_VAES		0
+#else
+# define DISABLE_VAES		(1 << (X86_FEATURE_VAES & 31))
+#endif
+
 #ifdef CONFIG_IOMMU_SUPPORT
 # define DISABLE_ENQCMD	0
 #else
@@ -88,7 +94,7 @@
 #define DISABLED_MASK14	0
 #define DISABLED_MASK15	0
 #define DISABLED_MASK16	(DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
-			 DISABLE_ENQCMD|DISABLE_VPCLMULQDQ)
+			 DISABLE_ENQCMD|DISABLE_VPCLMULQDQ|DISABLE_VAES)
 #define DISABLED_MASK17	0
 #define DISABLED_MASK18	0
 #define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 19)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 70d1d35..3043849 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -649,6 +649,18 @@ config CRYPTO_GHASH_CLMUL_NI_AVX512
 	depends on CRYPTO_GHASH_CLMUL_NI_INTEL
 	depends on AS_VPCLMULQDQ
 
+# We default CRYPTO_AES_CTR_AVX512 to Y but depend on CRYPTO_AVX512 in
+# order to have a singular option (CRYPTO_AVX512) select multiple algorithms
+# when supported. Specifically, if the platform and/or toolset does not
+# support VPLMULQDQ. Then this algorithm should not be supported as part of
+# the set that CRYPTO_AVX512 selects.
+config CRYPTO_AES_CTR_AVX512
+	bool
+	default y
+	depends on CRYPTO_AVX512
+	depends on CRYPTO_AES_NI_INTEL
+	depends on AS_VAES_AVX512
+
 config CRYPTO_CRC32C_SPARC64
 	tristate "CRC32c CRC algorithm (SPARC64)"
 	depends on SPARC64
-- 
2.7.4





[Index of Archives]     [Kernel]     [Gnu Classpath]     [Gnu Crypto]     [DM Crypt]     [Netfilter]     [Bugtraq]

  Powered by Linux