Patch replaces 'movb' instructions with 'movzbl' to break false register dependencies and interleaves instructions better for out-of-order scheduling. Also move common round code to separate function to reduce object size. Tested on Core i5-2450M. Cc: Johannes Goetzfried <Johannes.Goetzfried@xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx> Signed-off-by: Jussi Kivilinna <jussi.kivilinna@xxxxxxxx> --- arch/x86/crypto/twofish-avx-x86_64-asm_64.S | 144 +++++++++++++++++---------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S index 35f4557..42b27b7 100644 --- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -47,15 +47,22 @@ #define RC2 %xmm6 #define RD2 %xmm7 -#define RX %xmm8 -#define RY %xmm9 +#define RX0 %xmm8 +#define RY0 %xmm9 -#define RK1 %xmm10 -#define RK2 %xmm11 +#define RX1 %xmm10 +#define RY1 %xmm11 + +#define RK1 %xmm12 +#define RK2 %xmm13 + +#define RT %xmm14 #define RID1 %rax +#define RID1d %eax #define RID1b %al #define RID2 %rbx +#define RID2d %ebx #define RID2b %bl #define RGI1 %rdx @@ -73,40 +80,45 @@ #define RGS3d %r10d -#define lookup_32bit(t0, t1, t2, t3, src, dst) \ - movb src ## bl, RID1b; \ - movb src ## bh, RID2b; \ +#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \ + movzbl src ## bl, RID1d; \ + movzbl src ## bh, RID2d; \ + shrq $16, src; \ movl t0(CTX, RID1, 4), dst ## d; \ xorl t1(CTX, RID2, 4), dst ## d; \ - shrq $16, src; \ - movb src ## bl, RID1b; \ - movb src ## bh, RID2b; \ + movzbl src ## bl, RID1d; \ + movzbl src ## bh, RID2d; \ + interleave_op(il_reg); \ xorl t2(CTX, RID1, 4), dst ## d; \ xorl t3(CTX, RID2, 4), dst ## d; +#define dummy(d) /* do nothing */ + +#define shr_next(reg) \ + shrq $16, reg; + #define G(a, x, t0, t1, t2, t3) \ vmovq a, RGI1; \ - vpsrldq $8, a, x; \ - vmovq x, RGI2; \ + vpextrq $1, a, RGI2; \ \ - lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \ - shrq $16, RGI1; \ - lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \ - shlq $32, RGS2; \ - orq RGS1, RGS2; \ + lookup_32bit(t0, t1, t2, t3, RGI1, RGS1, shr_next, RGI1); \ + vmovd RGS1d, x; \ + lookup_32bit(t0, t1, t2, t3, RGI1, RGS2, dummy, none); \ + vpinsrd $1, RGS2d, x, x; \ \ - lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \ - shrq $16, RGI2; \ - lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \ - shlq $32, RGS3; \ - orq RGS1, RGS3; \ - \ - vmovq RGS2, x; \ - vpinsrq $1, RGS3, x, x; + lookup_32bit(t0, t1, t2, t3, RGI2, RGS1, shr_next, RGI2); \ + vpinsrd $2, RGS1d, x, x; \ + lookup_32bit(t0, t1, t2, t3, RGI2, RGS3, dummy, none); \ + vpinsrd $3, RGS3d, x, x; + +#define encround_g1g2(a, b, c, d, x, y) \ + G(a, x, s0, s1, s2, s3); \ + G(b, y, s1, s2, s3, s0); -#define encround(a, b, c, d, x, y) \ - G(a, x, s0, s1, s2, s3); \ - G(b, y, s1, s2, s3, s0); \ +#define encround_end(a, b, c, d, x, y) \ + vpslld $1, d, RT; \ + vpsrld $(32 - 1), d, d; \ + vpor d, RT, d; \ vpaddd x, y, x; \ vpaddd y, x, y; \ vpaddd x, RK1, x; \ @@ -115,14 +127,16 @@ vpsrld $1, c, x; \ vpslld $(32 - 1), c, c; \ vpor c, x, c; \ - vpslld $1, d, x; \ - vpsrld $(32 - 1), d, d; \ - vpor d, x, d; \ vpxor d, y, d; -#define decround(a, b, c, d, x, y) \ - G(a, x, s0, s1, s2, s3); \ - G(b, y, s1, s2, s3, s0); \ +#define decround_g1g2(a, b, c, d, x, y) \ + G(a, x, s0, s1, s2, s3); \ + G(b, y, s1, s2, s3, s0); + +#define decround_end(a, b, c, d, x, y) \ + vpslld $1, c, RT; \ + vpsrld $(32 - 1), c, c; \ + vpor c, RT, c; \ vpaddd x, y, x; \ vpaddd y, x, y; \ vpaddd y, RK2, y; \ @@ -130,23 +144,50 @@ vpsrld $1, d, y; \ vpslld $(32 - 1), d, d; \ vpor d, y, d; \ - vpslld $1, c, y; \ - vpsrld $(32 - 1), c, c; \ - vpor c, y, c; \ vpaddd x, RK1, x; \ vpxor x, c, c; +.align 4 +encround_RARBRCRD: + encround_g1g2(RA1, RB1, RC1, RD1, RX0, RY0); + encround_g1g2(RA2, RB2, RC2, RD2, RX1, RY1); + encround_end(RA1, RB1, RC1, RD1, RX0, RY0); + encround_end(RA2, RB2, RC2, RD2, RX1, RY1); + ret; + +.align 4 +encround_RCRDRARB: + encround_g1g2(RC1, RD1, RA1, RB1, RX0, RY0); + encround_g1g2(RC2, RD2, RA2, RB2, RX1, RY1); + encround_end(RC1, RD1, RA1, RB1, RX0, RY0); + encround_end(RC2, RD2, RA2, RB2, RX1, RY1); + ret; + #define encrypt_round(n, a, b, c, d) \ vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ - encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ - encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); + call encround_ ## a ## b ## c ## d; + +.align 4 +decround_RARBRCRD: + decround_g1g2(RA1, RB1, RC1, RD1, RX0, RY0); + decround_g1g2(RA2, RB2, RC2, RD2, RX1, RY1); + decround_end(RA1, RB1, RC1, RD1, RX0, RY0); + decround_end(RA2, RB2, RC2, RD2, RX1, RY1); + ret; + +.align 4 +decround_RCRDRARB: + decround_g1g2(RC1, RD1, RA1, RB1, RX0, RY0); + decround_g1g2(RC2, RD2, RA2, RB2, RX1, RY1); + decround_end(RC1, RD1, RA1, RB1, RX0, RY0); + decround_end(RC2, RD2, RA2, RB2, RX1, RY1); + ret; #define decrypt_round(n, a, b, c, d) \ vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ - decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ - decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); + call decround_ ## a ## b ## c ## d; #define encrypt_cycle(n) \ encrypt_round((2*n), RA, RB, RC, RD); \ @@ -156,7 +197,6 @@ decrypt_round(((2*n) + 1), RC, RD, RA, RB); \ decrypt_round((2*n), RA, RB, RC, RD); - #define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ vpunpckldq x1, x0, t0; \ vpunpckhdq x1, x0, t2; \ @@ -222,8 +262,8 @@ __twofish_enc_blk_8way: vmovdqu w(CTX), RK1; leaq (4*4*4)(%rdx), %rax; - inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); - inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); + inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); + inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); xorq RID1, RID1; xorq RID2, RID2; @@ -247,14 +287,14 @@ __twofish_enc_blk_8way: testb %cl, %cl; jnz __enc_xor8; - outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); - outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); + outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); ret; __enc_xor8: - outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); - outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); + outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); ret; @@ -274,8 +314,8 @@ twofish_dec_blk_8way: vmovdqu (w+4*4)(CTX), RK1; leaq (4*4*4)(%rdx), %rax; - inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); - inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2); + inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2); xorq RID1, RID1; xorq RID2, RID2; @@ -294,7 +334,7 @@ twofish_dec_blk_8way: popq %rbx; leaq (4*4*4)(%rsi), %rax; - outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); - outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); + outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2); + outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2); ret; -- To unsubscribe from this list: send the line "unsubscribe linux-crypto" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
