This adds a scalar implementation of AES, based on the precomputed tables
that are exposed by the generic AES code. Since rotates are cheap on arm64,
this implementation only uses the 4 core tables (of 1 KB each), and avoids
the prerotated ones, reducing the D-cache footprint by 75%.
On Cortex-A57, this code manages 13.0 cycles per byte, which is ~34% faster
than the generic C code. (Note that this is still >13x slower than the code
that uses the optional ARMv8 Crypto Extensions, which manages <1 cycles per
byte.)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@xxxxxxxxxx>
---
Raw performance data after the patch, which was generated on a 2 GHz
Cortex-A57 (AMD Seattle B1).
arch/arm64/crypto/Kconfig | 4 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/aes-cipher-core.S | 126 ++++++++++++++++++++
arch/arm64/crypto/aes-cipher-glue.c | 69 +++++++++++
4 files changed, 202 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0bf0f531f539..0826f8e599a6 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -41,6 +41,10 @@ config CRYPTO_CRC32_ARM64_CE
depends on KERNEL_MODE_NEON && CRC32
select CRYPTO_HASH
+config CRYPTO_AES_ARM64
+ tristate "AES core cipher using scalar instructions"
+ select CRYPTO_AES
+
config CRYPTO_AES_ARM64_CE
tristate "AES core cipher using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 9d2826c5fccf..a893507629eb 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -44,6 +44,9 @@ sha512-arm64-y := sha512-glue.o sha512-core.o
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
+obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
+aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..22d1bc46feba
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -0,0 +1,126 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@xxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+ .align 5
+
+ rk .req x0
+ out .req x1
+ in .req x2
+ rounds .req x3
+ tt .req x4
+ lt .req x2
+
+ .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
+ ldp \out0, \out1, [rk], #8
+
+ ubfx w13, \in0, #0, #8
+ ubfx w14, \in1, #8, #8
+ ldr w13, [tt, w13, uxtw #2]
+ ldr w14, [tt, w14, uxtw #2]
+
+ ubfx w15, \in2, #16, #8
+ ubfx w16, \in3, #24, #8
+ ldr w15, [tt, w15, uxtw #2]
+ ldr w16, [tt, w16, uxtw #2]
+
+ .if \enc
+ ubfx w17, \in1, #0, #8
+ ubfx w18, \in2, #8, #8
+ .else
+ ubfx w17, \in3, #0, #8
+ ubfx w18, \in0, #8, #8
+ .endif
+ ldr w17, [tt, w17, uxtw #2]
+ ldr w18, [tt, w18, uxtw #2]
+
+ .if \enc
+ ubfx \t0, \in3, #16, #8
+ ubfx \t1, \in0, #24, #8
+ .else
+ ubfx \t0, \in1, #16, #8
+ ubfx \t1, \in2, #24, #8
+ .endif
+ ldr \t0, [tt, \t0, uxtw #2]
+ ldr \t1, [tt, \t1, uxtw #2]
+
+ eor \out0, \out0, w13
+ eor \out1, \out1, w17
+ eor \out0, \out0, w14, ror #24
+ eor \out1, \out1, w18, ror #24
+ eor \out0, \out0, w15, ror #16
+ eor \out1, \out1, \t0, ror #16
+ eor \out0, \out0, w16, ror #8
+ eor \out1, \out1, \t1, ror #8
+ .endm
+
+ .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
+ __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
+ .endm
+
+ .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
+ __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
+ .endm
+
+ .macro do_crypt, round, ttab, ltab
+ ldp w5, w6, [in]
+ ldp w7, w8, [in, #8]
+ ldp w9, w10, [rk], #16
+ ldp w11, w12, [rk, #-8]
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ eor w5, w5, w9
+ eor w6, w6, w10
+ eor w7, w7, w11
+ eor w8, w8, w12
+
+ ldr tt, =\ttab
+ ldr lt, =\ltab
+
+ tbnz rounds, #1, 1f
+
+0: \round w9, w10, w11, w12, w5, w6, w7, w8
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+
+1: subs rounds, rounds, #4
+ \round w9, w10, w11, w12, w5, w6, w7, w8
+ csel tt, tt, lt, hi
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+ b.hi 0b
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ stp w5, w6, [out]
+ stp w7, w8, [out, #8]
+ ret
+
+ .align 4
+ .ltorg
+ .endm
+
+ENTRY(__aes_arm64_encrypt)
+ do_crypt fround, crypto_ft_tab, crypto_fl_tab
+ENDPROC(__aes_arm64_encrypt)
+
+ENTRY(__aes_arm64_decrypt)
+ do_crypt iround, crypto_it_tab, crypto_il_tab
+ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-cipher-glue.c b/arch/arm64/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..7288e7cbebff
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-glue.c
@@ -0,0 +1,69 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@xxxxxxxxxx>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_encrypt);
+
+asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_decrypt);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_encrypt(ctx->key_enc, out, in, rounds);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_decrypt(ctx->key_dec, out, in, rounds);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-arm64",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = crypto_aes_set_key,
+ .cra_cipher.cia_encrypt = aes_encrypt,
+ .cra_cipher.cia_decrypt = aes_decrypt
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Scalar AES cipher for arm64");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@xxxxxxxxxx>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("aes");
--
2.7.4
testing speed of async ecb(aes) (ecb(aes-generic)) encryption
test 0 (128 bit key, 16 byte blocks): 4594689 operations in 1 seconds (73515024 bytes)
test 1 (128 bit key, 64 byte blocks): 1585137 operations in 1 seconds (101448768 bytes)
test 2 (128 bit key, 256 byte blocks): 435173 operations in 1 seconds (111404288 bytes)
test 3 (128 bit key, 1024 byte blocks): 111505 operations in 1 seconds (114181120 bytes)
test 4 (128 bit key, 8192 byte blocks): 14093 operations in 1 seconds (115449856 bytes)
test 5 (192 bit key, 16 byte blocks): 4078345 operations in 1 seconds (65253520 bytes)
test 6 (192 bit key, 64 byte blocks): 1349425 operations in 1 seconds (86363200 bytes)
test 7 (192 bit key, 256 byte blocks): 365631 operations in 1 seconds (93601536 bytes)
test 8 (192 bit key, 1024 byte blocks): 93362 operations in 1 seconds (95602688 bytes)
test 9 (192 bit key, 8192 byte blocks): 11729 operations in 1 seconds (96083968 bytes)
test 10 (256 bit key, 16 byte blocks): 3692945 operations in 1 seconds (59087120 bytes)
test 11 (256 bit key, 64 byte blocks): 1182522 operations in 1 seconds (75681408 bytes)
test 12 (256 bit key, 256 byte blocks): 317285 operations in 1 seconds (81224960 bytes)
test 13 (256 bit key, 1024 byte blocks): 80459 operations in 1 seconds (82390016 bytes)
test 14 (256 bit key, 8192 byte blocks): 10138 operations in 1 seconds (83050496 bytes)
testing speed of async ecb(aes) (ecb(aes-arm64)) encryption
test 0 (128 bit key, 16 byte blocks): 5455304 operations in 1 seconds (87284864 bytes)
test 1 (128 bit key, 64 byte blocks): 2000321 operations in 1 seconds (128020544 bytes)
test 2 (128 bit key, 256 byte blocks): 574174 operations in 1 seconds (146988544 bytes)
test 3 (128 bit key, 1024 byte blocks): 148497 operations in 1 seconds (152060928 bytes)
test 4 (128 bit key, 8192 byte blocks): 18836 operations in 1 seconds (154304512 bytes)
test 5 (192 bit key, 16 byte blocks): 4962478 operations in 1 seconds (79399648 bytes)
test 6 (192 bit key, 64 byte blocks): 1740157 operations in 1 seconds (111370048 bytes)
test 7 (192 bit key, 256 byte blocks): 490443 operations in 1 seconds (125553408 bytes)
test 8 (192 bit key, 1024 byte blocks): 126165 operations in 1 seconds (129192960 bytes)
test 9 (192 bit key, 8192 byte blocks): 15897 operations in 1 seconds (130228224 bytes)
test 10 (256 bit key, 16 byte blocks): 4527784 operations in 1 seconds (72444544 bytes)
test 11 (256 bit key, 64 byte blocks): 1527235 operations in 1 seconds (97743040 bytes)
test 12 (256 bit key, 256 byte blocks): 425302 operations in 1 seconds (108877312 bytes)
test 13 (256 bit key, 1024 byte blocks): 109013 operations in 1 seconds (111629312 bytes)
test 14 (256 bit key, 8192 byte blocks): 13778 operations in 1 seconds (112869376 bytes)
--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
