Now that algo implementations that require it will set the new
CRYPTO_ALG_NEED_DMA_ALIGNMENT flag, we can reduce the static footprint
of the skcipher request structure, and increase the request context size
to include the headroom needed for DMA alignment, but only when it is
actually needed.
On arm64, this (and reordering the 'base' field) reduces the size of the
skcipher request structure from 128 bytes to 72 bytes. It also reduces
the minimum alignment of the struct type, which is relevant because the
SYNC_SKCIPHER_ON_STACK() macro places such a struct on the stack, and
here, the alignment requirement increases the memory footprint
substantially, given that the stack pointer can only be assumed to be
aligned to 16 bytes, as per the AArch64 ABI.
Since DMA to the stack is never allowed, we can ignore the flag here,
and simply align the whole allocation to CRYPTO_REQ_MINALIGN.
Signed-off-by: Ard Biesheuvel <ardb@xxxxxxxxxx>
---
include/crypto/internal/skcipher.h | 13 +++++++++++--
include/crypto/skcipher.h | 8 ++++----
include/linux/crypto.h | 14 ++++++++++++++
3 files changed, 29 insertions(+), 6 deletions(-)
diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h
index a2339f80a615..8c825bef2b43 100644
--- a/include/crypto/internal/skcipher.h
+++ b/include/crypto/internal/skcipher.h
@@ -119,7 +119,15 @@ static inline struct crypto_skcipher *crypto_spawn_skcipher(
static inline void crypto_skcipher_set_reqsize(
struct crypto_skcipher *skcipher, unsigned int reqsize)
{
- skcipher->reqsize = reqsize;
+ unsigned int align = crypto_tfm_alg_req_alignmask(&skcipher->base) + 1;
+
+ /*
+ * The request structure itself is only aligned to CRYPTO_REQ_MINALIGN,
+ * so we need to add some headroom, allowing us to return a suitably
+ * aligned context buffer pointer. We also need to round up the size so
+ * we don't end up sharing a cacheline at the end of the buffer.
+ */
+ skcipher->reqsize = ALIGN(reqsize, align) + align - CRYPTO_REQ_MINALIGN;
}
int crypto_register_skcipher(struct skcipher_alg *alg);
@@ -153,7 +161,8 @@ static inline void *crypto_skcipher_ctx(struct crypto_skcipher *tfm)
static inline void *skcipher_request_ctx(struct skcipher_request *req)
{
- return req->__ctx;
+ return PTR_ALIGN(&req->__ctx,
+ crypto_tfm_alg_req_alignmask(req->base.tfm) + 1);
}
static inline u32 skcipher_request_flags(struct skcipher_request *req)
diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h
index 39f5b67c3069..0b477878f100 100644
--- a/include/crypto/skcipher.h
+++ b/include/crypto/skcipher.h
@@ -26,6 +26,8 @@ struct scatterlist;
* @__ctx: Start of private context data
*/
struct skcipher_request {
+ struct crypto_async_request base;
+
unsigned int cryptlen;
u8 *iv;
@@ -33,9 +35,7 @@ struct skcipher_request {
struct scatterlist *src;
struct scatterlist *dst;
- struct crypto_async_request base;
-
- void *__ctx[] CRYPTO_MINALIGN_ATTR;
+ void *__ctx[] CRYPTO_REQ_MINALIGN_ATTR;
};
struct crypto_skcipher {
@@ -132,7 +132,7 @@ struct skcipher_alg {
MAX_SYNC_SKCIPHER_REQSIZE + \
(!(sizeof((struct crypto_sync_skcipher *)1 == \
(typeof(tfm))1))) \
- ] CRYPTO_MINALIGN_ATTR; \
+ ] CRYPTO_REQ_MINALIGN_ATTR;\
struct skcipher_request *name = (void *)__##name##_desc
/**
diff --git a/include/linux/crypto.h b/include/linux/crypto.h
index f2e95fb6cedb..d02a112914c3 100644
--- a/include/linux/crypto.h
+++ b/include/linux/crypto.h
@@ -178,6 +178,10 @@
#define CRYPTO_MINALIGN_ATTR __attribute__ ((__aligned__(CRYPTO_MINALIGN)))
+/* minimum alignment for request structures */
+#define CRYPTO_REQ_MINALIGN ARCH_SLAB_MINALIGN
+#define CRYPTO_REQ_MINALIGN_ATTR __attribute__ ((__aligned__(CRYPTO_REQ_MINALIGN)))
+
struct scatterlist;
struct crypto_async_request;
struct crypto_tfm;
@@ -706,6 +710,16 @@ static inline unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm)
return tfm->__crt_alg->cra_alignmask;
}
+static inline unsigned int crypto_tfm_alg_req_alignmask(struct crypto_tfm *tfm)
+{
+#ifdef ARCH_DMA_MINALIGN
+ if (ARCH_DMA_MINALIGN > CRYPTO_REQ_MINALIGN &&
+ (tfm->__crt_alg->cra_flags & CRYPTO_ALG_NEED_DMA_ALIGNMENT))
+ return ARCH_DMA_MINALIGN - 1;
+#endif
+ return CRYPTO_REQ_MINALIGN - 1;
+}
+
static inline u32 crypto_tfm_get_flags(struct crypto_tfm *tfm)
{
return tfm->crt_flags;
--
2.30.2
