Now that we have moved all the scratch buffers that must be allocated
on the heap out of the request context, we can move the request context
itself to the stack if we are instantiating a synchronous version of
the chacha20poly1305 transformation. This allows users of the AEAD to
allocate the request structure on the stack, removing the need for
per-packet heap allocations on the en/decryption hot path.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@xxxxxxxxxx>
---
crypto/chacha20poly1305.c | 51 ++++++++++++--------
1 file changed, 32 insertions(+), 19 deletions(-)
diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c
index 71496a8107f5..d171a0c9e837 100644
--- a/crypto/chacha20poly1305.c
+++ b/crypto/chacha20poly1305.c
@@ -49,13 +49,14 @@ struct chachapoly_req_ctx {
};
static inline void async_done_continue(struct aead_request *req, int err,
- int (*cont)(struct aead_request *))
+ int (*cont)(struct aead_request *,
+ struct chachapoly_req_ctx *))
{
if (!err) {
struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
- err = cont(req);
+ err = cont(req, rctx);
}
if (err != -EINPROGRESS && err != -EBUSY)
@@ -74,11 +75,11 @@ static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)
}
static int poly_generate_tag(struct aead_request *req, u8 *poly_tag,
- struct scatterlist *crypt)
+ struct scatterlist *crypt,
+ struct chachapoly_req_ctx *rctx)
{
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
- struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
u32 chacha_state[CHACHA_BLOCK_SIZE / sizeof(u32)];
SHASH_DESC_ON_STACK(desc, ctx->poly);
u8 poly_key[POLY1305_KEY_SIZE];
@@ -148,13 +149,13 @@ static int poly_generate_tag(struct aead_request *req, u8 *poly_tag,
return 0;
}
-static int poly_append_tag(struct aead_request *req)
+static int poly_append_tag(struct aead_request *req,
+ struct chachapoly_req_ctx *rctx)
{
- struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
u8 poly_tag[POLY1305_DIGEST_SIZE];
int err;
- err = poly_generate_tag(req, poly_tag, req->dst);
+ err = poly_generate_tag(req, poly_tag, req->dst, rctx);
if (err)
return err;
@@ -171,12 +172,17 @@ static void chacha_encrypt_done(struct crypto_async_request *areq, int err)
static int chachapoly_encrypt(struct aead_request *req)
{
- struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+ struct chachapoly_req_ctx stack_rctx CRYPTO_MINALIGN_ATTR;
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
+ struct chachapoly_req_ctx *rctx = &stack_rctx;
struct chacha_req *creq = &rctx->chacha;
struct scatterlist *src, *dst;
int err;
+ if (unlikely(crypto_aead_reqsize(tfm) > 0))
+ rctx = aead_request_ctx(req);
+
rctx->cryptlen = req->cryptlen;
rctx->flags = aead_request_flags(req);
@@ -200,7 +206,7 @@ static int chachapoly_encrypt(struct aead_request *req)
return err;
skip:
- return poly_append_tag(req);
+ return poly_append_tag(req, rctx);
}
static void chacha_decrypt_done(struct crypto_async_request *areq, int err)
@@ -210,18 +216,23 @@ static void chacha_decrypt_done(struct crypto_async_request *areq, int err)
static int chachapoly_decrypt(struct aead_request *req)
{
- struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
+ struct chachapoly_req_ctx stack_rctx CRYPTO_MINALIGN_ATTR;
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
+ struct chachapoly_req_ctx *rctx = &stack_rctx;
struct chacha_req *creq = &rctx->chacha;
u8 calculated_tag[POLY1305_DIGEST_SIZE];
u8 provided_tag[POLY1305_DIGEST_SIZE];
struct scatterlist *src, *dst;
int err;
+ if (unlikely(crypto_aead_reqsize(tfm) > 0))
+ rctx = aead_request_ctx(req);
+
rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
rctx->flags = aead_request_flags(req);
- err = poly_generate_tag(req, calculated_tag, req->src);
+ err = poly_generate_tag(req, calculated_tag, req->src, rctx);
if (err)
return err;
scatterwalk_map_and_copy(provided_tag, req->src,
@@ -314,12 +325,14 @@ static int chachapoly_init(struct crypto_aead *tfm)
align = crypto_aead_alignmask(tfm);
align &= ~(crypto_tfm_ctx_alignment() - 1);
- crypto_aead_set_reqsize(
- tfm,
- align +
- offsetof(struct chachapoly_req_ctx, chacha.req) +
- sizeof(struct skcipher_request) +
- crypto_skcipher_reqsize(chacha));
+ if (crypto_aead_alignmask(tfm) > 0 ||
+ (crypto_aead_get_flags(tfm) & CRYPTO_ALG_ASYNC))
+ crypto_aead_set_reqsize(
+ tfm,
+ align +
+ offsetof(struct chachapoly_req_ctx, chacha.req) +
+ sizeof(struct skcipher_request) +
+ crypto_skcipher_reqsize(chacha));
return 0;
}
--
2.20.1
