On Wed, 23 Dec 2020 at 09:12, Eric Biggers <ebiggers@xxxxxxxxxx> wrote:
>
> From: Eric Biggers <ebiggers@xxxxxxxxxx>
>
> Add helper functions for shash implementations of BLAKE2s to
> include/crypto/internal/blake2s.h, taking advantage of
> __blake2s_update() and __blake2s_final() that were added by the previous
> patch to share more code between the library and shash implementations.
>
> crypto_blake2s_setkey() and crypto_blake2s_init() are usable as
> shash_alg::setkey and shash_alg::init directly, while
> crypto_blake2s_update() and crypto_blake2s_final() take an extra
> 'blake2s_compress_t' function pointer parameter. This allows the
> implementation of the compression function to be overridden, which is
> the only part that optimized implementations really care about.
>
> The new functions are inline functions (similar to those in sha1_base.h,
> sha256_base.h, and sm3_base.h) because this avoids needing to add a new
> module blake2s_helpers.ko, they aren't *too* long, and this avoids
> indirect calls which are expensive these days. Note that they can't go
> in blake2s_generic.ko, as that would require selecting CRYPTO_BLAKE2S
> from CRYPTO_BLAKE2S_X86, which would cause a recursive dependency.
>
> Finally, use these new helper functions in the x86 implementation of
> BLAKE2s. (This part should be a separate patch, but unfortunately the
> x86 implementation used the exact same function names like
> "crypto_blake2s_update()", so it had to be updated at the same time.)
>
> Signed-off-by: Eric Biggers <ebiggers@xxxxxxxxxx>
Acked-by: Ard Biesheuvel <ardb@xxxxxxxxxx>
> ---
> arch/x86/crypto/blake2s-glue.c | 74 +++---------------------------
> crypto/blake2s_generic.c | 76 ++++---------------------------
> include/crypto/internal/blake2s.h | 65 ++++++++++++++++++++++++--
> 3 files changed, 76 insertions(+), 139 deletions(-)
>
> diff --git a/arch/x86/crypto/blake2s-glue.c b/arch/x86/crypto/blake2s-glue.c
> index 4dcb2ee89efc9..a40365ab301ee 100644
> --- a/arch/x86/crypto/blake2s-glue.c
> +++ b/arch/x86/crypto/blake2s-glue.c
> @@ -58,75 +58,15 @@ void blake2s_compress_arch(struct blake2s_state *state,
> }
> EXPORT_SYMBOL(blake2s_compress_arch);
>
> -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key,
> - unsigned int keylen)
> +static int crypto_blake2s_update_x86(struct shash_desc *desc,
> + const u8 *in, unsigned int inlen)
> {
> - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm);
> -
> - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE)
> - return -EINVAL;
> -
> - memcpy(tctx->key, key, keylen);
> - tctx->keylen = keylen;
> -
> - return 0;
> -}
> -
> -static int crypto_blake2s_init(struct shash_desc *desc)
> -{
> - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
> - struct blake2s_state *state = shash_desc_ctx(desc);
> - const int outlen = crypto_shash_digestsize(desc->tfm);
> -
> - if (tctx->keylen)
> - blake2s_init_key(state, outlen, tctx->key, tctx->keylen);
> - else
> - blake2s_init(state, outlen);
> -
> - return 0;
> -}
> -
> -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in,
> - unsigned int inlen)
> -{
> - struct blake2s_state *state = shash_desc_ctx(desc);
> - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
> -
> - if (unlikely(!inlen))
> - return 0;
> - if (inlen > fill) {
> - memcpy(state->buf + state->buflen, in, fill);
> - blake2s_compress_arch(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
> - state->buflen = 0;
> - in += fill;
> - inlen -= fill;
> - }
> - if (inlen > BLAKE2S_BLOCK_SIZE) {
> - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
> - /* Hash one less (full) block than strictly possible */
> - blake2s_compress_arch(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
> - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
> - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
> - }
> - memcpy(state->buf + state->buflen, in, inlen);
> - state->buflen += inlen;
> -
> - return 0;
> + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_arch);
> }
>
> -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out)
> +static int crypto_blake2s_final_x86(struct shash_desc *desc, u8 *out)
> {
> - struct blake2s_state *state = shash_desc_ctx(desc);
> -
> - blake2s_set_lastblock(state);
> - memset(state->buf + state->buflen, 0,
> - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
> - blake2s_compress_arch(state, state->buf, 1, state->buflen);
> - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
> - memcpy(out, state->h, state->outlen);
> - memzero_explicit(state, sizeof(*state));
> -
> - return 0;
> + return crypto_blake2s_final(desc, out, blake2s_compress_arch);
> }
>
> #define BLAKE2S_ALG(name, driver_name, digest_size) \
> @@ -141,8 +81,8 @@ static int crypto_blake2s_final(struct shash_desc *desc, u8 *out)
> .digestsize = digest_size, \
> .setkey = crypto_blake2s_setkey, \
> .init = crypto_blake2s_init, \
> - .update = crypto_blake2s_update, \
> - .final = crypto_blake2s_final, \
> + .update = crypto_blake2s_update_x86, \
> + .final = crypto_blake2s_final_x86, \
> .descsize = sizeof(struct blake2s_state), \
> }
>
> diff --git a/crypto/blake2s_generic.c b/crypto/blake2s_generic.c
> index b89536c3671cf..72fe480f9bd67 100644
> --- a/crypto/blake2s_generic.c
> +++ b/crypto/blake2s_generic.c
> @@ -1,5 +1,7 @@
> // SPDX-License-Identifier: GPL-2.0 OR MIT
> /*
> + * shash interface to the generic implementation of BLAKE2s
> + *
> * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@xxxxxxxxx>. All Rights Reserved.
> */
>
> @@ -10,75 +12,15 @@
> #include <linux/kernel.h>
> #include <linux/module.h>
>
> -static int crypto_blake2s_setkey(struct crypto_shash *tfm, const u8 *key,
> - unsigned int keylen)
> +static int crypto_blake2s_update_generic(struct shash_desc *desc,
> + const u8 *in, unsigned int inlen)
> {
> - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm);
> -
> - if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE)
> - return -EINVAL;
> -
> - memcpy(tctx->key, key, keylen);
> - tctx->keylen = keylen;
> -
> - return 0;
> + return crypto_blake2s_update(desc, in, inlen, blake2s_compress_generic);
> }
>
> -static int crypto_blake2s_init(struct shash_desc *desc)
> +static int crypto_blake2s_final_generic(struct shash_desc *desc, u8 *out)
> {
> - struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
> - struct blake2s_state *state = shash_desc_ctx(desc);
> - const int outlen = crypto_shash_digestsize(desc->tfm);
> -
> - if (tctx->keylen)
> - blake2s_init_key(state, outlen, tctx->key, tctx->keylen);
> - else
> - blake2s_init(state, outlen);
> -
> - return 0;
> -}
> -
> -static int crypto_blake2s_update(struct shash_desc *desc, const u8 *in,
> - unsigned int inlen)
> -{
> - struct blake2s_state *state = shash_desc_ctx(desc);
> - const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
> -
> - if (unlikely(!inlen))
> - return 0;
> - if (inlen > fill) {
> - memcpy(state->buf + state->buflen, in, fill);
> - blake2s_compress_generic(state, state->buf, 1, BLAKE2S_BLOCK_SIZE);
> - state->buflen = 0;
> - in += fill;
> - inlen -= fill;
> - }
> - if (inlen > BLAKE2S_BLOCK_SIZE) {
> - const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
> - /* Hash one less (full) block than strictly possible */
> - blake2s_compress_generic(state, in, nblocks - 1, BLAKE2S_BLOCK_SIZE);
> - in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
> - inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
> - }
> - memcpy(state->buf + state->buflen, in, inlen);
> - state->buflen += inlen;
> -
> - return 0;
> -}
> -
> -static int crypto_blake2s_final(struct shash_desc *desc, u8 *out)
> -{
> - struct blake2s_state *state = shash_desc_ctx(desc);
> -
> - blake2s_set_lastblock(state);
> - memset(state->buf + state->buflen, 0,
> - BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
> - blake2s_compress_generic(state, state->buf, 1, state->buflen);
> - cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
> - memcpy(out, state->h, state->outlen);
> - memzero_explicit(state, sizeof(*state));
> -
> - return 0;
> + return crypto_blake2s_final(desc, out, blake2s_compress_generic);
> }
>
> #define BLAKE2S_ALG(name, driver_name, digest_size) \
> @@ -93,8 +35,8 @@ static int crypto_blake2s_final(struct shash_desc *desc, u8 *out)
> .digestsize = digest_size, \
> .setkey = crypto_blake2s_setkey, \
> .init = crypto_blake2s_init, \
> - .update = crypto_blake2s_update, \
> - .final = crypto_blake2s_final, \
> + .update = crypto_blake2s_update_generic, \
> + .final = crypto_blake2s_final_generic, \
> .descsize = sizeof(struct blake2s_state), \
> }
>
> diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h
> index 42deba4b8ceef..2ea0a8f5e7f41 100644
> --- a/include/crypto/internal/blake2s.h
> +++ b/include/crypto/internal/blake2s.h
> @@ -1,16 +1,16 @@
> /* SPDX-License-Identifier: GPL-2.0 OR MIT */
> +/*
> + * Helper functions for BLAKE2s implementations.
> + * Keep this in sync with the corresponding BLAKE2b header.
> + */
>
> #ifndef BLAKE2S_INTERNAL_H
> #define BLAKE2S_INTERNAL_H
>
> #include <crypto/blake2s.h>
> +#include <crypto/internal/hash.h>
> #include <linux/string.h>
>
> -struct blake2s_tfm_ctx {
> - u8 key[BLAKE2S_KEY_SIZE];
> - unsigned int keylen;
> -};
> -
> void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
> size_t nblocks, const u32 inc);
>
> @@ -27,6 +27,8 @@ static inline void blake2s_set_lastblock(struct blake2s_state *state)
> typedef void (*blake2s_compress_t)(struct blake2s_state *state,
> const u8 *block, size_t nblocks, u32 inc);
>
> +/* Helper functions for BLAKE2s shared by the library and shash APIs */
> +
> static inline void __blake2s_update(struct blake2s_state *state,
> const u8 *in, size_t inlen,
> blake2s_compress_t compress)
> @@ -64,4 +66,57 @@ static inline void __blake2s_final(struct blake2s_state *state, u8 *out,
> memcpy(out, state->h, state->outlen);
> }
>
> +/* Helper functions for shash implementations of BLAKE2s */
> +
> +struct blake2s_tfm_ctx {
> + u8 key[BLAKE2S_KEY_SIZE];
> + unsigned int keylen;
> +};
> +
> +static inline int crypto_blake2s_setkey(struct crypto_shash *tfm,
> + const u8 *key, unsigned int keylen)
> +{
> + struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(tfm);
> +
> + if (keylen == 0 || keylen > BLAKE2S_KEY_SIZE)
> + return -EINVAL;
> +
> + memcpy(tctx->key, key, keylen);
> + tctx->keylen = keylen;
> +
> + return 0;
> +}
> +
> +static inline int crypto_blake2s_init(struct shash_desc *desc)
> +{
> + const struct blake2s_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
> + struct blake2s_state *state = shash_desc_ctx(desc);
> + unsigned int outlen = crypto_shash_digestsize(desc->tfm);
> +
> + if (tctx->keylen)
> + blake2s_init_key(state, outlen, tctx->key, tctx->keylen);
> + else
> + blake2s_init(state, outlen);
> + return 0;
> +}
> +
> +static inline int crypto_blake2s_update(struct shash_desc *desc,
> + const u8 *in, unsigned int inlen,
> + blake2s_compress_t compress)
> +{
> + struct blake2s_state *state = shash_desc_ctx(desc);
> +
> + __blake2s_update(state, in, inlen, compress);
> + return 0;
> +}
> +
> +static inline int crypto_blake2s_final(struct shash_desc *desc, u8 *out,
> + blake2s_compress_t compress)
> +{
> + struct blake2s_state *state = shash_desc_ctx(desc);
> +
> + __blake2s_final(state, out, compress);
> + return 0;
> +}
> +
> #endif /* BLAKE2S_INTERNAL_H */
> --
> 2.29.2
>
