Added rfc4106(gcm(aes)) cipher. Signed-off-by: Nagadheeraj Rottela <rnagadheeraj@xxxxxxxxxxx> Reviewed-by: Srikanth Jampala <jsrikanth@xxxxxxxxxxx> --- drivers/crypto/cavium/nitrox/nitrox_aead.c | 337 ++++++++++++++++++++++------- drivers/crypto/cavium/nitrox/nitrox_req.h | 46 +++- 2 files changed, 300 insertions(+), 83 deletions(-) diff --git a/drivers/crypto/cavium/nitrox/nitrox_aead.c b/drivers/crypto/cavium/nitrox/nitrox_aead.c index 4f43eacd2557..e4841eb2a09f 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_aead.c +++ b/drivers/crypto/cavium/nitrox/nitrox_aead.c @@ -18,26 +18,6 @@ #define GCM_AES_SALT_SIZE 4 -/** - * struct nitrox_crypt_params - Params to set nitrox crypto request. - * @cryptlen: Encryption/Decryption data length - * @authlen: Assoc data length + Cryptlen - * @srclen: Input buffer length - * @dstlen: Output buffer length - * @iv: IV data - * @ivsize: IV data length - * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) - */ -struct nitrox_crypt_params { - unsigned int cryptlen; - unsigned int authlen; - unsigned int srclen; - unsigned int dstlen; - u8 *iv; - int ivsize; - u8 ctrl_arg; -}; - union gph_p3 { struct { #ifdef __BIG_ENDIAN_BITFIELD @@ -94,36 +74,40 @@ static int nitrox_aead_setauthsize(struct crypto_aead *aead, return 0; } -static int alloc_src_sglist(struct aead_request *areq, char *iv, int ivsize, +static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *src, char *iv, int ivsize, int buflen) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - int nents = sg_nents_for_len(areq->src, buflen) + 1; + int nents = sg_nents_for_len(src, buflen); int ret; if (nents < 0) return nents; + /* IV entry */ + nents += 1; /* Allocate buffer to hold IV and input scatterlist array */ ret = alloc_src_req_buf(nkreq, nents, ivsize); if (ret) return ret; nitrox_creq_copy_iv(nkreq->src, iv, ivsize); - nitrox_creq_set_src_sg(nkreq, nents, ivsize, areq->src, buflen); + nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen); return 0; } -static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen) +static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq, + struct scatterlist *dst, int ivsize, int buflen) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - int nents = sg_nents_for_len(areq->dst, buflen) + 3; + int nents = sg_nents_for_len(dst, buflen); int ret; if (nents < 0) return nents; + /* IV, ORH, COMPLETION entries */ + nents += 3; /* Allocate buffer to hold ORH, COMPLETION and output scatterlist * array */ @@ -133,61 +117,54 @@ static int alloc_dst_sglist(struct aead_request *areq, int ivsize, int buflen) nitrox_creq_set_orh(nkreq); nitrox_creq_set_comp(nkreq); - nitrox_creq_set_dst_sg(nkreq, nents, ivsize, areq->dst, buflen); + nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen); return 0; } -static void free_src_sglist(struct aead_request *areq) +static void free_src_sglist(struct nitrox_kcrypt_request *nkreq) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - kfree(nkreq->src); } -static void free_dst_sglist(struct aead_request *areq) +static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - kfree(nkreq->dst); } -static int nitrox_set_creq(struct aead_request *areq, - struct nitrox_crypt_params *params) +static int nitrox_set_creq(struct nitrox_aead_rctx *rctx) { - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; - struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; union gph_p3 param3; - struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); int ret; - creq->flags = areq->base.flags; - creq->gfp = (areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; + creq->flags = rctx->flags; + creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : + GFP_ATOMIC; creq->ctrl.value = 0; creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; - creq->ctrl.s.arg = params->ctrl_arg; + creq->ctrl.s.arg = rctx->ctrl_arg; - creq->gph.param0 = cpu_to_be16(params->cryptlen); - creq->gph.param1 = cpu_to_be16(params->authlen); - creq->gph.param2 = cpu_to_be16(params->ivsize + areq->assoclen); + creq->gph.param0 = cpu_to_be16(rctx->cryptlen); + creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen); + creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen); param3.iv_offset = 0; - param3.auth_offset = params->ivsize; + param3.auth_offset = rctx->ivsize; creq->gph.param3 = cpu_to_be16(param3.param); - creq->ctx_handle = nctx->u.ctx_handle; + creq->ctx_handle = rctx->ctx_handle; creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); - ret = alloc_src_sglist(areq, params->iv, params->ivsize, - params->srclen); + ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize, + rctx->srclen); if (ret) return ret; - ret = alloc_dst_sglist(areq, params->ivsize, params->dstlen); + ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize, + rctx->dstlen); if (ret) { - free_src_sglist(areq); + free_src_sglist(&rctx->nkreq); return ret; } @@ -197,9 +174,10 @@ static int nitrox_set_creq(struct aead_request *areq, static void nitrox_aead_callback(void *arg, int err) { struct aead_request *areq = arg; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); - free_src_sglist(areq); - free_dst_sglist(areq); + free_src_sglist(&rctx->nkreq); + free_dst_sglist(&rctx->nkreq); if (err) { pr_err_ratelimited("request failed status 0x%0x\n", err); err = -EINVAL; @@ -212,23 +190,25 @@ static int nitrox_aes_gcm_enc(struct aead_request *areq) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; struct flexi_crypto_context *fctx = nctx->u.fctx; - struct nitrox_crypt_params params; int ret; memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); - memset(¶ms, 0, sizeof(params)); - params.cryptlen = areq->cryptlen; - params.authlen = areq->assoclen + params.cryptlen; - params.srclen = params.authlen; - params.dstlen = params.srclen + aead->authsize; - params.iv = &areq->iv[GCM_AES_SALT_SIZE]; - params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; - params.ctrl_arg = ENCRYPT; - ret = nitrox_set_creq(areq, ¶ms); + rctx->cryptlen = areq->cryptlen; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->assoclen + areq->cryptlen; + rctx->dstlen = rctx->srclen + aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = ENCRYPT; + ret = nitrox_set_creq(rctx); if (ret) return ret; @@ -241,23 +221,25 @@ static int nitrox_aes_gcm_dec(struct aead_request *areq) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); - struct nitrox_kcrypt_request *nkreq = aead_request_ctx(areq); - struct se_crypto_request *creq = &nkreq->creq; + struct nitrox_aead_rctx *rctx = aead_request_ctx(areq); + struct se_crypto_request *creq = &rctx->nkreq.creq; struct flexi_crypto_context *fctx = nctx->u.fctx; - struct nitrox_crypt_params params; int ret; memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE); - memset(¶ms, 0, sizeof(params)); - params.cryptlen = areq->cryptlen - aead->authsize; - params.authlen = areq->assoclen + params.cryptlen; - params.srclen = areq->cryptlen + areq->assoclen; - params.dstlen = params.srclen - aead->authsize; - params.iv = &areq->iv[GCM_AES_SALT_SIZE]; - params.ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; - params.ctrl_arg = DECRYPT; - ret = nitrox_set_creq(areq, ¶ms); + rctx->cryptlen = areq->cryptlen - aead->authsize; + rctx->assoclen = areq->assoclen; + rctx->srclen = areq->cryptlen + areq->assoclen; + rctx->dstlen = rctx->srclen - aead->authsize; + rctx->iv = &areq->iv[GCM_AES_SALT_SIZE]; + rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE; + rctx->flags = areq->base.flags; + rctx->ctx_handle = nctx->u.ctx_handle; + rctx->src = areq->src; + rctx->dst = areq->dst; + rctx->ctrl_arg = DECRYPT; + ret = nitrox_set_creq(rctx); if (ret) return ret; @@ -290,7 +272,7 @@ static int nitrox_aead_init(struct crypto_aead *aead) return 0; } -static int nitrox_aes_gcm_init(struct crypto_aead *aead) +static int nitrox_gcm_common_init(struct crypto_aead *aead) { int ret; struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); @@ -308,8 +290,20 @@ static int nitrox_aes_gcm_init(struct crypto_aead *aead) flags->w0.auth_input_type = 1; flags->f = be64_to_cpu(flags->f); - crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + - sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static int nitrox_aes_gcm_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, + sizeof(struct aead_request) + + sizeof(struct nitrox_aead_rctx)); return 0; } @@ -332,6 +326,166 @@ static void nitrox_aead_exit(struct crypto_aead *aead) nctx->ndev = NULL; } +static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key, + unsigned int keylen) +{ + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct flexi_crypto_context *fctx = nctx->u.fctx; + int ret; + + if (keylen < GCM_AES_SALT_SIZE) + return -EINVAL; + + keylen -= GCM_AES_SALT_SIZE; + ret = nitrox_aes_gcm_setkey(aead, key, keylen); + if (ret) + return ret; + + memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE); + return 0; +} + +static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead, + unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return nitrox_aead_setauthsize(aead, authsize); +} + +static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq) +{ + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + struct scatterlist *sg; + + if (areq->assoclen != 16 && areq->assoclen != 20) + return -EINVAL; + + scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0); + sg_init_table(rctx->src, 3); + sg_set_buf(rctx->src, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen); + if (sg != rctx->src + 1) + sg_chain(rctx->src, 2, sg); + + if (areq->src != areq->dst) { + sg_init_table(rctx->dst, 3); + sg_set_buf(rctx->dst, rctx->assoc, assoclen); + sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen); + if (sg != rctx->dst + 1) + sg_chain(rctx->dst, 2, sg); + } + + aead_rctx->src = rctx->src; + aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst; + + return 0; +} + +static void nitrox_rfc4106_callback(void *arg, int err) +{ + struct aead_request *areq = arg; + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq; + + free_src_sglist(nkreq); + free_dst_sglist(nkreq); + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + + areq->base.complete(&areq->base, err); +} + +static int nitrox_rfc4106_enc(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen; + aead_rctx->dstlen = aead_rctx->srclen + aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = ENCRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_dec(struct aead_request *areq) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(areq); + struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead); + struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq); + struct nitrox_aead_rctx *aead_rctx = &rctx->base; + struct se_crypto_request *creq = &aead_rctx->nkreq.creq; + int ret; + + aead_rctx->cryptlen = areq->cryptlen - aead->authsize; + aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE; + aead_rctx->srclen = + areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen; + aead_rctx->dstlen = aead_rctx->srclen - aead->authsize; + aead_rctx->iv = areq->iv; + aead_rctx->ivsize = GCM_RFC4106_IV_SIZE; + aead_rctx->flags = areq->base.flags; + aead_rctx->ctx_handle = nctx->u.ctx_handle; + aead_rctx->ctrl_arg = DECRYPT; + + ret = nitrox_rfc4106_set_aead_rctx_sglist(areq); + if (ret) + return ret; + + ret = nitrox_set_creq(aead_rctx); + if (ret) + return ret; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_rfc4106_callback, areq); +} + +static int nitrox_rfc4106_init(struct crypto_aead *aead) +{ + int ret; + + ret = nitrox_gcm_common_init(aead); + if (ret) + return ret; + + crypto_aead_set_reqsize(aead, sizeof(struct aead_request) + + sizeof(struct nitrox_rfc4106_rctx)); + + return 0; +} + static struct aead_alg nitrox_aeads[] = { { .base = { .cra_name = "gcm(aes)", @@ -351,6 +505,25 @@ static struct aead_alg nitrox_aeads[] = { { .exit = nitrox_aead_exit, .ivsize = GCM_AES_IV_SIZE, .maxauthsize = AES_BLOCK_SIZE, +}, { + .base = { + .cra_name = "rfc4106(gcm(aes))", + .cra_driver_name = "n5_rfc4106", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .setkey = nitrox_rfc4106_setkey, + .setauthsize = nitrox_rfc4106_setauthsize, + .encrypt = nitrox_rfc4106_enc, + .decrypt = nitrox_rfc4106_dec, + .init = nitrox_rfc4106_init, + .exit = nitrox_aead_exit, + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = AES_BLOCK_SIZE, } }; int nitrox_register_aeads(void) diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h index 76c0f0be7233..efdbd0fc3e3b 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_req.h +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -212,6 +212,50 @@ struct nitrox_kcrypt_request { }; /** + * struct nitrox_aead_rctx - AEAD request context + * @nkreq: Base request context + * @cryptlen: Encryption/Decryption data length + * @assoclen: AAD length + * @srclen: Input buffer length + * @dstlen: Output buffer length + * @iv: IV data + * @ivsize: IV data length + * @flags: AEAD req flags + * @ctx_handle: Device context handle + * @src: Source sglist + * @dst: Destination sglist + * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT) + */ +struct nitrox_aead_rctx { + struct nitrox_kcrypt_request nkreq; + unsigned int cryptlen; + unsigned int assoclen; + unsigned int srclen; + unsigned int dstlen; + u8 *iv; + int ivsize; + u32 flags; + u64 ctx_handle; + struct scatterlist *src; + struct scatterlist *dst; + u8 ctrl_arg; +}; + +/** + * struct nitrox_rfc4106_rctx - rfc4106 cipher request context + * @base: AEAD request context + * @src: Source sglist + * @dst: Destination sglist + * @assoc: AAD + */ +struct nitrox_rfc4106_rctx { + struct nitrox_aead_rctx base; + struct scatterlist src[3]; + struct scatterlist dst[3]; + u8 assoc[20]; +}; + +/** * struct pkt_instr_hdr - Packet Instruction Header * @g: Gather used * When [G] is set and [GSZ] != 0, the instruction is @@ -512,7 +556,7 @@ static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg, struct scatterlist *sg = to_sg; unsigned int sglen; - for (; buflen; buflen -= sglen) { + for (; buflen && from_sg; buflen -= sglen) { sglen = from_sg->length; if (sglen > buflen) sglen = buflen; -- 2.13.6