Re: [PATCH v3 08/10] crypto: marvell: Add load balancing between engines

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On Tue, 21 Jun 2016 10:08:38 +0200
Romain Perier <romain.perier@xxxxxxxxxxxxxxxxxx> wrote:

> This commits adds support for fine grained load balancing on
> multi-engine IPs. The engine is pre-selected based on its current load
> and on the weight of the crypto request that is about to be processed.
> The global crypto queue is also moved to each engine. These changes are
> required to allow chaining crypto requests at the DMA level. By using
> a crypto queue per engine, we make sure that we keep the state of the
> tdma chain synchronized with the crypto queue. We also reduce contention
> on 'cesa_dev->lock' and improve parallelism.
> 
> Signed-off-by: Romain Perier <romain.perier@xxxxxxxxxxxxxxxxxx>

Acked-by: Boris Brezillon <boris.brezillon@xxxxxxxxxxxxxxxxxx>

> ---
> 
> Changes in v3:
> 
>   - Renamed mv_cesa_dequeue_req_unlocked => mv_cesa_dequeue_req_locked
> 
> Changes in v2:
> 
>   - Reworded the commit message
>   - Moved the code about SRAM I/O operations from this commit to
>     a separated commit (see PATCH 07/10).
> 
>  drivers/crypto/marvell/cesa.c   | 34 ++++++++++++------------
>  drivers/crypto/marvell/cesa.h   | 29 +++++++++++++++++----
>  drivers/crypto/marvell/cipher.c | 57 ++++++++++++++++++-----------------------
>  drivers/crypto/marvell/hash.c   | 50 ++++++++++++++----------------------
>  4 files changed, 84 insertions(+), 86 deletions(-)
> 
> diff --git a/drivers/crypto/marvell/cesa.c b/drivers/crypto/marvell/cesa.c
> index af96426..c0497ac 100644
> --- a/drivers/crypto/marvell/cesa.c
> +++ b/drivers/crypto/marvell/cesa.c
> @@ -40,16 +40,14 @@ MODULE_PARM_DESC(allhwsupport, "Enable support for all hardware (even it if over
>  
>  struct mv_cesa_dev *cesa_dev;
>  
> -static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine)
> +static void mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine)
>  {
>  	struct crypto_async_request *req, *backlog;
>  	struct mv_cesa_ctx *ctx;
>  
> -	spin_lock_bh(&cesa_dev->lock);
> -	backlog = crypto_get_backlog(&cesa_dev->queue);
> -	req = crypto_dequeue_request(&cesa_dev->queue);
> +	backlog = crypto_get_backlog(&engine->queue);
> +	req = crypto_dequeue_request(&engine->queue);
>  	engine->req = req;
> -	spin_unlock_bh(&cesa_dev->lock);
>  
>  	if (!req)
>  		return;
> @@ -58,7 +56,6 @@ static void mv_cesa_dequeue_req_unlocked(struct mv_cesa_engine *engine)
>  		backlog->complete(backlog, -EINPROGRESS);
>  
>  	ctx = crypto_tfm_ctx(req->tfm);
> -	ctx->ops->prepare(req, engine);
>  	ctx->ops->step(req);
>  }
>  
> @@ -96,7 +93,7 @@ static irqreturn_t mv_cesa_int(int irq, void *priv)
>  			if (res != -EINPROGRESS) {
>  				spin_lock_bh(&engine->lock);
>  				engine->req = NULL;
> -				mv_cesa_dequeue_req_unlocked(engine);
> +				mv_cesa_dequeue_req_locked(engine);
>  				spin_unlock_bh(&engine->lock);
>  				ctx->ops->complete(req);
>  				ctx->ops->cleanup(req);
> @@ -116,21 +113,19 @@ int mv_cesa_queue_req(struct crypto_async_request *req,
>  		      struct mv_cesa_req *creq)
>  {
>  	int ret;
> -	int i;
> +	struct mv_cesa_engine *engine = creq->engine;
>  
> -	spin_lock_bh(&cesa_dev->lock);
> -	ret = crypto_enqueue_request(&cesa_dev->queue, req);
> -	spin_unlock_bh(&cesa_dev->lock);
> +	spin_lock_bh(&engine->lock);
> +	ret = crypto_enqueue_request(&engine->queue, req);
> +	spin_unlock_bh(&engine->lock);
>  
>  	if (ret != -EINPROGRESS)
>  		return ret;
>  
> -	for (i = 0; i < cesa_dev->caps->nengines; i++) {
> -		spin_lock_bh(&cesa_dev->engines[i].lock);
> -		if (!cesa_dev->engines[i].req)
> -			mv_cesa_dequeue_req_unlocked(&cesa_dev->engines[i]);
> -		spin_unlock_bh(&cesa_dev->engines[i].lock);
> -	}
> +	spin_lock_bh(&engine->lock);
> +	if (!engine->req)
> +		mv_cesa_dequeue_req_locked(engine);
> +	spin_unlock_bh(&engine->lock);
>  
>  	return -EINPROGRESS;
>  }
> @@ -425,7 +420,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
>  		return -ENOMEM;
>  
>  	spin_lock_init(&cesa->lock);
> -	crypto_init_queue(&cesa->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN);
> +
>  	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
>  	cesa->regs = devm_ioremap_resource(dev, res);
>  	if (IS_ERR(cesa->regs))
> @@ -498,6 +493,9 @@ static int mv_cesa_probe(struct platform_device *pdev)
>  						engine);
>  		if (ret)
>  			goto err_cleanup;
> +
> +		crypto_init_queue(&engine->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN);
> +		atomic_set(&engine->load, 0);
>  	}
>  
>  	cesa_dev = cesa;
> diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h
> index c463528..644be35 100644
> --- a/drivers/crypto/marvell/cesa.h
> +++ b/drivers/crypto/marvell/cesa.h
> @@ -400,7 +400,6 @@ struct mv_cesa_dev_dma {
>   * @regs:	device registers
>   * @sram_size:	usable SRAM size
>   * @lock:	device lock
> - * @queue:	crypto request queue
>   * @engines:	array of engines
>   * @dma:	dma pools
>   *
> @@ -412,7 +411,6 @@ struct mv_cesa_dev {
>  	struct device *dev;
>  	unsigned int sram_size;
>  	spinlock_t lock;
> -	struct crypto_queue queue;
>  	struct mv_cesa_engine *engines;
>  	struct mv_cesa_dev_dma *dma;
>  };
> @@ -431,6 +429,8 @@ struct mv_cesa_dev {
>   * @int_mask:		interrupt mask cache
>   * @pool:		memory pool pointing to the memory region reserved in
>   *			SRAM
> + * @queue:		fifo of the pending crypto requests
> + * @load:		engine load counter, useful for load balancing
>   *
>   * Structure storing CESA engine information.
>   */
> @@ -446,11 +446,12 @@ struct mv_cesa_engine {
>  	size_t max_req_len;
>  	u32 int_mask;
>  	struct gen_pool *pool;
> +	struct crypto_queue queue;
> +	atomic_t load;
>  };
>  
>  /**
>   * struct mv_cesa_req_ops - CESA request operations
> - * @prepare:	prepare a request to be executed on the specified engine
>   * @process:	process a request chunk result (should return 0 if the
>   *		operation, -EINPROGRESS if it needs more steps or an error
>   *		code)
> @@ -460,8 +461,6 @@ struct mv_cesa_engine {
>   * 		when it is needed.
>   */
>  struct mv_cesa_req_ops {
> -	void (*prepare)(struct crypto_async_request *req,
> -			struct mv_cesa_engine *engine);
>  	int (*process)(struct crypto_async_request *req, u32 status);
>  	void (*step)(struct crypto_async_request *req);
>  	void (*cleanup)(struct crypto_async_request *req);
> @@ -690,6 +689,26 @@ static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op)
>  int mv_cesa_queue_req(struct crypto_async_request *req,
>  		      struct mv_cesa_req *creq);
>  
> +static inline struct mv_cesa_engine *mv_cesa_select_engine(int weight)
> +{
> +	int i;
> +	u32 min_load = U32_MAX;
> +	struct mv_cesa_engine *selected = NULL;
> +
> +	for (i = 0; i < cesa_dev->caps->nengines; i++) {
> +		struct mv_cesa_engine *engine = cesa_dev->engines + i;
> +		u32 load = atomic_read(&engine->load);
> +		if (load < min_load) {
> +			min_load = load;
> +			selected = engine;
> +		}
> +	}
> +
> +	atomic_add(weight, &selected->load);
> +
> +	return selected;
> +}
> +
>  /*
>   * Helper function that indicates whether a crypto request needs to be
>   * cleaned up or not after being enqueued using mv_cesa_queue_req().
> diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c
> index 79d4175..28894be 100644
> --- a/drivers/crypto/marvell/cipher.c
> +++ b/drivers/crypto/marvell/cipher.c
> @@ -214,6 +214,7 @@ mv_cesa_ablkcipher_complete(struct crypto_async_request *req)
>  	struct mv_cesa_engine *engine = creq->base.engine;
>  	unsigned int ivsize;
>  
> +	atomic_sub(ablkreq->nbytes, &engine->load);
>  	ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq));
>  
>  	if (mv_cesa_req_get_type(&creq->base) == CESA_DMA_REQ) {
> @@ -231,7 +232,6 @@ mv_cesa_ablkcipher_complete(struct crypto_async_request *req)
>  static const struct mv_cesa_req_ops mv_cesa_ablkcipher_req_ops = {
>  	.step = mv_cesa_ablkcipher_step,
>  	.process = mv_cesa_ablkcipher_process,
> -	.prepare = mv_cesa_ablkcipher_prepare,
>  	.cleanup = mv_cesa_ablkcipher_req_cleanup,
>  	.complete = mv_cesa_ablkcipher_complete,
>  };
> @@ -456,29 +456,41 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req,
>  	return ret;
>  }
>  
> -static int mv_cesa_des_op(struct ablkcipher_request *req,
> -			  struct mv_cesa_op_ctx *tmpl)
> +static int mv_cesa_ablkcipher_queue_req(struct ablkcipher_request *req,
> +					struct mv_cesa_op_ctx *tmpl)
>  {
> -	struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req);
> -	struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
>  	int ret;
> -
> -	mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES,
> -			      CESA_SA_DESC_CFG_CRYPTM_MSK);
> -
> -	memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE);
> +	struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req);
> +	struct mv_cesa_engine *engine;
>  
>  	ret = mv_cesa_ablkcipher_req_init(req, tmpl);
>  	if (ret)
>  		return ret;
>  
> +	engine = mv_cesa_select_engine(req->nbytes);
> +	mv_cesa_ablkcipher_prepare(&req->base, engine);
> +
>  	ret = mv_cesa_queue_req(&req->base, &creq->base);
> +
>  	if (mv_cesa_req_needs_cleanup(&req->base, ret))
>  		mv_cesa_ablkcipher_cleanup(req);
>  
>  	return ret;
>  }
>  
> +static int mv_cesa_des_op(struct ablkcipher_request *req,
> +			  struct mv_cesa_op_ctx *tmpl)
> +{
> +	struct mv_cesa_des_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
> +
> +	mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_DES,
> +			      CESA_SA_DESC_CFG_CRYPTM_MSK);
> +
> +	memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES_KEY_SIZE);
> +
> +	return mv_cesa_ablkcipher_queue_req(req, tmpl);
> +}
> +
>  static int mv_cesa_ecb_des_encrypt(struct ablkcipher_request *req)
>  {
>  	struct mv_cesa_op_ctx tmpl;
> @@ -580,24 +592,14 @@ struct crypto_alg mv_cesa_cbc_des_alg = {
>  static int mv_cesa_des3_op(struct ablkcipher_request *req,
>  			   struct mv_cesa_op_ctx *tmpl)
>  {
> -	struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req);
>  	struct mv_cesa_des3_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
> -	int ret;
>  
>  	mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_CRYPTM_3DES,
>  			      CESA_SA_DESC_CFG_CRYPTM_MSK);
>  
>  	memcpy(tmpl->ctx.blkcipher.key, ctx->key, DES3_EDE_KEY_SIZE);
>  
> -	ret = mv_cesa_ablkcipher_req_init(req, tmpl);
> -	if (ret)
> -		return ret;
> -
> -	ret = mv_cesa_queue_req(&req->base, &creq->base);
> -	if (mv_cesa_req_needs_cleanup(&req->base, ret))
> -		mv_cesa_ablkcipher_cleanup(req);
> -
> -	return ret;
> +	return mv_cesa_ablkcipher_queue_req(req, tmpl);
>  }
>  
>  static int mv_cesa_ecb_des3_ede_encrypt(struct ablkcipher_request *req)
> @@ -707,9 +709,8 @@ struct crypto_alg mv_cesa_cbc_des3_ede_alg = {
>  static int mv_cesa_aes_op(struct ablkcipher_request *req,
>  			  struct mv_cesa_op_ctx *tmpl)
>  {
> -	struct mv_cesa_ablkcipher_req *creq = ablkcipher_request_ctx(req);
>  	struct mv_cesa_aes_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
> -	int ret, i;
> +	int i;
>  	u32 *key;
>  	u32 cfg;
>  
> @@ -732,15 +733,7 @@ static int mv_cesa_aes_op(struct ablkcipher_request *req,
>  			      CESA_SA_DESC_CFG_CRYPTM_MSK |
>  			      CESA_SA_DESC_CFG_AES_LEN_MSK);
>  
> -	ret = mv_cesa_ablkcipher_req_init(req, tmpl);
> -	if (ret)
> -		return ret;
> -
> -	ret = mv_cesa_queue_req(&req->base, &creq->base);
> -	if (mv_cesa_req_needs_cleanup(&req->base, ret))
> -		mv_cesa_ablkcipher_cleanup(req);
> -
> -	return ret;
> +	return mv_cesa_ablkcipher_queue_req(req, tmpl);
>  }
>  
>  static int mv_cesa_ecb_aes_encrypt(struct ablkcipher_request *req)
> diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c
> index e1f8acd..b7cfc42 100644
> --- a/drivers/crypto/marvell/hash.c
> +++ b/drivers/crypto/marvell/hash.c
> @@ -335,6 +335,8 @@ static void mv_cesa_ahash_complete(struct crypto_async_request *req)
>  				result[i] = cpu_to_be32(creq->state[i]);
>  		}
>  	}
> +
> +	atomic_sub(ahashreq->nbytes, &engine->load);
>  }
>  
>  static void mv_cesa_ahash_prepare(struct crypto_async_request *req,
> @@ -365,7 +367,6 @@ static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req)
>  static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = {
>  	.step = mv_cesa_ahash_step,
>  	.process = mv_cesa_ahash_process,
> -	.prepare = mv_cesa_ahash_prepare,
>  	.cleanup = mv_cesa_ahash_req_cleanup,
>  	.complete = mv_cesa_ahash_complete,
>  };
> @@ -682,13 +683,13 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached)
>  	return ret;
>  }
>  
> -static int mv_cesa_ahash_update(struct ahash_request *req)
> +static int mv_cesa_ahash_queue_req(struct ahash_request *req)
>  {
>  	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
> +	struct mv_cesa_engine *engine;
>  	bool cached = false;
>  	int ret;
>  
> -	creq->len += req->nbytes;
>  	ret = mv_cesa_ahash_req_init(req, &cached);
>  	if (ret)
>  		return ret;
> @@ -696,61 +697,48 @@ static int mv_cesa_ahash_update(struct ahash_request *req)
>  	if (cached)
>  		return 0;
>  
> +	engine = mv_cesa_select_engine(req->nbytes);
> +	mv_cesa_ahash_prepare(&req->base, engine);
> +
>  	ret = mv_cesa_queue_req(&req->base, &creq->base);
> +
>  	if (mv_cesa_req_needs_cleanup(&req->base, ret))
>  		mv_cesa_ahash_cleanup(req);
>  
>  	return ret;
>  }
>  
> +static int mv_cesa_ahash_update(struct ahash_request *req)
> +{
> +	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
> +
> +	creq->len += req->nbytes;
> +
> +	return mv_cesa_ahash_queue_req(req);
> +}
> +
>  static int mv_cesa_ahash_final(struct ahash_request *req)
>  {
>  	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
>  	struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl;
> -	bool cached = false;
> -	int ret;
>  
>  	mv_cesa_set_mac_op_total_len(tmpl, creq->len);
>  	creq->last_req = true;
>  	req->nbytes = 0;
>  
> -	ret = mv_cesa_ahash_req_init(req, &cached);
> -	if (ret)
> -		return ret;
> -
> -	if (cached)
> -		return 0;
> -
> -	ret = mv_cesa_queue_req(&req->base, &creq->base);
> -	if (mv_cesa_req_needs_cleanup(&req->base, ret))
> -		mv_cesa_ahash_cleanup(req);
> -
> -	return ret;
> +	return mv_cesa_ahash_queue_req(req);
>  }
>  
>  static int mv_cesa_ahash_finup(struct ahash_request *req)
>  {
>  	struct mv_cesa_ahash_req *creq = ahash_request_ctx(req);
>  	struct mv_cesa_op_ctx *tmpl = &creq->op_tmpl;
> -	bool cached = false;
> -	int ret;
>  
>  	creq->len += req->nbytes;
>  	mv_cesa_set_mac_op_total_len(tmpl, creq->len);
>  	creq->last_req = true;
>  
> -	ret = mv_cesa_ahash_req_init(req, &cached);
> -	if (ret)
> -		return ret;
> -
> -	if (cached)
> -		return 0;
> -
> -	ret = mv_cesa_queue_req(&req->base, &creq->base);
> -	if (mv_cesa_req_needs_cleanup(&req->base, ret))
> -		mv_cesa_ahash_cleanup(req);
> -
> -	return ret;
> +	return mv_cesa_ahash_queue_req(req);
>  }
>  
>  static int mv_cesa_ahash_export(struct ahash_request *req, void *hash,

--
To unsubscribe from this list: send the line "unsubscribe linux-crypto" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html



[Index of Archives]     [Kernel]     [Gnu Classpath]     [Gnu Crypto]     [DM Crypt]     [Netfilter]     [Bugtraq]

  Powered by Linux