Reviewed-by: Gaurav Jain <gaurav.jain@xxxxxxx>
> -----Original Message-----
> From: Meenakshi Aggarwal <meenakshi.aggarwal@xxxxxxx>
> Sent: Wednesday, July 12, 2023 11:37 AM
> To: Horia Geanta <horia.geanta@xxxxxxx>; Varun Sethi <V.Sethi@xxxxxxx>;
> Pankaj Gupta <pankaj.gupta@xxxxxxx>; Gaurav Jain <gaurav.jain@xxxxxxx>;
> herbert@xxxxxxxxxxxxxxxxxxx; davem@xxxxxxxxxxxxx; linux-
> crypto@xxxxxxxxxxxxxxx; linux-kernel@xxxxxxxxxxxxxxx
> Cc: Victoria Milhoan <vicki.milhoan@xxxxxxxxxxxxx>; Dan Douglass
> <dan.douglass@xxxxxxx>; Vipul Kumar <vipul_kumar@xxxxxxxxxx>; Franck
> Lenormand <franck.lenormand@xxxxxxx>; Meenakshi Aggarwal
> <meenakshi.aggarwal@xxxxxxx>
> Subject: [PATCH 2/2] crypto: caam - add power management support
>
> From: Horia Geanta <horia.geanta@xxxxxxx>
>
> Add support for suspend and resume operation for PM in CAAM driver.
>
> When the CAAM goes in suspend, the hardware is considered to do nothing.
>
> On some platforms, the power of the CAAM is not turned off so it keeps its
> configuration.
> On other platforms, it doesn't so it is necessary to save the state of the CAAM:
> - JRs MID
> - Address of input and output rings
>
> Signed-off-by: Horia Geanta <horia.geanta@xxxxxxx>
> Signed-off-by: Victoria Milhoan <vicki.milhoan@xxxxxxxxxxxxx>
> Signed-off-by: Dan Douglass <dan.douglass@xxxxxxx>
> Signed-off-by: Vipul Kumar <vipul_kumar@xxxxxxxxxx>
> Signed-off-by: Franck LENORMAND <franck.lenormand@xxxxxxx>
> Signed-off-by: Meenakshi Aggarwal <meenakshi.aggarwal@xxxxxxx>
> ---
> drivers/crypto/caam/ctrl.c | 114 ++++++++++++++++++++
> drivers/crypto/caam/intern.h | 33 +++++-
> drivers/crypto/caam/jr.c | 199 +++++++++++++++++++++++++++++++----
> drivers/crypto/caam/regs.h | 3 +-
> 4 files changed, 328 insertions(+), 21 deletions(-)
>
> diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index
> ff9ddbbca377..64b6cd09d8a8 100644
> --- a/drivers/crypto/caam/ctrl.c
> +++ b/drivers/crypto/caam/ctrl.c
> @@ -740,6 +740,113 @@ static int caam_ctrl_rng_init(struct device *dev)
> return 0;
> }
>
> +#ifdef CONFIG_PM_SLEEP
> +
> +/* Indicate if the internal state of the CAAM is lost during PM */
> +static int caam_off_during_pm(void) {
> + bool not_off_during_pm = of_machine_is_compatible("fsl,imx6q") ||
> + of_machine_is_compatible("fsl,imx6qp") ||
> + of_machine_is_compatible("fsl,imx6dl");
> +
> + return not_off_during_pm ? 0 : 1;
> +}
> +
> +static void caam_state_save(struct device *dev) {
> + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
> + struct caam_ctl_state *state = &ctrlpriv->state;
> + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
> + u32 deco_inst, jr_inst;
> + int i;
> +
> + state->mcr = rd_reg32(&ctrl->mcr);
> + state->scfgr = rd_reg32(&ctrl->scfgr);
> +
> + deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
> + CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT;
> + for (i = 0; i < deco_inst; i++) {
> + state->deco_mid[i].liodn_ms =
> + rd_reg32(&ctrl->deco_mid[i].liodn_ms);
> + state->deco_mid[i].liodn_ls =
> + rd_reg32(&ctrl->deco_mid[i].liodn_ls);
> + }
> +
> + jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
> + CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT;
> + for (i = 0; i < jr_inst; i++) {
> + state->jr_mid[i].liodn_ms =
> + rd_reg32(&ctrl->jr_mid[i].liodn_ms);
> + state->jr_mid[i].liodn_ls =
> + rd_reg32(&ctrl->jr_mid[i].liodn_ls);
> + }
> +}
> +
> +static void caam_state_restore(const struct device *dev) {
> + const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
> + const struct caam_ctl_state *state = &ctrlpriv->state;
> + struct caam_ctrl __iomem *ctrl = ctrlpriv->ctrl;
> + u32 deco_inst, jr_inst;
> + int i;
> +
> + wr_reg32(&ctrl->mcr, state->mcr);
> + wr_reg32(&ctrl->scfgr, state->scfgr);
> +
> + deco_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
> + CHA_ID_MS_DECO_MASK) >> CHA_ID_MS_DECO_SHIFT;
> + for (i = 0; i < deco_inst; i++) {
> + wr_reg32(&ctrl->deco_mid[i].liodn_ms,
> + state->deco_mid[i].liodn_ms);
> + wr_reg32(&ctrl->deco_mid[i].liodn_ls,
> + state->deco_mid[i].liodn_ls);
> + }
> +
> + jr_inst = (rd_reg32(&ctrl->perfmon.cha_num_ms) &
> + CHA_ID_MS_JR_MASK) >> CHA_ID_MS_JR_SHIFT;
> + for (i = 0; i < jr_inst; i++) {
> + wr_reg32(&ctrl->jr_mid[i].liodn_ms,
> + state->jr_mid[i].liodn_ms);
> + wr_reg32(&ctrl->jr_mid[i].liodn_ls,
> + state->jr_mid[i].liodn_ls);
> + }
> +
> + if (ctrlpriv->virt_en == 1)
> + clrsetbits_32(&ctrl->jrstart, 0, JRSTART_JR0_START |
> + JRSTART_JR1_START | JRSTART_JR2_START |
> + JRSTART_JR3_START);
> +}
> +
> +static int caam_ctrl_suspend(struct device *dev) {
> + const struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
> +
> + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en)
> + caam_state_save(dev);
> +
> + return 0;
> +}
> +
> +static int caam_ctrl_resume(struct device *dev) {
> + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev);
> + int ret = 0;
> +
> + if (ctrlpriv->caam_off_during_pm && !ctrlpriv->optee_en) {
> + caam_state_restore(dev);
> +
> + /* HW and rng will be reset so deinstantiation can be removed
> */
> + devm_remove_action(dev, devm_deinstantiate_rng, dev);
> + ret = caam_ctrl_rng_init(dev);
> + }
> +
> + return ret;
> +}
> +
> +static SIMPLE_DEV_PM_OPS(caam_ctrl_pm_ops, caam_ctrl_suspend,
> +caam_ctrl_resume);
> +
> +#endif /* CONFIG_PM_SLEEP */
> +
> /* Probe routine for CAAM top (controller) level */ static int caam_probe(struct
> platform_device *pdev) { @@ -771,6 +878,10 @@ static int caam_probe(struct
> platform_device *pdev)
>
> caam_imx = (bool)imx_soc_match;
>
> +#ifdef CONFIG_PM_SLEEP
> + ctrlpriv->caam_off_during_pm = caam_imx && caam_off_during_pm();
> +#endif
> +
> if (imx_soc_match) {
> /*
> * Until Layerscape and i.MX OP-TEE get in sync, @@ -1033,6
> +1144,9 @@ static struct platform_driver caam_driver = {
> .driver = {
> .name = "caam",
> .of_match_table = caam_match,
> +#ifdef CONFIG_PM_SLEEP
> + .pm = &caam_ctrl_pm_ops,
> +#endif
> },
> .probe = caam_probe,
> };
> diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index
> b4f7bf77f487..fcf0a080b5d6 100644
> --- a/drivers/crypto/caam/intern.h
> +++ b/drivers/crypto/caam/intern.h
> @@ -4,7 +4,7 @@
> * Private/internal definitions between modules
> *
> * Copyright 2008-2011 Freescale Semiconductor, Inc.
> - * Copyright 2019 NXP
> + * Copyright 2019, 2023 NXP
> */
>
> #ifndef INTERN_H
> @@ -47,6 +47,18 @@ struct caam_jrentry_info {
> u32 desc_size; /* Stored size for postprocessing, header derived */
> };
>
> +#ifdef CONFIG_PM_SLEEP
> +struct caam_jr_state {
> + dma_addr_t inpbusaddr;
> + dma_addr_t outbusaddr;
> +};
> +#endif
> +
> +struct caam_jr_dequeue_params {
> + struct device *dev;
> + int enable_itr;
> +};
> +
> /* Private sub-storage for a single JobR */ struct caam_drv_private_jr {
> struct list_head list_node; /* Job Ring device list */
> @@ -54,6 +66,7 @@ struct caam_drv_private_jr {
> int ridx;
> struct caam_job_ring __iomem *rregs; /* JobR's register space */
> struct tasklet_struct irqtask;
> + struct caam_jr_dequeue_params tasklet_params;
> int irq; /* One per queue */
> bool hwrng;
>
> @@ -71,7 +84,20 @@ struct caam_drv_private_jr {
> int tail; /* entinfo (s/w ring) tail index */
> void *outring; /* Base of output ring, DMA-safe */
> struct crypto_engine *engine;
> +
> +#ifdef CONFIG_PM_SLEEP
> + struct caam_jr_state state; /* State of the JR during PM */
> +#endif
> +};
> +
> +#ifdef CONFIG_PM_SLEEP
> +struct caam_ctl_state {
> + struct masterid deco_mid[16];
> + struct masterid jr_mid[4];
> + u32 mcr;
> + u32 scfgr;
> };
> +#endif
>
> /*
> * Driver-private storage for a single CAAM block instance @@ -116,6 +142,11
> @@ struct caam_drv_private {
> struct dentry *ctl; /* controller dir */
> struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap,
> ctl_tdsk_wrap; #endif
> +
> +#ifdef CONFIG_PM_SLEEP
> + int caam_off_during_pm; /* If the CAAM is reset after
> suspend */
> + struct caam_ctl_state state; /* State of the CTL during PM */
> +#endif
> };
>
> #ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API
> diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index
> 96dea5304d22..1801638eb1c8 100644
> --- a/drivers/crypto/caam/jr.c
> +++ b/drivers/crypto/caam/jr.c
> @@ -117,6 +117,25 @@ static int caam_jr_flush(struct device *dev)
> return caam_jr_stop_processing(dev, JRCR_RESET); }
>
> +#ifdef CONFIG_PM_SLEEP
> +/* The resume can be used after a park or a flush if CAAM has not been
> +reset */ static int caam_jr_restart_processing(struct device *dev) {
> + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
> + u32 halt_status = rd_reg32(&jrp->rregs->jrintstatus) &
> + JRINT_ERR_HALT_MASK;
> +
> + /* Check that the flush/park is completed */
> + if (halt_status != JRINT_ERR_HALT_COMPLETE)
> + return -1;
> +
> + /* Resume processing of jobs */
> + clrsetbits_32(&jrp->rregs->jrintstatus, 0, JRINT_ERR_HALT_COMPLETE);
> +
> + return 0;
> +}
> +#endif /* CONFIG_PM_SLEEP */
> +
> static int caam_reset_hw_jr(struct device *dev) {
> struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); @@ -245,7
> +264,8 @@ static irqreturn_t caam_jr_interrupt(int irq, void *st_dev) static void
> caam_jr_dequeue(unsigned long devarg) {
> int hw_idx, sw_idx, i, head, tail;
> - struct device *dev = (struct device *)devarg;
> + struct caam_jr_dequeue_params *params = (void *)devarg;
> + struct device *dev = params->dev;
> struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
> void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
> u32 *userdesc, userstatus;
> @@ -319,8 +339,9 @@ static void caam_jr_dequeue(unsigned long devarg)
> outring_used--;
> }
>
> - /* reenable / unmask IRQs */
> - clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
> + if (params->enable_itr)
> + /* reenable / unmask IRQs */
> + clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
> }
>
> /**
> @@ -470,6 +491,29 @@ int caam_jr_enqueue(struct device *dev, u32 *desc, }
> EXPORT_SYMBOL(caam_jr_enqueue);
>
> +static void caam_jr_init_hw(struct device *dev, dma_addr_t inpbusaddr,
> + dma_addr_t outbusaddr)
> +{
> + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
> +
> + wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
> + wr_reg64(&jrp->rregs->outring_base, outbusaddr);
> + wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
> + wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
> +
> + /* Select interrupt coalescing parameters */
> + clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
> + (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
> + (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); }
> +
> +static void caam_jr_reset_index(struct caam_drv_private_jr *jrp) {
> + jrp->out_ring_read_index = 0;
> + jrp->head = 0;
> + jrp->tail = 0;
> +}
> +
> /*
> * Init JobR independent of platform property detection
> */
> @@ -506,25 +550,16 @@ static int caam_jr_init(struct device *dev)
> jrp->entinfo[i].desc_addr_dma = !0;
>
> /* Setup rings */
> - jrp->out_ring_read_index = 0;
> - jrp->head = 0;
> - jrp->tail = 0;
> -
> - wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
> - wr_reg64(&jrp->rregs->outring_base, outbusaddr);
> - wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
> - wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
> -
> + caam_jr_reset_index(jrp);
> jrp->inpring_avail = JOBR_DEPTH;
> + caam_jr_init_hw(dev, inpbusaddr, outbusaddr);
>
> spin_lock_init(&jrp->inplock);
>
> - /* Select interrupt coalescing parameters */
> - clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
> - (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
> - (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
> -
> - tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
> + jrp->tasklet_params.dev = dev;
> + jrp->tasklet_params.enable_itr = 1;
> + tasklet_init(&jrp->irqtask, caam_jr_dequeue,
> + (unsigned long)&jrp->tasklet_params);
>
> /* Connect job ring interrupt handler. */
> error = devm_request_irq(dev, jrp->irq, caam_jr_interrupt,
> IRQF_SHARED, @@ -635,11 +670,136 @@ static int caam_jr_probe(struct
> platform_device *pdev)
>
> atomic_set(&jrpriv->tfm_count, 0);
>
> + device_init_wakeup(&pdev->dev, 1);
> + device_set_wakeup_enable(&pdev->dev, false);
> +
> register_algs(jrpriv, jrdev->parent);
>
> return 0;
> }
>
> +#ifdef CONFIG_PM_SLEEP
> +static void caam_jr_get_hw_state(struct device *dev) {
> + struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
> +
> + jrp->state.inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
> + jrp->state.outbusaddr = rd_reg64(&jrp->rregs->outring_base);
> +}
> +
> +static int caam_jr_suspend(struct device *dev) {
> + struct platform_device *pdev = to_platform_device(dev);
> + struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev);
> + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent);
> + struct caam_jr_dequeue_params suspend_params = {
> + .dev = dev,
> + .enable_itr = 0,
> + };
> +
> + /* Remove the node from Physical JobR list maintained by driver */
> + spin_lock(&driver_data.jr_alloc_lock);
> + list_del(&jrpriv->list_node);
> + spin_unlock(&driver_data.jr_alloc_lock);
> +
> + if (jrpriv->hwrng)
> + caam_rng_exit(dev->parent);
> +
> + if (ctrlpriv->caam_off_during_pm) {
> + int err;
> +
> + tasklet_disable(&jrpriv->irqtask);
> +
> + /* mask itr to call flush */
> + clrsetbits_32(&jrpriv->rregs->rconfig_lo, 0, JRCFG_IMSK);
> +
> + /* Invalid job in process */
> + err = caam_jr_flush(dev);
> + if (err) {
> + dev_err(dev, "Failed to flush\n");
> + return err;
> + }
> +
> + /* Dequeing jobs flushed */
> + caam_jr_dequeue((unsigned long)&suspend_params);
> +
> + /* Save state */
> + caam_jr_get_hw_state(dev);
> + } else if (device_may_wakeup(&pdev->dev)) {
> + enable_irq_wake(jrpriv->irq);
> + }
> +
> + return 0;
> +}
> +
> +static int caam_jr_resume(struct device *dev) {
> + struct platform_device *pdev = to_platform_device(dev);
> + struct caam_drv_private_jr *jrpriv = platform_get_drvdata(pdev);
> + struct caam_drv_private *ctrlpriv = dev_get_drvdata(dev->parent);
> +
> + if (ctrlpriv->caam_off_during_pm) {
> + u64 inp_addr;
> + int err;
> +
> + /*
> + * Check if the CAAM has been resetted checking the address of
> + * the input ring
> + */
> + inp_addr = rd_reg64(&jrpriv->rregs->inpring_base);
> + if (inp_addr != 0) {
> + /* JR still has some configuration */
> + if (inp_addr == jrpriv->state.inpbusaddr) {
> + /* JR has not been resetted */
> + err = caam_jr_restart_processing(dev);
> + if (err) {
> + dev_err(dev,
> + "Restart processing failed\n");
> + return err;
> + }
> +
> + tasklet_enable(&jrpriv->irqtask);
> +
> + clrsetbits_32(&jrpriv->rregs->rconfig_lo,
> + JRCFG_IMSK, 0);
> +
> + goto add_jr;
> + } else if (ctrlpriv->optee_en) {
> + /* JR has been used by OPTEE, reset it */
> + err = caam_reset_hw_jr(dev);
> + if (err) {
> + dev_err(dev, "Failed to reset JR\n");
> + return err;
> + }
> + } else {
> + /* No explanation, return error */
> + return -EIO;
> + }
> + }
> +
> + caam_jr_reset_index(jrpriv);
> + caam_jr_init_hw(dev, jrpriv->state.inpbusaddr,
> + jrpriv->state.outbusaddr);
> +
> + tasklet_enable(&jrpriv->irqtask);
> + } else if (device_may_wakeup(&pdev->dev)) {
> + disable_irq_wake(jrpriv->irq);
> + }
> +
> +add_jr:
> + spin_lock(&driver_data.jr_alloc_lock);
> + list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
> + spin_unlock(&driver_data.jr_alloc_lock);
> +
> + if (jrpriv->hwrng)
> + jrpriv->hwrng = !caam_rng_init(dev->parent);
> +
> + return 0;
> +}
> +
> +static SIMPLE_DEV_PM_OPS(caam_jr_pm_ops, caam_jr_suspend,
> +caam_jr_resume); #endif /* CONFIG_PM_SLEEP */
> +
> static const struct of_device_id caam_jr_match[] = {
> {
> .compatible = "fsl,sec-v4.0-job-ring", @@ -655,6 +815,9 @@
> static struct platform_driver caam_jr_driver = {
> .driver = {
> .name = "caam_jr",
> .of_match_table = caam_jr_match,
> +#ifdef CONFIG_PM_SLEEP
> + .pm = &caam_jr_pm_ops,
> +#endif
> },
> .probe = caam_jr_probe,
> .remove = caam_jr_remove,
> diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index
> 0f87bd365582..873df9de9890 100644
> --- a/drivers/crypto/caam/regs.h
> +++ b/drivers/crypto/caam/regs.h
> @@ -584,8 +584,7 @@ struct caam_ctrl {
> u32 deco_rsr; /* DECORSR - Deco Request Source */
> u32 rsvd11;
> u32 deco_rq; /* DECORR - DECO Request */
> - struct masterid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */
> - u32 rsvd5[22];
> + struct masterid deco_mid[16]; /* DECOxLIODNR - 1 per DECO */
>
> /* DECO Availability/Reset Section 120-3ff */
> u32 deco_avail; /* DAR - DECO availability */
> --
> 2.25.1
