Re: [PATCH v3 3/8] accel/qaic: Add MHI controller

[Jeffrey Hugo <quic_jhugo@xxxxxxxxxxx>]

On 3/13/2023 7:39 AM, Jacek Lawrynowicz wrote:
> Hi,
>
> On 06.03.2023 22:33, Jeffrey Hugo wrote:
> > An AIC100 device contains a MHI interface with a number of different
> > channels for controlling different aspects of the device. The MHI
> > controller works with the MHI bus to enable and drive that interface.
> >
> > AIC100 uses the BHI protocol in PBL to load SBL. The MHI controller
> > expects the SBL to be located at /lib/firmware/qcom/aic100/sbl.bin and
> > expects the MHI bus to manage the process of loading and sending SBL to
> > the device.
> >
> > Signed-off-by: Jeffrey Hugo <quic_jhugo@xxxxxxxxxxx>
> > Reviewed-by: Carl Vanderlip <quic_carlv@xxxxxxxxxxx>
> > Reviewed-by: Pranjal Ramajor Asha Kanojiya <quic_pkanojiy@xxxxxxxxxxx>
> > ---
> >   drivers/accel/qaic/mhi_controller.c | 563 ++++++++++++++++++++++++++++++++++++
> >   drivers/accel/qaic/mhi_controller.h |  16 +
> >   2 files changed, 579 insertions(+)
> >   create mode 100644 drivers/accel/qaic/mhi_controller.c
> >   create mode 100644 drivers/accel/qaic/mhi_controller.h
> >
> > diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c
> > new file mode 100644
> > index 0000000..f16dbb7
> > --- /dev/null
> > +++ b/drivers/accel/qaic/mhi_controller.c
> > @@ -0,0 +1,563 @@
> > +// SPDX-License-Identifier: GPL-2.0-only
> > +
> > +/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
> > +/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
> > +
> > +#include <linux/delay.h>
> > +#include <linux/err.h>
> > +#include <linux/memblock.h>
> > +#include <linux/mhi.h>
> > +#include <linux/moduleparam.h>
> > +#include <linux/pci.h>
> > +#include <linux/sizes.h>
> > +
> > +#include "mhi_controller.h"
> > +#include "qaic.h"
> > +
> > +#define MAX_RESET_TIME_SEC 25
> > +
> > +static unsigned int mhi_timeout = 2000; /* 2 sec default */
> > +module_param(mhi_timeout, uint, 0600);
>
> Consider documenting the param with MODULE_PARM_DESC() and adding _ms postfix to
> indicate that time units it is using.

Will do.

> > +
> > +static struct mhi_channel_config aic100_channels[] = {
> > +	{
> > +		.name = "QAIC_LOOPBACK",
> > +		.num = 0,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_LOOPBACK",
> > +		.num = 1,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_SAHARA",
> > +		.num = 2,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_SAHARA",
> > +		.num = 3,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_DIAG",
> > +		.num = 4,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_DIAG",
> > +		.num = 5,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_SSR",
> > +		.num = 6,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_SSR",
> > +		.num = 7,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_QDSS",
> > +		.num = 8,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_QDSS",
> > +		.num = 9,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_CONTROL",
> > +		.num = 10,
> > +		.num_elements = 128,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_CONTROL",
> > +		.num = 11,
> > +		.num_elements = 128,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_LOGGING",
> > +		.num = 12,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_LOGGING",
> > +		.num = 13,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_STATUS",
> > +		.num = 14,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_STATUS",
> > +		.num = 15,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_TELEMETRY",
> > +		.num = 16,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_TELEMETRY",
> > +		.num = 17,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_DEBUG",
> > +		.num = 18,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_DEBUG",
> > +		.num = 19,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.name = "QAIC_TIMESYNC",
> > +		.num = 20,
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_TO_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +	{
> > +		.num = 21,
> > +		.name = "QAIC_TIMESYNC",
> > +		.num_elements = 32,
> > +		.local_elements = 0,
> > +		.event_ring = 0,
> > +		.dir = DMA_FROM_DEVICE,
> > +		.ee_mask = MHI_CH_EE_SBL | MHI_CH_EE_AMSS,
> > +		.pollcfg = 0,
> > +		.doorbell = MHI_DB_BRST_DISABLE,
> > +		.lpm_notify = false,
> > +		.offload_channel = false,
> > +		.doorbell_mode_switch = false,
> > +		.auto_queue = false,
> > +		.wake_capable = false,
> > +	},
> > +};
> > +
> > +static struct mhi_event_config aic100_events[] = {
> > +	{
> > +		.num_elements = 32,
> > +		.irq_moderation_ms = 0,
> > +		.irq = 0,
> > +		.channel = U32_MAX,
> > +		.priority = 1,
> > +		.mode = MHI_DB_BRST_DISABLE,
> > +		.data_type = MHI_ER_CTRL,
> > +		.hardware_event = false,
> > +		.client_managed = false,
> > +		.offload_channel = false,
> > +	},
> > +};
> > +
> > +static struct mhi_controller_config aic100_config = {
> > +	.max_channels = 128,
> > +	.timeout_ms = 0, /* controlled by mhi_timeout */
> > +	.buf_len = 0,
> > +	.num_channels = ARRAY_SIZE(aic100_channels),
> > +	.ch_cfg = aic100_channels,
> > +	.num_events = ARRAY_SIZE(aic100_events),
> > +	.event_cfg = aic100_events,
> > +	.use_bounce_buf = false,
> > +	.m2_no_db = false,
> > +};
> > +
> > +static int mhi_read_reg(struct mhi_controller *mhi_cntl, void __iomem *addr, u32 *out)
> > +{
> > +	u32 tmp = readl_relaxed(addr);
> > +
> > +	if (tmp == U32_MAX)
> > +		return -EIO;
> > +
> > +	*out = tmp;
> > +
> > +	return 0;
> > +}
> > +
> > +static void mhi_write_reg(struct mhi_controller *mhi_cntl, void __iomem *addr, u32 val)
> > +{
> > +	writel_relaxed(val, addr);
> > +}
> > +
> > +static int mhi_runtime_get(struct mhi_controller *mhi_cntl)
> > +{
> > +	return 0;
> > +}
> > +
> > +static void mhi_runtime_put(struct mhi_controller *mhi_cntl)
> > +{
> > +}
> > +
> > +static void mhi_status_cb(struct mhi_controller *mhi_cntl, enum mhi_callback reason)
> > +{
> > +	struct qaic_device *qdev = pci_get_drvdata(to_pci_dev(mhi_cntl->cntrl_dev));
> > +
> > +	/* this event occurs in atomic context */
> > +	if (reason == MHI_CB_FATAL_ERROR)
> > +		pci_err(qdev->pdev, "Fatal error received from device. Attempting to recover\n");
> > +	/* this event occurs in non-atomic context */
> > +	if (reason == MHI_CB_SYS_ERROR && !qdev->in_reset)
>
> Looks like qdev->in_reset should be protected by qdev->dev_lock.

Will check.

> > +		qaic_dev_reset_clean_local_state(qdev, true);
> > +}
> > +
> > +static int mhi_reset_and_async_power_up(struct mhi_controller *mhi_cntl)
> > +{
> > +	char time_sec = 1;
> > +	int current_ee;
> > +	int ret;
> > +
> > +	/* Reset the device to bring the device in PBL EE */
> > +	mhi_soc_reset(mhi_cntl);
> > +
> > +	/*
> > +	 * Keep checking the execution environment(EE) after every 1 second
> > +	 * interval.
> > +	 */
> > +	do {
> > +		msleep(1000)> +		current_ee = mhi_get_exec_env(mhi_cntl);
> > +	} while (current_ee != MHI_EE_PBL && time_sec++ <= MAX_RESET_TIME_SEC);
> > +
> > +	/* If the device is in PBL EE retry power up */
> > +	if (current_ee == MHI_EE_PBL)
> > +		ret = mhi_async_power_up(mhi_cntl);
> > +	else
> > +		ret = -EIO;
> > +
> > +	return ret;
> > +}
> > +
> > +struct mhi_controller *qaic_mhi_register_controller(struct pci_dev *pci_dev, void __iomem *mhi_bar,
> > +						    int mhi_irq)
> > +{
> > +	struct mhi_controller *mhi_cntl;
> > +	int ret;
> > +
> > +	mhi_cntl = devm_kzalloc(&pci_dev->dev, sizeof(*mhi_cntl), GFP_KERNEL);
> > +	if (!mhi_cntl)
> > +		return ERR_PTR(-ENOMEM);
> > +
> > +	mhi_cntl->cntrl_dev = &pci_dev->dev;
> > +
> > +	/*
> > +	 * Covers the entire possible physical ram region. Remote side is
> > +	 * going to calculate a size of this range, so subtract 1 to prevent
> > +	 * rollover.
> > +	 */
> > +	mhi_cntl->iova_start = 0;
> > +	mhi_cntl->iova_stop = PHYS_ADDR_MAX - 1;
> > +	mhi_cntl->status_cb = mhi_status_cb;
> > +	mhi_cntl->runtime_get = mhi_runtime_get;
> > +	mhi_cntl->runtime_put = mhi_runtime_put;
> > +	mhi_cntl->read_reg = mhi_read_reg;
> > +	mhi_cntl->write_reg = mhi_write_reg;
> > +	mhi_cntl->regs = mhi_bar;
> > +	mhi_cntl->reg_len = SZ_4K;
> > +	mhi_cntl->nr_irqs = 1;
> > +	mhi_cntl->irq = devm_kmalloc(&pci_dev->dev, sizeof(*mhi_cntl->irq), GFP_KERNEL);
> > +
> > +	if (!mhi_cntl->irq)
> > +		return ERR_PTR(-ENOMEM);
> > +
> > +	mhi_cntl->irq[0] = mhi_irq;
> > +	mhi_cntl->fw_image = "qcom/aic100/sbl.bin";
> > +
> > +	/* use latest configured timeout */
> > +	aic100_config.timeout_ms = mhi_timeout;
> > +	ret = mhi_register_controller(mhi_cntl, &aic100_config);
> > +	if (ret) {
> > +		pci_err(pci_dev, "mhi_register_controller failed %d\n", ret);
> > +		return ERR_PTR(ret);
> > +	}
> > +
> > +	ret = mhi_prepare_for_power_up(mhi_cntl);
> > +	if (ret) {
> > +		pci_err(pci_dev, "mhi_prepare_for_power_up failed %d\n", ret);
> > +		goto prepare_power_up_fail;
> > +	}
> > +
> > +	ret = mhi_async_power_up(mhi_cntl);
> > +	/*
> > +	 * If EIO is returned it is possible that device is in SBL EE, which is
> > +	 * undesired. SOC reset the device and try to power up again.
> > +	 */
> > +	if (ret == -EIO && MHI_EE_SBL == mhi_get_exec_env(mhi_cntl)) {
> > +		pci_err(pci_dev, "Device is not expected to be SBL EE. SOC resetting the device to put it in PBL EE and again trying mhi async power up. Error %d\n",

Will have a look.