On Fri, Feb 07, 2025 at 04:58:57AM +0530, Krishna Chaitanya Chundru wrote:
> The current implementation requires iATU for every configuration
> space access which increases latency & cpu utilization.
>
> Designware databook 5.20a, section 3.10.10.3 says about CFG Shift Feature,
> which shifts/maps the BDF (bits [31:16] of the third header DWORD, which
> would be matched against the Base and Limit addresses) of the incoming
> CfgRd0/CfgWr0 down to bits[27:12]of the translated address.
>
> Configuring iATU in config shift feature enables ECAM feature to access the
> config space, which avoids iATU configuration for every config access.
>
> Add "ctrl2" into struct dw_pcie_ob_atu_cfg to enable config shift feature.
>
> As DBI comes under config space, this avoids remapping of DBI space
> separately. Instead, it uses the mapped config space address returned from
> ECAM initialization. Change the order of dw_pcie_get_resources() execution
> to achieve this.
>
> Enable the ECAM feature if the config space size is equal to size required
> to represent number of buses in the bus range property, add a function
> which checks this. The DWC glue drivers uses this function and decide to
> enable ECAM mode or not.
>
> Signed-off-by: Krishna Chaitanya Chundru <krishna.chundru@xxxxxxxxxxxxxxxx>
> ---
> drivers/pci/controller/dwc/Kconfig | 1 +
> drivers/pci/controller/dwc/pcie-designware-host.c | 133 +++++++++++++++++++---
> drivers/pci/controller/dwc/pcie-designware.c | 2 +-
> drivers/pci/controller/dwc/pcie-designware.h | 11 ++
> 4 files changed, 132 insertions(+), 15 deletions(-)
>
> diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
> index b6d6778b0698..73c3aed6b60a 100644
> --- a/drivers/pci/controller/dwc/Kconfig
> +++ b/drivers/pci/controller/dwc/Kconfig
> @@ -9,6 +9,7 @@ config PCIE_DW
> config PCIE_DW_HOST
> bool
> select PCIE_DW
> + select PCI_HOST_COMMON
>
> config PCIE_DW_EP
> bool
> diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
> index ffaded8f2df7..826ff9338646 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-host.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-host.c
> @@ -418,6 +418,66 @@ static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
> }
> }
>
> +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> + struct dw_pcie_ob_atu_cfg atu = {0};
> + resource_size_t bus_range_max;
> + struct resource_entry *bus;
> + int ret;
> +
> + bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> +
> + /*
> + * Root bus under the host bridge doesn't require any iATU configuration
> + * as DBI space will represent Root bus configuration space.
> + * Immediate bus under Root Bus, needs type 0 iATU configuration and
> + * remaining buses need type 1 iATU configuration.
> + */
> + atu.index = 0;
> + atu.type = PCIE_ATU_TYPE_CFG0;
> + atu.cpu_addr = pp->cfg0_base + SZ_1M;
Why this 1MiB hole needed here? For DBI?
> + atu.size = SZ_1M;
Add a comment stating that this 1M corresponds to 1 bus, 32 devices and 8
functions. Like,
/* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */
> + atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> + ret = dw_pcie_prog_outbound_atu(pci, &atu);
> + if (ret)
> + return ret;
> +
> + bus_range_max = resource_size(bus->res);
> +
> + if (bus_range_max < 2)
> + return 0;
> +
> + /* Configure remaining buses in type 1 iATU configuration */
> + atu.index = 1;
> + atu.type = PCIE_ATU_TYPE_CFG1;
> + atu.cpu_addr = pp->cfg0_base + SZ_2M;
> + atu.size = (SZ_1M * (bus_range_max - 2));
Nit: Use,
atu.size = (SZ_1M * bus_range_max) - SZ_2M;
> + atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
> +
> + return dw_pcie_prog_outbound_atu(pci, &atu);
> +}
> +
> +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> + struct device *dev = pci->dev;
> + struct resource_entry *bus;
> +
> + bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
> + if (!bus)
> + return -ENODEV;
> +
> + pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
> + if (IS_ERR(pp->cfg))
> + return PTR_ERR(pp->cfg);
> +
> + pci->dbi_base = pp->cfg->win;
> + pci->dbi_phys_addr = res->start;
Why can't we use the existing 'dbi' region?
> +
> + return 0;
> +}
> +
> int dw_pcie_host_init(struct dw_pcie_rp *pp)
> {
> struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> @@ -431,10 +491,6 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>
> raw_spin_lock_init(&pp->lock);
>
> - ret = dw_pcie_get_resources(pci);
> - if (ret)
> - return ret;
> -
> res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
> if (!res) {
> dev_err(dev, "Missing \"config\" reg space\n");
> @@ -444,9 +500,28 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
> pp->cfg0_size = resource_size(res);
> pp->cfg0_base = res->start;
>
> - pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
> - if (IS_ERR(pp->va_cfg0_base))
> - return PTR_ERR(pp->va_cfg0_base);
> + if (pp->ecam_mode) {
> + ret = dw_pcie_create_ecam_window(pp, res);
> + if (ret)
> + return ret;
> +
> + bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
> + pp->bridge->sysdata = pp->cfg;
> + pp->cfg->priv = pp;
> + } else {
> + pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
> + if (IS_ERR(pp->va_cfg0_base))
> + return PTR_ERR(pp->va_cfg0_base);
> +
> + /* Set default bus ops */
> + bridge->ops = &dw_pcie_ops;
> + bridge->child_ops = &dw_child_pcie_ops;
> + bridge->sysdata = pp;
> + }
> +
> + ret = dw_pcie_get_resources(pci);
> + if (ret)
> + return ret;
>
> bridge = devm_pci_alloc_host_bridge(dev, 0);
> if (!bridge)
> @@ -462,14 +537,10 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
> pp->io_base = pci_pio_to_address(win->res->start);
> }
>
> - /* Set default bus ops */
> - bridge->ops = &dw_pcie_ops;
> - bridge->child_ops = &dw_child_pcie_ops;
> -
> if (pp->ops->init) {
> ret = pp->ops->init(pp);
> if (ret)
> - return ret;
> + goto err_free_ecam;
> }
>
> if (pci_msi_enabled()) {
> @@ -504,6 +575,14 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
>
> dw_pcie_iatu_detect(pci);
>
> + if (pp->ecam_mode) {
> + ret = dw_pcie_config_ecam_iatu(pp);
> + if (ret) {
> + dev_err(dev, "Failed to confuure iATU\n");
*configure iATU in ECAM mode
> + goto err_free_msi;
> + }
> + }
> +
> /*
> * Allocate the resource for MSG TLP before programming the iATU
> * outbound window in dw_pcie_setup_rc(). Since the allocation depends
> @@ -539,8 +618,6 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
> /* Ignore errors, the link may come up later */
> dw_pcie_wait_for_link(pci);
>
> - bridge->sysdata = pp;
> -
> ret = pci_host_probe(bridge);
> if (ret)
> goto err_stop_link;
> @@ -564,6 +641,10 @@ int dw_pcie_host_init(struct dw_pcie_rp *pp)
> if (pp->ops->deinit)
> pp->ops->deinit(pp);
>
> +err_free_ecam:
> + if (pp->cfg)
> + pci_ecam_free(pp->cfg);
> +
> return ret;
> }
> EXPORT_SYMBOL_GPL(dw_pcie_host_init);
> @@ -584,6 +665,9 @@ void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
>
> if (pp->ops->deinit)
> pp->ops->deinit(pp);
> +
> + if (pp->cfg)
> + pci_ecam_free(pp->cfg);
> }
> EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
>
> @@ -999,3 +1083,24 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci)
> return ret;
> }
> EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
> +
> +bool dw_pcie_ecam_supported(struct dw_pcie_rp *pp)
Add Kernel doc since this is an exported API. Clearly mention the requirement
needed (config region size) to support the ECAM mode.
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
> + struct platform_device *pdev = to_platform_device(pci->dev);
> + struct resource *config_res, *bus_range;
> + u64 bus_config_space_count;
> +
> + bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res;
> + if (!bus_range)
> + return false;
> +
> + config_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
> + if (!config_res)
> + return false;
> +
> + bus_config_space_count = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT;
> +
> + return !!(bus_config_space_count >= resource_size(bus_range));
> +}
> +EXPORT_SYMBOL_GPL(dw_pcie_ecam_supported);
> diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
> index 145e7f579072..523ca7f267fb 100644
> --- a/drivers/pci/controller/dwc/pcie-designware.c
> +++ b/drivers/pci/controller/dwc/pcie-designware.c
> @@ -509,7 +509,7 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci,
> val = dw_pcie_enable_ecrc(val);
> dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val);
>
> - val = PCIE_ATU_ENABLE;
> + val = PCIE_ATU_ENABLE | atu->ctrl2;
> if (atu->type == PCIE_ATU_TYPE_MSG) {
> /* The data-less messages only for now */
> val |= PCIE_ATU_INHIBIT_PAYLOAD | atu->code;
> diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
> index 501d9ddfea16..d0ba8855ba2a 100644
> --- a/drivers/pci/controller/dwc/pcie-designware.h
> +++ b/drivers/pci/controller/dwc/pcie-designware.h
> @@ -20,6 +20,7 @@
> #include <linux/irq.h>
> #include <linux/msi.h>
> #include <linux/pci.h>
> +#include <linux/pci-ecam.h>
> #include <linux/reset.h>
>
> #include <linux/pci-epc.h>
> @@ -171,6 +172,7 @@
> #define PCIE_ATU_REGION_CTRL2 0x004
> #define PCIE_ATU_ENABLE BIT(31)
> #define PCIE_ATU_BAR_MODE_ENABLE BIT(30)
> +#define PCIE_ATU_CFG_SHIFT_MODE_ENABLE BIT(28)
> #define PCIE_ATU_INHIBIT_PAYLOAD BIT(22)
> #define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19)
> #define PCIE_ATU_LOWER_BASE 0x008
> @@ -343,6 +345,7 @@ struct dw_pcie_ob_atu_cfg {
> u8 func_no;
> u8 code;
> u8 routing;
> + u32 ctrl2;
> u64 cpu_addr;
> u64 pci_addr;
> u64 size;
> @@ -381,6 +384,8 @@ struct dw_pcie_rp {
> int msg_atu_index;
> struct resource *msg_res;
> bool use_linkup_irq;
> + bool ecam_mode;
> + struct pci_config_window *cfg;
> };
>
> struct dw_pcie_ep_ops {
> @@ -686,6 +691,7 @@ void dw_pcie_host_deinit(struct dw_pcie_rp *pp);
> int dw_pcie_allocate_domains(struct dw_pcie_rp *pp);
> void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn,
> int where);
> +bool dw_pcie_ecam_supported(struct dw_pcie_rp *pp);
> #else
> static inline int dw_pcie_suspend_noirq(struct dw_pcie *pci)
> {
> @@ -726,6 +732,11 @@ static inline void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus,
> {
> return NULL;
> }
> +
> +static inline bool dw_pcie_ecam_supported(struct dw_pcie_rp *pp)
> +{
> + return 0;
return false
- Mani
--
மணிவண்ணன் சதாசிவம்
