On Fri, Jan 31, 2025 at 07:29:53PM +0100, Niklas Cassel wrote:
> The DWC databook specifies three different BARn_SIZING_SCHEME_N
> - Fixed Mask (0)
> - Programmable Mask (1)
> - Resizable BAR (2)
>
> Each of these sizing schemes have different instructions for how to
> initialize the BAR.
>
> The DWC driver currently does not support resizable BARs.
>
> Instead, in order to somewhat support resizable BARs, the DWC EP driver
> currently has an ugly hack that force sets a resizable BAR to 1 MB, if
> such a BAR is detected.
>
> Additionally, this hack only works if the DWC glue driver also has lied
> in their EPC features, and claimed that the resizable BAR is a 1 MB fixed
> size BAR.
>
> This is unintuitive (as you somehow need to know that you need to lie in
> your EPC features), but other than that it is overly restrictive, since a
> resizable BAR is capable of supporting sizes different than 1 MB.
>
> Add proper support for resizable BARs in the DWC EP driver.
>
> Note that the pci_epc_set_bar() API takes a struct pci_epf_bar which tells
> the EPC driver how it wants to configure the BAR.
>
> struct pci_epf_bar only has a single size struct member.
>
> This means that an EPC driver will only be able to set a single supported
> size. This is perfectly fine, as we do not need the complexity of allowing
> a host to change the size of the BAR. If someone ever wants to support
> resizing a resizable BAR, the pci_epc_set_bar() API can be extended in the
> future.
>
> With these changes, we allow an EPF driver to configure the size of
> Resizable BARs, rather than forcing them to a 1 MB size.
>
> This means that an EPC driver does not need to lie in EPC features, and an
> EPF driver will be able to set an arbitrary size (not be forced to a 1 MB
> size), just like BAR_PROGRAMMABLE.
>
> Signed-off-by: Niklas Cassel <cassel@xxxxxxxxxx>
Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@xxxxxxxxxx>
- Mani
> ---
> .../pci/controller/dwc/pcie-designware-ep.c | 182 ++++++++++++++++--
> 1 file changed, 167 insertions(+), 15 deletions(-)
>
> diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
> index 6b494781da42..72418160e658 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
> @@ -223,6 +223,125 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
> ep->bar_to_atu[bar] = 0;
> }
>
> +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci,
> + enum pci_barno bar)
> +{
> + u32 reg, bar_index;
> + unsigned int offset, nbars;
> + int i;
> +
> + offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
> + if (!offset)
> + return offset;
> +
> + reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> + nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT;
> +
> + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) {
> + reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> + bar_index = reg & PCI_REBAR_CTRL_BAR_IDX;
> + if (bar_index == bar)
> + return offset;
> + }
> +
> + return 0;
> +}
> +
> +static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no,
> + struct pci_epf_bar *epf_bar)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> + enum pci_barno bar = epf_bar->barno;
> + size_t size = epf_bar->size;
> + int flags = epf_bar->flags;
> + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> + unsigned int rebar_offset;
> + u32 rebar_cap, rebar_ctrl;
> + int ret;
> +
> + rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar);
> + if (!rebar_offset)
> + return -EINVAL;
> +
> + ret = pci_epc_bar_size_to_rebar_cap(size, &rebar_cap);
> + if (ret)
> + return ret;
> +
> + dw_pcie_dbi_ro_wr_en(pci);
> +
> + /*
> + * A BAR mask should not be written for a resizable BAR. The BAR mask
> + * is automatically derived by the controller every time the "selected
> + * size" bits are updated, see "Figure 3-26 Resizable BAR Example for
> + * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write
> + * BIT(0) to set the BAR enable bit.
> + */
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0);
> + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> + }
> +
> + /*
> + * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes
> + * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes"
> + * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB.
> + */
> + rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL);
> + rebar_ctrl &= ~GENMASK(31, 16);
> + dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl);
> +
> + /*
> + * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically
> + * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR
> + * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a.
> + */
> + dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap);
> +
> + dw_pcie_dbi_ro_wr_dis(pci);
> +
> + return 0;
> +}
> +
> +static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no,
> + struct pci_epf_bar *epf_bar)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> + enum pci_barno bar = epf_bar->barno;
> + size_t size = epf_bar->size;
> + int flags = epf_bar->flags;
> + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> +
> + dw_pcie_dbi_ro_wr_en(pci);
> +
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> + }
> +
> + dw_pcie_dbi_ro_wr_dis(pci);
> +
> + return 0;
> +}
> +
> +static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep,
> + enum pci_barno bar)
> +{
> + const struct pci_epc_features *epc_features;
> +
> + if (!ep->ops->get_features)
> + return BAR_PROGRAMMABLE;
> +
> + epc_features = ep->ops->get_features(ep);
> +
> + return epc_features->bar[bar].type;
> +}
> +
> static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
> struct pci_epf_bar *epf_bar)
> {
> @@ -230,9 +349,9 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
> struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> enum pci_barno bar = epf_bar->barno;
> size_t size = epf_bar->size;
> + enum pci_epc_bar_type bar_type;
> int flags = epf_bar->flags;
> int ret, type;
> - u32 reg;
>
> /*
> * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs
> @@ -264,19 +383,30 @@ static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
> goto config_atu;
> }
>
> - reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> -
> - dw_pcie_dbi_ro_wr_en(pci);
> -
> - dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1));
> - dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> -
> - if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> - dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1));
> - dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> + bar_type = dw_pcie_ep_get_bar_type(ep, bar);
> + switch (bar_type) {
> + case BAR_FIXED:
> + /*
> + * There is no need to write a BAR mask for a fixed BAR (except
> + * to write 1 to the LSB of the BAR mask register, to enable the
> + * BAR). Write the BAR mask regardless. (The fixed bits in the
> + * BAR mask register will be read-only anyway.)
> + */
> + fallthrough;
> + case BAR_PROGRAMMABLE:
> + ret = dw_pcie_ep_set_bar_programmable(ep, func_no, epf_bar);
> + break;
> + case BAR_RESIZABLE:
> + ret = dw_pcie_ep_set_bar_resizable(ep, func_no, epf_bar);
> + break;
> + default:
> + ret = -EINVAL;
> + dev_err(pci->dev, "Invalid BAR type\n");
> + break;
> }
>
> - dw_pcie_dbi_ro_wr_dis(pci);
> + if (ret)
> + return ret;
>
> config_atu:
> if (!(flags & PCI_BASE_ADDRESS_SPACE))
> @@ -710,9 +840,11 @@ EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit);
>
> static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
> {
> + struct dw_pcie_ep *ep = &pci->ep;
> unsigned int offset;
> unsigned int nbars;
> - u32 reg, i;
> + enum pci_barno bar;
> + u32 reg, i, val;
>
> offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
>
> @@ -727,9 +859,29 @@ static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci)
> * PCIe r6.0, sec 7.8.6.2 require us to support at least one
> * size in the range from 1 MB to 512 GB. Advertise support
> * for 1 MB BAR size only.
> + *
> + * For a BAR that has been configured via dw_pcie_ep_set_bar(),
> + * advertise support for only that size instead.
> */
> - for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL)
> - dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, BIT(4));
> + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) {
> + /*
> + * While the RESBAR_CAP_REG_* fields are sticky, the
> + * RESBAR_CTRL_REG_BAR_SIZE field is non-sticky (it is
> + * sticky in certain versions of DWC PCIe, but not all).
> + *
> + * RESBAR_CTRL_REG_BAR_SIZE is updated automatically by
> + * the controller when RESBAR_CAP_REG is written, which
> + * is why RESBAR_CAP_REG is written here.
> + */
> + val = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> + bar = val & PCI_REBAR_CTRL_BAR_IDX;
> + if (ep->epf_bar[bar])
> + pci_epc_bar_size_to_rebar_cap(ep->epf_bar[bar]->size, &val);
> + else
> + val = BIT(4);
> +
> + dw_pcie_writel_dbi(pci, offset + PCI_REBAR_CAP, val);
> + }
> }
>
> dw_pcie_setup(pci);
> --
> 2.48.1
>
--
மணிவண்ணன் சதாசிவம்
