On Mon, Jan 13, 2025 at 11:27:34AM +0100, Niklas Cassel wrote:
Maybe the subject should be reworded as:
PCI: dwc: endpoint: Allow EPF drivers to configure the size of Resizable BARs
> The DWC databook specifies three different BARn_SIZING_SCHEME_N
> - Fixed Mask (0)
> - Programmable Mask (1)
> - Resizable BAR (2)
>
> Each of these sizing schemes have different instructions for how to
> initialize the BAR.
>
> The DWC driver currently does not support resizable BARs.
>
> Instead, in order to somewhat support resizable BARs, the DWC EP driver
> currently has an ugly hack that force sets a resizable BAR to 1 MB, if
> such a BAR is detected.
>
> Additionally, this hack only works if the DWC glue driver also has lied
> in their EPC features, and claimed that the resizable BAR is a 1 MB fixed
> size BAR.
>
> This is unintuitive (as you somehow need to know that you need to lie in
> your EPC features), but other than that it is overly restrictive, since a
> resizable BAR is capable of supporting sizes different than 1 MB.
>
> Add proper support for resizable BARs in the DWC EP driver.
>
> Note that the pci_epc_set_bar() API takes a struct pci_epf_bar which tells
> the EPC driver how it wants to configure the BAR.
>
> struct pci_epf_bar only has a single size struct member.
>
> This means that an EPC driver will only be able to set a single supported
> size. This is perfectly fine, as we do not need the complexity of allowing
> a host to change the size of the BAR. If someone ever wants to support
> resizing a resizable BAR, the pci_epc_set_bar() API can be extended in the
> future.
>
> With these changes, the DWC EP driver will be able to support resizable
> BARs (we intentionally only support a single supported resizable BAR size).
Same comment as patch 1.
> This means that an EPC driver does not need to lie in EPC features, and an
> EPF driver will be able to set an arbitrary size (not be forced to a 1 MB
> size), just like BAR_PROGRAMMABLE.
>
> Signed-off-by: Niklas Cassel <cassel@xxxxxxxxxx>
> ---
> .../pci/controller/dwc/pcie-designware-ep.c | 195 ++++++++++++++++--
> 1 file changed, 180 insertions(+), 15 deletions(-)
>
> diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
> index 6b494781da42..34c3ae7219f4 100644
> --- a/drivers/pci/controller/dwc/pcie-designware-ep.c
> +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
> @@ -223,6 +223,138 @@ static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
> ep->bar_to_atu[bar] = 0;
> }
>
> +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie *pci,
> + enum pci_barno bar)
> +{
> + u32 reg, bar_index;
> + unsigned int offset, nbars;
> + int i;
> +
> + offset = dw_pcie_ep_find_ext_capability(pci, PCI_EXT_CAP_ID_REBAR);
> + if (!offset)
> + return offset;
> +
> + reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> + nbars = (reg & PCI_REBAR_CTRL_NBAR_MASK) >> PCI_REBAR_CTRL_NBAR_SHIFT;
> +
> + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) {
> + reg = dw_pcie_readl_dbi(pci, offset + PCI_REBAR_CTRL);
> + bar_index = reg & PCI_REBAR_CTRL_BAR_IDX;
> + if (bar_index == bar)
> + return offset;
> + }
> +
> + return 0;
> +}
> +
> +static u32 dw_pcie_ep_bar_size_to_rebar_cap(size_t size)
> +{
> + u32 val;
> +
> + /*
> + * According to PCIe base spec, min size for a resizable BAR is 1 MB,
> + * thus disallow a requested BAR size smaller than 1 MB.
> + * Disallow a requested BAR size larger than 128 TB.
> + */
> + if (size < SZ_1M || (u64)size > (SZ_128G * 1024))
> + return 0;
Could you please move this check to pci_epc_set_bar()?
> +
> + val = ilog2(size);
> + val -= 20;
val = ilog2(size) - ilog2(SZ_1M);
> +
> + /* Sizes in REBAR_CAP start at BIT(4). */
> + return BIT(val + 4);
> +}
This function is also not DWC specific, so this could also be moved to
pci-epc-core.
> +
> +static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no,
> + struct pci_epf_bar *epf_bar)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> + enum pci_barno bar = epf_bar->barno;
> + size_t size = epf_bar->size;
> + int flags = epf_bar->flags;
> + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> + unsigned int rebar_offset;
> + u32 rebar_cap, rebar_ctrl;
> +
> + rebar_offset = dw_pcie_ep_get_rebar_offset(pci, bar);
> + if (!rebar_offset)
> + return -EINVAL;
> +
> + rebar_cap = dw_pcie_ep_bar_size_to_rebar_cap(size);
> + if (!rebar_cap)
> + return -EINVAL;
> +
> + dw_pcie_dbi_ro_wr_en(pci);
> +
> + /*
> + * You should not write a BAR mask for a resizable BAR. The BAR mask
Nit: Avoid using 'you'
> + * is automatically derived by the controller every time the "selected
> + * size" bits are updated, see "Figure 3-26 Resizable BAR Example for
> + * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write
> + * BIT(0) to set the BAR enable bit.
> + */
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0);
> + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> + }
> +
> + /*
> + * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes
> + * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes"
> + * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB.
> + */
> + rebar_ctrl = dw_pcie_readl_dbi(pci, rebar_offset + PCI_REBAR_CTRL);
> + rebar_ctrl &= ~GENMASK(31, 16);
> + dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl);
> +
> + /*
> + * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically
> + * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR
> + * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a.
> + */
> + dw_pcie_writel_dbi(pci, rebar_offset + PCI_REBAR_CAP, rebar_cap);
> +
> + dw_pcie_dbi_ro_wr_dis(pci);
> +
> + return 0;
> +}
> +
> +static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no,
> + struct pci_epf_bar *epf_bar)
> +{
> + struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
> + enum pci_barno bar = epf_bar->barno;
> + size_t size = epf_bar->size;
> + int flags = epf_bar->flags;
> + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar);
> +
> + dw_pcie_dbi_ro_wr_en(pci);
> +
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags);
> +
> + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
> + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1));
> + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0);
> + }
> +
> + dw_pcie_dbi_ro_wr_dis(pci);
> +
> + return 0;
> +}
> +
> +static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep,
> + enum pci_barno bar)
> +{
> + const struct pci_epc_features *epc_features = ep->ops->get_features(ep);
'epc_features' can be NULL.
- Mani
--
மணிவண்ணன் சதாசிவம்
