On Mon, Jan 31, 2022 at 9:03 AM Jonathan Cameron
<Jonathan.Cameron@xxxxxxxxxx> wrote:
>
> On Sun, 23 Jan 2022 16:30:30 -0800
> Dan Williams <dan.j.williams@xxxxxxxxx> wrote:
>
> > The core houses infrastructure for decoder resources. A CXL port's
> > dports are more closely related to decoder infrastructure than topology
> > enumeration. Implement generic PCI based dport enumeration in the core,
> > i.e. arrange for existing root port enumeration from cxl_acpi to share
> > code with switch port enumeration which is just amounts to a small
>
> which just amounts
Yup, thanks.
>
> > difference in a pci_walk_bus() invocation once the appropriate 'struct
> > pci_bus' has been retrieved.
> >
> > This also simplifies assumptions about the state of a cxl_port relative
> > to when its dports are populated. Previously threads racing enumeration
> > and port lookup could find the port in partially initialized state with
> > respect to its dports. Now it can assume that the arrival of decoder
> > objects indicates the dport description is stable.
>
> Possibly worth clarifying if that race caused any known bugs, or
> if you just mean it's removal leads to simplifications
Yeah, that point is a bit confusing because what I am comparing is the
difference between this patch series and earlier versions. Upstream
has not enumerated switch ports up to this point, so the difference
between how cxl_acpi registered dports relative to decoders vs this
new common way is not something upstream ever dealt with.
>
> A few additional comment inline.
>
> Jonathan
>
> >
> > Signed-off-by: Dan Williams <dan.j.williams@xxxxxxxxx>
> > ---
> > drivers/cxl/acpi.c | 71 ++++------------------------
> > drivers/cxl/core/Makefile | 1
> > drivers/cxl/core/pci.c | 104 +++++++++++++++++++++++++++++++++++++++++
> > drivers/cxl/core/port.c | 91 +++++++++++++++++++++---------------
> > drivers/cxl/cxl.h | 16 ++----
> > drivers/cxl/cxlpci.h | 1
> > tools/testing/cxl/Kbuild | 3 +
> > tools/testing/cxl/mock_acpi.c | 78 -------------------------------
> > tools/testing/cxl/test/cxl.c | 67 ++++++++++++++++++--------
> > tools/testing/cxl/test/mock.c | 45 +++++++-----------
> > tools/testing/cxl/test/mock.h | 6 ++
> > 11 files changed, 243 insertions(+), 240 deletions(-)
> > create mode 100644 drivers/cxl/core/pci.c
> >
> > diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c
> > index 3485ae9d3baf..259441245687 100644
> > --- a/drivers/cxl/acpi.c
> > +++ b/drivers/cxl/acpi.c
> > @@ -130,48 +130,6 @@ static int cxl_parse_cfmws(union acpi_subtable_headers *header, void *arg,
> > return 0;
> > }
> >
> > -__mock int match_add_root_ports(struct pci_dev *pdev, void *data)
> > -{
> > - resource_size_t creg = CXL_RESOURCE_NONE;
> > - struct cxl_walk_context *ctx = data;
> > - struct pci_bus *root_bus = ctx->root;
> > - struct cxl_port *port = ctx->port;
> > - int type = pci_pcie_type(pdev);
> > - struct device *dev = ctx->dev;
> > - struct cxl_register_map map;
> > - u32 lnkcap, port_num;
> > - int rc;
> > -
> > - if (pdev->bus != root_bus)
> > - return 0;
> > - if (!pci_is_pcie(pdev))
> > - return 0;
> > - if (type != PCI_EXP_TYPE_ROOT_PORT)
> > - return 0;
> > - if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
> > - &lnkcap) != PCIBIOS_SUCCESSFUL)
> > - return 0;
> > -
> > - /* The driver doesn't rely on component registers for Root Ports yet. */
> > - rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
> > - if (!rc)
> > - dev_info(&pdev->dev, "No component register block found\n");
> > -
> > - creg = cxl_regmap_to_base(pdev, &map);
> > -
> > - port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
> > - rc = cxl_add_dport(port, &pdev->dev, port_num, creg);
> > - if (rc) {
> > - ctx->error = rc;
> > - return rc;
> > - }
> > - ctx->count++;
> > -
> > - dev_dbg(dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev));
> > -
> > - return 0;
> > -}
> > -
> > static struct cxl_dport *find_dport_by_dev(struct cxl_port *port, struct device *dev)
> > {
> > struct cxl_dport *dport;
> > @@ -210,7 +168,6 @@ static int add_host_bridge_uport(struct device *match, void *arg)
> > struct device *host = root_port->dev.parent;
> > struct acpi_device *bridge = to_cxl_host_bridge(host, match);
> > struct acpi_pci_root *pci_root;
> > - struct cxl_walk_context ctx;
> > int single_port_map[1], rc;
> > struct cxl_decoder *cxld;
> > struct cxl_dport *dport;
> > @@ -240,18 +197,10 @@ static int add_host_bridge_uport(struct device *match, void *arg)
> > return PTR_ERR(port);
> > dev_dbg(host, "%s: add: %s\n", dev_name(match), dev_name(&port->dev));
> >
> > - ctx = (struct cxl_walk_context){
> > - .dev = host,
> > - .root = pci_root->bus,
> > - .port = port,
> > - };
> > - pci_walk_bus(pci_root->bus, match_add_root_ports, &ctx);
> > -
> > - if (ctx.count == 0)
> > - return -ENODEV;
> > - if (ctx.error)
> > - return ctx.error;
> > - if (ctx.count > 1)
> > + rc = devm_cxl_port_enumerate_dports(host, port);
> > + if (rc < 0)
> > + return rc;
> > + if (rc > 1)
> > return 0;
> >
> > /* TODO: Scan CHBCR for HDM Decoder resources */
> > @@ -311,9 +260,9 @@ static int cxl_get_chbcr(union acpi_subtable_headers *header, void *arg,
> >
> > static int add_host_bridge_dport(struct device *match, void *arg)
> > {
> > - int rc;
> > acpi_status status;
> > unsigned long long uid;
> > + struct cxl_dport *dport;
> > struct cxl_chbs_context ctx;
> > struct cxl_port *root_port = arg;
> > struct device *host = root_port->dev.parent;
> > @@ -342,13 +291,13 @@ static int add_host_bridge_dport(struct device *match, void *arg)
> > return 0;
> > }
> >
> > - device_lock(&root_port->dev);
> > - rc = cxl_add_dport(root_port, match, uid, ctx.chbcr);
> > - device_unlock(&root_port->dev);
> > - if (rc) {
> > + cxl_device_lock(&root_port->dev);
>
> Ah. This is putting back the cxl_device_lock dropped in previous patch I think...
Correct, rebase error now fixed up.
>
> > + dport = devm_cxl_add_dport(host, root_port, match, uid, ctx.chbcr);
> > + cxl_device_unlock(&root_port->dev);
> > + if (IS_ERR(dport)) {
> > dev_err(host, "failed to add downstream port: %s\n",
> > dev_name(match));
> > - return rc;
> > + return PTR_ERR(dport);
> > }
> > dev_dbg(host, "add dport%llu: %s\n", uid, dev_name(match));
> > return 0;
> > diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile
> > index a90202ac88d2..91057f0ec763 100644
> > --- a/drivers/cxl/core/Makefile
> > +++ b/drivers/cxl/core/Makefile
> > @@ -7,3 +7,4 @@ cxl_core-y += pmem.o
> > cxl_core-y += regs.o
> > cxl_core-y += memdev.o
> > cxl_core-y += mbox.o
> > +cxl_core-y += pci.o
> > diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
> > new file mode 100644
> > index 000000000000..48c9a004ae8e
> > --- /dev/null
> > +++ b/drivers/cxl/core/pci.c
> > @@ -0,0 +1,104 @@
> > +// SPDX-License-Identifier: GPL-2.0-only
> > +/* Copyright(c) 2021 Intel Corporation. All rights reserved. */
> > +#include <linux/device.h>
> > +#include <linux/pci.h>
> > +#include <cxlpci.h>
> > +#include <cxl.h>
> > +#include "core.h"
> > +
> > +/**
> > + * DOC: cxl core pci
> > + *
> > + * Compute Express Link protocols are layered on top of PCIe. CXL core provides
> > + * a set of helpers for CXL interactions which occur via PCIe.
> > + */
> > +
> > +struct cxl_walk_context {
> > + struct pci_bus *bus;
> > + struct device *host;
> > + struct cxl_port *port;
> > + int type;
> > + int error;
> > + int count;
> > +};
> > +
> > +static int match_add_dports(struct pci_dev *pdev, void *data)
> > +{
> > + struct cxl_walk_context *ctx = data;
> > + struct cxl_port *port = ctx->port;
> > + struct device *host = ctx->host;
> > + struct pci_bus *bus = ctx->bus;
> > + int type = pci_pcie_type(pdev);
> > + struct cxl_register_map map;
> > + int match_type = ctx->type;
> > + struct cxl_dport *dport;
> > + u32 lnkcap, port_num;
> > + int rc;
> > +
> > + if (pdev->bus != bus)
> if (pdev->bus != ctx->bus) seems just as clear to me and the local
> variable bus isn't used elsewhere.
>
> > + return 0;
> > + if (!pci_is_pcie(pdev))
> > + return 0;
> > + if (type != match_type)
>
> if (pci_pcie_type(pdev) != ctx->type)
>
> is probably easier to follow than with the local variables.
> (note I've not read the rest of the series yet so this might make
> sense if there are additional changes in here)
No, I think the local variables can go.
>
> > + return 0;
> > + if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP,
> > + &lnkcap) != PCIBIOS_SUCCESSFUL)
>
> We could take this opportunity to just compare with 0 as we do in lots
> of other places.
Sure.
>
> > + return 0;
> > +
> > + rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
> > + if (rc)
> > + dev_dbg(&port->dev, "failed to find component registers\n");
> > +
> > + port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap);
> > + cxl_device_lock(&port->dev);
> > + dport = devm_cxl_add_dport(host, port, &pdev->dev, port_num,
> > + cxl_regmap_to_base(pdev, &map));
> > + cxl_device_unlock(&port->dev);
> > + if (IS_ERR(dport)) {
> > + ctx->error = PTR_ERR(dport);
> > + return PTR_ERR(dport);
> > + }
> > + ctx->count++;
> > +
> > + dev_dbg(&port->dev, "add dport%d: %s\n", port_num, dev_name(&pdev->dev));
> > +
> > + return 0;
> > +}
> > +
> > +/**
> > + * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port
> > + * @host: devm context
> > + * @port: cxl_port whose ->uport is the upstream of dports to be enumerated
> > + *
> > + * Returns a positive number of dports enumerated or a negative error
> > + * code.
> > + */
> > +int devm_cxl_port_enumerate_dports(struct device *host, struct cxl_port *port)
> > +{
> > + struct pci_bus *bus = cxl_port_to_pci_bus(port);
> > + struct cxl_walk_context ctx;
> > + int type;
> > +
> > + if (!bus)
> > + return -ENXIO;
> > +
> > + if (pci_is_root_bus(bus))
> > + type = PCI_EXP_TYPE_ROOT_PORT;
> > + else
> > + type = PCI_EXP_TYPE_DOWNSTREAM;
> > +
> > + ctx = (struct cxl_walk_context) {
> > + .host = host,
> > + .port = port,
> > + .bus = bus,
> > + .type = type,
> > + };
> > + pci_walk_bus(bus, match_add_dports, &ctx);
> > +
> > + if (ctx.count == 0)
> > + return -ENODEV;
> > + if (ctx.error)
> > + return ctx.error;
> > + return ctx.count;
> > +}
> > +EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL);
> > diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
> > index c51a10154e29..777de6d91dde 100644
> > --- a/drivers/cxl/core/port.c
> > +++ b/drivers/cxl/core/port.c
>
> ...
>
> >
> > @@ -529,51 +506,87 @@ static int add_dport(struct cxl_port *port, struct cxl_dport *new)
> > return dup ? -EEXIST : 0;
> > }
> >
> > +static void cxl_dport_remove(void *data)
> > +{
> > + struct cxl_dport *dport = data;
> > + struct cxl_port *port = dport->port;
> > +
> > + cxl_device_lock(&port->dev);
> > + list_del_init(&dport->list);
>
> Why _init?
I think at some point I rebased this from something that would look at
the state of ->list at release time, so I wanted to make sure that
list_empty() returned true, but that got simplified along the way, so
this can become plain list_del().
>
> > + cxl_device_unlock(&port->dev);
> > + put_device(dport->dport);
>
> For this unwinding, could we do the put_device(dport->dport)
> before the rest. I don't think we need to hold the reference
> whilst doing the rest of this unwinding and it would more closely
> 'reverse' the setup order below.
True, and ok.
