Add support for CXL.mem Timeout & Isolation interrupts. A CXL root port
under isolation will not complete writes and will return an exception
response (i.e. poison) on reads (CXL 3.0 12.3.2). Therefore, when a
CXL-enabled PCIe root port enters isolation, we assume that the memory
under the port is unreachable and will need to be unmapped.
When an isolation interrupt occurs, the CXL Timeout & Isolation service
driver will mark the port (cxl_dport) as isolated, destroy any CXL memory
regions under the *bridge* (cxl_port), and attempt to unmap the memory
backing the CXL region(s). If the memory was already in use, the mapping
is not guaranteed to succeed.
Signed-off-by: Ben Cheatham <Benjamin.Cheatham@xxxxxxx>
---
drivers/cxl/core/pci.c | 5 +
drivers/cxl/core/port.c | 80 ++++++++++++++++
drivers/cxl/core/region.c | 9 ++
drivers/cxl/cxl.h | 10 ++
drivers/cxl/cxlpci.h | 9 ++
drivers/pci/pcie/cxl_timeout.c | 165 ++++++++++++++++++++++++++++++++-
drivers/pci/pcie/portdrv.h | 1 +
7 files changed, 278 insertions(+), 1 deletion(-)
diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c
index 6c9c8d92f8f7..95b6a5f0d0cc 100644
--- a/drivers/cxl/core/pci.c
+++ b/drivers/cxl/core/pci.c
@@ -64,6 +64,11 @@ static int match_add_dports(struct pci_dev *pdev, void *data)
}
ctx->count++;
+ if (type == PCI_EXP_TYPE_ROOT_PORT && !pcie_cxlt_register_dport(dport))
+ return devm_add_action_or_reset(dport->dport_dev,
+ pcie_cxlt_unregister_dport,
+ dport);
+
return 0;
}
diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c
index e59d9d37aa65..88d114c67596 100644
--- a/drivers/cxl/core/port.c
+++ b/drivers/cxl/core/port.c
@@ -2000,6 +2000,86 @@ int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
}
EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
+/* Checks to see if a dport is above an endpoint */
+static bool cxl_dport_is_parent(struct cxl_dport *parent, struct cxl_ep *ep)
+{
+ struct cxl_dport *ep_dport = ep->dport;
+ struct cxl_port *ep_port = ep_dport->port;
+
+ while (!is_cxl_root(ep_port)) {
+ if (ep_dport == parent)
+ return true;
+
+ ep_dport = ep_port->parent_dport;
+ ep_port = ep_dport->port;
+ }
+
+ return false;
+}
+
+bool cxl_dport_is_in_region(struct cxl_dport *dport,
+ struct cxl_region_ref *rr)
+{
+ struct cxl_region_params *p = &rr->region->params;
+ struct cxl_ep *ep;
+ int i;
+
+ for (i = 0; i < p->nr_targets; i++) {
+ if (!p->targets[i])
+ continue;
+
+ ep = cxl_ep_load(dport->port, cxled_to_memdev(p->targets[i]));
+
+ if (ep && cxl_dport_is_parent(dport, ep))
+ return true;
+ }
+
+ return false;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_dport_is_in_region, CXL);
+
+void cxl_port_kill_regions(struct cxl_port *port)
+{
+ struct cxl_endpoint_decoder *ep_decoder;
+ struct cxl_region_params *p;
+ struct cxl_region_ref *ref;
+ unsigned long index;
+ struct cxl_ep *ep;
+ int i;
+
+ xa_for_each(&port->regions, index, ref) {
+ p = &ref->region->params;
+
+ for (i = 0; i < p->nr_targets; i++) {
+ ep_decoder = p->targets[i];
+ if (!ep_decoder)
+ continue;
+
+ ep = cxl_ep_load(port, cxled_to_memdev(ep_decoder));
+ if (ep)
+ cxl_decoder_kill_region(ep_decoder);
+ }
+ }
+}
+EXPORT_SYMBOL_NS_GPL(cxl_port_kill_regions, CXL);
+
+bool cxl_port_is_isolated(struct cxl_port *port)
+{
+ struct cxl_dport *dport = port->parent_dport;
+
+ while (!is_cxl_root(port) && dport) {
+ if (dport->isolated || !dport->port)
+ return true;
+
+ dport = dport->port->parent_dport;
+ port = dport->port;
+ }
+
+
+ return false;
+}
+EXPORT_SYMBOL_NS_GPL(cxl_port_is_isolated, CXL);
+
/**
* __cxl_driver_register - register a driver for the cxl bus
* @cxl_drv: cxl driver structure to attach
diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c
index 0f05692bfec3..f9aef17db26c 100644
--- a/drivers/cxl/core/region.c
+++ b/drivers/cxl/core/region.c
@@ -1699,6 +1699,12 @@ static int cxl_region_attach(struct cxl_region *cxlr,
return -ENXIO;
}
+ if (cxl_port_is_isolated(ep_port)) {
+ dev_err(&cxlr->dev, "%s:%s endpoint is under a dport in error isolation\n",
+ dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
+ return -EBUSY;
+ }
+
if (cxled->cxld.target_type != cxlr->type) {
dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
@@ -2782,6 +2788,9 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
struct resource *res;
int rc;
+ if (cxl_port_is_isolated(cxlmd->endpoint))
+ return ERR_PTR(-EBUSY);
+
do {
cxlr = __create_region(cxlrd, cxled->mode,
atomic_read(&cxlrd->region_id));
diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
index 3b5645ec95b9..1bee2560446a 100644
--- a/drivers/cxl/cxl.h
+++ b/drivers/cxl/cxl.h
@@ -138,6 +138,10 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
#define CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_MASK GENMASK(3, 0)
#define CXL_TIMEOUT_CONTROL_MEM_TIMEOUT_ENABLE BIT(4)
#define CXL_TIMEOUT_CONTROL_MEM_ISO_ENABLE BIT(16)
+#define CXL_TIMEOUT_CONTROL_MEM_INTR_ENABLE BIT(26)
+#define CXL_TIMEOUT_STATUS_OFFSET 0xC
+#define CXL_TIMEOUT_STATUS_MEM_TIMEOUT BIT(0)
+#define CXL_TIMEOUT_STATUS_MEM_ISO BIT(8)
#define CXL_TIMEOUT_CAPABILITY_LENGTH 0x10
/* CXL 3.0 8.2.4.23.2 CXL Timeout and Isolation Control Register, bits 3:0 */
@@ -700,8 +704,11 @@ struct cxl_dport {
struct access_coordinate sw_coord;
struct access_coordinate hb_coord;
long link_latency;
+ bool isolated;
};
+bool cxl_port_is_isolated(struct cxl_port *port);
+
/**
* struct cxl_ep - track an endpoint's interest in a port
* @ep: device that hosts a generic CXL endpoint (expander or accelerator)
@@ -735,6 +742,9 @@ struct cxl_region_ref {
int nr_targets;
};
+bool cxl_dport_is_in_region(struct cxl_dport *dport, struct cxl_region_ref *ref);
+void cxl_port_kill_regions(struct cxl_port *port);
+
/*
* The platform firmware device hosting the root is also the top of the
* CXL port topology. All other CXL ports have another CXL port as their
diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
index 711b05d9a370..7100e23a1819 100644
--- a/drivers/cxl/cxlpci.h
+++ b/drivers/cxl/cxlpci.h
@@ -106,4 +106,13 @@ void read_cdat_data(struct cxl_port *port);
void cxl_cor_error_detected(struct pci_dev *pdev);
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state);
+
+#ifdef CONFIG_PCIE_CXL_TIMEOUT
+int pcie_cxlt_register_dport(struct cxl_dport *dport);
+void pcie_cxlt_unregister_dport(struct cxl_dport *dport);
+#else
+int pcie_cxlt_register_dport(void *dport) { return -ENXIO; }
+void pcie_cxlt_unregister_dport(void *dport) {}
+#endif
+
#endif /* __CXL_PCI_H__ */
diff --git a/drivers/pci/pcie/cxl_timeout.c b/drivers/pci/pcie/cxl_timeout.c
index 352d9370a999..2193e872b4b7 100644
--- a/drivers/pci/pcie/cxl_timeout.c
+++ b/drivers/pci/pcie/cxl_timeout.c
@@ -22,6 +22,8 @@
#define NUM_CXL_TIMEOUT_RANGES 9
+static u32 num_cxlt_devs;
+
struct cxl_timeout {
struct pcie_device *dev;
void __iomem *regs;
@@ -141,6 +143,141 @@ static struct pcie_cxlt_data *cxlt_create_pdata(struct pcie_device *dev)
return data;
}
+int pcie_cxlt_register_dport(struct cxl_dport *dport)
+{
+ struct device *dev = dport->dport_dev;
+ struct pcie_device *pcie_dev;
+ struct pcie_cxlt_data *pdata;
+ struct pci_dev *pdev;
+
+ if (!dev_is_pci(dev))
+ return -ENXIO;
+
+ pdev = to_pci_dev(dev);
+
+ dev = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_CXLT);
+ if (!dev) {
+ dev_warn(dev,
+ "Device is not registered with cxl_timeout driver.\n");
+ return -ENODEV;
+ }
+
+ pcie_dev = to_pcie_device(dev);
+
+ pdata = get_service_data(pcie_dev);
+ pdata->dport = dport;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(pcie_cxlt_register_dport);
+
+void pcie_cxlt_unregister_dport(struct cxl_dport *dport)
+{
+ struct device *dev = dport->dport_dev;
+ struct pcie_device *pcie_dev;
+ struct pcie_cxlt_data *pdata;
+ struct pci_dev *pdev;
+
+ if (!dev_is_pci(dev))
+ return;
+
+ pdev = to_pci_dev(dev);
+
+ dev = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_CXLT);
+ if (!dev) {
+ dev_dbg(dev,
+ "Device was not registered with cxl_timeout driver.\n");
+ return;
+ }
+
+ pcie_dev = to_pcie_device(dev);
+ pdata = get_service_data(pcie_dev);
+ pdata->dport = NULL;
+}
+EXPORT_SYMBOL_GPL(pcie_cxlt_unregister_dport);
+
+struct cxl_timeout_wq_data {
+ struct work_struct w;
+ struct cxl_dport *dport;
+};
+
+static struct workqueue_struct *cxl_timeout_wq;
+
+static void cxl_timeout_handler(struct work_struct *w)
+{
+ struct cxl_timeout_wq_data *data =
+ container_of(w, struct cxl_timeout_wq_data, w);
+ struct cxl_dport *dport = data->dport;
+ struct cxl_port *port;
+ struct cxl_region_ref *ref;
+ unsigned long index;
+ bool kill_regions;
+
+ if (!dport || !dport->port)
+ return;
+
+ port = dport->port;
+
+ xa_for_each(&port->regions, index, ref)
+ if (cxl_dport_is_in_region(dport, ref))
+ kill_regions = true;
+
+ if (kill_regions)
+ cxl_port_kill_regions(port);
+
+ kfree(data);
+}
+
+irqreturn_t cxl_timeout_thread(int irq, void *data)
+{
+ struct cxl_timeout_wq_data *wq_data;
+ struct cxl_timeout *cxlt = data;
+ struct pcie_device *pcie_dev = cxlt->dev;
+ struct pcie_cxlt_data *pdata;
+ struct cxl_dport *dport;
+ u32 status;
+
+ /*
+ * If the CXL core didn't register a cxl_dport with this PCIe device,
+ * then dport enumeration failed and there's nothing to do CXL-wise.
+ */
+ pdata = get_service_data(pcie_dev);
+ if (!pdata || !pdata->dport)
+ return IRQ_HANDLED;
+
+ dport = pdata->dport;
+
+ status = readl(cxlt->regs + CXL_TIMEOUT_STATUS_OFFSET);
+ if (!(status & CXL_TIMEOUT_STATUS_MEM_ISO
+ || status & CXL_TIMEOUT_STATUS_MEM_TIMEOUT))
+ return IRQ_HANDLED;
+
+ dport->isolated = true;
+
+ wq_data = kzalloc(sizeof(struct cxl_timeout_wq_data), GFP_NOWAIT);
+ if (!wq_data)
+ return IRQ_NONE;
+
+ wq_data->dport = dport;
+
+ INIT_WORK(&wq_data->w, cxl_timeout_handler);
+ queue_work(cxl_timeout_wq, &wq_data->w);
+
+ return IRQ_HANDLED;
+}
+
+static int cxl_enable_interrupts(struct pcie_device *dev,
+ struct cxl_timeout *cxlt)
+{
+ if (!cxlt || !FIELD_GET(CXL_TIMEOUT_CAP_INTR_SUPP, cxlt->cap))
+ return -ENXIO;
+
+ return devm_request_threaded_irq(&dev->device, dev->irq, NULL,
+ cxl_timeout_thread,
+ IRQF_SHARED | IRQF_ONESHOT, "cxltdrv",
+ cxlt);
+}
+
static bool cxl_validate_timeout_range(struct cxl_timeout *cxlt, u8 range)
{
u8 timeout_ranges = FIELD_GET(CXL_TIMEOUT_CAP_MEM_TIMEOUT_MASK,
@@ -405,9 +542,28 @@ static int cxl_timeout_probe(struct pcie_device *dev)
if (rc && !timeout_enabled) {
pci_info(dev->port,
"Failed to enable CXL.mem timeout and isolation.\n");
+ return rc;
}
- return rc;
+ rc = cxl_enable_interrupts(dev, cxlt);
+ if (rc) {
+ pci_info(dev->port,
+ "Failed to enable CXL.mem timeout & isolation interrupts: %d\n",
+ rc);
+ } else {
+ pci_info(port, "enabled with IRQ %d\n", dev->irq);
+ }
+
+ num_cxlt_devs++;
+ return 0;
+}
+
+static void cxl_timeout_remove(struct pcie_device *dev)
+{
+ num_cxlt_devs--;
+
+ if (!num_cxlt_devs)
+ destroy_workqueue(cxl_timeout_wq);
}
static struct pcie_port_service_driver cxltdriver = {
@@ -416,6 +572,7 @@ static struct pcie_port_service_driver cxltdriver = {
.service = PCIE_PORT_SERVICE_CXLT,
.probe = cxl_timeout_probe,
+ .remove = cxl_timeout_remove,
.driver = {
.dev_groups = cxl_timeout_attribute_groups,
},
@@ -423,6 +580,12 @@ static struct pcie_port_service_driver cxltdriver = {
int __init pcie_cxlt_init(void)
{
+ cxl_timeout_wq = alloc_ordered_workqueue("cxl_timeout", 0);
+ if (!cxl_timeout_wq)
+ return -ENOMEM;
+
+ num_cxlt_devs = 0;
+
return pcie_port_service_register(&cxltdriver);
}
diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h
index f89e7366e986..c56c629cf563 100644
--- a/drivers/pci/pcie/portdrv.h
+++ b/drivers/pci/pcie/portdrv.h
@@ -10,6 +10,7 @@
#define _PORTDRV_H_
#include <linux/compiler.h>
+#include <linux/errno.h>
/* Service Type */
#define PCIE_PORT_SERVICE_PME_SHIFT 0 /* Power Management Event */
--
2.34.1
