ira.weiny@ wrote:
> From: Davidlohr Bueso <dave@xxxxxxxxxxxx>
>
> Currently the only CXL features targeted for irq support require their
> message numbers to be within the first 16 entries. The device may
> however support less than 16 entries depending on the support it
> provides.
>
> Attempt to allocate these 16 irq vectors. If the device supports less
> then the PCI infrastructure will allocate that number. Upon successful
> allocation, users can plug in their respective isr at any point
> thereafter.
>
> CXL device events are signaled via interrupts. Each event log may have
> a different interrupt message number. These message numbers are
> reported in the Get Event Interrupt Policy mailbox command.
>
> Add interrupt support for event logs. Interrupts are allocated as
> shared interrupts. Therefore, all or some event logs can share the same
> message number.
>
> In addition all logs are queried on any interrupt in order of the most
> to least severe based on the status register.
>
> Finally place all event configuration logic into cxl_event_config().
> Previously the logic was a simple 'read all' on start up. But
> interrupts must be configured prior to any reads to ensure no events are
> missed. A single event configuration function results in a cleaner over
> all implementation.
>
> Cc: Bjorn Helgaas <helgaas@xxxxxxxxxx>
> Cc: Jonathan Cameron <Jonathan.Cameron@xxxxxxxxxx>
> Co-developed-by: Ira Weiny <ira.weiny@xxxxxxxxx>
> Signed-off-by: Davidlohr Bueso <dave@xxxxxxxxxxxx>
> Reviewed-by: Jonathan Cameron <Jonathan.Cameron@xxxxxxxxxx>
> Signed-off-by: Ira Weiny <ira.weiny@xxxxxxxxx>
> ---
> Changes in v7:
> <no change>
> ---
> drivers/cxl/cxl.h | 4 +
> drivers/cxl/cxlmem.h | 19 +++++
> drivers/cxl/cxlpci.h | 6 ++
> drivers/cxl/pci.c | 221 ++++++++++++++++++++++++++++++++++++++++++++++++---
> 4 files changed, 240 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h
> index 5974d1082210..b3964149c77b 100644
> --- a/drivers/cxl/cxl.h
> +++ b/drivers/cxl/cxl.h
> @@ -168,6 +168,10 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
> CXLDEV_EVENT_STATUS_FAIL | \
> CXLDEV_EVENT_STATUS_FATAL)
>
> +/* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */
> +#define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0)
> +#define CXLDEV_EVENT_INT_MSGNUM_MASK GENMASK(7, 4)
> +
> /* CXL 2.0 8.2.8.4 Mailbox Registers */
> #define CXLDEV_MBOX_CAPS_OFFSET 0x00
> #define CXLDEV_MBOX_CAP_PAYLOAD_SIZE_MASK GENMASK(4, 0)
> diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h
> index 0f272206e294..5bef485cd8d9 100644
> --- a/drivers/cxl/cxlmem.h
> +++ b/drivers/cxl/cxlmem.h
> @@ -194,6 +194,23 @@ struct cxl_endpoint_dvsec_info {
> struct range dvsec_range[2];
> };
>
> +/**
> + * Event Interrupt Policy
> + *
> + * CXL rev 3.0 section 8.2.9.2.4; Table 8-52
> + */
> +enum cxl_event_int_mode {
> + CXL_INT_NONE = 0x00,
> + CXL_INT_MSI_MSIX = 0x01,
> + CXL_INT_FW = 0x02
> +};
> +struct cxl_event_interrupt_policy {
> + u8 info_settings;
> + u8 warn_settings;
> + u8 failure_settings;
> + u8 fatal_settings;
> +} __packed;
> +
> /**
> * struct cxl_event_state - Event log driver state
> *
> @@ -288,6 +305,8 @@ enum cxl_opcode {
> CXL_MBOX_OP_RAW = CXL_MBOX_OP_INVALID,
> CXL_MBOX_OP_GET_EVENT_RECORD = 0x0100,
> CXL_MBOX_OP_CLEAR_EVENT_RECORD = 0x0101,
> + CXL_MBOX_OP_GET_EVT_INT_POLICY = 0x0102,
> + CXL_MBOX_OP_SET_EVT_INT_POLICY = 0x0103,
> CXL_MBOX_OP_GET_FW_INFO = 0x0200,
> CXL_MBOX_OP_ACTIVATE_FW = 0x0202,
> CXL_MBOX_OP_GET_SUPPORTED_LOGS = 0x0400,
> diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h
> index 77dbdb980b12..a8ea04f536ab 100644
> --- a/drivers/cxl/cxlpci.h
> +++ b/drivers/cxl/cxlpci.h
> @@ -53,6 +53,12 @@
> #define CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK GENMASK(15, 8)
> #define CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK GENMASK(31, 16)
>
> +/*
> + * NOTE: Currently all the functions which are enabled for CXL require their
> + * vectors to be in the first 16. Use this as the default max.
> + */
> +#define CXL_PCI_DEFAULT_MAX_VECTORS 16
> +
> /* Register Block Identifier (RBI) */
> enum cxl_regloc_type {
> CXL_REGLOC_RBI_EMPTY = 0,
> diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c
> index 0fd7742ee70c..a4ee073fd16d 100644
> --- a/drivers/cxl/pci.c
> +++ b/drivers/cxl/pci.c
> @@ -447,6 +447,209 @@ static int cxl_mem_alloc_event_buf(struct cxl_dev_state *cxlds)
> return 0;
> }
>
> +static int cxl_alloc_irq_vectors(struct pci_dev *pdev)
> +{
> + int nvecs;
> +
> + /*
> + * Per CXL 3.0 3.1.1 CXL.io Endpoint a function on a CXL device must
> + * not generate INTx messages if that function participates in
> + * CXL.cache or CXL.mem.
> + *
> + * Additionally pci_alloc_irq_vectors() handles calling
> + * pci_free_irq_vectors() automatically despite not being called
> + * pcim_*. See pci_setup_msi_context().
> + */
> + nvecs = pci_alloc_irq_vectors(pdev, 1, CXL_PCI_DEFAULT_MAX_VECTORS,
> + PCI_IRQ_MSIX | PCI_IRQ_MSI);
> + if (nvecs < 1) {
> + dev_dbg(&pdev->dev, "Failed to alloc irq vectors: %d\n", nvecs);
> + return -ENXIO;
> + }
> + return 0;
> +}
> +
> +struct cxl_dev_id {
> + struct cxl_dev_state *cxlds;
> +};
> +
> +static irqreturn_t cxl_event_thread(int irq, void *id)
> +{
> + struct cxl_dev_id *dev_id = id;
> + struct cxl_dev_state *cxlds = dev_id->cxlds;
> + u32 status;
> +
> + do {
> + /*
> + * CXL 3.0 8.2.8.3.1: The lower 32 bits are the status;
> + * ignore the reserved upper 32 bits
> + */
> + status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET);
> + /* Ignore logs unknown to the driver */
> + status &= CXLDEV_EVENT_STATUS_ALL;
> + if (!status)
> + break;
> + cxl_mem_get_event_records(cxlds, status);
> + cond_resched();
> + } while (status);
> +
> + return IRQ_HANDLED;
> +}
> +
> +static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
> +{
> + struct device *dev = cxlds->dev;
> + struct pci_dev *pdev = to_pci_dev(dev);
> + struct cxl_dev_id *dev_id;
> + int irq;
> +
> + if (FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting) != CXL_INT_MSI_MSIX)
> + return -ENXIO;
> +
> + /* dev_id must be globally unique and must contain the cxlds */
> + dev_id = devm_kzalloc(dev, sizeof(*dev_id), GFP_KERNEL);
> + if (!dev_id)
> + return -ENOMEM;
> + dev_id->cxlds = cxlds;
> +
> + irq = pci_irq_vector(pdev,
> + FIELD_GET(CXLDEV_EVENT_INT_MSGNUM_MASK, setting));
> + if (irq < 0)
> + return irq;
> +
> + return devm_request_threaded_irq(dev, irq, NULL, cxl_event_thread,
> + IRQF_SHARED, NULL, dev_id);
> +}
> +
> +static int cxl_event_get_int_policy(struct cxl_dev_state *cxlds,
> + struct cxl_event_interrupt_policy *policy)
> +{
> + struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {
At declaration time no need for the cast, I'll zap it on applying.
Rest of this patch looks good me.
