On Fri, Nov 11, 2022 at 02:58:51PM +0100, Thomas Gleixner wrote:
> MSI-X vectors can be allocated after the initial MSI-X enablement, but this
> needs explicit support of the underlying interrupt domains.
>
> Provide a function to query the ability and functions to allocate/free
> individual vectors post-enable.
>
> The allocation can either request a specific index in the MSI-X table or
> with the index argument MSI_ANY_INDEX it allocates the next free vector.
>
> The return value is a struct msi_map which on success contains both index
> and the Linux interrupt number. In case of failure index is negative and
> the Linux interrupt number is 0.
>
> The allocation function is for a single MSI-X index at a time as that's
> sufficient for the most urgent use case VFIO to get rid of the 'disable
> MSI-X, reallocate, enable-MSI-X' cycle which is prone to lost interrupts
> and redirections to the legacy and obviously unhandled INTx.
>
> Also for the use cases Jason Gunthorpe pointed a single index allocation
> is sufficient.
Maybe a URL or outline the use cases so this means something in a few
years? I haven't followed this discussion, so it doesn't even mean
anything to me now :)
> Signed-off-by: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Acked-by: Bjorn Helgaas <bhelgaas@xxxxxxxxxx>
> ---
> drivers/pci/msi/api.c | 67 ++++++++++++++++++++++++++++++++++++++++++++
> drivers/pci/msi/irqdomain.c | 3 +
> include/linux/pci.h | 6 +++
> 3 files changed, 75 insertions(+), 1 deletion(-)
>
> --- a/drivers/pci/msi/api.c
> +++ b/drivers/pci/msi/api.c
> @@ -113,6 +113,73 @@ int pci_enable_msix_range(struct pci_dev
> EXPORT_SYMBOL(pci_enable_msix_range);
>
> /**
> + * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
> + * MSI-X is supported
> + *
> + * @dev: PCI device to operate on
> + *
> + * Return: True if supported, false otherwise
> + */
> +bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
> +{
> + if (!dev->msix_cap)
> + return false;
> +
> + return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);
> +}
> +EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
> +
> +/**
> + * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
> + * at a given MSI-X vector index or any free vector index
> + *
> + * @dev: PCI device to operate on
> + * @index: Index to allocate. If @index == MSI_ANY_INDEX this allocates
> + * the next free index in the MSI-X table
> + * @affdesc: Optional pointer to an affinity descriptor structure. NULL otherwise
> + *
> + * Return: A struct msi_map
> + *
> + * On success msi_map::index contains the allocated index (>= 0) and
> + * msi_map::virq the allocated Linux interrupt number (> 0).
> + *
> + * On fail msi_map::index contains the error code and msi_map::virq
> + * is set to 0.
> + */
> +struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
> + const struct irq_affinity_desc *affdesc)
> +{
> + struct msi_map map = { .index = -ENOTSUPP };
> +
> + if (!dev->msix_enabled)
> + return map;
> +
> + if (!pci_msix_can_alloc_dyn(dev))
> + return map;
> +
> + return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
> +}
> +EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
> +
> +/**
> + * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain
> + * which was allocated via pci_msix_alloc_irq_at()
> + *
> + * @dev: The PCI device to operate on
> + * @map: A struct msi_map describing the interrupt to free
> + * as returned from the allocation function.
> + */
> +void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
> +{
> + if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
> + return;
> + if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
> + return;
> + msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);
> +}
> +EXPORT_SYMBOL_GPL(pci_msix_free_irq);
> +
> +/**
> * pci_disable_msix() - Disable MSI-X interrupt mode on device
> * @dev: the PCI device to operate on
> *
> --- a/drivers/pci/msi/irqdomain.c
> +++ b/drivers/pci/msi/irqdomain.c
> @@ -225,7 +225,8 @@ static struct msi_domain_template pci_ms
> },
>
> .info = {
> - .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX,
> + .flags = MSI_COMMON_FLAGS | MSI_FLAG_PCI_MSIX |
> + MSI_FLAG_PCI_MSIX_ALLOC_DYN,
> .bus_token = DOMAIN_BUS_PCI_DEVICE_MSIX,
> },
> };
> --- a/include/linux/pci.h
> +++ b/include/linux/pci.h
> @@ -38,6 +38,7 @@
> #include <linux/interrupt.h>
> #include <linux/io.h>
> #include <linux/resource_ext.h>
> +#include <linux/msi_api.h>
> #include <uapi/linux/pci.h>
>
> #include <linux/pci_ids.h>
> @@ -1559,6 +1560,11 @@ int pci_alloc_irq_vectors_affinity(struc
> unsigned int max_vecs, unsigned int flags,
> struct irq_affinity *affd);
>
> +bool pci_msix_can_alloc_dyn(struct pci_dev *dev);
> +struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
> + const struct irq_affinity_desc *affdesc);
> +void pci_msix_free_irq(struct pci_dev *pdev, struct msi_map map);
> +
> void pci_free_irq_vectors(struct pci_dev *dev);
> int pci_irq_vector(struct pci_dev *dev, unsigned int nr);
> const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec);
>
