On Mon, 29 Apr 2019 13:56:11 -0500
Bjorn Helgaas <helgaas@xxxxxxxxxx> wrote:
> From: Bjorn Helgaas <bhelgaas@xxxxxxxxxx>
>
> This reverts commit e8303bb7a75c113388badcc49b2a84b4121c1b3e.
>
> e8303bb7a75c added logging whenever a link changed speed or width to a
> state that is considered degraded. Unfortunately, it cannot differentiate
> signal integrity-related link changes from those intentionally initiated by
> an endpoint driver, including drivers that may live in userspace or VMs
> when making use of vfio-pci. Some GPU drivers actively manage the link
> state to save power, which generates a stream of messages like this:
>
> vfio-pci 0000:07:00.0: 32.000 Gb/s available PCIe bandwidth, limited by 2.5 GT/s x16 link at 0000:00:02.0 (capable of 64.000 Gb/s with 5 GT/s x16 link)
>
> We really *do* want to be alerted when the link bandwidth is reduced
> because of hardware failures, but degradation by intentional link state
> management is probably far more common, so the signal-to-noise ratio is
> currently low.
>
> Until we figure out a way to identify the real problems or silence the
> intentional situations, revert the following commits, which include the
> initial implementation (e8303bb7a75c) and subsequent fixes:
>
> e8303bb7a75c ("PCI/LINK: Report degraded links via link bandwidth notification")
> 3e82a7f9031f ("PCI/LINK: Supply IRQ handler so level-triggered IRQs are acked")
> 55397ce8df48 ("PCI/LINK: Clear bandwidth notification interrupt before enabling it")
> 0fa635aec9ab ("PCI/LINK: Deduplicate bandwidth reports for multi-function devices")
>
> Link: https://lore.kernel.org/lkml/155597243666.19387.1205950870601742062.stgit@xxxxxxxxxx
> Link: https://lore.kernel.org/lkml/155605909349.3575.13433421148215616375.stgit@xxxxxxxxxx
> Signed-off-by: Bjorn Helgaas <bhelgaas@xxxxxxxxxx>
> CC: Alexandru Gagniuc <mr.nuke.me@xxxxxxxxx>
> CC: Lukas Wunner <lukas@xxxxxxxxx>
> CC: Alex Williamson <alex.williamson@xxxxxxxxxx>
Unfortunate, but a good choice for 5.1 and hopefully it can be brought
up to par for 5.2. Some sort of more structured reporting channel,
possibly also with admin &/ driver hooks would be good.
Reviewed-by: Alex Williamson <alex.williamson@xxxxxxxxxx>
Thanks,
Alex
> ---
> drivers/pci/pci.h | 1 -
> drivers/pci/pcie/Makefile | 1 -
> drivers/pci/pcie/bw_notification.c | 121 -----------------------------
> drivers/pci/pcie/portdrv.h | 6 +-
> drivers/pci/pcie/portdrv_core.c | 17 ++--
> drivers/pci/pcie/portdrv_pci.c | 1 -
> drivers/pci/probe.c | 2 +-
> 7 files changed, 7 insertions(+), 142 deletions(-)
> delete mode 100644 drivers/pci/pcie/bw_notification.c
>
> diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
> index d994839a3e24..224d88634115 100644
> --- a/drivers/pci/pci.h
> +++ b/drivers/pci/pci.h
> @@ -273,7 +273,6 @@ enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev);
> u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
> enum pcie_link_width *width);
> void __pcie_print_link_status(struct pci_dev *dev, bool verbose);
> -void pcie_report_downtraining(struct pci_dev *dev);
>
> /* Single Root I/O Virtualization */
> struct pci_sriov {
> diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile
> index f1d7bc1e5efa..ab514083d5d4 100644
> --- a/drivers/pci/pcie/Makefile
> +++ b/drivers/pci/pcie/Makefile
> @@ -3,7 +3,6 @@
> # Makefile for PCI Express features and port driver
>
> pcieportdrv-y := portdrv_core.o portdrv_pci.o err.o
> -pcieportdrv-y += bw_notification.o
>
> obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
>
> diff --git a/drivers/pci/pcie/bw_notification.c b/drivers/pci/pcie/bw_notification.c
> deleted file mode 100644
> index 4fa9e3523ee1..000000000000
> --- a/drivers/pci/pcie/bw_notification.c
> +++ /dev/null
> @@ -1,121 +0,0 @@
> -// SPDX-License-Identifier: GPL-2.0+
> -/*
> - * PCI Express Link Bandwidth Notification services driver
> - * Author: Alexandru Gagniuc <mr.nuke.me@xxxxxxxxx>
> - *
> - * Copyright (C) 2019, Dell Inc
> - *
> - * The PCIe Link Bandwidth Notification provides a way to notify the
> - * operating system when the link width or data rate changes. This
> - * capability is required for all root ports and downstream ports
> - * supporting links wider than x1 and/or multiple link speeds.
> - *
> - * This service port driver hooks into the bandwidth notification interrupt
> - * and warns when links become degraded in operation.
> - */
> -
> -#include "../pci.h"
> -#include "portdrv.h"
> -
> -static bool pcie_link_bandwidth_notification_supported(struct pci_dev *dev)
> -{
> - int ret;
> - u32 lnk_cap;
> -
> - ret = pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnk_cap);
> - return (ret == PCIBIOS_SUCCESSFUL) && (lnk_cap & PCI_EXP_LNKCAP_LBNC);
> -}
> -
> -static void pcie_enable_link_bandwidth_notification(struct pci_dev *dev)
> -{
> - u16 lnk_ctl;
> -
> - pcie_capability_write_word(dev, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS);
> -
> - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl);
> - lnk_ctl |= PCI_EXP_LNKCTL_LBMIE;
> - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl);
> -}
> -
> -static void pcie_disable_link_bandwidth_notification(struct pci_dev *dev)
> -{
> - u16 lnk_ctl;
> -
> - pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &lnk_ctl);
> - lnk_ctl &= ~PCI_EXP_LNKCTL_LBMIE;
> - pcie_capability_write_word(dev, PCI_EXP_LNKCTL, lnk_ctl);
> -}
> -
> -static irqreturn_t pcie_bw_notification_irq(int irq, void *context)
> -{
> - struct pcie_device *srv = context;
> - struct pci_dev *port = srv->port;
> - u16 link_status, events;
> - int ret;
> -
> - ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status);
> - events = link_status & PCI_EXP_LNKSTA_LBMS;
> -
> - if (ret != PCIBIOS_SUCCESSFUL || !events)
> - return IRQ_NONE;
> -
> - pcie_capability_write_word(port, PCI_EXP_LNKSTA, events);
> - pcie_update_link_speed(port->subordinate, link_status);
> - return IRQ_WAKE_THREAD;
> -}
> -
> -static irqreturn_t pcie_bw_notification_handler(int irq, void *context)
> -{
> - struct pcie_device *srv = context;
> - struct pci_dev *port = srv->port;
> - struct pci_dev *dev;
> -
> - /*
> - * Print status from downstream devices, not this root port or
> - * downstream switch port.
> - */
> - down_read(&pci_bus_sem);
> - list_for_each_entry(dev, &port->subordinate->devices, bus_list)
> - pcie_report_downtraining(dev);
> - up_read(&pci_bus_sem);
> -
> - return IRQ_HANDLED;
> -}
> -
> -static int pcie_bandwidth_notification_probe(struct pcie_device *srv)
> -{
> - int ret;
> -
> - /* Single-width or single-speed ports do not have to support this. */
> - if (!pcie_link_bandwidth_notification_supported(srv->port))
> - return -ENODEV;
> -
> - ret = request_threaded_irq(srv->irq, pcie_bw_notification_irq,
> - pcie_bw_notification_handler,
> - IRQF_SHARED, "PCIe BW notif", srv);
> - if (ret)
> - return ret;
> -
> - pcie_enable_link_bandwidth_notification(srv->port);
> -
> - return 0;
> -}
> -
> -static void pcie_bandwidth_notification_remove(struct pcie_device *srv)
> -{
> - pcie_disable_link_bandwidth_notification(srv->port);
> - free_irq(srv->irq, srv);
> -}
> -
> -static struct pcie_port_service_driver pcie_bandwidth_notification_driver = {
> - .name = "pcie_bw_notification",
> - .port_type = PCIE_ANY_PORT,
> - .service = PCIE_PORT_SERVICE_BWNOTIF,
> - .probe = pcie_bandwidth_notification_probe,
> - .remove = pcie_bandwidth_notification_remove,
> -};
> -
> -int __init pcie_bandwidth_notification_init(void)
> -{
> - return pcie_port_service_register(&pcie_bandwidth_notification_driver);
> -}
> diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h
> index 1d50dc58ac40..fbbf00b0992e 100644
> --- a/drivers/pci/pcie/portdrv.h
> +++ b/drivers/pci/pcie/portdrv.h
> @@ -20,10 +20,8 @@
> #define PCIE_PORT_SERVICE_HP (1 << PCIE_PORT_SERVICE_HP_SHIFT)
> #define PCIE_PORT_SERVICE_DPC_SHIFT 3 /* Downstream Port Containment */
> #define PCIE_PORT_SERVICE_DPC (1 << PCIE_PORT_SERVICE_DPC_SHIFT)
> -#define PCIE_PORT_SERVICE_BWNOTIF_SHIFT 4 /* Bandwidth notification */
> -#define PCIE_PORT_SERVICE_BWNOTIF (1 << PCIE_PORT_SERVICE_BWNOTIF_SHIFT)
>
> -#define PCIE_PORT_DEVICE_MAXSERVICES 5
> +#define PCIE_PORT_DEVICE_MAXSERVICES 4
>
> #ifdef CONFIG_PCIEAER
> int pcie_aer_init(void);
> @@ -49,8 +47,6 @@ int pcie_dpc_init(void);
> static inline int pcie_dpc_init(void) { return 0; }
> #endif
>
> -int pcie_bandwidth_notification_init(void);
> -
> /* Port Type */
> #define PCIE_ANY_PORT (~0)
>
> diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c
> index 7d04f9d087a6..f458ac9cb70c 100644
> --- a/drivers/pci/pcie/portdrv_core.c
> +++ b/drivers/pci/pcie/portdrv_core.c
> @@ -99,7 +99,7 @@ static int pcie_message_numbers(struct pci_dev *dev, int mask,
> */
> static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
> {
> - int nr_entries, nvec, pcie_irq;
> + int nr_entries, nvec;
> u32 pme = 0, aer = 0, dpc = 0;
>
> /* Allocate the maximum possible number of MSI/MSI-X vectors */
> @@ -135,13 +135,10 @@ static int pcie_port_enable_irq_vec(struct pci_dev *dev, int *irqs, int mask)
> return nr_entries;
> }
>
> - /* PME, hotplug and bandwidth notification share an MSI/MSI-X vector */
> - if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP |
> - PCIE_PORT_SERVICE_BWNOTIF)) {
> - pcie_irq = pci_irq_vector(dev, pme);
> - irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pcie_irq;
> - irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pcie_irq;
> - irqs[PCIE_PORT_SERVICE_BWNOTIF_SHIFT] = pcie_irq;
> + /* PME and hotplug share an MSI/MSI-X vector */
> + if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP)) {
> + irqs[PCIE_PORT_SERVICE_PME_SHIFT] = pci_irq_vector(dev, pme);
> + irqs[PCIE_PORT_SERVICE_HP_SHIFT] = pci_irq_vector(dev, pme);
> }
>
> if (mask & PCIE_PORT_SERVICE_AER)
> @@ -253,10 +250,6 @@ static int get_port_device_capability(struct pci_dev *dev)
> pci_aer_available() && services & PCIE_PORT_SERVICE_AER)
> services |= PCIE_PORT_SERVICE_DPC;
>
> - if (pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM ||
> - pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT)
> - services |= PCIE_PORT_SERVICE_BWNOTIF;
> -
> return services;
> }
>
> diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c
> index 0a87091a0800..99d2abe88d0b 100644
> --- a/drivers/pci/pcie/portdrv_pci.c
> +++ b/drivers/pci/pcie/portdrv_pci.c
> @@ -240,7 +240,6 @@ static void __init pcie_init_services(void)
> pcie_pme_init();
> pcie_dpc_init();
> pcie_hp_init();
> - pcie_bandwidth_notification_init();
> }
>
> static int __init pcie_portdrv_init(void)
> diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
> index 7e12d0163863..2ec0df04e0dc 100644
> --- a/drivers/pci/probe.c
> +++ b/drivers/pci/probe.c
> @@ -2388,7 +2388,7 @@ static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
> return dev;
> }
>
> -void pcie_report_downtraining(struct pci_dev *dev)
> +static void pcie_report_downtraining(struct pci_dev *dev)
> {
> if (!pci_is_pcie(dev))
> return;
