On Wed, May 06, 2020 at 05:41:39PM +0200, Niklas Schnelle wrote:
> On s390 PCI Virtual Functions (VFs) are scanned by firmware and are made
> available to Linux via the hot-plug interface. As such the common code
> path of doing the scan directly using the parent Physical Function (PF)
> is not used and fenced off with the no_vf_scan attribute.
>
> Even if the partition created the VFs itself e.g. using the sriov_numvfs
> attribute of a PF, the PF/VF links thus need to be established after the
> fact. To do this when a VF is plugged we scan through all functions on
> the same zbus and test whether they are the parent PF in which case we
> establish the necessary links.
>
> With these links established there is now no more need to fence off
> pci_iov_remove_virtfn() for pdev->no_vf_scan as the common code now
> works fine.
>
> Signed-off-by: Niklas Schnelle <schnelle@xxxxxxxxxxxxx>
Acked-by: Bjorn Helgaas <bhelgaas@xxxxxxxxxx>
I assume these will go via the s390 tree.
> ---
> arch/s390/include/asm/pci.h | 3 +-
> arch/s390/include/asm/pci_clp.h | 3 +-
> arch/s390/pci/pci_bus.c | 69 ++++++++++++++++++++++++++++++++-
> arch/s390/pci/pci_clp.c | 1 +
> drivers/pci/iov.c | 3 --
> 5 files changed, 73 insertions(+), 6 deletions(-)
>
> diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
> index c1558cf071b8..99b92c3e46b0 100644
> --- a/arch/s390/include/asm/pci.h
> +++ b/arch/s390/include/asm/pci.h
> @@ -131,7 +131,8 @@ struct zpci_dev {
> u8 port;
> u8 rid_available : 1;
> u8 has_hp_slot : 1;
> - u8 reserved : 6;
> + u8 is_physfn : 1;
> + u8 reserved : 5;
> unsigned int devfn; /* DEVFN part of the RID*/
>
> struct mutex lock;
> diff --git a/arch/s390/include/asm/pci_clp.h b/arch/s390/include/asm/pci_clp.h
> index 896ee41e23e3..1651b5610b0f 100644
> --- a/arch/s390/include/asm/pci_clp.h
> +++ b/arch/s390/include/asm/pci_clp.h
> @@ -95,7 +95,8 @@ struct clp_rsp_query_pci {
> u16 vfn; /* virtual fn number */
> u16 : 3;
> u16 rid_avail : 1;
> - u16 : 2;
> + u16 is_physfn : 1;
> + u16 reserved : 1;
> u16 mio_addr_avail : 1;
> u16 util_str_avail : 1; /* utility string available? */
> u16 pfgid : 8; /* pci function group id */
> diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c
> index 542c6b8f56df..52d79a2f6722 100644
> --- a/arch/s390/pci/pci_bus.c
> +++ b/arch/s390/pci/pci_bus.c
> @@ -126,6 +126,64 @@ static struct zpci_bus *zpci_bus_alloc(int pchid)
> return zbus;
> }
>
> +#ifdef CONFIG_PCI_IOV
> +static int zpci_bus_link_virtfn(struct pci_dev *pdev,
> + struct pci_dev *virtfn, int vfid)
> +{
> + int rc;
> +
> + virtfn->physfn = pci_dev_get(pdev);
> + rc = pci_iov_sysfs_link(pdev, virtfn, vfid);
> + if (rc) {
> + pci_dev_put(pdev);
> + virtfn->physfn = NULL;
> + return rc;
> + }
> + return 0;
> +}
> +
> +static int zpci_bus_setup_virtfn(struct zpci_bus *zbus,
> + struct pci_dev *virtfn, int vfn)
> +{
> + int i, cand_devfn;
> + struct zpci_dev *zdev;
> + struct pci_dev *pdev;
> + int vfid = vfn - 1; /* Linux' vfid's start at 0 vfn at 1*/
> + int rc = 0;
> +
> + virtfn->is_virtfn = 1;
> + virtfn->multifunction = 0;
> + WARN_ON(vfid < 0);
> + /* If the parent PF for the given VF is also configured in the
> + * instance, it must be on the same zbus.
> + * We can then identify the parent PF by checking what
> + * devfn the VF would have if it belonged to that PF using the PF's
> + * stride and offset. Only if this candidate devfn matches the
> + * actual devfn will we link both functions.
> + */
> + for (i = 0; i < ZPCI_FUNCTIONS_PER_BUS; i++) {
> + zdev = zbus->function[i];
> + if (zdev && zdev->is_physfn) {
> + pdev = pci_get_slot(zbus->bus, zdev->devfn);
> + cand_devfn = pci_iov_virtfn_devfn(pdev, vfid);
> + if (cand_devfn == virtfn->devfn) {
> + rc = zpci_bus_link_virtfn(pdev, virtfn, vfid);
> + break;
> + }
> + }
> + }
> + return rc;
> +}
> +#else
> +static inline int zpci_bus_setup_virtfn(struct zpci_bus *zbus,
> + struct pci_dev *virtfn, int vfn)
> +{
> + virtfn->is_virtfn = 1;
> + virtfn->multifunction = 0;
> + return 0;
> +}
> +#endif
> +
> static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
> {
> struct pci_bus *bus;
> @@ -157,11 +215,20 @@ static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
>
> pdev = pci_scan_single_device(bus, zdev->devfn);
> if (pdev) {
> - pdev->multifunction = 1;
> + if (!zdev->is_physfn) {
> + rc = zpci_bus_setup_virtfn(zbus, pdev, zdev->vfn);
> + if (rc)
> + goto failed_with_pdev;
> + }
> pci_bus_add_device(pdev);
> }
>
> return 0;
> +
> +failed_with_pdev:
> + pci_stop_and_remove_bus_device(pdev);
> + pci_dev_put(pdev);
> + return rc;
> }
>
> static void zpci_bus_add_devices(struct zpci_bus *zbus)
> diff --git a/arch/s390/pci/pci_clp.c b/arch/s390/pci/pci_clp.c
> index 9b318824a134..d7bd3c287cf7 100644
> --- a/arch/s390/pci/pci_clp.c
> +++ b/arch/s390/pci/pci_clp.c
> @@ -159,6 +159,7 @@ static int clp_store_query_pci_fn(struct zpci_dev *zdev,
> zdev->uid = response->uid;
> zdev->fmb_length = sizeof(u32) * response->fmb_len;
> zdev->rid_available = response->rid_avail;
> + zdev->is_physfn = response->is_physfn;
> if (!s390_pci_no_rid && zdev->rid_available)
> zdev->devfn = response->rid & ZPCI_RID_MASK_DEVFN;
>
> diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
> index d0ddf5f5484c..d932495b0407 100644
> --- a/drivers/pci/iov.c
> +++ b/drivers/pci/iov.c
> @@ -569,9 +569,6 @@ static void sriov_del_vfs(struct pci_dev *dev)
> struct pci_sriov *iov = dev->sriov;
> int i;
>
> - if (dev->no_vf_scan)
> - return;
> -
> for (i = 0; i < iov->num_VFs; i++)
> pci_iov_remove_virtfn(dev, i);
> }
> --
> 2.17.1
>
