On Mon, 7 Feb 2022 19:22:13 +0200
Yishai Hadas <yishaih@xxxxxxxxxx> wrote:
> This patch adds support for vfio_pci driver for mlx5 devices.
>
> It uses vfio_pci_core to register to the VFIO subsystem and then
> implements the mlx5 specific logic in the migration area.
>
> The migration implementation follows the definition from uapi/vfio.h and
> uses the mlx5 VF->PF command channel to achieve it.
>
> This patch implements the suspend/resume flows.
>
> Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxx>
> Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxx>
> Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxx>
> ---
> MAINTAINERS | 6 +
> drivers/vfio/pci/Kconfig | 3 +
> drivers/vfio/pci/Makefile | 2 +
> drivers/vfio/pci/mlx5/Kconfig | 10 +
> drivers/vfio/pci/mlx5/Makefile | 4 +
> drivers/vfio/pci/mlx5/cmd.h | 1 +
> drivers/vfio/pci/mlx5/main.c | 623 +++++++++++++++++++++++++++++++++
> 7 files changed, 649 insertions(+)
> create mode 100644 drivers/vfio/pci/mlx5/Kconfig
> create mode 100644 drivers/vfio/pci/mlx5/Makefile
> create mode 100644 drivers/vfio/pci/mlx5/main.c
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index ea3e6c914384..5c5216f5e43d 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -20260,6 +20260,12 @@ L: kvm@xxxxxxxxxxxxxxx
> S: Maintained
> F: drivers/vfio/platform/
>
> +VFIO MLX5 PCI DRIVER
> +M: Yishai Hadas <yishaih@xxxxxxxxxx>
> +L: kvm@xxxxxxxxxxxxxxx
> +S: Maintained
> +F: drivers/vfio/pci/mlx5/
> +
> VGA_SWITCHEROO
> R: Lukas Wunner <lukas@xxxxxxxxx>
> S: Maintained
> diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig
> index 860424ccda1b..187b9c259944 100644
> --- a/drivers/vfio/pci/Kconfig
> +++ b/drivers/vfio/pci/Kconfig
> @@ -43,4 +43,7 @@ config VFIO_PCI_IGD
>
> To enable Intel IGD assignment through vfio-pci, say Y.
> endif
> +
> +source "drivers/vfio/pci/mlx5/Kconfig"
> +
> endif
> diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile
> index 349d68d242b4..ed9d6f2e0555 100644
> --- a/drivers/vfio/pci/Makefile
> +++ b/drivers/vfio/pci/Makefile
> @@ -7,3 +7,5 @@ obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o
> vfio-pci-y := vfio_pci.o
> vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
> obj-$(CONFIG_VFIO_PCI) += vfio-pci.o
> +
> +obj-$(CONFIG_MLX5_VFIO_PCI) += mlx5/
> diff --git a/drivers/vfio/pci/mlx5/Kconfig b/drivers/vfio/pci/mlx5/Kconfig
> new file mode 100644
> index 000000000000..29ba9c504a75
> --- /dev/null
> +++ b/drivers/vfio/pci/mlx5/Kconfig
> @@ -0,0 +1,10 @@
> +# SPDX-License-Identifier: GPL-2.0-only
> +config MLX5_VFIO_PCI
> + tristate "VFIO support for MLX5 PCI devices"
> + depends on MLX5_CORE
> + depends on VFIO_PCI_CORE
> + help
> + This provides migration support for MLX5 devices using the VFIO
> + framework.
> +
> + If you don't know what to do here, say N.
> diff --git a/drivers/vfio/pci/mlx5/Makefile b/drivers/vfio/pci/mlx5/Makefile
> new file mode 100644
> index 000000000000..689627da7ff5
> --- /dev/null
> +++ b/drivers/vfio/pci/mlx5/Makefile
> @@ -0,0 +1,4 @@
> +# SPDX-License-Identifier: GPL-2.0-only
> +obj-$(CONFIG_MLX5_VFIO_PCI) += mlx5-vfio-pci.o
> +mlx5-vfio-pci-y := main.o cmd.o
> +
> diff --git a/drivers/vfio/pci/mlx5/cmd.h b/drivers/vfio/pci/mlx5/cmd.h
> index 69a1481ed953..1392a11a9cc0 100644
> --- a/drivers/vfio/pci/mlx5/cmd.h
> +++ b/drivers/vfio/pci/mlx5/cmd.h
> @@ -12,6 +12,7 @@
> struct mlx5_vf_migration_file {
> struct file *filp;
> struct mutex lock;
> + bool disabled;
>
> struct sg_append_table table;
> size_t total_length;
> diff --git a/drivers/vfio/pci/mlx5/main.c b/drivers/vfio/pci/mlx5/main.c
> new file mode 100644
> index 000000000000..acd205bcff70
> --- /dev/null
> +++ b/drivers/vfio/pci/mlx5/main.c
> @@ -0,0 +1,623 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved
> + */
> +
> +#include <linux/device.h>
> +#include <linux/eventfd.h>
> +#include <linux/file.h>
> +#include <linux/interrupt.h>
> +#include <linux/iommu.h>
> +#include <linux/module.h>
> +#include <linux/mutex.h>
> +#include <linux/notifier.h>
> +#include <linux/pci.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/types.h>
> +#include <linux/uaccess.h>
> +#include <linux/vfio.h>
> +#include <linux/sched/mm.h>
> +#include <linux/vfio_pci_core.h>
> +#include <linux/anon_inodes.h>
> +
> +#include "cmd.h"
> +
> +/* Arbitrary to prevent userspace from consuming endless memory */
> +#define MAX_MIGRATION_SIZE (512*1024*1024)
> +
> +struct mlx5vf_pci_core_device {
> + struct vfio_pci_core_device core_device;
> + u8 migrate_cap:1;
> + /* protect migration state */
> + struct mutex state_mutex;
> + enum vfio_device_mig_state mig_state;
> + u16 vhca_id;
> + struct mlx5_vf_migration_file *resuming_migf;
> + struct mlx5_vf_migration_file *saving_migf;
> +};
Nit, migrate_cap and vhca_id could minimally be contiguous for better
packing of this struct. Thanks,
Alex
