From: Edward Srouji <edwards@xxxxxxxxxx>
Support opening an mlx5 device and creating a context over VFIO by
adding Mlx5VfioContext class based on Mlx5Context to allow using DevX
APIs.
Reviewed-by: Ido Kalir <idok@xxxxxxxxxx>
Signed-off-by: Edward Srouji <edwards@xxxxxxxxxx>
---
pyverbs/providers/mlx5/CMakeLists.txt | 3 +-
pyverbs/providers/mlx5/libmlx5.pxd | 8 +++
pyverbs/providers/mlx5/mlx5_vfio.pxd | 15 +++++
pyverbs/providers/mlx5/mlx5_vfio.pyx | 116 ++++++++++++++++++++++++++++++++
pyverbs/providers/mlx5/mlx5dv.pxd | 3 +
pyverbs/providers/mlx5/mlx5dv_enums.pxd | 3 +
6 files changed, 147 insertions(+), 1 deletion(-)
create mode 100644 pyverbs/providers/mlx5/mlx5_vfio.pxd
create mode 100644 pyverbs/providers/mlx5/mlx5_vfio.pyx
diff --git a/pyverbs/providers/mlx5/CMakeLists.txt b/pyverbs/providers/mlx5/CMakeLists.txt
index 4763c61..c0b5869 100644
--- a/pyverbs/providers/mlx5/CMakeLists.txt
+++ b/pyverbs/providers/mlx5/CMakeLists.txt
@@ -7,8 +7,9 @@ rdma_cython_module(pyverbs/providers/mlx5 mlx5
dr_matcher.pyx
dr_rule.pyx
dr_table.pyx
- mlx5dv.pyx
mlx5_enums.pyx
+ mlx5_vfio.pyx
+ mlx5dv.pyx
mlx5dv_flow.pyx
mlx5dv_mkey.pyx
mlx5dv_objects.pyx
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index af034ad..e0904f5 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -223,6 +223,11 @@ cdef extern from 'infiniband/mlx5dv.h':
uint64_t pgsz_bitmap
uint64_t comp_mask
+ cdef struct mlx5dv_vfio_context_attr:
+ const char *pci_name
+ uint32_t flags
+ uint64_t comp_mask
+
cdef struct mlx5dv_pd:
uint32_t pdn
uint64_t comp_mask
@@ -372,6 +377,9 @@ cdef extern from 'infiniband/mlx5dv.h':
uint64_t device_timestamp)
int mlx5dv_get_clock_info(v.ibv_context *ctx_in, mlx5dv_clock_info *clock_info)
int mlx5dv_map_ah_to_qp(v.ibv_ah *ah, uint32_t qp_num)
+ v.ibv_device **mlx5dv_get_vfio_device_list(mlx5dv_vfio_context_attr *attr)
+ int mlx5dv_vfio_get_events_fd(v.ibv_context *ibctx)
+ int mlx5dv_vfio_process_events(v.ibv_context *context)
# DevX APIs
mlx5dv_devx_uar *mlx5dv_devx_alloc_uar(v.ibv_context *context, uint32_t flags)
diff --git a/pyverbs/providers/mlx5/mlx5_vfio.pxd b/pyverbs/providers/mlx5/mlx5_vfio.pxd
new file mode 100644
index 0000000..0e9facd
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5_vfio.pxd
@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from pyverbs.providers.mlx5.mlx5dv cimport Mlx5Context
+cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base cimport PyverbsObject
+
+
+cdef class Mlx5VfioContext(Mlx5Context):
+ pass
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+ cdef dv.mlx5dv_vfio_context_attr attr
diff --git a/pyverbs/providers/mlx5/mlx5_vfio.pyx b/pyverbs/providers/mlx5/mlx5_vfio.pyx
new file mode 100644
index 0000000..2978b61
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5_vfio.pyx
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+from cpython.mem cimport PyMem_Malloc, PyMem_Free
+from libc.string cimport strcpy
+import weakref
+
+from pyverbs.pyverbs_error import PyverbsRDMAError
+cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.base cimport close_weakrefs
+from pyverbs.device cimport Context
+cimport pyverbs.libibverbs as v
+
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+ """
+ Mlx5VfioAttr class, represents mlx5dv_vfio_context_attr C struct.
+ """
+ def __init__(self, pci_name, flags=0, comp_mask=0):
+ self.pci_name = pci_name
+ self.attr.flags = flags
+ self.attr.comp_mask = comp_mask
+
+ def __dealloc__(self):
+ if self.attr.pci_name != NULL:
+ PyMem_Free(<void*>self.attr.pci_name)
+ self.attr.pci_name = NULL
+
+ @property
+ def flags(self):
+ return self.attr.flags
+ @flags.setter
+ def flags(self, val):
+ self.attr.flags = val
+
+ @property
+ def comp_mask(self):
+ return self.attr.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.attr.comp_mask = val
+
+ @property
+ def pci_name(self):
+ return self.attr.pci_name[:]
+ @pci_name.setter
+ def pci_name(self, val):
+ if self.attr.pci_name != NULL:
+ PyMem_Free(<void*>self.attr.pci_name)
+ pci_name_bytes = val.encode()
+ self.attr.pci_name = <char*>PyMem_Malloc(len(pci_name_bytes))
+ strcpy(<char*>self.attr.pci_name, pci_name_bytes)
+
+
+cdef class Mlx5VfioContext(Mlx5Context):
+ """
+ Mlx5VfioContext class is used to easily initialize and open a context over
+ a mlx5 vfio device.
+ It is initialized based on the passed mlx5 vfio attributes (Mlx5VfioAttr),
+ by getting the relevant vfio device and opening it (creating a context).
+ """
+ def __init__(self, Mlx5VfioAttr attr):
+ super(Context, self).__init__()
+ cdef v.ibv_device **dev_list
+
+ self.name = attr.pci_name
+ self.pds = weakref.WeakSet()
+ self.devx_umems = weakref.WeakSet()
+ self.devx_objs = weakref.WeakSet()
+ self.uars = weakref.WeakSet()
+
+ dev_list = dv.mlx5dv_get_vfio_device_list(&attr.attr)
+ if dev_list == NULL:
+ raise PyverbsRDMAErrno('Failed to get VFIO device list')
+ self.device = dev_list[0]
+ if self.device == NULL:
+ raise PyverbsRDMAError('Failed to get VFIO device')
+ try:
+ self.context = v.ibv_open_device(self.device)
+ if self.context == NULL:
+ raise PyverbsRDMAErrno('Failed to open mlx5 VFIO device '
+ f'({self.device.name.decode()})')
+ finally:
+ v.ibv_free_device_list(dev_list)
+
+ def get_events_fd(self):
+ """
+ Gets the file descriptor to manage driver events.
+ :return: The file descriptor to be used for managing driver events.
+ """
+ fd = dv.mlx5dv_vfio_get_events_fd(self.context)
+ if fd < 0:
+ raise PyverbsRDMAError('Failed to get VFIO events FD', -fd)
+ return fd
+
+ def process_events(self):
+ """
+ Process events on the vfio device.
+ This method should run from application thread to maintain device events.
+ :return: None
+ """
+ rc = dv.mlx5dv_vfio_process_events(self.context)
+ if rc:
+ raise PyverbsRDMAError('VFIO process events failed', rc)
+
+ cpdef close(self):
+ if self.context != NULL:
+ self.logger.debug('Closing Mlx5VfioContext')
+ close_weakrefs([self.pds, self.devx_objs, self.devx_umems, self.uars])
+ rc = v.ibv_close_device(self.context)
+ if rc != 0:
+ raise PyverbsRDMAErrno(f'Failed to close device {self.name}')
+ self.context = NULL
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index 968cbdb..490c697 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -86,3 +86,6 @@ cdef class Mlx5DevxObj(PyverbsCM):
cdef class Mlx5Cqe64(PyverbsObject):
cdef dv.mlx5_cqe64 *cqe
+
+cdef class Mlx5VfioAttr(PyverbsObject):
+ cdef dv.mlx5dv_vfio_context_attr attr
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
index 60713e8..94599ab 100644
--- a/pyverbs/providers/mlx5/mlx5dv_enums.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -215,6 +215,9 @@ cdef extern from 'infiniband/mlx5dv.h':
MLX5_SEND_WQE_BB
MLX5_SEND_WQE_SHIFT
+ cpdef enum mlx5dv_vfio_context_attr_flags:
+ MLX5DV_VFIO_CTX_FLAGS_INIT_LINK_DOWN
+
cpdef unsigned long long MLX5DV_RES_TYPE_QP
cpdef unsigned long long MLX5DV_RES_TYPE_RWQ
cpdef unsigned long long MLX5DV_RES_TYPE_DBR
--
1.8.3.1
