From: Edward Srouji <edwards@xxxxxxxxxx>
A DevX object represents some underlay firmware object, the input
command to create it is some raw data given by the user application
which should match the device specification.
A new class Mlx5DevxObj was added that would present any DevX object
that could be creating using the create_obj DevX API.
The command's output is stored in the object's instance, and the object
can be modified/queried and destroyed.
Reviewed-by: Ido Kalir <idok@xxxxxxxxxx>
Signed-off-by: Edward Srouji <edwards@xxxxxxxxxx>
---
Documentation/pyverbs.md | 34 ++++++++++++
pyverbs/providers/mlx5/libmlx5.pxd | 7 +++
pyverbs/providers/mlx5/mlx5dv.pxd | 6 +++
pyverbs/providers/mlx5/mlx5dv.pyx | 103 ++++++++++++++++++++++++++++++++++++-
4 files changed, 149 insertions(+), 1 deletion(-)
diff --git a/Documentation/pyverbs.md b/Documentation/pyverbs.md
index c7fa761..bcbde05 100644
--- a/Documentation/pyverbs.md
+++ b/Documentation/pyverbs.md
@@ -745,3 +745,37 @@ Below is the output when printing the spec.
Dst mac : de:de:de:00:de:de mask: ff:ff:ff:ff:ff:ff
Ether type : 8451 mask: 65535
Vlan tag : 8961 mask: 65535
+
+
+##### MLX5 DevX Objects
+A DevX object represents some underlay firmware object, the input command to
+create it is some raw data given by the user application which should match the
+device specification.
+Upon successful creation, the output buffer includes the raw data from the device
+according to its specification and is stored in the Mlx5DevxObj instance. This
+data can be used as part of related firmware commands to this object.
+In addition to creation, the user can query/modify and destroy the object.
+
+Although weakrefs and DevX objects closure are added and handled by
+Pyverbs, the users must manually close these objects when finished, and
+should not let them be handled by the GC, or by closing the Mlx5Context directly,
+since there's no guarantee that the DevX objects are closed in the correct order,
+because Mlx5DevxObj is a general class that can be any of the device's available
+objects.
+But Pyverbs does guarantee to close DevX UARs and UMEMs in order, and after
+closing the other DevX objects.
+
+The following code snippet shows how to allocate and destroy a PD object over DevX.
+```python
+from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, Mlx5DevxObj
+import pyverbs.providers.mlx5.mlx5_enums as dve
+import struct
+
+attr = Mlx5DVContextAttr(dve.MLX5DV_CONTEXT_FLAGS_DEVX)
+ctx = Mlx5Context(attr, 'rocep8s0f0')
+MLX5_CMD_OP_ALLOC_PD = 0x800
+MLX5_CMD_OP_ALLOC_PD_OUTLEN = 0x10
+cmd_in = struct.pack('!H14s', MLX5_CMD_OP_ALLOC_PD, bytes(0))
+pd = Mlx5DevxObj(ctx, cmd_in, MLX5_CMD_OP_ALLOC_PD_OUTLEN)
+pd.close()
+```
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index ba2c6ec..34691a9 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -319,6 +319,13 @@ cdef extern from 'infiniband/mlx5dv.h':
mlx5dv_devx_umem_in *umem_in)
int mlx5dv_devx_umem_dereg(mlx5dv_devx_umem *umem)
int mlx5dv_devx_query_eqn(v.ibv_context *context, uint32_t vector, uint32_t *eqn)
+ mlx5dv_devx_obj *mlx5dv_devx_obj_create(v.ibv_context *context, const void *_in,
+ size_t inlen, void *out, size_t outlen)
+ int mlx5dv_devx_obj_query(mlx5dv_devx_obj *obj, const void *in_,
+ size_t inlen, void *out, size_t outlen)
+ int mlx5dv_devx_obj_modify(mlx5dv_devx_obj *obj, const void *in_,
+ size_t inlen, void *out, size_t outlen)
+ int mlx5dv_devx_obj_destroy(mlx5dv_devx_obj *obj)
# Mkey setters
void mlx5dv_wr_mkey_configure(mlx5dv_qp_ex *mqp, mlx5dv_mkey *mkey,
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index 154a117..2b758fe 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -12,6 +12,7 @@ from pyverbs.cq cimport CQEX
cdef class Mlx5Context(Context):
cdef object devx_umems
+ cdef object devx_objs
cdef add_ref(self, obj)
cpdef close(self)
@@ -77,3 +78,8 @@ cdef class Mlx5UMEM(PyverbsCM):
cdef Context context
cdef void *addr
cdef object is_user_addr
+
+cdef class Mlx5DevxObj(PyverbsCM):
+ cdef dv.mlx5dv_devx_obj *obj
+ cdef Context context
+ cdef object out_view
diff --git a/pyverbs/providers/mlx5/mlx5dv.pyx b/pyverbs/providers/mlx5/mlx5dv.pyx
index 2c47cb6..ab0bd4a 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pyx
+++ b/pyverbs/providers/mlx5/mlx5dv.pyx
@@ -140,6 +140,104 @@ cdef class Mlx5DVContextAttr(PyverbsObject):
self.attr.comp_mask = val
+cdef class Mlx5DevxObj(PyverbsCM):
+ """
+ Represents mlx5dv_devx_obj C struct.
+ """
+ def __init__(self, Context context, in_, outlen):
+ """
+ Creates a DevX object.
+ If the object was successfully created, the command's output would be
+ stored as a memoryview in self.out_view.
+ :param in_: Bytes of the obj_create command's input data provided in a
+ device specification format.
+ (Stream of bytes or __bytes__ is implemented)
+ :param outlen: Expected output length in bytes
+ """
+ super().__init__()
+ in_bytes = bytes(in_)
+ cdef char *in_mailbox = _prepare_devx_inbox(in_bytes)
+ cdef char *out_mailbox = _prepare_devx_outbox(outlen)
+ self.obj = dv.mlx5dv_devx_obj_create(context.context, in_mailbox,
+ len(in_bytes), out_mailbox, outlen)
+ try:
+ if self.obj == NULL:
+ raise PyverbsRDMAErrno('Failed to create DevX object')
+ self.out_view = memoryview(out_mailbox[:outlen])
+ status = hex(self.out_view[0])
+ syndrome = self.out_view[4:8].hex()
+ if status != hex(0):
+ raise PyverbsRDMAError('Failed to create DevX object with status'
+ f'({status}) and syndrome (0x{syndrome})')
+ finally:
+ free(in_mailbox)
+ free(out_mailbox)
+ self.context = context
+ self.context.add_ref(self)
+
+ def query(self, in_, outlen):
+ """
+ Queries the DevX object.
+ :param in_: Bytes of the obj_query command's input data provided in a
+ device specification format.
+ (Stream of bytes or __bytes__ is implemented)
+ :param outlen: Expected output length in bytes
+ :return: Bytes of the command's output
+ """
+ in_bytes = bytes(in_)
+ cdef char *in_mailbox = _prepare_devx_inbox(in_bytes)
+ cdef char *out_mailbox = _prepare_devx_outbox(outlen)
+ rc = dv.mlx5dv_devx_obj_query(self.obj, in_mailbox, len(in_bytes),
+ out_mailbox, outlen)
+ try:
+ if rc:
+ raise PyverbsRDMAError('Failed to query DevX object', rc)
+ out = <bytes>out_mailbox[:outlen]
+ finally:
+ free(in_mailbox)
+ free(out_mailbox)
+ return out
+
+ def modify(self, in_, outlen):
+ """
+ Modifies the DevX object.
+ :param in_: Bytes of the obj_modify command's input data provided in a
+ device specification format.
+ (Stream of bytes or __bytes__ is implemented)
+ :param outlen: Expected output length in bytes
+ :return: Bytes of the command's output
+ """
+ in_bytes = bytes(in_)
+ cdef char *in_mailbox = _prepare_devx_inbox(in_bytes)
+ cdef char *out_mailbox = _prepare_devx_outbox(outlen)
+ rc = dv.mlx5dv_devx_obj_modify(self.obj, in_mailbox, len(in_bytes),
+ out_mailbox, outlen)
+ try:
+ if rc:
+ raise PyverbsRDMAError('Failed to modify DevX object', rc)
+ out = <bytes>out_mailbox[:outlen]
+ finally:
+ free(in_mailbox)
+ free(out_mailbox)
+ return out
+
+ @property
+ def out_view(self):
+ return self.out_view
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ if self.obj != NULL:
+ self.logger.debug('Closing Mlx5DvexObj')
+ rc = dv.mlx5dv_devx_obj_destroy(self.obj)
+ if rc:
+ raise PyverbsRDMAError('Failed to destroy a DevX object', rc)
+ self.obj = NULL
+ self.context = None
+
+
cdef class Mlx5Context(Context):
"""
Represent mlx5 context, which extends Context.
@@ -159,6 +257,7 @@ cdef class Mlx5Context(Context):
raise PyverbsRDMAErrno('Failed to open mlx5 context on {dev}'
.format(dev=self.name))
self.devx_umems = weakref.WeakSet()
+ self.devx_objs = weakref.WeakSet()
def query_mlx5_device(self, comp_mask=-1):
"""
@@ -280,6 +379,8 @@ cdef class Mlx5Context(Context):
except PyverbsError:
if isinstance(obj, Mlx5UMEM):
self.devx_umems.add(obj)
+ elif isinstance(obj, Mlx5DevxObj):
+ self.devx_objs.add(obj)
else:
raise PyverbsError('Unrecognized object type')
@@ -288,7 +389,7 @@ cdef class Mlx5Context(Context):
cpdef close(self):
if self.context != NULL:
- close_weakrefs([self.pps, self.devx_umems])
+ close_weakrefs([self.pps, self.devx_objs, self.devx_umems])
super(Mlx5Context, self).close()
--
1.8.3.1
