Add the needed declarations and structs to allow users to create a QP
via the mlx5dv interface.
Creation process is similar to that of a legacy QP but the user shall
use an Mlx5Context and provide an Mlx5DVQPInitAttr object as well as
a QPInitAttrEx object.
Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx>
---
pyverbs/providers/mlx5/libmlx5.pxd | 15 ++
pyverbs/providers/mlx5/mlx5dv.pxd | 11 ++
pyverbs/providers/mlx5/mlx5dv.pyx | 192 +++++++++++++++++++++++-
pyverbs/providers/mlx5/mlx5dv_enums.pxd | 21 +++
4 files changed, 236 insertions(+), 3 deletions(-)
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index aaf75620a9bb..2c5be241a3ec 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
+include 'mlx5dv_enums.pxd'
+
from libcpp cimport bool
cimport pyverbs.libibverbs as v
@@ -40,7 +42,20 @@ cdef extern from 'infiniband/mlx5dv.h':
unsigned int flow_action_flags
unsigned int dc_odp_caps
+ cdef struct mlx5dv_dc_init_attr:
+ mlx5dv_dc_type dc_type
+ unsigned long dct_access_key
+
+ cdef struct mlx5dv_qp_init_attr:
+ unsigned long comp_mask
+ unsigned int create_flags
+ mlx5dv_dc_init_attr dc_init_attr
+ unsigned long send_ops_flags
+
bool mlx5dv_is_supported(v.ibv_device *device)
v.ibv_context* mlx5dv_open_device(v.ibv_device *device,
mlx5dv_context_attr *attr)
int mlx5dv_query_device(v.ibv_context *ctx, mlx5dv_context *attrs_out)
+ v.ibv_qp *mlx5dv_create_qp(v.ibv_context *context,
+ v.ibv_qp_init_attr_ex *qp_attr,
+ mlx5dv_qp_init_attr *mlx5_qp_attr)
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index d9fea82af894..918065012d30 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -6,6 +6,8 @@
cimport pyverbs.providers.mlx5.libmlx5 as dv
from pyverbs.base cimport PyverbsObject
from pyverbs.device cimport Context
+from pyverbs.qp cimport QP
+
cdef class Mlx5Context(Context):
pass
@@ -15,3 +17,12 @@ cdef class Mlx5DVContextAttr(PyverbsObject):
cdef class Mlx5DVContext(PyverbsObject):
cdef dv.mlx5dv_context dv
+
+cdef class Mlx5DVDCInitAttr(PyverbsObject):
+ cdef dv.mlx5dv_dc_init_attr attr
+
+cdef class Mlx5DVQPInitAttr(PyverbsObject):
+ cdef dv.mlx5dv_qp_init_attr attr
+
+cdef class Mlx5QP(QP):
+ cdef object dc_type
diff --git a/pyverbs/providers/mlx5/mlx5dv.pyx b/pyverbs/providers/mlx5/mlx5dv.pyx
index dadc9cdcceee..b4a971a5b935 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pyx
+++ b/pyverbs/providers/mlx5/mlx5dv.pyx
@@ -6,7 +6,8 @@ cimport pyverbs.providers.mlx5.mlx5dv_enums as dve
cimport pyverbs.providers.mlx5.libmlx5 as dv
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
-
+from pyverbs.qp cimport QPInitAttrEx
+from pyverbs.pd cimport PD
cdef class Mlx5DVContextAttr(PyverbsObject):
"""
@@ -179,8 +180,159 @@ cdef class Mlx5DVContext(PyverbsObject):
print_format.format('DC ODP caps', self.dv.dc_odp_caps)
+cdef class Mlx5DVDCInitAttr(PyverbsObject):
+ """
+ Represents mlx5dv_dc_init_attr struct, which defines initial attributes
+ for DC QP creation.
+ """
+ def __cinit__(self, dc_type=dve.MLX5DV_DCTYPE_DCI, dct_access_key=0):
+ """
+ Initializes an Mlx5DVDCInitAttr object with the given DC type and DCT
+ access key.
+ :param dc_type: Which DC QP to create (DCI/DCT).
+ :param dct_access_key: Access key to be used by the DCT
+ :return: An initializes object
+ """
+ self.attr.dc_type = dc_type
+ self.attr.dct_access_key = dct_access_key
+
+ def __str__(self):
+ print_format = '{:20}: {:<20}\n'
+ return print_format.format('DC type', dc_type_to_str(self.attr.dc_type)) +\
+ print_format.format('DCT access key', self.attr.dct_access_key)
+
+ @property
+ def dc_type(self):
+ return self.attr.dc_type
+ @dc_type.setter
+ def dc_type(self, val):
+ self.attr.dc_type = val
+
+ @property
+ def dct_access_key(self):
+ return self.attr.dct_access_key
+ @dct_access_key.setter
+ def dct_access_key(self, val):
+ self.attr.dct_access_key = val
+
+
+cdef class Mlx5DVQPInitAttr(PyverbsObject):
+ """
+ Represents mlx5dv_qp_init_attr struct, initial attributes used for mlx5 QP
+ creation.
+ """
+ def __cinit__(self, comp_mask=0, create_flags=0,
+ Mlx5DVDCInitAttr dc_init_attr=None, send_ops_flags=0):
+ """
+ Initializes an Mlx5DVQPInitAttr object with the given user data.
+ :param comp_mask: A bitmask specifying which fields are valid
+ :param create_flags: A bitwise OR of mlx5dv_qp_create_flags
+ :param dc_init_attr: Mlx5DVDCInitAttr object
+ :param send_ops_flags: A bitwise OR of mlx5dv_qp_create_send_ops_flags
+ :return: An initialized Mlx5DVQPInitAttr object
+ """
+ self.attr.comp_mask = comp_mask
+ self.attr.create_flags = create_flags
+ self.attr.send_ops_flags = send_ops_flags
+ if dc_init_attr is not None:
+ self.attr.dc_init_attr.dc_type = dc_init_attr.dc_type
+ self.attr.dc_init_attr.dct_access_key = dc_init_attr.dct_access_key
+
+ def __str__(self):
+ print_format = '{:20}: {:<20}\n'
+ return print_format.format('Comp mask',
+ qp_comp_mask_to_str(self.attr.comp_mask)) +\
+ print_format.format('Create flags',
+ qp_create_flags_to_str(self.attr.create_flags)) +\
+ 'DC init attr:\n' +\
+ print_format.format(' DC type',
+ dc_type_to_str(self.attr.dc_init_attr.dc_type)) +\
+ print_format.format(' DCT access key',
+ self.attr.dc_init_attr.dct_access_key) +\
+ print_format.format('Send ops flags',
+ send_ops_flags_to_str(self.attr.send_ops_flags))
+
+ @property
+ def comp_mask(self):
+ return self.attr.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.attr.comp_mask = val
+
+ @property
+ def create_flags(self):
+ return self.attr.create_flags
+ @create_flags.setter
+ def create_flags(self, val):
+ self.attr.create_flags = val
+
+ @property
+ def send_ops_flags(self):
+ return self.attr.send_ops_flags
+ @send_ops_flags.setter
+ def send_ops_flags(self, val):
+ self.attr.send_ops_flags = val
+
+ @property
+ def dc_type(self):
+ return self.attr.dc_init_attr.dc_type
+ @dc_type.setter
+ def dc_type(self, val):
+ self.attr.dc_init_attr.dc_type = val
+
+ @property
+ def dct_access_key(self):
+ return self.attr.dc_init_attr.dct_access_key
+ @dct_access_key.setter
+ def dct_access_key(self, val):
+ self.attr.dc_init_attr.dct_access_key = val
+
+
+cdef class Mlx5QP(QP):
+ def __cinit__(self, Mlx5Context context, QPInitAttrEx init_attr,
+ Mlx5DVQPInitAttr dv_init_attr):
+ """
+ Initializes an mlx5 QP according to the user-provided data.
+ :param context: mlx5 Context object
+ :param init_attr: QPInitAttrEx object
+ :param dv_init_attr: Mlx5DVQPInitAttr object
+ :return: An initialized Mlx5QP
+ """
+ cdef PD pd
+ self.dc_type = dv_init_attr.dc_type if dv_init_attr else 0
+ if init_attr.pd is not None:
+ pd = <PD>init_attr.pd
+ pd.add_ref(self)
+ self.qp = \
+ dv.mlx5dv_create_qp(context.context,
+ &init_attr.attr,
+ &dv_init_attr.attr if dv_init_attr is not None
+ else NULL)
+ if self.qp == NULL:
+ raise PyverbsRDMAErrno('Failed to create MLX5 QP.\nQPInitAttrEx '
+ 'attributes:\n{}\nMLX5DVQPInitAttr:\n{}'.
+ format(init_attr, dv_init_attr))
+
+ def _get_comp_mask(self, dst):
+ masks = {dve.MLX5DV_DCTYPE_DCT: {'INIT': e.IBV_QP_PKEY_INDEX |
+ e.IBV_QP_PORT | e.IBV_QP_ACCESS_FLAGS,
+ 'RTR': e.IBV_QP_AV |\
+ e.IBV_QP_PATH_MTU |\
+ e.IBV_QP_MIN_RNR_TIMER},
+ dve.MLX5DV_DCTYPE_DCI: {'INIT': e.IBV_QP_PKEY_INDEX |\
+ e.IBV_QP_PORT,
+ 'RTR': e.IBV_QP_PATH_MTU,
+ 'RTS': e.IBV_QP_TIMEOUT |\
+ e.IBV_QP_RETRY_CNT |\
+ e.IBV_QP_RNR_RETRY | e.IBV_QP_SQ_PSN |\
+ e.IBV_QP_MAX_QP_RD_ATOMIC}}
+ if self.dc_type == 0:
+ return super()._get_comp_mask(dst)
+ return masks[self.dc_type][dst] | e.IBV_QP_STATE
+
+
def qpts_to_str(qp_types):
- numberic_types = qp_types
+ numeric_types = qp_types
qpts_str = ''
qpts = {e.IBV_QPT_RC: 'RC', e.IBV_QPT_UC: 'UC', e.IBV_QPT_UD: 'UD',
e.IBV_QPT_RAW_PACKET: 'Raw Packet', e.IBV_QPT_XRC_SEND: 'XRC Send',
@@ -191,7 +343,7 @@ def qpts_to_str(qp_types):
qp_types -= t
if qp_types == 0:
break
- return qpts_str[:-2] + ' ({})'.format(numberic_types)
+ return qpts_str[:-2] + ' ({})'.format(numeric_types)
def bitmask_to_str(bits, values):
@@ -251,3 +403,37 @@ def tunnel_offloads_to_str(tun):
dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP:\
'Ctrl word + MPLS over UDP'}
return bitmask_to_str(tun, l)
+
+
+def dc_type_to_str(dctype):
+ l = {dve.MLX5DV_DCTYPE_DCT: 'DCT', dve.MLX5DV_DCTYPE_DCI: 'DCI'}
+ try:
+ return l[dctype]
+ except KeyError:
+ return 'Unknown DC type ({dc})'.format(dc=dctype)
+
+
+def qp_comp_mask_to_str(flags):
+ l = {dve.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS: 'Create flags',
+ dve.MLX5DV_QP_INIT_ATTR_MASK_DC: 'DC',
+ dve.MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS: 'Send ops flags'}
+ return bitmask_to_str(flags, l)
+
+
+def qp_create_flags_to_str(flags):
+ l = {dve.MLX5DV_QP_CREATE_TUNNEL_OFFLOADS: 'Tunnel offloads',
+ dve.MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC:
+ 'Allow UC self loopback',
+ dve.MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_MC:
+ 'Allow MC self loopback',
+ dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE: 'Disable scatter to CQE',
+ dve.MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE: 'Allow scatter to CQE',
+ dve.MLX5DV_QP_CREATE_PACKET_BASED_CREDIT_MODE:
+ 'Packet based credit mode'}
+ return bitmask_to_str(flags, l)
+
+
+def send_ops_flags_to_str(flags):
+ l = {dve.MLX5DV_QP_EX_WITH_MR_INTERLEAVED: 'With MR interleaved',
+ dve.MLX5DV_QP_EX_WITH_MR_LIST: 'With MR list'}
+ return bitmask_to_str(flags, l)
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
index 038a49111a3b..3f7f591e23a5 100644
--- a/pyverbs/providers/mlx5/mlx5dv_enums.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -45,3 +45,24 @@ cdef extern from 'infiniband/mlx5dv.h':
MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING = 1 << 2
MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD = 1 << 3
MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN = 1 << 4
+
+ cpdef enum mlx5dv_qp_init_attr_mask:
+ MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0
+ MLX5DV_QP_INIT_ATTR_MASK_DC = 1 << 1
+ MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS = 1 << 2
+
+ cpdef enum mlx5dv_qp_create_flags:
+ MLX5DV_QP_CREATE_TUNNEL_OFFLOADS = 1 << 0
+ MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_UC = 1 << 1
+ MLX5DV_QP_CREATE_TIR_ALLOW_SELF_LOOPBACK_MC = 1 << 2
+ MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE = 1 << 3
+ MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE = 1 << 4
+ MLX5DV_QP_CREATE_PACKET_BASED_CREDIT_MODE = 1 << 5
+
+ cpdef enum mlx5dv_dc_type:
+ MLX5DV_DCTYPE_DCT = 1
+ MLX5DV_DCTYPE_DCI = 2
+
+ cpdef enum mlx5dv_qp_create_send_ops_flags:
+ MLX5DV_QP_EX_WITH_MR_INTERLEAVED = 1 << 0
+ MLX5DV_QP_EX_WITH_MR_LIST = 1 << 1
--
2.21.0
