Allow users to use the mlx5-specific query_device() function via
Mlx5Context's query_mlx5_device method. This device querying exposes
device properties that are not available via legacy query such as
software parsing and CQE compression capabilities.
This method returns an Mlx5DVContext object which supports a
user-friendly print format.
Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx>
---
pyverbs/providers/mlx5/CMakeLists.txt | 1 +
pyverbs/providers/mlx5/libmlx5.pxd | 29 ++++
pyverbs/providers/mlx5/mlx5_enums.pyx | 1 +
pyverbs/providers/mlx5/mlx5dv.pxd | 3 +
pyverbs/providers/mlx5/mlx5dv.pyx | 196 ++++++++++++++++++++++++
pyverbs/providers/mlx5/mlx5dv_enums.pxd | 47 ++++++
6 files changed, 277 insertions(+)
create mode 120000 pyverbs/providers/mlx5/mlx5_enums.pyx
create mode 100644 pyverbs/providers/mlx5/mlx5dv_enums.pxd
diff --git a/pyverbs/providers/mlx5/CMakeLists.txt b/pyverbs/providers/mlx5/CMakeLists.txt
index f6536de8a932..d9b0849f02dc 100644
--- a/pyverbs/providers/mlx5/CMakeLists.txt
+++ b/pyverbs/providers/mlx5/CMakeLists.txt
@@ -3,4 +3,5 @@
rdma_cython_module(pyverbs/providers/mlx5 mlx5
mlx5dv.pyx
+ mlx5_enums.pyx
)
diff --git a/pyverbs/providers/mlx5/libmlx5.pxd b/pyverbs/providers/mlx5/libmlx5.pxd
index 54d91e288590..aaf75620a9bb 100644
--- a/pyverbs/providers/mlx5/libmlx5.pxd
+++ b/pyverbs/providers/mlx5/libmlx5.pxd
@@ -12,6 +12,35 @@ cdef extern from 'infiniband/mlx5dv.h':
unsigned int flags
unsigned long comp_mask
+ cdef struct mlx5dv_cqe_comp_caps:
+ unsigned int max_num
+ unsigned int supported_format
+
+ cdef struct mlx5dv_sw_parsing_caps:
+ unsigned int sw_parsing_offloads
+ unsigned int supported_qpts
+
+ cdef struct mlx5dv_striding_rq_caps:
+ unsigned int min_single_stride_log_num_of_bytes
+ unsigned int max_single_stride_log_num_of_bytes
+ unsigned int min_single_wqe_log_num_of_strides
+ unsigned int max_single_wqe_log_num_of_strides
+ unsigned int supported_qpts
+
+ cdef struct mlx5dv_context:
+ unsigned char version
+ unsigned long flags
+ unsigned long comp_mask
+ mlx5dv_cqe_comp_caps cqe_comp_caps
+ mlx5dv_sw_parsing_caps sw_parsing_caps
+ mlx5dv_striding_rq_caps striding_rq_caps
+ unsigned int tunnel_offloads_caps
+ unsigned int max_dynamic_bfregs
+ unsigned long max_clock_info_update_nsec
+ unsigned int flow_action_flags
+ unsigned int dc_odp_caps
+
bool mlx5dv_is_supported(v.ibv_device *device)
v.ibv_context* mlx5dv_open_device(v.ibv_device *device,
mlx5dv_context_attr *attr)
+ int mlx5dv_query_device(v.ibv_context *ctx, mlx5dv_context *attrs_out)
diff --git a/pyverbs/providers/mlx5/mlx5_enums.pyx b/pyverbs/providers/mlx5/mlx5_enums.pyx
new file mode 120000
index 000000000000..ba0e916f9e62
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5_enums.pyx
@@ -0,0 +1 @@
+mlx5dv_enums.pxd
\ No newline at end of file
diff --git a/pyverbs/providers/mlx5/mlx5dv.pxd b/pyverbs/providers/mlx5/mlx5dv.pxd
index 6ab94b6484b0..d9fea82af894 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pxd
+++ b/pyverbs/providers/mlx5/mlx5dv.pxd
@@ -12,3 +12,6 @@ cdef class Mlx5Context(Context):
cdef class Mlx5DVContextAttr(PyverbsObject):
cdef dv.mlx5dv_context_attr attr
+
+cdef class Mlx5DVContext(PyverbsObject):
+ cdef dv.mlx5dv_context dv
diff --git a/pyverbs/providers/mlx5/mlx5dv.pyx b/pyverbs/providers/mlx5/mlx5dv.pyx
index 0c6b28be1d5a..dadc9cdcceee 100644
--- a/pyverbs/providers/mlx5/mlx5dv.pyx
+++ b/pyverbs/providers/mlx5/mlx5dv.pyx
@@ -2,7 +2,10 @@
# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
from pyverbs.pyverbs_error import PyverbsUserError
+cimport pyverbs.providers.mlx5.mlx5dv_enums as dve
cimport pyverbs.providers.mlx5.libmlx5 as dv
+from pyverbs.base import PyverbsRDMAErrno
+cimport pyverbs.libibverbs_enums as e
cdef class Mlx5DVContextAttr(PyverbsObject):
@@ -55,3 +58,196 @@ cdef class Mlx5Context(Context):
if not dv.mlx5dv_is_supported(self.device):
raise PyverbsUserError('This is not an MLX5 device')
self.context = dv.mlx5dv_open_device(self.device, &attr.attr)
+
+ def query_mlx5_device(self, comp_mask=-1):
+ """
+ Queries the provider for device-specific attributes.
+ :param comp_mask: Which attributes to query. Default value is -1. If
+ not changed by user, pyverbs will pass a bitwise OR
+ of all available enum entries.
+ :return: A Mlx5DVContext containing the attributes.
+ """
+ dv_attr = Mlx5DVContext()
+ if comp_mask == -1:
+ dv_attr.comp_mask = \
+ dve.MLX5DV_CONTEXT_MASK_CQE_COMPRESION |\
+ dve.MLX5DV_CONTEXT_MASK_SWP |\
+ dve.MLX5DV_CONTEXT_MASK_STRIDING_RQ |\
+ dve.MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS |\
+ dve.MLX5DV_CONTEXT_MASK_DYN_BFREGS |\
+ dve.MLX5DV_CONTEXT_MASK_CLOCK_INFO_UPDATE |\
+ dve.MLX5DV_CONTEXT_MASK_FLOW_ACTION_FLAGS
+ else:
+ dv_attr.comp_mask = comp_mask
+ rc = dv.mlx5dv_query_device(self.context, &dv_attr.dv)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to query mlx5 device {name}, got {rc}'.
+ format(name=self.name, rc=rc))
+ return dv_attr
+
+
+cdef class Mlx5DVContext(PyverbsObject):
+ """
+ Represents mlx5dv_context struct, which exposes mlx5-specific capabilities,
+ reported by mlx5dv_query_device.
+ """
+ @property
+ def version(self):
+ return self.dv.version
+
+ @property
+ def flags(self):
+ return self.dv.flags
+
+ @property
+ def comp_mask(self):
+ return self.dv.comp_mask
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.dv.comp_mask = val
+
+ @property
+ def cqe_comp_caps(self):
+ return self.dv.cqe_comp_caps
+
+ @property
+ def sw_parsing_caps(self):
+ return self.dv.sw_parsing_caps
+
+ @property
+ def striding_rq_caps(self):
+ return self.dv.striding_rq_caps
+
+ @property
+ def tunnel_offload_caps(self):
+ return self.dv.tunnel_offloads_caps
+
+ @property
+ def max_dynamic_bfregs(self):
+ return self.dv.max_dynamic_bfregs
+
+ @property
+ def max_clock_info_update_nsec(self):
+ return self.dv.max_clock_info_update_nsec
+
+ @property
+ def flow_action_flags(self):
+ return self.dv.flow_action_flags
+
+ @property
+ def dc_odp_caps(self):
+ return self.dv.dc_odp_caps
+
+ def __str__(self):
+ print_format = '{:20}: {:<20}\n'
+ ident_format = ' {:20}: {:<20}\n'
+ cqe = 'CQE compression caps:\n' +\
+ ident_format.format('max num',
+ self.dv.cqe_comp_caps.max_num) +\
+ ident_format.format('supported formats',
+ cqe_comp_to_str(self.dv.cqe_comp_caps.supported_format))
+ swp = 'SW parsing caps:\n' +\
+ ident_format.format('SW parsing offloads',
+ swp_to_str(self.dv.sw_parsing_caps.sw_parsing_offloads)) +\
+ ident_format.format('supported QP types',
+ qpts_to_str(self.dv.sw_parsing_caps.supported_qpts))
+ strd = 'Striding RQ caps:\n' +\
+ ident_format.format('min single stride log num of bytes',
+ self.dv.striding_rq_caps.min_single_stride_log_num_of_bytes) +\
+ ident_format.format('max single stride log num of bytes',
+ self.dv.striding_rq_caps.max_single_stride_log_num_of_bytes) +\
+ ident_format.format('min single wqe log num of strides',
+ self.dv.striding_rq_caps.min_single_wqe_log_num_of_strides) +\
+ ident_format.format('max single wqe log num of strides',
+ self.dv.striding_rq_caps.max_single_wqe_log_num_of_strides) +\
+ ident_format.format('supported QP types',
+ qpts_to_str(self.dv.striding_rq_caps.supported_qpts))
+ return print_format.format('Version', self.dv.version) +\
+ print_format.format('Flags',
+ context_flags_to_str(self.dv.flags)) +\
+ print_format.format('comp mask',
+ context_comp_mask_to_str(self.dv.comp_mask)) +\
+ cqe + swp + strd +\
+ print_format.format('Tunnel offloads caps',
+ tunnel_offloads_to_str(self.dv.tunnel_offloads_caps)) +\
+ print_format.format('Max dynamic BF registers',
+ self.dv.max_dynamic_bfregs) +\
+ print_format.format('Max clock info update [nsec]',
+ self.dv.max_clock_info_update_nsec) +\
+ print_format.format('Flow action flags',
+ self.dv.flow_action_flags) +\
+ print_format.format('DC ODP caps', self.dv.dc_odp_caps)
+
+
+def qpts_to_str(qp_types):
+ numberic_types = qp_types
+ qpts_str = ''
+ qpts = {e.IBV_QPT_RC: 'RC', e.IBV_QPT_UC: 'UC', e.IBV_QPT_UD: 'UD',
+ e.IBV_QPT_RAW_PACKET: 'Raw Packet', e.IBV_QPT_XRC_SEND: 'XRC Send',
+ e.IBV_QPT_XRC_RECV: 'XRC Recv', e.IBV_QPT_DRIVER: 'Driver QPT'}
+ for t in qpts.keys():
+ if (1 << t) & qp_types:
+ qpts_str += qpts[t] + ', '
+ qp_types -= t
+ if qp_types == 0:
+ break
+ return qpts_str[:-2] + ' ({})'.format(numberic_types)
+
+
+def bitmask_to_str(bits, values):
+ numeric_bits = bits
+ res = ''
+ for t in values.keys():
+ if t & bits:
+ res += values[t] + ', '
+ bits -= t
+ if bits == 0:
+ break
+ return res[:-2] + ' ({})'.format(numeric_bits) # Remove last comma and space
+
+
+def context_comp_mask_to_str(mask):
+ l = {dve.MLX5DV_CONTEXT_MASK_CQE_COMPRESION: 'CQE compression',
+ dve.MLX5DV_CONTEXT_MASK_SWP: 'SW parsing',
+ dve.MLX5DV_CONTEXT_MASK_STRIDING_RQ: 'Striding RQ',
+ dve.MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS: 'Tunnel offloads',
+ dve.MLX5DV_CONTEXT_MASK_DYN_BFREGS: 'Dynamic BF regs',
+ dve.MLX5DV_CONTEXT_MASK_CLOCK_INFO_UPDATE: 'Clock info update',
+ dve.MLX5DV_CONTEXT_MASK_FLOW_ACTION_FLAGS: 'Flow action flags'}
+ return bitmask_to_str(mask, l)
+
+
+def context_flags_to_str(flags):
+ l = {dve.MLX5DV_CONTEXT_FLAGS_CQE_V1: 'CQE v1',
+ dve.MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED: 'Multi packet WQE allowed',
+ dve.MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW: 'Enhanced multi packet WQE',
+ dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP: 'Support CQE 128B compression',
+ dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD: 'Support CQE 128B padding',
+ dve.MLX5DV_CONTEXT_FLAGS_PACKET_BASED_CREDIT_MODE:
+ 'Support packet based credit mode (in RC QP)'}
+ return bitmask_to_str(flags, l)
+
+
+def swp_to_str(swps):
+ l = {dve.MLX5DV_SW_PARSING: 'SW Parsing',
+ dve.MLX5DV_SW_PARSING_CSUM: 'SW Parsing CSUM',
+ dve.MLX5DV_SW_PARSING_LSO: 'SW Parsing LSO'}
+ return bitmask_to_str(swps, l)
+
+
+def cqe_comp_to_str(cqe):
+ l = {dve.MLX5DV_CQE_RES_FORMAT_HASH: 'with hash',
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM: 'with RX checksum CSUM',
+ dve.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX: 'with stride index'}
+ return bitmask_to_str(cqe, l)
+
+
+def tunnel_offloads_to_str(tun):
+ l = {dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN: 'VXLAN',
+ dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE: 'GRE',
+ dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE: 'Geneve',
+ dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE:\
+ 'Ctrl word + MPLS over GRE',
+ dve.MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP:\
+ 'Ctrl word + MPLS over UDP'}
+ return bitmask_to_str(tun, l)
diff --git a/pyverbs/providers/mlx5/mlx5dv_enums.pxd b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
new file mode 100644
index 000000000000..038a49111a3b
--- /dev/null
+++ b/pyverbs/providers/mlx5/mlx5dv_enums.pxd
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. See COPYING file
+
+#cython: language_level=3
+
+cdef extern from 'infiniband/mlx5dv.h':
+
+ cpdef enum mlx5dv_context_comp_mask:
+ MLX5DV_CONTEXT_MASK_CQE_COMPRESION = 1 << 0
+ MLX5DV_CONTEXT_MASK_SWP = 1 << 1
+ MLX5DV_CONTEXT_MASK_STRIDING_RQ = 1 << 2
+ MLX5DV_CONTEXT_MASK_TUNNEL_OFFLOADS = 1 << 3
+ MLX5DV_CONTEXT_MASK_DYN_BFREGS = 1 << 4
+ MLX5DV_CONTEXT_MASK_CLOCK_INFO_UPDATE = 1 << 5
+ MLX5DV_CONTEXT_MASK_FLOW_ACTION_FLAGS = 1 << 6
+
+ cpdef enum mlx5dv_context_flags:
+ MLX5DV_CONTEXT_FLAGS_CQE_V1 = 1 << 0
+ MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED = 1 << 2
+ MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW = 1 << 3
+ MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP = 1 << 4
+ MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD = 1 << 5
+ MLX5DV_CONTEXT_FLAGS_PACKET_BASED_CREDIT_MODE = 1 << 6
+
+ cpdef enum mlx5dv_sw_parsing_offloads:
+ MLX5DV_SW_PARSING = 1 << 0
+ MLX5DV_SW_PARSING_CSUM = 1 << 1
+ MLX5DV_SW_PARSING_LSO = 1 << 2
+
+ cpdef enum mlx5dv_cqe_comp_res_format:
+ MLX5DV_CQE_RES_FORMAT_HASH = 1 << 0
+ MLX5DV_CQE_RES_FORMAT_CSUM = 1 << 1
+ MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX = 1 << 2
+
+ cpdef enum mlx5dv_tunnel_offloads:
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_VXLAN = 1 << 0
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GRE = 1 << 1
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_GENEVE = 1 << 2
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE = 1 << 3
+ MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_UDP = 1 << 4
+
+ cpdef enum mlx5dv_flow_action_cap_flags:
+ MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM = 1 << 0
+ MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_REQ_METADATA = 1 << 1
+ MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_SPI_STEERING = 1 << 2
+ MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_FULL_OFFLOAD = 1 << 3
+ MLX5DV_FLOW_ACTION_FLAGS_ESP_AES_GCM_TX_IV_IS_ESN = 1 << 4
--
2.21.0
