From: Ido Kalir <idok@xxxxxxxxxxxx>
ECE (enhanced connection establishment) is a mechanism that gives an
option to different libibverbs providers to advertise and use various
provider-specific QP configuration options.
Add those verbs:
ibv_query_ece - get QPs ece.
ibv_set_ece - set the QPs ece.
rdma_set_local_ece - set the local CMs ece.
rdma_get_remote_ece - get the remote CM ece from the connection request.
Signed-off-by: Maxim Chicherin <maximc@xxxxxxxxxxxx>
Signed-off-by: Ido Kalir <idok@xxxxxxxxxxxx>
Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxxxx>
---
pyverbs/cmid.pyx | 23 ++++++++++++++++++++-
pyverbs/libibverbs.pxd | 7 +++++++
pyverbs/librdmacm.pxd | 2 ++
pyverbs/qp.pxd | 3 +++
pyverbs/qp.pyx | 45 ++++++++++++++++++++++++++++++++++++++++--
5 files changed, 77 insertions(+), 3 deletions(-)
diff --git a/pyverbs/cmid.pyx b/pyverbs/cmid.pyx
index 66d73268..2505ec70 100755
--- a/pyverbs/cmid.pyx
+++ b/pyverbs/cmid.pyx
@@ -1,7 +1,7 @@
from libc.string cimport memset
from pyverbs.pyverbs_error import PyverbsUserError
-from pyverbs.qp cimport QPInitAttr, QPAttr
+from pyverbs.qp cimport QPInitAttr, QPAttr, ECE
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs_enums as e
cimport pyverbs.librdmacm_enums as ce
@@ -424,6 +424,27 @@ cdef class CMID(PyverbsCM):
if ret != 0:
raise PyverbsRDMAErrno('Failed to Complete an active connection request')
+ def set_local_ece(self, ECE ece):
+ """
+ Set local ECE paraemters to be used for REQ/REP communication.
+ :param ece: ECE object with the requested configuration
+ :return: None
+ """
+ rc = cm.rdma_set_local_ece(self.id, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to set local ECE')
+
+ def get_remote_ece(self):
+ """
+ Get ECE parameters as were received from the communication peer.
+ :return: ECE object with the ece configuration
+ """
+ ece = ECE()
+ rc = cm.rdma_get_remote_ece(self.id, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to get remote ECE')
+ return ece
+
def create_qp(self, QPInitAttr qp_init not None):
"""
Create a QP, which is associated with CMID.
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 6ffa303c..52f51f07 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -475,6 +475,11 @@ cdef extern from 'infiniband/verbs.h':
uint64_t wr_id
unsigned int wr_flags
+ cdef struct ibv_ece:
+ uint32_t vendor_id
+ uint32_t options
+ uint32_t comp_mask
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -599,3 +604,5 @@ cdef extern from 'infiniband/verbs.h':
cdef extern from 'infiniband/driver.h':
int ibv_query_gid_type(ibv_context *context, uint8_t port_num,
unsigned int index, ibv_gid_type *type)
+ int ibv_set_ece(ibv_qp *qp, ibv_ece *ece)
+ int ibv_query_ece(ibv_qp *qp, ibv_ece *ece)
diff --git a/pyverbs/librdmacm.pxd b/pyverbs/librdmacm.pxd
index 03c0cddc..ff579205 100755
--- a/pyverbs/librdmacm.pxd
+++ b/pyverbs/librdmacm.pxd
@@ -97,6 +97,8 @@ cdef extern from '<rdma/rdma_cma.h>':
int rdma_create_id(rdma_event_channel *channel, rdma_cm_id **id,
void *context, rdma_port_space ps)
int rdma_destroy_id(rdma_cm_id *id)
+ int rdma_get_remote_ece(rdma_cm_id *id, ibv_ece *ece)
+ int rdma_set_local_ece(rdma_cm_id *id, ibv_ece *ece)
int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id)
int rdma_bind_addr(rdma_cm_id *id, sockaddr *addr)
int rdma_resolve_addr(rdma_cm_id *id, sockaddr *src_addr,
diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd
index 209a2438..1294560a 100644
--- a/pyverbs/qp.pxd
+++ b/pyverbs/qp.pxd
@@ -43,3 +43,6 @@ cdef class DataBuffer(PyverbsCM):
cdef class QPEx(QP):
cdef v.ibv_qp_ex *qp_ex
+
+cdef class ECE(PyverbsCM):
+ cdef v.ibv_ece ece
diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx
index 95ef554c..fd851443 100755
--- a/pyverbs/qp.pyx
+++ b/pyverbs/qp.pyx
@@ -4,9 +4,8 @@
from libc.stdlib cimport malloc, free
from libc.string cimport memcpy
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
-from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, \
- PyverbsRDMAError
from pyverbs.utils import access_flags_to_str, mig_state_to_str
from pyverbs.base import PyverbsRDMAErrno
from pyverbs.wr cimport RecvWR, SendWR, SGE
@@ -871,6 +870,27 @@ cdef class QPAttr(PyverbsObject):
print_format.format('Rate limit', self.attr.rate_limit)
+cdef class ECE(PyverbsCM):
+ def __init__(self, vendor_id=0, options=0, comp_mask=0):
+ """
+ :param vendor_id: Unique identifier of the provider vendor.
+ :param options: Provider specific attributes which are supported or
+ needed to be enabled by ECE users.
+ :param comp_mask: A bitmask specifying which ECE options should be
+ valid.
+ """
+ super().__init__()
+ self.ece.vendor_id = vendor_id
+ self.ece.options = options
+ self.ece.comp_mask = comp_mask
+
+ def __str__(self):
+ print_format = '{:22}: {:<20}\n'
+ print_format.format('Vendor ID', self.ece.vendor_id) +\
+ print_format.format('Options', self.ece.options) +\
+ print_format.format('Comp Mask', self.ece.comp_mask)
+
+
cdef class QP(PyverbsCM):
def __init__(self, object creator not None, object init_attr not None,
QPAttr qp_attr=None):
@@ -1133,6 +1153,27 @@ cdef class QP(PyverbsCM):
memcpy(&bad_wr.send_wr, my_bad_wr, sizeof(bad_wr.send_wr))
raise PyverbsRDMAError('Failed to post send', rc)
+ def set_ece(self, ECE ece):
+ """
+ Set ECE options and use them for QP configuration stage
+ :param ece: The requested ECE values.
+ :return: None
+ """
+ rc = v.ibv_set_ece(self.qp, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to set ECE', rc)
+
+ def query_ece(self):
+ """
+ Query QPs ECE options
+ :return: ECE object with this QP ece configuration.
+ """
+ ece = ECE()
+ rc = v.ibv_query_ece(self.qp, &ece.ece)
+ if rc != 0:
+ raise PyverbsRDMAError('Failed to query ECE', rc)
+ return ece
+
@property
def qp_type(self):
return self.qp.qp_type
--
2.25.2
