When using events, the CQ has to ack all the retrieved events prior to
destruction. Since this call uses a mutex, the events are usually acked
together at the end of the datapath to avoid performance degradation.
In Python, each exception triggers a teardown of all pyverbs' resources,
including CQs (which may haven't acked their events yet). Exception can
be caused by something as trivial as a syntax error.
To avoid that, let CQ object keep track of the number of events waiting
to be acked. This number will be incremented by the completion channel
during get_cq_event() and decremented by the CQ during ack_events().
During close(), if there are still events waiting to be acked, the CQ
will ack them prior to its destruction.
Also add a reference from a CQ object to its completion channel object
to allow users to know if it is connected to one or not during poll.
Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx>
Reviewed-by: Edward Srouji <edwards@xxxxxxxxxxxx>
---
pyverbs/cq.pxd | 2 ++
pyverbs/cq.pyx | 14 +++++++++++++-
2 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd
index 8eeb2e1fd0c2..9704b96f3ff7 100644
--- a/pyverbs/cq.pxd
+++ b/pyverbs/cq.pxd
@@ -20,6 +20,8 @@ cdef class CQ(PyverbsCM):
cdef add_ref(self, obj)
cdef object qps
cdef object srqs
+ cdef object channel
+ cdef object num_events
cdef class CqInitAttrEx(PyverbsObject):
cdef v.ibv_cq_init_attr_ex attr
diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx
index dda47207507f..defb37646034 100755
--- a/pyverbs/cq.pyx
+++ b/pyverbs/cq.pyx
@@ -47,7 +47,7 @@ cdef class CompChannel(PyverbsCM):
def get_cq_event(self, CQ expected_cq):
"""
Waits for the next completion event in the completion event channel
- :param expected_cq: The CQ that got the event
+ :param expected_cq: The CQ that is expected to get the event
:return: None
"""
cdef v.ibv_cq *cq
@@ -58,6 +58,7 @@ cdef class CompChannel(PyverbsCM):
raise PyverbsRDMAErrno('Failed to get CQ event')
if cq != expected_cq.cq:
raise PyverbsRDMAErrno('Received event on an unexpected CQ')
+ expected_cq.num_events += 1
cdef add_ref(self, obj):
if isinstance(obj, CQ) or isinstance(obj, CQEX):
@@ -87,15 +88,18 @@ cdef class CQ(PyverbsCM):
self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context,
channel.cc, comp_vector)
channel.add_ref(self)
+ self.channel = channel
else:
self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context,
NULL, comp_vector)
+ self.channel = None
if self.cq == NULL:
raise PyverbsRDMAErrno('Failed to create a CQ')
self.context = context
context.add_ref(self)
self.qps = weakref.WeakSet()
self.srqs = weakref.WeakSet()
+ self.num_events = 0
self.logger.debug('Created a CQ')
cdef add_ref(self, obj):
@@ -112,12 +116,15 @@ cdef class CQ(PyverbsCM):
cpdef close(self):
self.logger.debug('Closing CQ')
close_weakrefs([self.qps, self.srqs])
+ if self.num_events:
+ self.ack_events(self.num_events)
if self.cq != NULL:
rc = v.ibv_destroy_cq(self.cq)
if rc != 0:
raise PyverbsRDMAErrno('Failed to close CQ')
self.cq = NULL
self.context = None
+ self.channel = None
def poll(self, num_entries=1):
"""
@@ -166,6 +173,7 @@ cdef class CQ(PyverbsCM):
:return: None
"""
v.ibv_ack_cq_events(self.cq, num_events)
+ self.num_events -= num_events
def __str__(self):
print_format = '{:22}: {:<20}\n'
@@ -173,6 +181,10 @@ cdef class CQ(PyverbsCM):
print_format.format('Handle', self.cq.handle) +\
print_format.format('CQEs', self.cq.cqe)
+ @property
+ def comp_channel(self):
+ return self.channel
+
cdef class CqInitAttrEx(PyverbsObject):
def __init__(self, cqe = 100, CompChannel channel = None, comp_vector = 0,
--
2.21.0
