When using events, the CQ has to ack all the retrieved events prior to destruction. Since this call uses a mutex, the events are usually acked together at the end of the datapath to avoid performance degradation. In Python, each exception triggers a teardown of all pyverbs' resources, including CQs (which may haven't acked their events yet). Exception can be caused by something as trivial as a syntax error. To avoid that, let CQ object keep track of the number of events waiting to be acked. This number will be incremented by the completion channel during get_cq_event() and decremented by the CQ during ack_events(). During close(), if there are still events waiting to be acked, the CQ will ack them prior to its destruction. Also add a reference from a CQ object to its completion channel object to allow users to know if it is connected to one or not during poll. Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx> Reviewed-by: Edward Srouji <edwards@xxxxxxxxxxxx> --- pyverbs/cq.pxd | 2 ++ pyverbs/cq.pyx | 14 +++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyverbs/cq.pxd b/pyverbs/cq.pxd index 8eeb2e1fd0c2..9704b96f3ff7 100644 --- a/pyverbs/cq.pxd +++ b/pyverbs/cq.pxd @@ -20,6 +20,8 @@ cdef class CQ(PyverbsCM): cdef add_ref(self, obj) cdef object qps cdef object srqs + cdef object channel + cdef object num_events cdef class CqInitAttrEx(PyverbsObject): cdef v.ibv_cq_init_attr_ex attr diff --git a/pyverbs/cq.pyx b/pyverbs/cq.pyx index dda47207507f..defb37646034 100755 --- a/pyverbs/cq.pyx +++ b/pyverbs/cq.pyx @@ -47,7 +47,7 @@ cdef class CompChannel(PyverbsCM): def get_cq_event(self, CQ expected_cq): """ Waits for the next completion event in the completion event channel - :param expected_cq: The CQ that got the event + :param expected_cq: The CQ that is expected to get the event :return: None """ cdef v.ibv_cq *cq @@ -58,6 +58,7 @@ cdef class CompChannel(PyverbsCM): raise PyverbsRDMAErrno('Failed to get CQ event') if cq != expected_cq.cq: raise PyverbsRDMAErrno('Received event on an unexpected CQ') + expected_cq.num_events += 1 cdef add_ref(self, obj): if isinstance(obj, CQ) or isinstance(obj, CQEX): @@ -87,15 +88,18 @@ cdef class CQ(PyverbsCM): self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context, channel.cc, comp_vector) channel.add_ref(self) + self.channel = channel else: self.cq = v.ibv_create_cq(context.context, cqe, <void*>cq_context, NULL, comp_vector) + self.channel = None if self.cq == NULL: raise PyverbsRDMAErrno('Failed to create a CQ') self.context = context context.add_ref(self) self.qps = weakref.WeakSet() self.srqs = weakref.WeakSet() + self.num_events = 0 self.logger.debug('Created a CQ') cdef add_ref(self, obj): @@ -112,12 +116,15 @@ cdef class CQ(PyverbsCM): cpdef close(self): self.logger.debug('Closing CQ') close_weakrefs([self.qps, self.srqs]) + if self.num_events: + self.ack_events(self.num_events) if self.cq != NULL: rc = v.ibv_destroy_cq(self.cq) if rc != 0: raise PyverbsRDMAErrno('Failed to close CQ') self.cq = NULL self.context = None + self.channel = None def poll(self, num_entries=1): """ @@ -166,6 +173,7 @@ cdef class CQ(PyverbsCM): :return: None """ v.ibv_ack_cq_events(self.cq, num_events) + self.num_events -= num_events def __str__(self): print_format = '{:22}: {:<20}\n' @@ -173,6 +181,10 @@ cdef class CQ(PyverbsCM): print_format.format('Handle', self.cq.handle) +\ print_format.format('CQEs', self.cq.cqe) + @property + def comp_channel(self): + return self.channel + cdef class CqInitAttrEx(PyverbsObject): def __init__(self, cqe = 100, CompChannel channel = None, comp_vector = 0, -- 2.21.0