In order to use memory window, some additions were needed:
- Allow pyverbs users to create MWBindInfo, the Python representation
of ibv_mw_bind_info.
- Expose ibv_inc_rkey, which creates a new rkey from a given one by
increasing its 8 LSBs while keeping the same index.
- Expose the memory window's rkey and handle properties.
Signed-off-by: Ahmad Ghazawi <ahmadg@xxxxxxxxxxxx>
Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx>
Reviewed-by: Edward Srouji <edwards@xxxxxxxxxxxx>
---
pyverbs/base.pyx | 8 +++++++-
pyverbs/libibverbs.pxd | 1 +
pyverbs/mr.pxd | 6 ++++++
pyverbs/mr.pyx | 19 +++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx
index c5b16795ddb6..790ba4153dea 100644
--- a/pyverbs/base.pyx
+++ b/pyverbs/base.pyx
@@ -1,9 +1,15 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from libc.errno cimport errno
import logging
+
from pyverbs.pyverbs_error import PyverbsRDMAError
-from libc.errno cimport errno
+cimport pyverbs.libibverbs as v
+
+
+def inc_rkey(rkey):
+ return v.ibv_inc_rkey(rkey)
cpdef PyverbsRDMAErrno(str msg):
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index ad8d8bacc541..fea8a1b408da 100755
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -551,3 +551,4 @@ cdef extern from 'infiniband/verbs.h':
ibv_recv_wr **bad_recv_wr)
ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
ibv_parent_domain_init_attr *attr)
+ uint32_t ibv_inc_rkey(uint32_t rkey)
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index fb46611e6f42..402df4492425 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -12,6 +12,12 @@ cdef class MR(PyverbsCM):
cdef v.ibv_mr *mr
cdef void *buf
cpdef read(self, length, offset)
+ cdef add_ref(self, obj)
+ cdef object bind_infos
+
+cdef class MWBindInfo(PyverbsCM):
+ cdef v.ibv_mw_bind_info info
+ cdef object mr
cdef class MW(PyverbsCM):
cdef object pd
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 6b28c8173ef8..9b1277f0882f 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -3,10 +3,12 @@
import resource
import logging
+import weakref
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
from posix.stdlib cimport posix_memalign
+from pyverbs.base cimport close_weakrefs
from libc.string cimport memcpy, memset
from libc.stdint cimport uintptr_t
from pyverbs.device cimport DM
@@ -29,6 +31,7 @@ cdef class MR(PyverbsCM):
:return: The newly created MR on success
"""
super().__init__()
+ self.bind_infos = weakref.WeakSet()
if self.mr != NULL:
return
#We want to enable registering an MR of size 0 but this fails with a
@@ -61,6 +64,7 @@ cdef class MR(PyverbsCM):
:return: None
"""
self.logger.debug('Closing MR')
+ close_weakrefs([self.bind_infos])
if self.mr != NULL:
rc = v.ibv_dereg_mr(self.mr)
if rc != 0:
@@ -96,6 +100,10 @@ cdef class MR(PyverbsCM):
data = <char*>(self.buf + off)
return data[:length]
+ cdef add_ref(self, obj):
+ if isinstance(obj, MWBindInfo):
+ self.bind_infos.add(obj)
+
@property
def buf(self):
return <uintptr_t>self.buf
@@ -109,6 +117,17 @@ cdef class MR(PyverbsCM):
return self.mr.rkey
+cdef class MWBindInfo(PyverbsCM):
+ def __init__(self, MR mr not None, addr, length, mw_access_flags):
+ super().__init__()
+ self.mr = mr
+ self.info.mr = mr.mr
+ self.info.addr = addr
+ self.info.length = length
+ self.info.mw_access_flags = mw_access_flags
+ mr.add_ref(self)
+
+
cdef class MW(PyverbsCM):
def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
"""
--
2.21.0
