Expose DM and DMMR classes.
DM represents ibv_dm.
DMMR is similar to a regular MR but its buffer is allocated on the
device and not by the OS.
Signed-off-by: Daria Velikovsky <daria@xxxxxxxxxxxx>
Signed-off-by: Noa Osherovich <noaos@xxxxxxxxxxxx>
---
pyverbs/device.pxd | 10 ++++
pyverbs/device.pyx | 112 ++++++++++++++++++++++++++++++++++++++++-
pyverbs/libibverbs.pxd | 17 +++++++
pyverbs/mr.pxd | 3 ++
pyverbs/mr.pyx | 43 +++++++++++++++-
pyverbs/pd.pyx | 5 +-
6 files changed, 185 insertions(+), 5 deletions(-)
diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd
index c81e8176eef9..c9e346539e58 100644
--- a/pyverbs/device.pxd
+++ b/pyverbs/device.pxd
@@ -10,6 +10,7 @@ cdef class Context(PyverbsCM):
cdef object name
cdef add_ref(self, obj)
cdef object pds
+ cdef object dms
cdef class DeviceAttr(PyverbsObject):
cdef v.ibv_device_attr dev_attr
@@ -37,3 +38,12 @@ cdef class TSOCaps(PyverbsObject):
cdef class DeviceAttrEx(PyverbsObject):
cdef v.ibv_device_attr_ex dev_attr
+
+cdef class AllocDmAttr(PyverbsObject):
+ cdef v.ibv_alloc_dm_attr alloc_dm_attr
+
+cdef class DM(PyverbsCM):
+ cdef v.ibv_dm *dm
+ cdef object dm_mrs
+ cdef object context
+ cdef add_ref(self, obj)
diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx
index 948909753738..7bfcda8bc177 100644
--- a/pyverbs/device.pyx
+++ b/pyverbs/device.pyx
@@ -13,13 +13,21 @@ from .pyverbs_error import PyverbsUserError
from pyverbs.base import PyverbsRDMAErrno
cimport pyverbs.libibverbs as v
from pyverbs.addr cimport GID
+from pyverbs.mr import DMMR
from pyverbs.pd cimport PD
cdef extern from 'errno.h':
int errno
-
cdef extern from 'endian.h':
unsigned long be64toh(unsigned long host_64bits);
+cdef extern from 'stdlib.h':
+ void free(void *ptr)
+cdef extern from 'string.h':
+ void *memset(void *s, int c, size_t n)
+cdef extern from 'malloc.h':
+ void *malloc(size_t size)
+cdef extern from 'stdint.h':
+ ctypedef int uint64_t
class Device(PyverbsObject):
@@ -75,6 +83,8 @@ cdef class Context(PyverbsCM):
cdef v.ibv_device **dev_list
self.pds = weakref.WeakSet()
+ self.dms = weakref.WeakSet()
+
dev_name = kwargs.get('name')
if dev_name is not None:
@@ -110,7 +120,7 @@ cdef class Context(PyverbsCM):
cpdef close(self):
self.logger.debug('Closing Context')
- self.close_weakrefs([self.pds])
+ self.close_weakrefs([self.dms, self.pds])
if self.context != NULL:
rc = v.ibv_close_device(self.context)
if rc != 0:
@@ -158,6 +168,8 @@ cdef class Context(PyverbsCM):
cdef add_ref(self, obj):
if isinstance(obj, PD):
self.pds.add(obj)
+ elif isinstance(obj, DM):
+ self.dms.add(obj)
else:
raise PyverbsError('Unrecognized object type')
@@ -477,6 +489,102 @@ cdef class DeviceAttrEx(PyverbsObject):
return self.dev_attr.max_dm_size
+cdef class AllocDmAttr(PyverbsObject):
+ def __cinit__(self, length, log_align_req = 0, comp_mask = 0):
+ """
+ Creates an AllocDmAttr object with the given parameters. This object
+ can than be used to create a DM object.
+ :param length: Length of the future device memory
+ :param log_align_req: log2 of address alignment requirement
+ :param comp_mask: compatibility mask
+ :return: An AllocDmAttr object
+ """
+ self.alloc_dm_attr.length = length
+ self.alloc_dm_attr.log_align_req = log_align_req
+ self.alloc_dm_attr.comp_mask = comp_mask
+
+ @property
+ def length(self):
+ return self.alloc_dm_attr.length
+
+ @length.setter
+ def length(self, val):
+ self.alloc_dm_attr.length = val
+
+ @property
+ def log_align_req(self):
+ return self.alloc_dm_attr.log_align_req
+
+ @log_align_req.setter
+ def log_align_req(self, val):
+ self.alloc_dm_attr.log_align_req = val
+
+ @property
+ def comp_mask(self):
+ return self.alloc_dm_attr.comp_mask
+
+ @comp_mask.setter
+ def comp_mask(self, val):
+ self.alloc_dm_attr.comp_mask = val
+
+
+cdef class DM(PyverbsCM):
+ def __cinit__(self, Context context, AllocDmAttr dm_attr not None):
+ """
+ Allocate a device (direct) memory.
+ :param context: The context of the device on which to allocate memory
+ :param dm_attr: Attributes that define the DM
+ :return: A DM object on success
+ """
+ self.dm_mrs = weakref.WeakSet()
+ device_attr = context.query_device_ex()
+ if device_attr.max_dm_size <= 0:
+ raise PyverbsUserError('Device doesn\'t support dm allocation')
+ self.dm = v.ibv_alloc_dm(<v.ibv_context*>context.context,
+ &dm_attr.alloc_dm_attr)
+ if self.dm == NULL:
+ raise PyverbsRDMAErrno('Failed to allocate device memory of size '
+ '{size}. Max available size {max}.'
+ .format(size=dm_attr.length,
+ max=device_attr.max_dm_size))
+ self.context = context
+ context.add_ref(self)
+
+ def __dealloc__(self):
+ self.close()
+
+ cpdef close(self):
+ self.logger.debug('Closing DM')
+ self.close_weakrefs([self.dm_mrs])
+ if self.dm != NULL:
+ rc = v.ibv_free_dm(self.dm)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to free dm')
+ self.dm = NULL
+ self.context = None
+
+ cdef add_ref(self, obj):
+ if isinstance(obj, DMMR):
+ self.dm_mrs.add(obj)
+
+ def copy_to_dm(self, dm_offset, data, length):
+ rc = v.ibv_memcpy_to_dm(<v.ibv_dm *>self.dm, <uint64_t>dm_offset,
+ <char *>data, <size_t>length)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to copy to dm')
+
+ def copy_from_dm(self, dm_offset, length):
+ cdef char *data =<char*>malloc(length)
+ memset(data, 0, length)
+ rc = v.ibv_memcpy_from_dm(<void *>data, <v.ibv_dm *>self.dm,
+ <uint64_t>dm_offset, <size_t>length)
+ if rc != 0:
+ raise PyverbsRDMAErrno('Failed to copy from dm')
+ res = data[:length]
+ free(data)
+ return res
+
+
def guid_format(num):
"""
Get GUID representation of the given number, including change of endianness.
diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd
index 7eb1df2407c9..5dd3ba013e5e 100644
--- a/pyverbs/libibverbs.pxd
+++ b/pyverbs/libibverbs.pxd
@@ -139,6 +139,15 @@ cdef extern from 'infiniband/verbs.h':
unsigned int handle
ibv_mw_type mw_type
+ cdef struct ibv_alloc_dm_attr:
+ size_t length
+ unsigned int log_align_req
+ unsigned int comp_mask
+
+ cdef struct ibv_dm:
+ ibv_context *context
+ unsigned int comp_mask
+
ibv_device **ibv_get_device_list(int *n)
void ibv_free_device_list(ibv_device **list)
ibv_context *ibv_open_device(ibv_device *device)
@@ -156,3 +165,11 @@ cdef extern from 'infiniband/verbs.h':
int ibv_dereg_mr(ibv_mr *mr)
ibv_mw *ibv_alloc_mw(ibv_pd *pd, ibv_mw_type type)
int ibv_dealloc_mw(ibv_mw *mw)
+ ibv_dm *ibv_alloc_dm(ibv_context *context, ibv_alloc_dm_attr *attr)
+ int ibv_free_dm(ibv_dm *dm)
+ ibv_mr *ibv_reg_dm_mr(ibv_pd *pd, ibv_dm *dm, unsigned long dm_offset,
+ size_t length, unsigned int access)
+ int ibv_memcpy_to_dm(ibv_dm *dm, unsigned long dm_offset, void *host_addr,
+ size_t length)
+ int ibv_memcpy_from_dm(void *host_addr, ibv_dm *dm, unsigned long dm_offset,
+ size_t length)
diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd
index 91d662070135..2d76f2dfbe7c 100644
--- a/pyverbs/mr.pxd
+++ b/pyverbs/mr.pxd
@@ -14,3 +14,6 @@ cdef class MR(PyverbsCM):
cdef class MW(PyverbsCM):
cdef object pd
cdef v.ibv_mw *mw
+
+cdef class DMMR(MR):
+ cdef object dm
diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx
index 1ab4bd809dee..92b195f75ee2 100644
--- a/pyverbs/mr.pyx
+++ b/pyverbs/mr.pyx
@@ -3,6 +3,7 @@
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.device cimport DM
from .pd cimport PD
import resource
@@ -22,7 +23,7 @@ cdef class MR(PyverbsCM):
MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing
it is done in close().
"""
- def __cinit__(self, PD pd not None, length, access):
+ def __cinit__(self, PD pd not None, length, access, **kwargs):
"""
Allocate a user-level buffer of length <length> and register a Memory
Region of the given length and access flags.
@@ -31,6 +32,8 @@ cdef class MR(PyverbsCM):
:param access: Access flags, see ibv_access_flags enum
:return: The newly created MR on success
"""
+ if len(kwargs) != 0:
+ return
#We want to enable registering an MR of size 0 but this fails with a
#buffer of size 0, so in this case lets increase the buffer
if length == 0:
@@ -146,6 +149,44 @@ cdef class MW(PyverbsCM):
self.pd = None
+cdef class DMMR(MR):
+ def __cinit__(self, PD pd not None, length, access, **kwargs):
+ """
+ Initializes a DMMR (Device Memory Memory Region) of the given length
+ and access flags using the given PD and DM objects.
+ :param pd: A PD object
+ :param length: Length in bytes
+ :param access: Access flags, see ibv_access_flags enum
+ :param kwargs: see below
+ :return: The newly create DMMR
+
+ :keyword Arguments:
+ * *dm* (DM)
+ A DM (device memory) object to be used for this DMMR
+ * *offset*
+ Byte offset from the beginning of the allocated device memory
+ buffer
+ """
+ dm = <DM>kwargs['dm']
+ offset = kwargs['offset']
+ self.mr = v.ibv_reg_dm_mr(pd.pd, (<DM>dm).dm, offset, length, access)
+ if self.mr == NULL:
+ raise PyverbsRDMAErrno('Failed to register a device MR. length: {len}, access flags: {flags}'.
+ format(len=length, flags=access,))
+ self.pd = pd
+ self.dm = dm
+ pd.add_ref(self)
+ dm.add_ref(self)
+ self.logger.debug('Registered device ibv_mr. Length: {len}, access flags {flags}'.
+ format(len=length, flags=access))
+
+ def write(self, data, length):
+ return self.dm.copy_to_dm(0, data, length)
+
+ cpdef read(self, length, offset):
+ return self.dm.copy_from_dm(offset, length)
+
+
def mwtype2str(mw_type):
mw_types = {1:'IBV_MW_TYPE_1', 2:'IBV_MW_TYPE_2'}
try:
diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx
index da580423cd81..65cc851b0ecf 100644
--- a/pyverbs/pd.pyx
+++ b/pyverbs/pd.pyx
@@ -4,7 +4,8 @@ import weakref
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from pyverbs.base import PyverbsRDMAErrno
-from .mr cimport MR, MW
+from pyverbs.device cimport Context, DM
+from .mr cimport MR, MW, DMMR
cdef extern from 'errno.h':
int errno
@@ -52,7 +53,7 @@ cdef class PD(PyverbsCM):
self.ctx = None
cdef add_ref(self, obj):
- if isinstance(obj, MR):
+ if isinstance(obj, MR) or isinstance(obj, DMMR):
self.mrs.add(obj)
elif isinstance(obj, MW):
self.mws.add(obj)
--
2.17.2
