Re: [PATCH rdma-core v7 4/6] pyverbs: Add dma-buf based MR support

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 25/01/2021 21:57, Jianxin Xiong wrote:
> Define a new sub-class of 'MR' that uses dma-buf object for the memory
> region. Define a new class 'DmaBuf' as a wrapper for dma-buf allocation
> mechanism implemented in C.
> 
> Update the cmake function for cython modules to allow building modules
> with mixed cython and c source files.
> 
> Signed-off-by: Jianxin Xiong <jianxin.xiong@xxxxxxxxx>
> ---
>  buildlib/pyverbs_functions.cmake |  78 +++++++----
>  pyverbs/CMakeLists.txt           |  11 +-
>  pyverbs/dmabuf.pxd               |  15 +++
>  pyverbs/dmabuf.pyx               |  73 ++++++++++
>  pyverbs/dmabuf_alloc.c           | 278 +++++++++++++++++++++++++++++++++++++++
>  pyverbs/dmabuf_alloc.h           |  19 +++
>  pyverbs/libibverbs.pxd           |   2 +
>  pyverbs/mr.pxd                   |   6 +
>  pyverbs/mr.pyx                   | 105 ++++++++++++++-
>  9 files changed, 557 insertions(+), 30 deletions(-)
>  create mode 100644 pyverbs/dmabuf.pxd
>  create mode 100644 pyverbs/dmabuf.pyx
>  create mode 100644 pyverbs/dmabuf_alloc.c
>  create mode 100644 pyverbs/dmabuf_alloc.h
> 
> diff --git a/buildlib/pyverbs_functions.cmake b/buildlib/pyverbs_functions.cmake
> index 953cec2..0792410 100644
> --- a/buildlib/pyverbs_functions.cmake
> +++ b/buildlib/pyverbs_functions.cmake
> @@ -1,35 +1,61 @@
>  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
>  # Copyright (c) 2018, Mellanox Technologies. All rights reserved.  See COPYING file
> +# Copyright (c) 2020, Intel Corporation. All rights reserved.  See COPYING file
> +
> +function(build_module_from_cfiles PY_MODULE MODULE_NAME ALL_CFILES LINKER_FLAGS)
> +  string(REGEX REPLACE "\\.so$" "" SONAME "${MODULE_NAME}${CMAKE_PYTHON_SO_SUFFIX}")
> +  add_library(${SONAME} SHARED ${ALL_CFILES})
> +  set_target_properties(${SONAME} PROPERTIES
> +    COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations ${NO_VAR_TRACKING_FLAGS}"
> +    LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"
> +    PREFIX "")
> +  target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
> +  install(TARGETS ${SONAME}
> +    DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
> +endfunction()
>  
>  function(rdma_cython_module PY_MODULE LINKER_FLAGS)
> -  foreach(PYX_FILE ${ARGN})
> -    get_filename_component(FILENAME ${PYX_FILE} NAME_WE)
> -    get_filename_component(DIR ${PYX_FILE} DIRECTORY)
> -	if (DIR)
> -		set(PYX "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}/${FILENAME}.pyx")
> -	else()
> -	    set(PYX "${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}.pyx")
> -	endif()
> -    set(CFILE "${CMAKE_CURRENT_BINARY_DIR}/${FILENAME}.c")
> -    include_directories(${PYTHON_INCLUDE_DIRS})
> -    add_custom_command(
> -      OUTPUT "${CFILE}"
> -      MAIN_DEPENDENCY "${PYX}"
> -      COMMAND ${CYTHON_EXECUTABLE} "${PYX}" -o "${CFILE}"
> -      "-I${PYTHON_INCLUDE_DIRS}"
> -      COMMENT "Cythonizing ${PYX}"
> +  set(ALL_CFILES "")
> +  set(MODULE_NAME "")
> +  foreach(SRC_FILE ${ARGN})
> +    get_filename_component(FILENAME ${SRC_FILE} NAME_WE)
> +    get_filename_component(DIR ${SRC_FILE} DIRECTORY)
> +    get_filename_component(EXT ${SRC_FILE} EXT)
> +    if (DIR)
> +      set(SRC_PATH "${CMAKE_CURRENT_SOURCE_DIR}/${DIR}")
> +    else()
> +      set(SRC_PATH "${CMAKE_CURRENT_SOURCE_DIR}")
> +    endif()
> +    if (${EXT} STREQUAL ".pyx")
> +      # each .pyx file starts a new module, finish the previous module first
> +      if (ALL_CFILES AND MODULE_NAME)
> +        build_module_from_cfiles(${PY_MODULE} ${MODULE_NAME} "${ALL_CFILES}" "${LINKER_FLAGS}")
> +      endif()
> +      set(PYX "${SRC_PATH}/${FILENAME}.pyx")
> +      set(CFILE "${CMAKE_CURRENT_BINARY_DIR}/${FILENAME}.c")
> +      include_directories(${PYTHON_INCLUDE_DIRS})
> +      add_custom_command(
> +        OUTPUT "${CFILE}"
> +        MAIN_DEPENDENCY "${PYX}"
> +        COMMAND ${CYTHON_EXECUTABLE} "${PYX}" -o "${CFILE}"
> +        "-I${PYTHON_INCLUDE_DIRS}"
> +        COMMENT "Cythonizing ${PYX}"
>        )
> -
> -    string(REGEX REPLACE "\\.so$" "" SONAME "${FILENAME}${CMAKE_PYTHON_SO_SUFFIX}")
> -    add_library(${SONAME} SHARED ${CFILE})
> -    set_target_properties(${SONAME} PROPERTIES
> -      COMPILE_FLAGS "${CMAKE_C_FLAGS} -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations ${NO_VAR_TRACKING_FLAGS}"
> -      LIBRARY_OUTPUT_DIRECTORY "${BUILD_PYTHON}/${PY_MODULE}"
> -      PREFIX "")
> -    target_link_libraries(${SONAME} LINK_PRIVATE ${PYTHON_LIBRARIES} ibverbs rdmacm ${LINKER_FLAGS})
> -    install(TARGETS ${SONAME}
> -      DESTINATION ${CMAKE_INSTALL_PYTHON_ARCH_LIB}/${PY_MODULE})
> +      set(MODULE_NAME ${FILENAME})
> +      set(ALL_CFILES "${CFILE}")
> +    elseif(${EXT} STREQUAL ".c")
> +      # .c files belong to the same module as the most recent .pyx file,
> +      # ignored if appearing before all .pyx files
> +      set(CFILE "${SRC_PATH}/${FILENAME}.c")
> +      set(ALL_CFILES "${ALL_CFILES};${CFILE}")
> +    else()
> +      continue()
> +    endif()
>    endforeach()
> +  # finish the last module
> +  if (ALL_CFILES AND MODULE_NAME)
> +    build_module_from_cfiles(${PY_MODULE} ${MODULE_NAME} "${ALL_CFILES}" "${LINKER_FLAGS}")
> +  endif()
>  endfunction()
>  
>  function(rdma_python_module PY_MODULE)
> diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt
> index 9542c4b..6fd7625 100644
> --- a/pyverbs/CMakeLists.txt
> +++ b/pyverbs/CMakeLists.txt
> @@ -1,5 +1,10 @@
>  # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
>  # Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
> +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
> +
> +publish_internal_headers(""
> +  dmabuf_alloc.h
> +)
>  
>  rdma_cython_module(pyverbs ""
>    addr.pyx
> @@ -8,15 +13,17 @@ rdma_cython_module(pyverbs ""
>    cmid.pyx
>    cq.pyx
>    device.pyx
> +  dmabuf.pyx
> +  dmabuf_alloc.c
>    enums.pyx
>    mem_alloc.pyx
>    mr.pyx
>    pd.pyx
>    qp.pyx
> +  srq.pyx
>    wr.pyx
>    xrcd.pyx
> -  srq.pyx
> -  )
> +)
>  
>  rdma_python_module(pyverbs
>    __init__.py
> diff --git a/pyverbs/dmabuf.pxd b/pyverbs/dmabuf.pxd
> new file mode 100644
> index 0000000..a063acb
> --- /dev/null
> +++ b/pyverbs/dmabuf.pxd
> @@ -0,0 +1,15 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
> +
> +#cython: language_level=3
> +
> +cdef class DmaBuf:
> +    cdef int drm_fd
> +    cdef int handle
> +    cdef int fd
> +    cdef unsigned long size
> +    cdef unsigned long map_offset
> +    cdef void *dmabuf
> +    cdef object dmabuf_mrs
> +    cdef add_ref(self, obj)
> +    cpdef close(self)
> diff --git a/pyverbs/dmabuf.pyx b/pyverbs/dmabuf.pyx
> new file mode 100644
> index 0000000..b9406bd
> --- /dev/null
> +++ b/pyverbs/dmabuf.pyx
> @@ -0,0 +1,73 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
> +# Copyright (c) 2020, Intel Corporation. All rights reserved. See COPYING file
> +
> +#cython: language_level=3
> +
> +import weakref
> +
> +from pyverbs.base cimport close_weakrefs
> +from pyverbs.base import PyverbsRDMAErrno
> +from pyverbs.mr cimport DmaBufMR
> +
> +cdef extern from "dmabuf_alloc.h":
> +    cdef struct dmabuf:
> +        pass
> +    dmabuf *dmabuf_alloc(unsigned long size, int unit, int gtt)
> +    void dmabuf_free(dmabuf *dmabuf)
> +    int dmabuf_get_drm_fd(dmabuf *dmabuf)
> +    int dmabuf_get_fd(dmabuf *dmabuf)
> +    unsigned long dmabuf_get_offset(dmabuf *dmabuf)
> +
> +
> +cdef class DmaBuf:
> +    def __init__(self, size, unit=0, gtt=0):
> +        """
> +        Allocate DmaBuf object from a GPU device. This is done through the
> +        DRI device interface. Usually this requires the effective user id
> +        being a member of the 'render' group.
> +        :param size: The size (in number of bytes) of the buffer.
> +        :param unit: The unit number of the GPU to allocate the buffer from.
> +        :param gtt: Allocate from GTT instead of VRAM.
> +        :return: The newly created DmaBuf object on success.
> +        """
> +        self.dmabuf_mrs = weakref.WeakSet()
> +        self.dmabuf = dmabuf_alloc(size, unit, gtt)
> +        if self.dmabuf == NULL:
> +            raise PyverbsRDMAErrno(f'Failed to allocate dmabuf of size {size} on unit {unit}')
> +        self.drm_fd = dmabuf_get_drm_fd(<dmabuf *>self.dmabuf)
> +        self.fd = dmabuf_get_fd(<dmabuf *>self.dmabuf)
> +        self.map_offset = dmabuf_get_offset(<dmabuf *>self.dmabuf)
> +
> +    def __dealloc__(self):
> +        self.close()
> +
> +    cpdef close(self):
> +        if self.dmabuf == NULL:
> +            return None
> +        close_weakrefs([self.dmabuf_mrs])
> +        dmabuf_free(<dmabuf *>self.dmabuf)
> +        self.dmabuf = NULL
> +
> +    cdef add_ref(self, obj):
> +        if isinstance(obj, DmaBufMR):
> +            self.dmabuf_mrs.add(obj)
> +
> +    @property
> +    def drm_fd(self):
> +        return self.drm_fd
> +
> +    @property
> +    def handle(self):
> +        return self.handle
> +
> +    @property
> +    def fd(self):
> +        return self.fd
> +
> +    @property
> +    def size(self):
> +        return self.size
> +
> +    @property
> +    def map_offset(self):
> +        return self.map_offset
> diff --git a/pyverbs/dmabuf_alloc.c b/pyverbs/dmabuf_alloc.c
> new file mode 100644
> index 0000000..05eae75
> --- /dev/null
> +++ b/pyverbs/dmabuf_alloc.c
> @@ -0,0 +1,278 @@
> +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
> +/*
> + * Copyright 2020 Intel Corporation. All rights reserved. See COPYING file
> + */
> +
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <stdint.h>
> +#include <unistd.h>
> +#include <string.h>
> +#include <errno.h>
> +#include <drm/drm.h>
> +#include <drm/i915_drm.h>
> +#include <drm/amdgpu_drm.h>
> +#include <drm/radeon_drm.h>

I assume these should come from the kernel headers package, right?
We're getting new compilation errors on SLES 15 for some reason, although the
package is installed.

FAILED: pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o 
/usr/bin/cc -Ddmabuf_cpython_36m_x86_64_linux_gnu_EXPORTS -Iinclude -I/usr/include/libnl3 -I/usr/include/python3.6m -Wall -Wextra -Wno-sign-compare -Wno-unused-parameter -Wmissing-prototypes -Wmissing-declarations -Wwrite-strings -Wformat=2 -Wformat-nonliteral -Wdate-time -Wnested-externs -Wshadow -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -O2 -g  -fPIC    -Wall -Wextra -Wno-sign-compare -Wno-unused-parameter -Wmissing-prototypes -Wmissing-declarations -Wwrite-strings -Wformat=2 -Wformat-nonliteral -Wdate-time -Wnested-externs -Wshadow -Wstrict-prototypes -Wold-style-definition -Wredundant-decls -fPIC -fno-strict-aliasing -Wno-unused-function -Wno-redundant-decls -Wno-shadow -Wno-cast-function-type -Wno-implicit-fallthrough -Wno-unknown-warning -Wno-unknown-warning-option -Wno-deprecated-declarations  -fno-var-tracking-assignments -std=gnu11 -MD -MT pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o -MF pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o.d -o pyverbs/CMakeFiles/dmabuf.cpython-36m-x86_64-linux-gnu.dir/dmabuf_alloc.c.o   -c ../pyverbs/dmabuf_alloc.c
 ../pyverbs/dmabuf_alloc.c:12:10: fatal error: drm/drm.h: No such file or directory
  #include <drm/drm.h>
           ^~~~~~~~~~~
 compilation terminated.




[Index of Archives]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Photo]     [Yosemite News]     [Yosemite Photos]     [Linux Kernel]     [Linux SCSI]     [XFree86]

  Powered by Linux