On Mon, May 10, 2021 at 08:48:03PM +0800, Weihang Li wrote:
> From: Xi Wang <wangxi11@xxxxxxxxxx>
>
> The hip09 introduces the DCA(Dynamic context attachment) feature which
> supports many RC QPs to share the WQE buffer in a memory pool, this will
> reduce the memory consumption when there are too many QPs are inactive.
>
> If a QP enables DCA feature, the WQE's buffer will not be allocated when
> creating. But when the users start to post WRs, the hns driver will
> allocate a buffer from the memory pool and then fill WQEs which tagged with
> this QP's number.
>
> The hns ROCEE will stop accessing the WQE buffer when the user polled all
> of the CQEs for a DCA QP, then the driver will recycle this WQE's buffer
> to the memory pool.
>
> This patch adds a group of methods to support the user space register
> buffers to a memory pool which belongs to the user context. The hns kernel
> driver will update the pages state in this pool when the user calling the
> post/poll methods and the user driver can get the QP's WQE buffer address
> by the key and offset which queried from kernel.
>
> Signed-off-by: Xi Wang <wangxi11@xxxxxxxxxx>
> Signed-off-by: Weihang Li <liweihang@xxxxxxxxxx>
> ---
> drivers/infiniband/hw/hns/Makefile | 2 +-
> drivers/infiniband/hw/hns/hns_roce_dca.c | 381 ++++++++++++++++++++++++++++
> drivers/infiniband/hw/hns/hns_roce_dca.h | 22 ++
> drivers/infiniband/hw/hns/hns_roce_device.h | 9 +
> drivers/infiniband/hw/hns/hns_roce_main.c | 27 +-
> include/uapi/rdma/hns-abi.h | 27 ++
> 6 files changed, 465 insertions(+), 3 deletions(-)
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_dca.c
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_dca.h
>
> diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
> index e105945..9962b23 100644
> --- a/drivers/infiniband/hw/hns/Makefile
> +++ b/drivers/infiniband/hw/hns/Makefile
> @@ -6,7 +6,7 @@
> ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3
>
> hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
> - hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
> + hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o hns_roce_dca.o \
> hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
>
> ifdef CONFIG_INFINIBAND_HNS_HIP06
> diff --git a/drivers/infiniband/hw/hns/hns_roce_dca.c b/drivers/infiniband/hw/hns/hns_roce_dca.c
> new file mode 100644
> index 0000000..2a03cf3
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_dca.c
> @@ -0,0 +1,381 @@
> +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
> +/*
> + * Copyright (c) 2021 Hisilicon Limited. All rights reserved.
> + */
> +
> +#include <rdma/ib_user_verbs.h>
> +#include <rdma/ib_verbs.h>
> +#include <rdma/uverbs_types.h>
> +#include <rdma/uverbs_ioctl.h>
> +#include <rdma/uverbs_std_types.h>
> +#include <rdma/ib_umem.h>
> +#include "hns_roce_device.h"
> +#include "hns_roce_dca.h"
> +
> +#define UVERBS_MODULE_NAME hns_ib
> +#include <rdma/uverbs_named_ioctl.h>
> +
> +/* DCA memory */
> +struct dca_mem {
> +#define DCA_MEM_FLAGS_ALLOCED BIT(0)
> +#define DCA_MEM_FLAGS_REGISTERED BIT(1)
> + u32 flags;
> + struct list_head list; /* link to mem list in dca context */
> + spinlock_t lock; /* protect the @flags and @list */
> + int page_count; /* page count in this mem obj */
> + u64 key; /* register by caller */
> + u32 size; /* bytes in this mem object */
> + struct hns_dca_page_state *states; /* record each page's state */
> + void *pages; /* memory handle for getting dma address */
> +};
> +
> +struct dca_mem_attr {
> + u64 key;
> + u64 addr;
> + u32 size;
> +};
> +
> +static inline bool dca_mem_is_free(struct dca_mem *mem)
> +{
> + return mem->flags == 0;
> +}
> +
> +static inline void set_dca_mem_free(struct dca_mem *mem)
> +{
> + mem->flags = 0;
> +}
> +
> +static inline void set_dca_mem_alloced(struct dca_mem *mem)
> +{
> + mem->flags |= DCA_MEM_FLAGS_ALLOCED;
> +}
> +
> +static inline void set_dca_mem_registered(struct dca_mem *mem)
> +{
> + mem->flags |= DCA_MEM_FLAGS_REGISTERED;
> +}
> +
> +static inline void clr_dca_mem_registered(struct dca_mem *mem)
> +{
> + mem->flags &= ~DCA_MEM_FLAGS_REGISTERED;
> +}
> +
> +static void free_dca_pages(void *pages)
> +{
> + ib_umem_release(pages);
> +}
All this oneline madness should go.
Thanks
