> -----Original Message-----
> From: Cheng Xu <chengyou@xxxxxxxxxxxxxxxxx>
> Sent: Friday, December 24, 2021 12:25 PM
> To: leon@xxxxxxxxxx
> Cc: dledford@xxxxxxxxxx; jgg@xxxxxxxxxxxx; linux-rdma@xxxxxxxxxxxxxxx;
> KaiShen@xxxxxxxxxxxxxxxxx; chengyou@xxxxxxxxxxxxxxxxx
> Subject: [PATCH rdma-core 1/5] RDMA-CORE/erdma: Add userspace verbs
> related header files.
>
> Add the userspace verbs implementation related header files: 'erdma_hw.h'
> for hardware interface definitions, 'erdma_verbs.h' for verbs related
> definitions and 'erdma_db.h' for doorbell records related definitions.
>
> Signed-off-by: Cheng Xu <chengyou@xxxxxxxxxxxxxxxxx>
> ---
> providers/erdma/erdma_db.h | 17 +++
> providers/erdma/erdma_hw.h | 206
> ++++++++++++++++++++++++++++++++++
> providers/erdma/erdma_verbs.h | 134 ++++++++++++++++++++++
> 3 files changed, 357 insertions(+)
> create mode 100644 providers/erdma/erdma_db.h create mode 100644
> providers/erdma/erdma_hw.h create mode 100644
> providers/erdma/erdma_verbs.h
>
> diff --git a/providers/erdma/erdma_db.h b/providers/erdma/erdma_db.h
> new file mode 100644 index 00000000..c302cb7a
> --- /dev/null
> +++ b/providers/erdma/erdma_db.h
> @@ -0,0 +1,17 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@xxxxxxxxxxxxxxxxx>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_DB_H__
> +#define __ERDMA_DB_H__
> +
> +#include <inttypes.h>
> +
> +#include "erdma.h"
> +
> +uint64_t *erdma_alloc_dbrecords(struct erdma_context *ctx); void
> +erdma_dealloc_dbrecords(struct erdma_context *ctx, uint64_t
> +*dbrecords);
> +
> +#endif
> diff --git a/providers/erdma/erdma_hw.h b/providers/erdma/erdma_hw.h
> new file mode 100644 index 00000000..f373f7f8
> --- /dev/null
> +++ b/providers/erdma/erdma_hw.h
> @@ -0,0 +1,206 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@xxxxxxxxxxxxxxxxx>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_HW_H__
> +#define __ERDMA_HW_H__
> +
> +#include <stdint.h>
> +
> +#define ERDMA_SDB_PAGE 0
> +#define ERDMA_SDB_ENTRY 1
> +#define ERDMA_SDB_SHARED 2
> +
> +#define ERDMA_SQDB_SIZE 128
> +#define ERDMA_CQDB_SIZE 8
> +#define ERDMA_RQDB_SIZE 8
> +#define ERDMA_RQDB_SPACE_SIZE 32
> +
> +/* WQE related. */
> +#define EQE_SIZE 16
> +#define EQE_SHIFT 4
> +#define RQE_SIZE 32
> +#define RQE_SHIFT 5
> +#define CQE_SIZE 32
> +#define CQE_SHIFT 5
> +#define SQEBB_SIZE 32
> +#define SQEBB_SHIFT 5
> +#define SQEBB_MASK (~(SQEBB_SIZE - 1))
> +#define SQEBB_ALIGN(size) ((size + SQEBB_SIZE - 1) & SQEBB_MASK)
> +#define SQEBB_COUNT(size) (SQEBB_ALIGN(size) >> SQEBB_SHIFT)
> +
> +#define MAX_WQEBB_PER_SQE 4
> +
> +enum erdma_opcode {
> + ERDMA_OP_WRITE = 0,
> + ERDMA_OP_READ = 1,
> + ERDMA_OP_SEND = 2,
> + ERDMA_OP_SEND_WITH_IMM = 3,
> +
> + ERDMA_OP_RECEIVE = 4,
> + ERDMA_OP_RECV_IMM = 5,
> + ERDMA_OP_RECV_INV = 6,
> +
> + ERDMA_OP_REQ_ERR = 7,
> + ERDNA_OP_READ_RESPONSE = 8,
> + ERDMA_OP_WRITE_WITH_IMM = 9,
> +
> + ERDMA_OP_RECV_ERR = 10,
> +
> + ERDMA_OP_INVALIDATE = 11,
> + ERDMA_OP_RSP_SEND_IMM = 12,
> + ERDMA_OP_SEND_WITH_INV = 13,
> +
> + ERDMA_OP_REG_MR = 14,
> + ERDMA_OP_LOCAL_INV = 15,
> + ERDMA_OP_READ_WITH_INV = 16,
> + ERDMA_NUM_OPCODES = 17,
> + ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
> +};
> +
> +/*
> + * Inline data are kept within the work request itself occupying
> + * the space of sge[1] .. sge[n]. Therefore, inline data cannot be
> + * supported if ERDMA_MAX_SGE is below 2 elements.
> + */
> +#define ERDMA_MAX_INLINE (sizeof(struct erdma_sge) *
> (ERDMA_MAX_SEND_SGE))
> +
> +enum erdma_wc_status {
> + ERDMA_WC_SUCCESS = 0,
> + ERDMA_WC_GENERAL_ERR = 1,
> + ERDMA_WC_RECV_WQE_FORMAT_ERR = 2,
> + ERDMA_WC_RECV_STAG_INVALID_ERR = 3,
> + ERDMA_WC_RECV_ADDR_VIOLATION_ERR = 4,
> + ERDMA_WC_RECV_RIGHT_VIOLATION_ERR = 5,
> + ERDMA_WC_RECV_PDID_ERR = 6,
> + ERDMA_WC_RECV_WARRPING_ERR = 7,
> + ERDMA_WC_SEND_WQE_FORMAT_ERR = 8,
> + ERDMA_WC_SEND_WQE_ORD_EXCEED = 9,
> + ERDMA_WC_SEND_STAG_INVALID_ERR = 10,
> + ERDMA_WC_SEND_ADDR_VIOLATION_ERR = 11,
> + ERDMA_WC_SEND_RIGHT_VIOLATION_ERR = 12,
> + ERDMA_WC_SEND_PDID_ERR = 13,
> + ERDMA_WC_SEND_WARRPING_ERR = 14,
> + ERDMA_WC_FLUSH_ERR = 15,
> + ERDMA_WC_RETRY_EXC_ERR = 16,
> + ERDMA_NUM_WC_STATUS
> +};
> +
> +enum erdma_vendor_err {
> + ERDMA_WC_VENDOR_NO_ERR = 0,
> + ERDMA_WC_VENDOR_INVALID_RQE = 1,
> + ERDMA_WC_VENDOR_RQE_INVALID_STAG = 2,
> + ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION = 3,
> + ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR = 4,
> + ERDMA_WC_VENDOR_RQE_INVALID_PD = 5,
> + ERDMA_WC_VENDOR_RQE_WRAP_ERR = 6,
> + ERDMA_WC_VENDOR_INVALID_SQE = 0x20,
> + ERDMA_WC_VENDOR_ZERO_ORD = 0x21,
> + ERDMA_WC_VENDOR_SQE_INVALID_STAG = 0x30,
> + ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION = 0x31,
> + ERDMA_WC_VENDOR_SQE_ACCESS_ERR = 0x32,
> + ERDMA_WC_VENDOR_SQE_INVALID_PD = 0x33,
> + ERDMA_WC_VENDOR_SQE_WARP_ERR = 0x34
> +};
> +
> +
> +/* Doorbell related. */
> +#define ERDMA_CQDB_EQN_MASK GENMASK_ULL(63, 56)
> +#define ERDMA_CQDB_CQN_MASK GENMASK_ULL(55, 32)
> +#define ERDMA_CQDB_ARM_MASK BIT_ULL(31)
> +#define ERDMA_CQDB_SOL_MASK BIT_ULL(30)
> +#define ERDMA_CQDB_CMDSN_MASK GENMASK_ULL(29, 28)
> +#define ERDMA_CQDB_CI_MASK GENMASK_ULL(23, 0)
> +
> +#define ERDMA_CQE_QTYPE_SQ 0
> +#define ERDMA_CQE_QTYPE_RQ 1
> +#define ERDMA_CQE_QTYPE_CMDQ 2
> +
> +/* CQE hdr */
> +#define ERDMA_CQE_HDR_OWNER_MASK BIT(31)
> +#define ERDMA_CQE_HDR_OPCODE_MASK GENMASK(23, 16)
> +#define ERDMA_CQE_HDR_QTYPE_MASK GENMASK(15, 8)
> +#define ERDMA_CQE_HDR_SYNDROME_MASK GENMASK(7, 0)
> +
> +#define ERDMA_CQE_QTYPE_SQ 0
> +#define ERDMA_CQE_QTYPE_RQ 1
> +#define ERDMA_CQE_QTYPE_CMDQ 2
> +
> +
> +struct erdma_cqe {
> + __be32 hdr;
> + __be32 qe_idx;
> + __be32 qpn;
> + __le32 imm_data;
> + __be32 size;
> + __be32 rsvd[3];
> +};
> +
> +struct erdma_sge {
> + __aligned_le64 laddr;
> + __le32 length;
> + __le32 lkey;
> +};
> +
> +/* Receive Queue Element */
> +struct erdma_rqe {
> + __le16 qe_idx;
> + __le16 rsvd;
> + __le32 qpn;
> + __le32 rsvd2;
> + __le32 rsvd3;
> + __le64 to;
> + __le32 length;
> + __le32 stag;
> +};
> +
> +/* SQE */
> +#define ERDMA_SQE_HDR_SGL_LEN_MASK GENMASK_ULL(63, 56)
> +#define ERDMA_SQE_HDR_WQEBB_CNT_MASK GENMASK_ULL(54, 52)
> +#define ERDMA_SQE_HDR_QPN_MASK GENMASK_ULL(51, 32)
> +#define ERDMA_SQE_HDR_OPCODE_MASK GENMASK_ULL(31, 27)
> +#define ERDMA_SQE_HDR_DWQE_MASK BIT_ULL(26)
> +#define ERDMA_SQE_HDR_INLINE_MASK BIT_ULL(25)
> +#define ERDMA_SQE_HDR_FENCE_MASK BIT_ULL(24)
> +#define ERDMA_SQE_HDR_SE_MASK BIT_ULL(23)
> +#define ERDMA_SQE_HDR_CE_MASK BIT_ULL(22)
> +#define ERDMA_SQE_HDR_WQEBB_INDEX_MASK GENMASK_ULL(15, 0)
> +
> +struct erdma_write_sqe {
> + __le64 hdr;
> + __be32 imm_data;
> + __le32 length;
> +
> + __le32 sink_stag;
> + /* avoid sink_to not 8-byte aligned. */
> + __le32 sink_to_low;
> + __le32 sink_to_high;
> +
> + __le32 rsvd;
> +
> + struct erdma_sge sgl[0];
> +};
> +
> +struct erdma_send_sqe {
> + __le64 hdr;
> + __be32 imm_data;
> + __le32 length;
> + struct erdma_sge sgl[0];
> +};
> +
> +struct erdma_readreq_sqe {
> + __le64 hdr;
> + __le32 invalid_stag;
> + __le32 length;
> + __le32 sink_stag;
> + /* avoid sink_to not 8-byte aligned. */
> + __le32 sink_to_low;
> + __le32 sink_to_high;
> + __le32 rsvd0;
> + struct erdma_sge sgl;
> +};
> +
> +
> +#endif
> diff --git a/providers/erdma/erdma_verbs.h
> b/providers/erdma/erdma_verbs.h new file mode 100644 index
> 00000000..944d29f5
> --- /dev/null
> +++ b/providers/erdma/erdma_verbs.h
> @@ -0,0 +1,134 @@
> +/* SPDX-License-Identifier: GPL-2.0 or OpenIB.org BSD (MIT) See COPYING
> +file */
> +/*
> + * Authors: Cheng Xu <chengyou@xxxxxxxxxxxxxxxxx>
> + * Copyright (c) 2020-2021, Alibaba Group.
> + */
> +
> +#ifndef __ERDMA_VERBS_H__
> +#define __ERDMA_VERBS_H__
> +
> +#include <pthread.h>
> +#include <inttypes.h>
> +#include <stddef.h>
> +
> +#include "erdma.h"
> +#include "erdma_hw.h"
> +
> +#define ERDMA_MAX_SEND_SGE 6
> +#define ERDMA_MAX_RECV_SGE 1
> +
> +struct erdma_queue {
> + void *qbuf;
> + void *db;
> +
> + uint16_t rsvd0;
> + uint16_t depth;
> + uint32_t size;
> +
> + uint16_t pi;
> + uint16_t ci;
> +
> + uint32_t rsvd1;
> + uint64_t *wr_tbl;
> +
> + void *db_record;
> +};
> +
> +struct erdma_qp {
> + struct ibv_qp base_qp;
> + struct erdma_device *erdma_dev;
> +
> + uint32_t id; /* qpn */
> +
> + pthread_spinlock_t sq_lock;
> + pthread_spinlock_t rq_lock;
> +
> + int sq_sig_all;
> +
> + struct erdma_queue sq;
> + struct erdma_queue rq;
> +
> + void *qbuf;
> + uint64_t *db_records;
> +};
> +
> +struct erdma_cq {
> + struct ibv_cq base_cq;
> + struct erdma_device *erdma_dev;
> + uint32_t id;
> +
> + uint32_t event_stats;
> +
> + uint32_t depth;
> + uint32_t ci;
> + uint32_t owner;
> + struct erdma_cqe *queue;
> +
> + void *db;
> + uint16_t db_offset;
> +
> + void *db_record;
> + uint32_t cmdsn;
> + int comp_vector;
> +
> + pthread_spinlock_t lock;
> +};
> +
> +static inline struct erdma_qp *to_eqp(struct ibv_qp *base) {
> + return container_of(base, struct erdma_qp, base_qp); }
> +
> +static inline struct erdma_cq *to_ecq(struct ibv_cq *base) {
> + return container_of(base, struct erdma_cq, base_cq); }
> +
> +static inline void *get_sq_wqebb(struct erdma_qp *qp, uint16_t idx) {
> + idx &= (qp->sq.depth - 1);
> + return qp->sq.qbuf + (idx << SQEBB_SHIFT); }
> +
> +static inline void __kick_sq_db(struct erdma_qp *qp, uint16_t pi) {
> + uint64_t db_data;
> +
> + db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, qp->id) |
> + FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);
> +
> + *(__le64 *)qp->sq.db_record = htole64(db_data);
> + udma_to_device_barrier();
> + mmio_write64_le(qp->sq.db, htole64(db_data)); }
Standard function definition format
Func ()
{
}
Update all over the place.
> +
> +struct ibv_pd *erdma_alloc_pd(struct ibv_context *ctx); int
> +erdma_free_pd(struct ibv_pd *pd);
> +
> +int erdma_query_device(struct ibv_context *ctx,
> + const struct ibv_query_device_ex_input *input,
> + struct ibv_device_attr_ex *attr, size_t attr_size); int
> +erdma_query_port(struct ibv_context *ctx, uint8_t port, struct
> +ibv_port_attr *attr);
> +
> +struct ibv_mr *erdma_reg_mr(struct ibv_pd *pd, void *addr, size_t len,
> + uint64_t hca_va, int access);
> +int erdma_dereg_mr(struct verbs_mr *vmr);
> +
> +struct ibv_qp *erdma_create_qp(struct ibv_pd *pd, struct
> +ibv_qp_init_attr *attr); int erdma_modify_qp(struct ibv_qp *base_qp,
> +struct ibv_qp_attr *attr, int attr_mask); int erdma_query_qp(struct ibv_qp
> *qp, struct ibv_qp_attr *attr,
> + int attr_mask, struct ibv_qp_init_attr *init_attr); int
> +erdma_post_send(struct ibv_qp *base_qp, struct ibv_send_wr *wr, struct
> +ibv_send_wr **bad_wr); int erdma_post_recv(struct ibv_qp *base_qp,
> +struct ibv_recv_wr *wr, struct ibv_recv_wr **bad_wr); int
> +erdma_destroy_qp(struct ibv_qp *base_qp);
> +
> +void erdma_free_context(struct ibv_context *ibv_ctx);
> +
> +struct ibv_cq *erdma_create_cq(struct ibv_context *ctx, int num_cqe,
> + struct ibv_comp_channel *channel, int
> comp_vector); int
> +erdma_destroy_cq(struct ibv_cq *base_cq); int erdma_notify_cq(struct
> +ibv_cq *ibcq, int solicited); void erdma_cq_event(struct ibv_cq *ibcq);
> +int erdma_poll_cq(struct ibv_cq *ibcq, int num_entries, struct ibv_wc
> +*wc);
> +
> +#endif
> --
> 2.27.0
