From: Wenpeng Liang <liangwenpeng@xxxxxxxxxx>
Implement the ibv_create_srq_ex verbs to support for XRC and more
features in the future.
Signed-off-by: Wenpeng Liang <liangwenpeng@xxxxxxxxxx>
Signed-off-by: Weihang Li <liweihang@xxxxxxxxxx>
---
providers/hns/hns_roce_u.c | 23 +++--
providers/hns/hns_roce_u.h | 21 +++-
providers/hns/hns_roce_u_abi.h | 3 +
providers/hns/hns_roce_u_verbs.c | 212 +++++++++++++++++++++++++++++++--------
4 files changed, 208 insertions(+), 51 deletions(-)
diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c
index 6def926..d103808b 100644
--- a/providers/hns/hns_roce_u.c
+++ b/providers/hns/hns_roce_u.c
@@ -81,6 +81,7 @@ static const struct verbs_context_ops hns_common_ops = {
.reg_mr = hns_roce_u_reg_mr,
.rereg_mr = hns_roce_u_rereg_mr,
.create_srq = hns_roce_u_create_srq,
+ .create_srq_ex = hns_roce_u_create_srq_ex,
.modify_srq = hns_roce_u_modify_srq,
.query_srq = hns_roce_u_query_srq,
.destroy_srq = hns_roce_u_destroy_srq,
@@ -110,21 +111,29 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev,
&resp.ibv_resp, sizeof(resp)))
goto err_free;
+ if (!resp.cqe_size)
+ context->cqe_size = HNS_ROCE_CQE_SIZE;
+ else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE)
+ context->cqe_size = resp.cqe_size;
+ else
+ context->cqe_size = HNS_ROCE_V3_CQE_SIZE;
+
context->num_qps = resp.qp_tab_size;
+ context->num_srqs = resp.srq_tab_size;
+
context->qp_table_shift = ffs(context->num_qps) - 1 -
HNS_ROCE_QP_TABLE_BITS;
context->qp_table_mask = (1 << context->qp_table_shift) - 1;
-
pthread_mutex_init(&context->qp_table_mutex, NULL);
for (i = 0; i < HNS_ROCE_QP_TABLE_SIZE; ++i)
context->qp_table[i].refcnt = 0;
- if (!resp.cqe_size)
- context->cqe_size = HNS_ROCE_CQE_SIZE;
- else if (resp.cqe_size <= HNS_ROCE_V3_CQE_SIZE)
- context->cqe_size = resp.cqe_size;
- else
- context->cqe_size = HNS_ROCE_V3_CQE_SIZE;
+ context->srq_table_shift = ffs(context->num_srqs) - 1 -
+ HNS_ROCE_SRQ_TABLE_BITS;
+ context->srq_table_mask = (1 << context->srq_table_shift) - 1;
+ pthread_mutex_init(&context->srq_table_mutex, NULL);
+ for (i = 0; i < HNS_ROCE_SRQ_TABLE_SIZE; ++i)
+ context->srq_table[i].refcnt = 0;
if (hns_roce_u_query_device(&context->ibv_ctx.context, NULL,
container_of(&dev_attrs,
diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h
index 97e5d54..de6d0f3 100644
--- a/providers/hns/hns_roce_u.h
+++ b/providers/hns/hns_roce_u.h
@@ -105,6 +105,9 @@ enum {
HNS_ROCE_QP_TABLE_SIZE = 1 << HNS_ROCE_QP_TABLE_BITS,
};
+#define HNS_ROCE_SRQ_TABLE_BITS 8
+#define HNS_ROCE_SRQ_TABLE_SIZE BIT(HNS_ROCE_SRQ_TABLE_BITS)
+
/* operation type list */
enum {
/* rq&srq operation */
@@ -153,13 +156,20 @@ struct hns_roce_context {
struct hns_roce_qp **table;
int refcnt;
} qp_table[HNS_ROCE_QP_TABLE_SIZE];
-
pthread_mutex_t qp_table_mutex;
-
int num_qps;
int qp_table_shift;
int qp_table_mask;
+ struct {
+ struct hns_roce_srq **table;
+ int refcnt;
+ } srq_table[HNS_ROCE_SRQ_TABLE_SIZE];
+ pthread_mutex_t srq_table_mutex;
+ int num_srqs;
+ int srq_table_shift;
+ int srq_table_mask;
+
struct hns_roce_db_page *db_list[HNS_ROCE_DB_TYPE_NUM];
pthread_mutex_t db_list_mutex;
@@ -376,10 +386,15 @@ void hns_roce_u_cq_event(struct ibv_cq *cq);
struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
struct ibv_srq_init_attr *srq_init_attr);
+struct ibv_srq *hns_roce_u_create_srq_ex(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *attr);
int hns_roce_u_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
int srq_attr_mask);
int hns_roce_u_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
-int hns_roce_u_destroy_srq(struct ibv_srq *srq);
+struct hns_roce_srq *hns_roce_find_srq(struct hns_roce_context *ctx,
+ uint32_t srqn);
+int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq);
+
struct ibv_qp *hns_roce_u_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *attr);
struct ibv_qp *
diff --git a/providers/hns/hns_roce_u_abi.h b/providers/hns/hns_roce_u_abi.h
index 4341207..e56f9d3 100644
--- a/providers/hns/hns_roce_u_abi.h
+++ b/providers/hns/hns_roce_u_abi.h
@@ -53,4 +53,7 @@ DECLARE_DRV_CMD(hns_roce_create_qp_ex, IB_USER_VERBS_EX_CMD_CREATE_QP,
DECLARE_DRV_CMD(hns_roce_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp);
+DECLARE_DRV_CMD(hns_roce_create_srq_ex, IB_USER_VERBS_CMD_CREATE_XSRQ,
+ hns_roce_ib_create_srq, hns_roce_ib_create_srq_resp);
+
#endif /* _HNS_ROCE_U_ABI_H */
diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c
index 3329812..e51b9d5 100644
--- a/providers/hns/hns_roce_u_verbs.c
+++ b/providers/hns/hns_roce_u_verbs.c
@@ -391,6 +391,46 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq)
return ret;
}
+static int hns_roce_store_srq(struct hns_roce_context *ctx,
+ struct hns_roce_srq *srq)
+{
+ int tind = (srq->srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+ if (!ctx->srq_table[tind].refcnt) {
+ ctx->srq_table[tind].table =
+ calloc(ctx->srq_table_mask + 1,
+ sizeof(struct hns_roce_srq *));
+ if (!ctx->srq_table[tind].table)
+ return -ENOMEM;
+ }
+
+ ++ctx->srq_table[tind].refcnt;
+ ctx->srq_table[tind].table[srq->srqn & ctx->srq_table_mask] = srq;
+
+ return 0;
+}
+
+struct hns_roce_srq *hns_roce_find_srq(struct hns_roce_context *ctx,
+ uint32_t srqn)
+{
+ int tind = (srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+ if (ctx->srq_table[tind].refcnt)
+ return ctx->srq_table[tind].table[srqn & ctx->srq_table_mask];
+ else
+ return NULL;
+}
+
+static void hns_roce_clear_srq(struct hns_roce_context *ctx, uint32_t srqn)
+{
+ int tind = (srqn & (ctx->num_srqs - 1)) >> ctx->srq_table_shift;
+
+ if (!--ctx->srq_table[tind].refcnt)
+ free(ctx->srq_table[tind].table);
+ else
+ ctx->srq_table[tind].table[srqn & ctx->srq_table_mask] = NULL;
+}
+
static int hns_roce_create_idx_que(struct hns_roce_srq *srq)
{
struct hns_roce_idx_que *idx_que = &srq->idx_que;
@@ -442,18 +482,71 @@ static int hns_roce_alloc_srq_buf(struct hns_roce_srq *srq)
return 0;
}
-struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
- struct ibv_srq_init_attr *init_attr)
+static int hns_roce_verify_srq(struct hns_roce_context *context,
+ struct ibv_srq_init_attr_ex *init_attr)
+{
+ if (init_attr->srq_type != IBV_SRQT_BASIC &&
+ init_attr->srq_type != IBV_SRQT_XRC)
+ return -EINVAL;
+
+ if (!init_attr->attr.max_wr || !init_attr->attr.max_sge ||
+ init_attr->attr.max_wr > context->max_srq_wr ||
+ init_attr->attr.max_sge > context->max_srq_sge)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int exec_srq_create_cmd(struct ibv_context *context,
+ struct hns_roce_srq *srq,
+ struct ibv_srq_init_attr_ex *init_attr)
+{
+ struct hns_roce_create_srq_resp resp = {};
+ struct hns_roce_create_srq_ex cmd_ex = {};
+ struct hns_roce_ib_create_srq *cmd_drv;
+ struct hns_roce_create_srq cmd = {};
+ bool is_basic_srq;
+ int ret;
+
+ is_basic_srq = (init_attr->srq_type == IBV_SRQT_BASIC) ||
+ (!(init_attr->comp_mask & IBV_SRQ_INIT_ATTR_TYPE));
+
+ if (is_basic_srq)
+ cmd_drv = &cmd.drv_payload;
+ else
+ cmd_drv = &cmd_ex.drv_payload;
+
+ cmd_drv->buf_addr = (uintptr_t)srq->buf.buf;
+ cmd_drv->que_addr = (uintptr_t)srq->idx_que.buf.buf;
+ cmd_drv->db_addr = (uintptr_t)srq->db;
+
+ if (is_basic_srq)
+ ret = ibv_cmd_create_srq(init_attr->pd, &srq->verbs_srq.srq,
+ (struct ibv_srq_init_attr *)init_attr,
+ &cmd.ibv_cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp));
+ else
+ ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, init_attr,
+ &cmd_ex.ibv_cmd, sizeof(cmd_ex),
+ &resp.ibv_resp, sizeof(resp));
+
+ if (ret)
+ return ret;
+
+ srq->srqn = resp.srqn;
+
+ return 0;
+}
+
+static struct ibv_srq *create_srq(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *init_attr)
{
- struct hns_roce_context *ctx = to_hr_ctx(pd->context);
- struct hns_roce_create_srq_resp resp;
- struct hns_roce_create_srq cmd;
+ struct hns_roce_context *ctx = to_hr_ctx(context);
+ struct ibv_srq_attr *attr = &init_attr->attr;
struct hns_roce_srq *srq;
int ret;
- if (!init_attr->attr.max_wr || !init_attr->attr.max_sge ||
- init_attr->attr.max_wr > ctx->max_srq_wr ||
- init_attr->attr.max_sge > ctx->max_srq_sge)
+ if (hns_roce_verify_srq(ctx, init_attr))
return NULL;
srq = calloc(1, sizeof(*srq));
@@ -461,48 +554,52 @@ struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
return NULL;
if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
- goto out;
+ goto err_free_srq;
- if (to_hr_dev(pd->context->device)->hw_version == HNS_ROCE_HW_VER2)
+ if (to_hr_dev(context->device)->hw_version == HNS_ROCE_HW_VER2)
srq->rsv_sge = 1;
- srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1);
- srq->max_gs = roundup_pow_of_two(init_attr->attr.max_sge + srq->rsv_sge);
- init_attr->attr.max_sge = srq->max_gs;
+ srq->wqe_cnt = roundup_pow_of_two(attr->max_wr + 1);
+ srq->max_gs = roundup_pow_of_two(attr->max_sge + srq->rsv_sge);
+ attr->max_sge = srq->max_gs;
ret = hns_roce_create_idx_que(srq);
if (ret)
- goto out;
+ goto err_free_srq;
ret = hns_roce_alloc_srq_buf(srq);
if (ret)
goto err_idx_que;
- srq->db = hns_roce_alloc_db(to_hr_ctx(pd->context),
- HNS_ROCE_QP_TYPE_DB);
+ srq->db = hns_roce_alloc_db(ctx, HNS_ROCE_QP_TYPE_DB);
if (!srq->db)
goto err_srq_buf;
*(srq->db) = 0;
- cmd.buf_addr = (uintptr_t)srq->buf.buf;
- cmd.que_addr = (uintptr_t)srq->idx_que.buf.buf;
- cmd.db_addr = (uintptr_t)srq->db;
- ret = ibv_cmd_create_srq(pd, &srq->verbs_srq.srq, init_attr,
- &cmd.ibv_cmd, sizeof(cmd), &resp.ibv_resp,
- sizeof(resp));
+ pthread_mutex_lock(&ctx->srq_table_mutex);
+
+ ret = exec_srq_create_cmd(context, srq, init_attr);
if (ret)
goto err_srq_db;
- srq->srqn = resp.srqn;
- srq->max_gs = init_attr->attr.max_sge;
- init_attr->attr.max_sge =
- min(init_attr->attr.max_sge - srq->rsv_sge, ctx->max_srq_sge);
+ ret = hns_roce_store_srq(ctx, srq);
+ if (ret)
+ goto err_destroy_srq;
+
+ pthread_mutex_unlock(&ctx->srq_table_mutex);
+
+ srq->max_gs = attr->max_sge;
+ attr->max_sge = min(attr->max_sge - srq->rsv_sge, ctx->max_srq_sge);
return &srq->verbs_srq.srq;
+err_destroy_srq:
+ ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
+
err_srq_db:
- hns_roce_free_db(to_hr_ctx(pd->context), srq->db, HNS_ROCE_QP_TYPE_DB);
+ pthread_mutex_unlock(&ctx->srq_table_mutex);
+ hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
err_srq_buf:
free(srq->wrid);
@@ -511,11 +608,35 @@ err_srq_buf:
err_idx_que:
free(srq->idx_que.bitmap);
hns_roce_free_buf(&srq->idx_que.buf);
-out:
+
+err_free_srq:
free(srq);
+
return NULL;
}
+struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd,
+ struct ibv_srq_init_attr *attr)
+{
+ struct ibv_srq_init_attr_ex attrx = {};
+ struct ibv_srq *srq;
+
+ memcpy(&attrx, attr, sizeof(*attr));
+ attrx.pd = pd;
+
+ srq = create_srq(pd->context, &attrx);
+ if (srq)
+ memcpy(attr, &attrx, sizeof(*attr));
+
+ return srq;
+}
+
+struct ibv_srq *hns_roce_u_create_srq_ex(struct ibv_context *context,
+ struct ibv_srq_init_attr_ex *attr)
+{
+ return create_srq(context, attr);
+}
+
int hns_roce_u_modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
int srq_attr_mask)
{
@@ -536,21 +657,30 @@ int hns_roce_u_query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr)
return ret;
}
-int hns_roce_u_destroy_srq(struct ibv_srq *srq)
+int hns_roce_u_destroy_srq(struct ibv_srq *ibv_srq)
{
+ struct hns_roce_context *ctx = to_hr_ctx(ibv_srq->context);
+ struct hns_roce_srq *srq = to_hr_srq(ibv_srq);
int ret;
- ret = ibv_cmd_destroy_srq(srq);
- if (ret)
+ pthread_mutex_lock(&ctx->srq_table_mutex);
+
+ ret = ibv_cmd_destroy_srq(ibv_srq);
+ if (ret) {
+ pthread_mutex_unlock(&ctx->srq_table_mutex);
return ret;
+ }
+
+ hns_roce_clear_srq(ctx, srq->srqn);
- hns_roce_free_db(to_hr_ctx(srq->context), to_hr_srq(srq)->db,
- HNS_ROCE_QP_TYPE_DB);
- hns_roce_free_buf(&to_hr_srq(srq)->buf);
- free(to_hr_srq(srq)->wrid);
- hns_roce_free_buf(&to_hr_srq(srq)->idx_que.buf);
- free(to_hr_srq(srq)->idx_que.bitmap);
- free(to_hr_srq(srq));
+ pthread_mutex_unlock(&ctx->srq_table_mutex);
+
+ hns_roce_free_db(ctx, srq->db, HNS_ROCE_QP_TYPE_DB);
+ hns_roce_free_buf(&srq->buf);
+ free(srq->wrid);
+ hns_roce_free_buf(&srq->idx_que.buf);
+ free(srq->idx_que.bitmap);
+ free(srq);
return 0;
}
@@ -868,9 +998,9 @@ static int qp_alloc_db(struct ibv_qp_init_attr_ex *attr, struct hns_roce_qp *qp,
return 0;
}
-static int hns_roce_store_qp(struct hns_roce_context *ctx, uint32_t qpn,
- struct hns_roce_qp *qp)
+static int hns_roce_store_qp(struct hns_roce_context *ctx, struct hns_roce_qp *qp)
{
+ uint32_t qpn = qp->verbs_qp.qp.qp_num;
uint32_t tind = (qpn & (ctx->num_qps - 1)) >> ctx->qp_table_shift;
pthread_mutex_lock(&ctx->qp_table_mutex);
@@ -987,7 +1117,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibv_ctx,
if (ret)
goto err_cmd;
- ret = hns_roce_store_qp(context, qp->verbs_qp.qp.qp_num, qp);
+ ret = hns_roce_store_qp(context, qp);
if (ret)
goto err_store;
--
2.8.1
