On Tue, Apr 24, 2018 at 10:55:22PM +0530, Raju Rangoju wrote: > Reference for the corresponding kernel mode srq submission > > https://www.spinics.net/lists/linux-rdma/msg63695.html > https://www.spinics.net/lists/linux-rdma/msg63696.html > https://www.spinics.net/lists/linux-rdma/msg63697.html > > This patch adds necessary changes for supporting srq feature to chelsio > adapters. > > Signed-off-by: Raju Rangoju <rajur@xxxxxxxxxxx> > Reviewed-by: Steve Wise <swise@xxxxxxxxxxxxxxxxxxxxx> > kernel-headers/rdma/cxgb4-abi.h | 28 ++++- > providers/cxgb4/cq.c | 168 +++++++++++++++++++++++++++-- > providers/cxgb4/cxgb4-abi.h | 5 +- > providers/cxgb4/dev.c | 2 + > providers/cxgb4/libcxgb4.h | 22 ++++ > providers/cxgb4/qp.c | 228 ++++++++++++++++++++++++++++++--------- > providers/cxgb4/t4.h | 164 +++++++++++++++++++++++++++- > providers/cxgb4/t4_regs.h | 4 + > providers/cxgb4/t4fw_api.h | 2 + > providers/cxgb4/t4fw_ri_api.h | 20 ++++ > providers/cxgb4/verbs.c | 232 ++++++++++++++++++++++++++++++++-------- > 11 files changed, 761 insertions(+), 114 deletions(-) > > diff --git a/kernel-headers/rdma/cxgb4-abi.h b/kernel-headers/rdma/cxgb4-abi.h > index 1fefd014..55959158 100644 > +++ b/kernel-headers/rdma/cxgb4-abi.h > @@ -44,6 +44,16 @@ > * In particular do not use pointer types -- pass pointers in __aligned_u64 > * instead. > */ > + > +enum { > + C4IW_64B_CQE = (1 << 0) > +}; > + > +struct c4iw_create_cq { > + __u32 flags; > + __u32 reserved; > +}; > + > struct c4iw_create_cq_resp { > __aligned_u64 key; > __aligned_u64 gts_key; > @@ -51,7 +61,7 @@ struct c4iw_create_cq_resp { > __u32 cqid; > __u32 size; > __u32 qid_mask; > - __u32 reserved; /* explicit padding (optional for i386) */ > + __u32 flags; > }; > > enum { > @@ -84,4 +94,20 @@ struct c4iw_alloc_pd_resp { > __u32 pdid; > }; > > +struct c4iw_create_srq_resp { > + //struct ibv_create_srq_resp ibv_resp; > + __u64 srq_key; > + __u64 srq_db_gts_key; > + __u64 srq_memsize; These are supposed to be __aligned_u64 in the kernel patches > + __u32 srqid; > + __u32 srq_size; > + __u32 rqt_abs_idx; > + __u32 qid_mask; > + __u32 flags; > +}; > + > +enum { > + T4_SRQ_LIMIT_SUPPORT = (1<<0), /* HW supports SRQ_LIMIT_REACHED event */ () is unneeded The above need to be fixed in the kernel patches. Changes to kernel-headers/rdma/* need to be done via the kernel-headers/update script which creates a dedicate patch for the kernel header sync. > diff --git a/providers/cxgb4/cxgb4-abi.h b/providers/cxgb4/cxgb4-abi.h > index 14fe8feb..63945719 100644 > +++ b/providers/cxgb4/cxgb4-abi.h > @@ -58,12 +58,13 @@ struct c4iw_create_qp_resp_v0 _c4iw_create_qp_resp_v0; > DECLARE_DRV_CMD(uc4iw_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD, > empty, c4iw_alloc_pd_resp); > DECLARE_DRV_CMD(uc4iw_create_cq, IB_USER_VERBS_CMD_CREATE_CQ, > - empty, c4iw_create_cq_resp); > + c4iw_create_cq, c4iw_create_cq_resp); How are you handling compatibility here? > DECLARE_DRV_CMD(uc4iw_create_qp, IB_USER_VERBS_CMD_CREATE_QP, > empty, c4iw_create_qp_resp); > DECLARE_DRV_CMD(uc4iw_create_qp_v0, IB_USER_VERBS_CMD_CREATE_QP, > empty, c4iw_create_qp_resp_v0); > DECLARE_DRV_CMD(uc4iw_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT, > empty, c4iw_alloc_ucontext_resp); > - > +DECLARE_DRV_CMD(uc4iw_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ, > + empty, c4iw_create_srq_resp); This list is sorted by IB_USER_VERBS_CMD_xxx > +struct t4_srq { > + union t4_recv_wr *queue; > + struct t4_swrqe *sw_rq; > + volatile u32 *udb; use of volatile is probably wrong, what is this? > +static inline void t4_ring_srq_db(struct t4_srq *srq, u16 inc, u8 len16, > + union t4_recv_wr *wqe) > +{ > + mmio_wc_start(); > + if (inc == 1 && srq->wc_reg_available) { > + PDBG("%s: WC srq->pidx = %d; len16=%d\n", > + __func__, srq->pidx, len16); > + copy_wqe_to_udb(srq->udb + 14, wqe); > + } else { > + PDBG("%s: DB srq->pidx = %d; len16=%d\n", > + __func__, srq->pidx, len16); > + writel(QID_V(srq->bar2_qid) | PIDX_T5_V(inc), srq->udb); > + } > + mmio_flush_writes(); > + return; > +} Are you sure that shouldn't be in a lock? > diff --git a/providers/cxgb4/verbs.c b/providers/cxgb4/verbs.c > index 3c493697..435bb238 100644 > +++ b/providers/cxgb4/verbs.c > @@ -168,6 +168,7 @@ int c4iw_dereg_mr(struct ibv_mr *mr) > struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, > struct ibv_comp_channel *channel, int comp_vector) > { > + struct uc4iw_create_cq cmd; > struct uc4iw_create_cq_resp resp; > struct c4iw_cq *chp; > struct c4iw_dev *dev = to_c4iw_dev(context->device); > @@ -178,16 +179,22 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, > return NULL; > } > > - resp.reserved = 0; > + resp.flags = 0; > + memset(&cmd, 0, sizeof cmd); > + cmd.flags = C4IW_64B_CQE; > + > ret = ibv_cmd_create_cq(context, cqe, channel, comp_vector, > - &chp->ibv_cq, NULL, 0, > + &chp->ibv_cq, &cmd.ibv_cmd, sizeof(cmd), > &resp.ibv_resp, sizeof resp); > if (ret) > goto err1; > > - if (resp.reserved) > - PDBG("%s c4iw_create_cq_resp reserved field modified by kernel\n", > - __FUNCTION__); > + if (!resp.flags) { > + fprintf(stderr, "libcxgb4 FATAL ERROR: downlevel iw_cxgb4 " > + "module. Cannot support RDMA with this driver/lib " > + "combination. Update your drivers!\n"); Oh, no, we don't do this. You have to support old kernels too. Jason -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html
