This patch adds user mode t4_srq structures and support functions
- Also adds srq resp structures and helper functions
Signed-off-by: Raju Rangoju <rajur@xxxxxxxxxxx>
Reviewed-by: Steve Wise <swise@xxxxxxxxxxxxxxxxxxxxx>
---
providers/cxgb4/cxgb4-abi.h | 2 +
providers/cxgb4/libcxgb4.h | 22 ++++++
providers/cxgb4/t4.h | 155 +++++++++++++++++++++++++++++++++++++++++-
providers/cxgb4/t4_regs.h | 4 ++
providers/cxgb4/t4fw_api.h | 2 +
providers/cxgb4/t4fw_ri_api.h | 19 ++++++
6 files changed, 201 insertions(+), 3 deletions(-)
diff --git a/providers/cxgb4/cxgb4-abi.h b/providers/cxgb4/cxgb4-abi.h
index 67d44512..d5140660 100644
--- a/providers/cxgb4/cxgb4-abi.h
+++ b/providers/cxgb4/cxgb4-abi.h
@@ -59,6 +59,8 @@ DECLARE_DRV_CMD(uc4iw_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
empty, c4iw_alloc_pd_resp);
DECLARE_DRV_CMD(uc4iw_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
c4iw_create_cq, c4iw_create_cq_resp);
+DECLARE_DRV_CMD(uc4iw_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
+ empty, c4iw_create_srq_resp);
DECLARE_DRV_CMD(uc4iw_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
empty, c4iw_create_qp_resp);
DECLARE_DRV_CMD(uc4iw_create_qp_v0, IB_USER_VERBS_CMD_CREATE_QP,
diff --git a/providers/cxgb4/libcxgb4.h b/providers/cxgb4/libcxgb4.h
index d6e0d785..e3cac866 100644
--- a/providers/cxgb4/libcxgb4.h
+++ b/providers/cxgb4/libcxgb4.h
@@ -59,6 +59,7 @@ struct c4iw_dev {
struct c4iw_qp **qpid2ptr;
int max_cq;
struct c4iw_cq **cqid2ptr;
+ struct list_head srq_list;
pthread_spinlock_t lock;
struct list_node list;
int abi_version;
@@ -117,11 +118,29 @@ struct c4iw_qp {
struct t4_wq wq;
pthread_spinlock_t lock;
int sq_sig_all;
+ struct c4iw_srq *srq;
};
#define to_c4iw_xxx(xxx, type) \
container_of(ib##xxx, struct c4iw_##type, ibv_##xxx)
+struct c4iw_srq {
+ struct ibv_srq ibv_srq;
+ int type; /* must be 2nd in this struct */
+ struct c4iw_dev *rhp;
+ struct t4_srq wq;
+ struct list_node list;
+ pthread_spinlock_t lock;
+ uint32_t srq_limit;
+ int armed;
+ __u32 flags;
+};
+
+static inline struct c4iw_srq *to_c4iw_srq(struct ibv_srq *ibsrq)
+{
+ return to_c4iw_xxx(srq, srq);
+}
+
static inline struct c4iw_dev *to_c4iw_dev(struct ibv_device *ibdev)
{
return container_of(ibdev, struct c4iw_dev, ibv_dev.device);
@@ -201,6 +220,7 @@ int c4iw_destroy_srq(struct ibv_srq *srq);
int c4iw_post_srq_recv(struct ibv_srq *ibsrq,
struct ibv_recv_wr *wr,
struct ibv_recv_wr **bad_wr);
+int c4iw_query_srq(struct ibv_srq *ibsrq, struct ibv_srq_attr *attr);
struct ibv_qp *c4iw_create_qp(struct ibv_pd *pd,
struct ibv_qp_init_attr *attr);
@@ -229,6 +249,8 @@ void c4iw_flush_hw_cq(struct c4iw_cq *chp, struct c4iw_qp *flush_qhp);
int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
void c4iw_flush_sq(struct c4iw_qp *qhp);
void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
+void c4iw_copy_wr_to_srq(struct t4_srq *srq, union t4_recv_wr *wqe, u8 len16);
+void c4iw_flush_srqidx(struct c4iw_qp *qhp, u32 srqidx);
#define FW_MAJ 0
#define FW_MIN 0
diff --git a/providers/cxgb4/t4.h b/providers/cxgb4/t4.h
index 38500a5d..67f411d9 100644
--- a/providers/cxgb4/t4.h
+++ b/providers/cxgb4/t4.h
@@ -102,10 +102,12 @@ struct t4_status_page {
__be16 pidx;
u8 qp_err; /* flit 1 - sw owns */
u8 db_off;
- u8 pad;
+ u8 pad[2];
u16 host_wq_pidx;
u16 host_cidx;
u16 host_pidx;
+ u16 pad2;
+ u32 srqidx;
};
#define T4_EQ_ENTRY_SIZE 64
@@ -297,6 +299,7 @@ union t4_cqe {
/* used for RQ completion processing */
#define CQE_WRID_STAG(x) (be32toh((x)->u.rcqe.stag))
#define CQE_WRID_MSN(x) (be32toh((x)->u.rcqe.msn))
+#define CQE_ABS_RQE_IDX(x) (be32toh((x)->u.srcqe.abs_rqe_idx))
/* used for SQ completion processing */
#define CQE_WRID_SQ_IDX(x) (x)->u.scqe.cidx
@@ -398,6 +401,8 @@ struct t4_wq {
int error;
int flushed;
u8 *db_offp;
+ u8 *qp_errp;
+ u32 *srqidxp;
};
static inline int t4_rqes_posted(struct t4_wq *wq)
@@ -443,6 +448,138 @@ static inline void t4_rq_consume(struct t4_wq *wq)
wq->rq.queue[wq->rq.size].status.host_cidx = wq->rq.cidx;
}
+struct t4_srq_pending_wr {
+ u64 wr_id;
+ union t4_recv_wr wqe;
+ u8 len16;
+};
+
+struct t4_srq {
+ union t4_recv_wr *queue;
+ struct t4_swrqe *sw_rq;
+ volatile u32 *udb;
+ size_t memsize;
+ u32 qid;
+ u32 bar2_qid;
+ u32 msn;
+ u32 rqt_hwaddr;
+ u32 rqt_abs_idx;
+ u16 in_use;
+ u16 size;
+ u16 cidx;
+ u16 pidx;
+ u16 wq_pidx;
+ int wc_reg_available;
+ struct t4_srq_pending_wr *pending_wrs;
+ u16 pending_cidx;
+ u16 pending_pidx;
+ u16 pending_in_use;
+ u16 ooo_count;
+};
+
+static inline u32 t4_srq_avail(struct t4_srq *srq)
+{
+ return srq->size - 1 - srq->in_use;
+}
+
+static inline int t4_srq_empty(struct t4_srq *srq)
+{
+ return srq->in_use == 0;
+}
+
+static inline int t4_srq_cidx_at_end(struct t4_srq *srq)
+{
+ assert(srq->cidx != srq->pidx);
+ if (srq->cidx < srq->pidx)
+ return srq->cidx == (srq->pidx - 1);
+ else
+ return srq->cidx == (srq->size - 1) && srq->pidx == 0;
+}
+
+static inline int t4_srq_wrs_pending(struct t4_srq *srq)
+{
+ return srq->pending_cidx != srq->pending_pidx;
+}
+
+static inline void t4_srq_produce(struct t4_srq *srq, u8 len16)
+{
+ srq->in_use++;
+ assert(srq->in_use < srq->size);
+ if (++srq->pidx == srq->size)
+ srq->pidx = 0;
+ assert(srq->cidx != srq->pidx); /* overflow */
+ srq->wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE);
+ if (srq->wq_pidx >= srq->size * T4_RQ_NUM_SLOTS)
+ srq->wq_pidx %= srq->size * T4_RQ_NUM_SLOTS;
+ srq->queue[srq->size].status.host_pidx = srq->pidx;
+}
+
+static inline void t4_srq_produce_pending_wr(struct t4_srq *srq)
+{
+ srq->pending_in_use++;
+ srq->in_use++;
+ assert(srq->pending_in_use < srq->size);
+ assert(srq->in_use < srq->size);
+ assert(srq->pending_pidx < srq->size);
+ if (++srq->pending_pidx == srq->size)
+ srq->pending_pidx = 0;
+}
+
+static inline void t4_srq_consume_pending_wr(struct t4_srq *srq)
+{
+ assert(srq->pending_in_use > 0);
+ srq->pending_in_use--;
+ assert(srq->in_use > 0);
+ srq->in_use--;
+ if (++srq->pending_cidx == srq->size)
+ srq->pending_cidx = 0;
+ assert((srq->pending_cidx != srq->pending_pidx) || srq->pending_in_use == 0);
+}
+
+static inline void t4_srq_produce_ooo(struct t4_srq *srq)
+{
+ assert(srq->in_use > 0);
+ srq->in_use--;
+ srq->ooo_count++;
+ assert(srq->ooo_count < srq->size);
+}
+
+static inline void t4_srq_consume_ooo(struct t4_srq *srq)
+{
+ srq->cidx++;
+ if (srq->cidx == srq->size)
+ srq->cidx = 0;
+ srq->queue[srq->size].status.host_cidx = srq->cidx;
+ assert(srq->ooo_count > 0);
+ srq->ooo_count--;
+}
+
+static inline void t4_srq_consume(struct t4_srq *srq)
+{
+ assert(srq->in_use > 0);
+ srq->in_use--;
+ if (++srq->cidx == srq->size)
+ srq->cidx = 0;
+ assert((srq->cidx != srq->pidx) || srq->in_use == 0);
+ srq->queue[srq->size].status.host_cidx = srq->cidx;
+}
+
+static inline int t4_wq_in_error(struct t4_wq *wq)
+{
+ return wq->error || wq->rq.queue[wq->rq.size].status.qp_err;
+}
+
+static inline u32 t4_wq_srqidx(struct t4_wq *wq)
+{
+ u32 srqidx;
+
+ if (!wq->srqidxp)
+ return 0;
+ srqidx = *wq->srqidxp;
+ wq->srqidxp = 0;
+ return srqidx;
+}
+
static inline int t4_sq_empty(struct t4_wq *wq)
{
return wq->sq.in_use == 0;
@@ -586,9 +723,21 @@ static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t4, u8 len16,
writel(QID_V(wq->rq.qid & wq->qid_mask) | PIDX_V(inc), wq->rq.udb);
}
-static inline int t4_wq_in_error(struct t4_wq *wq)
+static inline void t4_ring_srq_db(struct t4_srq *srq, u16 inc, u8 len16,
+ union t4_recv_wr *wqe)
{
- return wq->error || wq->rq.queue[wq->rq.size].status.qp_err;
+ mmio_wc_start();
+ if (t5_en_wc && inc == 1 && srq->wc_reg_available) {
+ PDBG("%s: WC srq->pidx = %d; len16=%d\n",
+ __func__, srq->pidx, len16);
+ copy_wqe_to_udb(srq->udb + 14, wqe);
+ } else {
+ PDBG("%s: DB srq->pidx = %d; len16=%d\n",
+ __func__, srq->pidx, len16);
+ writel(QID_V(srq->bar2_qid) | PIDX_T5_V(inc), srq->udb);
+ }
+ mmio_flush_writes();
+ return;
}
static inline void t4_set_wq_in_error(struct t4_wq *wq)
diff --git a/providers/cxgb4/t4_regs.h b/providers/cxgb4/t4_regs.h
index 9fea255c..73516226 100644
--- a/providers/cxgb4/t4_regs.h
+++ b/providers/cxgb4/t4_regs.h
@@ -1436,6 +1436,10 @@
#define TP_MIB_INDEX_A 0x7e50
#define TP_MIB_DATA_A 0x7e54
#define TP_INT_CAUSE_A 0x7e74
+#define SRQTABLEPERR_S 1
+#define SRQTABLEPERR_V(x) ((x) << SRQTABLEPERR_S)
+#define SRQTABLEPERR_F SRQTABLEPERR_V(1U)
+
#define FLMTXFLSTEMPTY_S 30
#define FLMTXFLSTEMPTY_V(x) ((x) << FLMTXFLSTEMPTY_S)
diff --git a/providers/cxgb4/t4fw_api.h b/providers/cxgb4/t4fw_api.h
index 49bbca18..fabb16c7 100644
--- a/providers/cxgb4/t4fw_api.h
+++ b/providers/cxgb4/t4fw_api.h
@@ -1152,6 +1152,8 @@ enum fw_params_param_pfvf {
FW_PARAMS_PARAM_PFVF_SQRQ_END = 0x16,
FW_PARAMS_PARAM_PFVF_CQ_START = 0x17,
FW_PARAMS_PARAM_PFVF_CQ_END = 0x18,
+ FW_PARAMS_PARAM_PFVF_SRQ_START = 0x19,
+ FW_PARAMS_PARAM_PFVF_SRQ_END = 0x1A,
FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH = 0x20,
FW_PARAMS_PARAM_PFVF_VIID = 0x24,
FW_PARAMS_PARAM_PFVF_CPMASK = 0x25,
diff --git a/providers/cxgb4/t4fw_ri_api.h b/providers/cxgb4/t4fw_ri_api.h
index 1e266697..29e844f3 100644
--- a/providers/cxgb4/t4fw_ri_api.h
+++ b/providers/cxgb4/t4fw_ri_api.h
@@ -263,6 +263,7 @@ enum fw_ri_res_type {
FW_RI_RES_TYPE_SQ,
FW_RI_RES_TYPE_RQ,
FW_RI_RES_TYPE_CQ,
+ FW_RI_RES_TYPE_SRQ,
};
enum fw_ri_res_op {
@@ -296,6 +297,20 @@ struct fw_ri_res {
__be32 r6_lo;
__be64 r7;
} cq;
+ struct fw_ri_res_srq {
+ __u8 restype;
+ __u8 op;
+ __be16 r3;
+ __be32 eqid;
+ __be32 r4[2];
+ __be32 fetchszm_to_iqid;
+ __be32 dcaen_to_eqsize;
+ __be64 eqaddr;
+ __be32 srqid;
+ __be32 pdid;
+ __be32 hwsrqsize;
+ __be32 hwsrqaddr;
+ } srq;
} u;
};
@@ -695,6 +710,10 @@ enum fw_ri_init_p2ptype {
FW_RI_INIT_P2PTYPE_DISABLED = 0xf,
};
+enum fw_ri_init_rqeqid_srq {
+ FW_RI_INIT_RQEQID_SRQ = 1 << 31,
+};
+
struct fw_ri_wr {
__be32 op_compl;
__be32 flowid_len16;
--
2.13.0
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
