The shadow queue is used for wrid and flush wqe management.
The indices used in this queue are independent to what is
actually used by hardware. Thus, detaching the shadow queue
indices from hardware queue indices. This even more useful
when the hardware queue indices has alignment other than wqe
boundary.
Signed-off-by: Devesh Sharma <devesh.sharma@xxxxxxxxxxxx>
---
providers/bnxt_re/main.h | 20 ++++++
providers/bnxt_re/memory.h | 8 +--
providers/bnxt_re/verbs.c | 128 ++++++++++++++++++++++---------------
3 files changed, 101 insertions(+), 55 deletions(-)
diff --git a/providers/bnxt_re/main.h b/providers/bnxt_re/main.h
index 94d42958..ad660e1a 100644
--- a/providers/bnxt_re/main.h
+++ b/providers/bnxt_re/main.h
@@ -437,4 +437,24 @@ static inline void bnxt_re_change_cq_phase(struct bnxt_re_cq *cq)
if (!cq->cqq.head)
cq->phase = (~cq->phase & BNXT_RE_BCQE_PH_MASK);
}
+
+static inline void *bnxt_re_get_swqe(struct bnxt_re_joint_queue *jqq,
+ uint32_t *wqe_idx)
+{
+ if (wqe_idx)
+ *wqe_idx = jqq->start_idx;
+ return &jqq->swque[jqq->start_idx];
+}
+
+static inline void bnxt_re_jqq_mod_start(struct bnxt_re_joint_queue *jqq,
+ uint32_t idx)
+{
+ jqq->start_idx = jqq->swque[idx].next_idx;
+}
+
+static inline void bnxt_re_jqq_mod_last(struct bnxt_re_joint_queue *jqq,
+ uint32_t idx)
+{
+ jqq->last_idx = jqq->swque[idx].next_idx;
+}
#endif
diff --git a/providers/bnxt_re/memory.h b/providers/bnxt_re/memory.h
index 75564c43..5bcdef9a 100644
--- a/providers/bnxt_re/memory.h
+++ b/providers/bnxt_re/memory.h
@@ -97,14 +97,14 @@ static inline uint32_t bnxt_re_incr(uint32_t val, uint32_t max)
return (++val & (max - 1));
}
-static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que)
+static inline void bnxt_re_incr_tail(struct bnxt_re_queue *que, uint8_t cnt)
{
- que->tail = bnxt_re_incr(que->tail, que->depth);
+ que->tail = (que->tail + cnt) & (que->depth - 1);
}
-static inline void bnxt_re_incr_head(struct bnxt_re_queue *que)
+static inline void bnxt_re_incr_head(struct bnxt_re_queue *que, uint8_t cnt)
{
- que->head = bnxt_re_incr(que->head, que->depth);
+ que->head = (que->head + cnt) & (que->depth - 1);
}
#endif
diff --git a/providers/bnxt_re/verbs.c b/providers/bnxt_re/verbs.c
index 8507617b..9c3cfbd3 100644
--- a/providers/bnxt_re/verbs.c
+++ b/providers/bnxt_re/verbs.c
@@ -247,10 +247,12 @@ static uint8_t bnxt_re_poll_err_scqe(struct bnxt_re_qp *qp,
struct bnxt_re_wrid *swrid;
struct bnxt_re_psns *spsn;
struct bnxt_re_cq *scq;
- uint32_t head = sq->head;
uint8_t status;
+ uint32_t head;
scq = to_bnxt_re_cq(qp->ibvqp.send_cq);
+
+ head = qp->jsqq->last_idx;
cntx = to_bnxt_re_context(scq->ibvcq.context);
swrid = &qp->jsqq->swque[head];
spsn = swrid->psns;
@@ -267,7 +269,8 @@ static uint8_t bnxt_re_poll_err_scqe(struct bnxt_re_qp *qp,
BNXT_RE_PSNS_OPCD_MASK;
ibvwc->byte_len = 0;
- bnxt_re_incr_head(sq);
+ bnxt_re_incr_head(sq, swrid->slots);
+ bnxt_re_jqq_mod_last(qp->jsqq, head);
if (qp->qpst != IBV_QPS_ERR)
qp->qpst = IBV_QPS_ERR;
@@ -287,13 +290,14 @@ static uint8_t bnxt_re_poll_success_scqe(struct bnxt_re_qp *qp,
struct bnxt_re_queue *sq = qp->jsqq->hwque;
struct bnxt_re_wrid *swrid;
struct bnxt_re_psns *spsn;
- uint32_t head = sq->head;
uint8_t pcqe = false;
uint32_t cindx;
+ uint32_t head;
+ head = qp->jsqq->last_idx;
swrid = &qp->jsqq->swque[head];
spsn = swrid->psns;
- cindx = le32toh(scqe->con_indx);
+ cindx = le32toh(scqe->con_indx) & (qp->cap.max_swr - 1);
if (!(swrid->sig & IBV_SEND_SIGNALED)) {
*cnt = 0;
@@ -313,8 +317,10 @@ static uint8_t bnxt_re_poll_success_scqe(struct bnxt_re_qp *qp,
*cnt = 1;
}
- bnxt_re_incr_head(sq);
- if (sq->head != cindx)
+ bnxt_re_incr_head(sq, swrid->slots);
+ bnxt_re_jqq_mod_last(qp->jsqq, head);
+
+ if (qp->jsqq->last_idx != cindx)
pcqe = true;
return pcqe;
@@ -352,23 +358,29 @@ static void bnxt_re_release_srqe(struct bnxt_re_srq *srq, int tag)
static int bnxt_re_poll_err_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
struct bnxt_re_bcqe *hdr, void *cqe)
{
+ struct bnxt_re_context *cntx;
+ struct bnxt_re_wrid *swque;
struct bnxt_re_queue *rq;
+ uint8_t status, cnt = 0;
struct bnxt_re_cq *rcq;
- struct bnxt_re_context *cntx;
- uint8_t status;
+ uint32_t head = 0;
rcq = to_bnxt_re_cq(qp->ibvqp.recv_cq);
cntx = to_bnxt_re_context(rcq->ibvcq.context);
if (!qp->srq) {
rq = qp->jrqq->hwque;
- ibvwc->wr_id = qp->jrqq->swque[rq->head].wrid;
+ head = qp->jrqq->last_idx;
+ swque = &qp->jrqq->swque[head];
+ ibvwc->wr_id = swque->wrid;
+ cnt = swque->slots;
} else {
struct bnxt_re_srq *srq;
int tag;
srq = qp->srq;
rq = srq->srqq;
+ cnt = 1;
tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK;
ibvwc->wr_id = srq->srwrid[tag].wrid;
bnxt_re_release_srqe(srq, tag);
@@ -387,7 +399,10 @@ static int bnxt_re_poll_err_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
ibvwc->wc_flags = 0;
if (qp->qptyp == IBV_QPT_UD)
ibvwc->src_qp = 0;
- bnxt_re_incr_head(rq);
+
+ if (!qp->srq)
+ bnxt_re_jqq_mod_last(qp->jrqq, head);
+ bnxt_re_incr_head(rq, cnt);
if (!qp->srq) {
pthread_spin_lock(&cntx->fqlock);
@@ -417,14 +432,20 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
struct ibv_wc *ibvwc,
struct bnxt_re_bcqe *hdr, void *cqe)
{
- struct bnxt_re_queue *rq;
- struct bnxt_re_rc_cqe *rcqe;
uint8_t flags, is_imm, is_rdma;
+ struct bnxt_re_rc_cqe *rcqe;
+ struct bnxt_re_wrid *swque;
+ struct bnxt_re_queue *rq;
+ uint32_t head = 0;
+ uint8_t cnt = 0;
rcqe = cqe;
if (!qp->srq) {
rq = qp->jrqq->hwque;
- ibvwc->wr_id = qp->jrqq->swque[rq->head].wrid;
+ head = qp->jrqq->last_idx;
+ swque = &qp->jrqq->swque[head];
+ ibvwc->wr_id = swque->wrid;
+ cnt = swque->slots;
} else {
struct bnxt_re_srq *srq;
int tag;
@@ -433,6 +454,7 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
rq = srq->srqq;
tag = le32toh(hdr->qphi_rwrid) & BNXT_RE_BCQE_RWRID_MASK;
ibvwc->wr_id = srq->srwrid[tag].wrid;
+ cnt = 1;
bnxt_re_release_srqe(srq, tag);
}
@@ -463,7 +485,9 @@ static void bnxt_re_poll_success_rcqe(struct bnxt_re_qp *qp,
if (qp->qptyp == IBV_QPT_UD)
bnxt_re_fill_ud_cqe(ibvwc, hdr, cqe);
- bnxt_re_incr_head(rq);
+ if (!qp->srq)
+ bnxt_re_jqq_mod_last(qp->jrqq, head);
+ bnxt_re_incr_head(rq, cnt);
}
static uint8_t bnxt_re_poll_rcqe(struct bnxt_re_qp *qp, struct ibv_wc *ibvwc,
@@ -575,7 +599,7 @@ static int bnxt_re_poll_one(struct bnxt_re_cq *cq, int nwc, struct ibv_wc *wc)
*qp_handle = 0x0ULL; /* mark cqe as read */
qp_handle = NULL;
}
- bnxt_re_incr_head(&cq->cqq);
+ bnxt_re_incr_head(&cq->cqq, 1);
bnxt_re_change_cq_phase(cq);
skipp_real:
if (cnt) {
@@ -592,21 +616,21 @@ skipp_real:
return dqed;
}
-static int bnxt_re_poll_flush_wcs(struct bnxt_re_queue *que,
- struct bnxt_re_wrid *wridp,
+static int bnxt_re_poll_flush_wcs(struct bnxt_re_joint_queue *jqq,
struct ibv_wc *ibvwc, uint32_t qpid,
int nwc)
{
+ uint8_t opcode = IBV_WC_RECV;
+ struct bnxt_re_queue *que;
struct bnxt_re_wrid *wrid;
struct bnxt_re_psns *psns;
- uint32_t cnt = 0, head;
- uint8_t opcode = IBV_WC_RECV;
+ uint32_t cnt = 0;
+ que = jqq->hwque;
while (nwc) {
if (bnxt_re_is_que_empty(que))
break;
- head = que->head;
- wrid = &wridp[head];
+ wrid = &jqq->swque[jqq->last_idx];
if (wrid->psns) {
psns = wrid->psns;
opcode = (le32toh(psns->opc_spsn) >>
@@ -621,7 +645,8 @@ static int bnxt_re_poll_flush_wcs(struct bnxt_re_queue *que,
ibvwc->byte_len = 0;
ibvwc->wc_flags = 0;
- bnxt_re_incr_head(que);
+ bnxt_re_jqq_mod_last(jqq, jqq->last_idx);
+ bnxt_re_incr_head(que, wrid->slots);
nwc--;
cnt++;
ibvwc++;
@@ -636,8 +661,7 @@ static int bnxt_re_poll_flush_wqes(struct bnxt_re_cq *cq,
int32_t nwc)
{
struct bnxt_re_fque_node *cur, *tmp;
- struct bnxt_re_wrid *wridp;
- struct bnxt_re_queue *que;
+ struct bnxt_re_joint_queue *jqq;
struct bnxt_re_qp *qp;
bool sq_list = false;
uint32_t polled = 0;
@@ -648,18 +672,15 @@ static int bnxt_re_poll_flush_wqes(struct bnxt_re_cq *cq,
if (sq_list) {
qp = container_of(cur, struct bnxt_re_qp,
snode);
- que = qp->jsqq->hwque;
- wridp = qp->jsqq->swque;
+ jqq = qp->jsqq;
} else {
qp = container_of(cur, struct bnxt_re_qp,
rnode);
- que = qp->jrqq->hwque;
- wridp = qp->jrqq->swque;
+ jqq = qp->jrqq;
}
- if (bnxt_re_is_que_empty(que))
+ if (bnxt_re_is_que_empty(jqq->hwque))
continue;
- polled += bnxt_re_poll_flush_wcs(que, wridp,
- ibvwc + polled,
+ polled += bnxt_re_poll_flush_wcs(jqq, ibvwc + polled,
qp->qpid,
nwc - polled);
if (!(nwc - polled))
@@ -1164,14 +1185,17 @@ static void bnxt_re_fill_psns(struct bnxt_re_qp *qp, struct bnxt_re_wrid *wrid,
psns_ext->st_slot_idx = 0;
}
-static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, struct ibv_send_wr *wr,
- uint32_t len, uint8_t sqsig)
+static void bnxt_re_fill_wrid(struct bnxt_re_wrid *wrid, uint64_t wr_id,
+ uint32_t len, uint8_t sqsig, uint32_t st_idx,
+ uint8_t slots)
{
- wrid->wrid = wr->wr_id;
+ wrid->wrid = wr_id;
wrid->bytes = len;
wrid->sig = 0;
- if (wr->send_flags & IBV_SEND_SIGNALED || sqsig)
+ if (sqsig)
wrid->sig = IBV_SEND_SIGNALED;
+ wrid->st_slot_idx = st_idx;
+ wrid->slots = slots;
}
static int bnxt_re_build_send_sqe(struct bnxt_re_qp *qp, void *wqe,
@@ -1290,6 +1314,8 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
struct bnxt_re_bsqe *hdr;
int ret = 0, bytes = 0;
bool ring_db = false;
+ uint32_t swq_idx;
+ uint32_t sig;
void *sqe;
pthread_spin_lock(&sq->qlock);
@@ -1316,8 +1342,6 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
}
sqe = (void *)(sq->va + (sq->tail * sq->stride));
- wrid = &qp->jsqq->swque[sq->tail];
-
memset(sqe, 0, bnxt_re_get_sqe_sz());
hdr = sqe;
is_inline = bnxt_re_set_hdr_flags(hdr, wr->send_flags,
@@ -1365,9 +1389,12 @@ int bnxt_re_post_send(struct ibv_qp *ibvqp, struct ibv_send_wr *wr,
break;
}
- bnxt_re_fill_wrid(wrid, wr, bytes, qp->cap.sqsig);
+ wrid = bnxt_re_get_swqe(qp->jsqq, &swq_idx);
+ sig = ((wr->send_flags & IBV_SEND_SIGNALED) || qp->cap.sqsig);
+ bnxt_re_fill_wrid(wrid, wr->wr_id, bytes, sig, sq->tail, 1);
bnxt_re_fill_psns(qp, wrid, wr->opcode, bytes);
- bnxt_re_incr_tail(sq);
+ bnxt_re_jqq_mod_start(qp->jsqq, swq_idx);
+ bnxt_re_incr_tail(sq, 1);
qp->wqe_cnt++;
wr = wr->next;
ring_db = true;
@@ -1394,16 +1421,14 @@ bad_wr:
}
static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr,
- void *rqe)
+ void *rqe, uint32_t idx)
{
struct bnxt_re_brqe *hdr = rqe;
- struct bnxt_re_wrid *wrid;
struct bnxt_re_sge *sge;
int wqe_sz, len;
uint32_t hdrval;
sge = (rqe + bnxt_re_get_rqe_hdr_sz());
- wrid = &qp->jrqq->swque[qp->jrqq->hwque->tail];
len = bnxt_re_build_sge(sge, wr->sg_list, wr->num_sge, false);
wqe_sz = wr->num_sge + (bnxt_re_get_rqe_hdr_sz() >> 4); /* 16B align */
@@ -1415,12 +1440,7 @@ static int bnxt_re_build_rqe(struct bnxt_re_qp *qp, struct ibv_recv_wr *wr,
hdrval = BNXT_RE_WR_OPCD_RECV;
hdrval |= ((wqe_sz & BNXT_RE_HDR_WS_MASK) << BNXT_RE_HDR_WS_SHIFT);
hdr->rsv_ws_fl_wt = htole32(hdrval);
- hdr->wrid = htole32(qp->jrqq->hwque->tail);
-
- /* Fill wrid */
- wrid->wrid = wr->wr_id;
- wrid->bytes = len; /* N.A. for RQE */
- wrid->sig = 0; /* N.A. for RQE */
+ hdr->wrid = htole32(idx);
return len;
}
@@ -1430,6 +1450,8 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
{
struct bnxt_re_qp *qp = to_bnxt_re_qp(ibvqp);
struct bnxt_re_queue *rq = qp->jrqq->hwque;
+ struct bnxt_re_wrid *swque;
+ uint32_t swq_idx;
void *rqe;
int ret;
@@ -1451,14 +1473,18 @@ int bnxt_re_post_recv(struct ibv_qp *ibvqp, struct ibv_recv_wr *wr,
rqe = (void *)(rq->va + (rq->tail * rq->stride));
memset(rqe, 0, bnxt_re_get_rqe_sz());
- ret = bnxt_re_build_rqe(qp, wr, rqe);
+ swque = bnxt_re_get_swqe(qp->jrqq, &swq_idx);
+ ret = bnxt_re_build_rqe(qp, wr, rqe, swq_idx);
if (ret < 0) {
pthread_spin_unlock(&rq->qlock);
*bad = wr;
return ENOMEM;
}
- bnxt_re_incr_tail(rq);
+ swque = bnxt_re_get_swqe(qp->jrqq, NULL);
+ bnxt_re_fill_wrid(swque, wr->wr_id, ret, 0, rq->tail, 1);
+ bnxt_re_jqq_mod_start(qp->jrqq, swq_idx);
+ bnxt_re_incr_tail(rq, 1);
wr = wr->next;
bnxt_re_ring_rq_db(qp);
}
@@ -1666,7 +1692,7 @@ int bnxt_re_post_srq_recv(struct ibv_srq *ibvsrq, struct ibv_recv_wr *wr,
}
srq->start_idx = srq->srwrid[srq->start_idx].next_idx;
- bnxt_re_incr_tail(rq);
+ bnxt_re_incr_tail(rq, 1);
wr = wr->next;
bnxt_re_ring_srq_db(srq);
count++;
--
2.25.1
Attachment:
smime.p7s
Description: S/MIME Cryptographic Signature
