Extend rxe to support receiving XRC trafic.
- Extend lists of QP types where appropriate.
- Add an error state for invalid XRCETH header.
- Add code to handle srq, pd amd rcq coming from XRCSRQ for XRC packets
Save pointers in pkt info struct for later use and dropping refs.
Signed-off-by: Bob Pearson <rpearsonhpe@xxxxxxxxx>
---
drivers/infiniband/sw/rxe/rxe_av.c | 2 +
drivers/infiniband/sw/rxe/rxe_comp.c | 27 +++-
drivers/infiniband/sw/rxe/rxe_hdr.h | 9 +-
drivers/infiniband/sw/rxe/rxe_loc.h | 3 +-
drivers/infiniband/sw/rxe/rxe_mw.c | 6 +-
drivers/infiniband/sw/rxe/rxe_net.c | 8 ++
drivers/infiniband/sw/rxe/rxe_opcode.c | 1 -
drivers/infiniband/sw/rxe/rxe_recv.c | 22 ++-
drivers/infiniband/sw/rxe/rxe_req.c | 3 +-
drivers/infiniband/sw/rxe/rxe_resp.c | 191 ++++++++++++++++++++-----
drivers/infiniband/sw/rxe/rxe_verbs.h | 16 ++-
11 files changed, 232 insertions(+), 56 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_av.c b/drivers/infiniband/sw/rxe/rxe_av.c
index 05d4cb772dc6..673b65d415b8 100644
--- a/drivers/infiniband/sw/rxe/rxe_av.c
+++ b/drivers/infiniband/sw/rxe/rxe_av.c
@@ -122,9 +122,11 @@ struct rxe_av *rxe_get_av(struct rxe_pkt_info *pkt)
ah = rxe_pool_get_index(&pkt->rxe->ah_pool, ah_num);
if (!ah || ah->ah_num != ah_num ||
rxe_ah_pd(ah) != rxe_qp_pd(pkt->qp)) {
+ rxe_drop_ref(ah);
pr_warn("Unable to find AH matching ah_num\n");
return NULL;
}
+ pkt->ah = ah;
return &ah->av;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_comp.c b/drivers/infiniband/sw/rxe/rxe_comp.c
index 4d62e5bdf820..987bd8e67fb6 100644
--- a/drivers/infiniband/sw/rxe/rxe_comp.c
+++ b/drivers/infiniband/sw/rxe/rxe_comp.c
@@ -242,6 +242,23 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
return COMPST_ERROR;
}
break;
+
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_FIRST:
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_MIDDLE:
+ if (pkt->opcode != IB_OPCODE_XRC_RDMA_READ_RESPONSE_MIDDLE &&
+ pkt->opcode != IB_OPCODE_XRC_RDMA_READ_RESPONSE_LAST) {
+ if ((pkt->psn == wqe->first_psn &&
+ pkt->opcode ==
+ IB_OPCODE_XRC_RDMA_READ_RESPONSE_FIRST) ||
+ (wqe->first_psn == wqe->last_psn &&
+ pkt->opcode ==
+ IB_OPCODE_XRC_RDMA_READ_RESPONSE_ONLY))
+ break;
+
+ return COMPST_ERROR;
+ }
+ break;
+
default:
WARN_ON_ONCE(1);
}
@@ -251,6 +268,9 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
case IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST:
case IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST:
case IB_OPCODE_RC_RDMA_READ_RESPONSE_ONLY:
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_FIRST:
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_LAST:
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_ONLY:
syn = aeth_syn(pkt);
if ((syn & AETH_TYPE_MASK) != AETH_ACK)
@@ -260,6 +280,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
/* (IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE doesn't have an AETH)
*/
case IB_OPCODE_RC_RDMA_READ_RESPONSE_MIDDLE:
+ case IB_OPCODE_XRC_RDMA_READ_RESPONSE_MIDDLE:
if (wqe->wr.opcode != IB_WR_RDMA_READ &&
wqe->wr.opcode != IB_WR_RDMA_READ_WITH_INV) {
wqe->status = IB_WC_FATAL_ERR;
@@ -269,6 +290,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
return COMPST_READ;
case IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE:
+ case IB_OPCODE_XRC_ATOMIC_ACKNOWLEDGE:
syn = aeth_syn(pkt);
if ((syn & AETH_TYPE_MASK) != AETH_ACK)
@@ -281,6 +303,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
return COMPST_ATOMIC;
case IB_OPCODE_RC_ACKNOWLEDGE:
+ case IB_OPCODE_XRC_ACKNOWLEDGE:
syn = aeth_syn(pkt);
switch (syn & AETH_TYPE_MASK) {
case AETH_ACK:
@@ -570,8 +593,8 @@ int rxe_completer(void *arg)
state = COMPST_GET_ACK;
while (1) {
- pr_debug("qp#%d state = %s\n", rxe_qp_num(qp),
- comp_state_name[state]);
+ pr_debug("qp#%d type %d state = %s\n", rxe_qp_num(qp),
+ rxe_qp_type(qp), comp_state_name[state]);
switch (state) {
case COMPST_GET_ACK:
skb = skb_dequeue(&qp->resp_pkts);
diff --git a/drivers/infiniband/sw/rxe/rxe_hdr.h b/drivers/infiniband/sw/rxe/rxe_hdr.h
index 499807b11405..db6033f49697 100644
--- a/drivers/infiniband/sw/rxe/rxe_hdr.h
+++ b/drivers/infiniband/sw/rxe/rxe_hdr.h
@@ -14,7 +14,14 @@
struct rxe_pkt_info {
struct sk_buff *skb; /* back pointer to skb */
struct rxe_dev *rxe; /* device that owns packet */
- struct rxe_qp *qp; /* qp that owns packet */
+
+ /* these are objects that need references dropped when pkt freed */
+ struct rxe_ah *ah;
+ struct rxe_qp *qp;
+ struct rxe_srq *srq;
+
+ struct rxe_pd *pd;
+ struct rxe_cq *rcq;
struct rxe_send_wqe *wqe; /* send wqe */
u8 *hdr; /* points to bth */
u32 mask; /* useful info about pkt */
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index 790884a5e9d5..2580273c1806 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -94,7 +94,8 @@ int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
int rxe_dealloc_mw(struct ib_mw *ibmw);
int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe);
int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey);
-struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey);
+struct rxe_mw *rxe_lookup_mw(struct rxe_pd *pd, struct rxe_qp *qp,
+ int access, u32 rkey);
void rxe_mw_cleanup(struct rxe_pool_entry *arg);
/* rxe_net.c */
diff --git a/drivers/infiniband/sw/rxe/rxe_mw.c b/drivers/infiniband/sw/rxe/rxe_mw.c
index 17936a0b8320..0de65c89b7c5 100644
--- a/drivers/infiniband/sw/rxe/rxe_mw.c
+++ b/drivers/infiniband/sw/rxe/rxe_mw.c
@@ -305,10 +305,10 @@ int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
return ret;
}
-struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
+struct rxe_mw *rxe_lookup_mw(struct rxe_pd *pd, struct rxe_qp *qp,
+ int access, u32 rkey)
{
- struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
- struct rxe_pd *pd = rxe_qp_pd(qp);
+ struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
struct rxe_mw *mw;
int index = rkey >> 8;
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index b1353d456a4c..1597d5313af3 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -424,6 +424,14 @@ static int rxe_loopback(struct sk_buff *skb, struct rxe_pkt_info *pkt)
new_pkt = kzalloc(sizeof(*new_pkt), GFP_ATOMIC);
memcpy(new_pkt, pkt, sizeof(*pkt));
+
+ /* don't keep the references */
+ new_pkt->ah = NULL;
+ new_pkt->qp = NULL;
+ new_pkt->srq = NULL;
+ new_pkt->pd = NULL;
+ new_pkt->rcq = NULL;
+
RXE_CB(skb)->pkt = new_pkt;
new_pkt->skb = skb;
diff --git a/drivers/infiniband/sw/rxe/rxe_opcode.c b/drivers/infiniband/sw/rxe/rxe_opcode.c
index af8e05bc63b2..1685a29efaf7 100644
--- a/drivers/infiniband/sw/rxe/rxe_opcode.c
+++ b/drivers/infiniband/sw/rxe/rxe_opcode.c
@@ -88,7 +88,6 @@ struct rxe_wr_opcode_info rxe_wr_opcode_info[] = {
.name = "IB_WR_RDMA_READ_WITH_INV",
.mask = {
[IB_QPT_RC] = WR_READ_MASK,
- /* TODO get rid of this no such thing for RoCE */
},
},
[IB_WR_LOCAL_INV] = {
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index cf5ac6bba59c..7ac40857def3 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -12,10 +12,13 @@
void rxe_free_pkt(struct rxe_pkt_info *pkt)
{
struct sk_buff *skb = PKT_TO_SKB(pkt);
- struct rxe_qp *qp = pkt->qp;
- if (qp)
- rxe_drop_ref(qp);
+ if (pkt->qp)
+ rxe_drop_ref(pkt->qp);
+ if (pkt->srq)
+ rxe_drop_ref(pkt->srq);
+ if (pkt->ah)
+ rxe_drop_ref(pkt->ah);
ib_device_put(&pkt->rxe->ib_dev);
@@ -37,13 +40,13 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
switch (rxe_qp_type(qp)) {
case IB_QPT_RC:
if (unlikely(pkt_type != IB_OPCODE_RC)) {
- pr_warn_ratelimited("bad qp type\n");
+ pr_warn_ratelimited("bad qp type for RC packet\n");
goto err1;
}
break;
case IB_QPT_UC:
if (unlikely(pkt_type != IB_OPCODE_UC)) {
- pr_warn_ratelimited("bad qp type\n");
+ pr_warn_ratelimited("bad qp type for UC packet\n");
goto err1;
}
break;
@@ -51,7 +54,14 @@ static int check_type_state(struct rxe_dev *rxe, struct rxe_pkt_info *pkt,
case IB_QPT_SMI:
case IB_QPT_GSI:
if (unlikely(pkt_type != IB_OPCODE_UD)) {
- pr_warn_ratelimited("bad qp type\n");
+ pr_warn_ratelimited("bad qp type for UD packet\n");
+ goto err1;
+ }
+ break;
+ case IB_QPT_XRC_INI:
+ case IB_QPT_XRC_TGT:
+ if (unlikely(pkt_type != IB_OPCODE_XRC)) {
+ pr_warn_ratelimited("bad qp type for XRC packet\n");
goto err1;
}
break;
diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c
index 166d4aeef5e9..592101ea0461 100644
--- a/drivers/infiniband/sw/rxe/rxe_req.c
+++ b/drivers/infiniband/sw/rxe/rxe_req.c
@@ -451,6 +451,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp,
paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
/* pkt->hdr, port_num and mask are initialized in ifc layer */
+ memset(pkt, 0, sizeof(*pkt));
pkt->rxe = rxe;
pkt->opcode = opcode;
pkt->qp = qp;
@@ -507,7 +508,7 @@ static struct sk_buff *init_req_packet(struct rxe_qp *qp,
if (pkt->mask & RXE_ATMETH_MASK) {
atmeth_set_va(pkt, wqe->iova);
if (opcode == IB_OPCODE_RC_COMPARE_SWAP ||
- opcode == IB_OPCODE_RD_COMPARE_SWAP) {
+ opcode == IB_OPCODE_XRC_COMPARE_SWAP) {
atmeth_set_swap_add(pkt, ibwr->wr.atomic.swap);
atmeth_set_comp(pkt, ibwr->wr.atomic.compare_add);
} else {
diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c
index ac8d823eb416..4c57e5495d4c 100644
--- a/drivers/infiniband/sw/rxe/rxe_resp.c
+++ b/drivers/infiniband/sw/rxe/rxe_resp.c
@@ -35,6 +35,7 @@ enum resp_states {
RESPST_ERR_TOO_MANY_RDMA_ATM_REQ,
RESPST_ERR_RNR,
RESPST_ERR_RKEY_VIOLATION,
+ RESPST_ERR_INVALID_XRCETH,
RESPST_ERR_INVALIDATE_RKEY,
RESPST_ERR_LENGTH,
RESPST_ERR_CQ_OVERFLOW,
@@ -69,6 +70,7 @@ static char *resp_state_name[] = {
[RESPST_ERR_TOO_MANY_RDMA_ATM_REQ] = "ERR_TOO_MANY_RDMA_ATM_REQ",
[RESPST_ERR_RNR] = "ERR_RNR",
[RESPST_ERR_RKEY_VIOLATION] = "ERR_RKEY_VIOLATION",
+ [RESPST_ERR_INVALID_XRCETH] = "ERR_INVALID_XRCETH",
[RESPST_ERR_INVALIDATE_RKEY] = "ERR_INVALIDATE_RKEY_VIOLATION",
[RESPST_ERR_LENGTH] = "ERR_LENGTH",
[RESPST_ERR_CQ_OVERFLOW] = "ERR_CQ_OVERFLOW",
@@ -122,6 +124,7 @@ static enum resp_states check_psn(struct rxe_qp *qp,
switch (rxe_qp_type(qp)) {
case IB_QPT_RC:
+ case IB_QPT_XRC_TGT:
if (diff > 0) {
if (qp->resp.sent_psn_nak)
return RESPST_CLEANUP;
@@ -243,6 +246,47 @@ static enum resp_states check_op_seq(struct rxe_qp *qp,
}
break;
+ case IB_QPT_XRC_TGT:
+ switch (qp->resp.opcode) {
+ case IB_OPCODE_XRC_SEND_FIRST:
+ case IB_OPCODE_XRC_SEND_MIDDLE:
+ switch (pkt->opcode) {
+ case IB_OPCODE_XRC_SEND_MIDDLE:
+ case IB_OPCODE_XRC_SEND_LAST:
+ case IB_OPCODE_XRC_SEND_LAST_WITH_IMMEDIATE:
+ case IB_OPCODE_XRC_SEND_LAST_WITH_INVALIDATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_C;
+ }
+
+ case IB_OPCODE_XRC_RDMA_WRITE_FIRST:
+ case IB_OPCODE_XRC_RDMA_WRITE_MIDDLE:
+ switch (pkt->opcode) {
+ case IB_OPCODE_XRC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_XRC_RDMA_WRITE_LAST:
+ case IB_OPCODE_XRC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ return RESPST_CHK_OP_VALID;
+ default:
+ return RESPST_ERR_MISSING_OPCODE_LAST_C;
+ }
+
+ default:
+ switch (pkt->opcode) {
+ case IB_OPCODE_XRC_SEND_MIDDLE:
+ case IB_OPCODE_XRC_SEND_LAST:
+ case IB_OPCODE_XRC_SEND_LAST_WITH_IMMEDIATE:
+ case IB_OPCODE_XRC_SEND_LAST_WITH_INVALIDATE:
+ case IB_OPCODE_XRC_RDMA_WRITE_MIDDLE:
+ case IB_OPCODE_XRC_RDMA_WRITE_LAST:
+ case IB_OPCODE_XRC_RDMA_WRITE_LAST_WITH_IMMEDIATE:
+ return RESPST_ERR_MISSING_OPCODE_FIRST;
+ default:
+ return RESPST_CHK_OP_VALID;
+ }
+ }
+ break;
+
default:
return RESPST_CHK_OP_VALID;
}
@@ -253,6 +297,7 @@ static enum resp_states check_op_valid(struct rxe_qp *qp,
{
switch (rxe_qp_type(qp)) {
case IB_QPT_RC:
+ case IB_QPT_XRC_TGT:
if (((pkt->mask & RXE_READ_MASK) &&
!(qp->attr.qp_access_flags & IB_ACCESS_REMOTE_READ)) ||
((pkt->mask & RXE_WRITE_MASK) &&
@@ -286,9 +331,8 @@ static enum resp_states check_op_valid(struct rxe_qp *qp,
return RESPST_CHK_RESOURCE;
}
-static enum resp_states get_srq_wqe(struct rxe_qp *qp)
+static enum resp_states get_srq_wqe(struct rxe_qp *qp, struct rxe_srq *srq)
{
- struct rxe_srq *srq = rxe_qp_srq(qp);
struct rxe_queue *q = srq->rq.queue;
struct rxe_recv_wqe *wqe;
struct ib_event ev;
@@ -339,8 +383,11 @@ static enum resp_states get_srq_wqe(struct rxe_qp *qp)
static enum resp_states check_resource(struct rxe_qp *qp,
struct rxe_pkt_info *pkt)
{
- struct rxe_srq *srq = rxe_qp_srq(qp);
+ struct rxe_srq *srq = NULL;
+ /* can come here from get_req() with no packet
+ * to drain/flush recv work request queue
+ */
if (qp->resp.state == QP_STATE_ERROR) {
if (qp->resp.wqe) {
qp->resp.status = IB_WC_WR_FLUSH_ERR;
@@ -359,6 +406,37 @@ static enum resp_states check_resource(struct rxe_qp *qp,
}
}
+ /* not supposed to happen */
+ if (WARN_ON_ONCE(!pkt))
+ return RESPST_EXIT;
+
+ /*
+ * srq, pd and rcq can come from XRCSRQ or QP
+ * decide which and store in pkt
+ */
+ if (pkt->mask & RXE_XRCETH_MASK) {
+ int srq_num = xrceth_xrcsrq(pkt);
+
+ srq = rxe_pool_get_index(&pkt->rxe->srq_pool, srq_num);
+ if (!srq || rxe_qp_xrcd(qp) != rxe_srq_xrcd(srq)) {
+ pr_warn("Unable to get srq from xrceth\n");
+ return RESPST_ERR_INVALID_XRCETH;
+ }
+
+ pkt->srq = srq;
+ pkt->pd = rxe_srq_pd(srq);
+ pkt->rcq = rxe_srq_cq(srq);
+ } else {
+ srq = rxe_qp_srq(qp);
+ pkt->pd = rxe_qp_pd(qp);
+ pkt->rcq = rxe_qp_rcq(qp);
+ }
+
+ if (!pkt->pd)
+ pr_info("%s: no PD for pkt\n", __func__);
+ if (!pkt->rcq)
+ pr_info("%s: no RCQ for pkt\n", __func__);
+
if (pkt->mask & RXE_READ_OR_ATOMIC) {
/* it is the requesters job to not send
* too many read/atomic ops, we just
@@ -372,7 +450,7 @@ static enum resp_states check_resource(struct rxe_qp *qp,
if (pkt->mask & RXE_RWR_MASK) {
if (srq)
- return get_srq_wqe(qp);
+ return get_srq_wqe(qp, srq);
qp->resp.wqe = queue_head(qp->rq.queue,
QUEUE_TYPE_FROM_CLIENT);
@@ -387,6 +465,7 @@ static enum resp_states check_length(struct rxe_qp *qp,
{
switch (rxe_qp_type(qp)) {
case IB_QPT_RC:
+ case IB_QPT_XRC_TGT:
return RESPST_CHK_RKEY;
case IB_QPT_UC:
@@ -443,16 +522,16 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
pktlen = payload_size(pkt);
if (rkey_is_mw(rkey)) {
- mw = rxe_lookup_mw(qp, access, rkey);
+ mw = rxe_lookup_mw(pkt->pd, qp, access, rkey);
if (!mw) {
- pr_err("%s: no MW matches rkey %#x\n", __func__, rkey);
+ pr_warn("%s: no MW matches rkey %#x\n", __func__, rkey);
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
mr = mw->mr;
if (!mr) {
- pr_err("%s: MW doesn't have an MR\n", __func__);
+ pr_warn("%s: MW doesn't have an MR\n", __func__);
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
@@ -463,9 +542,9 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
rxe_drop_ref(mw);
rxe_add_ref(mr);
} else {
- mr = lookup_mr(rxe_qp_pd(qp), access, rkey, RXE_LOOKUP_REMOTE);
+ mr = lookup_mr(pkt->pd, access, rkey, RXE_LOOKUP_REMOTE);
if (!mr) {
- pr_err("%s: no MR matches rkey %#x\n", __func__, rkey);
+ pr_warn("%s: no MR matches rkey %#x\n", __func__, rkey);
state = RESPST_ERR_RKEY_VIOLATION;
goto err;
}
@@ -511,12 +590,12 @@ static enum resp_states check_rkey(struct rxe_qp *qp,
return state;
}
-static enum resp_states send_data_in(struct rxe_qp *qp, void *data_addr,
- int data_len)
+static enum resp_states send_data_in(struct rxe_pd *pd, struct rxe_qp *qp,
+ void *data_addr, int data_len)
{
int err;
- err = copy_data(rxe_qp_pd(qp), IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
+ err = copy_data(pd, IB_ACCESS_LOCAL_WRITE, &qp->resp.wqe->dma,
data_addr, data_len, RXE_TO_MR_OBJ);
if (unlikely(err))
return (err == -ENOSPC) ? RESPST_ERR_LENGTH
@@ -574,7 +653,7 @@ static enum resp_states process_atomic(struct rxe_qp *qp,
qp->resp.atomic_orig = *vaddr;
if (pkt->opcode == IB_OPCODE_RC_COMPARE_SWAP ||
- pkt->opcode == IB_OPCODE_RD_COMPARE_SWAP) {
+ pkt->opcode == IB_OPCODE_XRC_COMPARE_SWAP) {
if (*vaddr == atmeth_comp(pkt))
*vaddr = atmeth_swap_add(pkt);
} else {
@@ -705,6 +784,10 @@ static enum resp_states read_reply(struct rxe_qp *qp,
opcode = IB_OPCODE_RC_RDMA_READ_RESPONSE_LAST;
}
+ /* fixup opcode if this is XRC add 0xa0 */
+ if (rxe_qp_type(qp) == IB_QPT_XRC_TGT)
+ opcode += IB_OPCODE_XRC;
+
res->state = rdatm_res_state_next;
payload = min_t(int, res->read.resid, mtu);
@@ -717,7 +800,7 @@ static enum resp_states read_reply(struct rxe_qp *qp,
err = rxe_mr_copy(res->read.mr, res->read.va, payload_addr(&ack_pkt),
payload, RXE_FROM_MR_OBJ);
if (err)
- pr_err("Failed copying memory\n");
+ pr_warn("Failed copying memory\n");
if (bth_pad(&ack_pkt)) {
u8 *pad = payload_addr(&ack_pkt) + payload;
@@ -727,7 +810,7 @@ static enum resp_states read_reply(struct rxe_qp *qp,
err = rxe_xmit_packet(qp, &ack_pkt, skb);
if (err) {
- pr_err("Failed sending RDMA reply.\n");
+ pr_warn("Failed sending RDMA reply.\n");
return RESPST_ERR_RNR;
}
@@ -775,15 +858,17 @@ static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
sizeof(hdr.reserved));
memcpy(&hdr.roce4grh, ip_hdr(skb),
sizeof(hdr.roce4grh));
- err = send_data_in(qp, &hdr, sizeof(hdr));
+ err = send_data_in(pkt->pd, qp, &hdr,
+ sizeof(hdr));
} else {
- err = send_data_in(qp, ipv6_hdr(skb),
- sizeof(hdr));
+ err = send_data_in(pkt->pd, qp, ipv6_hdr(skb),
+ sizeof(hdr));
}
if (err)
return err;
}
- err = send_data_in(qp, payload_addr(pkt), payload_size(pkt));
+ err = send_data_in(pkt->pd, qp, payload_addr(pkt),
+ payload_size(pkt));
if (err)
return err;
} else if (pkt->mask & RXE_WRITE_MASK) {
@@ -822,7 +907,8 @@ static enum resp_states execute(struct rxe_qp *qp, struct rxe_pkt_info *pkt)
/* We successfully processed this new request. */
qp->resp.msn++;
return RESPST_COMPLETE;
- } else if (rxe_qp_type(qp) == IB_QPT_RC)
+ } else if (rxe_qp_type(qp) == IB_QPT_RC ||
+ rxe_qp_type(qp) == IB_QPT_XRC_TGT)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
@@ -836,13 +922,19 @@ static enum resp_states do_complete(struct rxe_qp *qp,
struct ib_uverbs_wc *uwc = &cqe.uibwc;
struct rxe_recv_wqe *wqe = qp->resp.wqe;
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+ struct rxe_cq *rcq;
+
+ /* can come here from check_resources to flush the
+ * recv work queue entries. Otherwise pkt will be there
+ */
+ rcq = pkt ? pkt->rcq : rxe_qp_rcq(qp);
if (!wqe)
goto finish;
memset(&cqe, 0, sizeof(cqe));
- if (rxe_qp_rcq(qp)->is_user) {
+ if (rcq->is_user) {
uwc->status = qp->resp.status;
uwc->qp_num = rxe_qp_num(qp);
uwc->wr_id = wqe->wr_id;
@@ -865,7 +957,7 @@ static enum resp_states do_complete(struct rxe_qp *qp,
/* fields after byte_len are different between kernel and user
* space
*/
- if (rxe_qp_rcq(qp)->is_user) {
+ if (rcq->is_user) {
uwc->wc_flags = IB_WC_GRH;
if (pkt->mask & RXE_IMMDT_MASK) {
@@ -917,12 +1009,12 @@ static enum resp_states do_complete(struct rxe_qp *qp,
}
/* have copy for srq and reference for !srq */
- if (!rxe_qp_srq(qp))
+ if (qp->rq.queue)
advance_consumer(qp->rq.queue, QUEUE_TYPE_FROM_CLIENT);
qp->resp.wqe = NULL;
- if (rxe_cq_post(rxe_qp_rcq(qp), &cqe, pkt ? bth_se(pkt) : 1))
+ if (rxe_cq_post(rcq, &cqe, pkt ? bth_se(pkt) : 1))
return RESPST_ERR_CQ_OVERFLOW;
finish:
@@ -930,7 +1022,8 @@ static enum resp_states do_complete(struct rxe_qp *qp,
return RESPST_CHK_RESOURCE;
if (unlikely(!pkt))
return RESPST_DONE;
- if (rxe_qp_type(qp) == IB_QPT_RC)
+ if (rxe_qp_type(qp) == IB_QPT_RC ||
+ rxe_qp_type(qp) == IB_QPT_XRC_TGT)
return RESPST_ACKNOWLEDGE;
else
return RESPST_CLEANUP;
@@ -942,8 +1035,13 @@ static int send_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
int err = 0;
struct rxe_pkt_info ack_pkt;
struct sk_buff *skb;
+ int opcode;
+
+ opcode = (rxe_qp_type(qp) == IB_QPT_XRC_TGT) ?
+ IB_OPCODE_XRC_ACKNOWLEDGE :
+ IB_OPCODE_RC_ACKNOWLEDGE;
- skb = prepare_ack_packet(qp, pkt, &ack_pkt, IB_OPCODE_RC_ACKNOWLEDGE,
+ skb = prepare_ack_packet(qp, pkt, &ack_pkt, opcode,
0, psn, syndrome);
if (!skb) {
err = -ENOMEM;
@@ -952,7 +1050,7 @@ static int send_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
err = rxe_xmit_packet(qp, &ack_pkt, skb);
if (err)
- pr_err_ratelimited("Failed sending ack\n");
+ pr_warn("Failed sending ack\n");
err1:
return err;
@@ -987,7 +1085,7 @@ static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
rc = rxe_xmit_packet(qp, &ack_pkt, skb);
if (rc) {
- pr_err_ratelimited("Failed sending ack\n");
+ pr_warn("Failed sending ack\n");
rxe_drop_ref(qp);
}
out:
@@ -997,7 +1095,8 @@ static int send_atomic_ack(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
static enum resp_states acknowledge(struct rxe_qp *qp,
struct rxe_pkt_info *pkt)
{
- if (rxe_qp_type(qp) != IB_QPT_RC)
+ if (rxe_qp_type(qp) != IB_QPT_RC &&
+ rxe_qp_type(qp) != IB_QPT_XRC_TGT)
return RESPST_CLEANUP;
if (qp->resp.aeth_syndrome != AETH_ACK_UNLIMITED)
@@ -1114,7 +1213,7 @@ static enum resp_states duplicate_request(struct rxe_qp *qp,
/* Resend the result. */
rc = rxe_xmit_packet(qp, pkt, res->atomic.skb);
if (rc) {
- pr_err("Failed resending result. This flow is not handled - skb ignored\n");
+ pr_warn("Failed resending result. This flow is not handled - skb ignored\n");
rc = RESPST_CLEANUP;
goto out;
}
@@ -1176,7 +1275,7 @@ static enum resp_states do_class_d1e_error(struct rxe_qp *qp)
static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
{
struct sk_buff *skb;
- struct rxe_queue *q = qp->rq.queue;
+ struct rxe_queue *q;
while ((skb = skb_dequeue(&qp->req_pkts)))
rxe_free_pkt(SKB_TO_PKT(skb));
@@ -1184,7 +1283,11 @@ static void rxe_drain_req_pkts(struct rxe_qp *qp, bool notify)
if (notify)
return;
- while (!rxe_qp_srq(qp) && q && queue_head(q, q->type))
+ q = qp->rq.queue;
+ if (!q)
+ return;
+
+ while (queue_head(q, q->type))
advance_consumer(q, q->type);
}
@@ -1216,8 +1319,8 @@ int rxe_responder(void *arg)
}
while (1) {
- pr_debug("qp#%d state = %s\n", rxe_qp_num(qp),
- resp_state_name[state]);
+ pr_debug("qp#%d type=%d state = %s\n", rxe_qp_num(qp),
+ rxe_qp_type(qp), resp_state_name[state]);
switch (state) {
case RESPST_GET_REQ:
state = get_req(qp, &pkt);
@@ -1279,7 +1382,8 @@ int rxe_responder(void *arg)
state = do_class_d1e_error(qp);
break;
case RESPST_ERR_RNR:
- if (rxe_qp_type(qp) == IB_QPT_RC) {
+ if (rxe_qp_type(qp) == IB_QPT_RC ||
+ rxe_qp_type(qp) == IB_QPT_XRC_TGT) {
rxe_counter_inc(rxe, RXE_CNT_SND_RNR);
/* RC - class B */
send_ack(qp, pkt, AETH_RNR_NAK |
@@ -1294,7 +1398,8 @@ int rxe_responder(void *arg)
break;
case RESPST_ERR_RKEY_VIOLATION:
- if (rxe_qp_type(qp) == IB_QPT_RC) {
+ if (rxe_qp_type(qp) == IB_QPT_RC ||
+ rxe_qp_type(qp) == IB_QPT_XRC_TGT) {
/* Class C */
do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
IB_WC_REM_ACCESS_ERR);
@@ -1312,6 +1417,17 @@ int rxe_responder(void *arg)
}
break;
+ case RESPST_ERR_INVALID_XRCETH:
+ if (rxe_qp_type(qp) == IB_QPT_XRC_TGT) {
+ do_class_ac_error(qp, AETH_NAK_REM_ACC_ERR,
+ IB_WC_REM_ACCESS_ERR);
+ state = RESPST_COMPLETE;
+ } else {
+ pr_info("can't happen\n");
+ state = RESPST_CLEANUP;
+ }
+ break;
+
case RESPST_ERR_INVALIDATE_RKEY:
/* RC - Class J. */
qp->resp.goto_error = 1;
@@ -1320,7 +1436,8 @@ int rxe_responder(void *arg)
break;
case RESPST_ERR_LENGTH:
- if (rxe_qp_type(qp) == IB_QPT_RC) {
+ if (rxe_qp_type(qp) == IB_QPT_RC ||
+ rxe_qp_type(qp) == IB_QPT_XRC_TGT) {
/* Class C */
do_class_ac_error(qp, AETH_NAK_INVALID_REQ,
IB_WC_REM_INV_REQ_ERR);
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 52599f398ddd..1fdd07d6a94d 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -475,11 +475,13 @@ static inline struct rxe_mw *to_rmw(struct ib_mw *mw)
return mw ? container_of(mw, struct rxe_mw, ibmw) : NULL;
}
+/* AH extractors */
static inline struct rxe_pd *rxe_ah_pd(struct rxe_ah *ah)
{
return to_rpd(ah->ibah.pd);
}
+/* MR extractors */
static inline struct rxe_pd *rxe_mr_pd(struct rxe_mr *mr)
{
return to_rpd(mr->ibmr.pd);
@@ -495,6 +497,7 @@ static inline u32 rxe_mr_rkey(struct rxe_mr *mr)
return mr->ibmr.rkey;
}
+/* MW extractors */
static inline struct rxe_pd *rxe_mw_pd(struct rxe_mw *mw)
{
return to_rpd(mw->ibmw.pd);
@@ -531,6 +534,11 @@ static inline struct rxe_srq *rxe_qp_srq(struct rxe_qp *qp)
return to_rsrq(qp->ibqp.srq);
}
+static inline struct rxe_xrcd *rxe_qp_xrcd(struct rxe_qp *qp)
+{
+ return to_rxrcd(qp->ibqp.xrcd);
+}
+
static inline enum ib_qp_state rxe_qp_state(struct rxe_qp *qp)
{
return qp->attr.qp_state;
@@ -557,14 +565,14 @@ static inline struct rxe_pd *rxe_srq_pd(struct rxe_srq *srq)
return to_rpd(srq->ibsrq.pd);
}
-static inline enum ib_srq_type rxe_srq_type(struct rxe_srq *srq)
+static inline struct rxe_xrcd *rxe_srq_xrcd(struct rxe_srq *srq)
{
- return srq->ibsrq.srq_type;
+ return to_rxrcd(srq->ibsrq.ext.xrc.xrcd);
}
-static inline struct rxe_xrcd *rxe_srq_xrcd(struct rxe_srq *srq)
+static inline enum ib_srq_type rxe_srq_type(struct rxe_srq *srq)
{
- return to_rxrcd(srq->ibsrq.ext.xrc.xrcd);
+ return srq->ibsrq.srq_type;
}
int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
--
2.30.2
