From: Bryan Tan <bryantan@xxxxxxxxxx>
From: Bryan Tan <bryantan@xxxxxxxxxx>
This is the accompanying userspace change to allow applications
use physical resource ids if provided by the driver/device.
The create QP command is modified to specify a flag that tells
the driver to send back the physical HCA's QPN and the hypervisor-
generated QP handle separately.
Reviewed-by: Jorgen Hansen <jhansen@xxxxxxxxxx>
Signed-off-by: Adit Ranadive <aditr@xxxxxxxxxx>
Signed-off-by: Bryan Tan <bryantan@xxxxxxxxxx>
---
providers/vmw_pvrdma/pvrdma-abi.h | 2 +-
providers/vmw_pvrdma/pvrdma.h | 1 +
providers/vmw_pvrdma/qp.c | 24 +++++++++++++-----------
3 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/providers/vmw_pvrdma/pvrdma-abi.h b/providers/vmw_pvrdma/pvrdma-abi.h
index 77db9ddd1bb7..1a4c3c8a98f2 100644
--- a/providers/vmw_pvrdma/pvrdma-abi.h
+++ b/providers/vmw_pvrdma/pvrdma-abi.h
@@ -55,7 +55,7 @@ DECLARE_DRV_CMD(user_pvrdma_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
DECLARE_DRV_CMD(user_pvrdma_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
pvrdma_create_cq, pvrdma_create_cq_resp);
DECLARE_DRV_CMD(user_pvrdma_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
- pvrdma_create_qp, empty);
+ pvrdma_create_qp, pvrdma_create_qp_resp);
DECLARE_DRV_CMD(user_pvrdma_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
pvrdma_create_srq, pvrdma_create_srq_resp);
DECLARE_DRV_CMD(user_pvrdma_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
diff --git a/providers/vmw_pvrdma/pvrdma.h b/providers/vmw_pvrdma/pvrdma.h
index d90bd8096664..bb4a2db08d14 100644
--- a/providers/vmw_pvrdma/pvrdma.h
+++ b/providers/vmw_pvrdma/pvrdma.h
@@ -170,6 +170,7 @@ struct pvrdma_qp {
struct pvrdma_wq sq;
struct pvrdma_wq rq;
int is_srq;
+ uint32_t qp_handle;
};
struct pvrdma_ah {
diff --git a/providers/vmw_pvrdma/qp.c b/providers/vmw_pvrdma/qp.c
index ef429db93a43..966480f5abaa 100644
--- a/providers/vmw_pvrdma/qp.c
+++ b/providers/vmw_pvrdma/qp.c
@@ -211,9 +211,8 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
{
struct pvrdma_device *dev = to_vdev(pd->context->device);
struct user_pvrdma_create_qp cmd;
- struct ib_uverbs_create_qp_resp resp;
+ struct user_pvrdma_create_qp_resp resp = {};
struct pvrdma_qp *qp;
- int ret;
int is_srq = !!(attr->srq);
attr->cap.max_send_sge = max_t(uint32_t, 1U, attr->cap.max_send_sge);
@@ -281,15 +280,18 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
cmd.rbuf_addr = (uintptr_t)qp->rbuf.buf;
cmd.rbuf_size = qp->rbuf.length;
cmd.qp_addr = (uintptr_t) qp;
+ cmd.flags = PVRDMA_USER_QP_CREATE_USE_RESP;
- ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr,
- &cmd.ibv_cmd, sizeof(cmd),
- &resp, sizeof(resp));
-
- if (ret)
+ if (ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp)))
goto err_free;
- to_vctx(pd->context)->qp_tbl[qp->ibv_qp.qp_num & 0xFFFF] = qp;
+ if (resp.drv_payload.qpn_valid == PVRDMA_USER_QP_CREATE_USE_RESP)
+ qp->qp_handle = resp.drv_payload.qp_handle;
+ else
+ qp->qp_handle = qp->ibv_qp.qp_num;
+
+ to_vctx(pd->context)->qp_tbl[qp->qp_handle & 0xFFFF] = qp;
/* If set, each WR submitted to the SQ generate a completion entry */
if (attr->sq_sig_all)
@@ -414,7 +416,7 @@ int pvrdma_destroy_qp(struct ibv_qp *ibqp)
free(qp->rq.wrid);
pvrdma_free_buf(&qp->rbuf);
pvrdma_free_buf(&qp->sbuf);
- ctx->qp_tbl[ibqp->qp_num & 0xFFFF] = NULL;
+ ctx->qp_tbl[qp->qp_handle & 0xFFFF] = NULL;
free(qp);
return 0;
@@ -547,7 +549,7 @@ out:
if (nreq) {
udma_to_device_barrier();
pvrdma_write_uar_qp(ctx->uar,
- PVRDMA_UAR_QP_SEND | ibqp->qp_num);
+ PVRDMA_UAR_QP_SEND | qp->qp_handle);
}
pthread_spin_unlock(&qp->sq.lock);
@@ -630,7 +632,7 @@ int pvrdma_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
out:
if (nreq)
pvrdma_write_uar_qp(ctx->uar,
- PVRDMA_UAR_QP_RECV | ibqp->qp_num);
+ PVRDMA_UAR_QP_RECV | qp->qp_handle);
pthread_spin_unlock(&qp->rq.lock);
return ret;
--
1.8.3.1
