From: Yossi Itigin <yosefe@xxxxxxxxxxxx>
The option is available via "-q" flag.
Signed-off-by: Yossi Itigin <yosefe@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
librdmacm/examples/rping.c | 112 ++++++++++++++++++++++++++++++++++++++-------
librdmacm/man/rping.1 | 3 ++
2 files changed, 99 insertions(+), 16 deletions(-)
diff --git a/librdmacm/examples/rping.c b/librdmacm/examples/rping.c
index 49f5dc5..993c5ad 100644
--- a/librdmacm/examples/rping.c
+++ b/librdmacm/examples/rping.c
@@ -145,6 +145,7 @@ struct rping_cb {
struct sockaddr_storage ssource;
__be16 port; /* dst port in NBO */
int verbose; /* verbose logging */
+ int self_create_qp; /* Create QP not via cma */
int count; /* ping count */
int size; /* ping data size */
int validate; /* validate ping data */
@@ -190,6 +191,12 @@ static int rping_cma_event_handler(struct rdma_cm_id *cma_id,
sem_post(&cb->sem);
break;
+ case RDMA_CM_EVENT_CONNECT_RESPONSE:
+ DEBUG_LOG("CONNECT_RESPONSE\n");
+ cb->state = CONNECTED;
+ sem_post(&cb->sem);
+ break;
+
case RDMA_CM_EVENT_ESTABLISHED:
DEBUG_LOG("ESTABLISHED\n");
@@ -347,13 +354,67 @@ error:
return ret;
}
+static void rping_init_conn_param(struct rping_cb *cb,
+ struct rdma_conn_param *conn_param)
+{
+ memset(conn_param, 0, sizeof(*conn_param));
+ conn_param->responder_resources = 1;
+ conn_param->initiator_depth = 1;
+ conn_param->retry_count = 7;
+ conn_param->rnr_retry_count = 7;
+ if (cb->self_create_qp)
+ conn_param->qp_num = cb->qp->qp_num;
+}
+
+
+static int rping_self_modify_qp(struct rping_cb *cb, struct rdma_cm_id *id)
+{
+ struct ibv_qp_attr qp_attr;
+ int qp_attr_mask, ret;
+
+ qp_attr.qp_state = IBV_QPS_INIT;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ ret = ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask);
+ if (ret)
+ return ret;
+
+ qp_attr.qp_state = IBV_QPS_RTR;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ ret = ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask);
+ if (ret)
+ return ret;
+
+ qp_attr.qp_state = IBV_QPS_RTS;
+ ret = rdma_init_qp_attr(id, &qp_attr, &qp_attr_mask);
+ if (ret)
+ return ret;
+
+ return ibv_modify_qp(cb->qp, &qp_attr, qp_attr_mask);
+}
+
static int rping_accept(struct rping_cb *cb)
{
+ struct rdma_conn_param conn_param;
int ret;
DEBUG_LOG("accepting client connection request\n");
- ret = rdma_accept(cb->child_cm_id, NULL);
+ if (cb->self_create_qp) {
+ ret = rping_self_modify_qp(cb, cb->child_cm_id);
+ if (ret)
+ return ret;
+
+ rping_init_conn_param(cb, &conn_param);
+ ret = rdma_accept(cb->child_cm_id, &conn_param);
+ } else {
+ ret = rdma_accept(cb->child_cm_id, NULL);
+ }
if (ret) {
perror("rdma_accept");
return ret;
@@ -480,6 +541,7 @@ static void rping_free_buffers(struct rping_cb *cb)
static int rping_create_qp(struct rping_cb *cb)
{
struct ibv_qp_init_attr init_attr;
+ struct rdma_cm_id *id;
int ret;
memset(&init_attr, 0, sizeof(init_attr));
@@ -491,16 +553,21 @@ static int rping_create_qp(struct rping_cb *cb)
init_attr.send_cq = cb->cq;
init_attr.recv_cq = cb->cq;
- if (cb->server) {
- ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr);
- if (!ret)
- cb->qp = cb->child_cm_id->qp;
- } else {
- ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr);
- if (!ret)
- cb->qp = cb->cm_id->qp;
+ if (cb->self_create_qp) {
+ cb->qp = ibv_create_qp(cb->pd, &init_attr);
+ if (!cb->qp) {
+ perror("ibv_create_qp");
+ return -1;
+ }
+ return 0;
}
+ id = cb->server ? cb->child_cm_id : cb->cm_id;
+ ret = rdma_create_qp(id, cb->pd, &init_attr);
+ if (!ret)
+ cb->qp = id->qp;
+ else
+ perror("rdma_create_qp");
return ret;
}
@@ -549,7 +616,6 @@ static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id)
ret = rping_create_qp(cb);
if (ret) {
- perror("rdma_create_qp");
goto err3;
}
DEBUG_LOG("created qp %p\n", cb->qp);
@@ -1011,11 +1077,7 @@ static int rping_connect_client(struct rping_cb *cb)
struct rdma_conn_param conn_param;
int ret;
- memset(&conn_param, 0, sizeof conn_param);
- conn_param.responder_resources = 1;
- conn_param.initiator_depth = 1;
- conn_param.retry_count = 7;
-
+ rping_init_conn_param(cb, &conn_param);
ret = rdma_connect(cb->cm_id, &conn_param);
if (ret) {
perror("rdma_connect");
@@ -1028,6 +1090,20 @@ static int rping_connect_client(struct rping_cb *cb)
return -1;
}
+ if (cb->self_create_qp) {
+ ret = rping_self_modify_qp(cb, cb->cm_id);
+ if (ret) {
+ perror("rping_modify_qp");
+ return ret;
+ }
+
+ ret = rdma_establish(cb->cm_id);
+ if (ret) {
+ perror("rdma_establish");
+ return ret;
+ }
+ }
+
DEBUG_LOG("rmda_connect successful\n");
return 0;
}
@@ -1160,6 +1236,7 @@ static void usage(const char *name)
printf("\t-a addr\t\taddress\n");
printf("\t-p port\t\tport\n");
printf("\t-P\t\tpersistent server mode allowing multiple connections\n");
+ printf("\t-q\t\tuse self-created, self-modified QP\n");
}
int main(int argc, char *argv[])
@@ -1182,7 +1259,7 @@ int main(int argc, char *argv[])
sem_init(&cb->sem, 0, 0);
opterr = 0;
- while ((op=getopt(argc, argv, "a:I:Pp:C:S:t:scvVd")) != -1) {
+ while ((op = getopt(argc, argv, "a:I:Pp:C:S:t:scvVdq")) != -1) {
switch (op) {
case 'a':
ret = get_addr(optarg, (struct sockaddr *) &cb->sin);
@@ -1236,6 +1313,9 @@ int main(int argc, char *argv[])
case 'd':
debug++;
break;
+ case 'q':
+ cb->self_create_qp = 1;
+ break;
default:
usage("rping");
ret = EINVAL;
diff --git a/librdmacm/man/rping.1 b/librdmacm/man/rping.1
index 8d4f157..7ec530e 100644
--- a/librdmacm/man/rping.1
+++ b/librdmacm/man/rping.1
@@ -52,6 +52,9 @@ The size of each message transferred, in bytes. (default 100)
\-P
Run the server in persistent mode. This allows multiple rping clients
to connect to a single server instance. The server will run until killed.
+.TP
+\-q
+Control QP Creation/Modification directly from the application, instead of rdma_cm.
.SH "NOTES"
Because this test maps RDMA resources to userspace, users must ensure
that they have available system resources and permissions. See the
--
1.8.3.1
