From: Guy Levi <guyle@xxxxxxxxxxxx>
Expose a new flag in ibv_rc_pingpong command line so send WRs will be
posted by the new post send method as introduced previously by
libibverbs. It can be used as a simple example for this API usage, as a
sanity for this API functionality or for any other goal.
We use this opportunity to fix the man page with device memory option
description which was missed.
Signed-off-by: Guy Levi <guyle@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
libibverbs/examples/rc_pingpong.c | 49 ++++++++++++++++++++++++++++++++++++---
libibverbs/man/ibv_rc_pingpong.1 | 10 ++++++--
2 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/libibverbs/examples/rc_pingpong.c b/libibverbs/examples/rc_pingpong.c
index d03dd7d..9781c4f 100644
--- a/libibverbs/examples/rc_pingpong.c
+++ b/libibverbs/examples/rc_pingpong.c
@@ -62,6 +62,7 @@ static int prefetch_mr;
static int use_ts;
static int validate_buf;
static int use_dm;
+static int use_new_send;
struct pingpong_context {
struct ibv_context *context;
@@ -74,6 +75,7 @@ struct pingpong_context {
struct ibv_cq_ex *cq_ex;
} cq_s;
struct ibv_qp *qp;
+ struct ibv_qp_ex *qpx;
char *buf;
int size;
int send_flags;
@@ -492,12 +494,35 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
.qp_type = IBV_QPT_RC
};
- ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
+ if (use_new_send) {
+ struct ibv_qp_init_attr_ex init_attr_ex = {};
+
+ init_attr_ex.send_cq = pp_cq(ctx);
+ init_attr_ex.recv_cq = pp_cq(ctx);
+ init_attr_ex.cap.max_send_wr = 1;
+ init_attr_ex.cap.max_recv_wr = rx_depth;
+ init_attr_ex.cap.max_send_sge = 1;
+ init_attr_ex.cap.max_recv_sge = 1;
+ init_attr_ex.qp_type = IBV_QPT_RC;
+
+ init_attr_ex.comp_mask |= IBV_QP_INIT_ATTR_PD |
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+ init_attr_ex.pd = ctx->pd;
+ init_attr_ex.send_ops_flags = IBV_QP_EX_WITH_SEND;
+
+ ctx->qp = ibv_create_qp_ex(ctx->context, &init_attr_ex);
+ } else {
+ ctx->qp = ibv_create_qp(ctx->pd, &init_attr);
+ }
+
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
goto clean_cq;
}
+ if (use_new_send)
+ ctx->qpx = ibv_qp_to_qp_ex(ctx->qp);
+
ibv_query_qp(ctx->qp, &attr, IBV_QP_CAP, &init_attr);
if (init_attr.cap.max_inline_data >= size && !use_dm)
ctx->send_flags |= IBV_SEND_INLINE;
@@ -640,7 +665,19 @@ static int pp_post_send(struct pingpong_context *ctx)
};
struct ibv_send_wr *bad_wr;
- return ibv_post_send(ctx->qp, &wr, &bad_wr);
+ if (use_new_send) {
+ ibv_wr_start(ctx->qpx);
+
+ ctx->qpx->wr_id = PINGPONG_SEND_WRID;
+ ctx->qpx->wr_flags = ctx->send_flags;
+
+ ibv_wr_send(ctx->qpx);
+ ibv_wr_set_sge(ctx->qpx, list.lkey, list.addr, list.length);
+
+ return ibv_wr_complete(ctx->qpx);
+ } else {
+ return ibv_post_send(ctx->qp, &wr, &bad_wr);
+ }
}
struct ts_params {
@@ -749,6 +786,7 @@ static void usage(const char *argv0)
printf(" -t, --ts get CQE with timestamp\n");
printf(" -c, --chk validate received buffer\n");
printf(" -j, --dm use device memory\n");
+ printf(" -N, --new_send use new post send WR API\n");
}
int main(int argc, char *argv[])
@@ -798,10 +836,11 @@ int main(int argc, char *argv[])
{ .name = "ts", .has_arg = 0, .val = 't' },
{ .name = "chk", .has_arg = 0, .val = 'c' },
{ .name = "dm", .has_arg = 0, .val = 'j' },
+ { .name = "new_send", .has_arg = 0, .val = 'N' },
{}
};
- c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:oOPtcj",
+ c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:oOPtcjN",
long_options, NULL);
if (c == -1)
@@ -881,6 +920,10 @@ int main(int argc, char *argv[])
use_dm = 1;
break;
+ case 'N':
+ use_new_send = 1;
+ break;
+
default:
usage(argv[0]);
return 1;
diff --git a/libibverbs/man/ibv_rc_pingpong.1 b/libibverbs/man/ibv_rc_pingpong.1
index 5561fe5..92554c0 100644
--- a/libibverbs/man/ibv_rc_pingpong.1
+++ b/libibverbs/man/ibv_rc_pingpong.1
@@ -8,12 +8,12 @@ ibv_rc_pingpong \- simple InfiniBand RC transport test
.B ibv_rc_pingpong
[\-p port] [\-d device] [\-i ib port] [\-s size] [\-m size]
[\-r rx depth] [\-n iters] [\-l sl] [\-e] [\-g gid index]
-[\-o] [\-P] [\-t] \fBHOSTNAME\fR
+[\-o] [\-P] [\-t] [\-j] [\-N] \fBHOSTNAME\fR
.B ibv_rc_pingpong
[\-p port] [\-d device] [\-i ib port] [\-s size] [\-m size]
[\-r rx depth] [\-n iters] [\-l sl] [\-e] [\-g gid index]
-[\-o] [\-P] [\-t]
+[\-o] [\-P] [\-t] [\-j] [\-N]
.SH DESCRIPTION
.PP
@@ -66,6 +66,12 @@ get CQE with timestamp
.TP
\fB\-c\fR, \fB\-\-chk\fR
validate received buffer
+.TP
+\fB\-j\fR, \fB\-\-dm\fR
+use device memory
+.TP
+\fB\-N\fR, \fB\-\-new_send\fR
+use new post send WR API
.SH SEE ALSO
.BR ibv_uc_pingpong (1),
--
1.8.3.1
