[PATCH rdma-core 5/6] verbs: Add option to register ODP MR in ibv_srq_pingpong

[Yishai Hadas <yishaih@xxxxxxxxxxxx>]

From: Moni Shoua <monis@xxxxxxxxxxxx>

Flag -o indicates that the MR should be registered as ODP. Since one
memory region is used for both send and receive operations the flag
applies to send WQEs as well as to receive WQEs.

Signed-off-by: Moni Shoua <monis@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
 libibverbs/examples/srq_pingpong.c | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/libibverbs/examples/srq_pingpong.c b/libibverbs/examples/srq_pingpong.c
index 230e9bf..2a91fd3 100644
--- a/libibverbs/examples/srq_pingpong.c
+++ b/libibverbs/examples/srq_pingpong.c
@@ -56,6 +56,7 @@ enum {
 
 static int page_size;
 static int validate_buf;
+static int use_odp;
 
 struct pingpong_context {
 	struct ibv_context	*context;
@@ -352,6 +353,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 {
 	struct pingpong_context *ctx;
 	int i;
+	int access_flags = IBV_ACCESS_LOCAL_WRITE;
 
 	ctx = calloc(1, sizeof *ctx);
 	if (!ctx)
@@ -376,6 +378,24 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 			ibv_get_device_name(ib_dev));
 		goto clean_buffer;
 	}
+	if (use_odp) {
+		struct ibv_device_attr_ex attrx;
+		const uint32_t rc_caps_mask = IBV_ODP_SUPPORT_SEND |
+					      IBV_ODP_SUPPORT_RECV |
+					      IBV_ODP_SUPPORT_SRQ_RECV;
+
+		if (ibv_query_device_ex(ctx->context, NULL, &attrx)) {
+			fprintf(stderr, "Couldn't query device for its features\n");
+			goto clean_device;
+		}
+		if (!(attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
+		    (attrx.odp_caps.per_transport_caps.rc_odp_caps & rc_caps_mask) != rc_caps_mask) {
+			fprintf(stderr, "The device isn't ODP capable or does not support RC send, receive and srq with ODP\n");
+			goto clean_device;
+		}
+		access_flags |= IBV_ACCESS_ON_DEMAND;
+	}
+
 
 	if (use_event) {
 		ctx->channel = ibv_create_comp_channel(ctx->context);
@@ -392,7 +412,7 @@ static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
 		goto clean_comp_channel;
 	}
 
-	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE);
+	ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, access_flags);
 	if (!ctx->mr) {
 		fprintf(stderr, "Couldn't register MR\n");
 		goto clean_pd;
@@ -617,6 +637,7 @@ static void usage(const char *argv0)
 	printf("  -l, --sl=<sl>          service level value\n");
 	printf("  -e, --events           sleep on CQ events (default poll)\n");
 	printf("  -g, --gid-idx=<gid index> local port gid index\n");
+	printf("  -o, --odp		    use on demand paging\n");
 	printf("  -c, --chk              validate received buffer\n");
 }
 
@@ -664,12 +685,13 @@ int main(int argc, char *argv[])
 			{ .name = "iters",    .has_arg = 1, .val = 'n' },
 			{ .name = "sl",       .has_arg = 1, .val = 'l' },
 			{ .name = "events",   .has_arg = 0, .val = 'e' },
+			{ .name = "odp",      .has_arg = 0, .val = 'o' },
 			{ .name = "gid-idx",  .has_arg = 1, .val = 'g' },
 			{ .name = "chk",      .has_arg = 0, .val = 'c' },
 			{}
 		};
 
-		c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:c",
+		c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eog:c",
 				long_options, NULL);
 		if (c == -1)
 			break;
@@ -735,6 +757,10 @@ int main(int argc, char *argv[])
 			gidx = strtol(optarg, NULL, 0);
 			break;
 
+		case 'o':
+			use_odp = 1;
+			break;
+
 		case 'c':
 			validate_buf = 1;
 			break;
-- 
1.8.3.1