[PATCH] dynamically link libibverbs and librdma

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



dynamically link libibverbs and librdma for using
stgt without having userspace IB (e.g tcp mode).

Signed-off-by: Doron Shoham <dorons@xxxxxxxxxxxx>
---
 usr/Makefile           |    8 +-
 usr/iscsi/iscsi_rdma.c |  214 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 196 insertions(+), 26 deletions(-)

diff --git a/usr/Makefile b/usr/Makefile
index 82ddf07..37b924b 100644
--- a/usr/Makefile
+++ b/usr/Makefile
@@ -14,13 +14,15 @@ TGTD_OBJS += $(addprefix iscsi/, conn.o param.o session.o \
 TGTD_OBJS += bs_rdwr.o bs_aio.o
 
 LIBS += -lcrypto
-ifneq ($(ISCSI_RDMA),)
-CFLAGS += -DISCSI_RDMA
+
+ifneq ($(findstring verbs.h,$(wildcard /usr/include/infiniband/*.h)), )
+ifneq ($(findstring rdma_cma.h,$(wildcard /usr/include/rdma/*.h)), )
 TGTD_OBJS += iscsi/iscsi_rdma.o
-LIBS += -libverbs -lrdmacm
 endif
 endif
 
+endif
+
 ifneq ($(FCP),)
 CFLAGS += -DFCP -DUSE_KERNEL
 TGTD_OBJS += $(addprefix fc/, fc.o)
diff --git a/usr/iscsi/iscsi_rdma.c b/usr/iscsi/iscsi_rdma.c
index 46e6ea8..ebcf050 100644
--- a/usr/iscsi/iscsi_rdma.c
+++ b/usr/iscsi/iscsi_rdma.c
@@ -30,6 +30,8 @@
 #include <sys/epoll.h>
 #include <infiniband/verbs.h>
 #include <rdma/rdma_cma.h>
+#include <dlfcn.h>
+#include <syslog.h>
 
 #include "util.h"
 #include "iscsid.h"
@@ -278,6 +280,67 @@ static int iscsi_rdma_show(struct iscsi_connection *conn, char *buf,
 			   int rest);
 static void iscsi_rdma_event_modify(struct iscsi_connection *conn, int events);
 
+
+static void *pverbs;
+static void *prdma;
+typedef void (*ibv_ack_cq_events_t)(struct ibv_cq *cq, unsigned int nevents);
+typedef struct ibv_pd *(*ibv_alloc_pd_t)(struct ibv_context *context);
+typedef struct ibv_comp_channel *(*ibv_create_comp_channel_t)(struct ibv_context *context);
+typedef struct ibv_cq *(*ibv_create_cq_t)(struct ibv_context *context, int cqe,
+                              void *cq_context,
+                              struct ibv_comp_channel *channel,
+                              int comp_vector);
+typedef int (*ibv_dereg_mr_t)(struct ibv_mr *mr);
+typedef int (*ibv_destroy_qp_t)(struct ibv_qp *qp);
+typedef int (*ibv_get_cq_event_t)(struct ibv_comp_channel *channel,
+                      struct ibv_cq **cq, void **cq_context);
+typedef int (*ibv_query_device_t)(struct ibv_context *context,
+                      struct ibv_device_attr *device_attr);
+typedef struct ibv_mr *(*ibv_reg_mr_t)(struct ibv_pd *pd, void *addr,
+                          size_t length, enum ibv_access_flags access);
+typedef int (*rdma_ack_cm_event_t)(struct rdma_cm_event *event);
+typedef int (*rdma_bind_addr_t)(struct rdma_cm_id *id, struct sockaddr *addr);
+typedef struct rdma_event_channel *(*rdma_create_event_channel_t)(void);
+typedef int (*rdma_create_id_t)(struct rdma_event_channel *channel,
+                   struct rdma_cm_id **id, void *context,
+                   enum rdma_port_space ps);
+typedef int (*rdma_create_qp_t)(struct rdma_cm_id *id, struct ibv_pd *pd,
+                   struct ibv_qp_init_attr *qp_init_attr);
+typedef int (*rdma_destroy_id_t)(struct rdma_cm_id *id);
+typedef int (*rdma_disconnect_t)(struct rdma_cm_id *id);
+typedef int (*rdma_get_cm_event_t)(struct rdma_event_channel *channel,
+                      struct rdma_cm_event **event);
+typedef int (*rdma_listen_t)(struct rdma_cm_id *id, int backlog);
+typedef int (*rdma_accept_t)(struct rdma_cm_id *id, struct rdma_conn_param *conn_param);
+typedef int (*rdma_reject_t)(struct rdma_cm_id *id, const void *private_data,
+                uint8_t private_data_len);
+
+typedef struct {
+	ibv_ack_cq_events_t ibv_ack_cq_events;
+	ibv_alloc_pd_t ibv_alloc_pd;
+	ibv_create_comp_channel_t ibv_create_comp_channel;
+	ibv_create_cq_t ibv_create_cq;
+	ibv_dereg_mr_t ibv_dereg_mr;
+	ibv_destroy_qp_t ibv_destroy_qp;
+	ibv_get_cq_event_t ibv_get_cq_event;
+	ibv_query_device_t ibv_query_device;
+	ibv_reg_mr_t ibv_reg_mr;
+	rdma_ack_cm_event_t rdma_ack_cm_event;
+	rdma_bind_addr_t rdma_bind_addr;
+	rdma_create_event_channel_t rdma_create_event_channel;
+	rdma_create_id_t rdma_create_id;
+	rdma_create_qp_t rdma_create_qp;
+	rdma_destroy_id_t rdma_destroy_id;
+	rdma_disconnect_t rdma_disconnect;
+	rdma_get_cm_event_t rdma_get_cm_event;
+	rdma_listen_t rdma_listen;
+	rdma_accept_t rdma_accept;
+	rdma_reject_t rdma_reject;
+} tgt_fptr_t;
+
+static tgt_fptr_t tgt_fptr;
+
+
 /*
  * Called when ready for full feature, builds resources.
  */
@@ -303,7 +366,7 @@ static int iser_init_comm(struct conn_info *conn)
 		goto out;
 	}
 
-	conn->srmr = ibv_reg_mr(conn->dev->pd, conn->srbuf, size,
+	conn->srmr = tgt_fptr.ibv_reg_mr(conn->dev->pd, conn->srbuf, size,
 				IBV_ACCESS_LOCAL_WRITE);
 	if (!conn->srmr) {
 		eprintf("register srbuf\n");
@@ -410,7 +473,7 @@ static int iser_init_comm_login(struct conn_info *conn)
 		goto out;
 	}
 
-	conn->srmr_login = ibv_reg_mr(conn->dev->pd, conn->srbuf_login, size,
+	conn->srmr_login = tgt_fptr.ibv_reg_mr(conn->dev->pd, conn->srbuf_login, size,
 				      IBV_ACCESS_LOCAL_WRITE);
 	if (!conn->srmr_login) {
 		eprintf("ibv_reg_mr srbuf failed\n");
@@ -490,7 +553,7 @@ static void iser_free_comm(struct conn_info *ci)
 
 	/* release mr and free the lists */
 	dprintf("dereg mr %p\n", ci->srmr);
-	ret = ibv_dereg_mr(ci->srmr);
+	ret = tgt_fptr.ibv_dereg_mr(ci->srmr);
 	if (ret)
 		eprintf("ibv_dereg_mr\n");
 	free(ci->srbuf);
@@ -510,7 +573,7 @@ static void iser_free_comm_login(struct conn_info *ci)
 	dprintf("freeing, login phase %d\n", ci->login_phase);
 
 	/* release mr and free the lists */
-	ret = ibv_dereg_mr(ci->srmr_login);
+	ret = tgt_fptr.ibv_dereg_mr(ci->srmr_login);
 	if (ret)
 		eprintf("ibv_dereg_mr\n");
 	free(ci->srbuf_login);
@@ -541,7 +604,7 @@ static int iser_init_mempool(struct iser_device *dev)
 		return -ENOMEM;
 	}
 
-	dev->mempool_mr = ibv_reg_mr(dev->pd, regbuf,
+	dev->mempool_mr = tgt_fptr.ibv_reg_mr(dev->pd, regbuf,
 				     mempool_num * mempool_size,
 				     IBV_ACCESS_LOCAL_WRITE);
 	if (!dev->mempool_mr) {
@@ -578,7 +641,7 @@ static int iser_device_init(struct iser_device *dev)
 	int ret = -1;
 
 	dprintf("dev %p\n", dev);
-	dev->pd = ibv_alloc_pd(dev->ibv_hndl);
+	dev->pd = tgt_fptr.ibv_alloc_pd(dev->ibv_hndl);
 	if (dev->pd == NULL) {
 		eprintf("ibv_alloc_pd failed\n");
 		goto out;
@@ -590,7 +653,7 @@ static int iser_device_init(struct iser_device *dev)
 		goto out;
 	}
 
-	ret = ibv_query_device(dev->ibv_hndl, &device_attr);
+	ret = tgt_fptr.ibv_query_device(dev->ibv_hndl, &device_attr);
 	if (ret < 0) {
 		eprintf("ibv_query_device: %m\n");
 		goto out;
@@ -599,13 +662,13 @@ static int iser_device_init(struct iser_device *dev)
 	dprintf("max %d CQEs\n", cqe_num);
 
 	ret = -1;
-	dev->cq_channel = ibv_create_comp_channel(dev->ibv_hndl);
+	dev->cq_channel = tgt_fptr.ibv_create_comp_channel(dev->ibv_hndl);
 	if (dev->cq_channel == NULL) {
 		eprintf("ibv_create_comp_channel failed: %m\n");
 		goto out;
 	}
 
-	dev->cq = ibv_create_cq(dev->ibv_hndl, cqe_num, NULL,
+	dev->cq = tgt_fptr.ibv_create_cq(dev->ibv_hndl, cqe_num, NULL,
 				dev->cq_channel, 0);
 	if (dev->cq == NULL) {
 		eprintf("ibv_create_cq failed: %m\n");
@@ -719,7 +782,7 @@ static void iser_accept_connection(struct rdma_cm_event *event)
 	/* only generate completion queue entries if requested */
 	qp_init_attr.sq_sig_all = 0;
 
-	ret = rdma_create_qp(ci->cma_id, dev->pd, &qp_init_attr);
+	ret = tgt_fptr.rdma_create_qp(ci->cma_id, dev->pd, &qp_init_attr);
 	if (ret) {
 		eprintf("create qp failed\n");
 		goto free_conn;
@@ -749,7 +812,7 @@ static void iser_accept_connection(struct rdma_cm_event *event)
 	}
 
 	/* now we can actually accept the connection */
-	ret = rdma_accept(ci->cma_id, &conn_param);
+	ret = tgt_fptr.rdma_accept(ci->cma_id, &conn_param);
 	if (ret) {
 		eprintf("rdma_accept failed\n");
 		iser_free_comm_login(ci);
@@ -762,7 +825,7 @@ free_conn:
 	conn_exit(conn);
 	free(ci);
 reject:
-	ret = rdma_reject(event->id, NULL, 0);
+	ret = tgt_fptr.rdma_reject(event->id, NULL, 0);
 	if (ret)
 		eprintf("rdma_reject failed: %s\n", strerror(-ret));
 }
@@ -826,7 +889,7 @@ static void iser_handle_rdmacm(int fd __attribute__((unused)),
 	struct rdma_cm_id *destroy_cm_id = NULL;
 
 	dprintf("entry\n");
-	ret = rdma_get_cm_event(rdma_evt_channel, &event);
+	ret = tgt_fptr.rdma_get_cm_event(rdma_evt_channel, &event);
 	if (ret) {
 		eprintf("rdma_get_cm_event failed\n");
 		return;
@@ -849,14 +912,14 @@ static void iser_handle_rdmacm(int fd __attribute__((unused)),
 		break;
 	}
 
-	ret = rdma_ack_cm_event(event);
+	ret = tgt_fptr.rdma_ack_cm_event(event);
 	if (ret) {
 		eprintf("ack cm event failed\n");
 		return;
 	}
 
 	if (destroy_cm_id) {
-		ret = rdma_destroy_id(destroy_cm_id);
+		ret = tgt_fptr.rdma_destroy_id(destroy_cm_id);
 		if (ret)
 			eprintf("rdma_destroy_id failed\n");
 	}
@@ -985,13 +1048,13 @@ static void iser_cqe_handler(int fd __attribute__((unused)),
 	void *cq_context;
 	struct iser_device *dev = data;
 
-	ret = ibv_get_cq_event(dev->cq_channel, &dev->cq, &cq_context);
+	ret = tgt_fptr.ibv_get_cq_event(dev->cq_channel, &dev->cq, &cq_context);
 	if (ret != 0) {
 		eprintf("notification, but no CQ event\n");
 		exit(1);
 	}
 
-	ibv_ack_cq_events(dev->cq, 1);
+    tgt_fptr.ibv_ack_cq_events(dev->cq, 1);
 
 	ret = ibv_req_notify_cq(dev->cq, 0);
 	if (ret) {
@@ -1118,14 +1181,14 @@ static int iscsi_rdma_init(void)
 	struct sockaddr_in sock_addr;
 	short int port = ISCSI_LISTEN_PORT;
 
-	rdma_evt_channel = rdma_create_event_channel();
+	rdma_evt_channel = tgt_fptr.rdma_create_event_channel();
 
 	if (!rdma_evt_channel) {
 		eprintf("cannot initialize RDMA; load kernel modules?\n");
 		return -1;
 	}
 
-	ret = rdma_create_id(rdma_evt_channel, &cma_listen_id, NULL,
+	ret = tgt_fptr.rdma_create_id(rdma_evt_channel, &cma_listen_id, NULL,
 			     RDMA_PS_TCP);
 	if (ret) {
 		eprintf("rdma_create_id: %s\n", strerror(ret));
@@ -1136,7 +1199,7 @@ static int iscsi_rdma_init(void)
 	sock_addr.sin_family = AF_INET;
 	sock_addr.sin_port = htons(port);
 	sock_addr.sin_addr.s_addr = INADDR_ANY;
-	ret = rdma_bind_addr(cma_listen_id, (struct sockaddr *) &sock_addr);
+	ret = tgt_fptr.rdma_bind_addr(cma_listen_id, (struct sockaddr *) &sock_addr);
 	if (ret) {
 		if (ret == -1)
 			eprintf("rdma_bind_addr -1: %m\n");
@@ -1146,7 +1209,7 @@ static int iscsi_rdma_init(void)
 	}
 
 	/* 0 == maximum backlog */
-	ret = rdma_listen(cma_listen_id, 0);
+	ret = tgt_fptr.rdma_listen(cma_listen_id, 0);
 	if (ret) {
 		if (ret == -1)
 			eprintf("rdma_listen -1: %m\n");
@@ -1559,7 +1622,7 @@ static size_t iscsi_rdma_close(struct iscsi_connection *conn)
 	struct conn_info *ci = RDMA_CONN(conn);
 	int ret;
 
-	ret = rdma_disconnect(ci->cma_id);
+	ret = tgt_fptr.rdma_disconnect(ci->cma_id);
 	if (ret)
 		eprintf("rdma_disconnect: %s\n", strerror(-ret));
 	dprintf("did rdma_disconnect\n");
@@ -1592,7 +1655,7 @@ static void iscsi_rdma_release(struct iscsi_connection *conn)
 		iser_free_comm(ci);
 
 	/* finally destory QP */
-	ret = ibv_destroy_qp(ci->qp_hndl);
+	ret = tgt_fptr.ibv_destroy_qp(ci->qp_hndl);
 	if (ret)
 		eprintf("ibv_destroy_qp: %s\n", strerror(-ret));
 
@@ -1729,7 +1792,112 @@ static struct iscsi_transport iscsi_iser = {
 	.ep_getpeername		= iscsi_rdma_getpeername,
 };
 
+
+int is_dlerror(const char *symbol)
+{
+	char *error;
+
+	if ((error = dlerror()) != NULL) {
+		syslog(LOG_ERR, "symbol %s not found - %s", symbol, error);
+		return 1;
+	}
+	return 0;
+}
+
+
+
 __attribute__((constructor)) static void iser_transport_init(void)
 {
+	pverbs = dlopen("libibverbs.so",RTLD_LAZY);
+	if (!pverbs) {
+		goto Exit; /* do not register iser transport */
+	}
+
+	prdma = dlopen("librdmacm.so",RTLD_LAZY);
+	if (!prdma) {
+		goto Exit; /* do not register iser transport */
+	}
+
+	/* initialize function pointers */
+	tgt_fptr.ibv_ack_cq_events = dlsym(pverbs, "ibv_ack_cq_events");
+	if (is_dlerror("ibv_ack_cq_events"))
+		goto Exit;
+	tgt_fptr.ibv_alloc_pd = dlsym(pverbs, "ibv_alloc_pd");
+	if (is_dlerror("ibv_alloc_pd"))
+		goto Exit;
+	tgt_fptr.ibv_create_comp_channel = dlsym(pverbs, "ibv_create_comp_channel");
+	if (is_dlerror("ibv_create_comp_channel"))
+		goto Exit;
+	tgt_fptr.ibv_create_cq = dlsym(pverbs, "ibv_create_cq");
+	if (is_dlerror("ibv_create_cq"))
+		goto Exit;
+	tgt_fptr.ibv_dereg_mr = dlsym(pverbs, "ibv_dereg_mr");
+	if (is_dlerror("ibv_dereg_mr"))
+		goto Exit;
+	tgt_fptr.ibv_destroy_qp = dlsym(pverbs, "ibv_destroy_qp");
+	if (is_dlerror("ibv_destroy_qp"))
+		goto Exit;
+	tgt_fptr.ibv_get_cq_event = dlsym(pverbs, "ibv_get_cq_event");
+	if (is_dlerror("ibv_get_cq_event"))
+		goto Exit;
+	tgt_fptr.ibv_query_device = dlsym(pverbs, "ibv_query_device");
+	if (is_dlerror("ibv_query_device"))
+		goto Exit;
+	tgt_fptr.ibv_reg_mr = dlsym(pverbs, "ibv_reg_mr");
+	if (is_dlerror("ibv_reg_mr"))
+		goto Exit;
+	tgt_fptr.rdma_ack_cm_event = dlsym(prdma, "rdma_ack_cm_event");
+	if (is_dlerror("rdma_ack_cm_event"))
+		goto Exit;
+	tgt_fptr.rdma_bind_addr = dlsym(prdma, "rdma_bind_addr");
+	if (is_dlerror("rdma_bind_addr"))
+		goto Exit;
+	tgt_fptr.rdma_create_event_channel = dlsym(prdma, "rdma_create_event_channel");
+	if (is_dlerror("rdma_create_event_channel"))
+		goto Exit;
+	tgt_fptr.rdma_create_id = dlsym(prdma, "rdma_create_id");
+	if (is_dlerror("rdma_create_id"))
+		goto Exit;
+	tgt_fptr.rdma_create_qp = dlsym(prdma, "rdma_create_qp");
+	if (is_dlerror("rdma_create_qp"))
+		goto Exit;
+	tgt_fptr.rdma_destroy_id = dlsym(prdma, "rdma_destroy_id");
+	if (is_dlerror("rdma_destroy_id"))
+		goto Exit;
+	tgt_fptr.rdma_disconnect = dlsym(prdma, "rdma_disconnect");
+	if (is_dlerror("rdma_disconnect"))
+		goto Exit;
+	tgt_fptr.rdma_get_cm_event = dlsym(prdma, "rdma_get_cm_event");
+	if (is_dlerror("rdma_get_cm_event"))
+		goto Exit;
+	tgt_fptr.rdma_listen = dlsym(prdma, "rdma_listen");
+	if (is_dlerror("rdma_listen"))
+		goto Exit;
+	tgt_fptr.rdma_accept = dlsym(prdma, "rdma_accept");
+	if (is_dlerror("rdma_accept"))
+		goto Exit;
+	tgt_fptr.rdma_reject = dlsym(prdma, "rdma_reject");
+	if (is_dlerror("rdma_reject"))
+		goto Exit;
+
+	syslog(LOG_INFO, "iser transport register");
 	iscsi_transport_register(&iscsi_iser);
+	return;
+
+Exit:
+	syslog(LOG_ERR, "%s - iser transport not used", dlerror());
+	if (pverbs)
+		dlclose(pverbs);
+	if (prdma)
+		dlclose(prdma);
 }
+
+__attribute__((destructor)) static void iser_transport_close(void)
+{
+	syslog(LOG_INFO, "iser transport register");
+	if (pverbs)
+		dlclose(pverbs);
+	if (prdma)
+		dlclose(prdma);
+}
+
-- 
1.5.3.8

--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[Index of Archives]     [Linux SCSI]     [Linux RAID]     [Linux Clusters]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]

  Powered by Linux