[PATCH] Add support for modular backends

[Ronnie Sahlberg <ronniesahlberg@xxxxxxxxx>]

Link tgtd with -E so all symbols are available for runtime linking
by dynamically loaded objects.
Build some modules, such as bs_{aio|rbd} as shared objects and
install them under /usr/lib/tgtd/backing-store
Add support so that when tgtd is started and initializes the backends
it will traverse /usr/lib/tgtd/backing-store and dynamically load
all object files.

This allows to build and distribute backends as separate packages than tgtd.
TGTD can be shipped with only base dependencies and basic backend support
and then those backends that include extra dependencies can be shipped
and installed separately.

Signed-off-by: Ronnie Sahlberg <ronniesahlberg@xxxxxxxxx>
---
 usr/Makefile |   35 ++++++++++++++++++++++++-----------
 usr/bs.c     |   52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 usr/bs_aio.c |    2 +-
 usr/bs_rbd.c |    2 +-
 4 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/usr/Makefile b/usr/Makefile
index 453eb1a..785850e 100644
--- a/usr/Makefile
+++ b/usr/Makefile
@@ -1,4 +1,5 @@
 sbindir ?= $(PREFIX)/sbin
+libdir ?= $(PREFIX)/lib/tgt
 
 ifneq ($(shell test -e /usr/include/linux/signalfd.h && echo 1),)
 CFLAGS += -DUSE_SIGNALFD
@@ -11,17 +12,14 @@ endif
 TGTD_OBJS += $(addprefix iscsi/, conn.o param.o session.o \
 		iscsid.o target.o chap.o sha1.o md5.o transport.o iscsi_tcp.o \
 		isns.o)
-TGTD_OBJS += bs_rdwr.o
 
 ifneq ($(CEPH_RBD),)
-TGTD_OBJS += bs_rbd.o
-LIBS += -lrados -lrbd
+MODULES += bs_rbd.so
 endif
 
 ifneq ($(shell test -e /usr/include/sys/eventfd.h && test -e /usr/include/libaio.h && echo 1),)
 CFLAGS += -DUSE_EVENTFD
-TGTD_OBJS += bs_aio.o
-LIBS += -laio
+MODULES += bs_aio.o
 endif
 
 ifneq ($(ISCSI_RDMA),)
@@ -40,22 +38,26 @@ CFLAGS += -g -O2 -fno-strict-aliasing
 endif
 CFLAGS += -Wall -Wstrict-prototypes -fPIC
 CFLAGS += -DTGT_VERSION=\"$(VERSION)$(EXTRAVERSION)\"
+CFLAGS += -DBSDIR=\"$(DESTDIR)$(libdir)/backing-store\"
 
-LIBS += -lpthread
+LIBS += -lpthread -ldl
 
 PROGRAMS += tgtd tgtadm tgtimg
 TGTD_OBJS += tgtd.o mgmt.o target.o scsi.o log.o driver.o util.o work.o \
 		concat_buf.o parser.o spc.o sbc.o mmc.o osd.o scc.o smc.o \
-		ssc.o bs_ssc.o libssc.o \
+		ssc.o libssc.o bs_rdwr.o bs_ssc.o \
 		bs_null.o bs_sg.o bs.o libcrc32c.o
 
 TGTD_DEP = $(TGTD_OBJS:.o=.d)
 
+LDFLAGS = -Wl,-E,-rpath=$(libdir)
+
 .PHONY:all
-all: $(PROGRAMS)
+all: $(PROGRAMS) $(MODULES)
 
 tgtd: $(TGTD_OBJS)
-	$(CC) $^ -o $@ $(LIBS)
+	echo $(CC) $^ -o $@ $(LIBS)
+	$(CC) $^ -o $@ $(LDFLAGS) $(LIBS)
 
 -include $(TGTD_DEP)
 
@@ -79,11 +81,22 @@ tgtimg: $(TGTIMG_OBJS)
 	$(CC) -c $(CFLAGS) $*.c -o $*.o
 	@$(CC) -MM $(CFLAGS) -MF $*.d -MT $*.o $*.c
 
+%.so: %.c
+	$(CC) -shared $(CFLAGS) $*.c -o $*.so
+
+bs_aio.so: bs_aio.c
+	$(CC) -shared $(CFLAGS) bs_aio.c -o bs_aio.so -laio
+
+bs_rbd.so: bs_rbd.c
+	$(CC) -shared $(CFLAGS) bs_rbd.c -o bs_rbd.so -lrados -lrbd
+
 .PHONY: install
-install: $(PROGRAMS)
+install: $(PROGRAMS) $(MODULES)
 	install -d -m 755 $(DESTDIR)$(sbindir)
 	install -m 755 $(PROGRAMS) $(DESTDIR)$(sbindir)
+	install -d -m 755 $(DESTDIR)$(libdir)/backing-store
+	install -m 755 $(MODULES) $(DESTDIR)$(libdir)/backing-store
 
 .PHONY: clean
 clean:
-	rm -f *.[od] $(PROGRAMS) iscsi/*.[od] ibmvio/*.[od] fc/*.[od]
+	rm -f *.[od] *.so $(PROGRAMS) iscsi/*.[od] ibmvio/*.[od] fc/*.[od]
diff --git a/usr/bs.c b/usr/bs.c
index 65c332e..739ab81 100644
--- a/usr/bs.c
+++ b/usr/bs.c
@@ -19,6 +19,12 @@
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  * 02110-1301 USA
  */
+#define _GNU_SOURCE
+#include <dirent.h>
+#include <dlfcn.h>
+#include <linux/fs.h>
+#include <sys/ioctl.h>
+#include <stdio.h>
 #include <errno.h>
 #include <string.h>
 #include <inttypes.h>
@@ -31,7 +37,9 @@
 #include <syscall.h>
 #include <sys/types.h>
 #include <sys/epoll.h>
+#include <sys/stat.h>
 #include <linux/types.h>
+#include <unistd.h>
 
 #include "list.h"
 #include "tgtd.h"
@@ -233,6 +241,50 @@ static int bs_init_signalfd(void)
 {
 	sigset_t mask;
 	int ret;
+	DIR *dir;
+
+	dir = opendir(BSDIR);
+	if (dir == NULL) {
+		eprintf("could not open backing-store module directory %s\n",
+			BSDIR); 
+	} else {
+		struct dirent *dirent;
+		void *handle;
+		while ((dirent = readdir(dir))) {
+			char *soname;
+			void (*register_bs_module)(void);
+
+			if (dirent->d_name[0] == '.') {
+				continue;
+			}
+
+			ret = asprintf(&soname, "%s/%s", BSDIR,
+					dirent->d_name);
+			if (ret == -1) {
+				eprintf("out of memory\n");
+				continue;
+			}
+			handle = dlopen(soname, RTLD_NOW|RTLD_LOCAL);
+			if (handle == NULL) {
+				eprintf("failed to dlopen backing-store "
+					"module %s error %s \n",
+					soname, dlerror());
+				free(soname);
+				continue;
+			}
+			register_bs_module = dlsym(handle, "register_bs_module");
+			if (register_bs_module == NULL) {
+				eprintf("could not find register_bs_module "
+					"symbol in module %s\n",
+					soname);
+				free(soname);
+				continue;
+			}
+			register_bs_module();
+			free(soname);
+		}
+		closedir(dir);
+	}
 
 	pthread_mutex_init(&finished_lock, NULL);
 
diff --git a/usr/bs_aio.c b/usr/bs_aio.c
index c0cbadd..cc59cf6 100644
--- a/usr/bs_aio.c
+++ b/usr/bs_aio.c
@@ -414,7 +414,7 @@ static struct backingstore_template aio_bst = {
 	.bs_cmd_submit  	= bs_aio_cmd_submit,
 };
 
-__attribute__((constructor)) static void bs_rdwr_constructor(void)
+void register_bs_module(void)
 {
 	register_backingstore_template(&aio_bst);
 }
diff --git a/usr/bs_rbd.c b/usr/bs_rbd.c
index b09090b..f1dd77d 100644
--- a/usr/bs_rbd.c
+++ b/usr/bs_rbd.c
@@ -529,7 +529,7 @@ static struct backingstore_template rbd_bst = {
 	.bs_oflags_supported    = O_SYNC | O_DIRECT,
 };
 
-static __attribute__((constructor)) void bs_rbd_constructor(void)
+void register_bs_module(void)
 {
 	register_backingstore_template(&rbd_bst);
 }
-- 
1.7.3.1

--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html