We use threads and pipe for event notification (I/O completion); I/O
threads signal the main thread when I/Os are complete. signalfd is
faster than it (signalfd is supported as of 2.6.22).
Signed-off-by: FUJITA Tomonori <fujita.tomonori@xxxxxxxxxxxxx>
---
usr/bs.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------
usr/list.h | 23 +++++++++++++
usr/util.h | 34 ++++++++++++++++++--
3 files changed, 148 insertions(+), 13 deletions(-)
diff --git a/usr/bs.c b/usr/bs.c
index cd19b86..df0aab8 100644
--- a/usr/bs.c
+++ b/usr/bs.c
@@ -23,7 +23,15 @@
#include <string.h>
#include <inttypes.h>
#include <pthread.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <syscall.h>
+#include <sys/types.h>
#include <sys/epoll.h>
+#include <linux/types.h>
+#include <linux/signalfd.h>
#include "list.h"
#include "tgtd.h"
@@ -33,6 +41,10 @@
static LIST_HEAD(bst_list);
+static int sig_fd = -1;
+static LIST_HEAD(sig_finished_list);
+static pthread_mutex_t sig_finished_lock;
+
int register_backingstore_template(struct backingstore_template *bst)
{
list_add(&bst->backingstore_siblings, &bst_list);
@@ -140,10 +152,39 @@ rewrite:
}
}
+void bs_sig_request_done(int fd, int events, void *data)
+{
+ int ret;
+ struct scsi_cmd *cmd;
+ struct signalfd_siginfo siginfo[16];
+ LIST_HEAD(list);
+
+ ret = read(fd, (char *)siginfo, sizeof(siginfo));
+ if (ret <= 0) {
+ return;
+ }
+
+ pthread_mutex_lock(&sig_finished_lock);
+ list_splice_init(&sig_finished_list, &list);
+ pthread_mutex_unlock(&sig_finished_lock);
+
+ while (!list_empty(&list)) {
+ cmd = list_first_entry(&list, struct scsi_cmd, bs_list);
+
+ list_del(&cmd->bs_list);
+
+ cmd->scsi_cmd_done(cmd, scsi_get_result(cmd));
+ }
+}
+
static void *bs_thread_worker_fn(void *arg)
{
struct bs_thread_info *info = arg;
struct scsi_cmd *cmd;
+ sigset_t set;
+
+ sigfillset(&set);
+ sigprocmask(SIG_BLOCK, &set, NULL);
pthread_mutex_lock(&info->startup_lock);
dprintf("started this thread\n");
@@ -171,21 +212,58 @@ static void *bs_thread_worker_fn(void *arg)
info->request_fn(cmd);
- pthread_mutex_lock(&info->finished_lock);
- list_add(&cmd->bs_list, &info->finished_list);
- pthread_mutex_unlock(&info->finished_lock);
+ if (sig_fd < 0) {
+ pthread_mutex_lock(&info->finished_lock);
+ list_add(&cmd->bs_list, &info->finished_list);
+ pthread_mutex_unlock(&info->finished_lock);
- pthread_cond_signal(&info->finished_cond);
+ pthread_cond_signal(&info->finished_cond);
+ } else {
+ pthread_mutex_lock(&sig_finished_lock);
+ list_add_tail(&cmd->bs_list, &sig_finished_list);
+ pthread_mutex_unlock(&sig_finished_lock);
+
+ kill(getpid(), SIGUSR2);
+ }
}
pthread_exit(NULL);
}
+static void bs_init(void)
+{
+ static int done = 0;
+ sigset_t mask;
+ int ret;
+
+ if (done)
+ return;
+ done++;
+
+ sigemptyset(&mask);
+ sigaddset(&mask, SIGUSR2);
+ sigprocmask(SIG_BLOCK, &mask, NULL);
+
+ sig_fd = __signalfd(-1, &mask, 0);
+ if (sig_fd < 0)
+ return;
+
+ ret = tgt_event_add(sig_fd, EPOLLIN, bs_sig_request_done, NULL);
+ if (ret < 0) {
+ close (sig_fd);
+ sig_fd = -1;
+ }
+
+ pthread_mutex_init(&sig_finished_lock, NULL);
+}
+
int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
int nr_threads)
{
int i, ret;
+ bs_init();
+
info->request_fn = rfn;
INIT_LIST_HEAD(&info->ack_list);
@@ -211,10 +289,13 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
goto close_command_fd;
}
- ret = tgt_event_add(info->done_fd[0], EPOLLIN, bs_thread_request_done, info);
- if (ret) {
- eprintf("failed to add epoll event\n");
- goto close_done_fd;
+ if (sig_fd < 0) {
+ ret = tgt_event_add(info->done_fd[0], EPOLLIN, bs_thread_request_done,
+ info);
+ if (ret) {
+ eprintf("failed to add epoll event\n");
+ goto close_done_fd;
+ }
}
ret = pthread_create(&info->ack_thread, NULL, bs_thread_ack_fn, info);
@@ -263,7 +344,9 @@ destroy_threads:
eprintf("stopped the worker thread %d\n", i - 1);
}
event_del:
- tgt_event_del(info->done_fd[0]);
+ if (sig_fd < 0)
+ tgt_event_del(info->done_fd[0]);
+
close_done_fd:
close(info->done_fd[0]);
close(info->done_fd[1]);
@@ -300,7 +383,8 @@ void bs_thread_close(struct bs_thread_info *info)
pthread_mutex_destroy(&info->pending_lock);
pthread_mutex_destroy(&info->startup_lock);
- tgt_event_del(info->done_fd[0]);
+ if (sig_fd < 0)
+ tgt_event_del(info->done_fd[0]);
close(info->done_fd[0]);
close(info->done_fd[1]);
diff --git a/usr/list.h b/usr/list.h
index 4d76057..f66ff36 100644
--- a/usr/list.h
+++ b/usr/list.h
@@ -92,4 +92,27 @@ static inline void list_del_init(struct list_head *entry)
INIT_LIST_HEAD(entry);
}
+static inline void __list_splice(const struct list_head *list,
+ struct list_head *prev,
+ struct list_head *next)
+{
+ struct list_head *first = list->next;
+ struct list_head *last = list->prev;
+
+ first->prev = prev;
+ prev->next = first;
+
+ last->next = next;
+ next->prev = last;
+}
+
+static inline void list_splice_init(struct list_head *list,
+ struct list_head *head)
+{
+ if (!list_empty(list)) {
+ __list_splice(list, head, head->next);
+ INIT_LIST_HEAD(list);
+ }
+}
+
#endif
diff --git a/usr/util.h b/usr/util.h
index c941b37..ae22ec0 100644
--- a/usr/util.h
+++ b/usr/util.h
@@ -2,12 +2,13 @@
#define __UTIL_H__
#include <byteswap.h>
+#include <endian.h>
+#include <errno.h>
#include <fcntl.h>
+#include <signal.h>
#include <syscall.h>
#include <unistd.h>
-#include <errno.h>
-#include <endian.h>
-#include <sys/syscall.h>
+
#include "be_byteshift.h"
#define roundup(x, y) ((((x) + ((y) - 1)) / (y)) * (y))
@@ -115,4 +116,31 @@ static inline int __sync_file_range(int fd, __off64_t offset, __off64_t bytes)
#define __sync_file_range(fd, offset, bytes) fsync(fd)
#endif
+#if defined(__NR_signalfd)
+static inline int __signalfd(int fd, const sigset_t *mask, int flags)
+{
+ int fd2, ret;
+
+ fd2 = syscall(__NR_signalfd, fd, mask, _NSIG / 8);
+ if (fd2 < 0)
+ return fd2;
+
+ ret = fcntl(fd2, F_GETFL);
+ if (ret < 0) {
+ close(fd2);
+ return -1;
+ }
+
+ ret = fcntl(fd2, F_SETFL, ret | O_NONBLOCK);
+ if (ret < 0) {
+ close(fd2);
+ return -1;
+ }
+
+ return fd2;
+}
+#else
+#define __signalfd() (-1)
+#endif
+
#endif
--
1.6.0.6
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
