[PATCH 2/2] Changes to support thread per connection

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Add thread per connection support. A new thread is created for
each connection to handle the tcp portion. cmd_done is handled
at the thread level.

Signed-off-by: Chandra Seetharaman <sekharan@xxxxxxxxxx>
---
 usr/iscsi/conn.c      |    4 +++
 usr/iscsi/iscsi_tcp.c |   78 +++++++++++++++++++++++++++++++++++++++++---------
 usr/iscsi/iscsid.c    |   27 +++++++++++++++--
 usr/iscsi/iscsid.h    |    4 +++
 usr/target.c          |    4 +-
 5 files changed, 98 insertions(+), 19 deletions(-)

Index: tgt-thread/usr/iscsi/iscsid.h
===================================================================
--- tgt-thread.orig/usr/iscsi/iscsid.h
+++ tgt-thread/usr/iscsi/iscsid.h
@@ -181,6 +181,7 @@ struct iscsi_connection {
 	struct iscsi_task *rx_task;
 	struct iscsi_task *tx_task;
 
+	pthread_mutex_t tx_lock;
 	struct list_head tx_clist;
 
 	struct list_head task_list;
@@ -199,6 +200,9 @@ struct iscsi_connection {
 	} auth;
 
 	struct iscsi_transport *tp;
+
+	struct thread_info conn_ti;
+	pthread_t thread;
 };
 
 #define STATE_FREE		0
Index: tgt-thread/usr/iscsi/iscsi_tcp.c
===================================================================
--- tgt-thread.orig/usr/iscsi/iscsi_tcp.c
+++ tgt-thread/usr/iscsi/iscsi_tcp.c
@@ -42,6 +42,10 @@ static void iscsi_tcp_event_handler(int 
 static int listen_fds[8];
 static struct iscsi_transport iscsi_tcp;
 
+#define NO_PTHREAD		0
+#define PTHREAD_PER_TARGET	1
+#define PTHREAD_PER_CONN	2
+
 struct iscsi_tcp_connection {
 	int fd;
 	int pthread;
@@ -168,15 +172,24 @@ static void iscsi_tcp_event_handler(int 
 
 	if (conn->state == STATE_CLOSE) {
 		dprintf("connection closed %p\n", conn);
-		if (tcp_conn->pthread) {
-			struct iscsi_target *target = conn->session->target;
+		if (tcp_conn->pthread != NO_PTHREAD) {
+			struct thread_info *ti;
 
-			do_tgt_event_del(&target->ti, tcp_conn->fd);
+			if (tcp_conn->pthread == PTHREAD_PER_TARGET) {
+				ti = &conn->session->target->ti;
+				do_tgt_event_del(ti, tcp_conn->fd);
+			} else {
+				ti = &conn->conn_ti;
+				do_tgt_event_del(ti, tcp_conn->fd);
+				conn->conn_ti.stop_pthread = 1;
+			}
 			/* let the main thread handle this */
-			tcp_conn->pthread = 0;
+			tcp_conn->pthread = NO_PTHREAD;
 			tgt_event_add(tcp_conn->fd, EPOLLIN,
 					iscsi_tcp_event_handler, conn);
 		} else {
+			if (conn->thread)
+				pthread_join(conn->thread, NULL);
 			conn_close(conn);
 		}
 	}
@@ -279,16 +292,39 @@ static int iscsi_tcp_conn_login_complete
 static void iscsi_tcp_conn_nexus_init(struct iscsi_connection *conn)
 {
 	struct iscsi_tcp_connection *tcp_conn = TCP_CONN(conn);
-	struct iscsi_target *target = conn->session->target;
+	struct thread_info *ti;
+	int ret;
 
 	if (iscsi_pthread_per_target()) {
 		/* remove the conn from the main thread. */
 		tgt_event_del(tcp_conn->fd);
 
-		tcp_conn->pthread = 1;
+		ti = &conn->session->target->ti;
 
-		do_tgt_event_add(&target->ti, tcp_conn->fd, EPOLLIN,
-				 iscsi_tcp_event_handler, conn);
+		tcp_conn->pthread = PTHREAD_PER_TARGET;
+
+		conn->conn_ti.bsf = NULL;
+		conn->conn_ti.stop_pthread = 0;
+		conn->conn_ti.start_pthread = 0;
+		INIT_LIST_HEAD(&conn->conn_ti.t_list);
+		pthread_rwlock_init(&conn->conn_ti.rwlock, NULL);
+
+		conn->conn_ti.efd = epoll_create(128);
+		if (conn->conn_ti.efd < 0)
+			goto thread_per_target;
+
+		ret = pthread_create(&conn->thread, NULL, thread_fn, &conn->conn_ti);
+		if (!ret) {
+			eprintf("created thread %u for connection %p\n",
+				(unsigned) conn->thread, conn);
+			ti = &conn->conn_ti;
+			tcp_conn->pthread = PTHREAD_PER_CONN;
+		}
+thread_per_target:
+		do_tgt_event_add(ti, tcp_conn->fd, EPOLLIN,
+					iscsi_tcp_event_handler, conn);
+		if (tcp_conn->pthread == PTHREAD_PER_CONN)
+			conn->conn_ti.start_pthread = 1;
 	} else
 		conn->tp->ep_event_modify(conn, EPOLLIN);
 }
@@ -364,16 +400,30 @@ static int iscsi_tcp_show(struct iscsi_c
 static void iscsi_event_modify(struct iscsi_connection *conn, int events)
 {
 	struct iscsi_tcp_connection *tcp_conn = TCP_CONN(conn);
+	struct thread_info *ti;
 	int ret;
 
-	if (tcp_conn->pthread) {
-		do_tgt_event_modify(&conn->session->target->ti,
-						tcp_conn->fd, events);
-	} else {
+	switch (tcp_conn->pthread) {
+	default:
+		eprintf(" Unknown connection pthread state %d\n",
+							tcp_conn->pthread);
+		return;
+	case NO_PTHREAD:
 		ret = tgt_event_modify(tcp_conn->fd, events);
-		if (ret)
-			eprintf("tgt_event_modify failed\n");
+		goto done;
+		break;
+	case PTHREAD_PER_TARGET:
+		ti = &conn->session->target->ti;
+		break;
+	case PTHREAD_PER_CONN:
+		ti = &conn->conn_ti;
+		break;
 	}
+
+	ret = do_tgt_event_modify(ti, tcp_conn->fd, events);
+done:
+	if (ret)
+		eprintf("event_modify failed\n");
 }
 
 static struct iscsi_task *iscsi_tcp_alloc_task(struct iscsi_connection *conn,
Index: tgt-thread/usr/target.c
===================================================================
--- tgt-thread.orig/usr/target.c
+++ tgt-thread/usr/target.c
@@ -856,7 +856,7 @@ static int cmd_enabled(struct tgt_cmd_qu
 
 static void cmd_post_perform(struct tgt_cmd_queue *q, struct scsi_cmd *cmd)
 {
-	q->active_cmd++;
+	__sync_fetch_and_add(&q->active_cmd, 1);
 	switch (cmd->attribute) {
 	case MSG_ORDERED_TAG:
 	case MSG_HEAD_TAG:
@@ -1029,7 +1029,7 @@ static void __cmd_done(struct target *ta
 		scsi_get_in_length(cmd), err);
 
 	q = &cmd->dev->cmd_queue;
-	q->active_cmd--;
+	__sync_fetch_and_sub(&q->active_cmd, 1);
 	switch (cmd->attribute) {
 	case MSG_ORDERED_TAG:
 	case MSG_HEAD_TAG:
Index: tgt-thread/usr/iscsi/conn.c
===================================================================
--- tgt-thread.orig/usr/iscsi/conn.c
+++ tgt-thread/usr/iscsi/conn.c
@@ -63,6 +63,7 @@ int conn_init(struct iscsi_connection *c
 
 	INIT_LIST_HEAD(&conn->clist);
 	INIT_LIST_HEAD(&conn->tx_clist);
+	pthread_mutex_init(&conn->tx_lock, NULL);
 	INIT_LIST_HEAD(&conn->task_list);
 
 	return 0;
@@ -121,6 +122,7 @@ void conn_close(struct iscsi_connection 
 		iscsi_free_task(task);
 	}
 
+	pthread_mutex_lock(&conn->tx_lock);
 	list_for_each_entry_safe(task, tmp, &conn->tx_clist, c_list) {
 		uint8_t op;
 
@@ -151,6 +153,8 @@ void conn_close(struct iscsi_connection 
 			break;
 		}
 	}
+	pthread_mutex_unlock(&conn->tx_lock);
+	pthread_mutex_destroy(&conn->tx_lock);
 
 	if (conn->rx_task) {
 		eprintf("Forcing release of rx task %p %" PRIx64 "\n",
Index: tgt-thread/usr/iscsi/iscsid.c
===================================================================
--- tgt-thread.orig/usr/iscsi/iscsid.c
+++ tgt-thread/usr/iscsi/iscsid.c
@@ -1210,7 +1210,9 @@ static int iscsi_scsi_cmd_done(uint64_t 
 		task->len = read_len;
 	}
 
+	pthread_mutex_lock(&task->conn->tx_lock);
 	list_add_tail(&task->c_list, &task->conn->tx_clist);
+	pthread_mutex_unlock(&task->conn->tx_lock);
 	task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
 
 	return 0;
@@ -1335,8 +1337,11 @@ int iscsi_scsi_cmd_execute(struct iscsi_
 	int ret = 0;
 
 	if ((req->flags & ISCSI_FLAG_CMD_WRITE) && task->r2t_count) {
-		if (!task->unsol_count)
+		if (!task->unsol_count) {
+			pthread_mutex_lock(&task->conn->tx_lock);
 			list_add_tail(&task->c_list, &task->conn->tx_clist);
+			pthread_mutex_unlock(&task->conn->tx_lock);
+		}
 		goto no_queuing;
 	}
 
@@ -1371,7 +1376,9 @@ static int iscsi_tm_done(struct mgmt_req
 		task->result = ISCSI_TMF_RSP_REJECTED;
 		break;
 	}
+	pthread_mutex_lock(&task->conn->tx_lock);
 	list_add_tail(&task->c_list, &task->conn->tx_clist);
+	pthread_mutex_unlock(&task->conn->tx_lock);
 	task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
 	return 0;
 }
@@ -1441,7 +1448,9 @@ static int iscsi_task_execute(struct isc
 	switch (op) {
 	case ISCSI_OP_NOOP_OUT:
 	case ISCSI_OP_LOGOUT:
+		pthread_mutex_lock(&task->conn->tx_lock);
 		list_add_tail(&task->c_list, &task->conn->tx_clist);
+		pthread_mutex_unlock(&task->conn->tx_lock);
 		task->conn->tp->ep_event_modify(task->conn, EPOLLIN | EPOLLOUT);
 		break;
 	case ISCSI_OP_SCSI_CMD:
@@ -1461,7 +1470,9 @@ static int iscsi_task_execute(struct isc
 	case ISCSI_OP_SCSI_TMFUNC:
 		err = iscsi_tm_execute(task);
 		if (err) {
+			pthread_mutex_lock(&task->conn->tx_lock);
 			list_add_tail(&task->c_list, &task->conn->tx_clist);
+			pthread_mutex_unlock(&task->conn->tx_lock);
 			task->conn->tp->ep_event_modify(task->conn,
 							EPOLLIN | EPOLLOUT);
 		}
@@ -1548,12 +1559,16 @@ static int iscsi_task_queue(struct iscsi
 
 		if (list_empty(&session->pending_cmd_list))
 			return 0;
+		pthread_mutex_lock(&task->conn->tx_lock);
 		task = list_first_entry(&session->pending_cmd_list,
 					struct iscsi_task, c_list);
-		if (be32_to_cpu(task->req.statsn) != cmd_sn)
+		if (be32_to_cpu(task->req.statsn) != cmd_sn) {
+			pthread_mutex_unlock(&task->conn->tx_lock);
 			return 0;
+		}
 
 		list_del(&task->c_list);
+		pthread_mutex_unlock(&task->conn->tx_lock);
 		clear_task_pending(task);
 		goto retry;
 	} else {
@@ -1828,7 +1843,9 @@ static int iscsi_scsi_cmd_tx_done(struct
 		    scsi_get_data_dir(&task->scmd) == DATA_BIDIRECTIONAL ||
 		    conn->tp->rdma) {
 			dprintf("more data or sense or bidir %x\n", hdr->itt);
+			pthread_mutex_lock(&task->conn->tx_lock);
 			list_add(&task->c_list, &task->conn->tx_clist);
+			pthread_mutex_unlock(&task->conn->tx_lock);
 			return 0;
 		}
 	case ISCSI_OP_SCSI_CMD_RSP:
@@ -1870,8 +1887,11 @@ static int iscsi_task_tx_start(struct is
 	struct iscsi_task *task;
 	int is_rsp, err = 0;
 
-	if (list_empty(&conn->tx_clist))
+	pthread_mutex_lock(&conn->tx_lock);
+	if (list_empty(&conn->tx_clist)) {
+		pthread_mutex_unlock(&conn->tx_lock);
 		goto nodata;
+	}
 
 	conn_write_pdu(conn);
 
@@ -1882,6 +1902,7 @@ static int iscsi_task_tx_start(struct is
 		task->r2t_count);
 
 	list_del(&task->c_list);
+	pthread_mutex_unlock(&conn->tx_lock);
 
 	switch (task->req.opcode & ISCSI_OPCODE_MASK) {
 	case ISCSI_OP_SCSI_CMD:
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html


[Index of Archives]     [Linux SCSI]     [Linux RAID]     [Linux Clusters]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]

  Powered by Linux