Re: [PATCH v3] tgtd: send/recv iSCSI PDUs in worker threads

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



At Sun, 24 Nov 2013 23:07:52 +0900,
Hitoshi Mitake wrote:
> 
> From: Hitoshi Mitake <mitake.hitoshi@xxxxxxxxxxxxx>
> 
> Current tgtd sends and receives iSCSI PDUs in its main event
> loop. This design can cause bottleneck when many iSCSI clients connect
> to single tgtd process. For example, we need multiple tgtd processes
> for utilizing fast network like 10 GbE because typical single
> processor core isn't fast enough for processing bunch of requests.
> 
> This patch lets tgtd send/receive iSCSI PDUs and check digests in its
> worker threads. After applying this patch, the bottleneck in the main
> event loop is removed and the performance is improved.
> 
> The improvement can be seen even if tgtd and iSCSI initiator are
> running on a single host. Below is a snippet of fio result on my
> linux box. The workload is 128MB random RW. Backingstore is sheepdog.
> 
> Original tgtd:
>   read : io=65008KB, bw=6259.9KB/s, iops=1564 , runt= 10385msec
>   write: io=66064KB, bw=6361.5KB/s, iops=1590 , runt= 10385msec
> 
> tgtd with this patch:
>   read : io=65416KB, bw=7295.2KB/s, iops=1823 , runt=  8967msec
>   write: io=65656KB, bw=7321.1KB/s, iops=1830 , runt=  8967msec
> 
> Signed-off-by: Hitoshi Mitake <mitake.hitoshi@xxxxxxxxxxxxx>
> ---

I benchmarked this patch and found that it has a problem of
performance stability. I'll refine it and send v4 in the near
future. Please ignore this version.

Thanks,
Hitoshi


> 
> v3:
>  - Exeucte iscsi_task_rx_start() and iscsi_task_tx_start() in a serialized
>    manner. Because they manipulate data structures related to iSCSI session.
> 
> v2:
>  - correct handling of connection closing based on a reference count of an iSCSI
>    connection
>  - a silly bug in iscsi_tcp_init() introduced in the previous patch is removed
> 
>  usr/iscsi/conn.c      |  12 ++-
>  usr/iscsi/iscsi_tcp.c | 290 +++++++++++++++++++++++++++++++++++++++++++++++---
>  usr/iscsi/iscsid.c    | 110 +++++++++++++------
>  usr/iscsi/iscsid.h    |   8 +-
>  4 files changed, 372 insertions(+), 48 deletions(-)
> 
> diff --git a/usr/iscsi/conn.c b/usr/iscsi/conn.c
> index 17aa3e3..21b552e 100644
> --- a/usr/iscsi/conn.c
> +++ b/usr/iscsi/conn.c
> @@ -83,7 +83,7 @@ void conn_exit(struct iscsi_connection *conn)
>  		session_put(session);
>  }
>  
> -void conn_close(struct iscsi_connection *conn)
> +int conn_close(struct iscsi_connection *conn)
>  {
>  	struct iscsi_task *task, *tmp;
>  	int ret;
> @@ -180,14 +180,18 @@ void conn_close(struct iscsi_connection *conn)
>  		iscsi_free_task(task);
>  	}
>  done:
> -	conn_put(conn);
> +	return conn_put(conn);
>  }
>  
> -void conn_put(struct iscsi_connection *conn)
> +int conn_put(struct iscsi_connection *conn)
>  {
>  	conn->refcount--;
> -	if (!conn->refcount)
> +	if (!conn->refcount) {
>  		conn->tp->ep_release(conn);
> +		return 0;
> +	}
> +
> +	return conn->refcount;
>  }
>  
>  int conn_get(struct iscsi_connection *conn)
> diff --git a/usr/iscsi/iscsi_tcp.c b/usr/iscsi/iscsi_tcp.c
> index 4a9532a..cf9d734 100644
> --- a/usr/iscsi/iscsi_tcp.c
> +++ b/usr/iscsi/iscsi_tcp.c
> @@ -31,6 +31,7 @@
>  #include <netinet/tcp.h>
>  #include <sys/epoll.h>
>  #include <sys/socket.h>
> +#include <pthread.h>
>  
>  #include "iscsid.h"
>  #include "tgtd.h"
> @@ -48,6 +49,20 @@ static long nop_ttt;
>  static int listen_fds[8];
>  static struct iscsi_transport iscsi_tcp;
>  
> +enum iscsi_tcp_work_state {
> +	ISCSI_TCP_WORK_INIT,
> +	ISCSI_TCP_WORK_RX,
> +	ISCSI_TCP_WORK_TX,
> +	ISCSI_TCP_WORK_TX_EAGAIN,
> +};
> +
> +struct iscsi_tcp_work {
> +	/* list: connected to iscsi_tcp_work_list or iscsi_tcp_finished_list */
> +	struct list_head list;
> +
> +	enum iscsi_tcp_work_state state;
> +};
> +
>  struct iscsi_tcp_connection {
>  	int fd;
>  
> @@ -59,13 +74,197 @@ struct iscsi_tcp_connection {
>  	long ttt;
>  
>  	struct iscsi_connection iscsi_conn;
> +	struct iscsi_tcp_work work;
> +
> +	int used_in_worker_thread;
>  };
>  
> +static LIST_HEAD(iscsi_tcp_work_list);
> +static pthread_mutex_t iscsi_tcp_work_mutex = PTHREAD_MUTEX_INITIALIZER;
> +static pthread_cond_t iscsi_tcp_work_cond = PTHREAD_COND_INITIALIZER;
> +
> +static LIST_HEAD(iscsi_tcp_work_finished_list);
> +static pthread_mutex_t iscsi_tcp_work_finished_mutex =
> +	PTHREAD_MUTEX_INITIALIZER;
> +
> +static int iscsi_tcp_work_done_fds[2];
> +
> +static pthread_mutex_t iscsi_tcp_worker_startup_mutex =
> +	PTHREAD_MUTEX_INITIALIZER;
> +
> +static int iscsi_tcp_worker_stop;
> +
> +static pthread_t *iscsi_tcp_worker_threads;
> +
> +static void iscsi_tcp_work_done_handler(int fd, int events, void *data)
> +{
> +	LIST_HEAD(list);
> +	struct iscsi_tcp_work *work;
> +	struct iscsi_connection *conn;
> +	struct iscsi_tcp_connection *tcp_conn;
> +	int ret;
> +	char dummy = 0;
> +
> +	ret = read(fd, &dummy, sizeof(dummy));
> +	if (ret != sizeof(dummy)) {
> +		eprintf("iscsi tcp work error: %m\n");
> +		exit(1);
> +	}
> +
> +	pthread_mutex_lock(&iscsi_tcp_work_finished_mutex);
> +	list_splice_init(&iscsi_tcp_work_finished_list, &list);
> +	pthread_mutex_unlock(&iscsi_tcp_work_finished_mutex);
> +
> +	while (!list_empty(&list)) {
> +		work = list_first_entry(&list, struct iscsi_tcp_work, list);
> +		list_del(&work->list);
> +
> +		tcp_conn =
> +			container_of(work, struct iscsi_tcp_connection, work);
> +		conn = &tcp_conn->iscsi_conn;
> +
> +		tcp_conn->used_in_worker_thread = 0;
> +
> +		ret = tgt_event_add(tcp_conn->fd, EPOLLOUT | EPOLLIN,
> +				    iscsi_tcp_event_handler, conn);
> +		if (ret) {
> +			eprintf("failed to add event: %m\n");
> +			exit(1);
> +		}
> +
> +		switch (work->state) {
> +		case ISCSI_TCP_WORK_RX:
> +			if (conn->state != STATE_CLOSE)
> +				iscsi_tcp_work_rx_done_fn(conn);
> +			break;
> +		case ISCSI_TCP_WORK_TX:
> +			iscsi_tcp_work_tx_done_fn(conn);
> +			break;
> +		case ISCSI_TCP_WORK_TX_EAGAIN:
> +			/* do nothing */
> +			break;
> +		default:
> +			eprintf("invalid state of iscsi work tcp: %d\n",
> +				work->state);
> +			exit(1);
> +		}
> +
> +		if (conn->state == STATE_CLOSE) {
> +			dprintf("connection closed %p\n", conn);
> +			if (conn_close(conn))
> +				/* reference count of the conn isn't zero */
> +				work->state = ISCSI_TCP_WORK_INIT;
> +		} else
> +			work->state = ISCSI_TCP_WORK_INIT;
> +	}
> +}
> +
> +static void *iscsi_tcp_worker_fn(void *arg)
> +{
> +	sigset_t set;
> +	struct iscsi_tcp_work *work;
> +	struct iscsi_connection *conn;
> +	struct iscsi_tcp_connection *tcp_conn;
> +	int ret;
> +	char dummy = 0;
> +
> +	sigfillset(&set);
> +	sigprocmask(SIG_BLOCK, &set, NULL);
> +
> +	pthread_mutex_lock(&iscsi_tcp_worker_startup_mutex);
> +	pthread_mutex_unlock(&iscsi_tcp_worker_startup_mutex);
> +
> +	dprintf("starting iscsi tcp worker thread: %lu\n", pthread_self());
> +
> +	while (!iscsi_tcp_worker_stop) {
> +		pthread_mutex_lock(&iscsi_tcp_work_mutex);
> +retest:
> +		if (list_empty(&iscsi_tcp_work_list)) {
> +			pthread_cond_wait(&iscsi_tcp_work_cond,
> +					  &iscsi_tcp_work_mutex);
> +
> +			if (iscsi_tcp_worker_stop) {
> +				pthread_mutex_unlock(&iscsi_tcp_work_mutex);
> +				pthread_exit(NULL);
> +			}
> +
> +			goto retest;
> +		}
> +
> +		work = list_first_entry(&iscsi_tcp_work_list,
> +				       struct iscsi_tcp_work, list);
> +
> +		list_del(&work->list);
> +		pthread_mutex_unlock(&iscsi_tcp_work_mutex);
> +
> +		tcp_conn =
> +			container_of(work, struct iscsi_tcp_connection, work);
> +		conn = &tcp_conn->iscsi_conn;
> +
> +		switch (work->state) {
> +		case ISCSI_TCP_WORK_RX:
> +			do {
> +				iscsi_rx_handler(conn);
> +			} while (!is_conn_rx_end(conn)
> +				 && conn->state != STATE_CLOSE);
> +
> +			break;
> +		case ISCSI_TCP_WORK_TX:
> +			do {
> +				ret = iscsi_tx_handler(conn);
> +				if (ret == -EAGAIN) {
> +					/* no data to send */
> +					work->state = ISCSI_TCP_WORK_TX_EAGAIN;
> +					break;
> +				}
> +			} while (!is_conn_tx_end(conn)
> +				 && conn->state != STATE_CLOSE);
> +
> +			break;
> +		default:
> +			eprintf("invalid state of iscsi tcp work: %d\n",
> +				work->state);
> +			exit(1);
> +		}
> +
> +		pthread_mutex_lock(&iscsi_tcp_work_finished_mutex);
> +		list_add_tail(&work->list, &iscsi_tcp_work_finished_list);
> +		pthread_mutex_unlock(&iscsi_tcp_work_finished_mutex);
> +
> +		ret = write(iscsi_tcp_work_done_fds[1], &dummy, sizeof(dummy));
> +		if (ret != sizeof(dummy)) {
> +			eprintf("iscsi tcp work error: %m\n");
> +			exit(1);
> +		}
> +	}
> +
> +	pthread_exit(NULL);
> +}
> +
>  static inline struct iscsi_tcp_connection *TCP_CONN(struct iscsi_connection *conn)
>  {
>  	return container_of(conn, struct iscsi_tcp_connection, iscsi_conn);
>  }
>  
> +static void queue_iscsi_tcp_work(struct iscsi_connection *conn)
> +{
> +	struct iscsi_tcp_connection *tcp_conn = TCP_CONN(conn);
> +	struct iscsi_tcp_work *work = &tcp_conn->work;
> +
> +	/*
> +	 * Delete from main event loop temporaly, because the fd is used by
> +	 * worker threads.
> +	 */
> +	tcp_conn->used_in_worker_thread = 1;
> +	tgt_event_del(tcp_conn->fd);
> +
> +	pthread_mutex_lock(&iscsi_tcp_work_mutex);
> +	list_add_tail(&work->list, &iscsi_tcp_work_list);
> +	pthread_mutex_unlock(&iscsi_tcp_work_mutex);
> +
> +	pthread_cond_signal(&iscsi_tcp_work_cond);
> +}
> +
>  static struct tgt_work nop_work;
>  
>  /* all iscsi connections */
> @@ -102,6 +301,9 @@ static void iscsi_tcp_nop_work_handler(void *data)
>  		if (tcp_conn->nop_tick > 0)
>  			continue;
>  
> +		if (tcp_conn->used_in_worker_thread)
> +			continue;
> +
>  		tcp_conn->nop_tick = tcp_conn->nop_interval;
>  
>  		tcp_conn->nop_inflight_count++;
> @@ -241,6 +443,9 @@ static void accept_connection(int afd, int events, void *data)
>  	if (!tcp_conn)
>  		goto out;
>  
> +	INIT_LIST_HEAD(&tcp_conn->work.list);
> +	tcp_conn->work.state = ISCSI_TCP_WORK_INIT;
> +
>  	conn = &tcp_conn->iscsi_conn;
>  
>  	ret = conn_init(conn);
> @@ -273,20 +478,20 @@ out:
>  static void iscsi_tcp_event_handler(int fd, int events, void *data)
>  {
>  	struct iscsi_connection *conn = (struct iscsi_connection *) data;
> +	struct iscsi_tcp_connection *tcp_conn = TCP_CONN(conn);
> +	struct iscsi_tcp_work *work = &tcp_conn->work;
>  
> -	if (events & EPOLLIN)
> -		iscsi_rx_handler(conn);
> -
> -	if (conn->state == STATE_CLOSE)
> -		dprintf("connection closed\n");
> +	if (work->state != ISCSI_TCP_WORK_INIT) {
> +		eprintf("invalid state of work: %d\n", work->state);
> +		exit(1);
> +	}
>  
> -	if (conn->state != STATE_CLOSE && events & EPOLLOUT)
> -		iscsi_tx_handler(conn);
> +	if (events & EPOLLIN)
> +		work->state = ISCSI_TCP_WORK_RX;
> +	else if (events & EPOLLOUT)
> +		work->state = ISCSI_TCP_WORK_TX;
>  
> -	if (conn->state == STATE_CLOSE) {
> -		dprintf("connection closed %p\n", conn);
> -		conn_close(conn);
> -	}
> +	queue_iscsi_tcp_work(conn);
>  }
>  
>  int iscsi_tcp_init_portal(char *addr, int port, int tpgt)
> @@ -423,6 +628,8 @@ int iscsi_delete_portal(char *addr, int port)
>  
>  static int iscsi_tcp_init(void)
>  {
> +	int ret = 0, i;
> +
>  	/* If we were passed any portals on the command line */
>  	if (portal_arguments)
>  		iscsi_param_parse_portals(portal_arguments, 1, 0);
> @@ -440,17 +647,76 @@ static int iscsi_tcp_init(void)
>  	nop_work.data = &nop_work;
>  	add_work(&nop_work, 1);
>  
> -	return 0;
> +	ret = pipe(iscsi_tcp_work_done_fds);
> +	if (ret < 0) {
> +		eprintf("failed to create pipe for tcp work: %m\n");
> +		return -1;
> +	}
> +
> +	ret = tgt_event_add(iscsi_tcp_work_done_fds[0], EPOLLIN,
> +			    iscsi_tcp_work_done_handler, NULL);
> +	if (ret < 0) {
> +		eprintf("failed to register iscsi_tcp_work_done_handler():"\
> +			" %m\n");
> +		ret = -1;
> +		goto close_done_fds;
> +	}
> +
> +	iscsi_tcp_worker_threads = calloc(nr_iothreads, sizeof(pthread_t));
> +	if (!iscsi_tcp_worker_threads) {
> +		eprintf("failed to allocate memory for pthread identifier:"\
> +			" %m\n");
> +		ret = -1;
> +
> +		goto close_done_fds;
> +	}
> +
> +	pthread_mutex_lock(&iscsi_tcp_worker_startup_mutex);
> +	for (i = 0; i < nr_iothreads; i++) {
> +		ret = pthread_create(&iscsi_tcp_worker_threads[i], NULL,
> +				     iscsi_tcp_worker_fn, NULL);
> +		if (ret) {
> +			eprintf("creating worker thread failed: %m\n");
> +			ret = -1;
> +
> +			goto terminate_workers;
> +		}
> +	}
> +
> +	pthread_mutex_unlock(&iscsi_tcp_worker_startup_mutex);
> +	goto out;
> +
> +terminate_workers:
> +	iscsi_tcp_worker_stop = 1;
> +	pthread_mutex_unlock(&iscsi_tcp_worker_startup_mutex);
> +
> +	for (; 0 <= i; i--)
> +		pthread_join(iscsi_tcp_worker_threads[i], NULL);
> +
> +close_done_fds:
> +	close(iscsi_tcp_work_done_fds[0]);
> +	close(iscsi_tcp_work_done_fds[1]);
> +
> +out:
> +	return ret;
>  }
>  
>  static void iscsi_tcp_exit(void)
>  {
> +	int i;
> +
>  	struct iscsi_portal *portal, *ptmp;
>  
>  	list_for_each_entry_safe(portal, ptmp, &iscsi_portals_list,
>  			    iscsi_portal_siblings) {
>  		iscsi_delete_portal(portal->addr, portal->port);
>  	}
> +
> +	iscsi_tcp_worker_stop = 1;
> +	for (i = 0; i < nr_iothreads; i++) {
> +		pthread_cond_signal(&iscsi_tcp_work_cond);
> +		pthread_join(iscsi_tcp_worker_threads[i], NULL);
> +	}
>  }
>  
>  static int iscsi_tcp_conn_login_complete(struct iscsi_connection *conn)
> diff --git a/usr/iscsi/iscsid.c b/usr/iscsi/iscsid.c
> index 30bd13f..c936553 100644
> --- a/usr/iscsi/iscsid.c
> +++ b/usr/iscsi/iscsid.c
> @@ -33,6 +33,8 @@
>  #include <unistd.h>
>  #include <sys/epoll.h>
>  
> +#include <pthread.h>
> +
>  #include "iscsid.h"
>  #include "tgtd.h"
>  #include "util.h"
> @@ -76,6 +78,16 @@ enum {
>  	IOSTATE_TX_END,
>  };
>  
> +int is_conn_rx_end(struct iscsi_connection *conn)
> +{
> +	return conn->rx_iostate == IOSTATE_RX_END;
> +}
> +
> +int is_conn_tx_end(struct iscsi_connection *conn)
> +{
> +	return conn->tx_iostate == IOSTATE_TX_END;
> +}
> +
>  void conn_read_pdu(struct iscsi_connection *conn)
>  {
>  	conn->rx_iostate = IOSTATE_RX_BHS;
> @@ -1754,6 +1766,19 @@ static int iscsi_task_rx_start(struct iscsi_connection *conn)
>  	uint8_t op;
>  	int err = 0;
>  
> +	/*
> +	 * Below sequence should be executed in a sequential manner. Because
> +	 * this function is called by multiple threads (iscsi_tcp_worker_fn())
> +	 * and updates shared data structures related to iSCSI session (e.g.
> +	 * conn->session->cmd_list).
> +	 *
> +	 * TODO: more fine grained locking
> +	 */
> +	static pthread_mutex_t iscsi_task_rx_start_lock =
> +		PTHREAD_MUTEX_INITIALIZER;
> +
> +	pthread_mutex_lock(&iscsi_task_rx_start_lock);
> +
>  	op = hdr->opcode & ISCSI_OPCODE_MASK;
>  	switch (op) {
>  	case ISCSI_OP_SCSI_CMD:
> @@ -1788,6 +1813,7 @@ static int iscsi_task_rx_start(struct iscsi_connection *conn)
>  		break;
>  	}
>  
> +	pthread_mutex_unlock(&iscsi_task_rx_start_lock);
>  	return err;
>  }
>  
> @@ -1976,8 +2002,21 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
>  	struct iscsi_task *task;
>  	int is_rsp, err = 0;
>  
> -	if (list_empty(&conn->tx_clist))
> -		goto nodata;
> +	/*
> +	 * Below sequence should be executed in a sequential manner for the
> +	 * same reason of iscsi_task_rx_start().
> +	 *
> +	 * TODO: more fine grained locking
> +	 */
> +	static pthread_mutex_t iscsi_task_tx_start_lock =
> +		PTHREAD_MUTEX_INITIALIZER;
> +
> +	pthread_mutex_lock(&iscsi_task_tx_start_lock);
> +
> +	if (list_empty(&conn->tx_clist)) {
> +		err = -EAGAIN;
> +		goto ret;
> +	}
>  
>  	conn_write_pdu(conn);
>  
> @@ -1998,8 +2037,10 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
>  		break;
>  	case ISCSI_OP_NOOP_OUT:
>  		err = iscsi_noop_out_tx_start(task, &is_rsp);
> -		if (!is_rsp)
> -			goto nodata;
> +		if (!is_rsp) {
> +			err = -EAGAIN;
> +			goto ret;
> +		}
>  		break;
>  	case ISCSI_OP_LOGOUT:
>  		err = iscsi_logout_tx_start(task);
> @@ -2010,12 +2051,13 @@ static int iscsi_task_tx_start(struct iscsi_connection *conn)
>  	}
>  
>  	conn->tx_task = task;
> -	return err;
> +ret:
> +	pthread_mutex_unlock(&iscsi_task_tx_start_lock);
>  
> -nodata:
> -	dprintf("no more data\n");
> -	conn->tp->ep_event_modify(conn, EPOLLIN);
> -	return -EAGAIN;
> +	if (err == -EAGAIN)
> +		dprintf("no more data\n");
> +
> +	return err;
>  }
>  
>  static int do_recv(struct iscsi_connection *conn, int next_state)
> @@ -2051,7 +2093,6 @@ void iscsi_rx_handler(struct iscsi_connection *conn)
>  	int ret = 0, hdigest, ddigest;
>  	uint32_t crc;
>  
> -
>  	if (conn->state == STATE_SCSI) {
>  		struct param *p = conn->session_param;
>  		hdigest = p[ISCSI_PARAM_HDRDGST_EN].val & DIGEST_CRC32C;
> @@ -2182,9 +2223,8 @@ again:
>  		exit(1);
>  	}
>  
> -	if (ret < 0 ||
> -	    conn->rx_iostate != IOSTATE_RX_END ||
> -	    conn->state == STATE_CLOSE)
> +	if (ret < 0 || conn->rx_iostate != IOSTATE_RX_END
> +	    || conn->state == STATE_CLOSE)
>  		return;
>  
>  	if (conn->rx_size) {
> @@ -2192,20 +2232,6 @@ again:
>  			conn->rx_size);
>  		exit(1);
>  	}
> -
> -	if (conn->state == STATE_SCSI) {
> -		ret = iscsi_task_rx_done(conn);
> -		if (ret)
> -			conn->state = STATE_CLOSE;
> -		else
> -			conn_read_pdu(conn);
> -	} else {
> -		conn_write_pdu(conn);
> -		conn->tp->ep_event_modify(conn, EPOLLOUT);
> -		ret = cmnd_execute(conn);
> -		if (ret)
> -			conn->state = STATE_CLOSE;
> -	}
>  }
>  
>  static int do_send(struct iscsi_connection *conn, int next_state)
> @@ -2373,6 +2399,33 @@ again:
>  finish:
>  	cmnd_finish(conn);
>  
> +out:
> +	return ret;
> +}
> +
> +void iscsi_tcp_work_rx_done_fn(struct iscsi_connection *conn)
> +{
> +	int ret;
> +
> +	if (conn->state == STATE_SCSI) {
> +		ret = iscsi_task_rx_done(conn);
> +		if (ret)
> +			conn->state = STATE_CLOSE;
> +		else
> +			conn_read_pdu(conn);
> +	} else {
> +		conn_write_pdu(conn);
> +		conn->tp->ep_event_modify(conn, EPOLLOUT);
> +		ret = cmnd_execute(conn);
> +		if (ret)
> +			conn->state = STATE_CLOSE;
> +	}
> +}
> +
> +void iscsi_tcp_work_tx_done_fn(struct iscsi_connection *conn)
> +{
> +	int ret;
> +
>  	switch (conn->state) {
>  	case STATE_KERNEL:
>  		ret = conn_take_fd(conn);
> @@ -2395,9 +2448,6 @@ finish:
>  		conn->tp->ep_event_modify(conn, EPOLLIN);
>  		break;
>  	}
> -
> -out:
> -	return ret;
>  }
>  
>  int iscsi_transportid(int tid, uint64_t itn_id, char *buf, int size)
> diff --git a/usr/iscsi/iscsid.h b/usr/iscsi/iscsid.h
> index c7f6801..aabbeca 100644
> --- a/usr/iscsi/iscsid.h
> +++ b/usr/iscsi/iscsid.h
> @@ -309,8 +309,8 @@ extern int cmnd_exec_auth_chap(struct iscsi_connection *conn);
>  /* conn.c */
>  extern int conn_init(struct iscsi_connection *conn);
>  extern void conn_exit(struct iscsi_connection *conn);
> -extern void conn_close(struct iscsi_connection *conn);
> -extern void conn_put(struct iscsi_connection *conn);
> +extern int conn_close(struct iscsi_connection *conn);
> +extern int conn_put(struct iscsi_connection *conn);
>  extern int conn_get(struct iscsi_connection *conn);
>  extern struct iscsi_connection * conn_find(struct iscsi_session *session, uint32_t cid);
>  extern int conn_take_fd(struct iscsi_connection *conn);
> @@ -336,6 +336,10 @@ extern int iscsi_param_parse_portals(char *p, int do_add, int do_delete);
>  extern void iscsi_update_conn_stats_rx(struct iscsi_connection *conn, int size, int opcode);
>  extern void iscsi_update_conn_stats_tx(struct iscsi_connection *conn, int size, int opcode);
>  extern void iscsi_rsp_set_residual(struct iscsi_cmd_rsp *rsp, struct scsi_cmd *scmd);
> +extern void iscsi_tcp_work_rx_done_fn(struct iscsi_connection *conn);
> +extern void iscsi_tcp_work_tx_done_fn(struct iscsi_connection *conn);
> +extern int is_conn_rx_end(struct iscsi_connection *conn);
> +extern int is_conn_tx_end(struct iscsi_connection *conn);
>  
>  /* iscsid.c iscsi_task */
>  extern void iscsi_free_task(struct iscsi_task *task);
> -- 
> 1.8.1.2
> 
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html




[Index of Archives]     [Linux SCSI]     [Linux RAID]     [Linux Clusters]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]

  Powered by Linux