RE: Worker Threads

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Fujita, I'm sorry that I haven't tried this yet, but we had to use our hardware for something else. However, in the following weeks, I 'll be trying this patch and will send feedback to this list.

Thanks for all your help!

jab

-----Original Message-----
From: Andy Grover [mailto:agrover@xxxxxxxxxx] 
Sent: Wednesday, September 21, 2011 2:29 PM
To: FUJITA Tomonori
Cc: Bennett, Jeffrey; stgt@xxxxxxxxxxxxxxx
Subject: Re: Worker Threads

On 07/27/2011 07:03 PM, FUJITA Tomonori wrote:
> On Wed, 27 Jul 2011 13:03:52 -0700
> "Bennett, Jeffrey" <jab@xxxxxxxx> wrote:
> 
>> Is there any "side effect" by doing this change?
> 
> NR_WORKER_THREADS is the number of threads that perform I/Os. So you 
> run more threads. You consume more memory.
> 
>> Is there any plan
>> to make this a configuration option that can be changed with the 
>> tgt-adm command by the user?
> 
> tgtd should adjust NR_WORKER_THREADS dynamically since the optimal 
> value depends on your workload, hardware, etc. Well, until we 
> implement that feature, I think that we can support the boot option to 
> specify the number of I/O threads.
> 
> The following patch works for you? You can specify the number in the 
> following way:

Hello Tomo-san,

I haven't tried it but this looks pretty handy to me! Will you be applying it?

Thanks -- Regards -- Andy

> 
> tgtd --nr_iothreads=128
> 
> 
> diff --git a/usr/bs.c b/usr/bs.c
> index d72d090..6f67d15 100644
> --- a/usr/bs.c
> +++ b/usr/bs.c
> @@ -24,6 +24,7 @@
>  #include <inttypes.h>
>  #include <pthread.h>
>  #include <stdio.h>
> +#include <stdlib.h>
>  #include <unistd.h>
>  #include <fcntl.h>
>  #include <signal.h>
> @@ -327,6 +328,11 @@ int bs_thread_open(struct bs_thread_info *info, 
> request_func_t *rfn,  {
>  	int i, ret;
>  
> +	info->worker_thread = zalloc(sizeof(pthread_t) * nr_threads);
> +	if (!info->worker_thread)
> +		return TGTADM_NOMEM;
> +
> +	eprintf("%d\n", nr_threads);
>  	info->request_fn = rfn;
>  
>  	INIT_LIST_HEAD(&info->pending_list);
> @@ -335,11 +341,6 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
>  	pthread_mutex_init(&info->pending_lock, NULL);
>  	pthread_mutex_init(&info->startup_lock, NULL);
>  
> -	if (nr_threads > ARRAY_SIZE(info->worker_thread)) {
> -		eprintf("too many threads %d\n", nr_threads);
> -		nr_threads = ARRAY_SIZE(info->worker_thread);
> -	}
> -
>  	pthread_mutex_lock(&info->startup_lock);
>  	for (i = 0; i < nr_threads; i++) {
>  		ret = pthread_create(&info->worker_thread[i], NULL, @@ -353,6 
> +354,7 @@ int bs_thread_open(struct bs_thread_info *info, request_func_t *rfn,
>  		}
>  	}
>  	pthread_mutex_unlock(&info->startup_lock);
> +	info->nr_worker_threads = nr_threads;
>  
>  	return 0;
>  destroy_threads:
> @@ -367,6 +369,7 @@ destroy_threads:
>  	pthread_cond_destroy(&info->pending_cond);
>  	pthread_mutex_destroy(&info->pending_lock);
>  	pthread_mutex_destroy(&info->startup_lock);
> +	free(info->worker_thread);
>  
>  	return TGTADM_NOMEM;
>  }
> @@ -378,13 +381,13 @@ void bs_thread_close(struct bs_thread_info *info)
>  	info->stop = 1;
>  	pthread_cond_broadcast(&info->pending_cond);
>  
> -	for (i = 0; info->worker_thread[i] &&
> -		     i < ARRAY_SIZE(info->worker_thread); i++)
> +	for (i = 0; info->worker_thread[i] && i < info->nr_worker_threads; 
> +i++)
>  		pthread_join(info->worker_thread[i], NULL);
>  
>  	pthread_cond_destroy(&info->pending_cond);
>  	pthread_mutex_destroy(&info->pending_lock);
>  	pthread_mutex_destroy(&info->startup_lock);
> +	free(info->worker_thread);
>  
>  	info->stop = 0;
>  }
> diff --git a/usr/bs_rdwr.c b/usr/bs_rdwr.c index 29db403..03b869d 
> 100644
> --- a/usr/bs_rdwr.c
> +++ b/usr/bs_rdwr.c
> @@ -148,11 +148,13 @@ static void bs_rdwr_close(struct scsi_lu *lu)
>  	close(lu->fd);
>  }
>  
> +int nr_iothreads = 16;
> +
>  static int bs_rdwr_init(struct scsi_lu *lu)  {
>  	struct bs_thread_info *info = BS_THREAD_I(lu);
>  
> -	return bs_thread_open(info, bs_rdwr_request, NR_WORKER_THREADS);
> +	return bs_thread_open(info, bs_rdwr_request, nr_iothreads);
>  }
>  
>  static void bs_rdwr_exit(struct scsi_lu *lu) diff --git 
> a/usr/bs_thread.h b/usr/bs_thread.h index d460032..50dd1c3 100644
> --- a/usr/bs_thread.h
> +++ b/usr/bs_thread.h
> @@ -1,9 +1,8 @@
> -#define NR_WORKER_THREADS	4
> -
>  typedef void (request_func_t) (struct scsi_cmd *);
>  
>  struct bs_thread_info {
> -	pthread_t worker_thread[NR_WORKER_THREADS];
> +	pthread_t *worker_thread;
> +	int nr_worker_threads;
>  
>  	/* wokers sleep on this and signaled by tgtd */
>  	pthread_cond_t pending_cond;
> diff --git a/usr/tgtd.c b/usr/tgtd.c
> index 5e07267..d5a512c 100644
> --- a/usr/tgtd.c
> +++ b/usr/tgtd.c
> @@ -54,13 +54,14 @@ static struct option const long_options[] =  {
>  	{"foreground", no_argument, 0, 'f'},
>  	{"control-port", required_argument, 0, 'C'},
> +	{"nr_iothreads", required_argument, 0, 't'},
>  	{"debug", required_argument, 0, 'd'},
>  	{"version", no_argument, 0, 'V'},
>  	{"help", no_argument, 0, 'h'},
>  	{0, 0, 0, 0},
>  };
>  
> -static char *short_options = "fC:d:Vh";
> +static char *short_options = "fC:d:t:Vh";
>  
>  static void usage(int status)
>  {
> @@ -72,6 +73,7 @@ static void usage(int status)  Target framework 
> daemon, version %s\n\
>    -f, --foreground        make the program run in the foreground\n\
>    -C, --control-port NNNN use port NNNN for the mgmt channel\n\
> +  -t, --nr_iothreads NNNN specify the number of I/O threads\n\
>    -d, --debug debuglevel  print debugging information\n\
>    -V, --version           print version and exit\n\
>    -h, --help              display this help and exit\n\
> @@ -500,6 +502,9 @@ int main(int argc, char **argv)
>  		case 'C':
>  			control_port = atoi(optarg);
>  			break;
> +		case 't':
> +			nr_iothreads = atoi(optarg);
> +			break;
>  		case 'd':
>  			is_debug = atoi(optarg);
>  			break;
> diff --git a/usr/tgtd.h b/usr/tgtd.h
> index 032cb1c..8f61a07 100644
> --- a/usr/tgtd.h
> +++ b/usr/tgtd.h
> @@ -210,6 +210,7 @@ enum mgmt_req_result {
>  
>  extern int system_active;
>  extern int is_debug;
> +extern int nr_iothreads;
>  
>  extern int kspace_send_tsk_mgmt_res(struct mgmt_req *mreq);  extern 
> int kspace_send_cmd_res(uint64_t nid, int result, struct scsi_cmd *);
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe stgt" in the 
> body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at  
> http://vger.kernel.org/majordomo-info.html


-----
No virus found in this message.
Checked by AVG - www.avg.com
Version: 10.0.1410 / Virus Database: 1520/3910 - Release Date: 09/21/11
--
To unsubscribe from this list: send the line "unsubscribe stgt" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html


[Index of Archives]     [Linux SCSI]     [Linux RAID]     [Linux Clusters]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]

  Powered by Linux