The following changes since commit a9bca4aa722a23d648e622da8a83f5ab3a943531: configure: check for zlib (2013-04-05 12:55:42 +0200) are available in the git repository at: git://git.kernel.dk/fio.git gfio Jens Axboe (14): Add filename_format option Add strcasestr() Use strcasestr() for matching filename_prefix keywords Man page formatting fixups Makefile: use ginstall on Solaris Makefile: use ginstall on Solaris Fio 2.0.15 Merge branch 'next' Merge branch 'master' into gfio server: bump protocol version gfio: update new options for gfio opt format gfio: add support for FIO_OPT_INT options with posval options: make unit_base be posval[] based parse: add posval support to FIO_OPT_INT Steven Noonan (4): num2str: add arguments to represent values in terms of bytes/bits implement 'unit_base' option to select between KB and Kbit et. al. stats: show summary bandwidth in terms of kb_base stat.c: make 'bw' summary line respect 'unit_base' option FIO-VERSION-GEN | 2 +- HOWTO | 28 +++++++++- Makefile | 9 +++- cconv.c | 4 + client.c | 3 + configure | 19 ++++++ engines/net.c | 2 +- eta.c | 9 ++- filesetup.c | 55 +++++++++--------- fio.1 | 29 ++++++++++ fio.h | 2 +- gclient.c | 34 ++++++------ goptions.c | 49 +++++++++++----- init.c | 147 +++++++++++++++++++++++++++++++++++++----------- ioengine.h | 1 + lib/num2str.c | 25 ++++++-- lib/strcasestr.c | 25 ++++++++ lib/strcasestr.h | 13 ++++ options.c | 33 +++++++++++ os/windows/install.wxs | 2 +- parse.c | 18 ++++++ server.c | 2 + server.h | 2 +- stat.c | 54 +++++++++++------- stat.h | 3 + thread_options.h | 4 + 26 files changed, 445 insertions(+), 129 deletions(-) create mode 100644 lib/strcasestr.c create mode 100644 lib/strcasestr.h --- Diff of recent changes: diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN index 6811d29..f9a8ec6 100755 --- a/FIO-VERSION-GEN +++ b/FIO-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=FIO-VERSION-FILE -DEF_VER=fio-2.0.14 +DEF_VER=fio-2.0.15 LF=' ' diff --git a/HOWTO b/HOWTO index 6a880a7..7dc7d48 100644 --- a/HOWTO +++ b/HOWTO @@ -285,6 +285,32 @@ filename=str Fio normally makes up a filename based on the job name, stdin or stdout. Which of the two depends on the read/write direction set. +filename_format=str + If sharing multiple files between jobs, it is usually necessary + to have fio generate the exact names that you want. By default, + fio will name a file based on the default file format + specification of jobname.jobnumber.filenumber. With this + option, that can be customized. Fio will recognize and replace + the following keywords in this string: + + $jobname + The name of the worker thread or process. + + $jobnum + The incremental number of the worker thread or + process. + + $filenum + The incremental number of the file for that worker + thread or process. + + To have dependent jobs share a set of files, this option can + be set to have fio generate filenames that are shared between + the two. For instance, if testfiles.$filenum is specified, + file number 4 for any job will be named testfiles.4. The + default of $jobname.$jobnum.$filenum will be used if + no other format specifier is given. + opendir=str Tell fio to recursively add any file it can find in this directory and down the file system tree. @@ -405,7 +431,7 @@ filesize=int Individual file sizes. May be a range, in which case fio fill_device=bool fill_fs=bool Sets size to something really large and waits for ENOSPC (no space left on device) as the terminating condition. Only makes - sense with sequential write. For a read workload, the mount + sense with sequential write. For a read workload, the mount point will be filled first then IO started on the result. This option doesn't make sense if operating on a raw device node, since the size of that is already known by the file system. diff --git a/Makefile b/Makefile index 21e6ad3..1f58d58 100644 --- a/Makefile +++ b/Makefile @@ -76,6 +76,9 @@ endif ifndef CONFIG_STRSEP SOURCE += lib/strsep.c endif +ifndef CONFIG_STRCASESTR + SOURCE += lib/strcasestr.c +endif ifndef CONFIG_GETOPT_LONG_ONLY SOURCE += lib/getopt_long.c endif @@ -173,7 +176,11 @@ ifndef V endif endif -INSTALL = install +ifeq ($(CONFIG_TARGET_OS), SunOS) + INSTALL = ginstall +else + INSTALL = install +endif prefix = /usr/local bindir = $(prefix)/bin diff --git a/cconv.c b/cconv.c index ea25e0a..57c76e3 100644 --- a/cconv.c +++ b/cconv.c @@ -27,6 +27,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, string_to_cpu(&o->name, top->name); string_to_cpu(&o->directory, top->directory); string_to_cpu(&o->filename, top->filename); + string_to_cpu(&o->filename_format, top->filename_format); string_to_cpu(&o->opendir, top->opendir); string_to_cpu(&o->ioengine, top->ioengine); string_to_cpu(&o->mmapfile, top->mmapfile); @@ -45,6 +46,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->td_ddir = le32_to_cpu(top->td_ddir); o->rw_seq = le32_to_cpu(top->rw_seq); o->kb_base = le32_to_cpu(top->kb_base); + o->unit_base = le32_to_cpu(top->kb_base); o->ddir_seq_nr = le32_to_cpu(top->ddir_seq_nr); o->ddir_seq_add = le64_to_cpu(top->ddir_seq_add); o->iodepth = le32_to_cpu(top->iodepth); @@ -214,6 +216,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, string_to_net(top->name, o->name); string_to_net(top->directory, o->directory); string_to_net(top->filename, o->filename); + string_to_net(top->filename_format, o->filename_format); string_to_net(top->opendir, o->opendir); string_to_net(top->ioengine, o->ioengine); string_to_net(top->mmapfile, o->mmapfile); @@ -232,6 +235,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->td_ddir = cpu_to_le32(o->td_ddir); top->rw_seq = cpu_to_le32(o->rw_seq); top->kb_base = cpu_to_le32(o->kb_base); + top->unit_base = cpu_to_le32(o->kb_base); top->ddir_seq_nr = cpu_to_le32(o->ddir_seq_nr); top->iodepth = cpu_to_le32(o->iodepth); top->iodepth_low = cpu_to_le32(o->iodepth_low); diff --git a/client.c b/client.c index fe6d75e..ffccc5d 100644 --- a/client.c +++ b/client.c @@ -755,6 +755,7 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src) dst->total_err_count = le64_to_cpu(src->total_err_count); dst->first_error = le32_to_cpu(src->first_error); dst->kb_base = le32_to_cpu(src->kb_base); + dst->unit_base = le32_to_cpu(src->unit_base); } static void convert_gs(struct group_run_stats *dst, struct group_run_stats *src) @@ -771,6 +772,7 @@ static void convert_gs(struct group_run_stats *dst, struct group_run_stats *src) } dst->kb_base = le32_to_cpu(src->kb_base); + dst->unit_base = le32_to_cpu(src->unit_base); dst->groupid = le32_to_cpu(src->groupid); dst->unified_rw_rep = le32_to_cpu(src->unified_rw_rep); } @@ -887,6 +889,7 @@ static void convert_jobs_eta(struct jobs_eta *je) je->eta_sec = le64_to_cpu(je->eta_sec); je->nr_threads = le32_to_cpu(je->nr_threads); je->is_pow2 = le32_to_cpu(je->is_pow2); + je->unit_base = le32_to_cpu(je->unit_base); } void fio_client_sum_jobs_eta(struct jobs_eta *dst, struct jobs_eta *je) diff --git a/configure b/configure index 4e2202f..1ca461d 100755 --- a/configure +++ b/configure @@ -872,6 +872,22 @@ fi echo "strsep $strsep" ########################################## +# strcasestr() probe +strcasestr="no" +cat > $TMPC << EOF +#include <string.h> +int main(int argc, char **argv) +{ + strcasestr(NULL, NULL); + return 0; +} +EOF +if compile_prog "" "" "strcasestr"; then + strcasestr="yes" +fi +echo "strcasestr $strcasestr" + +########################################## # getopt_long_only() probe getopt_long_only="no" cat > $TMPC << EOF @@ -1113,6 +1129,9 @@ fi if test "$strsep" = "yes" ; then output_sym "CONFIG_STRSEP" fi +if test "$strcasestr" = "yes" ; then + output_sym "CONFIG_STRCASESTR" +fi if test "$getopt_long_only" = "yes" ; then output_sym "CONFIG_GETOPT_LONG_ONLY" fi diff --git a/engines/net.c b/engines/net.c index a70e56e..566ad2d 100644 --- a/engines/net.c +++ b/engines/net.c @@ -978,7 +978,7 @@ static struct ioengine_ops ioengine_rw = { .options = options, .option_struct_size = sizeof(struct netio_options), .flags = FIO_SYNCIO | FIO_DISKLESSIO | FIO_UNIDIR | - FIO_PIPEIO, + FIO_PIPEIO | FIO_BIT_BASED, }; static int str_hostname_cb(void *data, const char *input) diff --git a/eta.c b/eta.c index 531b876..769ac14 100644 --- a/eta.c +++ b/eta.c @@ -319,6 +319,7 @@ int calc_thread_status(struct jobs_eta *je, int force) unified_rw_rep += td->o.unified_rw_rep; if (is_power_of_2(td->o.kb_base)) je->is_pow2 = 1; + je->unit_base = td->o.unit_base; if (td->o.bw_avg_time < bw_avg_time) bw_avg_time = td->o.bw_avg_time; if (td->runstate == TD_RUNNING || td->runstate == TD_VERIFYING @@ -449,8 +450,8 @@ void display_thread_status(struct jobs_eta *je) if (je->m_rate[0] || je->m_rate[1] || je->t_rate[0] || je->t_rate[1]) { char *tr, *mr; - mr = num2str(je->m_rate[0] + je->m_rate[1], 4, 0, je->is_pow2); - tr = num2str(je->t_rate[0] + je->t_rate[1], 4, 0, je->is_pow2); + mr = num2str(je->m_rate[0] + je->m_rate[1], 4, 0, je->is_pow2, 8); + tr = num2str(je->t_rate[0] + je->t_rate[1], 4, 0, je->is_pow2, 8); p += sprintf(p, ", CR=%s/%s KB/s", tr, mr); free(tr); free(mr); @@ -477,8 +478,8 @@ void display_thread_status(struct jobs_eta *je) for (ddir = DDIR_READ; ddir < DDIR_RWDIR_CNT; ddir++) { rate_str[ddir] = num2str(je->rate[ddir], 5, - 1024, je->is_pow2); - iops_str[ddir] = num2str(je->iops[ddir], 4, 1, 0); + 1024, je->is_pow2, je->unit_base); + iops_str[ddir] = num2str(je->iops[ddir], 4, 1, 0, 0); } left = sizeof(output) - (p - output) - 1; diff --git a/filesetup.c b/filesetup.c index 96be3d1..9edcac1 100644 --- a/filesetup.c +++ b/filesetup.c @@ -719,13 +719,14 @@ uint64_t get_start_offset(struct thread_data *td) int setup_files(struct thread_data *td) { unsigned long long total_size, extend_size; + struct thread_options *o = &td->o; struct fio_file *f; unsigned int i; int err = 0, need_extend; dprint(FD_FILE, "setup files\n"); - if (td->o.read_iolog_file) + if (o->read_iolog_file) goto done; /* @@ -753,15 +754,16 @@ int setup_files(struct thread_data *td) total_size += f->real_file_size; } - if (td->o.fill_device) + if (o->fill_device) td->fill_device_size = get_fs_free_counts(td); /* * device/file sizes are zero and no size given, punt */ - if ((!total_size || total_size == -1ULL) && !td->o.size && - !(td->io_ops->flags & FIO_NOIO) && !td->o.fill_device) { - log_err("%s: you need to specify size=\n", td->o.name); + if ((!total_size || total_size == -1ULL) && !o->size && + !(td->io_ops->flags & FIO_NOIO) && !o->fill_device && + !(o->nr_files && (o->file_size_low || o->file_size_high))) { + log_err("%s: you need to specify size=\n", o->name); td_verror(td, EINVAL, "total_file_size"); return 1; } @@ -776,27 +778,26 @@ int setup_files(struct thread_data *td) for_each_file(td, f, i) { f->file_offset = get_start_offset(td); - if (!td->o.file_size_low) { + if (!o->file_size_low) { /* * no file size range given, file size is equal to * total size divided by number of files. if that is * zero, set it to the real file size. */ - f->io_size = td->o.size / td->o.nr_files; + f->io_size = o->size / o->nr_files; if (!f->io_size) f->io_size = f->real_file_size - f->file_offset; - } else if (f->real_file_size < td->o.file_size_low || - f->real_file_size > td->o.file_size_high) { - if (f->file_offset > td->o.file_size_low) + } else if (f->real_file_size < o->file_size_low || + f->real_file_size > o->file_size_high) { + if (f->file_offset > o->file_size_low) goto err_offset; /* * file size given. if it's fixed, use that. if it's a * range, generate a random size in-between. */ - if (td->o.file_size_low == td->o.file_size_high) { - f->io_size = td->o.file_size_low - - f->file_offset; - } else { + if (o->file_size_low == o->file_size_high) + f->io_size = o->file_size_low - f->file_offset; + else { f->io_size = get_rand_file_size(td) - f->file_offset; } @@ -806,15 +807,15 @@ int setup_files(struct thread_data *td) if (f->io_size == -1ULL) total_size = -1ULL; else { - if (td->o.size_percent) - f->io_size = (f->io_size * td->o.size_percent) / 100; + if (o->size_percent) + f->io_size = (f->io_size * o->size_percent) / 100; total_size += f->io_size; } if (f->filetype == FIO_TYPE_FILE && (f->io_size + f->file_offset) > f->real_file_size && !(td->io_ops->flags & FIO_DISKLESSIO)) { - if (!td->o.create_on_open) { + if (!o->create_on_open) { need_extend++; extend_size += (f->io_size + f->file_offset); } else @@ -823,8 +824,8 @@ int setup_files(struct thread_data *td) } } - if (!td->o.size || td->o.size > total_size) - td->o.size = total_size; + if (!o->size || o->size > total_size) + o->size = total_size; /* * See if we need to extend some files @@ -833,7 +834,7 @@ int setup_files(struct thread_data *td) temp_stall_ts = 1; if (output_format == FIO_OUTPUT_NORMAL) log_info("%s: Laying out IO file(s) (%u file(s) /" - " %lluMB)\n", td->o.name, need_extend, + " %lluMB)\n", o->name, need_extend, extend_size >> 20); for_each_file(td, f, i) { @@ -844,7 +845,7 @@ int setup_files(struct thread_data *td) assert(f->filetype == FIO_TYPE_FILE); fio_file_clear_extend(f); - if (!td->o.fill_device) { + if (!o->fill_device) { old_len = f->real_file_size; extend_len = f->io_size + f->file_offset - old_len; @@ -867,23 +868,23 @@ int setup_files(struct thread_data *td) if (err) return err; - if (!td->o.zone_size) - td->o.zone_size = td->o.size; + if (!o->zone_size) + o->zone_size = o->size; /* * iolog already set the total io size, if we read back * stored entries. */ - if (!td->o.read_iolog_file) - td->total_io_size = td->o.size * td->o.loops; + if (!o->read_iolog_file) + td->total_io_size = o->size * o->loops; done: - if (td->o.create_only) + if (o->create_only) td->done = 1; return 0; err_offset: - log_err("%s: you need to specify valid offset=\n", td->o.name); + log_err("%s: you need to specify valid offset=\n", o->name); return 1; } diff --git a/fio.1 b/fio.1 index 2464dd5..dc13b98 100644 --- a/fio.1 +++ b/fio.1 @@ -151,6 +151,34 @@ a number of files by separating the names with a `:' character. `\-' is a reserved name, meaning stdin or stdout, depending on the read/write direction set. .TP +.BI filename_format \fR=\fPstr +If sharing multiple files between jobs, it is usually necessary to have +fio generate the exact names that you want. By default, fio will name a file +based on the default file format specification of +\fBjobname.jobnumber.filenumber\fP. With this option, that can be +customized. Fio will recognize and replace the following keywords in this +string: +.RS +.RS +.TP +.B $jobname +The name of the worker thread or process. +.TP +.B $jobnum +The incremental number of the worker thread or process. +.TP +.B $filenum +The incremental number of the file for that worker thread or process. +.RE +.P +To have dependent jobs share a set of files, this option can be set to +have fio generate filenames that are shared between the two. For instance, +if \fBtestfiles.$filenum\fR is specified, file number 4 for any job will +be named \fBtestfiles.4\fR. The default of \fB$jobname.$jobnum.$filenum\fR +will be used if no other format specifier is given. +.RE +.P +.TP .BI lockfile \fR=\fPstr Fio defaults to not locking any files before it does IO to them. If a file or file descriptor is shared, fio can serialize IO to that file to make the end @@ -169,6 +197,7 @@ Only one thread or process may do IO at the time, excluding all others. Read-write locking on the file. Many readers may access the file at the same time, but writes get exclusive access. .RE +.RE .P .BI opendir \fR=\fPstr Recursively open any files below directory \fIstr\fR. diff --git a/fio.h b/fio.h index 1cb44e6..c8c8b7a 100644 --- a/fio.h +++ b/fio.h @@ -442,7 +442,7 @@ extern void fio_options_mem_dupe(struct thread_data *); extern void options_mem_dupe(void *data, struct fio_option *options); extern void td_fill_rand_seeds(struct thread_data *); extern void add_job_opts(const char **, int); -extern char *num2str(unsigned long, int, int, int); +extern char *num2str(unsigned long, int, int, int, int); extern int ioengine_load(struct thread_data *); extern unsigned long page_mask; diff --git a/gclient.c b/gclient.c index b8c681a..6f7865c 100644 --- a/gclient.c +++ b/gclient.c @@ -400,13 +400,13 @@ static void gfio_update_client_eta(struct fio_client *client, struct jobs_eta *j sprintf(output, "%3.1f%% done", perc); } - rate_str[0] = num2str(je->rate[0], 5, 10, i2p); - rate_str[1] = num2str(je->rate[1], 5, 10, i2p); - rate_str[2] = num2str(je->rate[2], 5, 10, i2p); + rate_str[0] = num2str(je->rate[0], 5, 10, i2p, 0); + rate_str[1] = num2str(je->rate[1], 5, 10, i2p, 0); + rate_str[2] = num2str(je->rate[2], 5, 10, i2p, 0); - iops_str[0] = num2str(je->iops[0], 4, 1, 0); - iops_str[1] = num2str(je->iops[1], 4, 1, 0); - iops_str[2] = num2str(je->iops[2], 4, 1, 0); + iops_str[0] = num2str(je->iops[0], 4, 1, 0, 0); + iops_str[1] = num2str(je->iops[1], 4, 1, 0, 0); + iops_str[2] = num2str(je->iops[2], 4, 1, 0, 0); gtk_entry_set_text(GTK_ENTRY(ge->eta.read_bw), rate_str[0]); gtk_entry_set_text(GTK_ENTRY(ge->eta.read_iops), iops_str[0]); @@ -494,13 +494,13 @@ static void gfio_update_all_eta(struct jobs_eta *je) sprintf(output, "%3.1f%% done", perc); } - rate_str[0] = num2str(je->rate[0], 5, 10, i2p); - rate_str[1] = num2str(je->rate[1], 5, 10, i2p); - rate_str[2] = num2str(je->rate[2], 5, 10, i2p); + rate_str[0] = num2str(je->rate[0], 5, 10, i2p, 0); + rate_str[1] = num2str(je->rate[1], 5, 10, i2p, 0); + rate_str[2] = num2str(je->rate[2], 5, 10, i2p, 0); - iops_str[0] = num2str(je->iops[0], 4, 1, 0); - iops_str[1] = num2str(je->iops[1], 4, 1, 0); - iops_str[2] = num2str(je->iops[2], 4, 1, 0); + iops_str[0] = num2str(je->iops[0], 4, 1, 0, 0); + iops_str[1] = num2str(je->iops[1], 4, 1, 0, 0); + iops_str[2] = num2str(je->iops[2], 4, 1, 0, 0); gtk_entry_set_text(GTK_ENTRY(ui->eta.read_bw), rate_str[0]); gtk_entry_set_text(GTK_ENTRY(ui->eta.read_iops), iops_str[0]); @@ -1020,8 +1020,8 @@ static void gfio_show_lat(GtkWidget *vbox, const char *name, unsigned long min, if (!usec_to_msec(&min, &max, &mean, &dev)) base = "(msec)"; - minp = num2str(min, 6, 1, 0); - maxp = num2str(max, 6, 1, 0); + minp = num2str(min, 6, 1, 0, 0); + maxp = num2str(max, 6, 1, 0, 0); sprintf(tmp, "%s %s", name, base); frame = gtk_frame_new(tmp); @@ -1192,11 +1192,11 @@ static void gfio_show_ddir_status(struct gfio_client *gc, GtkWidget *mbox, runt = ts->runtime[ddir]; bw = (1000 * ts->io_bytes[ddir]) / runt; - io_p = num2str(ts->io_bytes[ddir], 6, 1, i2p); - bw_p = num2str(bw, 6, 1, i2p); + io_p = num2str(ts->io_bytes[ddir], 6, 1, i2p, 8); + bw_p = num2str(bw, 6, 1, i2p, ts->unit_base); iops = (1000 * (uint64_t)ts->total_io_u[ddir]) / runt; - iops_p = num2str(iops, 6, 1, 0); + iops_p = num2str(iops, 6, 1, 0, 0); box = gtk_hbox_new(FALSE, 3); gtk_box_pack_start(GTK_BOX(mbox), box, TRUE, FALSE, 3); diff --git a/goptions.c b/goptions.c index b78accd..a52410b 100644 --- a/goptions.c +++ b/goptions.c @@ -893,18 +893,29 @@ static void gopt_set_option(struct gopt_job_view *gjv, struct fio_option *o, gopt_int_set_val(i, *ullp); break; } - case FIO_OPT_INT: { - unsigned int *ip = NULL; - struct gopt_int *i; + case FIO_OPT_INT: + if (o->posval[0].ival) { + unsigned int *ip = NULL; + struct gopt_combo *c; - if (o->off1) - ip = td_var(to, o->off1); + if (o->off1) + ip = td_var(to, o->off1); - i = container_of(gopt, struct gopt_int, gopt); - if (ip) - gopt_int_set_val(i, *ip); - break; + c = container_of(gopt, struct gopt_combo, gopt); + if (ip) + gopt_combo_int_set_val(c, *ip); + } else { + unsigned int *ip = NULL; + struct gopt_int *i; + + if (o->off1) + ip = td_var(to, o->off1); + + i = container_of(gopt, struct gopt_int, gopt); + if (ip) + gopt_int_set_val(i, *ip); } + break; case FIO_OPT_STR_SET: case FIO_OPT_BOOL: { unsigned int *ip = NULL; @@ -1017,15 +1028,23 @@ static void gopt_add_option(struct gopt_job_view *gjv, GtkWidget *hbox, go = gopt_new_ullong(gjv, o, ullp, opt_index); break; } - case FIO_OPT_INT: { - unsigned int *ip = NULL; + case FIO_OPT_INT: + if (o->posval[0].ival) { + unsigned int *ip = NULL; - if (o->off1) - ip = td_var(to, o->off1); + if (o->off1) + ip = td_var(to, o->off1); - go = gopt_new_int(gjv, o, ip, opt_index); - break; + go = gopt_new_combo_int(gjv, o, ip, opt_index); + } else { + unsigned int *ip = NULL; + + if (o->off1) + ip = td_var(to, o->off1); + + go = gopt_new_int(gjv, o, ip, opt_index); } + break; case FIO_OPT_STR_SET: case FIO_OPT_BOOL: { unsigned int *ip = NULL; diff --git a/init.c b/init.c index 27370bb..8c80298 100644 --- a/init.c +++ b/init.c @@ -26,6 +26,7 @@ #include "idletime.h" #include "lib/getopt.h" +#include "lib/strcasestr.h" const char fio_version_string[] = FIO_VERSION; @@ -567,6 +568,13 @@ static int fixup_options(struct thread_data *td) } } + if (!o->unit_base) { + if (td->io_ops->flags & FIO_BIT_BASED) + o->unit_base = 1; + else + o->unit_base = 8; + } + #ifndef CONFIG_FDATASYNC if (o->fdatasync_blocks) { log_info("fio: this platform does not support fdatasync()" @@ -807,6 +815,82 @@ static int setup_random_seeds(struct thread_data *td) return 0; } +enum { + FPRE_NONE = 0, + FPRE_JOBNAME, + FPRE_JOBNUM, + FPRE_FILENUM +}; + +static struct fpre_keyword { + const char *keyword; + size_t strlen; + int key; +} fpre_keywords[] = { + { .keyword = "$jobname", .key = FPRE_JOBNAME, }, + { .keyword = "$jobnum", .key = FPRE_JOBNUM, }, + { .keyword = "$filenum", .key = FPRE_FILENUM, }, + { .keyword = NULL, }, + }; + +static char *make_filename(char *buf, struct thread_options *o, + const char *jobname, int jobnum, int filenum) +{ + struct fpre_keyword *f; + char copy[PATH_MAX]; + + if (!o->filename_format || !strlen(o->filename_format)) { + sprintf(buf, "%s.%d.%d", jobname, jobnum, filenum); + return NULL; + } + + for (f = &fpre_keywords[0]; f->keyword; f++) + f->strlen = strlen(f->keyword); + + strcpy(buf, o->filename_format); + memset(copy, 0, sizeof(copy)); + for (f = &fpre_keywords[0]; f->keyword; f++) { + do { + size_t pre_len, post_start = 0; + char *str, *dst = copy; + + str = strcasestr(buf, f->keyword); + if (!str) + break; + + pre_len = str - buf; + if (strlen(str) != f->strlen) + post_start = pre_len + f->strlen; + + if (pre_len) { + strncpy(dst, buf, pre_len); + dst += pre_len; + } + + switch (f->key) { + case FPRE_JOBNAME: + dst += sprintf(dst, "%s", jobname); + break; + case FPRE_JOBNUM: + dst += sprintf(dst, "%d", jobnum); + break; + case FPRE_FILENUM: + dst += sprintf(dst, "%d", filenum); + break; + default: + assert(0); + break; + } + + if (post_start) + strcpy(dst, buf + post_start); + + strcpy(buf, copy); + } while (1); + } + + return buf; +} /* * Adds a job to the list of things todo. Sanitizes the various options * to make sure we don't have conflicts, and initializes various @@ -818,6 +902,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, unsigned int i; char fname[PATH_MAX]; int numjobs, file_alloced; + struct thread_options *o = &td->o; /* * the def_thread is just for options, it's not a real job @@ -843,21 +928,18 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (ioengine_load(td)) goto err; - if (td->o.odirect) + if (o->odirect) td->io_ops->flags |= FIO_RAWIO; file_alloced = 0; - if (!td->o.filename && !td->files_index && !td->o.read_iolog_file) { + if (!o->filename && !td->files_index && !o->read_iolog_file) { file_alloced = 1; - if (td->o.nr_files == 1 && exists_and_not_file(jobname)) + if (o->nr_files == 1 && exists_and_not_file(jobname)) add_file(td, jobname); else { - for (i = 0; i < td->o.nr_files; i++) { - sprintf(fname, "%s.%d.%d", jobname, - td->thread_number, i); - add_file(td, fname); - } + for (i = 0; i < o->nr_files; i++) + add_file(td, make_filename(fname, o, jobname, td->thread_number, i)); } } @@ -882,9 +964,9 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, td->mutex = fio_mutex_init(FIO_MUTEX_LOCKED); - td->ts.clat_percentiles = td->o.clat_percentiles; - td->ts.percentile_precision = td->o.percentile_precision; - memcpy(td->ts.percentile_list, td->o.percentile_list, sizeof(td->o.percentile_list)); + td->ts.clat_percentiles = o->clat_percentiles; + td->ts.percentile_precision = o->percentile_precision; + memcpy(td->ts.percentile_list, o->percentile_list, sizeof(o->percentile_list)); for (i = 0; i < DDIR_RWDIR_CNT; i++) { td->ts.clat_stat[i].min_val = ULONG_MAX; @@ -892,9 +974,9 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, td->ts.lat_stat[i].min_val = ULONG_MAX; td->ts.bw_stat[i].min_val = ULONG_MAX; } - td->ddir_seq_nr = td->o.ddir_seq_nr; + td->ddir_seq_nr = o->ddir_seq_nr; - if ((td->o.stonewall || td->o.new_group) && prev_group_jobs) { + if ((o->stonewall || o->new_group) && prev_group_jobs) { prev_group_jobs = 0; groupid++; } @@ -910,18 +992,18 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (setup_rate(td)) goto err; - if (td->o.lat_log_file) { - setup_log(&td->lat_log, td->o.log_avg_msec, IO_LOG_TYPE_LAT); - setup_log(&td->slat_log, td->o.log_avg_msec, IO_LOG_TYPE_SLAT); - setup_log(&td->clat_log, td->o.log_avg_msec, IO_LOG_TYPE_CLAT); + if (o->lat_log_file) { + setup_log(&td->lat_log, o->log_avg_msec, IO_LOG_TYPE_LAT); + setup_log(&td->slat_log, o->log_avg_msec, IO_LOG_TYPE_SLAT); + setup_log(&td->clat_log, o->log_avg_msec, IO_LOG_TYPE_CLAT); } - if (td->o.bw_log_file) - setup_log(&td->bw_log, td->o.log_avg_msec, IO_LOG_TYPE_BW); - if (td->o.iops_log_file) - setup_log(&td->iops_log, td->o.log_avg_msec, IO_LOG_TYPE_IOPS); + if (o->bw_log_file) + setup_log(&td->bw_log, o->log_avg_msec, IO_LOG_TYPE_BW); + if (o->iops_log_file) + setup_log(&td->iops_log, o->log_avg_msec, IO_LOG_TYPE_IOPS); - if (!td->o.name) - td->o.name = strdup(jobname); + if (!o->name) + o->name = strdup(jobname); if (output_format == FIO_OUTPUT_NORMAL) { if (!job_add_num) { @@ -931,20 +1013,19 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, if (!(td->io_ops->flags & FIO_NOIO)) { char *c1, *c2, *c3, *c4, *c5, *c6; - c1 = fio_uint_to_kmg(td->o.min_bs[DDIR_READ]); - c2 = fio_uint_to_kmg(td->o.max_bs[DDIR_READ]); - c3 = fio_uint_to_kmg(td->o.min_bs[DDIR_WRITE]); - c4 = fio_uint_to_kmg(td->o.max_bs[DDIR_WRITE]); - c5 = fio_uint_to_kmg(td->o.min_bs[DDIR_TRIM]); - c6 = fio_uint_to_kmg(td->o.max_bs[DDIR_TRIM]); + c1 = fio_uint_to_kmg(o->min_bs[DDIR_READ]); + c2 = fio_uint_to_kmg(o->max_bs[DDIR_READ]); + c3 = fio_uint_to_kmg(o->min_bs[DDIR_WRITE]); + c4 = fio_uint_to_kmg(o->max_bs[DDIR_WRITE]); + c5 = fio_uint_to_kmg(o->min_bs[DDIR_TRIM]); + c6 = fio_uint_to_kmg(o->max_bs[DDIR_TRIM]); log_info("%s: (g=%d): rw=%s, bs=%s-%s/%s-%s/%s-%s," " ioengine=%s, iodepth=%u\n", td->o.name, td->groupid, - ddir_str(td->o.td_ddir), + ddir_str(o->td_ddir), c1, c2, c3, c4, c5, c6, - td->io_ops->name, - td->o.iodepth); + td->io_ops->name, o->iodepth); free(c1); free(c2); @@ -961,7 +1042,7 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, * recurse add identical jobs, clear numjobs and stonewall options * as they don't apply to sub-jobs */ - numjobs = td->o.numjobs; + numjobs = o->numjobs; while (--numjobs) { struct thread_data *td_new = get_new_job(0, td, 1); diff --git a/ioengine.h b/ioengine.h index 5503957..493d4b5 100644 --- a/ioengine.h +++ b/ioengine.h @@ -162,6 +162,7 @@ enum fio_ioengine_flags { FIO_PIPEIO = 1 << 7, /* input/output no seekable */ FIO_BARRIER = 1 << 8, /* engine supports barriers */ FIO_MEMALIGN = 1 << 9, /* engine wants aligned memory */ + FIO_BIT_BASED = 1 << 10, /* engine uses a bit base (e.g. uses Kbit as opposed to KB) */ }; /* diff --git a/lib/num2str.c b/lib/num2str.c index 559cbeb..a104192 100644 --- a/lib/num2str.c +++ b/lib/num2str.c @@ -5,12 +5,13 @@ /* * Cheesy number->string conversion, complete with carry rounding error. */ -char *num2str(unsigned long num, int maxlen, int base, int pow2) +char *num2str(unsigned long num, int maxlen, int base, int pow2, int unit_base) { - char postfix[] = { ' ', 'K', 'M', 'G', 'P', 'E' }; - unsigned int thousand[] = { 1000, 1024 }; + const char *postfix[] = { "", "K", "M", "G", "P", "E" }; + const char *byte_postfix[] = { "", "B", "bit" }; + const unsigned int thousand[] = { 1000, 1024 }; unsigned int modulo, decimals; - int post_index, carry = 0; + int byte_post_index = 0, post_index, carry = 0; char tmp[32]; char *buf; @@ -19,6 +20,16 @@ char *num2str(unsigned long num, int maxlen, int base, int pow2) for (post_index = 0; base > 1; post_index++) base /= thousand[!!pow2]; + switch (unit_base) { + case 1: + byte_post_index = 2; + num *= 8; + break; + case 8: + byte_post_index = 1; + break; + } + modulo = -1U; while (post_index < sizeof(postfix)) { sprintf(tmp, "%lu", num); @@ -33,7 +44,8 @@ char *num2str(unsigned long num, int maxlen, int base, int pow2) if (modulo == -1U) { done: - sprintf(buf, "%lu%c", num, postfix[post_index]); + sprintf(buf, "%lu%s%s", num, postfix[post_index], + byte_postfix[byte_post_index]); return buf; } @@ -53,6 +65,7 @@ done: modulo = (modulo + 9) / 10; } while (1); - sprintf(buf, "%lu.%u%c", num, modulo, postfix[post_index]); + sprintf(buf, "%lu.%u%s%s", num, modulo, postfix[post_index], + byte_postfix[byte_post_index]); return buf; } diff --git a/lib/strcasestr.c b/lib/strcasestr.c new file mode 100644 index 0000000..92cf24c --- /dev/null +++ b/lib/strcasestr.c @@ -0,0 +1,25 @@ +#include <ctype.h> +#include <stddef.h> + +char *strcasestr(const char *s1, const char *s2) +{ + const char *s = s1; + const char *p = s2; + + do { + if (!*p) + return (char *) s1; + if ((*p == *s) || + (tolower(*p) == tolower(*s))) { + ++p; + ++s; + } else { + p = s2; + if (!*s) + return NULL; + s = ++s1; + } + } while (1); + + return *p ? NULL : (char *) s1; +} diff --git a/lib/strcasestr.h b/lib/strcasestr.h new file mode 100644 index 0000000..43d61df --- /dev/null +++ b/lib/strcasestr.h @@ -0,0 +1,13 @@ +#ifdef CONFIG_STRCASESTR + +#include <string.h> + +#else + +#ifndef FIO_STRCASESTR_H +#define FIO_STRCASESTR_H + +char *strcasestr(const char *haystack, const char *needle); + +#endif +#endif diff --git a/options.c b/options.c index 00e542b..c838ee0 100644 --- a/options.c +++ b/options.c @@ -1148,6 +1148,16 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_FILENAME, }, { + .name = "filename_format", + .type = FIO_OPT_STR_STORE, + .off1 = td_var_offset(filename_format), + .prio = -1, /* must come after "directory" */ + .help = "Override default $jobname.$jobnum.$filenum naming", + .def = "$jobname.$jobnum.$filenum", + .category = FIO_OPT_C_FILE, + .group = FIO_OPT_G_FILENAME, + }, + { .name = "lockfile", .lname = "Lockfile", .type = FIO_OPT_STR, @@ -3070,6 +3080,29 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_INVALID, }, { + .name = "unit_base", + .type = FIO_OPT_INT, + .off1 = td_var_offset(unit_base), + .prio = 1, + .posval = { + { .ival = "0", + .oval = 0, + .help = "Auto-detect", + }, + { .ival = "8", + .oval = 8, + .help = "Normal (byte based)", + }, + { .ival = "1", + .oval = 1, + .help = "Bit based", + }, + }, + .help = "Bit multiple of result summary data (8 for byte, 1 for bit)", + .category = FIO_OPT_C_GENERAL, + .group = FIO_OPT_G_INVALID, + }, + { .name = "hugepage-size", .lname = "Hugepage size", .type = FIO_OPT_INT, diff --git a/os/windows/install.wxs b/os/windows/install.wxs index 37216b4..1494a64 100755 --- a/os/windows/install.wxs +++ b/os/windows/install.wxs @@ -10,7 +10,7 @@ <Product Id="*" Codepage="1252" Language="1033" Manufacturer="fio" Name="fio" - UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.0.14"> + UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.0.15"> <Package Description="Flexible IO Tester" InstallerVersion="301" Keywords="Installer,MSI,Database" diff --git a/parse.c b/parse.c index a701a5b..8885f59 100644 --- a/parse.c +++ b/parse.c @@ -464,6 +464,24 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data, " (%u min)\n", ull, o->minval); return 1; } + if (o->posval[0].ival) { + posval_sort(o, posval); + + ret = 1; + for (i = 0; i < PARSE_MAX_VP; i++) { + vp = &posval[i]; + if (!vp->ival || vp->ival[0] == '\0') + continue; + if (vp->oval == ull) { + ret = 0; + break; + } + } + if (ret) { + log_err("value %d not in allowed range\n",ull); + return 1; + } + } if (fn) ret = fn(data, &ull); diff --git a/server.c b/server.c index b7e9922..32c1d7a 100644 --- a/server.c +++ b/server.c @@ -936,6 +936,7 @@ static void convert_gs(struct group_run_stats *dst, struct group_run_stats *src) } dst->kb_base = cpu_to_le32(src->kb_base); + dst->unit_base = cpu_to_le32(src->unit_base); dst->groupid = cpu_to_le32(src->groupid); dst->unified_rw_rep = cpu_to_le32(src->unified_rw_rep); } @@ -1018,6 +1019,7 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs) p.ts.total_err_count = cpu_to_le64(ts->total_err_count); p.ts.first_error = cpu_to_le32(ts->first_error); p.ts.kb_base = cpu_to_le32(ts->kb_base); + p.ts.unit_base = cpu_to_le32(ts->unit_base); convert_gs(&p.rs, rs); diff --git a/server.h b/server.h index e84a709..067cdc0 100644 --- a/server.h +++ b/server.h @@ -38,7 +38,7 @@ struct fio_net_cmd_reply { }; enum { - FIO_SERVER_VER = 20, + FIO_SERVER_VER = 21, FIO_SERVER_MAX_FRAGMENT_PDU = 1024, diff --git a/stat.c b/stat.c index 402a73f..245197a 100644 --- a/stat.c +++ b/stat.c @@ -274,12 +274,12 @@ void show_group_stats(struct group_run_stats *rs) if (!rs->max_run[i]) continue; - p1 = num2str(rs->io_kb[i], 6, rs->kb_base, i2p); - p2 = num2str(rs->agg[i], 6, rs->kb_base, i2p); - p3 = num2str(rs->min_bw[i], 6, rs->kb_base, i2p); - p4 = num2str(rs->max_bw[i], 6, rs->kb_base, i2p); + p1 = num2str(rs->io_kb[i], 6, rs->kb_base, i2p, 8); + p2 = num2str(rs->agg[i], 6, rs->kb_base, i2p, rs->unit_base); + p3 = num2str(rs->min_bw[i], 6, rs->kb_base, i2p, rs->unit_base); + p4 = num2str(rs->max_bw[i], 6, rs->kb_base, i2p, rs->unit_base); - log_info("%s: io=%sB, aggrb=%sB/s, minb=%sB/s, maxb=%sB/s," + log_info("%s: io=%s, aggrb=%s/s, minb=%s/s, maxb=%s/s," " mint=%llumsec, maxt=%llumsec\n", rs->unified_rw_rep ? " MIXED" : ddir_str[i], p1, p2, p3, p4, rs->min_run[i], rs->max_run[i]); @@ -348,8 +348,8 @@ static void display_lat(const char *name, unsigned long min, unsigned long max, if (!usec_to_msec(&min, &max, &mean, &dev)) base = "(msec)"; - minp = num2str(min, 6, 1, 0); - maxp = num2str(max, 6, 1, 0); + minp = num2str(min, 6, 1, 0, 0); + maxp = num2str(max, 6, 1, 0, 0); log_info(" %s %s: min=%s, max=%s, avg=%5.02f," " stdev=%5.02f\n", name, base, minp, maxp, mean, dev); @@ -377,13 +377,13 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, runt = ts->runtime[ddir]; bw = (1000 * ts->io_bytes[ddir]) / runt; - io_p = num2str(ts->io_bytes[ddir], 6, 1, i2p); - bw_p = num2str(bw, 6, 1, i2p); + io_p = num2str(ts->io_bytes[ddir], 6, 1, i2p, 8); + bw_p = num2str(bw, 6, 1, i2p, ts->unit_base); iops = (1000 * (uint64_t)ts->total_io_u[ddir]) / runt; - iops_p = num2str(iops, 6, 1, 0); + iops_p = num2str(iops, 6, 1, 0, 0); - log_info(" %s: io=%sB, bw=%sB/s, iops=%s, runt=%6llumsec\n", + log_info(" %s: io=%s, bw=%s/s, iops=%s, runt=%6llumsec\n", rs->unified_rw_rep ? "mixed" : ddir_str[ddir], io_p, bw_p, iops_p, ts->runtime[ddir]); @@ -405,8 +405,15 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, ts->percentile_precision); } if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { - double p_of_agg = 100.0; - const char *bw_str = "KB"; + double p_of_agg = 100.0, fkb_base = (double)rs->kb_base; + const char *bw_str = (rs->unit_base == 1 ? "Kbit" : "KB"); + + if (rs->unit_base == 1) { + min *= 8.0; + max *= 8.0; + mean *= 8.0; + dev *= 8.0; + } if (rs->agg[ddir]) { p_of_agg = mean * 100 / (double) rs->agg[ddir]; @@ -414,15 +421,15 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, p_of_agg = 100.0; } - if (mean > 999999.9) { - min /= 1000.0; - max /= 1000.0; - mean /= 1000.0; - dev /= 1000.0; - bw_str = "MB"; + if (mean > fkb_base * fkb_base) { + min /= fkb_base; + max /= fkb_base; + mean /= fkb_base; + dev /= fkb_base; + bw_str = (rs->unit_base == 1 ? "Mbit" : "MB"); } - log_info(" bw (%s/s) : min=%5lu, max=%5lu, per=%3.2f%%," + log_info(" bw (%-4s/s): min=%5lu, max=%5lu, per=%3.2f%%," " avg=%5.02f, stdev=%5.02f\n", bw_str, min, max, p_of_agg, mean, dev); } @@ -1128,6 +1135,7 @@ void show_run_stats(void) struct thread_stat *threadstats, *ts; int i, j, nr_ts, last_ts, idx; int kb_base_warned = 0; + int unit_base_warned = 0; struct json_object *root = NULL; struct json_array *array = NULL; @@ -1204,11 +1212,16 @@ void show_run_stats(void) ts->pid = td->pid; ts->kb_base = td->o.kb_base; + ts->unit_base = td->o.unit_base; ts->unified_rw_rep = td->o.unified_rw_rep; } else if (ts->kb_base != td->o.kb_base && !kb_base_warned) { log_info("fio: kb_base differs for jobs in group, using" " %u as the base\n", ts->kb_base); kb_base_warned = 1; + } else if (ts->unit_base != td->o.unit_base && !unit_base_warned) { + log_info("fio: unit_base differs for jobs in group, using" + " %u as the base\n", ts->unit_base); + unit_base_warned = 1; } ts->continue_on_error = td->o.continue_on_error; @@ -1234,6 +1247,7 @@ void show_run_stats(void) ts = &threadstats[i]; rs = &runstats[ts->groupid]; rs->kb_base = ts->kb_base; + rs->unit_base = ts->unit_base; rs->unified_rw_rep += ts->unified_rw_rep; for (j = 0; j < DDIR_RWDIR_CNT; j++) { diff --git a/stat.h b/stat.h index a3b391c..76a71d4 100644 --- a/stat.h +++ b/stat.h @@ -9,6 +9,7 @@ struct group_run_stats { uint64_t io_kb[DDIR_RWDIR_CNT]; uint64_t agg[DDIR_RWDIR_CNT]; uint32_t kb_base; + uint32_t unit_base; uint32_t groupid; uint32_t unified_rw_rep; }; @@ -173,6 +174,7 @@ struct thread_stat { uint32_t first_error; uint32_t kb_base; + uint32_t unit_base; }; struct jobs_eta { @@ -187,6 +189,7 @@ struct jobs_eta { uint64_t elapsed_sec; uint64_t eta_sec; uint32_t is_pow2; + uint32_t unit_base; /* * Network 'copy' of run_str[] diff --git a/thread_options.h b/thread_options.h index d577f61..a794949 100644 --- a/thread_options.h +++ b/thread_options.h @@ -50,12 +50,14 @@ struct thread_options { char *name; char *directory; char *filename; + char *filename_format; char *opendir; char *ioengine; char *mmapfile; enum td_ddir td_ddir; unsigned int rw_seq; unsigned int kb_base; + unsigned int unit_base; unsigned int ddir_seq_nr; long ddir_seq_add; unsigned int iodepth; @@ -257,12 +259,14 @@ struct thread_options_pack { uint8_t name[FIO_TOP_STR_MAX]; uint8_t directory[FIO_TOP_STR_MAX]; uint8_t filename[FIO_TOP_STR_MAX]; + uint8_t filename_format[FIO_TOP_STR_MAX]; uint8_t opendir[FIO_TOP_STR_MAX]; uint8_t ioengine[FIO_TOP_STR_MAX]; uint8_t mmapfile[FIO_TOP_STR_MAX]; uint32_t td_ddir; uint32_t rw_seq; uint32_t kb_base; + uint32_t unit_base; uint32_t ddir_seq_nr; uint64_t ddir_seq_add; uint32_t iodepth; -- To unsubscribe from this list: send the line "unsubscribe fio" in the body of a message to majordomo@xxxxxxxxxxxxxxx More majordomo info at http://vger.kernel.org/majordomo-info.html