The following changes since commit 0b14f0a8bc16f949d9ff3eb425fa7a896513fb15:
Always exit if do_exit is set after parse (2013-05-23 21:27:54 +0200)
are available in the git repository at:
git://git.kernel.dk/fio.git master
Jens Axboe (4):
tiobench: ensure that ioengine is being specified
Merge branch 'master' of ssh://git.kernel.dk/data/git/fio
act: add slice support and pass/fail end criteria
Merge branch 'master' of ssh://git.kernel.dk/data/git/fio
Louwrentius (1):
Rework fio_generate_plots script
fio_generate_plots | 201 +++++++++++++++++++++++++++------------------------
profile.c | 2 +-
profiles/act.c | 139 ++++++++++++++++++++++++-----------
profiles/tiobench.c | 1 +
4 files changed, 203 insertions(+), 140 deletions(-)
---
Diff of recent changes:
diff --git a/fio_generate_plots b/fio_generate_plots
index 5e2febd..f65a326 100755
--- a/fio_generate_plots
+++ b/fio_generate_plots
@@ -1,115 +1,126 @@
-#! /bin/sh
-
-# Use gnuplot to generate plots from fio run with -l and/or -w
-
-if [ "$1"x = "x" ]; then
- echo "Usage: fio_generate_plots title [xres yres]"
+#!/bin/sh
+#
+# This script is an almost total rewrite by Louwrentius
+# of the original fio_generate_plots script provided as part of the FIO storage
+# benchmark utiliy. I only retained how GNUplot is used to generate graphs, as
+# that is something I know nothing about.
+#
+# The script uses the files generated by FIO to create nice graphs in the
+# SVG format. This output format is supported by most modern browsers and
+# allows resolution independant graphs to be generated.
+#
+# This script supports GNUPLOT 4.4 and higher.
+#
+# Version 1.0 @ 20121231
+#
+#
+#
+
+if [ -z "$1" ]; then
+ echo "Usage: fio_generate_plots subtitle [xres yres]"
exit 1
fi
GNUPLOT=$(which gnuplot)
-if [ ! -x "$GNUPLOT" ]; then
+if [ ! -x $GNUPLOT ]
+then
echo You need gnuplot installed to generate graphs
exit 1
fi
-TITLE=$1
+TITLE="$1"
# set resolution
-if [ "$2"x != "x" -a "$3"x != "x" ]; then
+if [ ! -z "$2" ] && [ ! -z "$3" ]
+then
XRES="$2"
YRES="$3"
else
- XRES=1024
+ XRES=1280
YRES=768
fi
-PLOT_LINE=""
-for i in *bw.log; do
- if [ ! -r $i ]; then
- continue
- fi
- PT=$(echo $i | sed s/_bw.log//g)
- if [ "$PLOT_LINE"x != "x" ]; then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines"
-done
-
-if [ "$PLOT_LINE"x != "x" ]; then
- echo Making bw logs
- echo "set title 'Bandwidth - $TITLE'; set xlabel 'time (msec)'; set ylabel 'KB/sec'; set terminal png size $XRES,$YRES; set output '$TITLE-bw.png'; plot " $PLOT_LINE | $GNUPLOT -
+if [ -z "$SAMPLE_DURATION" ]
+then
+ SAMPLE_DURATION="*"
fi
-PLOT_LINE=""
-for i in *iops.log; do
- if [ ! -r $i ]; then
- continue
- fi
- PT=$(echo $i | sed s/_iops.log//g)
- if [ "$PLOT_LINE"x != "x" ]; then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines"
-done
-
-if [ "$PLOT_LINE"x != "x" ]; then
- echo Making bw logs
- echo "set title 'IOPS - $TITLE'; set xlabel 'time (msec)'; set ylabel 'IOPS'; set terminal png size $XRES,$YRES; set output '$TITLE-IOPS.png'; plot " $PLOT_LINE | $GNUPLOT -
-fi
+DEFAULT_GRID_LINE_TYPE=3
+DEFAULT_LINE_WIDTH=2
+DEFAULT_LINE_COLORS="
+set object 1 rectangle from screen 0,0 to screen 1,1 fillcolor rgb\"#ffffff\" behind
+set style line 1 lc rgb \"#E41A1C\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 2 lc rgb \"#377EB8\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 3 lc rgb \"#4DAF4A\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 4 lc rgb \"#984EA3\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 5 lc rgb \"#FF7F00\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 6 lc rgb \"#DADA33\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 7 lc rgb \"#A65628\" lw $DEFAULT_LINE_WIDTH lt 1;
+set style line 20 lc rgb \"#000000\" lt $DEFAULT_GRID_LINE_TYPE lw $DEFAULT_LINE_WIDTH;
+"
+
+DEFAULT_TERMINAL="set terminal svg enhanced dashed size $XRES,$YRES dynamic"
+DEFAULT_TITLE_FONT="\"Helvetica,28\""
+DEFAULT_AXIS_FONT="\"Helvetica,14\""
+DEFAULT_AXIS_LABEL_FONT="\"Helvetica,16\""
+DEFAULT_XLABEL="set xlabel \"Time (sec)\" font $DEFAULT_AXIS_LABEL_FONT"
+DEFAULT_XTIC="set xtics font $DEFAULT_AXIS_FONT"
+DEFAULT_YTIC="set ytics font $DEFAULT_AXIS_FONT"
+DEFAULT_MXTIC="set mxtics 0"
+DEFAULT_MYTIC="set mytics 2"
+DEFAULT_XRANGE="set xrange [0:$SAMPLE_DURATION]"
+DEFAULT_YRANGE="set yrange [0:*]"
+DEFAULT_GRID="set grid ls 20"
+DEFAULT_KEY="set key outside bottom center ; set key box enhanced spacing 2.0 samplen 3 horizontal width 4 height 1.2 "
+DEFAULT_SOURCE="set label 30 \"Data source: http://example.com\"; font $DEFAULT_AXIS_FONT tc rgb \"#00000f\" at screen 0.976,0.175 right"
+DEFAULT_OPTS="$DEFAULT_LINE_COLORS ; $DEFAULT_GRID_LINE ; $DEFAULT_GRID ; $DEFAULT_GRID_MINOR ; $DEFAULT_XLABEL ; $DEFAULT_XRANGE ; $DEFAULT_YRANGE ; $DEFAULT_XTIC ; $DEFAULT_YTIC ; $DEFAULT_MXTIC ; $DEFAULT_MYTIC ; $DEFAULT_KEY ; $DEFAULT_TERMINAL ; $DEFAULT_SOURCE"
+
+plot () {
+
+ if [ -z "$TITLE" ]
+ then
+ PLOT_TITLE=" set title \"$1\" font $DEFAULT_TITLE_FONT"
+ else
+ PLOT_TITLE=" set title \"$TITLE\\\n\\\n{/*0.6 "$1"}\" font $DEFAULT_TITLE_FONT"
+ fi
+ FILETYPE="$2"
+ YAXIS="set ylabel \"$3\" font $DEFAULT_AXIS_LABEL_FONT"
+ SCALE=$4
+
+ echo "Title: $PLOT_TITLE"
+ echo "File type: $FILETYPE"
+ echo "yaxis: $YAXIS"
+
+ i=0
+
+ for x in *_"$FILETYPE".log
+ do
+ i=$((i+1))
+ PT=$(echo $x | sed s/_"$FILETYPE".log//g)
+ if [ ! -z "$PLOT_LINE" ]
+ then
+ PLOT_LINE=$PLOT_LINE", "
+ fi
+
+ DEPTH=$(echo $PT | cut -d "-" -f 4)
+ PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i"
+
+ done
+
+ OUTPUT="set output \"$TITLE-$FILETYPE.svg\" "
+
+ echo " $PLOT_TITLE ; $YAXIS ; $DEFAULT_OPTS ; show style lines ; $OUTPUT ; plot " $PLOT_LINE | $GNUPLOT -
+ unset PLOT_LINE
+}
+
+#
+# plot <sub title> <file name tag> <y axis label> <y axis scale>
+#
+
+plot "I/O Latency" lat "Time (msec)" 1000
+plot "I/O Operations Per Second" iops "IOPS" 1
+plot "I/O Submission Latency" slat "Time (μsec)" 1
+plot "I/O Completion Latency" clat "Time (msec)" 1000
+plot "I/O Bandwidth" bw "Throughput (KB/s)" 1
-PLOT_LINE=""
-for i in *slat.log; do
- if [ ! -r $i ]; then
- continue
- fi
- PT=$(echo $i | sed s/_slat.log//g)
- if [ "$PLOT_LINE"x != "x" ]; then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines"
-done
-
-if [ "$PLOT_LINE"x != "x" ]; then
- echo Making slat logs $PLOT_LINE
- echo "set title 'Submission latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-slat.png'; plot " $PLOT_LINE | $GNUPLOT -
-fi
-PLOT_LINE=""
-for i in *clat.log; do
- if [ ! -r $i ]; then
- continue
- fi
- PT=$(echo $i | sed s/_clat.log//g)
- if [ "$PLOT_LINE"x != "x" ]; then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines"
-done
-
-if [ "$PLOT_LINE"x != "x" ]; then
- echo Making clat logs $PLOT_LINE
- echo "set title 'Completion latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-clat.png'; plot " $PLOT_LINE | $GNUPLOT -
-fi
-
-PLOT_LINE=""
-for i in *_lat.log; do
- if [ ! -r $i ]; then
- continue
- fi
- PT=$(echo $i | sed s/_lat.log//g)
- if [ "$PLOT_LINE"x != "x" ]; then
- PLOT_LINE=$PLOT_LINE", "
- fi
-
- PLOT_LINE=$PLOT_LINE"'$i' title '$PT' with lines"
-done
-
-if [ "$PLOT_LINE"x != "x" ]; then
- echo Making lat logs $PLOT_LINE
- echo "set title 'Latency - $TITLE'; set xlabel 'time (msec)'; set ylabel 'latency (msec)'; set terminal png size $XRES,$YRES; set output '$TITLE-lat.png'; plot " $PLOT_LINE | $GNUPLOT -
-fi
diff --git a/profile.c b/profile.c
index 92f2fc4..5d0b866 100644
--- a/profile.c
+++ b/profile.c
@@ -31,7 +31,7 @@ int load_profile(const char *profile)
ops = find_profile(profile);
if (ops) {
if (ops->prep_cmd()) {
- log_err("fio: profile prep failed\n");
+ log_err("fio: profile %s prep failed\n", profile);
return 1;
}
add_job_opts(ops->cmdline, FIO_CLIENT_TYPE_CLI);
diff --git a/profiles/act.c b/profiles/act.c
index 2080602..73cde85 100644
--- a/profiles/act.c
+++ b/profiles/act.c
@@ -31,29 +31,34 @@ static struct act_pass_criteria act_pass[ACT_MAX_CRIT] = {
},
};
+struct act_slice {
+ uint64_t lat_buckets[ACT_MAX_CRIT];
+ uint64_t total_ios;
+};
+
struct act_run_data {
struct fio_mutex *mutex;
unsigned int pending;
- uint64_t lat_buckets[ACT_MAX_CRIT];
- uint64_t total_ios;
+ struct act_slice *slices;
+ unsigned int nr_slices;
};
static struct act_run_data *act_run_data;
struct act_prof_data {
struct timeval sample_tv;
- uint64_t lat_buckets[ACT_MAX_CRIT];
- uint64_t total_ios;
- uint64_t cum_lat_buckets[ACT_MAX_CRIT];
- uint64_t cum_total_ios;
+ struct act_slice *slices;
+ unsigned int cur_slice;
+ unsigned int nr_slices;
};
static char *device_names;
-static unsigned int load = 1;
+static unsigned int load;
static unsigned int prep;
static unsigned int threads_per_queue;
static unsigned int num_read_blocks;
static unsigned int write_size;
+static unsigned long long test_duration;
#define ACT_MAX_OPTS 128
static const char *act_opts[ACT_MAX_OPTS] = {
@@ -90,6 +95,16 @@ static struct fio_option options[] = {
.group = FIO_OPT_G_ACT,
},
{
+ .name = "test-duration",
+ .lname = "Test duration",
+ .type = FIO_OPT_STR_VAL_TIME,
+ .roff1 = &test_duration,
+ .help = "How long the entire test takes to run",
+ .def = "24h",
+ .category = FIO_OPT_C_PROFILE,
+ .group = FIO_OPT_G_ACT,
+ },
+ {
.name = "threads-per-queue",
.lname = "Number of read IO threads per device",
.type = FIO_OPT_INT,
@@ -222,7 +237,7 @@ static int act_add_dev(const char *dev)
if (prep)
return act_add_dev_prep(dev);
- if (act_add_opt("runtime=24h"))
+ if (act_add_opt("runtime=%llus", test_duration))
return 1;
if (act_add_opt("time_based=1"))
return 1;
@@ -241,7 +256,8 @@ static int act_add_dev(const char *dev)
static int act_prep_cmdline(void)
{
if (!device_names) {
- log_err("act: need device-names\n");
+ log_err("act: you need to set IO target(s) with the "
+ "device-names option.\n");
return 1;
}
@@ -266,17 +282,26 @@ static int act_prep_cmdline(void)
static int act_io_u_lat(struct thread_data *td, uint64_t usec)
{
struct act_prof_data *apd = td->prof_data;
+ struct act_slice *slice;
int i, ret = 0;
double perm;
if (prep)
return 0;
- apd->total_ios++;
+ /*
+ * Really should not happen, but lets not let jitter at the end
+ * ruin our day.
+ */
+ if (apd->cur_slice >= apd->nr_slices)
+ return 0;
+
+ slice = &apd->slices[apd->cur_slice];
+ slice->total_ios++;
for (i = ACT_MAX_CRIT - 1; i >= 0; i--) {
if (usec > act_pass[i].max_usec) {
- apd->lat_buckets[i]++;
+ slice->lat_buckets[i]++;
break;
}
}
@@ -286,7 +311,7 @@ static int act_io_u_lat(struct thread_data *td, uint64_t usec)
/* SAMPLE_SEC has passed, check criteria for pass */
for (i = 0; i < ACT_MAX_CRIT; i++) {
- perm = (1000.0 * apd->lat_buckets[i]) / apd->total_ios;
+ perm = (1000.0 * slice->lat_buckets[i]) / slice->total_ios;
if (perm < act_pass[i].max_perm)
continue;
@@ -295,15 +320,8 @@ static int act_io_u_lat(struct thread_data *td, uint64_t usec)
break;
}
- for (i = 0; i < ACT_MAX_CRIT; i++) {
- apd->cum_lat_buckets[i] += apd->lat_buckets[i];
- apd->lat_buckets[i] = 0;
- }
-
- apd->cum_total_ios += apd->total_ios;
- apd->total_ios = 0;
-
fio_gettime(&apd->sample_tv, NULL);
+ apd->cur_slice++;
return ret;
}
@@ -314,51 +332,76 @@ static void get_act_ref(void)
fio_mutex_up(act_run_data->mutex);
}
-static void act_show_all_stats(void)
+static int show_slice(struct act_slice *slice, unsigned int slice_num)
{
- unsigned int i;
-
- log_info(" trans device\n");
- log_info(" %%>(ms) %%>(ms)\n");
- log_info(" slice");
+ unsigned int i, failed = 0;
- for (i = 0; i < ACT_MAX_CRIT; i++)
- log_info("\t%2u", act_pass[i].max_usec / 1000);
- for (i = 0; i < ACT_MAX_CRIT; i++)
- log_info("\t%2u", act_pass[i].max_usec / 1000);
-
- log_info("\n");
- log_info(" ----- ------ ------ ------ ------ ------ ------\n");
- log_info(" 1");
+ log_info(" %2u", slice_num);
for (i = 0; i < ACT_MAX_CRIT; i++) {
- double perc;
+ double perc = 0.0;
- perc = 100.0 * (double) act_run_data->lat_buckets[i] / (double) act_run_data->total_ios;
+ if (slice->total_ios)
+ perc = 100.0 * (double) slice->lat_buckets[i] / (double) slice->total_ios;
+ if ((perc * 10.0) >= act_pass[i].max_perm)
+ failed++;
log_info("\t%2.2f", perc);
}
for (i = 0; i < ACT_MAX_CRIT; i++) {
- double perc;
+ double perc = 0.0;
- perc = 100.0 * (double) act_run_data->lat_buckets[i] / (double) act_run_data->total_ios;
+ if (slice->total_ios)
+ perc = 100.0 * (double) slice->lat_buckets[i] / (double) slice->total_ios;
log_info("\t%2.2f", perc);
}
+ log_info("\n");
+
+ return failed;
+}
+
+static void act_show_all_stats(void)
+{
+ unsigned int i, fails = 0;
+
+ log_info(" trans device\n");
+ log_info(" %%>(ms) %%>(ms)\n");
+ log_info(" slice");
+
+ for (i = 0; i < ACT_MAX_CRIT; i++)
+ log_info("\t %2u", act_pass[i].max_usec / 1000);
+ for (i = 0; i < ACT_MAX_CRIT; i++)
+ log_info("\t %2u", act_pass[i].max_usec / 1000);
+
+ log_info("\n");
+ log_info(" ----- ----- ----- ------ ----- ----- ------\n");
+
+ for (i = 0; i < act_run_data->nr_slices; i++)
+ fails += show_slice(&act_run_data->slices[i], i + 1);
+ log_info("\nact: test complete, device(s): %s\n", fails ? "FAILED" : "PASSED");
}
static void put_act_ref(struct thread_data *td)
{
struct act_prof_data *apd = td->prof_data;
- unsigned int i;
+ unsigned int i, slice;
fio_mutex_down(act_run_data->mutex);
- for (i = 0; i < ACT_MAX_CRIT; i++) {
- act_run_data->lat_buckets[i] += apd->cum_lat_buckets[i];
- act_run_data->lat_buckets[i] += apd->lat_buckets[i];
+ if (!act_run_data->slices) {
+ act_run_data->slices = calloc(sizeof(struct act_slice), apd->nr_slices);
+ act_run_data->nr_slices = apd->nr_slices;
}
- act_run_data->total_ios += apd->cum_total_ios + apd->total_ios;
+ for (slice = 0; slice < apd->nr_slices; slice++) {
+ struct act_slice *dst = &act_run_data->slices[slice];
+ struct act_slice *src = &apd->slices[slice];
+
+ dst->total_ios += src->total_ios;
+
+ for (i = 0; i < ACT_MAX_CRIT; i++)
+ dst->lat_buckets[i] += src->lat_buckets[i];
+ }
if (!--act_run_data->pending)
act_show_all_stats();
@@ -369,10 +412,14 @@ static void put_act_ref(struct thread_data *td)
static int act_td_init(struct thread_data *td)
{
struct act_prof_data *apd;
+ unsigned int nr_slices;
get_act_ref();
apd = calloc(sizeof(*apd), 1);
+ nr_slices = (test_duration + SAMPLE_SEC - 1) / SAMPLE_SEC;
+ apd->slices = calloc(sizeof(struct act_slice), nr_slices);
+ apd->nr_slices = nr_slices;
fio_gettime(&apd->sample_tv, NULL);
td->prof_data = apd;
return 0;
@@ -380,8 +427,11 @@ static int act_td_init(struct thread_data *td)
static void act_td_exit(struct thread_data *td)
{
+ struct act_prof_data *apd = td->prof_data;
+
put_act_ref(td);
- free(td->prof_data);
+ free(apd->slices);
+ free(apd);
td->prof_data = NULL;
}
@@ -416,6 +466,7 @@ static void fio_exit act_unregister(void)
unregister_profile(&act_profile);
fio_mutex_remove(act_run_data->mutex);
+ free(act_run_data->slices);
free(act_run_data);
act_run_data = NULL;
}
diff --git a/profiles/tiobench.c b/profiles/tiobench.c
index 0d2ce80..bdb5985 100644
--- a/profiles/tiobench.c
+++ b/profiles/tiobench.c
@@ -14,6 +14,7 @@ static const char *tb_opts[] = {
"buffered=0", sz_idx, bs_idx, loop_idx, dir_idx, t_idx,
"timeout=600", "group_reporting", "thread", "overwrite=1",
"filename=.fio.tio.1:.fio.tio.2:.fio.tio.3:.fio.tio.4",
+ "ioengine=sync",
"name=seqwrite", "rw=write", "end_fsync=1",
"name=randwrite", "stonewall", "rw=randwrite", "end_fsync=1",
"name=seqread", "stonewall", "rw=read",
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
