The following changes since commit 518dac097ec305d76fab3f0f45ce785a3849d8b5:
log: fix use-after-free (2014-07-02 13:36:34 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to f302710c4b7dd09fa6beb61a983345b2eda2b8d4:
iolog: fix bug when decompressing chunks with different sequence numbers (2014-07-03 21:57:29 -0600)
----------------------------------------------------------------
Jeff Moyer (1):
fio.1 - escape the escape character so it shows up
Jens Axboe (16):
Add support for runtime log compression
tp: remove debug start/shutdown printfs
Add support for storing compressed logs
Move tp.[ch] to lib/
configure: remove typo for zlib
iolog: remove dead code that stored a gzip header
glusterfs: fix leak in error path
rbd: fix leak in error path
iolog: do stat() after fopen()
init: fix dead check for !td
configure: add a note to install zlib-devel, if not there
iolog: get rid of one section of CONFIG_ZLIB ifdef
iolog: get rid of ic->nofree
iolog: use a define instead of a raw bit mask
flist: add flist_first_entry()
iolog: fix bug when decompressing chunks with different sequence numbers
Tiziano Müller (2):
Pass O_DIRECT/O_SYNC to glfs_open if requested.
Bail out if glfs_creat failed
HOWTO | 20 +++
Makefile | 3 +-
README | 1 +
backend.c | 19 ++-
cconv.c | 4 +
client.c | 8 +-
configure | 4 +
diskutil.c | 4 +-
engines/glusterfs.c | 25 +--
engines/rbd.c | 2 +
fio.1 | 21 ++-
fio.h | 3 +
flist.h | 19 +++
gclient.c | 2 +-
gfio.c | 2 +-
goptions.c | 4 +-
graph.c | 12 +-
init.c | 98 ++++++++---
io_u.c | 2 +-
iolog.c | 450 ++++++++++++++++++++++++++++++++++++++++++++++++---
iolog.h | 52 +++++-
lib/tp.c | 99 ++++++++++++
lib/tp.h | 32 ++++
options.c | 22 +++
server.c | 8 +-
stat.c | 28 ++--
thread_options.h | 4 +
trim.c | 2 +-
verify.c | 4 +-
29 files changed, 857 insertions(+), 97 deletions(-)
create mode 100644 lib/tp.c
create mode 100644 lib/tp.h
---
Diff of recent changes:
diff --git a/HOWTO b/HOWTO
index 3001fe7..fbc455d 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1336,6 +1336,26 @@ log_avg_msec=int By default, fio will log an entry in the iops, latency,
log_offset=int If this is set, the iolog options will include the byte
offset for the IO entry as well as the other data values.
+log_compression=int If this is set, fio will compress the IO logs as
+ it goes, to keep the memory footprint lower. When a log
+ reaches the specified size, that chunk is removed and
+ compressed in the background. Given that IO logs are
+ fairly highly compressible, this yields a nice memory
+ savings for longer runs. The downside is that the
+ compression will consume some background CPU cycles, so
+ it may impact the run. This, however, is also true if
+ the logging ends up consuming most of the system memory.
+ So pick your poison. The IO logs are saved normally at the
+ end of a run, by decompressing the chunks and storing them
+ in the specified log file. This feature depends on the
+ availability of zlib.
+
+log_store_compressed=bool If set, and log_compression is also set,
+ fio will store the log files in a compressed format. They
+ can be decompressed with fio, using the --inflate-log
+ command line parameter. The files will be stored with a
+ .fz suffix.
+
lockmem=int Pin down the specified amount of memory with mlock(2). Can
potentially be used instead of removing memory or booting
with less memory to simulate a smaller amount of memory.
diff --git a/Makefile b/Makefile
index d49c8a4..65e95be 100644
--- a/Makefile
+++ b/Makefile
@@ -35,7 +35,8 @@ SOURCE := gettime.c ioengines.c init.c stat.c log.c time.c filesetup.c \
cconv.c lib/prio_tree.c json.c lib/zipf.c lib/axmap.c \
lib/lfsr.c gettime-thread.c helpers.c lib/flist_sort.c \
lib/hweight.c lib/getrusage.c idletime.c td_error.c \
- profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c
+ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \
+ lib/tp.c
ifdef CONFIG_64BIT_LLP64
CFLAGS += -DBITS_PER_LONG=32
diff --git a/README b/README
index 1f72876..457b83d 100644
--- a/README
+++ b/README
@@ -176,6 +176,7 @@ $ fio
--idle-prof=option Report cpu idleness on a system or percpu basis
(option=system,percpu) or run unit work
calibration only (option=calibrate).
+ --inflate-log=log Inflate and output compressed log
Any parameters following the options will be assumed to be job files,
diff --git a/backend.c b/backend.c
index 448fc59..68540ab 100644
--- a/backend.c
+++ b/backend.c
@@ -53,6 +53,7 @@
#include "lib/getrusage.h"
#include "idletime.h"
#include "err.h"
+#include "lib/tp.h"
static pthread_t disk_util_thread;
static struct fio_mutex *disk_thread_mutex;
@@ -1443,6 +1444,9 @@ static void *thread_main(void *data)
goto err;
}
+ if (td->flags & TD_F_COMPRESS_LOG)
+ tp_init(&td->tp_data);
+
fio_verify_init(td);
fio_gettime(&td->epoch, NULL);
@@ -1524,6 +1528,9 @@ static void *thread_main(void *data)
fio_writeout_logs(td);
+ if (td->flags & TD_F_COMPRESS_LOG)
+ tp_exit(&td->tp_data);
+
if (o->exec_postrun)
exec_string(o, o->exec_postrun, (const char *)"postrun");
@@ -2020,9 +2027,13 @@ int fio_backend(void)
return 0;
if (write_bw_log) {
- setup_log(&agg_io_log[DDIR_READ], 0, IO_LOG_TYPE_BW, 0, "agg-read_bw.log");
- setup_log(&agg_io_log[DDIR_WRITE], 0, IO_LOG_TYPE_BW, 0, "agg-write_bw.log");
- setup_log(&agg_io_log[DDIR_TRIM], 0, IO_LOG_TYPE_BW, 0, "agg-trim_bw.log");
+ struct log_params p = {
+ .log_type = IO_LOG_TYPE_BW,
+ };
+
+ setup_log(&agg_io_log[DDIR_READ], &p, "agg-read_bw.log");
+ setup_log(&agg_io_log[DDIR_WRITE], &p, "agg-write_bw.log");
+ setup_log(&agg_io_log[DDIR_TRIM], &p, "agg-trim_bw.log");
}
startup_mutex = fio_mutex_init(FIO_MUTEX_LOCKED);
@@ -2046,7 +2057,7 @@ int fio_backend(void)
for (i = 0; i < DDIR_RWDIR_CNT; i++) {
struct io_log *log = agg_io_log[i];
- __finish_log(log);
+ flush_log(log);
free_log(log);
}
}
diff --git a/cconv.c b/cconv.c
index d253975..d4fb158 100644
--- a/cconv.c
+++ b/cconv.c
@@ -152,6 +152,8 @@ void convert_thread_options_to_cpu(struct thread_options *o,
o->use_os_rand = le32_to_cpu(top->use_os_rand);
o->log_avg_msec = le32_to_cpu(top->log_avg_msec);
o->log_offset = le32_to_cpu(top->log_offset);
+ o->log_gz = le32_to_cpu(top->log_gz);
+ o->log_gz_store = le32_to_cpu(top->log_gz_store);
o->norandommap = le32_to_cpu(top->norandommap);
o->softrandommap = le32_to_cpu(top->softrandommap);
o->bs_unaligned = le32_to_cpu(top->bs_unaligned);
@@ -323,6 +325,8 @@ void convert_thread_options_to_net(struct thread_options_pack *top,
top->use_os_rand = cpu_to_le32(o->use_os_rand);
top->log_avg_msec = cpu_to_le32(o->log_avg_msec);
top->log_offset = cpu_to_le32(o->log_offset);
+ top->log_gz = cpu_to_le32(o->log_gz);
+ top->log_gz_store = cpu_to_le32(o->log_gz_store);
top->norandommap = cpu_to_le32(o->norandommap);
top->softrandommap = cpu_to_le32(o->softrandommap);
top->bs_unaligned = cpu_to_le32(o->bs_unaligned);
diff --git a/client.c b/client.c
index e70a27d..1f52734 100644
--- a/client.c
+++ b/client.c
@@ -1237,10 +1237,10 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd)
struct io_sample *s;
s = __get_sample(samples, ret->log_offset, i);
- s->time = le64_to_cpu(s->time);
- s->val = le64_to_cpu(s->val);
- s->ddir = le32_to_cpu(s->ddir);
- s->bs = le32_to_cpu(s->bs);
+ s->time = le64_to_cpu(s->time);
+ s->val = le64_to_cpu(s->val);
+ s->__ddir = le32_to_cpu(s->__ddir);
+ s->bs = le32_to_cpu(s->bs);
if (ret->log_offset) {
struct io_sample_offset *so = (void *) s;
diff --git a/configure b/configure
index f1e116e..1494dd7 100755
--- a/configure
+++ b/configure
@@ -1385,6 +1385,10 @@ if test "$gf_fadvise" = "yes" ; then
output_sym "CONFIG_GF_FADVISE"
fi
+if test "$zlib" = "no" ; then
+ echo "Consider installing zlib-dev (zlib-devel), some fio features depend on it."
+fi
+
echo "LIBS+=$LIBS" >> $config_host_mak
echo "CFLAGS+=$CFLAGS" >> $config_host_mak
echo "CC=$cc" >> $config_host_mak
diff --git a/diskutil.c b/diskutil.c
index cb285cf..c5c5ea6 100644
--- a/diskutil.c
+++ b/diskutil.c
@@ -30,7 +30,7 @@ static void disk_util_free(struct disk_util *du)
while (!flist_empty(&du->slaves)) {
struct disk_util *slave;
- slave = flist_entry(du->slaves.next, struct disk_util, slavelist);
+ slave = flist_first_entry(&du->slaves, struct disk_util, slavelist);
flist_del(&slave->slavelist);
slave->users--;
}
@@ -562,7 +562,7 @@ void disk_util_prune_entries(void)
while (!flist_empty(&disk_list)) {
struct disk_util *du;
- du = flist_entry(disk_list.next, struct disk_util, list);
+ du = flist_first_entry(&disk_list, struct disk_util, list);
flist_del(&du->list);
disk_util_free(du);
}
diff --git a/engines/glusterfs.c b/engines/glusterfs.c
index b233b20..52006b0 100644
--- a/engines/glusterfs.c
+++ b/engines/glusterfs.c
@@ -77,16 +77,12 @@ int fio_gf_setup(struct thread_data *td)
}
dprint(FD_FILE, "fio setup %p\n", g->fs);
td->io_ops->data = g;
+ return 0;
cleanup:
- if (r) {
- if (g) {
- if (g->fs) {
- glfs_fini(g->fs);
- }
- free(g);
- td->io_ops->data = NULL;
- }
- }
+ if (g->fs)
+ glfs_fini(g->fs);
+ free(g);
+ td->io_ops->data = NULL;
return r;
}
@@ -150,12 +146,19 @@ int fio_gf_open_file(struct thread_data *td, struct fio_file *f)
else
flags = O_RDONLY;
}
+
+ if (td->o.odirect)
+ flags |= OS_O_DIRECT;
+ if (td->o.sync_io)
+ flags |= O_SYNC;
+
dprint(FD_FILE, "fio file %s open mode %s td rw %s\n", f->file_name,
- flags == O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write");
+ flags & O_RDONLY ? "ro" : "rw", td_read(td) ? "read" : "write");
g->fd = glfs_creat(g->fs, f->file_name, flags, 0644);
if (!g->fd) {
- log_err("glfs_creat failed.\n");
ret = errno;
+ log_err("glfs_creat failed.\n");
+ return ret;
}
/* file for read doesn't exist or shorter than required, create/extend it */
if (td_read(td)) {
diff --git a/engines/rbd.c b/engines/rbd.c
index 32ce60c..85a705f 100644
--- a/engines/rbd.c
+++ b/engines/rbd.c
@@ -86,6 +86,8 @@ static int _fio_setup_rbd_data(struct thread_data *td,
return 0;
failed:
+ if (rbd_data)
+ free(rbd_data);
return 1;
}
diff --git a/fio.1 b/fio.1
index da44e57..c58e817 100644
--- a/fio.1
+++ b/fio.1
@@ -182,8 +182,8 @@ set. On Windows, disk devices are accessed as \\.\PhysicalDrive0 for the first
device, \\.\PhysicalDrive1 for the second etc. Note: Windows and FreeBSD
prevent write access to areas of the disk containing in-use data
(e.g. filesystems). If the wanted filename does need to include a colon, then
-escape that with a '\' character. For instance, if the filename is
-"/dev/dsk/foo@3,0:c", then you would use filename="/dev/dsk/foo@3,0\:c".
+escape that with a '\\' character. For instance, if the filename is
+"/dev/dsk/foo@3,0:c", then you would use filename="/dev/dsk/foo@3,0\\:c".
.TP
.BI filename_format \fR=\fPstr
If sharing multiple files between jobs, it is usually necessary to have
@@ -1214,6 +1214,23 @@ Defaults to 0.
If this is set, the iolog options will include the byte offset for the IO
entry as well as the other data values.
.TP
+.BI log_compression \fR=\fPint
+If this is set, fio will compress the IO logs as it goes, to keep the memory
+footprint lower. When a log reaches the specified size, that chunk is removed
+and compressed in the background. Given that IO logs are fairly highly
+compressible, this yields a nice memory savings for longer runs. The downside
+is that the compression will consume some background CPU cycles, so it may
+impact the run. This, however, is also true if the logging ends up consuming
+most of the system memory. So pick your poison. The IO logs are saved
+normally at the end of a run, by decompressing the chunks and storing them
+in the specified log file. This feature depends on the availability of zlib.
+.TP
+.BI log_store_compressed \fR=\fPbool
+If set, and \fBlog\fR_compression is also set, fio will store the log files in
+a compressed format. They can be decompressed with fio, using the
+\fB\-\-inflate-log\fR command line parameter. The files will be stored with a
+\fB\.fz\fR suffix.
+.TP
.BI disable_lat \fR=\fPbool
Disable measurements of total latency numbers. Useful only for cutting
back the number of calls to \fBgettimeofday\fR\|(2), as that does impact performance at
diff --git a/fio.h b/fio.h
index 4d4af0a..df0d020 100644
--- a/fio.h
+++ b/fio.h
@@ -73,6 +73,7 @@ enum {
TD_F_PROFILE_OPS = 64,
TD_F_COMPRESS = 128,
TD_F_NOIO = 256,
+ TD_F_COMPRESS_LOG = 512,
};
enum {
@@ -112,6 +113,8 @@ struct thread_data {
struct io_log *bw_log;
struct io_log *iops_log;
+ struct tp_data *tp_data;
+
uint64_t stat_io_bytes[DDIR_RWDIR_CNT];
struct timeval bw_sample_time;
diff --git a/flist.h b/flist.h
index 8e13041..d453e79 100644
--- a/flist.h
+++ b/flist.h
@@ -140,6 +140,22 @@ static inline void flist_splice(const struct flist_head *list,
__flist_splice(list, head, head->next);
}
+static inline void flist_splice_tail(struct flist_head *list,
+ struct flist_head *head)
+{
+ if (!flist_empty(list))
+ __flist_splice(list, head->prev, head);
+}
+
+static inline void flist_splice_tail_init(struct flist_head *list,
+ struct flist_head *head)
+{
+ if (!flist_empty(list)) {
+ __flist_splice(list, head->prev, head);
+ INIT_FLIST_HEAD(list);
+ }
+}
+
static inline void flist_splice_init(struct flist_head *list,
struct flist_head *head)
{
@@ -158,6 +174,9 @@ static inline void flist_splice_init(struct flist_head *list,
#define flist_entry(ptr, type, member) \
container_of(ptr, type, member)
+#define flist_first_entry(ptr, type, member) \
+ flist_entry((ptr)->next, type, member)
+
/**
* flist_for_each - iterate over a list
* @pos: the &struct flist_head to use as a loop counter.
diff --git a/gclient.c b/gclient.c
index d236f86..42bc761 100644
--- a/gclient.c
+++ b/gclient.c
@@ -694,7 +694,7 @@ static void gfio_client_job_start(struct fio_client *client, struct fio_net_cmd
static void gfio_client_iolog(struct fio_client *client, struct cmd_iolog_pdu *pdu)
{
- printf("got iolog: name=%s, type=%u, entries=%u\n", pdu->name, pdu->log_type, pdu->nr_samples);
+ printf("got iolog: name=%s, type=%u, entries=%lu\n", pdu->name, pdu->log_type, (unsigned long) pdu->nr_samples);
free(pdu);
}
diff --git a/gfio.c b/gfio.c
index 65302e6..37c1db6 100644
--- a/gfio.c
+++ b/gfio.c
@@ -444,7 +444,7 @@ static int send_job_file(struct gui_entry *ge)
while (!flist_empty(&gc->o_list)) {
struct gfio_client_options *gco;
- gco = flist_entry(gc->o_list.next, struct gfio_client_options, list);
+ gco = flist_first_entry(&gc->o_list, struct gfio_client_options, list);
flist_del(&gco->list);
free(gco);
}
diff --git a/goptions.c b/goptions.c
index 5b5c89e..c01b6cc 100644
--- a/goptions.c
+++ b/goptions.c
@@ -1433,7 +1433,7 @@ static int gopt_handle_changed_options(struct gopt_job_view *gjv)
goto done;
while (!flist_empty(&gjv->changed_list)) {
- gopt = flist_entry(gjv->changed_list.next, struct gopt, changed_list);
+ gopt = flist_first_entry(&gjv->changed_list, struct gopt, changed_list);
flist_del_init(&gopt->changed_list);
}
@@ -1577,7 +1577,7 @@ void gopt_get_options_window(GtkWidget *window, struct gfio_client *gc)
gjv = calloc(1, sizeof(*gjv));
INIT_FLIST_HEAD(&gjv->changed_list);
- gco = flist_entry(gc->o_list.next, struct gfio_client_options, list);
+ gco = flist_first_entry(&gc->o_list, struct gfio_client_options, list);
gjv->o = &gco->o;
gjv->dialog = dialog;
gjv->client = gc;
diff --git a/graph.c b/graph.c
index 5c865dc..c45954c 100644
--- a/graph.c
+++ b/graph.c
@@ -687,7 +687,7 @@ static void graph_value_drop(struct graph_label *l, struct graph_value *v)
*/
while (!(v->flags & GV_F_ON_PRIO)) {
assert(!flist_empty(&v->alias));
- v = flist_entry(v->alias.next, struct graph_value, alias);
+ v = flist_first_entry(&v->alias, struct graph_value, alias);
}
prio_tree_remove(&l->prio_tree, &v->node);
@@ -698,7 +698,7 @@ static void graph_value_drop(struct graph_label *l, struct graph_value *v)
while (!flist_empty(&v->alias)) {
struct graph_value *a;
- a = flist_entry(v->alias.next, struct graph_value, alias);
+ a = flist_first_entry(&v->alias, struct graph_value, alias);
flist_del_init(&a->alias);
__graph_value_drop(l, a);
@@ -773,7 +773,7 @@ static void graph_label_add_value(struct graph_label *i, void *value,
to_drop = 2;
while (to_drop-- && !flist_empty(&i->value_list)) {
- x = flist_entry(i->value_list.next, struct graph_value, list);
+ x = flist_first_entry(&i->value_list, struct graph_value, list);
graph_value_drop(i, x);
/*
@@ -836,7 +836,7 @@ static void graph_free_values(struct graph_label *l)
struct graph_value *i;
while (!flist_empty(&l->value_list)) {
- i = flist_entry(l->value_list.next, struct graph_value, list);
+ i = flist_first_entry(&l->value_list, struct graph_value, list);
graph_value_drop(l, i);
}
}
@@ -846,7 +846,7 @@ static void graph_free_labels(struct graph *g)
struct graph_label *i;
while (!flist_empty(&g->label_list)) {
- i = flist_entry(g->label_list.next, struct graph_label, list);
+ i = flist_first_entry(&g->label_list, struct graph_label, list);
flist_del(&i->list);
graph_free_values(i);
free(i);
@@ -1010,7 +1010,7 @@ const char *graph_find_tooltip(struct graph *g, int ix, int iy)
}
}
if (!flist_empty(&v->alias))
- v = flist_entry(v->alias.next, struct graph_value, alias);
+ v = flist_first_entry(&v->alias, struct graph_value, alias);
} while (v != rootv);
} while ((n = prio_tree_next(&iter)) != NULL);
diff --git a/init.c b/init.c
index c2d6109..8268ed5 100644
--- a/init.c
+++ b/init.c
@@ -170,6 +170,13 @@ static struct option l_opts[FIO_NR_OPTIONS] = {
.has_arg = required_argument,
.val = 'x' | FIO_CLIENT_FLAG,
},
+#ifdef CONFIG_ZLIB
+ {
+ .name = (char *) "inflate-log",
+ .has_arg = required_argument,
+ .val = 'X' | FIO_CLIENT_FLAG,
+ },
+#endif
{
.name = (char *) "alloc-size",
.has_arg = required_argument,
@@ -1145,25 +1152,70 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num,
goto err;
if (o->lat_log_file) {
- snprintf(logname, sizeof(logname), "%s_lat.log", o->lat_log_file);
- setup_log(&td->lat_log, o->log_avg_msec, IO_LOG_TYPE_LAT,
- o->log_offset, logname);
- snprintf(logname, sizeof(logname), "%s_slat.log", o->lat_log_file);
- setup_log(&td->slat_log, o->log_avg_msec, IO_LOG_TYPE_SLAT,
- o->log_offset, logname);
- snprintf(logname, sizeof(logname), "%s_clat.log", o->lat_log_file);
- setup_log(&td->clat_log, o->log_avg_msec, IO_LOG_TYPE_CLAT,
- o->log_offset, logname);
+ struct log_params p = {
+ .td = td,
+ .avg_msec = o->log_avg_msec,
+ .log_type = IO_LOG_TYPE_LAT,
+ .log_offset = o->log_offset,
+ .log_gz = o->log_gz,
+ .log_gz_store = o->log_gz_store,
+ };
+ const char *suf;
+
+ if (p.log_gz_store)
+ suf = "log.fz";
+ else
+ suf = "log";
+
+ snprintf(logname, sizeof(logname), "%s_lat.%s",
+ o->lat_log_file, suf);
+ setup_log(&td->lat_log, &p, logname);
+ snprintf(logname, sizeof(logname), "%s_slat.%s",
+ o->lat_log_file, suf);
+ setup_log(&td->slat_log, &p, logname);
+ snprintf(logname, sizeof(logname), "%s_clat.%s",
+ o->lat_log_file, suf);
+ setup_log(&td->clat_log, &p, logname);
}
if (o->bw_log_file) {
- snprintf(logname, sizeof(logname), "%s_bw.log", o->bw_log_file);
- setup_log(&td->bw_log, o->log_avg_msec, IO_LOG_TYPE_BW,
- o->log_offset, logname);
+ struct log_params p = {
+ .td = td,
+ .avg_msec = o->log_avg_msec,
+ .log_type = IO_LOG_TYPE_BW,
+ .log_offset = o->log_offset,
+ .log_gz = o->log_gz,
+ .log_gz_store = o->log_gz_store,
+ };
+ const char *suf;
+
+ if (p.log_gz_store)
+ suf = "log.fz";
+ else
+ suf = "log";
+
+ snprintf(logname, sizeof(logname), "%s_bw.%s",
+ o->bw_log_file, suf);
+ setup_log(&td->bw_log, &p, logname);
}
if (o->iops_log_file) {
- snprintf(logname, sizeof(logname), "%s_iops.log", o->iops_log_file);
- setup_log(&td->iops_log, o->log_avg_msec, IO_LOG_TYPE_IOPS,
- o->log_offset, logname);
+ struct log_params p = {
+ .td = td,
+ .avg_msec = o->log_avg_msec,
+ .log_type = IO_LOG_TYPE_IOPS,
+ .log_offset = o->log_offset,
+ .log_gz = o->log_gz,
+ .log_gz_store = o->log_gz_store,
+ };
+ const char *suf;
+
+ if (p.log_gz_store)
+ suf = "log.fz";
+ else
+ suf = "log";
+
+ snprintf(logname, sizeof(logname), "%s_iops.%s",
+ o->iops_log_file, suf);
+ setup_log(&td->iops_log, &p, logname);
}
if (!o->name)
@@ -1555,6 +1607,9 @@ static void usage(const char *name)
printf(" --idle-prof=option\tReport cpu idleness on a system or percpu basis\n"
"\t\t\t(option=system,percpu) or run unit work\n"
"\t\t\tcalibration only (option=calibrate)\n");
+#ifdef CONFIG_ZLIB
+ printf(" --inflate-log=log\tInflate and output compressed log\n");
+#endif
printf("\nFio was written by Jens Axboe <jens.axboe@xxxxxxxxxx>");
printf("\n Jens Axboe <jaxboe@xxxxxxxxxxxx>");
printf("\n Jens Axboe <axboe@xxxxxx>\n");
@@ -1886,6 +1941,13 @@ int parse_cmd_line(int argc, char *argv[], int client_type)
nr_job_sections++;
break;
}
+#ifdef CONFIG_ZLIB
+ case 'X':
+ exit_val = iolog_file_inflate(optarg);
+ did_arg++;
+ do_exit++;
+ break;
+#endif
case 'p':
did_arg = 1;
if (exec_profile)
@@ -1943,10 +2005,8 @@ int parse_cmd_line(int argc, char *argv[], int client_type)
if (!ret && !strcmp(opt, "ioengine")) {
free_ioengine(td);
if (ioengine_load(td)) {
- if (td) {
- put_job(td);
- td = NULL;
- }
+ put_job(td);
+ td = NULL;
do_exit++;
break;
}
diff --git a/io_u.c b/io_u.c
index 5b9d483..16b52d6 100644
--- a/io_u.c
+++ b/io_u.c
@@ -223,7 +223,7 @@ static int get_next_rand_offset(struct thread_data *td, struct fio_file *f,
if (!flist_empty(&td->next_rand_list)) {
struct rand_off *r;
fetch:
- r = flist_entry(td->next_rand_list.next, struct rand_off, list);
+ r = flist_first_entry(&td->next_rand_list, struct rand_off, list);
flist_del(&r->list);
*b = r->off;
free(r);
diff --git a/iolog.c b/iolog.c
index 96afec6..29997ce 100644
--- a/iolog.c
+++ b/iolog.c
@@ -6,11 +6,19 @@
#include <stdlib.h>
#include <libgen.h>
#include <assert.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#ifdef CONFIG_ZLIB
+#include <zlib.h>
+#endif
+
#include "flist.h"
#include "fio.h"
#include "verify.h"
#include "trim.h"
#include "filelock.h"
+#include "lib/tp.h"
static const char iolog_ver2[] = "fio version 2 iolog";
@@ -123,7 +131,7 @@ int read_iolog_get(struct thread_data *td, struct io_u *io_u)
while (!flist_empty(&td->io_log_list)) {
int ret;
- ipo = flist_entry(td->io_log_list.next, struct io_piece, list);
+ ipo = flist_first_entry(&td->io_log_list, struct io_piece, list);
flist_del(&ipo->list);
remove_trim_entry(td, ipo);
@@ -176,7 +184,7 @@ void prune_io_piece_log(struct thread_data *td)
}
while (!flist_empty(&td->io_hist_list)) {
- ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
+ ipo = flist_entry(&td->io_hist_list, struct io_piece, list);
flist_del(&ipo->list);
remove_trim_entry(td, ipo);
td->io_hist_len--;
@@ -539,19 +547,35 @@ int init_iolog(struct thread_data *td)
return ret;
}
-void setup_log(struct io_log **log, unsigned long avg_msec, int log_type,
- int log_offset, const char *filename)
+void setup_log(struct io_log **log, struct log_params *p,
+ const char *filename)
{
struct io_log *l = malloc(sizeof(*l));
memset(l, 0, sizeof(*l));
l->nr_samples = 0;
l->max_samples = 1024;
- l->log_type = log_type;
- l->log_offset = log_offset;
+ l->log_type = p->log_type;
+ l->log_offset = p->log_offset;
+ l->log_gz = p->log_gz;
+ l->log_gz_store = p->log_gz_store;
l->log = malloc(l->max_samples * log_entry_sz(l));
- l->avg_msec = avg_msec;
+ l->avg_msec = p->avg_msec;
l->filename = strdup(filename);
+ l->td = p->td;
+
+ if (l->log_offset)
+ l->log_ddir_mask = LOG_OFFSET_SAMPLE_BIT;
+
+ INIT_FLIST_HEAD(&l->chunk_list);
+
+ if (l->log_gz && !p->td)
+ l->log_gz = 0;
+ else if (l->log_gz) {
+ pthread_mutex_init(&l->chunk_lock, NULL);
+ p->td->flags |= TD_F_COMPRESS_LOG;
+ }
+
*log = l;
}
@@ -588,38 +612,276 @@ void free_log(struct io_log *log)
free(log);
}
-void __finish_log(struct io_log *log)
+static void flush_samples(FILE *f, void *samples, uint64_t sample_size)
{
- uint64_t i;
- void *buf;
- FILE *f;
+ struct io_sample *s;
+ int log_offset;
+ uint64_t i, nr_samples;
- f = fopen(log->filename, "a");
- if (!f) {
- perror("fopen log");
+ if (!sample_size)
return;
- }
- buf = set_file_buffer(f);
+ s = __get_sample(samples, 0, 0);
+ log_offset = (s->__ddir & LOG_OFFSET_SAMPLE_BIT) != 0;
+
+ nr_samples = sample_size / __log_entry_sz(log_offset);
- for (i = 0; i < log->nr_samples; i++) {
- struct io_sample *s = get_sample(log, i);
+ for (i = 0; i < nr_samples; i++) {
+ s = __get_sample(samples, log_offset, i);
- if (!log->log_offset) {
+ if (!log_offset) {
fprintf(f, "%lu, %lu, %u, %u\n",
(unsigned long) s->time,
(unsigned long) s->val,
- s->ddir, s->bs);
+ io_sample_ddir(s), s->bs);
} else {
struct io_sample_offset *so = (void *) s;
fprintf(f, "%lu, %lu, %u, %u, %llu\n",
(unsigned long) s->time,
(unsigned long) s->val,
- s->ddir, s->bs,
+ io_sample_ddir(s), s->bs,
(unsigned long long) so->offset);
}
}
+}
+
+#ifdef CONFIG_ZLIB
+
+struct iolog_flush_data {
+ struct tp_work work;
+ struct io_log *log;
+ void *samples;
+ uint64_t nr_samples;
+};
+
+struct iolog_compress {
+ struct flist_head list;
+ void *buf;
+ size_t len;
+ unsigned int seq;
+};
+
+#define GZ_CHUNK 131072
+
+static struct iolog_compress *get_new_chunk(unsigned int seq)
+{
+ struct iolog_compress *c;
+
+ c = malloc(sizeof(*c));
+ INIT_FLIST_HEAD(&c->list);
+ c->buf = malloc(GZ_CHUNK);
+ c->len = 0;
+ c->seq = seq;
+ return c;
+}
+
+static void free_chunk(struct iolog_compress *ic)
+{
+ free(ic->buf);
+ free(ic);
+}
+
+static int z_stream_init(z_stream *stream, int gz_hdr)
+{
+ int wbits = 15;
+
+ stream->zalloc = Z_NULL;
+ stream->zfree = Z_NULL;
+ stream->opaque = Z_NULL;
+ stream->next_in = Z_NULL;
+
+ if (gz_hdr)
+ wbits += 32;
+
+ if (inflateInit2(stream, wbits) != Z_OK)
+ return 1;
+
+ return 0;
+}
+
+struct flush_chunk_iter {
+ unsigned int seq;
+ void *buf;
+ size_t buf_size;
+ size_t buf_used;
+ size_t chunk_sz;
+};
+
+static void finish_chunk(z_stream *stream, FILE *f,
+ struct flush_chunk_iter *iter)
+{
+ int ret;
+
+ ret = inflateEnd(stream);
+ if (ret != Z_OK)
+ log_err("fio: failed to end log inflation (%d)\n", ret);
+
+ flush_samples(f, iter->buf, iter->buf_used);
+ free(iter->buf);
+ iter->buf = NULL;
+ iter->buf_size = iter->buf_used = 0;
+}
+
+static size_t flush_chunk(struct iolog_compress *ic, int gz_hdr, FILE *f,
+ z_stream *stream, struct flush_chunk_iter *iter)
+{
+ if (ic->seq != iter->seq) {
+ if (iter->seq)
+ finish_chunk(stream, f, iter);
+
+ z_stream_init(stream, gz_hdr);
+ iter->seq = ic->seq;
+ }
+
+ stream->avail_in = ic->len;
+ stream->next_in = ic->buf;
+
+ if (!iter->buf_size) {
+ iter->buf_size = iter->chunk_sz;
+ iter->buf = malloc(iter->buf_size);
+ }
+
+ while (stream->avail_in) {
+ size_t this_out = iter->buf_size - iter->buf_used;
+ int err;
+
+ stream->avail_out = this_out;
+ stream->next_out = iter->buf + iter->buf_used;
+
+ err = inflate(stream, Z_NO_FLUSH);
+ if (err < 0) {
+ log_err("fio: failed inflating log: %d\n", err);
+ break;
+ }
+
+ iter->buf_used += this_out - stream->avail_out;
+
+ if (!stream->avail_out) {
+ iter->buf_size += iter->chunk_sz;
+ iter->buf = realloc(iter->buf, iter->buf_size);
+ continue;
+ }
+
+ if (err == Z_STREAM_END)
+ break;
+ }
+
+ return (void *) stream->next_in - ic->buf;
+}
+
+static void flush_gz_chunks(struct io_log *log, FILE *f)
+{
+ struct flush_chunk_iter iter = { .chunk_sz = log->log_gz, };
+ z_stream stream;
+
+ while (!flist_empty(&log->chunk_list)) {
+ struct iolog_compress *ic;
+
+ ic = flist_first_entry(&log->chunk_list, struct iolog_compress, list);
+ flist_del(&ic->list);
+
+ if (log->log_gz_store)
+ fwrite(ic->buf, ic->len, 1, f);
+ else
+ flush_chunk(ic, log->log_gz_store, f, &stream, &iter);
+
+ free_chunk(ic);
+ }
+
+ if (iter.seq) {
+ finish_chunk(&stream, f, &iter);
+ free(iter.buf);
+ }
+}
+
+int iolog_file_inflate(const char *file)
+{
+ struct flush_chunk_iter iter = { .chunk_sz = 64 * 1024 * 1024, };
+ struct iolog_compress ic;
+ z_stream stream;
+ struct stat sb;
+ ssize_t ret;
+ size_t total;
+ void *buf;
+ FILE *f;
+
+ f = fopen(file, "r");
+ if (!f) {
+ perror("fopen");
+ return 1;
+ }
+
+ if (stat(file, &sb) < 0) {
+ fclose(f);
+ perror("stat");
+ return 1;
+ }
+
+ ic.buf = buf = malloc(sb.st_size);
+ ic.len = sb.st_size;
+ ic.seq = 1;
+
+ ret = fread(ic.buf, ic.len, 1, f);
+ if (ret < 0) {
+ perror("fread");
+ fclose(f);
+ return 1;
+ } else if (ret != 1) {
+ log_err("fio: short read on reading log\n");
+ fclose(f);
+ return 1;
+ }
+
+ fclose(f);
+
+ total = ic.len;
+ do {
+ size_t ret;
+
+ ret = flush_chunk(&ic, 1, stdout, &stream, &iter);
+ total -= ret;
+ if (!total)
+ break;
+
+ ic.seq++;
+ ic.len -= ret;
+ ic.buf += ret;
+ } while (1);
+
+ if (iter.seq) {
+ finish_chunk(&stream, stdout, &iter);
+ free(iter.buf);
+ }
+
+ free(buf);
+ return 0;
+}
+
+#else
+
+static void flush_gz_chunks(struct io_log *log, FILE *f)
+{
+}
+
+#endif
+
+void flush_log(struct io_log *log)
+{
+ void *buf;
+ FILE *f;
+
+ f = fopen(log->filename, "w");
+ if (!f) {
+ perror("fopen log");
+ return;
+ }
+
+ buf = set_file_buffer(f);
+
+ flush_gz_chunks(log, f);
+
+ flush_samples(f, log->log, log->nr_samples * log_entry_sz(log));
fclose(f);
clear_file_buffer(buf);
@@ -627,22 +889,162 @@ void __finish_log(struct io_log *log)
static int finish_log(struct thread_data *td, struct io_log *log, int trylock)
{
+ if (td->tp_data)
+ iolog_flush(log, 1);
+
if (trylock) {
if (fio_trylock_file(log->filename))
return 1;
} else
fio_lock_file(log->filename);
- if (td->client_type == FIO_CLIENT_TYPE_GUI) {
+ if (td->client_type == FIO_CLIENT_TYPE_GUI)
fio_send_iolog(td, log, log->filename);
- } else
- __finish_log(log);
+ else
+ flush_log(log);
fio_unlock_file(log->filename);
free_log(log);
return 0;
}
+#ifdef CONFIG_ZLIB
+
+static int gz_work(struct tp_work *work)
+{
+ struct iolog_flush_data *data;
+ struct iolog_compress *c;
+ struct flist_head list;
+ unsigned int seq;
+ z_stream stream;
+ size_t total = 0;
+ int ret;
+
+ INIT_FLIST_HEAD(&list);
+
+ data = container_of(work, struct iolog_flush_data, work);
+
+ stream.zalloc = Z_NULL;
+ stream.zfree = Z_NULL;
+ stream.opaque = Z_NULL;
+
+ ret = deflateInit(&stream, Z_DEFAULT_COMPRESSION);
+ if (ret != Z_OK) {
+ log_err("fio: failed to init gz stream\n");
+ return 0;
+ }
+
+ seq = ++data->log->chunk_seq;
+
+ stream.next_in = (void *) data->samples;
+ stream.avail_in = data->nr_samples * log_entry_sz(data->log);
+
+ do {
+ c = get_new_chunk(seq);
+ stream.avail_out = GZ_CHUNK;
+ stream.next_out = c->buf;
+ ret = deflate(&stream, Z_NO_FLUSH);
+ if (ret < 0) {
+ log_err("fio: deflate log (%d)\n", ret);
+ break;
+ }
+
+ c->len = GZ_CHUNK - stream.avail_out;
+ flist_add_tail(&c->list, &list);
+ total += c->len;
+ } while (stream.avail_in);
+
+ stream.next_out = c->buf + c->len;
+ stream.avail_out = GZ_CHUNK - c->len;
+
+ ret = deflate(&stream, Z_FINISH);
+ if (ret == Z_STREAM_END)
+ c->len = GZ_CHUNK - stream.avail_out;
+ else {
+ do {
+ c = get_new_chunk(seq);
+ stream.avail_out = GZ_CHUNK;
+ stream.next_out = c->buf;
+ ret = deflate(&stream, Z_FINISH);
+ c->len = GZ_CHUNK - stream.avail_out;
+ flist_add_tail(&c->list, &list);
+ } while (ret != Z_STREAM_END);
+ }
+
+ ret = deflateEnd(&stream);
+ if (ret != Z_OK)
+ log_err("fio: deflateEnd %d\n", ret);
+
+ free(data->samples);
+
+ if (!flist_empty(&list)) {
+ pthread_mutex_lock(&data->log->chunk_lock);
+ flist_splice_tail(&list, &data->log->chunk_list);
+ pthread_mutex_unlock(&data->log->chunk_lock);
+ }
+
+ if (work->wait) {
+ work->done = 1;
+ pthread_cond_signal(&work->cv);
+ } else
+ free(data);
+
+ return 0;
+}
+
+int iolog_flush(struct io_log *log, int wait)
+{
+ struct tp_data *tdat = log->td->tp_data;
+ struct iolog_flush_data *data;
+ size_t sample_size;
+
+ data = malloc(sizeof(*data));
+ if (!data)
+ return 1;
+
+ data->log = log;
+
+ sample_size = log->nr_samples * log_entry_sz(log);
+ data->samples = malloc(sample_size);
+ if (!data->samples) {
+ free(data);
+ return 1;
+ }
+
+ memcpy(data->samples, log->log, sample_size);
+ data->nr_samples = log->nr_samples;
+ data->work.fn = gz_work;
+ log->nr_samples = 0;
+
+ if (wait) {
+ pthread_mutex_init(&data->work.lock, NULL);
+ pthread_cond_init(&data->work.cv, NULL);
+ data->work.wait = 1;
+ } else
+ data->work.wait = 0;
+
+ tp_queue_work(tdat, &data->work);
+
+ if (wait) {
+ pthread_mutex_lock(&data->work.lock);
+ while (!data->work.done)
+ pthread_cond_wait(&data->work.cv, &data->work.lock);
+ pthread_mutex_unlock(&data->work.lock);
+ free(data);
+ }
+
+ return 0;
+}
+
+#else
+
+int iolog_flush(struct io_log *log, int wait)
+{
+ return 1;
+}
+
+#endif
+
static int write_iops_log(struct thread_data *td, int try)
{
struct io_log *log = td->iops_log;
diff --git a/iolog.h b/iolog.h
index f97d91f..fcd6794 100644
--- a/iolog.h
+++ b/iolog.h
@@ -24,7 +24,7 @@ struct io_stat {
struct io_sample {
uint64_t time;
uint64_t val;
- uint32_t ddir;
+ uint32_t __ddir;
uint32_t bs;
};
@@ -52,8 +52,12 @@ struct io_log {
uint64_t max_samples;
void *log;
+ unsigned int log_ddir_mask;
+
char *filename;
+ struct thread_data *td;
+
unsigned int log_type;
/*
@@ -67,14 +71,41 @@ struct io_log {
unsigned int log_offset;
/*
+ * Max size of log entries before a chunk is compressed
+ */
+ unsigned int log_gz;
+
+ /*
+ * Don't deflate for storing, just store the compressed bits
+ */
+ unsigned int log_gz_store;
+
+ /*
* Windowed average, for logging single entries average over some
* period of time.
*/
struct io_stat avg_window[DDIR_RWDIR_CNT];
unsigned long avg_msec;
unsigned long avg_last;
+
+ pthread_mutex_t chunk_lock;
+ unsigned int chunk_seq;
+ struct flist_head chunk_list;
};
+/*
+ * If the upper bit is set, then we have the offset as well
+ */
+#define LOG_OFFSET_SAMPLE_BIT 0x80000000U
+#define io_sample_ddir(io) ((io)->__ddir & ~LOG_OFFSET_SAMPLE_BIT)
+
+static inline void io_sample_set_ddir(struct io_log *log,
+ struct io_sample *io,
+ enum fio_ddir ddir)
+{
+ io->__ddir = ddir | log->log_ddir_mask;
+}
+
static inline size_t __log_entry_sz(int log_offset)
{
if (log_offset)
@@ -153,9 +184,23 @@ extern void queue_io_piece(struct thread_data *, struct io_piece *);
extern void prune_io_piece_log(struct thread_data *);
extern void write_iolog_close(struct thread_data *);
+#ifdef CONFIG_ZLIB
+extern int iolog_file_inflate(const char *);
+#endif
+
/*
* Logging
*/
+struct log_params {
+ struct thread_data *td;
+ unsigned long avg_msec;
+ int log_type;
+ int log_offset;
+ int log_gz;
+ int log_gz_store;
+ int log_compress;
+};
+
extern void finalize_logs(struct thread_data *td);
extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long,
unsigned int, uint64_t);
@@ -169,13 +214,14 @@ extern void add_iops_sample(struct thread_data *, enum fio_ddir, unsigned int,
struct timeval *);
extern void init_disk_util(struct thread_data *);
extern void update_rusage_stat(struct thread_data *);
-extern void setup_log(struct io_log **, unsigned long, int, int, const char *);
-extern void __finish_log(struct io_log *);
+extern void setup_log(struct io_log **, struct log_params *, const char *);
+extern void flush_log(struct io_log *);
extern void free_log(struct io_log *);
extern struct io_log *agg_io_log[DDIR_RWDIR_CNT];
extern int write_bw_log;
extern void add_agg_sample(unsigned long, enum fio_ddir, unsigned int);
extern void fio_writeout_logs(struct thread_data *);
+extern int iolog_flush(struct io_log *, int);
static inline void init_ipo(struct io_piece *ipo)
{
diff --git a/lib/tp.c b/lib/tp.c
new file mode 100644
index 0000000..5111910
--- /dev/null
+++ b/lib/tp.c
@@ -0,0 +1,99 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <unistd.h>
+#include <errno.h>
+#include <pthread.h>
+
+#include "../smalloc.h"
+#include "../log.h"
+#include "tp.h"
+
+static void tp_flush_work(struct flist_head *list)
+{
+ struct tp_work *work;
+
+ while (!flist_empty(list)) {
+ work = flist_entry(list->next, struct tp_work, list);
+ flist_del(&work->list);
+ work->fn(work);
+ }
+}
+
+static void *tp_thread(void *data)
+{
+ struct tp_data *tdat = data;
+ struct flist_head work_list;
+
+ INIT_FLIST_HEAD(&work_list);
+
+ while (1) {
+ pthread_mutex_lock(&tdat->lock);
+
+ if (!tdat->thread_exit && flist_empty(&tdat->work))
+ pthread_cond_wait(&tdat->cv, &tdat->lock);
+
+ if (!flist_empty(&tdat->work))
+ flist_splice_tail_init(&tdat->work, &work_list);
+
+ pthread_mutex_unlock(&tdat->lock);
+
+ if (flist_empty(&work_list)) {
+ if (tdat->thread_exit)
+ break;
+ continue;
+ }
+
+ tp_flush_work(&work_list);
+ }
+
+ return NULL;
+}
+
+void tp_queue_work(struct tp_data *tdat, struct tp_work *work)
+{
+ work->done = 0;
+
+ pthread_mutex_lock(&tdat->lock);
+ flist_add_tail(&work->list, &tdat->work);
+ pthread_cond_signal(&tdat->cv);
+ pthread_mutex_unlock(&tdat->lock);
+}
+
+void tp_init(struct tp_data **tdatp)
+{
+ struct tp_data *tdat;
+ int ret;
+
+ if (*tdatp)
+ return;
+
+ *tdatp = tdat = smalloc(sizeof(*tdat));
+ pthread_mutex_init(&tdat->lock, NULL);
+ INIT_FLIST_HEAD(&tdat->work);
+ pthread_cond_init(&tdat->cv, NULL);
+ pthread_cond_init(&tdat->sleep_cv, NULL);
+
+ ret = pthread_create(&tdat->thread, NULL, tp_thread, tdat);
+ if (ret)
+ log_err("fio: failed to create tp thread\n");
+}
+
+void tp_exit(struct tp_data **tdatp)
+{
+ struct tp_data *tdat = *tdatp;
+ void *ret;
+
+ if (!tdat)
+ return;
+
+ tdat->thread_exit = 1;
+ pthread_mutex_lock(&tdat->lock);
+ pthread_cond_signal(&tdat->cv);
+ pthread_mutex_unlock(&tdat->lock);
+
+ pthread_join(tdat->thread, &ret);
+
+ sfree(tdat);
+ *tdatp = NULL;
+}
diff --git a/lib/tp.h b/lib/tp.h
new file mode 100644
index 0000000..5b07cc6
--- /dev/null
+++ b/lib/tp.h
@@ -0,0 +1,32 @@
+#ifndef FIO_TP_H
+#define FIO_TP_H
+
+#include "../flist.h"
+
+struct tp_work;
+typedef int (tp_work_fn)(struct tp_work *);
+
+struct tp_work {
+ struct flist_head list;
+ tp_work_fn *fn;
+ int wait;
+ pthread_cond_t cv;
+ pthread_mutex_t lock;
+ volatile int done;
+};
+
+struct tp_data {
+ pthread_t thread;
+ pthread_cond_t cv;
+ pthread_mutex_t lock;
+ struct flist_head work;
+ volatile int thread_exit;
+ pthread_cond_t sleep_cv;
+ volatile int sleeping;
+};
+
+extern void tp_init(struct tp_data **);
+extern void tp_exit(struct tp_data **);
+extern void tp_queue_work(struct tp_data *, struct tp_work *);
+
+#endif
diff --git a/options.c b/options.c
index 6d326d4..3a3321f 100644
--- a/options.c
+++ b/options.c
@@ -3104,6 +3104,28 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.category = FIO_OPT_C_LOG,
.group = FIO_OPT_G_INVALID,
},
+#ifdef CONFIG_ZLIB
+ {
+ .name = "log_compression",
+ .lname = "Log compression",
+ .type = FIO_OPT_INT,
+ .off1 = td_var_offset(log_gz),
+ .help = "Log in compressed chunks of this size",
+ .minval = 32 * 1024 * 1024ULL,
+ .maxval = 512 * 1024 * 1024ULL,
+ .category = FIO_OPT_C_LOG,
+ .group = FIO_OPT_G_INVALID,
+ },
+ {
+ .name = "log_store_compressed",
+ .lname = "Log store compressed",
+ .type = FIO_OPT_BOOL,
+ .off1 = td_var_offset(log_gz_store),
+ .help = "Store logs in a compressed format",
+ .category = FIO_OPT_C_LOG,
+ .group = FIO_OPT_G_INVALID,
+ },
+#endif
{
.name = "bwavgtime",
.lname = "Bandwidth average time",
diff --git a/server.c b/server.c
index cd00cc6..e2a6f73 100644
--- a/server.c
+++ b/server.c
@@ -1225,10 +1225,10 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name)
for (i = 0; i < log->nr_samples; i++) {
struct io_sample *s = get_sample(log, i);
- s->time = cpu_to_le64(s->time);
- s->val = cpu_to_le64(s->val);
- s->ddir = cpu_to_le32(s->ddir);
- s->bs = cpu_to_le32(s->bs);
+ s->time = cpu_to_le64(s->time);
+ s->val = cpu_to_le64(s->val);
+ s->__ddir = cpu_to_le32(s->__ddir);
+ s->bs = cpu_to_le32(s->bs);
if (log->log_offset) {
struct io_sample_offset *so = (void *) s;
diff --git a/stat.c b/stat.c
index 58744a8..979c810 100644
--- a/stat.c
+++ b/stat.c
@@ -1565,7 +1565,7 @@ static void __add_log_sample(struct io_log *iolog, unsigned long val,
enum fio_ddir ddir, unsigned int bs,
unsigned long t, uint64_t offset)
{
- const int nr_samples = iolog->nr_samples;
+ uint64_t nr_samples = iolog->nr_samples;
struct io_sample *s;
if (iolog->disabled)
@@ -1579,21 +1579,31 @@ static void __add_log_sample(struct io_log *iolog, unsigned long val,
void *new_log;
new_size = 2 * iolog->max_samples * log_entry_sz(iolog);
- new_log = realloc(iolog->log, new_size);
- if (!new_log) {
- log_err("fio: failed extending iolog! Will stop logging.\n");
- iolog->disabled = 1;
- return;
+
+ if (iolog->log_gz && (new_size > iolog->log_gz)) {
+ if (iolog_flush(iolog, 0)) {
+ log_err("fio: failed flushing iolog! Will stop logging.\n");
+ iolog->disabled = 1;
+ return;
+ }
+ nr_samples = iolog->nr_samples;
+ } else {
+ new_log = realloc(iolog->log, new_size);
+ if (!new_log) {
+ log_err("fio: failed extending iolog! Will stop logging.\n");
+ iolog->disabled = 1;
+ return;
+ }
+ iolog->log = new_log;
+ iolog->max_samples <<= 1;
}
- iolog->log = new_log;
- iolog->max_samples <<= 1;
}
s = get_sample(iolog, nr_samples);
s->val = val;
s->time = t;
- s->ddir = ddir;
+ io_sample_set_ddir(iolog, s, ddir);
s->bs = bs;
if (iolog->log_offset) {
diff --git a/thread_options.h b/thread_options.h
index e53000a..e545a8f 100644
--- a/thread_options.h
+++ b/thread_options.h
@@ -109,6 +109,8 @@ struct thread_options {
unsigned int use_os_rand;
unsigned int log_avg_msec;
unsigned int log_offset;
+ unsigned int log_gz;
+ unsigned int log_gz_store;
unsigned int norandommap;
unsigned int softrandommap;
unsigned int bs_unaligned;
@@ -337,6 +339,8 @@ struct thread_options_pack {
uint32_t use_os_rand;
uint32_t log_avg_msec;
uint32_t log_offset;
+ uint32_t log_gz;
+ uint32_t log_gz_store;
uint32_t norandommap;
uint32_t softrandommap;
uint32_t bs_unaligned;
diff --git a/trim.c b/trim.c
index de792dc..a7f1b86 100644
--- a/trim.c
+++ b/trim.c
@@ -24,7 +24,7 @@ int get_next_trim(struct thread_data *td, struct io_u *io_u)
return 1;
assert(td->trim_entries);
- ipo = flist_entry(td->trim_list.next, struct io_piece, trim_list);
+ ipo = flist_first_entry(&td->trim_list, struct io_piece, trim_list);
remove_trim_entry(td, ipo);
io_u->offset = ipo->offset;
diff --git a/verify.c b/verify.c
index 2615701..11963e1 100644
--- a/verify.c
+++ b/verify.c
@@ -1082,7 +1082,7 @@ int get_next_verify(struct thread_data *td, struct io_u *io_u)
assert(ipo->flags & IP_F_ONRB);
ipo->flags &= ~IP_F_ONRB;
} else if (!flist_empty(&td->io_hist_list)) {
- ipo = flist_entry(td->io_hist_list.next, struct io_piece, list);
+ ipo = flist_first_entry(&td->io_hist_list, struct io_piece, list);
/*
* Ensure that the associated IO has completed
@@ -1187,7 +1187,7 @@ static void *verify_async_thread(void *data)
continue;
while (!flist_empty(&list)) {
- io_u = flist_entry(list.next, struct io_u, verify_list);
+ io_u = flist_first_entry(&list, struct io_u, verify_list);
flist_del(&io_u->verify_list);
ret = verify_io_u(td, io_u);
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
