Show correct file operation strings in log.
For file operation measurement, 'rw' field is set to 'read' in job file.
The corresponding 'rw' in log is also 'read'. This makes user confused
since file operation and IO read are totally different things.
In this commit, add 3 new file operations to 'rw' field.
For implementation, TD_DDIR_FILECREATE/TD_DDIR_FILESTAT/TD_DDIR_FILEDELETE
are added. Meanwhile, DDIR_FILE_OP is added to identify the file
operation and IO operation. DDIR_FILE_OP is same value as
DDIR_READ, then clat_stat[DDIR_FILE_READ] can be 'borrowed' for file
operation statistics. Just set DDIR_FILE_OP_MASK when get io_ddir_name().
This avoid big code change if add a new index to enum fio_ddir.
With this commit, user can explicitly specify the exact file operation
in job file 'rw' field. 'rw' string and 'ddir' string in log is also
correct.
Example:
$ fio filestat-ioengine.fio
t0: (g=0): rw=filestat, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=filestat, iodepth=1
t1: (g=0): rw=filestat, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=filestat, iodepth=1
fio-3.33
Starting 2 threads
t0: (groupid=0, jobs=1): err= 0: pid=4744: Thu Nov 10 18:24:00 2022
file_operation: IOPS=20.0k, BW=78.1MiB/s (81.9MB/s)(80.0KiB/1msec)
clat (nsec): min=12645, max=29394, avg=13751.35, stdev=3690.55
clat percentiles (nsec):
| 1.00th=[12608], 5.00th=[12608], 10.00th=[12736], 20.00th=[12736],
| 30.00th=[12736], 40.00th=[12736], 50.00th=[12864], 60.00th=[12992],
| 70.00th=[12992], 80.00th=[13120], 90.00th=[13120], 95.00th=[13760],
| 99.00th=[29312], 99.50th=[29312], 99.90th=[29312], 99.95th=[29312],
| 99.99th=[29312]
Signed-off-by: friendy-su <friendy.su@xxxxxxxx>
---
HOWTO.rst | 6 ++++++
backend.c | 3 +++
engines/fileoperations.c | 7 +++----
examples/filecreate-ioengine.fio | 1 +
examples/filedelete-ioengine.fio | 1 +
examples/filestat-ioengine.fio | 1 +
fio.1 | 10 ++++++++++
fio.h | 1 +
io_ddir.h | 26 ++++++++++++++++++++++++--
io_u.c | 7 ++++---
iolog.c | 2 +-
options.c | 12 ++++++++++++
stat.c | 21 +++++++++++++++++----
stat.h | 1 +
14 files changed, 85 insertions(+), 14 deletions(-)
diff --git a/HOWTO.rst b/HOWTO.rst
index e796f961..c176b7d3 100644
--- a/HOWTO.rst
+++ b/HOWTO.rst
@@ -1137,6 +1137,12 @@ I/O type
**randtrimwrite**
Like trimwrite, but uses random offsets rather
than sequential writes.
+ **filecreate**
+ Create a batch of files.
+ **filestat**
+ Stat a batch of files.
+ **filedelete**
+ Delete a batch of files.
Fio defaults to read if the option is not specified. For the mixed I/O
types, the default is to split them 50/50. For certain types of I/O the
diff --git a/backend.c b/backend.c
index ba954a6b..7024003a 100644
--- a/backend.c
+++ b/backend.c
@@ -1924,6 +1924,9 @@ static void *thread_main(void *data)
update_runtime(td, elapsed_us, DDIR_WRITE);
if (td_trim(td) && td->io_bytes[DDIR_TRIM])
update_runtime(td, elapsed_us, DDIR_TRIM);
+ if (td_fileoperate(td) && td->io_bytes[DDIR_FILE_OP])
+ update_runtime(td, elapsed_us, DDIR_FILE_OP);
+
fio_gettime(&td->start, NULL);
fio_sem_up(stat_sem);
diff --git a/engines/fileoperations.c b/engines/fileoperations.c
index 1db60da1..3b44d429 100644
--- a/engines/fileoperations.c
+++ b/engines/fileoperations.c
@@ -245,12 +245,11 @@ static int init(struct thread_data *td)
data = calloc(1, sizeof(*data));
- if (td_read(td))
- data->stat_ddir = DDIR_READ;
- else if (td_write(td))
- data->stat_ddir = DDIR_WRITE;
+ if (td_fileoperate(td))
+ data->stat_ddir = DDIR_FILE_OP;
td->io_ops_data = data;
+ td->file_op_flag = DDIR_FILE_OP_MASK;
return 0;
}
diff --git a/examples/filecreate-ioengine.fio b/examples/filecreate-ioengine.fio
index ec7caad5..86226b19 100644
--- a/examples/filecreate-ioengine.fio
+++ b/examples/filecreate-ioengine.fio
@@ -10,6 +10,7 @@
# IO will not really be done and the write latency numbers will only reflect the
# open times.
[global]
+rw=filecreate
create_on_open=1
nrfiles=31250
ioengine=filecreate
diff --git a/examples/filedelete-ioengine.fio b/examples/filedelete-ioengine.fio
index 3c0028f9..6fa2b6a7 100644
--- a/examples/filedelete-ioengine.fio
+++ b/examples/filedelete-ioengine.fio
@@ -5,6 +5,7 @@
# 'unlink' is better set to 0, since the file is deleted in measurement.
# the options disabled completion latency output such as 'disable_clat' and 'gtod_reduce' must not set.
[global]
+rw=filedelete
ioengine=filedelete
filesize=4k
nrfiles=200
diff --git a/examples/filestat-ioengine.fio b/examples/filestat-ioengine.fio
index 932fced8..d225795b 100644
--- a/examples/filestat-ioengine.fio
+++ b/examples/filestat-ioengine.fio
@@ -4,6 +4,7 @@
# 'filesize' must be set, then files will be created at setup stage.
[global]
+rw=filestat
ioengine=filestat
numjobs=1
filesize=4k
diff --git a/fio.1 b/fio.1
index 9e33c9e1..dc22e83d 100644
--- a/fio.1
+++ b/fio.1
@@ -909,6 +909,16 @@ other Fio options limiting the total bytes or number of I/O's.
Like
.B trimwrite ,
but uses random offsets rather than sequential writes.
+.TP
+.B filecreate
+Create a batch of files.
+.TP
+.B filestat
+Stat a batch of files.
+.TP
+.B filedelete
+Delete a batch of files.
+.TP
.RE
.P
Fio defaults to read if the option is not specified. For the mixed I/O
diff --git a/fio.h b/fio.h
index 8da77640..6e6ed780 100644
--- a/fio.h
+++ b/fio.h
@@ -191,6 +191,7 @@ struct thread_data {
unsigned int thread_number;
unsigned int subjob_number;
unsigned int groupid;
+ uint32_t file_op_flag;
struct thread_stat ts __attribute__ ((aligned(8)));
int client_type;
diff --git a/io_ddir.h b/io_ddir.h
index 217eb628..7de5926b 100644
--- a/io_ddir.h
+++ b/io_ddir.h
@@ -14,8 +14,17 @@ enum fio_ddir {
DDIR_RWDIR_CNT = 3,
DDIR_RWDIR_SYNC_CNT = 4,
+
+ /*
+ * file operation ddir 'borrows' DDIR_READ, then ts->clat_stat[0] is also borrowed
+ * only when print by io_ddir_name(), DDIR_FILE_OP_MASK is set to print "file_operation"
+ * string
+ */
+ DDIR_FILE_OP = 0,
};
+#define DDIR_FILE_OP_MASK 0x80
+
#define for_each_rw_ddir(ddir) for (enum fio_ddir ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++)
static inline const char *io_ddir_name(enum fio_ddir ddir)
@@ -24,7 +33,10 @@ static inline const char *io_ddir_name(enum fio_ddir ddir)
"datasync", "sync_file_range",
"wait", };
- if (ddir >= 0 && ddir < DDIR_LAST)
+
+ if (ddir & DDIR_FILE_OP_MASK)
+ return "file_operation";
+ else if (ddir >= 0 && ddir < DDIR_LAST)
return name[ddir];
return "invalid";
@@ -42,6 +54,12 @@ enum td_ddir {
TD_DDIR_RANDTRIM = TD_DDIR_TRIM | TD_DDIR_RAND,
TD_DDIR_TRIMWRITE = TD_DDIR_TRIM | TD_DDIR_WRITE,
TD_DDIR_RANDTRIMWRITE = TD_DDIR_RANDTRIM | TD_DDIR_WRITE,
+
+ TD_DDIR_FILEOP = 1 << 4,
+ TD_DDIR_FILEOP_MASK = TD_DDIR_FILEOP | (TD_DDIR_FILEOP - 1),
+ TD_DDIR_FILECREATE = TD_DDIR_FILEOP | 0,
+ TD_DDIR_FILESTAT = TD_DDIR_FILEOP | 1,
+ TD_DDIR_FILEDELETE = TD_DDIR_FILEOP | 2,
};
#define td_read(td) ((td)->o.td_ddir & TD_DDIR_READ)
@@ -54,6 +72,10 @@ enum td_ddir {
== TD_DDIR_TRIMWRITE)
#define td_randtrimwrite(td) (((td)->o.td_ddir & TD_DDIR_RANDTRIMWRITE) \
== TD_DDIR_RANDTRIMWRITE)
+#define td_filecreate(td) (((td)->o.td_ddir & TD_DDIR_FILEOP_MASK) == TD_DDIR_FILECREATE)
+#define td_filestat(td) (((td)->o.td_ddir & TD_DDIR_FILEOP_MASK) == TD_DDIR_FILESTAT)
+#define td_filedelete(td) (((td)->o.td_ddir & TD_DDIR_FILEOP_MASK) == TD_DDIR_FILEDELETE)
+#define td_fileoperate(td) ((td)->o.td_ddir & TD_DDIR_FILEOP)
static inline int ddir_sync(enum fio_ddir ddir)
{
@@ -71,7 +93,7 @@ static inline const char *ddir_str(enum td_ddir ddir)
static const char *__str[] = { NULL, "read", "write", "rw", "rand",
"randread", "randwrite", "randrw",
"trim", NULL, "trimwrite", NULL, "randtrim",
- NULL, "randtrimwrite" };
+ NULL, "randtrimwrite", NULL, "filecreate", "filestat", "filedelete" };
return __str[ddir];
}
diff --git a/io_u.c b/io_u.c
index 8035f4b7..aaea3fdd 100644
--- a/io_u.c
+++ b/io_u.c
@@ -674,7 +674,6 @@ static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir)
enum fio_ddir odir = ddir ^ 1;
uint64_t usec;
uint64_t now;
-
assert(ddir_rw(ddir));
now = utime_since_now(&td->epoch);
@@ -782,6 +781,8 @@ static enum fio_ddir get_rw_ddir(struct thread_data *td)
ddir = DDIR_WRITE;
else if (td_trim(td))
ddir = DDIR_TRIM;
+ else if (td_fileoperate(td))
+ ddir = DDIR_FILE_OP;
else
ddir = DDIR_INVAL;
@@ -1426,7 +1427,7 @@ static void lat_fatal(struct thread_data *td, struct io_u *io_u, struct io_compl
log_err("fio: latency of %llu nsec exceeds specified max (%llu nsec): %s %s %llu %llu\n",
tnsec, max_nsec,
io_u->file->file_name,
- io_ddir_name(io_u->ddir),
+ io_ddir_name(io_u->ddir | td->file_op_flag),
io_u->offset, io_u->buflen);
}
td_verror(td, ETIMEDOUT, "max latency exceeded");
@@ -1862,7 +1863,7 @@ static void __io_u_log_error(struct thread_data *td, struct io_u *io_u)
io_u->file ? " on file " : "",
io_u->file ? io_u->file->file_name : "",
strerror(io_u->error),
- io_ddir_name(io_u->ddir),
+ io_ddir_name(io_u->ddir | td->file_op_flag),
io_u->offset, io_u->xfer_buflen);
if (td->io_ops->errdetails) {
diff --git a/iolog.c b/iolog.c
index aa9c3bb1..7b9d7088 100644
--- a/iolog.c
+++ b/iolog.c
@@ -49,7 +49,7 @@ void log_io_u(const struct thread_data *td, const struct io_u *io_u)
fio_gettime(&now, NULL);
fprintf(td->iolog_f, "%llu %s %s %llu %llu\n",
(unsigned long long) utime_since_now(&td->io_log_start_time),
- io_u->file->file_name, io_ddir_name(io_u->ddir), io_u->offset,
+ io_u->file->file_name, io_ddir_name(io_u->ddir | td->file_op_flag), io_u->offset,
io_u->buflen);
}
diff --git a/options.c b/options.c
index 9e4d8cd1..bbe0eada 100644
--- a/options.c
+++ b/options.c
@@ -1951,6 +1951,18 @@ struct fio_option fio_options[FIO_MAX_OPTS] = {
.oval = TD_DDIR_RANDTRIMWRITE,
.help = "Randomly trim and write mix, trims preceding writes"
},
+ { .ival = "filecreate",
+ .oval = TD_DDIR_FILECREATE,
+ .help = "Create file with file size zero"
+ },
+ { .ival = "filestat",
+ .oval = TD_DDIR_FILESTAT,
+ .help = "Stat files"
+ },
+ { .ival = "filedelete",
+ .oval = TD_DDIR_FILEDELETE,
+ .help = "Delete files"
+ },
},
},
{
diff --git a/stat.c b/stat.c
index b963973a..c52c946c 100644
--- a/stat.c
+++ b/stat.c
@@ -517,6 +517,11 @@ static double convert_agg_kbytes_percent(struct group_run_stats *rs,
return p_of_agg;
}
+uint32_t get_file_op_flag_from_ts(struct thread_stat *ts)
+{
+ return ts->file_op_flag;
+}
+
static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
enum fio_ddir ddir, struct buf_output *out)
{
@@ -526,16 +531,19 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
char *io_p, *bw_p, *bw_p_alt, *iops_p, *post_st = NULL;
int i2p, i;
const char *clat_type = ts->lat_percentiles ? "lat" : "clat";
+ uint32_t file_op_flag;
+
+ file_op_flag = get_file_op_flag_from_ts(ts);
if (ddir_sync(ddir)) {
if (calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) {
log_buf(out, " %s:\n", "fsync/fdatasync/sync_file_range");
- display_lat(io_ddir_name(ddir), min, max, mean, dev, out);
+ display_lat(io_ddir_name(ddir | file_op_flag), min, max, mean, dev, out);
show_clat_percentiles(ts->io_u_sync_plat,
ts->sync_stat.samples,
ts->percentile_list,
ts->percentile_precision,
- io_ddir_name(ddir), out);
+ io_ddir_name(ddir | file_op_flag), out);
}
return;
}
@@ -569,7 +577,7 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts,
}
log_buf(out, " %s: IOPS=%s, BW=%s (%s)(%s/%llumsec)%s\n",
- (ts->unified_rw_rep == UNIFIED_MIXED) ? "mixed" : io_ddir_name(ddir),
+ (ts->unified_rw_rep == UNIFIED_MIXED) ? "mixed" : io_ddir_name(ddir | file_op_flag),
iops_p, bw_p, bw_p_alt, io_p,
(unsigned long long) ts->runtime[ddir],
post_st ? : "");
@@ -1441,6 +1449,9 @@ static void add_ddir_status_json(struct thread_stat *ts,
double mean, dev, iops;
struct json_object *dir_object, *tmp_object;
double p_of_agg = 100.0;
+ uint32_t file_op_flag;
+
+ file_op_flag = get_file_op_flag_from_ts(ts);
assert(ddir_rw(ddir) || ddir_sync(ddir));
@@ -1449,7 +1460,7 @@ static void add_ddir_status_json(struct thread_stat *ts,
dir_object = json_create_object();
json_object_add_value_object(parent,
- (ts->unified_rw_rep == UNIFIED_MIXED) ? "mixed" : io_ddir_name(ddir), dir_object);
+ (ts->unified_rw_rep == UNIFIED_MIXED) ? "mixed" : io_ddir_name(ddir | file_op_flag), dir_object);
if (ddir_rw(ddir)) {
bw_bytes = 0;
@@ -2487,6 +2498,8 @@ void __show_run_stats(void)
memcpy(ts->percentile_list, td->o.percentile_list, sizeof(td->o.percentile_list));
opt_lists[j] = &td->opt_list;
+ ts->file_op_flag = td->file_op_flag;
+
idx++;
if (ts->groupid == -1) {
diff --git a/stat.h b/stat.h
index 4c3bf71f..f67965f9 100644
--- a/stat.h
+++ b/stat.h
@@ -175,6 +175,7 @@ struct thread_stat {
uint32_t members;
uint32_t unified_rw_rep;
uint32_t disable_prio_stat;
+ uint32_t file_op_flag;
/*
* bandwidth and latency stats
--
2.17.1
