This engine is to measure performance of accessing file's meta data.
This is for the actual access pattern which does not do real IO, but
just look up the file and get file's attribute.
---
HOWTO | 5 ++
Makefile | 2 +-
engines/filestat.c | 116 +++++++++++++++++++++++++++++++++
examples/filestat-ioengine.fio | 19 ++++++
fio.1 | 5 ++
5 files changed, 146 insertions(+), 1 deletion(-)
create mode 100644 engines/filestat.c
create mode 100644 examples/filestat-ioengine.fio
diff --git a/HOWTO b/HOWTO
index 88dbb03f..5e424e84 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1996,6 +1996,11 @@ I/O engine
set `filesize` so that all the accounting still occurs, but no
actual I/O will be done other than creating the file.
+ **filestat**
+ Simply do stat() and do no I/O to the file. You need to set 'filesize'
+ and 'nrfiles', so that files will be created.
+ This engine is to measure file lookup and meta data access.
+
**libpmem**
Read and write using mmap I/O to a file on a filesystem
mounted with DAX on a persistent memory device through the PMDK
diff --git a/Makefile b/Makefile
index 7aab6abd..ab85ceac 100644
--- a/Makefile
+++ b/Makefile
@@ -45,7 +45,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \
pshared.c options.c \
smalloc.c filehash.c profile.c debug.c engines/cpu.c \
engines/mmap.c engines/sync.c engines/null.c engines/net.c \
- engines/ftruncate.c engines/filecreate.c \
+ engines/ftruncate.c engines/filecreate.c engines/filestat.c \
server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \
gettime-thread.c helpers.c json.c idletime.c td_error.c \
profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \
diff --git a/engines/filestat.c b/engines/filestat.c
new file mode 100644
index 00000000..c48f0d89
--- /dev/null
+++ b/engines/filestat.c
@@ -0,0 +1,116 @@
+/*
+ * filestat engine
+ *
+ * IO engine that doesn't do any IO, just stat files and tracks the latency
+ * of the file stat.
+ */
+#include <stdio.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include "../fio.h"
+
+struct fc_data {
+ enum fio_ddir stat_ddir;
+};
+
+static int stat_file(struct thread_data *td, struct fio_file *f)
+{
+ struct timespec start;
+ int do_lat = !td->o.disable_lat;
+ struct stat statbuf;
+ int ret;
+
+ dprint(FD_FILE, "fd stat %s\n", f->file_name);
+
+ if (f->filetype != FIO_TYPE_FILE) {
+ log_err("fio: only files are supported fallocate \n");
+ return 1;
+ }
+ if (!strcmp(f->file_name, "-")) {
+ log_err("fio: can't read/write to stdin/out\n");
+ return 1;
+ }
+
+ if (do_lat)
+ fio_gettime(&start, NULL);
+
+ ret = stat(f->file_name, &statbuf);
+
+ if (ret == -1) {
+ char buf[FIO_VERROR_SIZE];
+ int e = errno;
+
+ snprintf(buf, sizeof(buf), "stat(%s)", f->file_name);
+ td_verror(td, e, buf);
+ return 1;
+ }
+
+ if (do_lat) {
+ struct fc_data *data = td->io_ops_data;
+ uint64_t nsec;
+
+ nsec = ntime_since_now(&start);
+ add_clat_sample(td, data->stat_ddir, nsec, 0, 0);
+ }
+
+ return 0;
+}
+
+static enum fio_q_status queue_io(struct thread_data *td, struct io_u fio_unused *io_u)
+{
+ return FIO_Q_COMPLETED;
+}
+
+static int init(struct thread_data *td)
+{
+ struct fc_data *data;
+
+ data = calloc(1, sizeof(*data));
+
+ if (td_read(td))
+ data->stat_ddir = DDIR_READ;
+ else if (td_write(td))
+ data->stat_ddir = DDIR_WRITE;
+
+ td->io_ops_data = data;
+ return 0;
+}
+
+static void cleanup(struct thread_data *td)
+{
+ struct fc_data *data = td->io_ops_data;
+
+ free(data);
+}
+
+static int stat_invalidate(struct thread_data *td, struct fio_file *f)
+{
+ /* do nothing because file not opened */
+ return 0;
+}
+
+static struct ioengine_ops ioengine = {
+ .name = "filestat",
+ .version = FIO_IOOPS_VERSION,
+ .init = init,
+ .cleanup = cleanup,
+ .queue = queue_io,
+ .invalidate = stat_invalidate,
+ .get_file_size = generic_get_file_size,
+ .open_file = stat_file,
+ .flags = FIO_SYNCIO | FIO_FAKEIO |
+ FIO_NOSTATS | FIO_NOFILEHASH,
+};
+
+static void fio_init fio_filestat_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_filestat_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/examples/filestat-ioengine.fio b/examples/filestat-ioengine.fio
new file mode 100644
index 00000000..932fced8
--- /dev/null
+++ b/examples/filestat-ioengine.fio
@@ -0,0 +1,19 @@
+# Example filestat job
+
+# 'filestat' engine only do 'stat(filename)', file will not be open().
+# 'filesize' must be set, then files will be created at setup stage.
+
+[global]
+ioengine=filestat
+numjobs=1
+filesize=4k
+nrfiles=200
+thread
+
+[t0]
+[t1]
+[t2]
+[t3]
+[t4]
+[t5]
+
diff --git a/fio.1 b/fio.1
index 14569e9f..bb1e42f6 100644
--- a/fio.1
+++ b/fio.1
@@ -1760,6 +1760,11 @@ Simply create the files and do no I/O to them. You still need to set
\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be
done other than creating the file.
.TP
+.B filestat
+Simply do stat() and do no I/O to the file. You need to set 'filesize'
+and 'nrfiles', so that files will be created.
+This engine is to measure file lookup and meta data access.
+.TP
.B libpmem
Read and write using mmap I/O to a file on a filesystem
mounted with DAX on a persistent memory device through the PMDK
--
2.17.1
-----Original Message-----
From: Jens Axboe <axboe@xxxxxxxxx>
Sent: Monday, December 30, 2019 7:21 AM
To: Sitsofe Wheeler <sitsofe@xxxxxxxxx>; Su, Friendy <Friendy.Su@xxxxxxxx>
Cc: fio <fio@xxxxxxxxxxxxxxx>; Kobayashi, Kento (Sony) <Kento.A.Kobayashi@xxxxxxxx>
Subject: Re: [PATCH] engines: add engine for file stat
On 12/29/19 2:23 PM, Sitsofe Wheeler wrote:
> On Thu, 19 Dec 2019 at 10:20, Su, Friendy <Friendy.Su@xxxxxxxx> wrote:
>>
>> Thanks for your comments.
>> I fixed and resent patch. Pls see my reply in text below.
>
> From a brief scan it looks OK to me but I'd say you are better off
> posting patches in-line because it make review easier (anyone else who
> wants to see it will have to look at the parent of this mail). If this
> is troublesome you can always try submitting a pull request (e.g. via
> https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_axboe_fio_pulls&d=DwICaQ&c=fP4tf--1dS0biCFlB0saz0I0kjO5v7-GLPtvShAo4cc&r=x4w-xeFLV1q9QNvT-g51hIN_YdnTDh2iniRTDjOpGTk&m=n7aH04Az0Bke6LtP1bcGhKisgjZ4-_cX3Cm0aJyo62Y&s=HZGFwGTEsbbO6oWDd2fNQYzft3kP9Yv1DBiBq7dmDj0&e= )...
>
> Jens: If you can see the patch as an attachment (e.g. at the bottom of
> https://urldefense.proofpoint.com/v2/url?u=https-3A__www.spinics.net_lists_fio_msg08203.html&d=DwICaQ&c=fP4tf--1dS0biCFlB0saz0I0kjO5v7-GLPtvShAo4cc&r=x4w-xeFLV1q9QNvT-g51hIN_YdnTDh2iniRTDjOpGTk&m=n7aH04Az0Bke6LtP1bcGhKisgjZ4-_cX3Cm0aJyo62Y&s=gVr4lkUJChqpKsxpSPdguu8txoH_geku7Z5MQXzYK_k&e= ), what do you think?
With my io_uring hat on, there's a few ways to do stat since io_uring also supports it. Is the plan to enable a io_uring backend as well for that engine? Or should stat perhaps be a data direction instead that all io engines could then support, some of them in an async manner?
--
Jens Axboe
