The following changes since commit d9b7596a1fad5adf7f6731d067e1513c56eabe96: Merge branch 'master' of https://github.com/bvanassche/fio (2020-01-18 09:35:25 -0700) are available in the Git repository at: git://git.kernel.dk/fio.git master for you to fetch changes up to 4ee9c375b64de236356cc97843d84978edeba491: stat: ensure we align correctly (2020-01-22 19:53:14 -0700) ---------------------------------------------------------------- Jens Axboe (2): Merge branch 'priorityQueuing' of https://github.com/Seagate/fio stat: ensure we align correctly Phillip Chen (2): Whitespace standardization Per-command priority: Priority logging and libaio/io_uring cmdprio_percentage Su, Friendy (1): engines: add engine for file stat HOWTO | 43 +++++--- Makefile | 2 +- client.c | 10 +- engines/filecreate.c | 2 +- engines/filestat.c | 116 +++++++++++++++++++++ engines/io_uring.c | 47 +++++++++ engines/libaio.c | 54 ++++++++++ eta.c | 6 +- examples/filestat-ioengine.fio | 19 ++++ fio.1 | 32 ++++-- fio.h | 4 +- init.c | 3 + io_u.c | 8 +- io_u.h | 3 + ioengines.c | 1 + iolog.c | 8 +- iolog.h | 1 + server.c | 13 ++- server.h | 2 +- stat.c | 229 +++++++++++++++++++++++++++++++++++------ stat.h | 14 ++- 21 files changed, 541 insertions(+), 76 deletions(-) create mode 100644 engines/filestat.c create mode 100644 examples/filestat-ioengine.fio --- Diff of recent changes: diff --git a/HOWTO b/HOWTO index 41d32c04..0a366168 100644 --- a/HOWTO +++ b/HOWTO @@ -1996,6 +1996,11 @@ I/O engine set `filesize` so that all the accounting still occurs, but no actual I/O will be done other than creating the file. + **filestat** + Simply do stat() and do no I/O to the file. You need to set 'filesize' + and 'nrfiles', so that files will be created. + This engine is to measure file lookup and meta data access. + **libpmem** Read and write using mmap I/O to a file on a filesystem mounted with DAX on a persistent memory device through the PMDK @@ -2029,21 +2034,29 @@ In addition, there are some parameters which are only valid when a specific with the caveat that when used on the command line, they must come after the :option:`ioengine` that defines them is selected. -.. option:: hipri : [io_uring] +.. option:: cmdprio_percentage=int : [io_uring] [libaio] - If this option is set, fio will attempt to use polled IO completions. - Normal IO completions generate interrupts to signal the completion of - IO, polled completions do not. Hence they are require active reaping - by the application. The benefits are more efficient IO for high IOPS - scenarios, and lower latencies for low queue depth IO. + Set the percentage of I/O that will be issued with higher priority by setting + the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``. + This option cannot be used with the `prio` or `prioclass` options. For this + option to set the priority bit properly, NCQ priority must be supported and + enabled and :option:`direct`\=1 option must be used. .. option:: fixedbufs : [io_uring] - If fio is asked to do direct IO, then Linux will map pages for each - IO call, and release them when IO is done. If this option is set, the - pages are pre-mapped before IO is started. This eliminates the need to - map and release for each IO. This is more efficient, and reduces the - IO latency as well. + If fio is asked to do direct IO, then Linux will map pages for each + IO call, and release them when IO is done. If this option is set, the + pages are pre-mapped before IO is started. This eliminates the need to + map and release for each IO. This is more efficient, and reduces the + IO latency as well. + +.. option:: hipri : [io_uring] + + If this option is set, fio will attempt to use polled IO completions. + Normal IO completions generate interrupts to signal the completion of + IO, polled completions do not. Hence they are require active reaping + by the application. The benefits are more efficient IO for high IOPS + scenarios, and lower latencies for low queue depth IO. .. option:: registerfiles : [io_uring] @@ -2687,11 +2700,15 @@ Threads, processes and job synchronization Set the I/O priority value of this job. Linux limits us to a positive value between 0 and 7, with 0 being the highest. See man :manpage:`ionice(1)`. Refer to an appropriate manpage for other operating - systems since meaning of priority may differ. + systems since meaning of priority may differ. For per-command priority + setting, see I/O engine specific `cmdprio_percentage` and `hipri_percentage` + options. .. option:: prioclass=int - Set the I/O priority class. See man :manpage:`ionice(1)`. + Set the I/O priority class. See man :manpage:`ionice(1)`. For per-command + priority setting, see I/O engine specific `cmdprio_percentage` and + `hipri_percentage` options. .. option:: cpus_allowed=str diff --git a/Makefile b/Makefile index 3c5e0f5b..027b62bc 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ pshared.c options.c \ smalloc.c filehash.c profile.c debug.c engines/cpu.c \ engines/mmap.c engines/sync.c engines/null.c engines/net.c \ - engines/ftruncate.c engines/filecreate.c \ + engines/ftruncate.c engines/filecreate.c engines/filestat.c \ server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ diff --git a/client.c b/client.c index 93bca5df..4aed39e7 100644 --- a/client.c +++ b/client.c @@ -1032,6 +1032,14 @@ static void convert_ts(struct thread_stat *dst, struct thread_stat *src) dst->nr_block_infos = le64_to_cpu(src->nr_block_infos); for (i = 0; i < dst->nr_block_infos; i++) dst->block_infos[i] = le32_to_cpu(src->block_infos[i]); + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { + dst->io_u_plat_high_prio[i][j] = le64_to_cpu(src->io_u_plat_high_prio[i][j]); + dst->io_u_plat_prio[i][j] = le64_to_cpu(src->io_u_plat_prio[i][j]); + } + convert_io_stat(&dst->clat_high_prio_stat[i], &src->clat_high_prio_stat[i]); + convert_io_stat(&dst->clat_prio_stat[i], &src->clat_prio_stat[i]); + } dst->ss_dur = le64_to_cpu(src->ss_dur); dst->ss_state = le32_to_cpu(src->ss_state); @@ -1693,7 +1701,7 @@ static struct cmd_iolog_pdu *convert_iolog(struct fio_net_cmd *cmd, s->time = le64_to_cpu(s->time); s->data.val = le64_to_cpu(s->data.val); - s->__ddir = le32_to_cpu(s->__ddir); + s->__ddir = __le32_to_cpu(s->__ddir); s->bs = le64_to_cpu(s->bs); if (ret->log_offset) { diff --git a/engines/filecreate.c b/engines/filecreate.c index 39a29502..5fec8544 100644 --- a/engines/filecreate.c +++ b/engines/filecreate.c @@ -49,7 +49,7 @@ static int open_file(struct thread_data *td, struct fio_file *f) uint64_t nsec; nsec = ntime_since_now(&start); - add_clat_sample(td, data->stat_ddir, nsec, 0, 0); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0); } return 0; diff --git a/engines/filestat.c b/engines/filestat.c new file mode 100644 index 00000000..79525934 --- /dev/null +++ b/engines/filestat.c @@ -0,0 +1,116 @@ +/* + * filestat engine + * + * IO engine that doesn't do any IO, just stat files and tracks the latency + * of the file stat. + */ +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include "../fio.h" + +struct fc_data { + enum fio_ddir stat_ddir; +}; + +static int stat_file(struct thread_data *td, struct fio_file *f) +{ + struct timespec start; + int do_lat = !td->o.disable_lat; + struct stat statbuf; + int ret; + + dprint(FD_FILE, "fd stat %s\n", f->file_name); + + if (f->filetype != FIO_TYPE_FILE) { + log_err("fio: only files are supported\n"); + return 1; + } + if (!strcmp(f->file_name, "-")) { + log_err("fio: can't read/write to stdin/out\n"); + return 1; + } + + if (do_lat) + fio_gettime(&start, NULL); + + ret = stat(f->file_name, &statbuf); + + if (ret == -1) { + char buf[FIO_VERROR_SIZE]; + int e = errno; + + snprintf(buf, sizeof(buf), "stat(%s)", f->file_name); + td_verror(td, e, buf); + return 1; + } + + if (do_lat) { + struct fc_data *data = td->io_ops_data; + uint64_t nsec; + + nsec = ntime_since_now(&start); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0, 0); + } + + return 0; +} + +static enum fio_q_status queue_io(struct thread_data *td, struct io_u fio_unused *io_u) +{ + return FIO_Q_COMPLETED; +} + +static int init(struct thread_data *td) +{ + struct fc_data *data; + + data = calloc(1, sizeof(*data)); + + if (td_read(td)) + data->stat_ddir = DDIR_READ; + else if (td_write(td)) + data->stat_ddir = DDIR_WRITE; + + td->io_ops_data = data; + return 0; +} + +static void cleanup(struct thread_data *td) +{ + struct fc_data *data = td->io_ops_data; + + free(data); +} + +static int stat_invalidate(struct thread_data *td, struct fio_file *f) +{ + /* do nothing because file not opened */ + return 0; +} + +static struct ioengine_ops ioengine = { + .name = "filestat", + .version = FIO_IOOPS_VERSION, + .init = init, + .cleanup = cleanup, + .queue = queue_io, + .invalidate = stat_invalidate, + .get_file_size = generic_get_file_size, + .open_file = stat_file, + .flags = FIO_SYNCIO | FIO_FAKEIO | + FIO_NOSTATS | FIO_NOFILEHASH, +}; + +static void fio_init fio_filestat_register(void) +{ + register_ioengine(&ioengine); +} + +static void fio_exit fio_filestat_unregister(void) +{ + unregister_ioengine(&ioengine); +} diff --git a/engines/io_uring.c b/engines/io_uring.c index 329f2f07..f1ffc712 100644 --- a/engines/io_uring.c +++ b/engines/io_uring.c @@ -70,6 +70,7 @@ struct ioring_data { struct ioring_options { void *pad; unsigned int hipri; + unsigned int cmdprio_percentage; unsigned int fixedbufs; unsigned int registerfiles; unsigned int sqpoll_thread; @@ -108,6 +109,26 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_IOURING, }, +#ifdef FIO_HAVE_IOPRIO_CLASS + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_INT, + .off1 = offsetof(struct ioring_options, cmdprio_percentage), + .minval = 1, + .maxval = 100, + .help = "Send high priority I/O this percentage of the time", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_IOURING, + }, +#else + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_UNSUPPORTED, + .help = "Your platform does not support I/O priority classes", + }, +#endif { .name = "fixedbufs", .lname = "Fixed (pre-mapped) IO buffers", @@ -313,11 +334,23 @@ static int fio_ioring_getevents(struct thread_data *td, unsigned int min, return r < 0 ? r : events; } +static void fio_ioring_prio_prep(struct thread_data *td, struct io_u *io_u) +{ + struct ioring_options *o = td->eo; + struct ioring_data *ld = td->io_ops_data; + if (rand_between(&td->prio_state, 0, 99) < o->cmdprio_percentage) { + ld->sqes[io_u->index].ioprio = IOPRIO_CLASS_RT << IOPRIO_CLASS_SHIFT; + io_u->flags |= IO_U_F_PRIORITY; + } + return; +} + static enum fio_q_status fio_ioring_queue(struct thread_data *td, struct io_u *io_u) { struct ioring_data *ld = td->io_ops_data; struct io_sq_ring *ring = &ld->sq_ring; + struct ioring_options *o = td->eo; unsigned tail, next_tail; fio_ro_check(td, io_u); @@ -343,6 +376,8 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td, /* ensure sqe stores are ordered with tail update */ write_barrier(); + if (o->cmdprio_percentage) + fio_ioring_prio_prep(td, io_u); ring->array[tail & ld->sq_ring_mask] = io_u->index; *ring->tail = next_tail; write_barrier(); @@ -618,6 +653,7 @@ static int fio_ioring_init(struct thread_data *td) { struct ioring_options *o = td->eo; struct ioring_data *ld; + struct thread_options *to = &td->o; /* sqthread submission requires registered files */ if (o->sqpoll_thread) @@ -640,6 +676,17 @@ static int fio_ioring_init(struct thread_data *td) ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec)); td->io_ops_data = ld; + + /* + * Check for option conflicts + */ + if ((fio_option_is_set(to, ioprio) || fio_option_is_set(to, ioprio_class)) && + o->cmdprio_percentage != 0) { + log_err("%s: cmdprio_percentage option and mutually exclusive " + "prio or prioclass option is set, exiting\n", to->name); + td_verror(td, EINVAL, "fio_io_uring_init"); + return 1; + } return 0; } diff --git a/engines/libaio.c b/engines/libaio.c index b047b746..299798ae 100644 --- a/engines/libaio.c +++ b/engines/libaio.c @@ -16,7 +16,13 @@ #include "../optgroup.h" #include "../lib/memalign.h" +/* Should be defined in newest aio_abi.h */ +#ifndef IOCB_FLAG_IOPRIO +#define IOCB_FLAG_IOPRIO (1 << 1) +#endif + static int fio_libaio_commit(struct thread_data *td); +static int fio_libaio_init(struct thread_data *td); struct libaio_data { io_context_t aio_ctx; @@ -44,6 +50,7 @@ struct libaio_data { struct libaio_options { void *pad; unsigned int userspace_reap; + unsigned int cmdprio_percentage; }; static struct fio_option options[] = { @@ -56,6 +63,26 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_LIBAIO, }, +#ifdef FIO_HAVE_IOPRIO_CLASS + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_INT, + .off1 = offsetof(struct libaio_options, cmdprio_percentage), + .minval = 1, + .maxval = 100, + .help = "Send high priority I/O this percentage of the time", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_LIBAIO, + }, +#else + { + .name = "cmdprio_percentage", + .lname = "high priority percentage", + .type = FIO_OPT_UNSUPPORTED, + .help = "Your platform does not support I/O priority classes", + }, +#endif { .name = NULL, }, @@ -85,6 +112,17 @@ static int fio_libaio_prep(struct thread_data fio_unused *td, struct io_u *io_u) return 0; } +static void fio_libaio_prio_prep(struct thread_data *td, struct io_u *io_u) +{ + struct libaio_options *o = td->eo; + if (rand_between(&td->prio_state, 0, 99) < o->cmdprio_percentage) { + io_u->iocb.aio_reqprio = IOPRIO_CLASS_RT << IOPRIO_CLASS_SHIFT; + io_u->iocb.u.c.flags |= IOCB_FLAG_IOPRIO; + io_u->flags |= IO_U_F_PRIORITY; + } + return; +} + static struct io_u *fio_libaio_event(struct thread_data *td, int event) { struct libaio_data *ld = td->io_ops_data; @@ -188,6 +226,7 @@ static enum fio_q_status fio_libaio_queue(struct thread_data *td, struct io_u *io_u) { struct libaio_data *ld = td->io_ops_data; + struct libaio_options *o = td->eo; fio_ro_check(td, io_u); @@ -218,6 +257,9 @@ static enum fio_q_status fio_libaio_queue(struct thread_data *td, return FIO_Q_COMPLETED; } + if (o->cmdprio_percentage) + fio_libaio_prio_prep(td, io_u); + ld->iocbs[ld->head] = &io_u->iocb; ld->io_us[ld->head] = io_u; ring_inc(ld, &ld->head, 1); @@ -358,6 +400,8 @@ static int fio_libaio_post_init(struct thread_data *td) static int fio_libaio_init(struct thread_data *td) { struct libaio_data *ld; + struct thread_options *to = &td->o; + struct libaio_options *o = td->eo; ld = calloc(1, sizeof(*ld)); @@ -368,6 +412,16 @@ static int fio_libaio_init(struct thread_data *td) ld->io_us = calloc(ld->entries, sizeof(struct io_u *)); td->io_ops_data = ld; + /* + * Check for option conflicts + */ + if ((fio_option_is_set(to, ioprio) || fio_option_is_set(to, ioprio_class)) && + o->cmdprio_percentage != 0) { + log_err("%s: cmdprio_percentage option and mutually exclusive " + "prio or prioclass option is set, exiting\n", to->name); + td_verror(td, EINVAL, "fio_libaio_init"); + return 1; + } return 0; } diff --git a/eta.c b/eta.c index 9950ef30..13f61ba4 100644 --- a/eta.c +++ b/eta.c @@ -509,9 +509,9 @@ bool calc_thread_status(struct jobs_eta *je, int force) calc_rate(unified_rw_rep, rate_time, io_bytes, rate_io_bytes, je->rate); memcpy(&rate_prev_time, &now, sizeof(now)); - add_agg_sample(sample_val(je->rate[DDIR_READ]), DDIR_READ, 0); - add_agg_sample(sample_val(je->rate[DDIR_WRITE]), DDIR_WRITE, 0); - add_agg_sample(sample_val(je->rate[DDIR_TRIM]), DDIR_TRIM, 0); + add_agg_sample(sample_val(je->rate[DDIR_READ]), DDIR_READ, 0, 0); + add_agg_sample(sample_val(je->rate[DDIR_WRITE]), DDIR_WRITE, 0, 0); + add_agg_sample(sample_val(je->rate[DDIR_TRIM]), DDIR_TRIM, 0, 0); } disp_time = mtime_since(&disp_prev_time, &now); diff --git a/examples/filestat-ioengine.fio b/examples/filestat-ioengine.fio new file mode 100644 index 00000000..932fced8 --- /dev/null +++ b/examples/filestat-ioengine.fio @@ -0,0 +1,19 @@ +# Example filestat job + +# 'filestat' engine only do 'stat(filename)', file will not be open(). +# 'filesize' must be set, then files will be created at setup stage. + +[global] +ioengine=filestat +numjobs=1 +filesize=4k +nrfiles=200 +thread + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] + diff --git a/fio.1 b/fio.1 index cf5dd853..05896e61 100644 --- a/fio.1 +++ b/fio.1 @@ -1760,6 +1760,11 @@ Simply create the files and do no I/O to them. You still need to set \fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be done other than creating the file. .TP +.B filestat +Simply do stat() and do no I/O to the file. You need to set 'filesize' +and 'nrfiles', so that files will be created. +This engine is to measure file lookup and meta data access. +.TP .B libpmem Read and write using mmap I/O to a file on a filesystem mounted with DAX on a persistent memory device through the PMDK @@ -1790,12 +1795,12 @@ In addition, there are some parameters which are only valid when a specific with the caveat that when used on the command line, they must come after the \fBioengine\fR that defines them is selected. .TP -.BI (io_uring)hipri -If this option is set, fio will attempt to use polled IO completions. Normal IO -completions generate interrupts to signal the completion of IO, polled -completions do not. Hence they are require active reaping by the application. -The benefits are more efficient IO for high IOPS scenarios, and lower latencies -for low queue depth IO. +.BI (io_uring, libaio)cmdprio_percentage \fR=\fPint +Set the percentage of I/O that will be issued with higher priority by setting +the priority bit. Non-read I/O is likely unaffected by ``cmdprio_percentage``. +This option cannot be used with the `prio` or `prioclass` options. For this +option to set the priority bit properly, NCQ priority must be supported and +enabled and `direct=1' option must be used. .TP .BI (io_uring)fixedbufs If fio is asked to do direct IO, then Linux will map pages for each IO call, and @@ -1803,6 +1808,13 @@ release them when IO is done. If this option is set, the pages are pre-mapped before IO is started. This eliminates the need to map and release for each IO. This is more efficient, and reduces the IO latency as well. .TP +.BI (io_uring)hipri +If this option is set, fio will attempt to use polled IO completions. Normal IO +completions generate interrupts to signal the completion of IO, polled +completions do not. Hence they are require active reaping by the application. +The benefits are more efficient IO for high IOPS scenarios, and lower latencies +for low queue depth IO. +.TP .BI (io_uring)registerfiles With this option, fio registers the set of files being used with the kernel. This avoids the overhead of managing file counts in the kernel, making the @@ -2381,10 +2393,14 @@ priority class. Set the I/O priority value of this job. Linux limits us to a positive value between 0 and 7, with 0 being the highest. See man \fBionice\fR\|(1). Refer to an appropriate manpage for other operating -systems since meaning of priority may differ. +systems since meaning of priority may differ. For per-command priority +setting, see I/O engine specific `cmdprio_percentage` and `hipri_percentage` +options. .TP .BI prioclass \fR=\fPint -Set the I/O priority class. See man \fBionice\fR\|(1). +Set the I/O priority class. See man \fBionice\fR\|(1). For per-command +priority setting, see I/O engine specific `cmdprio_percentage` and `hipri_percent` +options. .TP .BI cpus_allowed \fR=\fPstr Controls the same options as \fBcpumask\fR, but accepts a textual diff --git a/fio.h b/fio.h index 6a5ead4d..2a9eef45 100644 --- a/fio.h +++ b/fio.h @@ -139,6 +139,7 @@ enum { FIO_RAND_ZONE_OFF, FIO_RAND_POISSON2_OFF, FIO_RAND_POISSON3_OFF, + FIO_RAND_PRIO_CMDS, FIO_RAND_NR_OFFS, }; @@ -258,6 +259,7 @@ struct thread_data { struct frand_state buf_state_prev; struct frand_state dedupe_state; struct frand_state zone_state; + struct frand_state prio_state; struct zone_split_index **zone_state_index; @@ -460,7 +462,7 @@ struct thread_data { CUdevice cu_dev; CUcontext cu_ctx; CUdeviceptr dev_mem_ptr; -#endif +#endif }; diff --git a/init.c b/init.c index 2f64726c..dca44bca 100644 --- a/init.c +++ b/init.c @@ -1042,6 +1042,7 @@ static void td_fill_rand_seeds_internal(struct thread_data *td, bool use64) init_rand_seed(&td->poisson_state[2], td->rand_seeds[FIO_RAND_POISSON3_OFF], 0); init_rand_seed(&td->dedupe_state, td->rand_seeds[FIO_DEDUPE_OFF], false); init_rand_seed(&td->zone_state, td->rand_seeds[FIO_RAND_ZONE_OFF], false); + init_rand_seed(&td->prio_state, td->rand_seeds[FIO_RAND_PRIO_CMDS], false); if (!td_random(td)) return; @@ -1518,6 +1519,8 @@ static int add_job(struct thread_data *td, const char *jobname, int job_add_num, td->ts.lat_stat[i].min_val = ULONG_MAX; td->ts.bw_stat[i].min_val = ULONG_MAX; td->ts.iops_stat[i].min_val = ULONG_MAX; + td->ts.clat_high_prio_stat[i].min_val = ULONG_MAX; + td->ts.clat_prio_stat[i].min_val = ULONG_MAX; } td->ts.sync_stat.min_val = ULONG_MAX; td->ddir_seq_nr = o->ddir_seq_nr; diff --git a/io_u.c b/io_u.c index 03f5c21f..bcb893c5 100644 --- a/io_u.c +++ b/io_u.c @@ -1541,7 +1541,7 @@ again: assert(io_u->flags & IO_U_F_FREE); io_u_clear(td, io_u, IO_U_F_FREE | IO_U_F_NO_FILE_PUT | IO_U_F_TRIMMED | IO_U_F_BARRIER | - IO_U_F_VER_LIST); + IO_U_F_VER_LIST | IO_U_F_PRIORITY); io_u->error = 0; io_u->acct_ddir = -1; @@ -1830,7 +1830,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, unsigned long long tnsec; tnsec = ntime_since(&io_u->start_time, &icd->time); - add_lat_sample(td, idx, tnsec, bytes, io_u->offset); + add_lat_sample(td, idx, tnsec, bytes, io_u->offset, io_u_is_prio(io_u)); if (td->flags & TD_F_PROFILE_OPS) { struct prof_io_ops *ops = &td->prof_io_ops; @@ -1849,7 +1849,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, if (ddir_rw(idx)) { if (!td->o.disable_clat) { - add_clat_sample(td, idx, llnsec, bytes, io_u->offset); + add_clat_sample(td, idx, llnsec, bytes, io_u->offset, io_u_is_prio(io_u)); io_u_mark_latency(td, llnsec); } @@ -2091,7 +2091,7 @@ void io_u_queued(struct thread_data *td, struct io_u *io_u) td = td->parent; add_slat_sample(td, io_u->ddir, slat_time, io_u->xfer_buflen, - io_u->offset); + io_u->offset, io_u_is_prio(io_u)); } } diff --git a/io_u.h b/io_u.h index e75993bd..0f63cdd0 100644 --- a/io_u.h +++ b/io_u.h @@ -24,6 +24,7 @@ enum { IO_U_F_TRIMMED = 1 << 5, IO_U_F_BARRIER = 1 << 6, IO_U_F_VER_LIST = 1 << 7, + IO_U_F_PRIORITY = 1 << 8, }; /* @@ -193,5 +194,7 @@ static inline enum fio_ddir acct_ddir(struct io_u *io_u) td_flags_clear((td), &(io_u->flags), (val)) #define io_u_set(td, io_u, val) \ td_flags_set((td), &(io_u)->flags, (val)) +#define io_u_is_prio(io_u) \ + (io_u->flags & (unsigned int) IO_U_F_PRIORITY) != 0 #endif diff --git a/ioengines.c b/ioengines.c index b9200ba9..2c7a0df9 100644 --- a/ioengines.c +++ b/ioengines.c @@ -318,6 +318,7 @@ enum fio_q_status td_io_queue(struct thread_data *td, struct io_u *io_u) sizeof(io_u->issue_time)); } + if (ddir_rw(ddir)) { if (!(io_u->flags & IO_U_F_VER_LIST)) { td->io_issues[ddir]++; diff --git a/iolog.c b/iolog.c index b72dcf97..917a446c 100644 --- a/iolog.c +++ b/iolog.c @@ -896,18 +896,18 @@ void flush_samples(FILE *f, void *samples, uint64_t sample_size) s = __get_sample(samples, log_offset, i); if (!log_offset) { - fprintf(f, "%lu, %" PRId64 ", %u, %llu\n", + fprintf(f, "%lu, %" PRId64 ", %u, %llu, %u\n", (unsigned long) s->time, s->data.val, - io_sample_ddir(s), (unsigned long long) s->bs); + io_sample_ddir(s), (unsigned long long) s->bs, s->priority_bit); } else { struct io_sample_offset *so = (void *) s; - fprintf(f, "%lu, %" PRId64 ", %u, %llu, %llu\n", + fprintf(f, "%lu, %" PRId64 ", %u, %llu, %llu, %u\n", (unsigned long) s->time, s->data.val, io_sample_ddir(s), (unsigned long long) s->bs, - (unsigned long long) so->offset); + (unsigned long long) so->offset, s->priority_bit); } } } diff --git a/iolog.h b/iolog.h index 17be908f..981081f9 100644 --- a/iolog.h +++ b/iolog.h @@ -42,6 +42,7 @@ struct io_sample { uint64_t time; union io_sample_data data; uint32_t __ddir; + uint8_t priority_bit; uint64_t bs; }; diff --git a/server.c b/server.c index 1a070e56..a5af5e74 100644 --- a/server.c +++ b/server.c @@ -1262,7 +1262,7 @@ static int handle_connection(struct sk_out *sk_out) _exit(ret); } -/* get the address on this host bound by the input socket, +/* get the address on this host bound by the input socket, * whether it is ipv6 or ipv4 */ static int get_my_addr_str(int sk) @@ -1574,6 +1574,15 @@ void fio_server_send_ts(struct thread_stat *ts, struct group_run_stats *rs) p.ts.cachehit = cpu_to_le64(ts->cachehit); p.ts.cachemiss = cpu_to_le64(ts->cachemiss); + for (i = 0; i < DDIR_RWDIR_CNT; i++) { + for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { + p.ts.io_u_plat_high_prio[i][j] = cpu_to_le64(ts->io_u_plat_high_prio[i][j]); + p.ts.io_u_plat_prio[i][j] = cpu_to_le64(ts->io_u_plat_prio[i][j]); + } + convert_io_stat(&p.ts.clat_high_prio_stat[i], &ts->clat_high_prio_stat[i]); + convert_io_stat(&p.ts.clat_prio_stat[i], &ts->clat_prio_stat[i]); + } + convert_gs(&p.rs, rs); dprint(FD_NET, "ts->ss_state = %d\n", ts->ss_state); @@ -1998,7 +2007,7 @@ int fio_send_iolog(struct thread_data *td, struct io_log *log, const char *name) s->time = cpu_to_le64(s->time); s->data.val = cpu_to_le64(s->data.val); - s->__ddir = cpu_to_le32(s->__ddir); + s->__ddir = __cpu_to_le32(s->__ddir); s->bs = cpu_to_le64(s->bs); if (log->log_offset) { diff --git a/server.h b/server.h index de1d7f9b..6ac75366 100644 --- a/server.h +++ b/server.h @@ -48,7 +48,7 @@ struct fio_net_cmd_reply { }; enum { - FIO_SERVER_VER = 80, + FIO_SERVER_VER = 81, FIO_SERVER_MAX_FRAGMENT_PDU = 1024, FIO_SERVER_MAX_CMD_MB = 2048, diff --git a/stat.c b/stat.c index 4d3b728c..9d93dcd1 100644 --- a/stat.c +++ b/stat.c @@ -482,9 +482,12 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, display_lat("clat", min, max, mean, dev, out); if (calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) display_lat(" lat", min, max, mean, dev, out); + if (calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) + display_lat("prio_clat", min, max, mean, dev, out); if (ts->clat_percentiles || ts->lat_percentiles) { const char *name = ts->clat_percentiles ? "clat" : " lat"; + char prio_name[32]; uint64_t samples; if (ts->clat_percentiles) @@ -496,6 +499,27 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, samples, ts->percentile_list, ts->percentile_precision, name, out); + + /* Only print this if some high and low priority stats were collected */ + if (ts->clat_high_prio_stat[ddir].samples > 0 && + ts->clat_prio_stat[ddir].samples > 0) + { + sprintf(prio_name, "high prio (%.2f%%) %s", + 100. * (double) ts->clat_high_prio_stat[ddir].samples / (double) samples, + name); + show_clat_percentiles(ts->io_u_plat_high_prio[ddir], + ts->clat_high_prio_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, prio_name, out); + + sprintf(prio_name, "low prio (%.2f%%) %s", + 100. * (double) ts->clat_prio_stat[ddir].samples / (double) samples, + name); + show_clat_percentiles(ts->io_u_plat_prio[ddir], + ts->clat_prio_stat[ddir].samples, + ts->percentile_list, + ts->percentile_precision, prio_name, out); + } } if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { double p_of_agg = 100.0, fkb_base = (double)rs->kb_base; @@ -1009,7 +1033,7 @@ static void show_thread_status_normal(struct thread_stat *ts, if (!ddir_rw_sum(ts->io_bytes) && !ddir_rw_sum(ts->total_io_u)) return; - + memset(time_buf, 0, sizeof(time_buf)); time(&time_p); @@ -1335,6 +1359,112 @@ static void add_ddir_status_json(struct thread_stat *ts, } } + + /* Only print PRIO latencies if some high priority samples were gathered */ + if (ts->clat_high_prio_stat[ddir].samples > 0) { + /* START OF HIGH PRIO CLAT */ + if (!calc_lat(&ts->clat_high_prio_stat[ddir], &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } + tmp_object = json_create_object(); + json_object_add_value_object(dir_object, "clat_prio", + tmp_object); + json_object_add_value_int(tmp_object, "samples", + ts->clat_high_prio_stat[ddir].samples); + json_object_add_value_int(tmp_object, "min", min); + json_object_add_value_int(tmp_object, "max", max); + json_object_add_value_float(tmp_object, "mean", mean); + json_object_add_value_float(tmp_object, "stddev", dev); + + if (ts->clat_percentiles) { + len = calc_clat_percentiles(ts->io_u_plat_high_prio[ddir], + ts->clat_high_prio_stat[ddir].samples, + ts->percentile_list, &ovals, &maxv, + &minv); + } else + len = 0; + + percentile_object = json_create_object(); + json_object_add_value_object(tmp_object, "percentile", percentile_object); + for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) { + if (i >= len) { + json_object_add_value_int(percentile_object, "0.00", 0); + continue; + } + snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f); + json_object_add_value_int(percentile_object, (const char *)buf, ovals[i]); + } + + if (output_format & FIO_OUTPUT_JSON_PLUS) { + clat_bins_object = json_create_object(); + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { + snprintf(buf, sizeof(buf), "%d", i); + json_object_add_value_int(clat_bins_object, (const char *)buf, + ts->io_u_plat_high_prio[ddir][i]); + } + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_BITS", + FIO_IO_U_PLAT_BITS); + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_VAL", + FIO_IO_U_PLAT_VAL); + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_NR", + FIO_IO_U_PLAT_NR); + } + /* END OF HIGH PRIO CLAT */ + + /* START OF PRIO CLAT */ + if (!calc_lat(&ts->clat_prio_stat[ddir], &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } + tmp_object = json_create_object(); + json_object_add_value_object(dir_object, "clat_low_prio", + tmp_object); + json_object_add_value_int(tmp_object, "samples", + ts->clat_prio_stat[ddir].samples); + json_object_add_value_int(tmp_object, "min", min); + json_object_add_value_int(tmp_object, "max", max); + json_object_add_value_float(tmp_object, "mean", mean); + json_object_add_value_float(tmp_object, "stddev", dev); + + if (ts->clat_percentiles) { + len = calc_clat_percentiles(ts->io_u_plat_prio[ddir], + ts->clat_prio_stat[ddir].samples, + ts->percentile_list, &ovals, &maxv, + &minv); + } else + len = 0; + + percentile_object = json_create_object(); + json_object_add_value_object(tmp_object, "percentile", percentile_object); + for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) { + if (i >= len) { + json_object_add_value_int(percentile_object, "0.00", 0); + continue; + } + snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f); + json_object_add_value_int(percentile_object, (const char *)buf, ovals[i]); + } + + if (output_format & FIO_OUTPUT_JSON_PLUS) { + clat_bins_object = json_create_object(); + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { + snprintf(buf, sizeof(buf), "%d", i); + json_object_add_value_int(clat_bins_object, (const char *)buf, + ts->io_u_plat_prio[ddir][i]); + } + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_BITS", + FIO_IO_U_PLAT_BITS); + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_VAL", + FIO_IO_U_PLAT_VAL); + json_object_add_value_int(clat_bins_object, "FIO_IO_U_PLAT_NR", + FIO_IO_U_PLAT_NR); + } + /* END OF PRIO CLAT */ + } + if (!ddir_rw(ddir)) return; @@ -1856,6 +1986,8 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, for (l = 0; l < DDIR_RWDIR_CNT; l++) { if (!dst->unified_rw_rep) { sum_stat(&dst->clat_stat[l], &src->clat_stat[l], first, false); + sum_stat(&dst->clat_high_prio_stat[l], &src->clat_high_prio_stat[l], first, false); + sum_stat(&dst->clat_prio_stat[l], &src->clat_prio_stat[l], first, false); sum_stat(&dst->slat_stat[l], &src->slat_stat[l], first, false); sum_stat(&dst->lat_stat[l], &src->lat_stat[l], first, false); sum_stat(&dst->bw_stat[l], &src->bw_stat[l], first, true); @@ -1867,6 +1999,8 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, dst->runtime[l] = src->runtime[l]; } else { sum_stat(&dst->clat_stat[0], &src->clat_stat[l], first, false); + sum_stat(&dst->clat_high_prio_stat[l], &src->clat_high_prio_stat[l], first, false); + sum_stat(&dst->clat_prio_stat[l], &src->clat_prio_stat[l], first, false); sum_stat(&dst->slat_stat[0], &src->slat_stat[l], first, false); sum_stat(&dst->lat_stat[0], &src->lat_stat[l], first, false); sum_stat(&dst->bw_stat[0], &src->bw_stat[l], first, true); @@ -1926,10 +2060,16 @@ void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, int m; for (m = 0; m < FIO_IO_U_PLAT_NR; m++) { - if (!dst->unified_rw_rep) + if (!dst->unified_rw_rep) { dst->io_u_plat[k][m] += src->io_u_plat[k][m]; - else + dst->io_u_plat_high_prio[k][m] += src->io_u_plat_high_prio[k][m]; + dst->io_u_plat_prio[k][m] += src->io_u_plat_prio[k][m]; + } else { dst->io_u_plat[0][m] += src->io_u_plat[k][m]; + dst->io_u_plat_high_prio[0][m] += src->io_u_plat_high_prio[k][m]; + dst->io_u_plat_prio[0][m] += src->io_u_plat_prio[k][m]; + } + } } @@ -1962,6 +2102,8 @@ void init_thread_stat(struct thread_stat *ts) ts->slat_stat[j].min_val = -1UL; ts->bw_stat[j].min_val = -1UL; ts->iops_stat[j].min_val = -1UL; + ts->clat_high_prio_stat[j].min_val = -1UL; + ts->clat_prio_stat[j].min_val = -1UL; } ts->sync_stat.min_val = -1UL; ts->groupid = -1; @@ -2542,7 +2684,7 @@ static struct io_logs *get_cur_log(struct io_log *iolog) static void __add_log_sample(struct io_log *iolog, union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, - unsigned long t, uint64_t offset) + unsigned long t, uint64_t offset, uint8_t priority_bit) { struct io_logs *cur_log; @@ -2561,6 +2703,7 @@ static void __add_log_sample(struct io_log *iolog, union io_sample_data data, s->time = t + (iolog->td ? iolog->td->unix_epoch : 0); io_sample_set_ddir(iolog, s, ddir); s->bs = bs; + s->priority_bit = priority_bit; if (iolog->log_offset) { struct io_sample_offset *so = (void *) s; @@ -2588,6 +2731,8 @@ void reset_io_stats(struct thread_data *td) int i, j; for (i = 0; i < DDIR_RWDIR_CNT; i++) { + reset_io_stat(&ts->clat_high_prio_stat[i]); + reset_io_stat(&ts->clat_prio_stat[i]); reset_io_stat(&ts->clat_stat[i]); reset_io_stat(&ts->slat_stat[i]); reset_io_stat(&ts->lat_stat[i]); @@ -2602,6 +2747,8 @@ void reset_io_stats(struct thread_data *td) for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { ts->io_u_plat[i][j] = 0; + ts->io_u_plat_high_prio[i][j] = 0; + ts->io_u_plat_prio[i][j] = 0; if (!i) ts->io_u_sync_plat[j] = 0; } @@ -2629,7 +2776,7 @@ void reset_io_stats(struct thread_data *td) } static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, - unsigned long elapsed, bool log_max) + unsigned long elapsed, bool log_max, uint8_t priority_bit) { /* * Note an entry in the log. Use the mean from the logged samples, @@ -2644,26 +2791,26 @@ static void __add_stat_to_log(struct io_log *iolog, enum fio_ddir ddir, else data.val = iolog->avg_window[ddir].mean.u.f + 0.50; - __add_log_sample(iolog, data, ddir, 0, elapsed, 0); + __add_log_sample(iolog, data, ddir, 0, elapsed, 0, priority_bit); } reset_io_stat(&iolog->avg_window[ddir]); } static void _add_stat_to_log(struct io_log *iolog, unsigned long elapsed, - bool log_max) + bool log_max, uint8_t priority_bit) { int ddir; for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) - __add_stat_to_log(iolog, ddir, elapsed, log_max); + __add_stat_to_log(iolog, ddir, elapsed, log_max, priority_bit); } static unsigned long add_log_sample(struct thread_data *td, struct io_log *iolog, union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { unsigned long elapsed, this_window; @@ -2676,7 +2823,7 @@ static unsigned long add_log_sample(struct thread_data *td, * If no time averaging, just add the log sample. */ if (!iolog->avg_msec) { - __add_log_sample(iolog, data, ddir, bs, elapsed, offset); + __add_log_sample(iolog, data, ddir, bs, elapsed, offset, priority_bit); return 0; } @@ -2700,7 +2847,7 @@ static unsigned long add_log_sample(struct thread_data *td, return diff; } - __add_stat_to_log(iolog, ddir, elapsed, td->o.log_max != 0); + _add_stat_to_log(iolog, elapsed, td->o.log_max != 0, priority_bit); iolog->avg_last[ddir] = elapsed - (this_window - iolog->avg_msec); return iolog->avg_msec; @@ -2713,18 +2860,19 @@ void finalize_logs(struct thread_data *td, bool unit_logs) elapsed = mtime_since_now(&td->epoch); if (td->clat_log && unit_logs) - _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->clat_log, elapsed, td->o.log_max != 0, 0); if (td->slat_log && unit_logs) - _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->slat_log, elapsed, td->o.log_max != 0, 0); if (td->lat_log && unit_logs) - _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->lat_log, elapsed, td->o.log_max != 0, 0); if (td->bw_log && (unit_logs == per_unit_log(td->bw_log))) - _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->bw_log, elapsed, td->o.log_max != 0, 0); if (td->iops_log && (unit_logs == per_unit_log(td->iops_log))) - _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0); + _add_stat_to_log(td->iops_log, elapsed, td->o.log_max != 0, 0); } -void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long long bs) +void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long long bs, + uint8_t priority_bit) { struct io_log *iolog; @@ -2732,7 +2880,7 @@ void add_agg_sample(union io_sample_data data, enum fio_ddir ddir, unsigned long return; iolog = agg_io_log[ddir]; - __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0); + __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0, priority_bit); } void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec) @@ -2745,17 +2893,23 @@ void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec) } static void add_clat_percentile_sample(struct thread_stat *ts, - unsigned long long nsec, enum fio_ddir ddir) + unsigned long long nsec, enum fio_ddir ddir, uint8_t priority_bit) { unsigned int idx = plat_val_to_idx(nsec); assert(idx < FIO_IO_U_PLAT_NR); ts->io_u_plat[ddir][idx]++; + + if (!priority_bit) { + ts->io_u_plat_prio[ddir][idx]++; + } else { + ts->io_u_plat_high_prio[ddir][idx]++; + } } void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); unsigned long elapsed, this_window; @@ -2767,12 +2921,19 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, add_stat_sample(&ts->clat_stat[ddir], nsec); + if (priority_bit) { + add_stat_sample(&ts->clat_high_prio_stat[ddir], nsec); + } else { + add_stat_sample(&ts->clat_prio_stat[ddir], nsec); + } + if (td->clat_log) add_log_sample(td, td->clat_log, sample_val(nsec), ddir, bs, - offset); + offset, priority_bit); - if (ts->clat_percentiles) - add_clat_percentile_sample(ts, nsec, ddir); + if (ts->clat_percentiles) { + add_clat_percentile_sample(ts, nsec, ddir, priority_bit); + } if (iolog && iolog->hist_msec) { struct io_hist *hw = &iolog->hist_window[ddir]; @@ -2782,7 +2943,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, if (!hw->hist_last) hw->hist_last = elapsed; this_window = elapsed - hw->hist_last; - + if (this_window >= iolog->hist_msec) { uint64_t *io_u_plat; struct io_u_plat_entry *dst; @@ -2800,7 +2961,7 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, FIO_IO_U_PLAT_NR * sizeof(uint64_t)); flist_add(&dst->list, &hw->list); __add_log_sample(iolog, sample_plat(dst), ddir, bs, - elapsed, offset); + elapsed, offset, priority_bit); /* * Update the last time we recorded as being now, minus @@ -2817,7 +2978,8 @@ void add_clat_sample(struct thread_data *td, enum fio_ddir ddir, } void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, - unsigned long usec, unsigned long long bs, uint64_t offset) + unsigned long usec, unsigned long long bs, uint64_t offset, + uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -2831,7 +2993,8 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, add_stat_sample(&ts->slat_stat[ddir], usec); if (td->slat_log) - add_log_sample(td, td->slat_log, sample_val(usec), ddir, bs, offset); + add_log_sample(td, td->slat_log, sample_val(usec), ddir, bs, offset, + priority_bit); if (needs_lock) __td_io_u_unlock(td); @@ -2839,7 +3002,7 @@ void add_slat_sample(struct thread_data *td, enum fio_ddir ddir, void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, unsigned long long nsec, unsigned long long bs, - uint64_t offset) + uint64_t offset, uint8_t priority_bit) { const bool needs_lock = td_async_processing(td); struct thread_stat *ts = &td->ts; @@ -2854,10 +3017,10 @@ void add_lat_sample(struct thread_data *td, enum fio_ddir ddir, if (td->lat_log) add_log_sample(td, td->lat_log, sample_val(nsec), ddir, bs, - offset); + offset, priority_bit); if (ts->lat_percentiles) - add_clat_percentile_sample(ts, nsec, ddir); + add_clat_percentile_sample(ts, nsec, ddir, priority_bit); if (needs_lock) __td_io_u_unlock(td); @@ -2882,7 +3045,7 @@ void add_bw_sample(struct thread_data *td, struct io_u *io_u, if (td->bw_log) add_log_sample(td, td->bw_log, sample_val(rate), io_u->ddir, - bytes, io_u->offset); + bytes, io_u->offset, io_u_is_prio(io_u)); td->stat_io_bytes[io_u->ddir] = td->this_io_bytes[io_u->ddir]; @@ -2936,7 +3099,7 @@ static int __add_samples(struct thread_data *td, struct timespec *parent_tv, if (td->o.min_bs[ddir] == td->o.max_bs[ddir]) bs = td->o.min_bs[ddir]; - next = add_log_sample(td, log, sample_val(rate), ddir, bs, 0); + next = add_log_sample(td, log, sample_val(rate), ddir, bs, 0, 0); next_log = min(next_log, next); } @@ -2976,7 +3139,7 @@ void add_iops_sample(struct thread_data *td, struct io_u *io_u, if (td->iops_log) add_log_sample(td, td->iops_log, sample_val(1), io_u->ddir, - bytes, io_u->offset); + bytes, io_u->offset, io_u_is_prio(io_u)); td->stat_io_blocks[io_u->ddir] = td->this_io_blocks[io_u->ddir]; diff --git a/stat.h b/stat.h index 2ce91ff0..9320c6bd 100644 --- a/stat.h +++ b/stat.h @@ -239,6 +239,11 @@ struct thread_stat { fio_fp64_t ss_deviation; fio_fp64_t ss_criterion; + uint64_t io_u_plat_high_prio[DDIR_RWDIR_CNT][FIO_IO_U_PLAT_NR] __attribute__((aligned(8)));; + uint64_t io_u_plat_prio[DDIR_RWDIR_CNT][FIO_IO_U_PLAT_NR]; + struct io_stat clat_high_prio_stat[DDIR_RWDIR_CNT] __attribute__((aligned(8))); + struct io_stat clat_prio_stat[DDIR_RWDIR_CNT]; + union { uint64_t *ss_iops_data; uint64_t pad4; @@ -323,12 +328,13 @@ extern void update_rusage_stat(struct thread_data *); extern void clear_rusage_stat(struct thread_data *); extern void add_lat_sample(struct thread_data *, enum fio_ddir, unsigned long long, - unsigned long long, uint64_t); + unsigned long long, uint64_t, uint8_t); extern void add_clat_sample(struct thread_data *, enum fio_ddir, unsigned long long, - unsigned long long, uint64_t); + unsigned long long, uint64_t, uint8_t); extern void add_slat_sample(struct thread_data *, enum fio_ddir, unsigned long, - unsigned long long, uint64_t); -extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long); + unsigned long long, uint64_t, uint8_t); +extern void add_agg_sample(union io_sample_data, enum fio_ddir, unsigned long long bs, + uint8_t priority_bit); extern void add_iops_sample(struct thread_data *, struct io_u *, unsigned int); extern void add_bw_sample(struct thread_data *, struct io_u *,
