The following changes since commit 9e4438fecd1d92b4d5221f35d5e73546f52c6ebf:
stat: don't trust per_unit_log() if log is NULL (2016-08-22 13:23:29 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 04d6530f6ecd50520e99732b0b6bb90f71ff131a:
file: fix numjobs > 1 and implied jobname as filename (2016-08-25 21:00:55 -0600)
----------------------------------------------------------------
Jens Axboe (2):
Merge branch 'epoch-histograms' of https://github.com/cronburg/fio
file: fix numjobs > 1 and implied jobname as filename
Karl Cronburg (1):
Give job file to fiologparser_hist.py so that it can auto detect log_hist_msec. This commit also adds handling of unix epoch timestamps by fiologparser_hist.py.
file.h | 1 +
filesetup.c | 43 +++++++++++++++++++++++++++++++----------
init.c | 20 -------------------
tools/hist/fiologparser_hist.py | 41 +++++++++++++++++++++++++++++++++++++--
4 files changed, 73 insertions(+), 32 deletions(-)
---
Diff of recent changes:
diff --git a/file.h b/file.h
index f7e5d20..aff3ce9 100644
--- a/file.h
+++ b/file.h
@@ -211,5 +211,6 @@ extern void free_release_files(struct thread_data *);
extern void filesetup_mem_free(void);
extern void fio_file_reset(struct thread_data *, struct fio_file *);
extern int fio_files_done(struct thread_data *);
+extern bool exists_and_not_regfile(const char *);
#endif
diff --git a/filesetup.c b/filesetup.c
index 5db44c2..fc9f306 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -1242,31 +1242,33 @@ static void get_file_type(struct fio_file *f)
}
}
-static int __is_already_allocated(const char *fname)
+static bool __is_already_allocated(const char *fname)
{
struct flist_head *entry;
- char *filename;
if (flist_empty(&filename_list))
- return 0;
+ return false;
flist_for_each(entry, &filename_list) {
- filename = flist_entry(entry, struct file_name, list)->filename;
+ struct file_name *fn;
- if (strcmp(filename, fname) == 0)
- return 1;
+ fn = flist_entry(entry, struct file_name, list);
+
+ if (!strcmp(fn->filename, fname))
+ return true;
}
- return 0;
+ return false;
}
-static int is_already_allocated(const char *fname)
+static bool is_already_allocated(const char *fname)
{
- int ret;
+ bool ret;
fio_file_hash_lock();
ret = __is_already_allocated(fname);
fio_file_hash_unlock();
+
return ret;
}
@@ -1327,6 +1329,26 @@ static struct fio_file *alloc_new_file(struct thread_data *td)
return f;
}
+bool exists_and_not_regfile(const char *filename)
+{
+ struct stat sb;
+
+ if (lstat(filename, &sb) == -1)
+ return false;
+
+#ifndef WIN32 /* NOT Windows */
+ if (S_ISREG(sb.st_mode))
+ return false;
+#else
+ /* \\.\ is the device namespace in Windows, where every file
+ * is a device node */
+ if (S_ISREG(sb.st_mode) && strncmp(filename, "\\\\.\\", 4) != 0)
+ return false;
+#endif
+
+ return true;
+}
+
int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
{
int cur_files = td->files_index;
@@ -1343,7 +1365,8 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
sprintf(file_name + len, "%s", fname);
/* clean cloned siblings using existing files */
- if (numjob && is_already_allocated(file_name))
+ if (numjob && is_already_allocated(file_name) &&
+ !exists_and_not_regfile(fname))
return 0;
f = alloc_new_file(td);
diff --git a/init.c b/init.c
index 5ff7385..0221ab2 100644
--- a/init.c
+++ b/init.c
@@ -903,26 +903,6 @@ static const char *get_engine_name(const char *str)
return p;
}
-static int exists_and_not_regfile(const char *filename)
-{
- struct stat sb;
-
- if (lstat(filename, &sb) == -1)
- return 0;
-
-#ifndef WIN32 /* NOT Windows */
- if (S_ISREG(sb.st_mode))
- return 0;
-#else
- /* \\.\ is the device namespace in Windows, where every file
- * is a device node */
- if (S_ISREG(sb.st_mode) && strncmp(filename, "\\\\.\\", 4) != 0)
- return 0;
-#endif
-
- return 1;
-}
-
static void init_rand_file_service(struct thread_data *td)
{
unsigned long nranges = td->o.nr_files << FIO_FSERVICE_SHIFT;
diff --git a/tools/hist/fiologparser_hist.py b/tools/hist/fiologparser_hist.py
index 778cc00..93dca01 100755
--- a/tools/hist/fiologparser_hist.py
+++ b/tools/hist/fiologparser_hist.py
@@ -255,6 +255,29 @@ def guess_max_from_bins(ctx, hist_cols):
def main(ctx):
+ if ctx.job_file:
+ try:
+ from configparser import SafeConfigParser
+ except ImportError:
+ from ConfigParser import SafeConfigParser
+
+ cp = SafeConfigParser(allow_no_value=True)
+ with open(ctx.job_file, 'r') as fp:
+ cp.read(fp)
+
+ if ctx.interval is None:
+ # Auto detect --interval value
+ for s in cp.sections():
+ try:
+ hist_msec = cp[s]['log_hist_msec']
+ if hist_msec is not None:
+ ctx.interval = int(hist_msec)
+ except KeyError:
+ pass
+
+ if ctx.interval is None:
+ ctx.interval = 1000
+
# Automatically detect how many columns are in the input files,
# calculate the corresponding 'coarseness' parameter used to generate
# those files, and calculate the appropriate bin latency values:
@@ -291,6 +314,14 @@ def main(ctx):
arr = arr.astype(int)
if arr.size > 0:
+ # Jump immediately to the start of the input, rounding
+ # down to the nearest multiple of the interval (useful when --log_unix_epoch
+ # was used to create these histograms):
+ if start == 0 and arr[0][0] - ctx.max_latency > end:
+ start = arr[0][0] - ctx.max_latency
+ start = start - (start % ctx.interval)
+ end = start + ctx.interval
+
process_interval(ctx, arr, start, end)
# Update arr to throw away samples we no longer need - samples which
@@ -321,9 +352,8 @@ if __name__ == '__main__':
help='number of seconds of data to process at a time')
arg('-i', '--interval',
- default=1000,
type=int,
- help='interval width (ms)')
+ help='interval width (ms), default 1000 ms')
arg('-d', '--divisor',
required=False,
@@ -347,5 +377,12 @@ if __name__ == '__main__':
type=int,
help='FIO_IO_U_PLAT_GROUP_NR as defined in stat.h')
+ arg('--job-file',
+ default=None,
+ type=str,
+ help='Optional argument pointing to the job file used to create the '
+ 'given histogram files. Useful for auto-detecting --log_hist_msec and '
+ '--log_unix_epoch (in fio) values.')
+
main(p.parse_args())
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
