The following changes since commit 477bd5ab362b65fe9c7749735236e6ed0d4346f8:
server: bump version (2015-05-06 15:21:19 -0600)
are available in the git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to 81f3d8e9b0c7e178fc308e6416fade2756898fed:
Fio 2.2.8 (2015-05-07 18:19:25 -0600)
----------------------------------------------------------------
Ben England (3):
allow --client to work with shared filesystem
allow --client parameter to be pathname containing client host IPs/names
document changes to --client syntax and behavior
Bruce Cran (3):
Add Windows ctime_r implementation and add empty ioctl.h header
Fix compiler warning and test progs linker errors on Windows
Fix Windows installer build: LICENSE is now MORAL-LICENSE
Jens Axboe (8):
Merge branch 'multiclnt-sharedfs-try3' of git://github.com/bengland2/fio
Ensure that set_name_idx() doesn't exceed target bounds
posixaio: use 'errno' for inline error
Merge branch 'client-hostfile' of git://github.com/bengland2/fio
Fix keyword replacement leaks
README: add git:// for github as well
Use PATH_MAX instead of _POSIX_HOST_NAME_MAX
Fio 2.2.8
FIO-VERSION-GEN | 2 +-
Makefile | 6 ++++++
README | 30 ++++++++++++++++++++++++++++--
engines/posixaio.c | 2 +-
filesetup.c | 2 +-
fio.1 | 31 +++++++++++++++++++++++++++++--
fio.c | 14 ++++++++++----
fio.h | 2 ++
init.c | 29 +++++++++++++++++++++++++++++
libfio.c | 5 +++++
options.c | 25 +++++++++++++++++++++++--
options.h | 4 +++-
os/os-windows.h | 1 +
os/windows/install.wxs | 6 +++---
os/windows/posix.c | 24 ++++++++++++++++++++++++
os/windows/posix/include/sys/ioctl.h | 7 +++++++
server.c | 1 +
17 files changed, 174 insertions(+), 17 deletions(-)
create mode 100644 os/windows/posix/include/sys/ioctl.h
---
Diff of recent changes:
diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN
index 22c9f68..25fad7d 100755
--- a/FIO-VERSION-GEN
+++ b/FIO-VERSION-GEN
@@ -1,7 +1,7 @@
#!/bin/sh
GVF=FIO-VERSION-FILE
-DEF_VER=fio-2.2.7
+DEF_VER=fio-2.2.8
LF='
'
diff --git a/Makefile b/Makefile
index 1b312cb..d9aedf5 100644
--- a/Makefile
+++ b/Makefile
@@ -218,6 +218,12 @@ T_OBJS += $(T_LFSR_TEST_OBJS)
T_OBJS += $(T_BTRACE_FIO_OBJS)
T_OBJS += $(T_DEDUPE_OBJS)
+ifneq (,$(findstring CYGWIN,$(CONFIG_TARGET_OS)))
+ T_DEDUPE_OBJS += os/windows/posix.o lib/hweight.o
+ T_SMALLOC_OBJS += os/windows/posix.o lib/hweight.o
+ T_LFSR_TEST_OBJS += os/windows/posix.o lib/hweight.o
+endif
+
T_TEST_PROGS = $(T_SMALLOC_PROGS)
T_TEST_PROGS += $(T_IEEE_PROGS)
T_PROGS += $(T_ZIPF_PROGS)
diff --git a/README b/README
index 18d1c4f..25189f8 100644
--- a/README
+++ b/README
@@ -35,6 +35,7 @@ down for some reason, either one of those is safe to use:
or
+ git://github.com/axboe/fio.git
https://github.com/axboe/fio.git
@@ -172,7 +173,7 @@ $ fio
--warnings-fatal Fio parser warnings are fatal
--max-jobs Maximum number of threads/processes to support
--server=args Start backend server. See Client/Server section.
- --client=host Connect to specified backend.
+ --client=host Connect to specified backend(s).
--remote-config=file Tell fio server to load this local file
--idle-prof=option Report cpu idleness on a system or percpu basis
(option=system,percpu) or run unit work
@@ -311,9 +312,34 @@ to load a local file as well. This is done by using --remote-config:
fio --client=server --remote-config /path/to/file.fio
-Then the fio server will open this local (to the server) job file instead
+Then fio will open this local (to the server) job file instead
of being passed one from the client.
+If you have many servers (example: 100 VMs/containers),
+you can input a pathname of a file containing host IPs/names as the parameter
+value for the --client option. For example, here is an example "host.list"
+file containing 2 hostnames:
+
+host1.your.dns.domain
+host2.your.dns.domain
+
+The fio command would then be:
+
+fio --client=host.list <job file(s)>
+
+In this mode, you cannot input server-specific parameters or job files -- all
+servers receive the same job file.
+
+In order to let fio --client runs use a shared filesystem
+from multiple hosts, fio --client now prepends the IP address of the
+server to the filename. For example, if fio is using directory /mnt/nfs/fio
+and is writing filename fileio.tmp, with a --client hostfile containing
+two hostnames h1 and h2 with IP addresses 192.168.10.120 and 192.168.10.121,
+then fio will create two files:
+
+ /mnt/nfs/fio/192.168.10.120.fileio.tmp
+ /mnt/nfs/fio/192.168.10.121.fileio.tmp
+
Platforms
---------
diff --git a/engines/posixaio.c b/engines/posixaio.c
index 8ab88fb..29bcc5a 100644
--- a/engines/posixaio.c
+++ b/engines/posixaio.c
@@ -198,7 +198,7 @@ static int fio_posixaio_queue(struct thread_data *td,
}
if (ret) {
- int aio_err = aio_error(aiocb);
+ int aio_err = errno;
/*
* At least OSX has a very low limit on the number of pending
diff --git a/filesetup.c b/filesetup.c
index 10fc415..aee7ece 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -1262,7 +1262,7 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc)
dprint(FD_FILE, "add file %s\n", fname);
if (td->o.directory)
- len = set_name_idx(file_name, td->o.directory, numjob);
+ len = set_name_idx(file_name, PATH_MAX, td->o.directory, numjob);
sprintf(file_name + len, "%s", fname);
diff --git a/fio.1 b/fio.1
index e54e419..e5451b1 100644
--- a/fio.1
+++ b/fio.1
@@ -93,7 +93,7 @@ Start a backend server, with \fIargs\fP specifying what to listen to. See client
Background a fio server, writing the pid to the given pid file.
.TP
.BI \-\-client \fR=\fPhost
-Instead of running the jobs locally, send and run them on the given host.
+Instead of running the jobs locally, send and run them on the given host or set of hosts. See client/server section.
.TP
.BI \-\-idle\-prof \fR=\fPoption
Report cpu idleness on a system or percpu basis (\fIoption\fP=system,percpu) or run unit work calibration only (\fIoption\fP=calibrate).
@@ -1911,8 +1911,35 @@ to load a local file as well. This is done by using \-\-remote-config:
fio \-\-client=server \-\-remote-config /path/to/file.fio
-Then the fio serer will open this local (to the server) job file instead
+Then fio will open this local (to the server) job file instead
of being passed one from the client.
+
+If you have many servers (example: 100 VMs/containers), you can input a pathname
+of a file containing host IPs/names as the parameter value for the \-\-client option.
+For example, here is an example "host.list" file containing 2 hostnames:
+
+host1.your.dns.domain
+.br
+host2.your.dns.domain
+
+The fio command would then be:
+
+fio \-\-client=host.list <job file>
+
+In this mode, you cannot input server-specific parameters or job files, and all
+servers receive the same job file.
+
+In order to enable fio \-\-client runs utilizing a shared filesystem from multiple hosts,
+fio \-\-client now prepends the IP address of the server to the filename. For example,
+if fio is using directory /mnt/nfs/fio and is writing filename fileio.tmp,
+with a \-\-client hostfile
+containing two hostnames h1 and h2 with IP addresses 192.168.10.120 and 192.168.10.121, then
+fio will create two files:
+
+/mnt/nfs/fio/192.168.10.120.fileio.tmp
+.br
+/mnt/nfs/fio/192.168.10.121.fileio.tmp
+
.SH AUTHORS
.B fio
diff --git a/fio.c b/fio.c
index 9adc29a..ee6eae2 100644
--- a/fio.c
+++ b/fio.c
@@ -30,6 +30,8 @@
int main(int argc, char *argv[], char *envp[])
{
+ int ret = 1;
+
if (initialize_fio(envp))
return 1;
@@ -38,7 +40,7 @@ int main(int argc, char *argv[], char *envp[])
#endif
if (parse_options(argc, argv))
- return 1;
+ goto done;
fio_time_init();
@@ -46,8 +48,12 @@ int main(int argc, char *argv[], char *envp[])
set_genesis_time();
if (fio_start_all_clients())
- return 1;
- return fio_handle_clients(&fio_client_ops);
+ goto done;
+ ret = fio_handle_clients(&fio_client_ops);
} else
- return fio_backend();
+ ret = fio_backend();
+
+done:
+ deinitialize_fio();
+ return ret;
}
diff --git a/fio.h b/fio.h
index 0d5a0ef..6659e28 100644
--- a/fio.h
+++ b/fio.h
@@ -464,6 +464,7 @@ extern void reset_fio_state(void);
extern void clear_io_state(struct thread_data *);
extern int fio_options_parse(struct thread_data *, char **, int, int);
extern void fio_keywords_init(void);
+extern void fio_keywords_exit(void);
extern int fio_cmd_option_parse(struct thread_data *, const char *, char *);
extern int fio_cmd_ioengine_option_parse(struct thread_data *, const char *, char *);
extern void fio_fill_default_options(struct thread_data *);
@@ -483,6 +484,7 @@ extern int fio_set_fd_nonblocking(int, const char *);
extern uintptr_t page_mask;
extern uintptr_t page_size;
extern int initialize_fio(char *envp[]);
+extern void deinitialize_fio(void);
#define FIO_GETOPT_JOB 0x89000000
#define FIO_GETOPT_IOENGINE 0x98000000
diff --git a/init.c b/init.c
index 1a5d4c9..fa50d3c 100644
--- a/init.c
+++ b/init.c
@@ -2242,6 +2242,35 @@ int parse_cmd_line(int argc, char *argv[], int client_type)
exit_val = 1;
break;
}
+ /* if --client parameter contains a pathname */
+ if (0 == access(optarg, R_OK)) {
+ /* file contains a list of host addrs or names */
+ char hostaddr[PATH_MAX] = {0};
+ char formatstr[8];
+ FILE * hostf = fopen(optarg, "r");
+ if (!hostf) {
+ log_err("fio: could not open client list file %s for read\n", optarg);
+ do_exit++;
+ exit_val = 1;
+ break;
+ }
+ sprintf(formatstr, "%%%ds", PATH_MAX - 1);
+ /*
+ * read at most PATH_MAX-1 chars from each
+ * record in this file
+ */
+ while (fscanf(hostf, formatstr, hostaddr) == 1) {
+ /* expect EVERY host in file to be valid */
+ if (fio_client_add(&fio_client_ops, hostaddr, &cur_client)) {
+ log_err("fio: failed adding client %s from file %s\n", hostaddr, optarg);
+ do_exit++;
+ exit_val = 1;
+ break;
+ }
+ }
+ fclose(hostf);
+ break; /* no possibility of job file for "this client only" */
+ }
if (fio_client_add(&fio_client_ops, optarg, &cur_client)) {
log_err("fio: failed adding client %s\n", optarg);
do_exit++;
diff --git a/libfio.c b/libfio.c
index ed26114..3a28f55 100644
--- a/libfio.c
+++ b/libfio.c
@@ -353,3 +353,8 @@ int initialize_fio(char *envp[])
fio_keywords_init();
return 0;
}
+
+void deinitialize_fio(void)
+{
+ fio_keywords_exit();
+}
diff --git a/options.c b/options.c
index 40b69ed..f21ed26 100644
--- a/options.c
+++ b/options.c
@@ -8,6 +8,7 @@
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include <netinet/in.h>
#include "fio.h"
#include "verify.h"
@@ -17,6 +18,8 @@
#include "crc/crc32c.h"
+char client_sockaddr_str[INET6_ADDRSTRLEN] = { 0 };
+
/*
* Check if mmap/mmaphuge has a :/foo/bar/file at the end. If so, return that.
*/
@@ -820,7 +823,7 @@ static int get_max_name_idx(char *input)
* Returns the directory at the index, indexes > entires will be
* assigned via modulo division of the index
*/
-int set_name_idx(char *target, char *input, int index)
+int set_name_idx(char *target, size_t tlen, char *input, int index)
{
unsigned int cur_idx;
int len;
@@ -832,7 +835,13 @@ int set_name_idx(char *target, char *input, int index)
for (cur_idx = 0; cur_idx <= index; cur_idx++)
fname = get_next_name(&str);
- len = sprintf(target, "%s/", fname);
+ if (client_sockaddr_str[0]) {
+ len = snprintf(target, tlen, "%s/%s.", fname,
+ client_sockaddr_str);
+ } else
+ len = snprintf(target, tlen, "%s/", fname);
+
+ target[tlen - 1] = '\0';
free(p);
return len;
@@ -3813,6 +3822,18 @@ static struct fio_keyword fio_keywords[] = {
},
};
+void fio_keywords_exit(void)
+{
+ struct fio_keyword *kw;
+
+ kw = &fio_keywords[0];
+ while (kw->word) {
+ free(kw->replace);
+ kw->replace = NULL;
+ kw++;
+ }
+}
+
void fio_keywords_init(void)
{
unsigned long long mb_memory;
diff --git a/options.h b/options.h
index 2cf435a..6805b31 100644
--- a/options.h
+++ b/options.h
@@ -18,7 +18,9 @@ void del_opt_posval(const char *, const char *);
struct thread_data;
void fio_options_free(struct thread_data *);
char *get_name_idx(char *, int);
-int set_name_idx(char *, char *, int);
+int set_name_idx(char *, size_t, char *, int);
+
+extern char client_sockaddr_str[]; /* used with --client option */
extern struct fio_option fio_options[FIO_MAX_OPTS];
diff --git a/os/os-windows.h b/os/os-windows.h
index 6603635..9e931c9 100644
--- a/os/os-windows.h
+++ b/os/os-windows.h
@@ -105,6 +105,7 @@ int fcntl(int fildes, int cmd, ...);
int fdatasync(int fildes);
int lstat(const char * path, struct stat * buf);
uid_t geteuid(void);
+char* ctime_r(const time_t *t, char *buf);
int nanosleep(const struct timespec *rqtp, struct timespec *rmtp);
ssize_t pread(int fildes, void *buf, size_t nbyte, off_t offset);
ssize_t pwrite(int fildes, const void *buf, size_t nbyte,
diff --git a/os/windows/install.wxs b/os/windows/install.wxs
index ba58b92..2a67a4d 100755
--- a/os/windows/install.wxs
+++ b/os/windows/install.wxs
@@ -10,7 +10,7 @@
<Product Id="*"
Codepage="1252" Language="1033"
Manufacturer="fio" Name="fio"
- UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.2.7">
+ UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="2.2.8">
<Package
Description="Flexible IO Tester"
InstallerVersion="301" Keywords="Installer,MSI,Database"
@@ -40,7 +40,7 @@
<File Id="COPYING" Name="COPYING.txt" Source="..\..\COPYING"/>
</Component>
<Component>
- <File Id="LICENSE" Name="LICENSE.txt" Source="..\..\LICENSE"/>
+ <File Id="MORAL_LICENSE" Name="MORAL-LICENSE.txt" Source="..\..\MORAL-LICENSE"/>
</Component>
<Directory Id="examples" Name="examples"/>
</Directory>
@@ -54,7 +54,7 @@
<ComponentRef Id="README"/>
<ComponentRef Id="REPORTING_BUGS"/>
<ComponentRef Id="COPYING"/>
- <ComponentRef Id="LICENSE"/>
+ <ComponentRef Id="MORAL_LICENSE"/>
<ComponentGroupRef Id="examples"/>
</Feature>
diff --git a/os/windows/posix.c b/os/windows/posix.c
index d238c64..41fc480 100755
--- a/os/windows/posix.c
+++ b/os/windows/posix.c
@@ -229,6 +229,30 @@ char *dlerror(void)
return dl_error;
}
+/* Copied from http://blogs.msdn.com/b/joshpoley/archive/2007/12/19/date-time-formats-and-conversions.aspx */
+void Time_tToSystemTime(time_t dosTime, SYSTEMTIME *systemTime)
+{
+ LARGE_INTEGER jan1970FT;
+ LARGE_INTEGER utcFT;
+ jan1970FT.QuadPart = 116444736000000000LL; // january 1st 1970
+ utcFT.QuadPart = ((unsigned __int64)dosTime) * 10000000 + jan1970FT.QuadPart;
+
+ FileTimeToSystemTime((FILETIME*)&utcFT, systemTime);
+}
+
+char* ctime_r(const time_t *t, char *buf)
+{
+ SYSTEMTIME systime;
+ const char * const dayOfWeek[] = { "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" };
+ const char * const monthOfYear[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
+
+ Time_tToSystemTime(*t, &systime);
+ /* We don't know how long `buf` is, but assume it's rounded up from the minimum of 25 to 32 */
+ StringCchPrintfA(buf, 32, "%s %s %d %02d:%02d:%02d %04d", dayOfWeek[systime.wDayOfWeek - 1], monthOfYear[systime.wMonth - 1],
+ systime.wDay, systime.wHour, systime.wMinute, systime.wSecond, systime.wYear);
+ return buf;
+}
+
int gettimeofday(struct timeval *restrict tp, void *restrict tzp)
{
FILETIME fileTime;
diff --git a/os/windows/posix/include/sys/ioctl.h b/os/windows/posix/include/sys/ioctl.h
new file mode 100644
index 0000000..a42247d
--- /dev/null
+++ b/os/windows/posix/include/sys/ioctl.h
@@ -0,0 +1,7 @@
+#ifndef IOCTL_H
+#define IOCTL_H
+
+/* This file is empty since it only needs to exist on Windows
+ but isn't otherwise used */
+
+#endif /* IOCTL_H */
\ No newline at end of file
diff --git a/server.c b/server.c
index 93c0987..7a9b0a4 100644
--- a/server.c
+++ b/server.c
@@ -1007,6 +1007,7 @@ static int accept_loop(int listen_sk)
}
/* exits */
+ strncpy(client_sockaddr_str, from, INET6_ADDRSTRLEN);
handle_connection(sk);
}
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
