The following changes since commit 595e17349ca6b5cb8562689a12c0cc0b551c35da:
Fix man page indentation (2012-12-05 21:15:01 +0100)
are available in the git repository at:
git://git.kernel.dk/fio.git master
Bruce Cran (3):
Free io_u related structures before killing IO engine
Fix windows out-of-memory handling
windowsaio: create a single completion port during init, associate files during open.
Jens Axboe (3):
parser: always match the correct option length for posval options
windowsaio: initialize and map windowsaio IO structure to io_u
Document the ioengine=net pingpong= option
HOWTO | 9 +++
backend.c | 16 +++++-
engines/windowsaio.c | 142 ++++++++++++++++++++++++++-----------------------
fio.1 | 9 +++
ioengine.h | 2 +
os/os-windows.h | 2 +
os/windows/posix.c | 5 ++
parse.c | 9 +++-
8 files changed, 124 insertions(+), 70 deletions(-)
---
Diff of recent changes:
diff --git a/HOWTO b/HOWTO
index 6d541af..b349620 100644
--- a/HOWTO
+++ b/HOWTO
@@ -1399,6 +1399,15 @@ that defines them is selected.
[net] listen For TCP network connections, tell fio to listen for incoming
connections rather than initiating an outgoing connection. The
hostname must be omitted if this option is used.
+[net] pingpong Normal a network writer will just continue writing data, and
+ a network reader will just consume packages. If pingpong=1
+ is set, a writer will send its normal payload to the reader,
+ then wait for the reader to send the same payload back. This
+ allows fio to measure network latencies. The submission
+ and completion latencies then measure local time spent
+ sending or receiving, and the completion latency measures
+ how long it took for the other end to receive and send back.
+
[e4defrag] donorname=str
File will be used as a block donor(swap extents between files)
[e4defrag] inplace=int
diff --git a/backend.c b/backend.c
index faa861c..a71d768 100644
--- a/backend.c
+++ b/backend.c
@@ -803,6 +803,10 @@ static void cleanup_io_u(struct thread_data *td)
io_u = flist_entry(entry, struct io_u, list);
flist_del(&io_u->list);
+
+ if (td->io_ops->io_u_free)
+ td->io_ops->io_u_free(td, io_u);
+
fio_memfree(io_u, sizeof(*io_u));
}
@@ -885,6 +889,16 @@ static int init_io_u(struct thread_data *td)
io_u->index = i;
io_u->flags = IO_U_F_FREE;
flist_add(&io_u->list, &td->io_u_freelist);
+
+ if (td->io_ops->io_u_init) {
+ int ret = td->io_ops->io_u_init(td, io_u);
+
+ if (ret) {
+ log_err("fio: failed to init engine data: %d\n", ret);
+ return 1;
+ }
+ }
+
p += max_bs;
}
@@ -1273,8 +1287,8 @@ err:
verify_async_exit(td);
close_and_free_files(td);
- close_ioengine(td);
cleanup_io_u(td);
+ close_ioengine(td);
cgroup_shutdown(td, &cgroup_mnt);
if (td->o.cpumask_set) {
diff --git a/engines/windowsaio.c b/engines/windowsaio.c
index db75730..edc390c 100644
--- a/engines/windowsaio.c
+++ b/engines/windowsaio.c
@@ -20,12 +20,11 @@ struct fio_overlapped {
OVERLAPPED o;
struct io_u *io_u;
BOOL io_complete;
- BOOL io_free;
};
struct windowsaio_data {
- struct fio_overlapped *ovls;
struct io_u **aio_events;
+ HANDLE iocp;
HANDLE iothread;
HANDLE iocomplete_event;
CANCELIOEX pCancelIoEx;
@@ -50,9 +49,9 @@ static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter);
static int fio_windowsaio_init(struct thread_data *td);
static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f);
static int fio_windowsaio_close_file(struct thread_data fio_unused *td, struct fio_file *f);
-static int win_to_poxix_error(DWORD winerr);
+static int win_to_posix_error(DWORD winerr);
-static int win_to_poxix_error(DWORD winerr)
+static int win_to_posix_error(DWORD winerr)
{
switch (winerr)
{
@@ -139,7 +138,6 @@ static int fio_windowsaio_init(struct thread_data *td)
struct windowsaio_data *wd;
HANDLE hKernel32Dll;
int rc = 0;
- int i;
wd = malloc(sizeof(struct windowsaio_data));
if (wd != NULL)
@@ -154,25 +152,6 @@ static int fio_windowsaio_init(struct thread_data *td)
}
if (!rc) {
- wd->ovls = malloc(td->o.iodepth * sizeof(struct fio_overlapped));
- if (wd->ovls == NULL)
- rc = 1;
- }
-
- if (!rc) {
- for (i = 0; i < td->o.iodepth; i++) {
- wd->ovls[i].io_free = TRUE;
- wd->ovls[i].io_complete = FALSE;
-
- wd->ovls[i].o.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
- if (wd->ovls[i].o.hEvent == NULL) {
- rc = 1;
- break;
- }
- }
- }
-
- if (!rc) {
/* Create an auto-reset event */
wd->iocomplete_event = CreateEvent(NULL, FALSE, FALSE, NULL);
if (wd->iocomplete_event == NULL)
@@ -181,8 +160,6 @@ static int fio_windowsaio_init(struct thread_data *td)
if (rc) {
if (wd != NULL) {
- if (wd->ovls != NULL)
- free(wd->ovls);
if (wd->aio_events != NULL)
free(wd->aio_events);
@@ -194,12 +171,46 @@ static int fio_windowsaio_init(struct thread_data *td)
wd->pCancelIoEx = (CANCELIOEX)GetProcAddress(hKernel32Dll, "CancelIoEx");
td->io_ops->data = wd;
+
+ if (!rc) {
+ struct thread_ctx *ctx;
+ struct windowsaio_data *wd;
+ HANDLE hFile;
+
+ hFile = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
+ if (hFile == INVALID_HANDLE_VALUE)
+ rc = 1;
+
+ wd = td->io_ops->data;
+ wd->iothread_running = TRUE;
+ wd->iocp = hFile;
+
+ if (!rc)
+ ctx = malloc(sizeof(struct thread_ctx));
+
+ if (!rc && ctx == NULL)
+ {
+ log_err("fio: out of memory in windowsaio\n");
+ CloseHandle(hFile);
+ rc = 1;
+ }
+
+ if (!rc)
+ {
+ ctx->iocp = hFile;
+ ctx->wd = wd;
+ wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, NULL);
+ }
+
+ if (rc || wd->iothread == NULL)
+ rc = 1;
+ }
+
return rc;
}
static void fio_windowsaio_cleanup(struct thread_data *td)
{
- int i;
struct windowsaio_data *wd;
wd = td->io_ops->data;
@@ -211,12 +222,7 @@ static void fio_windowsaio_cleanup(struct thread_data *td)
CloseHandle(wd->iothread);
CloseHandle(wd->iocomplete_event);
- for (i = 0; i < td->o.iodepth; i++) {
- CloseHandle(wd->ovls[i].o.hEvent);
- }
-
free(wd->aio_events);
- free(wd->ovls);
free(wd);
td->io_ops->data = NULL;
@@ -227,7 +233,6 @@ static void fio_windowsaio_cleanup(struct thread_data *td)
static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
{
int rc = 0;
- HANDLE hFile;
DWORD flags = FILE_FLAG_POSIX_SEMANTICS | FILE_FLAG_OVERLAPPED;
DWORD sharemode = FILE_SHARE_READ | FILE_SHARE_WRITE;
DWORD openmode = OPEN_ALWAYS;
@@ -279,23 +284,11 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f)
/* Only set up the completion port and thread if we're not just
* querying the device size */
if (!rc && td->io_ops->data != NULL) {
- struct thread_ctx *ctx;
struct windowsaio_data *wd;
- hFile = CreateIoCompletionPort(f->hFile, NULL, 0, 0);
-
wd = td->io_ops->data;
- wd->iothread_running = TRUE;
- if (!rc) {
- ctx = malloc(sizeof(struct thread_ctx));
- ctx->iocp = hFile;
- ctx->wd = wd;
-
- wd->iothread = CreateThread(NULL, 0, IoCompletionRoutine, ctx, 0, NULL);
- }
-
- if (rc || wd->iothread == NULL)
+ if (CreateIoCompletionPort(f->hFile, wd->iocp, 0, 0) == NULL)
rc = 1;
}
@@ -364,7 +357,6 @@ static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
if (fov->io_complete) {
fov->io_complete = FALSE;
- fov->io_free = TRUE;
ResetEvent(fov->o.hEvent);
wd->aio_events[dequeued] = io_u;
dequeued++;
@@ -389,32 +381,18 @@ static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min,
static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u)
{
- LPOVERLAPPED lpOvl = NULL;
- struct windowsaio_data *wd;
+ struct fio_overlapped *o = io_u->engine_data;
+ LPOVERLAPPED lpOvl = &o->o;
DWORD iobytes;
BOOL success = FALSE;
- int index;
int rc = FIO_Q_COMPLETED;
fio_ro_check(td, io_u);
- wd = td->io_ops->data;
-
- for (index = 0; index < td->o.iodepth; index++) {
- if (wd->ovls[index].io_free)
- break;
- }
-
- assert(index < td->o.iodepth);
-
- wd->ovls[index].io_free = FALSE;
- wd->ovls[index].io_u = io_u;
- lpOvl = &wd->ovls[index].o;
lpOvl->Internal = STATUS_PENDING;
lpOvl->InternalHigh = 0;
lpOvl->Offset = io_u->offset & 0xFFFFFFFF;
lpOvl->OffsetHigh = io_u->offset >> 32;
- io_u->engine_data = &wd->ovls[index];
switch (io_u->ddir) {
case DDIR_WRITE:
@@ -428,7 +406,7 @@ static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u)
case DDIR_SYNC_FILE_RANGE:
success = FlushFileBuffers(io_u->file->hFile);
if (!success)
- io_u->error = win_to_poxix_error(GetLastError());
+ io_u->error = win_to_posix_error(GetLastError());
return FIO_Q_COMPLETED;
break;
@@ -446,7 +424,7 @@ static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u)
if (success || GetLastError() == ERROR_IO_PENDING)
rc = FIO_Q_QUEUED;
else {
- io_u->error = win_to_poxix_error(GetLastError());
+ io_u->error = win_to_posix_error(GetLastError());
io_u->resid = io_u->xfer_buflen;
}
@@ -479,7 +457,7 @@ static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter)
io_u->error = 0;
} else {
io_u->resid = io_u->xfer_buflen;
- io_u->error = win_to_poxix_error(GetLastError());
+ io_u->error = win_to_posix_error(GetLastError());
}
fov->io_complete = TRUE;
@@ -510,6 +488,34 @@ static int fio_windowsaio_cancel(struct thread_data *td,
return rc;
}
+static void fio_windowsaio_io_u_free(struct thread_data *td, struct io_u *io_u)
+{
+ struct fio_overlapped *o = io_u->engine_data;
+
+ if (o) {
+ CloseHandle(o->o.hEvent);
+ io_u->engine_data = NULL;
+ free(o);
+ }
+}
+
+static int fio_windowsaio_io_u_init(struct thread_data *td, struct io_u *io_u)
+{
+ struct fio_overlapped *o;
+
+ o = malloc(sizeof(*o));
+ o->io_complete = FALSE:
+ o->io_u = io_u;
+ o->o.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
+ if (!o->o.hEvent) {
+ free(o);
+ return 1;
+ }
+
+ io_u->engine_data = o;
+ return 0;
+}
+
static struct ioengine_ops ioengine = {
.name = "windowsaio",
.version = FIO_IOOPS_VERSION,
@@ -521,7 +527,9 @@ static struct ioengine_ops ioengine = {
.cleanup = fio_windowsaio_cleanup,
.open_file = fio_windowsaio_open_file,
.close_file = fio_windowsaio_close_file,
- .get_file_size = generic_get_file_size
+ .get_file_size = generic_get_file_size,
+ .io_u_init = fio_windowsaio_io_u_init,
+ .io_u_free = fio_windowsaio_io_u_free,
};
static void fio_init fio_posixaio_register(void)
diff --git a/fio.1 b/fio.1
index 26a4650..b7abc4b 100644
--- a/fio.1
+++ b/fio.1
@@ -1162,6 +1162,15 @@ For TCP network connections, tell fio to listen for incoming
connections rather than initiating an outgoing connection. The
hostname must be omitted if this option is used.
.TP
+.BI (net, pingpong) \fR=\fPbool
+Normal a network writer will just continue writing data, and a network reader
+will just consume packages. If pingpong=1 is set, a writer will send its normal
+payload to the reader, then wait for the reader to send the same payload back.
+This allows fio to measure network latencies. The submission and completion
+latencies then measure local time spent sending or receiving, and the
+completion latency measures how long it took for the other end to receive and
+send back.
+.TP
.BI (e4defrag,donorname) \fR=\fPstr
File will be used as a block donor (swap extents between files)
.TP
diff --git a/ioengine.h b/ioengine.h
index 997f90a..0285b08 100644
--- a/ioengine.h
+++ b/ioengine.h
@@ -121,6 +121,8 @@ struct ioengine_ops {
int (*close_file)(struct thread_data *, struct fio_file *);
int (*get_file_size)(struct thread_data *, struct fio_file *);
void (*terminate)(struct thread_data *);
+ int (*io_u_init)(struct thread_data *, struct io_u *);
+ void (*io_u_free)(struct thread_data *, struct io_u *);
int option_struct_size;
struct fio_option *options;
void *data;
diff --git a/os/os-windows.h b/os/os-windows.h
index 7b61d16..ba93195 100644
--- a/os/os-windows.h
+++ b/os/os-windows.h
@@ -23,6 +23,7 @@
#define FIO_HAVE_FALLOCATE
#define FIO_HAVE_GETTID
#define FIO_HAVE_CLOCK_MONOTONIC
+#define FIO_HAVE_FADVISE
#define FIO_USE_GENERIC_RAND
#define FIO_PREFERRED_ENGINE "windowsaio"
@@ -83,6 +84,7 @@ typedef DWORD_PTR os_cpu_mask_t;
#define SIGCONT 0
#define SIGUSR1 1
+#define SIGUSR2 2
typedef int sigset_t;
typedef int siginfo_t;
diff --git a/os/windows/posix.c b/os/windows/posix.c
index 11500e4..f616e87 100755
--- a/os/windows/posix.c
+++ b/os/windows/posix.c
@@ -535,6 +535,11 @@ int getrusage(int who, struct rusage *r_usage)
return 0;
}
+int posix_fadvise(int fd, off_t offset, off_t len, int advice)
+{
+ return 0;
+}
+
int posix_madvise(void *addr, size_t len, int advice)
{
log_err("%s is not implemented\n", __func__);
diff --git a/parse.c b/parse.c
index 13783fa..18c4530 100644
--- a/parse.c
+++ b/parse.c
@@ -344,6 +344,11 @@ static int opt_len(const char *str)
return (int)(postfix - str);
}
+static int str_match_len(const struct value_pair *vp, const char *str)
+{
+ return max(strlen(vp->ival), opt_len(str));
+}
+
#define val_store(ptr, val, off, or, data) \
do { \
ptr = td_var((data), (off)); \
@@ -388,7 +393,7 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data,
if (!vp->ival || vp->ival[0] == '\0')
continue;
all_skipped = 0;
- if (!strncmp(vp->ival, ptr, opt_len(ptr))) {
+ if (!strncmp(vp->ival, ptr, str_match_len(vp, ptr))) {
ret = 0;
if (o->roff1) {
if (vp->or)
@@ -549,7 +554,7 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data,
if (!vp->ival || vp->ival[0] == '\0')
continue;
all_skipped = 0;
- if (!strncmp(vp->ival, ptr, opt_len(ptr))) {
+ if (!strncmp(vp->ival, ptr, str_match_len(vp, ptr))) {
char *rest;
ret = 0;
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
