The following changes since commit 8200b8c711cfabca65ce32e84f5985a2307a1ea9:
Wire up fusion-aw-sync engine (2012-09-18 23:55:43 +0200)
are available in the git repository at:
git://git.kernel.dk/fio.git master
Dmitry Monakhov (4):
export file_lookup_open
ioengine: allow several external ioengines
ioengine: Add fallocate ioengine
engine: add e4defrag engine
Jens Axboe (2):
fusion-aw-sync: fixup GPL disclaimer
Makefile: break long line
HOWTO | 2 +
Makefile | 5 +-
engines/e4defrag.c | 215 +++++++++++++++++++++++++++++++++++++++++++++++++++
engines/falloc.c | 121 +++++++++++++++++++++++++++++
engines/fusion-aw.c | 5 +-
examples/e4defrag | 32 ++++++++
file.h | 1 +
filesetup.c | 2 +-
ioengines.c | 4 +-
9 files changed, 379 insertions(+), 8 deletions(-)
create mode 100644 engines/e4defrag.c
create mode 100644 engines/falloc.c
create mode 100644 examples/e4defrag
---
Diff of recent changes:
diff --git a/HOWTO b/HOWTO
index 025443b..3eb5510 100644
--- a/HOWTO
+++ b/HOWTO
@@ -602,6 +602,8 @@ ioengine=str Defines how the job issues io to the file. The following
channel semantics (Send/Recv) for the
InfiniBand, RoCE and iWARP protocols.
+ falloc Perform fallocate/punch_hole to files
+
external Prefix to specify loading an external
IO engine object file. Append the engine
filename, eg ioengine=external:/tmp/foo.o
diff --git a/Makefile b/Makefile
index 5e90ca2..ccfa802 100644
--- a/Makefile
+++ b/Makefile
@@ -14,14 +14,15 @@ SOURCE := gettime.c fio.c ioengines.c init.c stat.c log.c time.c filesetup.c \
rbtree.c smalloc.c filehash.c profile.c debug.c lib/rand.c \
lib/num2str.c lib/ieee754.c $(wildcard crc/*.c) engines/cpu.c \
engines/mmap.c engines/sync.c engines/null.c engines/net.c \
- memalign.c server.c client.c iolog.c backend.c libfio.c flow.c json.c
+ memalign.c server.c client.c iolog.c backend.c libfio.c flow.c \
+ json.c
ifeq ($(UNAME), Linux)
SOURCE += diskutil.c fifo.c blktrace.c helpers.c cgroup.c trim.c \
engines/libaio.c engines/posixaio.c engines/sg.c \
engines/splice.c engines/syslet-rw.c engines/guasi.c \
engines/binject.c engines/rdma.c profiles/tiobench.c \
- engines/fusion-aw.c
+ engines/fusion-aw.c engines/falloc.c engines/e4defrag.c
LIBS += -lpthread -ldl -lrt -laio
LDFLAGS += -rdynamic
endif
diff --git a/engines/e4defrag.c b/engines/e4defrag.c
new file mode 100644
index 0000000..5affaa0
--- /dev/null
+++ b/engines/e4defrag.c
@@ -0,0 +1,215 @@
+/*
+ * ioe_e4defrag: ioengine for git://git.kernel.dk/fio.git
+ *
+ * IO engine that does regular EXT4_IOC_MOVE_EXT ioctls to simulate
+ * defragment activity
+ *
+ */
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <errno.h>
+#include <assert.h>
+#include <fcntl.h>
+
+#include "../fio.h"
+
+#ifndef EXT4_IOC_MOVE_EXT
+#define EXT4_IOC_MOVE_EXT _IOWR('f', 15, struct move_extent)
+struct move_extent {
+ __u32 reserved; /* should be zero */
+ __u32 donor_fd; /* donor file descriptor */
+ __u64 orig_start; /* logical start offset in block for orig */
+ __u64 donor_start; /* logical start offset in block for donor */
+ __u64 len; /* block length to be moved */
+ __u64 moved_len; /* moved block length */
+};
+#endif
+
+struct e4defrag_data {
+ int donor_fd;
+ int bsz;
+};
+
+struct e4defrag_options {
+ struct thread_data *td;
+ unsigned int inplace;
+ char * donor_name;
+};
+
+static struct fio_option options[] = {
+ {
+ .name = "donorname",
+ .type = FIO_OPT_STR_STORE,
+ .off1 = offsetof(struct e4defrag_options, donor_name),
+ .help = "File used as a block donor",
+ },
+ {
+ .name = "inplace",
+ .type = FIO_OPT_INT,
+ .off1 = offsetof(struct e4defrag_options, inplace),
+ .minval = 0,
+ .maxval = 1,
+ .help = "Alloc and free space inside defrag event",
+ },
+ {
+ .name = NULL,
+ },
+};
+
+static int fio_e4defrag_init(struct thread_data *td)
+{
+ int r, len = 0;
+ struct e4defrag_options *o = td->eo;
+ struct e4defrag_data *ed;
+ struct stat stub;
+ char donor_name[PATH_MAX];
+
+ if (!strlen(o->donor_name)) {
+ log_err("'donorname' options required\n");
+ return 1;
+ }
+
+ ed = malloc(sizeof(*ed));
+ if (!ed) {
+ td_verror(td, -ENOMEM, "io_queue_init");
+ return 1;
+ }
+ memset(ed, 0 ,sizeof(*ed));
+
+ if (td->o.directory)
+ len = sprintf(donor_name, "%s/", td->o.directory);
+ sprintf(donor_name + len, "%s", o->donor_name);
+
+ ed->donor_fd = open(donor_name, O_CREAT|O_WRONLY, 0644);
+ if (ed->donor_fd < 0) {
+ td_verror(td, ed->donor_fd, "io_queue_init");
+ log_err("Can't open donor file %s err:%d", ed->donor_fd);
+ free(ed);
+ return 1;
+ }
+
+ if (!o->inplace) {
+ long long len = td->o.file_size_high - td->o.start_offset;
+ r = fallocate(ed->donor_fd, 0, td->o.start_offset, len);
+ if (r)
+ goto err;
+ }
+ r = fstat(ed->donor_fd, &stub);
+ if (r)
+ goto err;
+
+ ed->bsz = stub.st_blksize;
+ td->io_ops->data = ed;
+ return 0;
+err:
+ td_verror(td, errno, "io_queue_init");
+ close(ed->donor_fd);
+ free(ed);
+ return 1;
+}
+
+static void fio_e4defrag_cleanup(struct thread_data *td)
+{
+ struct e4defrag_data *ed = td->io_ops->data;
+ if (ed) {
+ if (ed->donor_fd >= 0)
+ close(ed->donor_fd);
+ free(ed);
+ }
+}
+
+
+static int fio_e4defrag_queue(struct thread_data *td, struct io_u *io_u)
+{
+
+ int ret;
+ unsigned long long len;
+ struct move_extent me;
+ struct fio_file *f = io_u->file;
+ struct e4defrag_data *ed = td->io_ops->data;
+ struct e4defrag_options *o = td->eo;
+
+ fio_ro_check(td, io_u);
+
+ /* Theoretically defragmentation should not change data, but it
+ * changes data layout. So this function handle only DDIR_WRITE
+ * in order to satisfy strict read only access pattern
+ */
+ if (io_u->ddir != DDIR_WRITE) {
+ io_u->error = errno;
+ return FIO_Q_COMPLETED;
+ }
+
+ if (o->inplace) {
+ ret = fallocate(ed->donor_fd, 0, io_u->offset, io_u->xfer_buflen);
+ if (ret) {
+ io_u->error = errno;
+ goto out;
+ }
+ }
+
+ memset(&me, 0, sizeof(me));
+ me.donor_fd = ed->donor_fd;
+ me.orig_start = io_u->offset / ed->bsz;
+ me.donor_start = me.orig_start;
+ len = (io_u->offset + io_u->xfer_buflen + ed->bsz -1);
+ me.len = len / ed->bsz - me.orig_start;
+
+ ret = ioctl(f->fd, EXT4_IOC_MOVE_EXT, &me);
+ len = me.moved_len * ed->bsz;
+
+ if (io_u->file && len >= 0 && ddir_rw(io_u->ddir))
+ io_u->file->file_pos = io_u->offset + len;
+
+ if (len > io_u->xfer_buflen)
+ len = io_u->xfer_buflen;
+
+ if (len != io_u->xfer_buflen) {
+ io_u->resid = io_u->xfer_buflen - len;
+ io_u->error = 0;
+ }
+ if (ret)
+ io_u->error = errno;
+
+ if (o->inplace) {
+ ret = ftruncate(ed->donor_fd, 0);
+ if (ret)
+ io_u->error = errno;
+ }
+out:
+ if (io_u->error)
+ td_verror(td, errno, "xfer");
+
+
+ return FIO_Q_COMPLETED;
+}
+
+static struct ioengine_ops ioengine = {
+ .name = "e4defrag",
+ .version = FIO_IOOPS_VERSION,
+ .init = fio_e4defrag_init,
+ .queue = fio_e4defrag_queue,
+ .open_file = generic_open_file,
+ .close_file = generic_close_file,
+ .get_file_size = generic_get_file_size,
+ .flags = FIO_SYNCIO,
+ .cleanup = fio_e4defrag_cleanup,
+ .options = options,
+ .option_struct_size = sizeof(struct e4defrag_options),
+
+};
+
+static void fio_init fio_syncio_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_syncio_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/engines/falloc.c b/engines/falloc.c
new file mode 100644
index 0000000..cbb30cb
--- /dev/null
+++ b/engines/falloc.c
@@ -0,0 +1,121 @@
+/*
+ * falloc: ioengine for git://git.kernel.dk/fio.git
+ *
+ * IO engine that does regular fallocate to simulate data transfer
+ * as fio ioengine.
+ * DDIR_READ does fallocate(,mode = FALLOC_FL_KEEP_SIZE,)
+ * DDIR_WRITE does fallocate(,mode = 0) : fallocate with size extention
+ * DDIR_TRIM does fallocate(,mode = FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)
+ *
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/uio.h>
+#include <errno.h>
+#include <assert.h>
+#include <fcntl.h>
+
+#include "../fio.h"
+#include "../filehash.h"
+
+/*
+ * generic_open_file is not appropriate because does not allow to perform
+ * TRIM in to file
+ */
+int open_file(struct thread_data *td, struct fio_file *f)
+{
+ int from_hash = 0;
+
+ dprint(FD_FILE, "fd open %s\n", f->file_name);
+
+ if (f->filetype != FIO_TYPE_FILE) {
+ log_err("fio: only files are supported fallocate \n");
+ return 1;
+ }
+ if (!strcmp(f->file_name, "-")) {
+ log_err("fio: can't read/write to stdin/out\n");
+ return 1;
+ }
+
+open_again:
+ from_hash = file_lookup_open(f, O_CREAT|O_RDWR);
+
+ if (f->fd == -1) {
+ char buf[FIO_VERROR_SIZE];
+ int __e = errno;
+ snprintf(buf, sizeof(buf) - 1, "open(%s)", f->file_name);
+ td_verror(td, __e, buf);
+ }
+
+ if (!from_hash && f->fd != -1) {
+ if (add_file_hash(f)) {
+ int fio_unused ret;
+
+ /*
+ * OK to ignore, we haven't done anything with it
+ */
+ ret = generic_close_file(td, f);
+ goto open_again;
+ }
+ }
+
+ return 0;
+}
+
+#ifndef FALLOC_FL_KEEP_SIZE
+#define FALLOC_FL_KEEP_SIZE 0x01 /* default is extend size */
+#endif
+#ifndef FALLOC_FL_PUNCH_HOLE
+#define FALLOC_FL_PUNCH_HOLE 0x02 /* de-allocates range */
+#endif
+static int fio_fallocate_queue(struct thread_data *td, struct io_u *io_u)
+{
+ struct fio_file *f = io_u->file;
+ int ret;
+ int flags = 0;
+
+ fio_ro_check(td, io_u);
+
+ if (io_u->ddir == DDIR_READ)
+ flags = FALLOC_FL_KEEP_SIZE;
+ else if (io_u->ddir == DDIR_WRITE)
+ flags = 0;
+ else if (io_u->ddir == DDIR_TRIM)
+ flags = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
+
+ ret = fallocate(f->fd, flags, io_u->offset, io_u->xfer_buflen);
+
+ if (ret) {
+ io_u->error = errno;
+ if (io_u->error)
+ td_verror(td, io_u->error, "xfer");
+ }
+ if (io_u->ddir == DDIR_TRIM && !ret)
+ return io_u->xfer_buflen;
+
+ if (io_u->file && ret == 0 && ddir_rw(io_u->ddir))
+ io_u->file->file_pos = io_u->offset + ret;
+
+ return FIO_Q_COMPLETED;
+}
+
+static struct ioengine_ops ioengine = {
+ .name = "falloc",
+ .version = FIO_IOOPS_VERSION,
+ .queue = fio_fallocate_queue,
+ .open_file = open_file,
+ .close_file = generic_close_file,
+ .get_file_size = generic_get_file_size,
+ .flags = FIO_SYNCIO
+};
+
+static void fio_init fio_syncio_register(void)
+{
+ register_ioengine(&ioengine);
+}
+
+static void fio_exit fio_syncio_unregister(void)
+{
+ unregister_ioengine(&ioengine);
+}
diff --git a/engines/fusion-aw.c b/engines/fusion-aw.c
index 0f9025f..9aac43a 100644
--- a/engines/fusion-aw.c
+++ b/engines/fusion-aw.c
@@ -6,10 +6,7 @@
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
- * Software Foundation; under version 2 of the License. You may obtain a copy
- * of the source code for DirectFS by sending a request to Fusion-io, Inc.,
- * 2855 E. Cottonwood Parkway, Suite 100, Salt Lake City, UT 84121; Attention:
- * Legal Department.
+ * Software Foundation; under version 2 of the License.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
diff --git a/examples/e4defrag b/examples/e4defrag
new file mode 100644
index 0000000..d392149
--- /dev/null
+++ b/examples/e4defrag
@@ -0,0 +1,32 @@
+[global]
+
+direct=0
+buffered=0
+directory=/scratch
+
+nrfiles=1
+
+filesize=4G
+fadvise_hint=0
+
+group_reporting
+
+[defrag-fuzzer-8k]
+ioengine=e4defrag
+iodepth=1
+size=1G
+bs=8k
+donorname=file.def
+filename=file
+inplace=0
+rw=randwrite
+numjobs=1
+
+[random-aio-32k]
+ioengine=libaio
+iodepth=128
+bs=32k
+size=4G
+filename=file
+rw=randwrite
+numjobs=1
diff --git a/file.h b/file.h
index 68f9a6e..42fd58c 100644
--- a/file.h
+++ b/file.h
@@ -153,6 +153,7 @@ extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_f
extern int __must_check generic_open_file(struct thread_data *, struct fio_file *);
extern int __must_check generic_close_file(struct thread_data *, struct fio_file *);
extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *);
+extern int __must_check file_lookup_open(struct fio_file *f, int flags);
extern int __must_check pre_read_files(struct thread_data *);
extern int add_file(struct thread_data *, const char *);
extern int add_file_exclusive(struct thread_data *, const char *);
diff --git a/filesetup.c b/filesetup.c
index 64da8bb..06b7d7f 100644
--- a/filesetup.c
+++ b/filesetup.c
@@ -435,7 +435,7 @@ int generic_close_file(struct thread_data fio_unused *td, struct fio_file *f)
return ret;
}
-static int file_lookup_open(struct fio_file *f, int flags)
+int file_lookup_open(struct fio_file *f, int flags)
{
struct fio_file *__f;
int from_hash;
diff --git a/ioengines.c b/ioengines.c
index b43374e..c2a64cb 100644
--- a/ioengines.c
+++ b/ioengines.c
@@ -104,7 +104,9 @@ static struct ioengine_ops *dlopen_ioengine(struct thread_data *td,
* Unlike the included modules, external engines should have a
* non-static ioengine structure that we can reference.
*/
- ops = dlsym(dlhandle, "ioengine");
+ ops = dlsym(dlhandle, engine_lib);
+ if (!ops)
+ ops = dlsym(dlhandle, "ioengine");
if (!ops) {
td_vmsg(td, -1, dlerror(), "dlsym");
dlclose(dlhandle);
--
To unsubscribe from this list: send the line "unsubscribe fio" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
