The following changes since commit ac4f3d4e4cf16b1097249a819fe7111b2674b3f4:
aioring: remove IOCB_FLAG_HIPRI (2018-12-30 17:19:40 -0700)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to ac122fea7540ca115c157e0a835a74b891f10484:
aioring: update to newer API (2019-01-04 22:22:54 -0700)
----------------------------------------------------------------
Jens Axboe (4):
t/aio-ring: update to newer mmap() API
engines/aioring: update for newer mmap based API
t/aio-ring: use syscall defines
aioring: update to newer API
arch/arch-x86_64.h | 8 +-
engines/aioring.c | 267 ++++++++++++++++++++++++++++++++---------------------
t/aio-ring.c | 262 +++++++++++++++++++++++++++++++---------------------
3 files changed, 322 insertions(+), 215 deletions(-)
---
Diff of recent changes:
diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index d49bcd7f..d0a98b8b 100644
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -1,11 +1,11 @@
#ifndef ARCH_X86_64_H
#define ARCH_X86_64_H
-#ifndef __NR_sys_io_setup2
-#define __NR_sys_io_setup2 335
+#ifndef __NR_sys_io_uring_setup
+#define __NR_sys_io_uring_setup 335
#endif
-#ifndef __NR_sys_io_ring_enter
-#define __NR_sys_io_ring_enter 336
+#ifndef __NR_sys_io_uring_enter
+#define __NR_sys_io_uring_enter 336
#endif
static inline void do_cpuid(unsigned int *eax, unsigned int *ebx,
diff --git a/engines/aioring.c b/engines/aioring.c
index f836009d..ca60b281 100644
--- a/engines/aioring.c
+++ b/engines/aioring.c
@@ -22,13 +22,13 @@
#ifdef ARCH_HAVE_AIORING
/*
- * io_setup2(2) flags
+ * io_uring_setup(2) flags
*/
-#ifndef IOCTX_FLAG_IOPOLL
-#define IOCTX_FLAG_IOPOLL (1 << 0)
-#endif
#ifndef IOCTX_FLAG_SCQRING
-#define IOCTX_FLAG_SCQRING (1 << 1)
+#define IOCTX_FLAG_SCQRING (1 << 0)
+#endif
+#ifndef IOCTX_FLAG_IOPOLL
+#define IOCTX_FLAG_IOPOLL (1 << 1)
#endif
#ifndef IOCTX_FLAG_FIXEDBUFS
#define IOCTX_FLAG_FIXEDBUFS (1 << 2)
@@ -43,12 +43,15 @@
#define IOCTX_FLAG_SQPOLL (1 << 5)
#endif
+#define IORING_OFF_SQ_RING 0ULL
+#define IORING_OFF_CQ_RING 0x8000000ULL
+#define IORING_OFF_IOCB 0x10000000ULL
/*
- * io_ring_enter(2) flags
+ * io_uring_enter(2) flags
*/
-#ifndef IORING_FLAG_GETEVENTS
-#define IORING_FLAG_GETEVENTS (1 << 0)
+#ifndef IORING_ENTER_GETEVENTS
+#define IORING_ENTER_GETEVENTS (1 << 0)
#endif
typedef uint64_t u64;
@@ -59,43 +62,68 @@ typedef uint16_t u16;
#define IOEV_RES2_CACHEHIT (1 << 0)
+struct aio_sqring_offsets {
+ u32 head;
+ u32 tail;
+ u32 ring_mask;
+ u32 ring_entries;
+ u32 flags;
+ u32 array;
+};
+
+struct aio_cqring_offsets {
+ u32 head;
+ u32 tail;
+ u32 ring_mask;
+ u32 ring_entries;
+ u32 overflow;
+ u32 events;
+};
+
+struct aio_uring_params {
+ u32 sq_entries;
+ u32 cq_entries;
+ u32 flags;
+ u16 sq_thread_cpu;
+ u16 resv[9];
+ struct aio_sqring_offsets sq_off;
+ struct aio_cqring_offsets cq_off;
+};
+
struct aio_sq_ring {
- union {
- struct {
- u32 head;
- u32 tail;
- u32 nr_events;
- u16 sq_thread_cpu;
- u16 kflags;
- u64 iocbs;
- };
- u32 pad[16];
- };
- u32 array[0];
+ u32 *head;
+ u32 *tail;
+ u32 *ring_mask;
+ u32 *ring_entries;
+ u32 *flags;
+ u32 *array;
};
struct aio_cq_ring {
- union {
- struct {
- u32 head;
- u32 tail;
- u32 nr_events;
- };
- struct io_event pad;
- };
- struct io_event events[0];
+ u32 *head;
+ u32 *tail;
+ u32 *ring_mask;
+ u32 *ring_entries;
+ struct io_event *events;
+};
+
+struct aioring_mmap {
+ void *ptr;
+ size_t len;
};
struct aioring_data {
- io_context_t aio_ctx;
+ int ring_fd;
+
struct io_u **io_us;
struct io_u **io_u_index;
- struct aio_sq_ring *sq_ring;
+ struct aio_sq_ring sq_ring;
struct iocb *iocbs;
+ struct iovec *iovecs;
unsigned sq_ring_mask;
- struct aio_cq_ring *cq_ring;
+ struct aio_cq_ring cq_ring;
struct io_event *events;
unsigned cq_ring_mask;
@@ -105,6 +133,8 @@ struct aioring_data {
uint64_t cachehit;
uint64_t cachemiss;
+
+ struct aioring_mmap mmap[3];
};
struct aioring_options {
@@ -178,11 +208,11 @@ static struct fio_option options[] = {
},
};
-static int io_ring_enter(io_context_t ctx, unsigned int to_submit,
+static int io_uring_enter(struct aioring_data *ld, unsigned int to_submit,
unsigned int min_complete, unsigned int flags)
{
- return syscall(__NR_sys_io_ring_enter, ctx, to_submit, min_complete,
- flags);
+ return syscall(__NR_sys_io_uring_enter, ld->ring_fd, to_submit,
+ min_complete, flags);
}
static int fio_aioring_prep(struct thread_data *td, struct io_u *io_u)
@@ -220,7 +250,7 @@ static struct io_u *fio_aioring_event(struct thread_data *td, int event)
index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
- ev = &ld->cq_ring->events[index];
+ ev = &ld->cq_ring.events[index];
io_u = ev->data;
if (ev->res != io_u->xfer_buflen) {
@@ -245,19 +275,19 @@ static int fio_aioring_cqring_reap(struct thread_data *td, unsigned int events,
unsigned int max)
{
struct aioring_data *ld = td->io_ops_data;
- struct aio_cq_ring *ring = ld->cq_ring;
+ struct aio_cq_ring *ring = &ld->cq_ring;
u32 head, reaped = 0;
- head = ring->head;
+ head = *ring->head;
do {
read_barrier();
- if (head == ring->tail)
+ if (head == *ring->tail)
break;
reaped++;
head++;
} while (reaped + events < max);
- ring->head = head;
+ *ring->head = head;
write_barrier();
return reaped;
}
@@ -268,11 +298,11 @@ static int fio_aioring_getevents(struct thread_data *td, unsigned int min,
struct aioring_data *ld = td->io_ops_data;
unsigned actual_min = td->o.iodepth_batch_complete_min == 0 ? 0 : min;
struct aioring_options *o = td->eo;
- struct aio_cq_ring *ring = ld->cq_ring;
+ struct aio_cq_ring *ring = &ld->cq_ring;
unsigned events = 0;
int r;
- ld->cq_ring_off = ring->head;
+ ld->cq_ring_off = *ring->head;
do {
r = fio_aioring_cqring_reap(td, events, max);
if (r) {
@@ -281,12 +311,12 @@ static int fio_aioring_getevents(struct thread_data *td, unsigned int min,
}
if (!o->sqthread_poll) {
- r = io_ring_enter(ld->aio_ctx, 0, actual_min,
- IORING_FLAG_GETEVENTS);
+ r = io_uring_enter(ld, 0, actual_min,
+ IORING_ENTER_GETEVENTS);
if (r < 0) {
if (errno == EAGAIN)
continue;
- td_verror(td, errno, "io_ring_enter get");
+ td_verror(td, errno, "io_uring_enter");
break;
}
}
@@ -299,7 +329,7 @@ static enum fio_q_status fio_aioring_queue(struct thread_data *td,
struct io_u *io_u)
{
struct aioring_data *ld = td->io_ops_data;
- struct aio_sq_ring *ring = ld->sq_ring;
+ struct aio_sq_ring *ring = &ld->sq_ring;
unsigned tail, next_tail;
fio_ro_check(td, io_u);
@@ -317,14 +347,14 @@ static enum fio_q_status fio_aioring_queue(struct thread_data *td,
return FIO_Q_COMPLETED;
}
- tail = ring->tail;
+ tail = *ring->tail;
next_tail = tail + 1;
read_barrier();
- if (next_tail == ring->head)
+ if (next_tail == *ring->head)
return FIO_Q_BUSY;
ring->array[tail & ld->sq_ring_mask] = io_u->index;
- ring->tail = next_tail;
+ *ring->tail = next_tail;
write_barrier();
ld->queued++;
@@ -342,7 +372,8 @@ static void fio_aioring_queued(struct thread_data *td, int start, int nr)
fio_gettime(&now, NULL);
while (nr--) {
- int index = ld->sq_ring->array[start & ld->sq_ring_mask];
+ struct aio_sq_ring *ring = &ld->sq_ring;
+ int index = ring->array[start & ld->sq_ring_mask];
struct io_u *io_u = ld->io_u_index[index];
memcpy(&io_u->issue_time, &now, sizeof(now));
@@ -363,19 +394,19 @@ static int fio_aioring_commit(struct thread_data *td)
/* Nothing to do */
if (o->sqthread_poll) {
- struct aio_sq_ring *ring = ld->sq_ring;
+ struct aio_sq_ring *ring = &ld->sq_ring;
- if (ring->kflags & IORING_SQ_NEED_WAKEUP)
- io_ring_enter(ld->aio_ctx, ld->queued, 0, 0);
+ if (*ring->flags & IORING_SQ_NEED_WAKEUP)
+ io_uring_enter(ld, ld->queued, 0, 0);
ld->queued = 0;
return 0;
}
do {
- unsigned start = ld->sq_ring->head;
+ unsigned start = *ld->sq_ring.head;
long nr = ld->queued;
- ret = io_ring_enter(ld->aio_ctx, nr, 0, IORING_FLAG_GETEVENTS);
+ ret = io_uring_enter(ld, nr, 0, IORING_ENTER_GETEVENTS);
if (ret > 0) {
fio_aioring_queued(td, start, ret);
io_u_mark_submit(td, ret);
@@ -394,7 +425,7 @@ static int fio_aioring_commit(struct thread_data *td)
usleep(1);
continue;
}
- td_verror(td, errno, "io_ring_enter sumit");
+ td_verror(td, errno, "io_uring_enter submit");
break;
}
} while (ld->queued);
@@ -402,24 +433,13 @@ static int fio_aioring_commit(struct thread_data *td)
return ret;
}
-static size_t aioring_cq_size(struct thread_data *td)
+static void fio_aioring_unmap(struct aioring_data *ld)
{
- return sizeof(struct aio_cq_ring) + 2 * td->o.iodepth * sizeof(struct io_event);
-}
+ int i;
-static size_t aioring_sq_iocb(struct thread_data *td)
-{
- return sizeof(struct iocb) * td->o.iodepth;
-}
-
-static size_t aioring_sq_size(struct thread_data *td)
-{
- return sizeof(struct aio_sq_ring) + td->o.iodepth * sizeof(u32);
-}
-
-static unsigned roundup_pow2(unsigned depth)
-{
- return 1UL << __fls(depth - 1);
+ for (i = 0; i < ARRAY_SIZE(ld->mmap); i++)
+ munmap(ld->mmap[i].ptr, ld->mmap[i].len);
+ close(ld->ring_fd);
}
static void fio_aioring_cleanup(struct thread_data *td)
@@ -437,33 +457,76 @@ static void fio_aioring_cleanup(struct thread_data *td)
* speeding it up a lot.
*/
if (!(td->flags & TD_F_CHILD))
- io_destroy(ld->aio_ctx);
+ fio_aioring_unmap(ld);
+
free(ld->io_u_index);
free(ld->io_us);
- fio_memfree(ld->sq_ring, aioring_sq_size(td), false);
- fio_memfree(ld->iocbs, aioring_sq_iocb(td), false);
- fio_memfree(ld->cq_ring, aioring_cq_size(td), false);
+ free(ld->iovecs);
free(ld);
}
}
+static int fio_aioring_mmap(struct aioring_data *ld, struct aio_uring_params *p)
+{
+ struct aio_sq_ring *sring = &ld->sq_ring;
+ struct aio_cq_ring *cring = &ld->cq_ring;
+ void *ptr;
+
+ ld->mmap[0].len = p->sq_off.array + p->sq_entries * sizeof(u32);
+ ptr = mmap(0, ld->mmap[0].len, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, ld->ring_fd,
+ IORING_OFF_SQ_RING);
+ ld->mmap[0].ptr = ptr;
+ sring->head = ptr + p->sq_off.head;
+ sring->tail = ptr + p->sq_off.tail;
+ sring->ring_mask = ptr + p->sq_off.ring_mask;
+ sring->ring_entries = ptr + p->sq_off.ring_entries;
+ sring->flags = ptr + p->sq_off.flags;
+ sring->array = ptr + p->sq_off.array;
+ ld->sq_ring_mask = *sring->ring_mask;
+
+ ld->mmap[1].len = p->sq_entries * sizeof(struct iocb);
+ ld->iocbs = mmap(0, ld->mmap[1].len, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, ld->ring_fd,
+ IORING_OFF_IOCB);
+ ld->mmap[1].ptr = ld->iocbs;
+
+ ld->mmap[2].len = p->cq_off.events +
+ p->cq_entries * sizeof(struct io_event);
+ ptr = mmap(0, ld->mmap[2].len, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, ld->ring_fd,
+ IORING_OFF_CQ_RING);
+ ld->mmap[2].ptr = ptr;
+ cring->head = ptr + p->cq_off.head;
+ cring->tail = ptr + p->cq_off.tail;
+ cring->ring_mask = ptr + p->cq_off.ring_mask;
+ cring->ring_entries = ptr + p->cq_off.ring_entries;
+ cring->events = ptr + p->cq_off.events;
+ ld->cq_ring_mask = *cring->ring_mask;
+ return 0;
+}
+
static int fio_aioring_queue_init(struct thread_data *td)
{
struct aioring_data *ld = td->io_ops_data;
struct aioring_options *o = td->eo;
- int flags = IOCTX_FLAG_SCQRING;
int depth = td->o.iodepth;
+ struct aio_uring_params p;
+ int ret;
+
+ memset(&p, 0, sizeof(p));
+ p.flags = IOCTX_FLAG_SCQRING;
if (o->hipri)
- flags |= IOCTX_FLAG_IOPOLL;
+ p.flags |= IOCTX_FLAG_IOPOLL;
if (o->sqthread_set) {
- ld->sq_ring->sq_thread_cpu = o->sqthread;
- flags |= IOCTX_FLAG_SQTHREAD;
+ p.sq_thread_cpu = o->sqthread;
+ p.flags |= IOCTX_FLAG_SQTHREAD;
if (o->sqthread_poll)
- flags |= IOCTX_FLAG_SQPOLL;
+ p.flags |= IOCTX_FLAG_SQPOLL;
}
if (o->sqwq)
- flags |= IOCTX_FLAG_SQWQ;
+ p.flags |= IOCTX_FLAG_SQWQ;
if (o->fixedbufs) {
struct rlimit rlim = {
@@ -472,11 +535,15 @@ static int fio_aioring_queue_init(struct thread_data *td)
};
setrlimit(RLIMIT_MEMLOCK, &rlim);
- flags |= IOCTX_FLAG_FIXEDBUFS;
+ p.flags |= IOCTX_FLAG_FIXEDBUFS;
}
- return syscall(__NR_sys_io_setup2, depth, flags,
- ld->sq_ring, ld->cq_ring, &ld->aio_ctx);
+ ret = syscall(__NR_sys_io_uring_setup, depth, ld->iovecs, &p);
+ if (ret < 0)
+ return ret;
+
+ ld->ring_fd = ret;
+ return fio_aioring_mmap(ld, &p);
}
static int fio_aioring_post_init(struct thread_data *td)
@@ -484,22 +551,21 @@ static int fio_aioring_post_init(struct thread_data *td)
struct aioring_data *ld = td->io_ops_data;
struct aioring_options *o = td->eo;
struct io_u *io_u;
- struct iocb *iocb;
- int err = 0;
+ int err;
if (o->fixedbufs) {
int i;
for (i = 0; i < td->o.iodepth; i++) {
+ struct iovec *iov = &ld->iovecs[i];
+
io_u = ld->io_u_index[i];
- iocb = &ld->iocbs[i];
- iocb->u.c.buf = io_u->buf;
- iocb->u.c.nbytes = td_max_bs(td);
+ iov->iov_base = io_u->buf;
+ iov->iov_len = td_max_bs(td);
}
}
err = fio_aioring_queue_init(td);
-
if (err) {
td_verror(td, errno, "io_queue_init");
return 1;
@@ -508,6 +574,11 @@ static int fio_aioring_post_init(struct thread_data *td)
return 0;
}
+static unsigned roundup_pow2(unsigned depth)
+{
+ return 1UL << __fls(depth - 1);
+}
+
static int fio_aioring_init(struct thread_data *td)
{
struct aioring_data *ld;
@@ -522,19 +593,7 @@ static int fio_aioring_init(struct thread_data *td)
ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
- ld->iocbs = fio_memalign(page_size, aioring_sq_iocb(td), false);
- memset(ld->iocbs, 0, aioring_sq_iocb(td));
-
- ld->sq_ring = fio_memalign(page_size, aioring_sq_size(td), false);
- memset(ld->sq_ring, 0, aioring_sq_size(td));
- ld->sq_ring->nr_events = td->o.iodepth;
- ld->sq_ring->iocbs = (u64) (uintptr_t) ld->iocbs;
- ld->sq_ring_mask = td->o.iodepth - 1;
-
- ld->cq_ring = fio_memalign(page_size, aioring_cq_size(td), false);
- memset(ld->cq_ring, 0, aioring_cq_size(td));
- ld->cq_ring->nr_events = td->o.iodepth * 2;
- ld->cq_ring_mask = (2 * td->o.iodepth) - 1;
+ ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
td->io_ops_data = ld;
return 0;
diff --git a/t/aio-ring.c b/t/aio-ring.c
index c0c5009e..71978c68 100644
--- a/t/aio-ring.c
+++ b/t/aio-ring.c
@@ -14,6 +14,7 @@
#include <sys/ioctl.h>
#include <sys/syscall.h>
#include <sys/resource.h>
+#include <sys/mman.h>
#include <linux/fs.h>
#include <fcntl.h>
#include <unistd.h>
@@ -22,11 +23,14 @@
#include <pthread.h>
#include <sched.h>
-#define IOCTX_FLAG_IOPOLL (1 << 0)
-#define IOCTX_FLAG_SCQRING (1 << 1) /* Use SQ/CQ rings */
+#include "../arch/arch.h"
+
+#define IOCTX_FLAG_SCQRING (1 << 0) /* Use SQ/CQ rings */
+#define IOCTX_FLAG_IOPOLL (1 << 1)
#define IOCTX_FLAG_FIXEDBUFS (1 << 2)
#define IOCTX_FLAG_SQTHREAD (1 << 3) /* Use SQ thread */
#define IOCTX_FLAG_SQWQ (1 << 4) /* Use SQ wq */
+#define IOCTX_FLAG_SQPOLL (1 << 5)
#define IOEV_RES2_CACHEHIT (1 << 0)
@@ -38,33 +42,55 @@ typedef uint64_t u64;
typedef uint32_t u32;
typedef uint16_t u16;
+#define IORING_OFF_SQ_RING 0ULL
+#define IORING_OFF_CQ_RING 0x8000000ULL
+#define IORING_OFF_IOCB 0x10000000ULL
+
+struct aio_sqring_offsets {
+ u32 head;
+ u32 tail;
+ u32 ring_mask;
+ u32 ring_entries;
+ u32 flags;
+ u32 array;
+};
+
+struct aio_cqring_offsets {
+ u32 head;
+ u32 tail;
+ u32 ring_mask;
+ u32 ring_entries;
+ u32 overflow;
+ u32 events;
+};
+
+struct aio_uring_params {
+ u32 sq_entries;
+ u32 cq_entries;
+ u32 flags;
+ u16 sq_thread_cpu;
+ u16 resv[9];
+ struct aio_sqring_offsets sq_off;
+ struct aio_cqring_offsets cq_off;
+};
+
struct aio_sq_ring {
- union {
- struct {
- u32 head;
- u32 tail;
- u32 nr_events;
- u16 sq_thread_cpu;
- u64 iocbs;
- };
- u32 pad[16];
- };
- u32 array[0];
+ u32 *head;
+ u32 *tail;
+ u32 *ring_mask;
+ u32 *ring_entries;
+ u32 *array;
};
struct aio_cq_ring {
- union {
- struct {
- u32 head;
- u32 tail;
- u32 nr_events;
- };
- struct io_event pad;
- };
- struct io_event events[0];
+ u32 *head;
+ u32 *tail;
+ u32 *ring_mask;
+ u32 *ring_entries;
+ struct io_event *events;
};
-#define IORING_FLAG_GETEVENTS (1 << 0)
+#define IORING_ENTER_GETEVENTS (1 << 0)
#define DEPTH 32
@@ -73,17 +99,17 @@ struct aio_cq_ring {
#define BS 4096
-static unsigned sq_ring_mask = DEPTH - 1;
-static unsigned cq_ring_mask = (2 * DEPTH) - 1;
+static unsigned sq_ring_mask, cq_ring_mask;
struct submitter {
pthread_t thread;
unsigned long max_blocks;
- io_context_t ioc;
+ int fd;
struct drand48_data rand;
- struct aio_sq_ring *sq_ring;
+ struct aio_sq_ring sq_ring;
struct iocb *iocbs;
- struct aio_cq_ring *cq_ring;
+ struct iovec iovecs[DEPTH];
+ struct aio_cq_ring cq_ring;
int inflight;
unsigned long reaps;
unsigned long done;
@@ -96,23 +122,23 @@ struct submitter {
static struct submitter submitters[1];
static volatile int finish;
-static int polled = 1; /* use IO polling */
-static int fixedbufs = 1; /* use fixed user buffers */
-static int buffered = 0; /* use buffered IO, not O_DIRECT */
+static int polled = 0; /* use IO polling */
+static int fixedbufs = 0; /* use fixed user buffers */
+static int buffered = 1; /* use buffered IO, not O_DIRECT */
static int sq_thread = 0; /* use kernel submission thread */
static int sq_thread_cpu = 0; /* pin above thread to this CPU */
-static int io_setup2(unsigned int nr_events, unsigned int flags,
- struct aio_sq_ring *sq_ring, struct aio_cq_ring *cq_ring,
- io_context_t *ctx_idp)
+static int io_uring_setup(unsigned entries, struct iovec *iovecs,
+ struct aio_uring_params *p)
{
- return syscall(335, nr_events, flags, sq_ring, cq_ring, ctx_idp);
+ return syscall(__NR_sys_io_uring_setup, entries, iovecs, p);
}
-static int io_ring_enter(io_context_t ctx, unsigned int to_submit,
- unsigned int min_complete, unsigned int flags)
+static int io_uring_enter(struct submitter *s, unsigned int to_submit,
+ unsigned int min_complete, unsigned int flags)
{
- return syscall(336, ctx, to_submit, min_complete, flags);
+ return syscall(__NR_sys_io_uring_enter, s->fd, to_submit, min_complete,
+ flags);
}
static int gettid(void)
@@ -120,8 +146,9 @@ static int gettid(void)
return syscall(__NR_gettid);
}
-static void init_io(struct submitter *s, int fd, struct iocb *iocb)
+static void init_io(struct submitter *s, int fd, unsigned index)
{
+ struct iocb *iocb = &s->iocbs[index];
unsigned long offset;
long r;
@@ -130,34 +157,34 @@ static void init_io(struct submitter *s, int fd, struct iocb *iocb)
iocb->aio_fildes = fd;
iocb->aio_lio_opcode = IO_CMD_PREAD;
+ iocb->u.c.buf = s->iovecs[index].iov_base;
+ iocb->u.c.nbytes = BS;
iocb->u.c.offset = offset;
- if (!fixedbufs)
- iocb->u.c.nbytes = BS;
}
static int prep_more_ios(struct submitter *s, int fd, int max_ios)
{
- struct aio_sq_ring *ring = s->sq_ring;
+ struct aio_sq_ring *ring = &s->sq_ring;
u32 index, tail, next_tail, prepped = 0;
- next_tail = tail = ring->tail;
+ next_tail = tail = *ring->tail;
do {
next_tail++;
barrier();
- if (next_tail == ring->head)
+ if (next_tail == *ring->head)
break;
index = tail & sq_ring_mask;
- init_io(s, fd, &s->iocbs[index]);
- s->sq_ring->array[index] = index;
+ init_io(s, fd, index);
+ ring->array[index] = index;
prepped++;
tail = next_tail;
} while (prepped < max_ios);
- if (ring->tail != tail) {
+ if (*ring->tail != tail) {
/* order tail store with writes to iocbs above */
barrier();
- ring->tail = tail;
+ *ring->tail = tail;
barrier();
}
return prepped;
@@ -187,14 +214,14 @@ static int get_file_size(int fd, unsigned long *blocks)
static int reap_events(struct submitter *s)
{
- struct aio_cq_ring *ring = s->cq_ring;
+ struct aio_cq_ring *ring = &s->cq_ring;
struct io_event *ev;
u32 head, reaped = 0;
- head = ring->head;
+ head = *ring->head;
do {
barrier();
- if (head == ring->tail)
+ if (head == *ring->tail)
break;
ev = &ring->events[head & cq_ring_mask];
if (ev->res != BS) {
@@ -213,7 +240,7 @@ static int reap_events(struct submitter *s)
} while (1);
s->inflight -= reaped;
- ring->head = head;
+ *ring->head = head;
barrier();
return reaped;
}
@@ -262,8 +289,7 @@ submit:
else
to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
- ret = io_ring_enter(s->ioc, to_submit, to_wait,
- IORING_FLAG_GETEVENTS);
+ ret = io_uring_enter(s, to_submit, to_wait, IORING_ENTER_GETEVENTS);
s->calls++;
this_reap = reap_events(s);
@@ -288,7 +314,7 @@ submit:
prepped = 0;
continue;
} else if (ret < 0) {
- if ((ret == -1 && errno == EAGAIN) || ret == -EAGAIN) {
+ if (errno == EAGAIN) {
if (s->finish)
break;
if (this_reap)
@@ -296,10 +322,7 @@ submit:
to_submit = 0;
goto submit;
}
- if (ret == -1)
- printf("io_submit: %s\n", strerror(errno));
- else
- printf("io_submit: %s\n", strerror(-ret));
+ printf("io_submit: %s\n", strerror(errno));
break;
}
} while (!s->finish);
@@ -327,15 +350,74 @@ static void arm_sig_int(void)
sigaction(SIGINT, &act, NULL);
}
+static int setup_ring(struct submitter *s)
+{
+ struct aio_sq_ring *sring = &s->sq_ring;
+ struct aio_cq_ring *cring = &s->cq_ring;
+ struct aio_uring_params p;
+ void *ptr;
+ int fd;
+
+ memset(&p, 0, sizeof(p));
+
+ p.flags = IOCTX_FLAG_SCQRING;
+ if (polled)
+ p.flags |= IOCTX_FLAG_IOPOLL;
+ if (fixedbufs)
+ p.flags |= IOCTX_FLAG_FIXEDBUFS;
+ if (buffered)
+ p.flags |= IOCTX_FLAG_SQWQ;
+ else if (sq_thread) {
+ p.flags |= IOCTX_FLAG_SQTHREAD;
+ p.sq_thread_cpu = sq_thread_cpu;
+ }
+
+ if (fixedbufs)
+ fd = io_uring_setup(DEPTH, s->iovecs, &p);
+ else
+ fd = io_uring_setup(DEPTH, NULL, &p);
+ if (fd < 0) {
+ perror("io_uring_setup");
+ return 1;
+ }
+
+ s->fd = fd;
+
+ ptr = mmap(0, p.sq_off.array + p.sq_entries * sizeof(u32),
+ PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+ fd, IORING_OFF_SQ_RING);
+ printf("sq_ring ptr = 0x%p\n", ptr);
+ sring->head = ptr + p.sq_off.head;
+ sring->tail = ptr + p.sq_off.tail;
+ sring->ring_mask = ptr + p.sq_off.ring_mask;
+ sring->ring_entries = ptr + p.sq_off.ring_entries;
+ sring->array = ptr + p.sq_off.array;
+ sq_ring_mask = *sring->ring_mask;
+
+ s->iocbs = mmap(0, p.sq_entries * sizeof(struct iocb), PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_IOCB);
+ printf("iocbs ptr = 0x%p\n", s->iocbs);
+
+ ptr = mmap(0, p.cq_off.events + p.cq_entries * sizeof(struct io_event),
+ PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
+ fd, IORING_OFF_CQ_RING);
+ printf("cq_ring ptr = 0x%p\n", ptr);
+ cring->head = ptr + p.cq_off.head;
+ cring->tail = ptr + p.cq_off.tail;
+ cring->ring_mask = ptr + p.cq_off.ring_mask;
+ cring->ring_entries = ptr + p.cq_off.ring_entries;
+ cring->events = ptr + p.cq_off.events;
+ cq_ring_mask = *cring->ring_mask;
+ return 0;
+}
+
int main(int argc, char *argv[])
{
struct submitter *s = &submitters[0];
unsigned long done, calls, reap, cache_hit, cache_miss;
- int flags = 0, err;
- int j;
- size_t size;
- void *p, *ret;
+ int err, i;
struct rlimit rlim;
+ void *ret;
if (argc < 2) {
printf("%s: filename\n", argv[0]);
@@ -351,58 +433,24 @@ int main(int argc, char *argv[])
arm_sig_int();
- size = sizeof(struct iocb) * DEPTH;
- if (posix_memalign(&p, 4096, size))
- return 1;
- memset(p, 0, size);
- s->iocbs = p;
+ for (i = 0; i < DEPTH; i++) {
+ void *buf;
- size = sizeof(struct aio_sq_ring) + DEPTH * sizeof(u32);
- if (posix_memalign(&p, 4096, size))
- return 1;
- s->sq_ring = p;
- memset(p, 0, size);
- s->sq_ring->nr_events = DEPTH;
- s->sq_ring->iocbs = (u64) s->iocbs;
-
- /* CQ ring must be twice as big */
- size = sizeof(struct aio_cq_ring) +
- 2 * DEPTH * sizeof(struct io_event);
- if (posix_memalign(&p, 4096, size))
- return 1;
- s->cq_ring = p;
- memset(p, 0, size);
- s->cq_ring->nr_events = 2 * DEPTH;
-
- for (j = 0; j < DEPTH; j++) {
- struct iocb *iocb = &s->iocbs[j];
-
- if (posix_memalign(&iocb->u.c.buf, BS, BS)) {
+ if (posix_memalign(&buf, BS, BS)) {
printf("failed alloc\n");
return 1;
}
- iocb->u.c.nbytes = BS;
- }
-
- flags = IOCTX_FLAG_SCQRING;
- if (polled)
- flags |= IOCTX_FLAG_IOPOLL;
- if (fixedbufs)
- flags |= IOCTX_FLAG_FIXEDBUFS;
- if (buffered)
- flags |= IOCTX_FLAG_SQWQ;
- else if (sq_thread) {
- flags |= IOCTX_FLAG_SQTHREAD;
- s->sq_ring->sq_thread_cpu = sq_thread_cpu;
+ s->iovecs[i].iov_base = buf;
+ s->iovecs[i].iov_len = BS;
}
- err = io_setup2(DEPTH, flags, s->sq_ring, s->cq_ring, &s->ioc);
+ err = setup_ring(s);
if (err) {
- printf("ctx_init failed: %s, %d\n", strerror(errno), err);
+ printf("ring setup failed: %s, %d\n", strerror(errno), err);
return 1;
}
- printf("polled=%d, fixedbufs=%d, buffered=%d\n", polled, fixedbufs, buffered);
- printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, s->sq_ring->nr_events, s->cq_ring->nr_events);
+ printf("polled=%d, fixedbufs=%d, buffered=%d", polled, fixedbufs, buffered);
+ printf(" QD=%d, sq_ring=%d, cq_ring=%d\n", DEPTH, *s->sq_ring.ring_entries, *s->cq_ring.ring_entries);
strcpy(s->filename, argv[1]);
pthread_create(&s->thread, NULL, submitter_fn, s);
@@ -437,7 +485,7 @@ int main(int argc, char *argv[])
}
printf("IOPS=%lu, IOS/call=%lu/%lu, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
this_done - done, rpc, ipc, s->inflight,
- s->cq_ring->head, s->cq_ring->tail, hit);
+ *s->cq_ring.head, *s->cq_ring.tail, hit);
done = this_done;
calls = this_call;
reap = this_reap;
