Re: io_uring_prep_openat_direct() and link/drain

[Miklos Szeredi <miklos@xxxxxxxxxx>]

Next issue:  seems like file slot reuse is not working correctly.
Attached program compares reads using io_uring with plain reads of
proc files.

In the below example it is using two slots alternately but the number
of slots does not seem to matter, read is apparently always using a
stale file (the prior one to the most recent open on that slot).  See
how the sizes of the files lag by two lines:

root@kvm:~# ./procreads
procreads: /proc/1/stat: ok (313)
procreads: /proc/2/stat: ok (149)
procreads: /proc/3/stat: read size mismatch 313/150
procreads: /proc/4/stat: read size mismatch 149/154
procreads: /proc/5/stat: read size mismatch 150/161
procreads: /proc/6/stat: read size mismatch 154/171
...

Any ideas?

Thanks,
Miklos

#include <stdio.h>
#include <fcntl.h>
#include <string.h>
#include <stdlib.h>
#include <dirent.h>
#include <unistd.h>
#include <err.h>
#include "liburing.h"

#define CHECK_NEGERR(_expr) \
	({ typeof(_expr) _ret = (_expr); if (_ret < 0) { errno = -_ret; err(1, #_expr); } _ret; })
#define CHECK_NULL(_expr) \
	({ typeof(_expr) _ret = (_expr); if (_ret == NULL) { errx(1, #_expr " returned NULL"); } _ret; })

ssize_t readfile_uring(struct io_uring *ring, int slot,
		       const char *path, char *buf, size_t size)
{
	struct io_uring_sqe *sqe;
	struct io_uring_cqe *cqe;
	int ret, i;
	unsigned int sum = 0;

	for (i = 0; path[i]; i++)
		sum += path[i];

	sqe = io_uring_get_sqe(ring);
	io_uring_prep_openat_direct(sqe, AT_FDCWD, path, O_RDONLY, 0, slot);
	sqe->flags = IOSQE_IO_LINK | IOSQE_CQE_SKIP_SUCCESS;
	sqe->user_data = 0xdead0000 + sum;

	sqe = io_uring_get_sqe(ring);
	io_uring_prep_read(sqe, slot, buf, size, 0);
	sqe->flags = IOSQE_FIXED_FILE;
	sqe->user_data = 0xfeed0000 + sum;

	ret = CHECK_NEGERR(io_uring_submit_and_wait(ring, 1));
	if (ret < 2)
		warnx("short submit count: %i", ret);

	ret = CHECK_NEGERR(io_uring_wait_cqe(ring, &cqe));

	if ((cqe->user_data & 0xffff) != sum)
		warnx("wrong sum: %x (should be %x)",(unsigned int) cqe->user_data, sum);
	if (cqe->res >= 0 && (cqe->user_data & 0xffff0000) != 0xfeed0000)
		warnx("not skipped: %x", (unsigned int) cqe->user_data);

	ret = cqe->res;
	io_uring_cqe_seen(ring, cqe);
	if (ret < 0) {
		errno = -ret;
		warn("failed to open or read %s", path);
	}

	return ret;
}

static ssize_t readfile_plain(const char *path, char *buf, size_t size)
{
	int fd;
	ssize_t ret;

	fd = open(path, O_RDONLY);
	if (fd == -1)
		return -errno;

	ret = read(fd, buf, size);
	if (ret == -1)
		return -errno;

	close(fd);

	return ret;
}

int main(void)
{
	int fds[] = { -1, -1 };
	struct io_uring ring;
	char *name, path[4096], buf1[4096], buf2[4096];
	DIR *dp;
	struct dirent *de;
	ssize_t ret1, ret2;
	int slot = 0;
	unsigned int numslots = sizeof(fds)/sizeof(fds[0]);

	CHECK_NEGERR(io_uring_queue_init(32, &ring, 0));
	CHECK_NEGERR(io_uring_register_files(&ring, fds, numslots));

	dp = CHECK_NULL(opendir("/proc"));
	while ((de = readdir(dp))) {
		name = de->d_name;
		if (name[0] > '0' && name[0] <= '9') {
			sprintf(path, "/proc/%s/stat", name);
			ret1 = readfile_uring(&ring, slot, path, buf1, sizeof(buf1));
			ret2 = readfile_plain(path, buf2, sizeof(buf2));
			if (ret1 != ret2)
				warnx("%s: read size mismatch %zi/%zi",
				      path, ret1, ret2);
			else if (ret1 > 0 && memcmp(buf1, buf2, ret1))
				warnx("%s: data mismatch", path);
			else {
				warnx("%s: ok (%zi)", path, ret1);
			}

			slot = (slot + 1) % numslots;
		}
	}
	closedir(dp);
	return 0;
}