openat ignores changes to RLIMIT_NOFILE?

[Dmitry Kadashev <dkadashev@xxxxxxxxx>]

Hi,

It seems that openat calls issued via io_uring ignore changes to
RLIMIT_NOFILE. Maybe a wrong limit is checked. A short reproducer is
attached, it sets RLIMIT_NOFILE to a very low value and the sync
openat() call fails with "Too many open files", but io_uring one
succeeds. The resulting FD is completely usable, I've tried writing to
it successfully.

To be clear, originally I've encountered another side of this problem:
we increase the limit in our code, and io_uring's openat started to
fail after a while under load, while the sync calls executed on a
thread pool were working as expected. It's just easier to demo with
small limit.

Kernel 5.6-rc2, 5.6-rc6.

Hope it's the right place to report an issue like this.

Thanks.

-- 
Dmitry

#include <liburing.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/resource.h>
#include <unistd.h>

#define DIE(...) do {\
		fprintf(stderr, __VA_ARGS__);\
		abort();\
	} while(0);

static const int RSIZE = 2;
static const int OPEN_FLAGS = O_RDWR | O_CREAT;
static const mode_t OPEN_MODE = S_IRUSR | S_IWUSR;

void setup_rlimit()
{
	struct rlimit rlim;
	rlim.rlim_cur = rlim.rlim_max = 5; // 3 stdio ones, 1 for uring, 1 for dirfd
	if (setrlimit(RLIMIT_NOFILE, &rlim) == -1) {
		DIE("setrlimit nofile: %s\n", strerror(errno));
	}
}

void open_sync(int dfd, const char* fn)
{
	int fd = openat(dfd, fn, OPEN_FLAGS, OPEN_MODE);
	if (fd < 0) {
		fprintf(stderr, "sync open failed: %s\n", strerror(errno));
	}
	else {
		fprintf(stderr, "sync open succeeded\n");
		close(fd);
	}
}

void open_io_uring(struct io_uring *ring, int dfd, const char* fn)
{
	struct io_uring_sqe *sqe;
	sqe = io_uring_get_sqe(ring);
	if (!sqe) {
		fprintf(stderr, "failed to get sqe\n");
		return;
	}
	io_uring_prep_openat(sqe, dfd, fn, OPEN_FLAGS, OPEN_MODE);
	int ret = io_uring_submit(ring);
	if (ret < 0) {
		fprintf(stderr, "failed to submit openat: %s\n", strerror(-ret));
		return;
	}

	struct io_uring_cqe *cqe;
	ret = io_uring_wait_cqe(ring, &cqe);
	int fd = cqe->res;
	io_uring_cqe_seen(ring, cqe);
	if (ret < 0) {
		fprintf(stderr, "wait_cqe failed: %s\n", strerror(-ret));
	}
	else if (fd < 0) {
		fprintf(stderr, "io_uring openat failed: %s\n", strerror(-fd));
	}
	else {
		fprintf(stderr, "io_uring openat succeeded\n");
		close(fd);
	}
}

int main(int argc, const char *argv[])
{
	const char *mode = "io_uring";
	const char *fn = "io_uring_openat_test";
	setup_rlimit();
	int dfd = open("/tmp", O_RDONLY | O_DIRECTORY);
	if (dfd < 0) {
		DIE("open /tmp: %s\n", strerror(errno));
	}
	struct io_uring ring;
	int ret = io_uring_queue_init(RSIZE, &ring, 0);
	if (ret < 0) {
		DIE("failed to init io_uring: %s\n", strerror(-ret));
	}

	open_sync(dfd, fn);
	open_io_uring(&ring, dfd, fn);

	io_uring_queue_exit(&ring);
	close(dfd);
	return 0;
}