On 11/11/24 8:17 AM, Jens Axboe wrote:
> On 11/11/24 8:16 AM, Christoph Hellwig wrote:
>> On Mon, Nov 11, 2024 at 07:12:35AM -0700, Jens Axboe wrote:
>>> Ok thanks, let me take a look at that and create a test case that
>>> exercises that explicitly.
>>
>> Please add RWF_UNCACHED to fsstress.c in xfstests also. That is our
>> exerciser for concurrent issuing of different I/O types to hit these
>> kinds of corner cases.
>
> Sure, can do.
Not familiar with fsstress at all, but something like the below? Will
use it if available, if it gets EOPNOTSUPP it'll just fallback to
using writev_f()/readv_f() instead.
Did give it a quick test spin and I see uncached reads and writes on the
kernel that supports it.
diff --git a/ltp/fsstress.c b/ltp/fsstress.c
index 3d248ee25791..6430f10efbc7 100644
--- a/ltp/fsstress.c
+++ b/ltp/fsstress.c
@@ -82,6 +82,12 @@ static int renameat2(int dfd1, const char *path1,
#define RENAME_WHITEOUT (1 << 2) /* Whiteout source */
#endif
+#ifndef RWF_UNCACHED
+#define RWF_UNCACHED 0x80
+#endif
+
+static int have_rwf_uncached = 1;
+
#define FILELEN_MAX (32*4096)
typedef enum {
@@ -117,6 +123,7 @@ typedef enum {
OP_COLLAPSE,
OP_INSERT,
OP_READ,
+ OP_READ_UNCACHED,
OP_READLINK,
OP_READV,
OP_REMOVEFATTR,
@@ -143,6 +150,7 @@ typedef enum {
OP_URING_READ,
OP_URING_WRITE,
OP_WRITE,
+ OP_WRITE_UNCACHED,
OP_WRITEV,
OP_EXCHANGE_RANGE,
OP_LAST
@@ -248,6 +256,7 @@ void zero_f(opnum_t, long);
void collapse_f(opnum_t, long);
void insert_f(opnum_t, long);
void unshare_f(opnum_t, long);
+void read_uncached_f(opnum_t, long);
void read_f(opnum_t, long);
void readlink_f(opnum_t, long);
void readv_f(opnum_t, long);
@@ -273,6 +282,7 @@ void unlink_f(opnum_t, long);
void unresvsp_f(opnum_t, long);
void uring_read_f(opnum_t, long);
void uring_write_f(opnum_t, long);
+void write_uncached_f(opnum_t, long);
void write_f(opnum_t, long);
void writev_f(opnum_t, long);
void exchangerange_f(opnum_t, long);
@@ -315,6 +325,7 @@ struct opdesc ops[OP_LAST] = {
[OP_COLLAPSE] = {"collapse", collapse_f, 1, 1 },
[OP_INSERT] = {"insert", insert_f, 1, 1 },
[OP_READ] = {"read", read_f, 1, 0 },
+ [OP_READ_UNCACHED] = {"read_uncached", read_uncached_f, 1, 0 },
[OP_READLINK] = {"readlink", readlink_f, 1, 0 },
[OP_READV] = {"readv", readv_f, 1, 0 },
/* remove (delete) extended attribute */
@@ -346,6 +357,7 @@ struct opdesc ops[OP_LAST] = {
[OP_URING_WRITE] = {"uring_write", uring_write_f, 1, 1 },
[OP_WRITE] = {"write", write_f, 4, 1 },
[OP_WRITEV] = {"writev", writev_f, 4, 1 },
+ [OP_WRITE_UNCACHED]= {"write_uncaced", write_uncached_f,4, 1 },
[OP_EXCHANGE_RANGE]= {"exchangerange", exchangerange_f, 2, 1 },
}, *ops_end;
@@ -4635,6 +4647,76 @@ readv_f(opnum_t opno, long r)
close(fd);
}
+void
+read_uncached_f(opnum_t opno, long r)
+{
+ int e;
+ pathname_t f;
+ int fd;
+ int64_t lr;
+ off64_t off;
+ struct stat64 stb;
+ int v;
+ char st[1024];
+ struct iovec iov;
+
+ if (!have_rwf_uncached) {
+ readv_f(opno, r);
+ return;
+ }
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%lld: read - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, O_RDONLY);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%lld: read - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+ if (fstat64(fd, &stb) < 0) {
+ if (v)
+ printf("%d/%lld: read - fstat64 %s failed %d\n",
+ procid, opno, f.path, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ inode_info(st, sizeof(st), &stb, v);
+ if (stb.st_size == 0) {
+ if (v)
+ printf("%d/%lld: read - %s%s zero size\n", procid, opno,
+ f.path, st);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ lr = ((int64_t)random() << 32) + random();
+ off = (off64_t)(lr % stb.st_size);
+ iov.iov_len = (random() % FILELEN_MAX) + 1;
+ iov.iov_base = malloc(iov.iov_len);
+ e = preadv2(fd, &iov, 1, off, RWF_UNCACHED) < 0 ? errno : 0;
+ if (e == EOPNOTSUPP) {
+ have_rwf_uncached = 0;
+ e = 0;
+ }
+ free(iov.iov_base);
+ if (v)
+ printf("%d/%lld: read uncached %s%s [%lld,%d] %d\n",
+ procid, opno, f.path, st, (long long)off,
+ (int)iov.iov_len, e);
+ free_pathname(&f);
+ close(fd);
+}
+
void
removefattr_f(opnum_t opno, long r)
{
@@ -5509,6 +5591,70 @@ writev_f(opnum_t opno, long r)
close(fd);
}
+void
+write_uncached_f(opnum_t opno, long r)
+{
+ int e;
+ pathname_t f;
+ int fd;
+ int64_t lr;
+ off64_t off;
+ struct stat64 stb;
+ int v;
+ char st[1024];
+ struct iovec iov;
+
+ if (!have_rwf_uncached) {
+ writev_f(opno, r);
+ return;
+ }
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGm, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%lld: write - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, O_WRONLY);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%lld: write - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+ if (fstat64(fd, &stb) < 0) {
+ if (v)
+ printf("%d/%lld: write - fstat64 %s failed %d\n",
+ procid, opno, f.path, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ inode_info(st, sizeof(st), &stb, v);
+ lr = ((int64_t)random() << 32) + random();
+ off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
+ off %= maxfsize;
+ iov.iov_len = (random() % FILELEN_MAX) + 1;
+ iov.iov_base = malloc(iov.iov_len);
+ memset(iov.iov_base, nameseq & 0xff, iov.iov_len);
+ e = pwritev2(fd, &iov, 1, off, RWF_UNCACHED) < 0 ? errno : 0;
+ free(iov.iov_base);
+ if (v)
+ printf("%d/%lld: write uncached %s%s [%lld,%d] %d\n",
+ procid, opno, f.path, st, (long long)off,
+ (int)iov.iov_len, e);
+ free_pathname(&f);
+ close(fd);
+ if (e == EOPNOTSUPP) {
+ writev_f(opno, r);
+ have_rwf_uncached = 0;
+ }
+}
+
char *
xattr_flag_to_string(int flag)
{
--
Jens Axboe
