Make swap_readpage(), when accessing a swap file (SWP_FS_OPS) use
the ->direct_IO() method on the filesystem rather then ->readpage().
Suggested-by: Matthew Wilcox (Oracle) <willy@xxxxxxxxxxxxx>
Signed-off-by: David Howells <dhowells@xxxxxxxxxx>
---
include/linux/fs.h | 1
mm/page_io.c | 115 +++++++++++++++++++++++++++++++++++++++++++++++++---
2 files changed, 110 insertions(+), 6 deletions(-)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index b3e6a20f28ef..94c47b9b5b1c 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -336,6 +336,7 @@ struct kiocb {
union {
unsigned int ki_cookie; /* for ->iopoll */
struct wait_page_queue *ki_waitq; /* for async buffered IO */
+ struct page *ki_swap_page; /* For swapfile_read/write */
};
randomized_struct_fields_end
diff --git a/mm/page_io.c b/mm/page_io.c
index 62cabcdfcec6..92ec4a7b0545 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -26,6 +26,24 @@
#include <linux/uio.h>
#include <linux/sched/task.h>
+/*
+ * Keep track of the kiocb we're using to do async DIO. We have to
+ * refcount it until various things stop looking at the kiocb *after*
+ * calling ->ki_complete().
+ */
+struct swapfile_kiocb {
+ struct kiocb iocb;
+ refcount_t ki_refcnt;
+};
+
+static void swapfile_put_kiocb(struct swapfile_kiocb *ki)
+{
+ if (refcount_dec_and_test(&ki->ki_refcnt)) {
+ fput(ki->iocb.ki_filp);
+ kfree(ki);
+ }
+}
+
static void end_swap_bio_write(struct bio *bio)
{
struct page *page = bio_first_page_all(bio);
@@ -353,6 +371,96 @@ int __swap_writepage(struct page *page, struct writeback_control *wbc)
return 0;
}
+static void __swapfile_read_complete(struct kiocb *iocb, long ret, long ret2)
+{
+ struct page *page = iocb->ki_swap_page;
+
+ if (ret == PAGE_SIZE) {
+ count_vm_event(PSWPIN);
+ SetPageUptodate(page);
+ } else {
+ SetPageError(page);
+ pr_err_ratelimited("Read error (%ld) on dio swapfile (%llu)\n",
+ ret, page_file_offset(page));
+ }
+
+ unlock_page(page);
+}
+
+static void swapfile_read_complete(struct kiocb *iocb, long ret, long ret2)
+{
+ struct swapfile_kiocb *ki = container_of(iocb, struct swapfile_kiocb, iocb);
+
+ __swapfile_read_complete(iocb, ret, ret2);
+ swapfile_put_kiocb(ki);
+}
+
+static int swapfile_read_sync(struct swap_info_struct *sis, struct page *page)
+{
+ struct kiocb kiocb;
+ struct file *swap_file = sis->swap_file;
+ struct bio_vec bv = {
+ .bv_page = page,
+ .bv_len = thp_size(page),
+ .bv_offset = 0
+ };
+ struct iov_iter to;
+ int ret;
+
+ init_sync_kiocb(&kiocb, swap_file);
+ kiocb.ki_swap_page = page;
+ kiocb.ki_pos = page_file_offset(page);
+ kiocb.ki_filp = swap_file;
+ kiocb.ki_flags = IOCB_DIRECT | IOCB_SWAP;
+ /* Should set IOCB_HIPRI too, but the box becomes unresponsive whilst
+ * putting out occasional messages about the NFS sunrpc scheduling
+ * tasks being hung.
+ */
+
+ iov_iter_bvec(&to, READ, &bv, 1, thp_size(page));
+ ret = swap_file->f_mapping->a_ops->direct_IO(&kiocb, &to);
+
+ __swapfile_read_complete(&kiocb, ret, 0);
+ return (ret > 0) ? 0 : ret;
+}
+
+static int swapfile_read(struct swap_info_struct *sis, struct page *page,
+ bool synchronous)
+{
+ struct swapfile_kiocb *ki;
+ struct file *swap_file = sis->swap_file;
+ struct bio_vec bv = {
+ .bv_page = page,
+ .bv_len = thp_size(page),
+ .bv_offset = 0
+ };
+ struct iov_iter to;
+ int ret;
+
+ if (synchronous)
+ return swapfile_read_sync(sis, page);
+
+ ki = kzalloc(sizeof(*ki), GFP_KERNEL);
+ if (!ki)
+ return -ENOMEM;
+
+ refcount_set(&ki->ki_refcnt, 2);
+ init_sync_kiocb(&ki->iocb, swap_file);
+ ki->iocb.ki_swap_page = page;
+ ki->iocb.ki_flags = IOCB_DIRECT | IOCB_SWAP;
+ ki->iocb.ki_pos = page_file_offset(page);
+ ki->iocb.ki_filp = get_file(swap_file);
+ ki->iocb.ki_complete = swapfile_read_complete;
+
+ iov_iter_bvec(&to, READ, &bv, 1, thp_size(page));
+ ret = swap_file->f_mapping->a_ops->direct_IO(&ki->iocb, &to);
+
+ if (ret != -EIOCBQUEUED)
+ swapfile_read_complete(&ki->iocb, ret, 0);
+ swapfile_put_kiocb(ki);
+ return (ret > 0) ? 0 : ret;
+}
+
int swap_readpage(struct page *page, bool synchronous)
{
struct bio *bio;
@@ -380,12 +488,7 @@ int swap_readpage(struct page *page, bool synchronous)
}
if (data_race(sis->flags & SWP_FS_OPS)) {
- struct file *swap_file = sis->swap_file;
- struct address_space *mapping = swap_file->f_mapping;
-
- ret = mapping->a_ops->readpage(swap_file, page);
- if (!ret)
- count_vm_event(PSWPIN);
+ ret = swapfile_read(sis, page, synchronous);
goto out;
}
