In the .read_iter and .write_iter file operations, we're accessing
user-space memory while holding the inodes glock. There's a possibility
that the memory is mapped to the same file, in which case we'd recurse on
the same glock.
More complex scenarios can involve multiple glocks, processes, and even cluster
nodes.
Avoids these kinds of problems by disabling page faults while holding a glock.
If a page fault occurs, we either end up with a partial read or write, or with
-EFAULT if nothing could be read or written. In that case, we drop the glock,
fault in the requested pages manually, and repeat the operation.
This locking problem in gfs2 was originally reported by Jan Kara. Linus came
up with the proposal to disable page faults. Many thanks to Al Viro and
Matthew Wilcox for their feedback as well.
Signed-off-by: Andreas Gruenbacher <agruenba@xxxxxxxxxx>
---
fs/gfs2/file.c | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c
index 55ec1cadc9e6..f66ac7f56f6d 100644
--- a/fs/gfs2/file.c
+++ b/fs/gfs2/file.c
@@ -843,6 +843,12 @@ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
size_t written = 0;
ssize_t ret;
+ /*
+ * In this function, we disable page faults when we're holding the
+ * inode glock while doing I/O. If a page fault occurs, we drop the
+ * inode glock, fault in the pages manually, and then we retry.
+ */
+
if (iocb->ki_flags & IOCB_DIRECT) {
ret = gfs2_file_direct_read(iocb, to, &gh);
if (likely(ret != -ENOTBLK))
@@ -864,13 +870,19 @@ static ssize_t gfs2_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
}
ip = GFS2_I(iocb->ki_filp->f_mapping->host);
gfs2_holder_init(ip->i_gl, LM_ST_SHARED, 0, &gh);
+retry:
ret = gfs2_glock_nq(&gh);
if (ret)
goto out_uninit;
+ pagefault_disable();
ret = generic_file_read_iter(iocb, to);
+ pagefault_enable();
if (ret > 0)
written += ret;
gfs2_glock_dq(&gh);
+ if (unlikely(iov_iter_count(to) && (ret > 0 || ret == -EFAULT)) &&
+ fault_in_iov_iter(to))
+ goto retry;
out_uninit:
gfs2_holder_uninit(&gh);
return written ? written : ret;
@@ -882,9 +894,20 @@ static ssize_t gfs2_file_buffered_write(struct kiocb *iocb, struct iov_iter *fro
struct inode *inode = file_inode(file);
ssize_t ret;
+ /*
+ * In this function, we disable page faults when we're holding the
+ * inode glock while doing I/O. If a page fault occurs, we drop the
+ * inode glock, fault in the pages manually, and then we retry.
+ */
+
+retry:
current->backing_dev_info = inode_to_bdi(inode);
+ pagefault_disable();
ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops);
+ pagefault_enable();
current->backing_dev_info = NULL;
+ if (unlikely(ret == -EFAULT) && fault_in_iov_iter(from))
+ goto retry;
return ret;
}
--
2.26.3
