When an overlayfs file is opened as lower and then the file is copied up,
every operation on the overlayfs open file will open a temporary backing
file to the upper dentry and close it at the end of the operation.
The original (lower) real file is stored in file->private_data pointer.
We could have allocated another container struct for file->private_data
to potentially store two backing files, the lower and the upper.
However the original backing file struct is not very space optimized
(and it has no memcache pool), so add a private_data pointer to the
backing_file struct and store the optional second backing upper file in
there instead of opening a temporary upper file on every operation.
Signed-off-by: Amir Goldstein <amir73il@xxxxxxxxx>
---
fs/file_table.c | 7 +++++
fs/internal.h | 6 ++++
fs/overlayfs/file.c | 70 +++++++++++++++++++++++++++++++++++++++------
3 files changed, 74 insertions(+), 9 deletions(-)
diff --git a/fs/file_table.c b/fs/file_table.c
index eed5ffad9997..1c2c08a5a66a 100644
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -47,6 +47,7 @@ static struct percpu_counter nr_files __cacheline_aligned_in_smp;
struct backing_file {
struct file file;
struct path user_path;
+ void *private_data;
};
static inline struct backing_file *backing_file(struct file *f)
@@ -60,6 +61,12 @@ struct path *backing_file_user_path(struct file *f)
}
EXPORT_SYMBOL_GPL(backing_file_user_path);
+void **backing_file_private_ptr(struct file *f)
+{
+ return &backing_file(f)->private_data;
+}
+EXPORT_SYMBOL_GPL(backing_file_private_ptr);
+
static inline void file_free(struct file *f)
{
security_file_free(f);
diff --git a/fs/internal.h b/fs/internal.h
index 8c1b7acbbe8f..b1152a3e8ba2 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -100,6 +100,12 @@ extern void chroot_fs_refs(const struct path *, const struct path *);
struct file *alloc_empty_file(int flags, const struct cred *cred);
struct file *alloc_empty_file_noaccount(int flags, const struct cred *cred);
struct file *alloc_empty_backing_file(int flags, const struct cred *cred);
+void **backing_file_private_ptr(struct file *f);
+
+static inline void *backing_file_private(struct file *f)
+{
+ return READ_ONCE(*backing_file_private_ptr(f));
+}
static inline void file_put_write_access(struct file *file)
{
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index d40c10a6bfac..42f9bbdd65b4 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -14,6 +14,8 @@
#include <linux/backing-file.h>
#include "overlayfs.h"
+#include "../internal.h" /* for backing_file_private{,_ptr}() */
+
static char ovl_whatisit(struct inode *inode, struct inode *realinode)
{
if (realinode != ovl_inode_upper(inode))
@@ -89,29 +91,70 @@ static int ovl_change_flags(struct file *file, unsigned int flags)
return 0;
}
+static bool ovl_is_real_file(const struct file *realfile,
+ const struct path *realpath)
+{
+ return file_inode(realfile) == d_inode(realpath->dentry);
+}
+
static int ovl_real_fdget_path(const struct file *file, struct fd *real,
struct path *realpath)
{
struct file *realfile = file->private_data;
+ struct file *upperfile = backing_file_private(realfile);
- real->word = (unsigned long)realfile;
+ real->word = 0;
if (WARN_ON_ONCE(!realpath->dentry))
return -EIO;
- /* Has it been copied up since we'd opened it? */
- if (unlikely(file_inode(realfile) != d_inode(realpath->dentry))) {
- struct file *f = ovl_open_realfile(file, realpath);
- if (IS_ERR(f))
- return PTR_ERR(f);
- real->word = (unsigned long)f | FDPUT_FPUT;
- return 0;
+ /*
+ * Usually, if we operated on a stashed upperfile once, all following
+ * operations will operate on the stashed upperfile, but there is one
+ * exception - ovl_fsync(datasync = false) can populate the stashed
+ * upperfile to perform fsync on upper metadata inode. In this case,
+ * following read/write operations will not use the stashed upperfile.
+ */
+ if (upperfile && likely(ovl_is_real_file(upperfile, realpath))) {
+ realfile = upperfile;
+ goto checkflags;
}
+ /*
+ * If realfile is lower and has been copied up since we'd opened it,
+ * open the real upper file and stash it in backing_file_private().
+ */
+ if (unlikely(!ovl_is_real_file(realfile, realpath))) {
+ struct file *old;
+
+ /* Either stashed realfile or upperfile must match realinode */
+ if (WARN_ON_ONCE(upperfile))
+ return -EIO;
+
+ upperfile = ovl_open_realfile(file, realpath);
+ if (IS_ERR(upperfile))
+ return PTR_ERR(upperfile);
+
+ old = cmpxchg_release(backing_file_private_ptr(realfile), NULL,
+ upperfile);
+ if (old) {
+ fput(upperfile);
+ upperfile = old;
+ }
+
+ /* Stashed upperfile that won the race must match realinode */
+ if (WARN_ON_ONCE(!ovl_is_real_file(upperfile, realpath)))
+ return -EIO;
+
+ realfile = upperfile;
+ }
+
+checkflags:
/* Did the flags change since open? */
if (unlikely((file->f_flags ^ realfile->f_flags) & ~OVL_OPEN_FLAGS))
return ovl_change_flags(realfile, file->f_flags);
+ real->word = (unsigned long)realfile;
return 0;
}
@@ -192,7 +235,16 @@ static int ovl_open(struct inode *inode, struct file *file)
static int ovl_release(struct inode *inode, struct file *file)
{
- fput(file->private_data);
+ struct file *realfile = file->private_data;
+ struct file *upperfile = backing_file_private(realfile);
+
+ fput(realfile);
+ /*
+ * If realfile is lower and file was copied up and accessed, we need
+ * to put reference also on the stashed real upperfile.
+ */
+ if (upperfile)
+ fput(upperfile);
return 0;
}
--
2.34.1
