[RFC PATCH v4] ovl: drop negative dentry in upper layer

[Chengguang Xu <cgxu519@xxxxxxxxxxxx>]

Negative dentries of upper layer are useless after construction
of overlayfs' own dentry and may keep in the memory long time even
after unmount of overlayfs instance. This patch tries to drop
unnecessary negative dentry of upper layer to effectively reclaim
memory.

Signed-off-by: Chengguang Xu <cgxu519@xxxxxxxxxxxx>
---
v1->v2:
- Only drop negative dentry in slow path of lookup.

v2->v3:
- Drop negative dentry in vfs layer.
- Rebase on latest linus-tree(5.7.0-rc5).

v3->v4:
- Check negative dentry with dentry lock.
- Only drop negative dentry in upper layer.

 fs/overlayfs/namei.c | 45 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 7 deletions(-)

diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c
index 723d17744758..47cc79ec8205 100644
--- a/fs/overlayfs/namei.c
+++ b/fs/overlayfs/namei.c
@@ -191,16 +191,46 @@ static bool ovl_is_opaquedir(struct dentry *dentry)
 	return ovl_check_dir_xattr(dentry, OVL_XATTR_OPAQUE);
 }
 
+static struct dentry *ovl_lookup_positive_unlocked(const char *name,
+					struct dentry *base, int len)
+{
+	struct dentry *dentry;
+	bool drop = false;
+
+	dentry = lookup_one_len_unlocked(name, base, len);
+	if (!IS_ERR(dentry) && d_is_negative(dentry) &&
+	    dentry->d_lockref.count == 1) {
+		spin_lock(&dentry->d_lock);
+		/* Recheck condition under lock */
+		if (d_is_negative(dentry) && dentry->d_lockref.count == 1) {
+			__d_drop(dentry);
+			drop = true;
+		}
+		spin_unlock(&dentry->d_lock);
+
+		if (drop) {
+			dput(dentry);
+			dentry = ERR_PTR(-ENOENT);
+		}
+	}
+
+	return dentry;
+}
+
 static int ovl_lookup_single(struct dentry *base, struct ovl_lookup_data *d,
 			     const char *name, unsigned int namelen,
 			     size_t prelen, const char *post,
-			     struct dentry **ret)
+			     struct dentry **ret, bool drop_negative)
 {
 	struct dentry *this;
 	int err;
 	bool last_element = !post[0];
 
-	this = lookup_positive_unlocked(name, base, namelen);
+	if (drop_negative)
+		this = ovl_lookup_positive_unlocked(name, base, namelen);
+	else
+		this = lookup_positive_unlocked(name, base, namelen);
+
 	if (IS_ERR(this)) {
 		err = PTR_ERR(this);
 		this = NULL;
@@ -276,7 +306,7 @@ static int ovl_lookup_single(struct dentry *base, struct ovl_lookup_data *d,
 }
 
 static int ovl_lookup_layer(struct dentry *base, struct ovl_lookup_data *d,
-			    struct dentry **ret)
+			    struct dentry **ret, bool drop_negative)
 {
 	/* Counting down from the end, since the prefix can change */
 	size_t rem = d->name.len - 1;
@@ -285,7 +315,7 @@ static int ovl_lookup_layer(struct dentry *base, struct ovl_lookup_data *d,
 
 	if (d->name.name[0] != '/')
 		return ovl_lookup_single(base, d, d->name.name, d->name.len,
-					 0, "", ret);
+					 0, "", ret, drop_negative);
 
 	while (!IS_ERR_OR_NULL(base) && d_can_lookup(base)) {
 		const char *s = d->name.name + d->name.len - rem;
@@ -298,7 +328,8 @@ static int ovl_lookup_layer(struct dentry *base, struct ovl_lookup_data *d,
 			return -EIO;
 
 		err = ovl_lookup_single(base, d, s, thislen,
-					d->name.len - rem, next, &base);
+					d->name.len - rem, next, &base,
+					drop_negative);
 		dput(dentry);
 		if (err)
 			return err;
@@ -830,7 +861,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry,
 	old_cred = ovl_override_creds(dentry->d_sb);
 	upperdir = ovl_dentry_upper(dentry->d_parent);
 	if (upperdir) {
-		err = ovl_lookup_layer(upperdir, &d, &upperdentry);
+		err = ovl_lookup_layer(upperdir, &d, &upperdentry, true);
 		if (err)
 			goto out;
 
@@ -888,7 +919,7 @@ struct dentry *ovl_lookup(struct inode *dir, struct dentry *dentry,
 		else
 			d.last = lower.layer->idx == roe->numlower;
 
-		err = ovl_lookup_layer(lower.dentry, &d, &this);
+		err = ovl_lookup_layer(lower.dentry, &d, &this, false);
 		if (err)
 			goto out_put;
 
-- 
2.20.1