Use ->d_seq instead of grabbing ->d_lock; in case of shortname dentries
that avoids any stores to shared data objects and in case of long names
we are down to (unavoidable) atomic_inc on the external_name refcount.
Makes the thing safer as well - the areas where ->d_seq is held odd are
all nested inside the areas where ->d_lock is held, and the latter are
much more numerous.
NOTE: we no longer can have external_name.u.count and external_name.u.head
sharing space, now that we have lockless path that might try to grab
a reference on already doomed instance (kudos to Linus for spotting that).
For now just turn that external_name.u into a struct (instead of union)
to reduce the noise in this commit; the next commit will dissolve it.
Signed-off-by: Al Viro <viro@xxxxxxxxxxxxxxxxxx>
---
fs/dcache.c | 38 ++++++++++++++++++++++++++++----------
1 file changed, 28 insertions(+), 10 deletions(-)
diff --git a/fs/dcache.c b/fs/dcache.c
index 007e582c3e68..ae13e89ce7d7 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -296,9 +296,9 @@ static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *c
}
struct external_name {
- union {
- atomic_t count;
- struct rcu_head head;
+ struct {
+ atomic_t count; // ->count and ->head can't be combined
+ struct rcu_head head; // see take_dentry_name_snapshot()
} u;
unsigned char name[];
};
@@ -329,15 +329,33 @@ static inline int dname_external(const struct dentry *dentry)
void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
{
- spin_lock(&dentry->d_lock);
- name->name = dentry->d_name;
- if (unlikely(dname_external(dentry))) {
- atomic_inc(&external_name(dentry)->u.count);
- } else {
- name->inline_name_words = dentry->d_iname_words;
+ unsigned seq;
+ const unsigned char *s;
+
+ rcu_read_lock();
+retry:
+ seq = read_seqcount_begin(&dentry->d_seq);
+ s = READ_ONCE(dentry->d_name.name);
+ name->name.hash_len = dentry->d_name.hash_len;
+ if (likely(s == dentry->d_iname)) {
name->name.name = name->inline_name;
+ name->inline_name_words = dentry->d_iname_words;
+ if (read_seqcount_retry(&dentry->d_seq, seq))
+ goto retry;
+ } else {
+ struct external_name *p;
+ p = container_of(s, struct external_name, name[0]);
+ name->name.name = s;
+ // get a valid reference
+ if (unlikely(!atomic_inc_not_zero(&p->u.count)))
+ goto retry;
+ if (read_seqcount_retry(&dentry->d_seq, seq)) {
+ if (unlikely(atomic_dec_and_test(&p->u.count)))
+ kfree_rcu(p, u.head);
+ goto retry;
+ }
}
- spin_unlock(&dentry->d_lock);
+ rcu_read_unlock();
}
EXPORT_SYMBOL(take_dentry_name_snapshot);
--
2.39.5
