From: "Yan, Zheng" <zheng.z.yan@xxxxxxxxx>
Locks' states should not change between composing the cache rejoin ack
messages and sending the message. If Locker::eval_gather() is called
in MDCache::{inode,dentry}_remove_replica(), it may wake requests and
change locks' states.
Signed-off-by: Yan, Zheng <zheng.z.yan@xxxxxxxxx>
---
src/mds/MDCache.cc | 51 ++++++++++++++++++++++++++++++---------------------
src/mds/MDCache.h | 8 +++++---
2 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 19dc60b..0f6b842 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -3729,6 +3729,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
// possible response(s)
MMDSCacheRejoin *ack = 0; // if survivor
set<vinodeno_t> acked_inodes; // if survivor
+ set<SimpleLock *> gather_locks; // if survivor
bool survivor = false; // am i a survivor?
if (mds->is_clientreplay() || mds->is_active() || mds->is_stopping()) {
@@ -3851,7 +3852,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
assert(dnl->is_primary());
if (survivor && dn->is_replica(from))
- dentry_remove_replica(dn, from); // this induces a lock gather completion
+ dentry_remove_replica(dn, from, gather_locks); // this induces a lock gather completion
int dnonce = dn->add_replica(from);
dout(10) << " have " << *dn << dendl;
if (ack)
@@ -3864,7 +3865,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
assert(in);
if (survivor && in->is_replica(from))
- inode_remove_replica(in, from);
+ inode_remove_replica(in, from, gather_locks);
int inonce = in->add_replica(from);
dout(10) << " have " << *in << dendl;
@@ -3887,7 +3888,7 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
CInode *in = get_inode(*p);
assert(in); // hmm fixme wrt stray?
if (survivor && in->is_replica(from))
- inode_remove_replica(in, from); // this induces a lock gather completion
+ inode_remove_replica(in, from, gather_locks); // this induces a lock gather completion
int inonce = in->add_replica(from);
dout(10) << " have base " << *in << dendl;
@@ -3909,8 +3910,11 @@ void MDCache::handle_cache_rejoin_weak(MMDSCacheRejoin *weak)
ack->add_inode_base(in);
}
- rejoin_scour_survivor_replicas(from, ack, acked_inodes);
+ rejoin_scour_survivor_replicas(from, ack, gather_locks, acked_inodes);
mds->send_message(ack, weak->get_connection());
+
+ for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p)
+ mds->locker->eval_gather(*p);
} else {
// done?
assert(rejoin_gather.count(from));
@@ -4055,7 +4059,9 @@ bool MDCache::parallel_fetch_traverse_dir(inodeno_t ino, filepath& path,
* all validated replicas are acked with a strong nonce, etc. if that isn't in the
* ack, the replica dne, and we can remove it from our replica maps.
*/
-void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack, set<vinodeno_t>& acked_inodes)
+void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
+ set<SimpleLock *>& gather_locks,
+ set<vinodeno_t>& acked_inodes)
{
dout(10) << "rejoin_scour_survivor_replicas from mds." << from << dendl;
@@ -4070,7 +4076,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack, set
if (in->is_auth() &&
in->is_replica(from) &&
acked_inodes.count(p->second->vino()) == 0) {
- inode_remove_replica(in, from);
+ inode_remove_replica(in, from, gather_locks);
dout(10) << " rem " << *in << dendl;
}
@@ -4099,7 +4105,7 @@ void MDCache::rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack, set
if (dn->is_replica(from) &&
(ack->strong_dentries.count(dir->dirfrag()) == 0 ||
ack->strong_dentries[dir->dirfrag()].count(string_snap_t(dn->name, dn->last)) == 0)) {
- dentry_remove_replica(dn, from);
+ dentry_remove_replica(dn, from, gather_locks);
dout(10) << " rem " << *dn << dendl;
}
}
@@ -6189,6 +6195,7 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
return;
}
+ set<SimpleLock *> gather_locks;
// loop over realms
for (map<dirfrag_t,MCacheExpire::realm>::iterator p = m->realms.begin();
p != m->realms.end();
@@ -6255,7 +6262,7 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
// remove from our cached_by
dout(7) << " inode expire on " << *in << " from mds." << from
<< " cached_by was " << in->get_replicas() << dendl;
- inode_remove_replica(in, from);
+ inode_remove_replica(in, from, gather_locks);
}
else {
// this is an old nonce, ignore expire.
@@ -6332,7 +6339,7 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
if (nonce == dn->get_replica_nonce(from)) {
dout(7) << " dentry_expire on " << *dn << " from mds." << from << dendl;
- dentry_remove_replica(dn, from);
+ dentry_remove_replica(dn, from, gather_locks);
}
else {
dout(7) << " dentry_expire on " << *dn << " from mds." << from
@@ -6343,6 +6350,8 @@ void MDCache::handle_cache_expire(MCacheExpire *m)
}
}
+ for (set<SimpleLock*>::iterator p = gather_locks.begin(); p != gather_locks.end(); ++p)
+ mds->locker->eval_gather(*p);
// done
m->put();
@@ -6368,35 +6377,35 @@ void MDCache::discard_delayed_expire(CDir *dir)
delayed_expire.erase(dir);
}
-void MDCache::inode_remove_replica(CInode *in, int from)
+void MDCache::inode_remove_replica(CInode *in, int from, set<SimpleLock *>& gather_locks)
{
in->remove_replica(from);
in->mds_caps_wanted.erase(from);
// note: this code calls _eval more often than it needs to!
// fix lock
- if (in->authlock.remove_replica(from)) mds->locker->eval_gather(&in->authlock);
- if (in->linklock.remove_replica(from)) mds->locker->eval_gather(&in->linklock);
- if (in->dirfragtreelock.remove_replica(from)) mds->locker->eval_gather(&in->dirfragtreelock);
- if (in->filelock.remove_replica(from)) mds->locker->eval_gather(&in->filelock);
- if (in->snaplock.remove_replica(from)) mds->locker->eval_gather(&in->snaplock);
- if (in->xattrlock.remove_replica(from)) mds->locker->eval_gather(&in->xattrlock);
+ if (in->authlock.remove_replica(from)) gather_locks.insert(&in->authlock);
+ if (in->linklock.remove_replica(from)) gather_locks.insert(&in->linklock);
+ if (in->dirfragtreelock.remove_replica(from)) gather_locks.insert(&in->dirfragtreelock);
+ if (in->filelock.remove_replica(from)) gather_locks.insert(&in->filelock);
+ if (in->snaplock.remove_replica(from)) gather_locks.insert(&in->snaplock);
+ if (in->xattrlock.remove_replica(from)) gather_locks.insert(&in->xattrlock);
- if (in->nestlock.remove_replica(from)) mds->locker->eval_gather(&in->nestlock);
- if (in->flocklock.remove_replica(from)) mds->locker->eval_gather(&in->flocklock);
- if (in->policylock.remove_replica(from)) mds->locker->eval_gather(&in->policylock);
+ if (in->nestlock.remove_replica(from)) gather_locks.insert(&in->nestlock);
+ if (in->flocklock.remove_replica(from)) gather_locks.insert(&in->flocklock);
+ if (in->policylock.remove_replica(from)) gather_locks.insert(&in->policylock);
// trim?
maybe_eval_stray(in);
}
-void MDCache::dentry_remove_replica(CDentry *dn, int from)
+void MDCache::dentry_remove_replica(CDentry *dn, int from, set<SimpleLock *>& gather_locks)
{
dn->remove_replica(from);
// fix lock
if (dn->lock.remove_replica(from))
- mds->locker->eval_gather(&dn->lock);
+ gather_locks.insert(&dn->lock);
CDentry::linkage_t *dnl = dn->get_projected_linkage();
if (dnl->is_primary())
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index f07ea74..a9f05c6 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -406,7 +406,9 @@ protected:
CDir* rejoin_invent_dirfrag(dirfrag_t df);
bool rejoin_fetch_dirfrags(MMDSCacheRejoin *m);
void handle_cache_rejoin_strong(MMDSCacheRejoin *m);
- void rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack, set<vinodeno_t>& acked_inodes);
+ void rejoin_scour_survivor_replicas(int from, MMDSCacheRejoin *ack,
+ set<SimpleLock *>& gather_locks,
+ set<vinodeno_t>& acked_inodes);
void handle_cache_rejoin_ack(MMDSCacheRejoin *m);
void handle_cache_rejoin_purge(MMDSCacheRejoin *m);
void handle_cache_rejoin_missing(MMDSCacheRejoin *m);
@@ -607,8 +609,8 @@ public:
}
protected:
- void inode_remove_replica(CInode *in, int rep);
- void dentry_remove_replica(CDentry *dn, int rep);
+ void inode_remove_replica(CInode *in, int rep, set<SimpleLock *>& gather_locks);
+ void dentry_remove_replica(CDentry *dn, int rep, set<SimpleLock *>& gather_locks);
void rename_file(CDentry *srcdn, CDentry *destdn);
--
1.7.11.7
--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
