Re: [PATCH 3/5] ceph: flush mdlog before umounting

[Xiubo Li <xiubli@xxxxxxxxxx>]

On 6/29/21 11:34 PM, Jeff Layton wrote:
> On Tue, 2021-06-29 at 12:42 +0800, xiubli@xxxxxxxxxx wrote:
> > From: Xiubo Li <xiubli@xxxxxxxxxx>
> >
> > Signed-off-by: Xiubo Li <xiubli@xxxxxxxxxx>
> > ---
> >   fs/ceph/mds_client.c         | 29 +++++++++++++++++++++++++++++
> >   fs/ceph/mds_client.h         |  1 +
> >   include/linux/ceph/ceph_fs.h |  1 +
> >   3 files changed, 31 insertions(+)
> >
> > diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
> > index 96bef289f58f..2db87a5c68d4 100644
> > --- a/fs/ceph/mds_client.c
> > +++ b/fs/ceph/mds_client.c
> > @@ -4689,6 +4689,34 @@ static void wait_requests(struct ceph_mds_client *mdsc)
> >   	dout("wait_requests done\n");
> >   }
> >   
> > +static void send_flush_mdlog(struct ceph_mds_session *s)
> > +{
> > +	u64 seq = s->s_seq;
> > +	struct ceph_msg *msg;
> > +
> The s_seq field is protected by the s_mutex (at least, AFAICT). I think
> you probably need to take it before fetching the s_seq and release it
> after calling ceph_con_send.

Will fix it.


> Long term, we probably need to rethink how the session sequence number
> handling is done. The s_mutex is a terribly heavyweight mechanism for
> this.

Yeah, makes sense.


> > +	/*
> > +	 * For the MDS daemons lower than Luminous will crash when it
> > +	 * saw this unknown session request.
> > +	 */
> > +	if (!CEPH_HAVE_FEATURE(s->s_con.peer_features, SERVER_LUMINOUS))
> > +		return;
> > +
> > +	dout("send_flush_mdlog to mds%d (%s)s seq %lld\n",
> > +	     s->s_mds, ceph_session_state_name(s->s_state), seq);
> > +	msg = ceph_create_session_msg(CEPH_SESSION_REQUEST_FLUSH_MDLOG, seq);
> > +	if (!msg) {
> > +		pr_err("failed to send_flush_mdlog to mds%d (%s)s seq %lld\n",
> > +		       s->s_mds, ceph_session_state_name(s->s_state), seq);
> > +	} else {
> > +		ceph_con_send(&s->s_con, msg);
> > +	}
> > +}
> > +
> > +void flush_mdlog(struct ceph_mds_client *mdsc)
> > +{
> > +	ceph_mdsc_iterate_sessions(mdsc, send_flush_mdlog, true);
> > +}
> > +
> >   /*
> >    * called before mount is ro, and before dentries are torn down.
> >    * (hmm, does this still race with new lookups?)
> > @@ -4698,6 +4726,7 @@ void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc)
> >   	dout("pre_umount\n");
> >   	mdsc->stopping = 1;
> >   
> > +	flush_mdlog(mdsc);
> >   	ceph_mdsc_iterate_sessions(mdsc, lock_unlock_session, false);
> >   	ceph_flush_dirty_caps(mdsc);
> >   	wait_requests(mdsc);
> > diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h
> > index fca2cf427eaf..79d5b8ed62bf 100644
> > --- a/fs/ceph/mds_client.h
> > +++ b/fs/ceph/mds_client.h
> > @@ -537,6 +537,7 @@ extern int ceph_iterate_session_caps(struct ceph_mds_session *session,
> >   				     int (*cb)(struct inode *,
> >   					       struct ceph_cap *, void *),
> >   				     void *arg);
> > +extern void flush_mdlog(struct ceph_mds_client *mdsc);
> >   extern void ceph_mdsc_pre_umount(struct ceph_mds_client *mdsc);
> >   
> >   static inline void ceph_mdsc_free_path(char *path, int len)
> > diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h
> > index 57e5bd63fb7a..ae60696fe40b 100644
> > --- a/include/linux/ceph/ceph_fs.h
> > +++ b/include/linux/ceph/ceph_fs.h
> > @@ -300,6 +300,7 @@ enum {
> >   	CEPH_SESSION_FLUSHMSG_ACK,
> >   	CEPH_SESSION_FORCE_RO,
> >   	CEPH_SESSION_REJECT,
> > +	CEPH_SESSION_REQUEST_FLUSH_MDLOG,
> >   };
> >   
> >   extern const char *ceph_session_op_name(int op);