On Mon, 29 Jan 2024, Jeff Layton wrote:
> On Mon, 2024-01-29 at 14:29 +1100, NeilBrown wrote:
> > When there is layout state on a filesystem that is being "unlocked" that
> > is now revoked, which involves closing the nfsd_file and releasing the
> > vfs lease.
> >
> > To avoid races, ->ls_file can now be accessed either:
> > - under ->fi_lock for the state's sc_file or
> > - under rcu_read_lock() if nfsd_file_get() is used.
> > To support this, ->fence_client and nfsd4_cb_layout_fail() now take a
> > second argument being the nfsd_file.
> >
> > Signed-off-by: NeilBrown <neilb@xxxxxxx>
> > ---
> > fs/nfsd/blocklayout.c | 4 ++--
> > fs/nfsd/nfs4layouts.c | 43 ++++++++++++++++++++++++++++++++-----------
> > fs/nfsd/nfs4state.c | 11 +++++++++--
> > fs/nfsd/pnfs.h | 8 +++++++-
> > 4 files changed, 50 insertions(+), 16 deletions(-)
> >
> > diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
> > index 46fd74d91ea9..3c040c81c77d 100644
> > --- a/fs/nfsd/blocklayout.c
> > +++ b/fs/nfsd/blocklayout.c
> > @@ -328,10 +328,10 @@ nfsd4_scsi_proc_layoutcommit(struct inode *inode,
> > }
> >
> > static void
> > -nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls)
> > +nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > {
> > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > - struct block_device *bdev = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> > + struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev;
> >
> > bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY,
> > nfsd4_scsi_pr_key(clp), 0, true);
> > diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
> > index 857b822450b4..1cfd61db2472 100644
> > --- a/fs/nfsd/nfs4layouts.c
> > +++ b/fs/nfsd/nfs4layouts.c
> > @@ -152,6 +152,23 @@ void nfsd4_setup_layout_type(struct svc_export *exp)
> > #endif
> > }
> >
> > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > +{
> > + struct nfsd_file *fl;
> > +
> > + spin_lock(&ls->ls_stid.sc_file->fi_lock);
> > + fl = ls->ls_file;
> > + ls->ls_file = NULL;
> > + spin_unlock(&ls->ls_stid.sc_file->fi_lock);
> > +
> > + if (fl) {
> > + if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > + vfs_setlease(fl->nf_file, F_UNLCK, NULL,
> > + (void **)&ls);
> > + nfsd_file_put(fl);
> > + }
> > +}
> > +
> > static void
> > nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > {
> > @@ -169,9 +186,7 @@ nfsd4_free_layout_stateid(struct nfs4_stid *stid)
> > list_del_init(&ls->ls_perfile);
> > spin_unlock(&fp->fi_lock);
> >
> > - if (!nfsd4_layout_ops[ls->ls_layout_type]->disable_recalls)
> > - vfs_setlease(ls->ls_file->nf_file, F_UNLCK, NULL, (void **)&ls);
> > - nfsd_file_put(ls->ls_file);
> > + nfsd4_close_layout(ls);
> >
> > if (ls->ls_recalled)
> > atomic_dec(&ls->ls_stid.sc_file->fi_lo_recalls);
> > @@ -605,7 +620,7 @@ nfsd4_return_all_file_layouts(struct nfs4_client *clp, struct nfs4_file *fp)
> > }
> >
> > static void
> > -nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> > +nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls, struct nfsd_file *file)
> > {
> > struct nfs4_client *clp = ls->ls_stid.sc_client;
> > char addr_str[INET6_ADDRSTRLEN];
> > @@ -627,7 +642,7 @@ nfsd4_cb_layout_fail(struct nfs4_layout_stateid *ls)
> >
> > argv[0] = (char *)nfsd_recall_failed;
> > argv[1] = addr_str;
> > - argv[2] = ls->ls_file->nf_file->f_path.mnt->mnt_sb->s_id;
> > + argv[2] = file->nf_file->f_path.mnt->mnt_sb->s_id;
> > argv[3] = NULL;
> >
> > error = call_usermodehelper(nfsd_recall_failed, argv, envp,
> > @@ -657,6 +672,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > struct nfsd_net *nn;
> > ktime_t now, cutoff;
> > const struct nfsd4_layout_ops *ops;
> > + struct nfsd_file *fl;
> >
> > trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
> > switch (task->tk_status) {
> > @@ -688,12 +704,17 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
> > * Unknown error or non-responding client, we'll need to fence.
> > */
> > trace_nfsd_layout_recall_fail(&ls->ls_stid.sc_stateid);
> > -
> > - ops = nfsd4_layout_ops[ls->ls_layout_type];
> > - if (ops->fence_client)
> > - ops->fence_client(ls);
> > - else
> > - nfsd4_cb_layout_fail(ls);
> > + rcu_read_lock();
> > + fl = nfsd_file_get(ls->ls_file);
> > + rcu_read_unlock();
>
> What is the rcu_read_lock protecting here? AFAICT, you don't need it
> since you hold a reference to "ls".
The new nfsd4_close_layout() can be called asynchronously (by admin
unlocking a filesystem) and will set ->ls_file to NULL, and then
nfsd_file_put() the file.
Without rcu_read_lock(), the pointer we get when dereferencing ls->ls_file
could have been passed to nfsd_file_put() and thence nfsd_file_free()
before nfsd_file_get() is called.
The rcu_read_lock() ensures that ->nf_ref from the pointer is still a valid
refcount and that it is safe for refcount_inc_not_zero() to be called on
it in nfsd_file_get().
Thanks for all you Reviewed-by!
NeilBrown
>
> > + if (fl) {
> > + ops = nfsd4_layout_ops[ls->ls_layout_type];
> > + if (ops->fence_client)
> > + ops->fence_client(ls, fl);
> > + else
> > + nfsd4_cb_layout_fail(ls, fl);
> > + nfsd_file_put(fl);
> > + }
> > return 1;
> > case -NFS4ERR_NOMATCHING_LAYOUT:
> > trace_nfsd_layout_recall_done(&ls->ls_stid.sc_stateid);
> > diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> > index e749d5c0e23a..268b47a6f3b6 100644
> > --- a/fs/nfsd/nfs4state.c
> > +++ b/fs/nfsd/nfs4state.c
> > @@ -1721,7 +1721,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > unsigned int idhashval;
> > unsigned int sc_types;
> >
> > - sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG;
> > + sc_types = SC_TYPE_OPEN | SC_TYPE_LOCK | SC_TYPE_DELEG | SC_TYPE_LAYOUT;
> >
> > spin_lock(&nn->client_lock);
> > for (idhashval = 0; idhashval < CLIENT_HASH_MASK; idhashval++) {
> > @@ -1734,6 +1734,7 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > if (stid) {
> > struct nfs4_ol_stateid *stp;
> > struct nfs4_delegation *dp;
> > + struct nfs4_layout_stateid *ls;
> >
> > spin_unlock(&nn->client_lock);
> > switch (stid->sc_type) {
> > @@ -1789,6 +1790,10 @@ void nfsd4_revoke_states(struct net *net, struct super_block *sb)
> > if (dp)
> > revoke_delegation(dp);
> > break;
> > + case SC_TYPE_LAYOUT:
> > + ls = layoutstateid(stid);
> > + nfsd4_close_layout(ls);
> > + break;
> > }
> > nfs4_put_stid(stid);
> > spin_lock(&nn->client_lock);
> > @@ -2868,7 +2873,6 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
> > struct nfsd_file *file;
> >
> > ls = container_of(st, struct nfs4_layout_stateid, ls_stid);
> > - file = ls->ls_file;
> >
> > seq_puts(s, "- ");
> > nfs4_show_stateid(s, &st->sc_stateid);
> > @@ -2876,12 +2880,15 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st)
> >
> > /* XXX: What else would be useful? */
> >
> > + spin_lock(&ls->ls_stid.sc_file->fi_lock);
> > + file = ls->ls_file;
> > if (file) {
> > seq_puts(s, ", ");
> > nfs4_show_superblock(s, file);
> > seq_puts(s, ", ");
> > nfs4_show_fname(s, file);
> > }
> > + spin_unlock(&ls->ls_stid.sc_file->fi_lock);
> > if (st->sc_status & SC_STATUS_ADMIN_REVOKED)
> > seq_puts(s, ", admin-revoked");
> > seq_puts(s, " }\n");
> > diff --git a/fs/nfsd/pnfs.h b/fs/nfsd/pnfs.h
> > index de1e0dfed06a..925817f66917 100644
> > --- a/fs/nfsd/pnfs.h
> > +++ b/fs/nfsd/pnfs.h
> > @@ -37,7 +37,8 @@ struct nfsd4_layout_ops {
> > __be32 (*proc_layoutcommit)(struct inode *inode,
> > struct nfsd4_layoutcommit *lcp);
> >
> > - void (*fence_client)(struct nfs4_layout_stateid *ls);
> > + void (*fence_client)(struct nfs4_layout_stateid *ls,
> > + struct nfsd_file *file);
> > };
> >
> > extern const struct nfsd4_layout_ops *nfsd4_layout_ops[];
> > @@ -72,11 +73,13 @@ void nfsd4_setup_layout_type(struct svc_export *exp);
> > void nfsd4_return_all_client_layouts(struct nfs4_client *);
> > void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > struct nfs4_file *fp);
> > +void nfsd4_close_layout(struct nfs4_layout_stateid *ls);
> > int nfsd4_init_pnfs(void);
> > void nfsd4_exit_pnfs(void);
> > #else
> > struct nfs4_client;
> > struct nfs4_file;
> > +struct nfs4_layout_stateid;
> >
> > static inline void nfsd4_setup_layout_type(struct svc_export *exp)
> > {
> > @@ -89,6 +92,9 @@ static inline void nfsd4_return_all_file_layouts(struct nfs4_client *clp,
> > struct nfs4_file *fp)
> > {
> > }
> > +static inline void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
> > +{
> > +}
> > static inline void nfsd4_exit_pnfs(void)
> > {
> > }
>
> --
> Jeff Layton <jlayton@xxxxxxxxxx>
>
>
