> On Jun 22, 2022, at 2:15 PM, Dai Ngo <dai.ngo@xxxxxxxxxx> wrote:
>
> Currently the idle timeout for courtesy client is fixed at 1 day.
> If there are lots of courtesy clients remain in the system it can
> cause memory resource shortage that effects the operations of other
> modules in the kernel. This problem can be observed by running pynfs
> nfs4.0 CID5 test in a loop. Eventually system runs out of memory
> and rpc.gssd fails to add new watch:
>
> rpc.gssd[3851]: ERROR: inotify_add_watch failed for nfsd4_cb/clnt6c2e:
> No space left on device
>
> and also alloc_inode fails with out of memory:
>
> Call Trace:
> <TASK>
> dump_stack_lvl+0x33/0x42
> dump_header+0x4a/0x1ed
> oom_kill_process+0x80/0x10d
> out_of_memory+0x237/0x25f
> __alloc_pages_slowpath.constprop.0+0x617/0x7b6
> __alloc_pages+0x132/0x1e3
> alloc_slab_page+0x15/0x33
> allocate_slab+0x78/0x1ab
> ? alloc_inode+0x38/0x8d
> ___slab_alloc+0x2af/0x373
> ? alloc_inode+0x38/0x8d
> ? slab_pre_alloc_hook.constprop.0+0x9f/0x158
> ? alloc_inode+0x38/0x8d
> __slab_alloc.constprop.0+0x1c/0x24
> kmem_cache_alloc_lru+0x8c/0x142
> alloc_inode+0x38/0x8d
> iget_locked+0x60/0x126
> kernfs_get_inode+0x18/0x105
> kernfs_iop_lookup+0x6d/0xbc
> __lookup_slow+0xb7/0xf9
> lookup_slow+0x3a/0x52
> walk_component+0x90/0x100
> ? inode_permission+0x87/0x128
> link_path_walk.part.0.constprop.0+0x266/0x2ea
> ? path_init+0x101/0x2f2
> path_lookupat+0x4c/0xfa
> filename_lookup+0x63/0xd7
> ? getname_flags+0x32/0x17a
> ? kmem_cache_alloc+0x11f/0x144
> ? getname_flags+0x16c/0x17a
> user_path_at_empty+0x37/0x4b
> do_readlinkat+0x61/0x102
> __x64_sys_readlinkat+0x18/0x1b
> do_syscall_64+0x57/0x72
> entry_SYSCALL_64_after_hwframe+0x46/0xb0
> RIP: 0033:0x7fce5410340e
>
> This patch adds a simple policy to dynamically adjust the idle
> timeout based on the percentage of available memory in the system
> as follow:
>
> . > 70% : unlimited. Courtesy clients are allowed to remain valid
> as long as memory availability is above 70%
> . 60% - 70%: 1 day.
> . 50% - 60%: 1hr
> . 40% - 50%: 30mins
> . 30% - 40%: 15mins
> . < 30%: disable. Expire all existing courtesy clients and donot
> allow new courtesey client
I thought our plan was to add a shrinker to do this.
> Signed-off-by: Dai Ngo <dai.ngo@xxxxxxxxxx>
> ---
> fs/nfsd/nfs4state.c | 41 +++++++++++++++++++++++++++++++++++++++--
> fs/nfsd/nfsd.h | 5 ++++-
> 2 files changed, 43 insertions(+), 3 deletions(-)
>
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 9409a0dc1b76..a7feea9d07cf 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -5788,12 +5788,47 @@ nfs4_anylock_blockers(struct nfs4_client *clp)
> return false;
> }
>
> +static bool
> +nfs4_allow_courtesy_client(struct nfsd_net *nn, unsigned int *idle_timeout)
> +{
> + unsigned long avail;
> + bool ret = true;
> + unsigned int courtesy_expire = 0;
> + struct sysinfo si;
> +
> + si_meminfo(&si);
> + avail = (si.freeram * 10) / (si.totalram - si.totalhigh);
> + switch (avail) {
> + case 7: case 8: case 9: case 10:
> + courtesy_expire = 0; /* unlimit */
> + break;
> + case 6:
> + courtesy_expire = NFSD_COURTESY_CLIENT_TO_1DAY;
> + break;
> + case 5:
> + courtesy_expire = NFSD_COURTESY_CLIENT_TO_1HR;
> + break;
> + case 4:
> + courtesy_expire = NFSD_COURTESY_CLIENT_TO_30MINS;
> + break;
> + case 3:
> + courtesy_expire = NFSD_COURTESY_CLIENT_TO_15MINS;
> + break;
> + default:
> + ret = false; /* disallow CC */
> + }
> + *idle_timeout = courtesy_expire;
> + return ret;
> +}
> +
> static void
> nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
> struct laundry_time *lt)
> {
> struct list_head *pos, *next;
> struct nfs4_client *clp;
> + unsigned int exptime;
> + bool allow_cc = nfs4_allow_courtesy_client(nn, &exptime);
>
> INIT_LIST_HEAD(reaplist);
> spin_lock(&nn->client_lock);
> @@ -5803,11 +5838,13 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
> goto exp_client;
> if (!state_expired(lt, clp->cl_time))
> break;
> + if (!allow_cc)
> + goto exp_client;
> if (!atomic_read(&clp->cl_rpc_users))
> clp->cl_state = NFSD4_COURTESY;
> if (!client_has_state(clp) ||
> - ktime_get_boottime_seconds() >=
> - (clp->cl_time + NFSD_COURTESY_CLIENT_TIMEOUT))
> + (exptime && ktime_get_boottime_seconds() >=
> + (clp->cl_time + exptime)))
> goto exp_client;
> if (nfs4_anylock_blockers(clp)) {
> exp_client:
> diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
> index 847b482155ae..9d4a5708f852 100644
> --- a/fs/nfsd/nfsd.h
> +++ b/fs/nfsd/nfsd.h
> @@ -340,7 +340,10 @@ void nfsd_lockd_shutdown(void);
> #define COMPOUND_ERR_SLACK_SPACE 16 /* OP_SETATTR */
>
> #define NFSD_LAUNDROMAT_MINTIMEOUT 1 /* seconds */
> -#define NFSD_COURTESY_CLIENT_TIMEOUT (24 * 60 * 60) /* seconds */
> +#define NFSD_COURTESY_CLIENT_TO_1DAY (24 * 60 * 60) /* seconds */
> +#define NFSD_COURTESY_CLIENT_TO_1HR (60 * 60)
> +#define NFSD_COURTESY_CLIENT_TO_30MINS (30 * 60)
> +#define NFSD_COURTESY_CLIENT_TO_15MINS (15 * 60)
>
> /*
> * The following attributes are currently not supported by the NFSv4 server:
> --
> 2.9.5
>
--
Chuck Lever
