On Mon, 23 Jul 2012 13:58:51 -0400
Jeff Layton <jlayton@xxxxxxxxxx> wrote:
> We've had some reports of a deadlock where rpciod ends up with a stack
> trace like this:
>
> PID: 2507 TASK: ffff88103691ab40 CPU: 14 COMMAND: "rpciod/14"
> #0 [ffff8810343bf2f0] schedule at ffffffff814dabd9
> #1 [ffff8810343bf3b8] nfs_wait_bit_killable at ffffffffa038fc04 [nfs]
> #2 [ffff8810343bf3c8] __wait_on_bit at ffffffff814dbc2f
> #3 [ffff8810343bf418] out_of_line_wait_on_bit at ffffffff814dbcd8
> #4 [ffff8810343bf488] nfs_commit_inode at ffffffffa039e0c1 [nfs]
> #5 [ffff8810343bf4f8] nfs_release_page at ffffffffa038bef6 [nfs]
> #6 [ffff8810343bf528] try_to_release_page at ffffffff8110c670
> #7 [ffff8810343bf538] shrink_page_list.clone.0 at ffffffff81126271
> #8 [ffff8810343bf668] shrink_inactive_list at ffffffff81126638
> #9 [ffff8810343bf818] shrink_zone at ffffffff8112788f
> #10 [ffff8810343bf8c8] do_try_to_free_pages at ffffffff81127b1e
> #11 [ffff8810343bf958] try_to_free_pages at ffffffff8112812f
> #12 [ffff8810343bfa08] __alloc_pages_nodemask at ffffffff8111fdad
> #13 [ffff8810343bfb28] kmem_getpages at ffffffff81159942
> #14 [ffff8810343bfb58] fallback_alloc at ffffffff8115a55a
> #15 [ffff8810343bfbd8] ____cache_alloc_node at ffffffff8115a2d9
> #16 [ffff8810343bfc38] kmem_cache_alloc at ffffffff8115b09b
> #17 [ffff8810343bfc78] sk_prot_alloc at ffffffff81411808
> #18 [ffff8810343bfcb8] sk_alloc at ffffffff8141197c
> #19 [ffff8810343bfce8] inet_create at ffffffff81483ba6
> #20 [ffff8810343bfd38] __sock_create at ffffffff8140b4a7
> #21 [ffff8810343bfd98] xs_create_sock at ffffffffa01f649b [sunrpc]
> #22 [ffff8810343bfdd8] xs_tcp_setup_socket at ffffffffa01f6965 [sunrpc]
> #23 [ffff8810343bfe38] worker_thread at ffffffff810887d0
> #24 [ffff8810343bfee8] kthread at ffffffff8108dd96
> #25 [ffff8810343bff48] kernel_thread at ffffffff8100c1ca
>
> rpciod is trying to allocate memory for a new socket to talk to the
> server. The VM ends up calling ->releasepage to get more memory, and it
> tries to do a blocking commit. That commit can't succeed however without
> a connected socket, so we deadlock.
>
> Fix this by setting PF_FSTRANS on the workqueue task prior to doing the
> socket allocation, and having nfs_release_page check for that flag when
> deciding whether to do a commit call. Also, set PF_FSTRANS
> unconditionally in rpc_async_schedule since that function can also do
> allocations sometimes.
>
> Signed-off-by: Jeff Layton <jlayton@xxxxxxxxxx>
> ---
> fs/nfs/file.c | 7 +++++--
> net/sunrpc/sched.c | 2 ++
> net/sunrpc/xprtrdma/transport.c | 3 ++-
> net/sunrpc/xprtsock.c | 10 ++++++++++
> 4 files changed, 19 insertions(+), 3 deletions(-)
>
> diff --git a/fs/nfs/file.c b/fs/nfs/file.c
> index a6708e6b..9075769 100644
> --- a/fs/nfs/file.c
> +++ b/fs/nfs/file.c
> @@ -459,8 +459,11 @@ static int nfs_release_page(struct page *page, gfp_t gfp)
>
> dfprintk(PAGECACHE, "NFS: release_page(%p)\n", page);
>
> - /* Only do I/O if gfp is a superset of GFP_KERNEL */
> - if (mapping && (gfp & GFP_KERNEL) == GFP_KERNEL) {
> + /* Only do I/O if gfp is a superset of GFP_KERNEL, and we're not
> + * doing this memory reclaim for a fs-related allocation.
> + */
> + if (mapping && (gfp & GFP_KERNEL) == GFP_KERNEL &&
> + !(current->flags & PF_FSTRANS)) {
> int how = FLUSH_SYNC;
>
> /* Don't let kswapd deadlock waiting for OOM RPC calls */
> diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c
> index 994cfea..eda32ae 100644
> --- a/net/sunrpc/sched.c
> +++ b/net/sunrpc/sched.c
> @@ -790,7 +790,9 @@ void rpc_execute(struct rpc_task *task)
>
> static void rpc_async_schedule(struct work_struct *work)
> {
> + current->flags |= PF_FSTRANS;
> __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
> + current->flags &= ~PF_FSTRANS;
> }
>
> /**
> diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c
> index b446e10..06cdbff 100644
> --- a/net/sunrpc/xprtrdma/transport.c
> +++ b/net/sunrpc/xprtrdma/transport.c
> @@ -200,6 +200,7 @@ xprt_rdma_connect_worker(struct work_struct *work)
> int rc = 0;
>
> if (!xprt->shutdown) {
> + current->flags |= PF_FSTRANS;
> xprt_clear_connected(xprt);
>
> dprintk("RPC: %s: %sconnect\n", __func__,
> @@ -212,10 +213,10 @@ xprt_rdma_connect_worker(struct work_struct *work)
>
> out:
> xprt_wake_pending_tasks(xprt, rc);
> -
> out_clear:
> dprintk("RPC: %s: exit\n", __func__);
> xprt_clear_connecting(xprt);
> + current->flags &= ~PF_FSTRANS;
> }
>
> /*
> diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
> index 890b03f..b88c6bf 100644
> --- a/net/sunrpc/xprtsock.c
> +++ b/net/sunrpc/xprtsock.c
> @@ -1895,6 +1895,8 @@ static void xs_local_setup_socket(struct work_struct *work)
> if (xprt->shutdown)
> goto out;
>
> + current->flags |= PF_FSTRANS;
> +
> clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
> status = __sock_create(xprt->xprt_net, AF_LOCAL,
> SOCK_STREAM, 0, &sock, 1);
> @@ -1928,6 +1930,7 @@ static void xs_local_setup_socket(struct work_struct *work)
> out:
> xprt_clear_connecting(xprt);
> xprt_wake_pending_tasks(xprt, status);
> + current->flags &= ~PF_FSTRANS;
> }
>
> static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock)
> @@ -1970,6 +1973,8 @@ static void xs_udp_setup_socket(struct work_struct *work)
> if (xprt->shutdown)
> goto out;
>
> + current->flags |= PF_FSTRANS;
> +
> /* Start by resetting any existing state */
> xs_reset_transport(transport);
> sock = xs_create_sock(xprt, transport,
> @@ -1988,6 +1993,7 @@ static void xs_udp_setup_socket(struct work_struct *work)
> out:
> xprt_clear_connecting(xprt);
> xprt_wake_pending_tasks(xprt, status);
> + current->flags &= ~PF_FSTRANS;
> }
>
> /*
> @@ -2113,6 +2119,8 @@ static void xs_tcp_setup_socket(struct work_struct *work)
> if (xprt->shutdown)
> goto out;
>
> + current->flags |= PF_FSTRANS;
> +
> if (!sock) {
> clear_bit(XPRT_CONNECTION_ABORT, &xprt->state);
> sock = xs_create_sock(xprt, transport,
> @@ -2162,6 +2170,7 @@ static void xs_tcp_setup_socket(struct work_struct *work)
> case -EINPROGRESS:
> case -EALREADY:
> xprt_clear_connecting(xprt);
> + current->flags &= ~PF_FSTRANS;
> return;
> case -EINVAL:
> /* Happens, for instance, if the user specified a link
> @@ -2174,6 +2183,7 @@ out_eagain:
> out:
> xprt_clear_connecting(xprt);
> xprt_wake_pending_tasks(xprt, status);
> + current->flags &= ~PF_FSTRANS;
> }
>
> /**
FWIW, we have had a large customer roll out a RHEL6 kernel with a
backport of this patch to a large farm of machines that were seeing
this pretty regularly. It seems to have corrected the problem for them.
Do we also want to consider this for stable?
--
Jeff Layton <jlayton@xxxxxxxxxx>
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
