On Tue, Nov 01, 2022 at 10:46:45AM -0400, Jeff Layton wrote:
> The filecache refcounting is a bit non-standard for something searchable
> by RCU, in that we maintain a sentinel reference while it's hashed. This
> in turn requires that we have to do things differently in the "put"
> depending on whether its hashed, which we believe to have led to races.
>
> There are other problems in here too. nfsd_file_close_inode_sync can end
> up freeing an nfsd_file while there are still outstanding references to
> it, and there are a number of subtle ToC/ToU races.
>
> Rework the code so that the refcount is what drives the lifecycle. When
> the refcount goes to zero, then unhash and rcu free the object.
>
> With this change, the LRU carries a reference. Take special care to
> deal with it when removing an entry from the list.
>
> Signed-off-by: Jeff Layton <jlayton@xxxxxxxxxx>
Our test team is getting crashes that bisection pointed at this
patch. It seems like there are multiple parallel crash reports so the
whole thing is a mess to read:
[ 875.548965] BUG: kernel NULL pointer dereference, address: 00000000000000d0
[ 875.548968] ------------[ cut here ]------------
[ 875.548972] refcount_t: underflow; use-after-free.
[ 875.548992] WARNING: CPU: 4 PID: 12145 at lib/refcount.c:28 refcount_warn_saturate+0xd8/0xe0
[ 875.549851] #PF: supervisor read access in kernel mode
[ 875.550158] Modules linked in:
[ 875.550752] #PF: error_code(0x0000) - not-present page
[ 875.551269] nfsd
[ 875.551878] PGD 0
[ 875.552069] iptable_raw
[ 875.552677] P4D 0
[ 875.552824] bonding mlx5_vfio_pci
[ 875.553095]
[ 875.553255] rdma_ucm ipip
[ 875.553525] Oops: 0000 [#1] SMP
[ 875.553733] tunnel4
[ 875.553941] CPU: 0 PID: 12147 Comm: nfsd Not tainted 6.1.0-rc7_ac3a2585f018 #1
[ 875.554109] ip_gre ib_umad
[ 875.554517] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.554656] nf_tables vfio_pci
[ 875.555508] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.555695] vfio_pci_core vfio_virqfd
[ 875.557015] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.557209] vfio_iommu_type1
[ 875.557406] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.557634] mlx5_ib
[ 875.558446]
[ 875.558628] vfio
[ 875.558862] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.559006] ib_uverbs
[ 875.559092] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.559218] ib_ipoib
[ 875.559557] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.559704] mlx5_core
[ 875.560021] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.560165] ip6_gre
[ 875.560488] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.560634] gre
[ 875.560959] FS: 0000000000000000(0000) GS:ffff88852c800000(0000) knlGS:0000000000000000
[ 875.561108] ip6_tunnel
[ 875.561432] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.561554] tunnel6
[ 875.561928] CR2: 00000000000000d0 CR3: 00000001ca27d001 CR4: 0000000000372eb0
[ 875.562084] geneve
[ 875.562349] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.562493] nfnetlink_cttimeout
[ 875.562822] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.562962] openvswitch
[ 875.563292] Call Trace:
[ 875.563298] <TASK>
[ 875.563503] nsh
[ 875.563839] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.563997] vhost_net
[ 875.564124] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.564262] vhost
[ 875.564357] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.564661] vhost_iotlb
[ 875.564798] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.565084] tap
[ 875.565187] svc_process_common+0x2b6/0x4d0
[ 875.565483] ip6table_mangle
[ 875.565607] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.565878] ip6table_nat
[ 875.565972] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.566228] iptable_mangle
[ 875.566371] svc_process+0xd4/0xf0
[ 875.566622] ip6table_filter
[ 875.566748] nfsd+0xcb/0x180 [nfsd]
[ 875.567063] ip6_tables
[ 875.567194] kthread+0xb9/0xe0
[ 875.567412] xt_conntrack
[ 875.567557] ? kthread_complete_and_exit+0x20/0x20
[ 875.567776] xt_MASQUERADE
[ 875.567892] ret_from_fork+0x1f/0x30
[ 875.568084] nf_conntrack_netlink
[ 875.568212] </TASK>
[ 875.568500] nfnetlink
[ 875.568631] Modules linked in:
[ 875.568853] xt_addrtype
[ 875.569025] nfsd
[ 875.569167] iptable_nat
[ 875.569270] iptable_raw
[ 875.569464] nf_nat
[ 875.569572] bonding
[ 875.569701] br_netfilter
[ 875.569810] mlx5_vfio_pci
[ 875.569971] overlay
[ 875.570064] rdma_ucm
[ 875.570211] rpcrdma
[ 875.570317] ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs
[ 875.570492] ib_iser
[ 875.570590] ib_ipoib
[ 875.570737] libiscsi
[ 875.570834] mlx5_core
[ 875.571499] scsi_transport_iscsi
[ 875.571592] ip6_gre
[ 875.571736] rdma_cm
[ 875.571835] gre
[ 875.571984] iw_cm
[ 875.572126] ip6_tunnel
[ 875.572272] ib_cm
[ 875.572366] tunnel6
[ 875.572489] ib_core
[ 875.572581] geneve
[ 875.572738] fuse
[ 875.572834] nfnetlink_cttimeout
[ 875.572978] [last unloaded: nf_tables]
[ 875.573076] openvswitch
[ 875.573214]
[ 875.573299] nsh
[ 875.573503] CPU: 4 PID: 12145 Comm: nfsd Not tainted 6.1.0-rc7_ac3a2585f018 #1
[ 875.573666] vhost_net
[ 875.573826] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.573898] vhost
[ 875.574022] RIP: 0010:refcount_warn_saturate+0xd8/0xe0
[ 875.574318] vhost_iotlb
[ 875.574469] Code: ff 48 c7 c7 70 60 26 82 c6 05 d7 aa fb 00 01 e8 35 3c 4a 00 0f 0b c3 48 c7 c7 18 60 26 82 c6 05 c3 aa fb 00 01 e8 1f 3c 4a 00 <0f> 0b c3 0f 1f 44 00 00 8b 07 3d 00 00 00 c0 74 12 83 f8 01 74 13
[ 875.574913] tap
[ 875.575056] RSP: 0018:ffff88811d127db0 EFLAGS: 00010282
[ 875.575265] ip6table_mangle
[ 875.575426]
[ 875.576149] ip6table_nat
[ 875.576274] RAX: 0000000000000000 RBX: ffff88810669f138 RCX: ffff88852ca1b548
[ 875.576486] iptable_mangle
[ 875.576669] RDX: 00000000ffffffd8 RSI: 0000000000000027 RDI: ffff88852ca1b540
[ 875.576740] ip6table_filter
[ 875.576915] RBP: ffff8881028410f0 R08: 80000000fffff3b3 R09: ffff88811d127d48
[ 875.577203] ip6_tables
[ 875.577379] R10: 0000000000000b16 R11: 0000000000000001 R12: 0000000000000000
[ 875.577663] xt_conntrack
[ 875.577846] R13: ffff8881dd380e00 R14: ffff888110e5a028 R15: ffff888110e5a1a0
[ 875.578136] xt_MASQUERADE
[ 875.578294] FS: 0000000000000000(0000) GS:ffff88852ca00000(0000) knlGS:0000000000000000
[ 875.578579] nf_conntrack_netlink
[ 875.578754] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.579040] nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm
[ 875.579218] CR2: 00007fd8ec0017a8 CR3: 000000016a0db004 CR4: 0000000000372ea0
[ 875.579538] ib_cm
[ 875.579745] Call Trace:
[ 875.579977] ib_core
[ 875.580688] <TASK>
[ 875.580983] fuse
[ 875.581118] nfsd_file_free+0x1c4/0x200 [nfsd]
[ 875.581222] [last unloaded: nf_tables]
[ 875.581366] destroy_unhashed_deleg+0xac/0xc0 [nfsd]
[ 875.581457]
[ 875.581586] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.581775] CR2: 00000000000000d0
[ 875.582012] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.582216] ---[ end trace 0000000000000000 ]---
[ 875.582217] BUG: kernel NULL pointer dereference, address: 00000000000000d0
[ 875.582219] #PF: supervisor read access in kernel mode
[ 875.582220] #PF: error_code(0x0000) - not-present page
[ 875.582222] PGD 0 P4D 0
[ 875.582228] Oops: 0000 [#2] SMP
[ 875.582229] CPU: 8 PID: 12143 Comm: nfsd Tainted: G D 6.1.0-rc7_ac3a2585f018 #1
[ 875.582231] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.582232] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.582236] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.582238] RSP: 0018:ffff888119c97db0 EFLAGS: 00010246
[ 875.582239] RAX: 0000000000000000 RBX: ffff88824866c690 RCX: ffff888119c97dd8
[ 875.582241] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560b200
[ 875.582242] RBP: ffff88812560b200 R08: ffff8881dd381800 R09: ffffffff82407758
[ 875.582243] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.582244] R13: ffff8881dd381800 R14: ffff8881010cc028 R15: ffff8881010cc1a0
[ 875.582245] FS: 0000000000000000(0000) GS:ffff88852cc00000(0000) knlGS:0000000000000000
[ 875.582247] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.582248] CR2: 00000000000000d0 CR3: 000000011c26d004 CR4: 0000000000372ea0
[ 875.582250] Call Trace:
[ 875.582251] <TASK>
[ 875.582253] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.582268] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.582280] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.582292] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.582302] svc_process_common+0x2b6/0x4d0
[ 875.582305] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.582315] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.582323] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.582339] svc_process_common+0x2b6/0x4d0
[ 875.582343] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.582357] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.582371] svc_process+0xd4/0xf0
[ 875.582374] nfsd+0xcb/0x180 [nfsd]
[ 875.582388] kthread+0xb9/0xe0
[ 875.582392] ? kthread_complete_and_exit+0x20/0x20
[ 875.582398] ret_from_fork+0x1f/0x30
[ 875.582401] </TASK>
[ 875.582402] ---[ end trace 0000000000000000 ]---
[ 875.582518] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.582673] svc_process+0xd4/0xf0
[ 875.582873] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.583075] nfsd+0xcb/0x180 [nfsd]
[ 875.583350] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.583559] kthread+0xb9/0xe0
[ 875.583767]
[ 875.583878] ? kthread_complete_and_exit+0x20/0x20
[ 875.584013] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.584363] ret_from_fork+0x1f/0x30
[ 875.584810] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.584990] </TASK>
[ 875.585722] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.585936] Modules linked in:
[ 875.586219] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.586507] nfsd
[ 875.586793] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.587094] iptable_raw
[ 875.587383] FS: 0000000000000000(0000) GS:ffff88852c800000(0000) knlGS:0000000000000000
[ 875.587703] bonding
[ 875.587935] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.588223] mlx5_vfio_pci
[ 875.588331] CR2: 00000000000000d0 CR3: 00000001ca27d001 CR4: 0000000000372eb0
[ 875.588426] rdma_ucm
[ 875.588631] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.588835] ipip
[ 875.589035] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.589215] tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.650660] CR2: 00000000000000d0
[ 875.650952] ---[ end trace 0000000000000000 ]---
[ 875.650952] BUG: unable to handle page fault for address: 000000012547108f
[ 875.651147] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.651493] #PF: supervisor read access in kernel mode
[ 875.651670] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.651920] #PF: error_code(0x0000) - not-present page
[ 875.652644] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.652902] PGD 0
[ 875.653070]
[ 875.653329] P4D 0
[ 875.653418] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.653501]
[ 875.653593] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.653944] Oops: 0000 [#3] SMP
[ 875.654013] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.654359] CPU: 7 PID: 12140 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.654491] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.654837] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.655184] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.655529] RIP: 0010:vfs_setlease+0x1d/0x70
[ 875.655970] FS: 0000000000000000(0000) GS:ffff88852cc00000(0000) knlGS:0000000000000000
[ 875.656318] Code: 19 37 76 00 49 89 ee e9 ac fc ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f <48> 8b 45 28 4c 89 e2 48 89 ef 48 8b 80 d0 00 00 00 48 85 c0 74 2c
[ 875.656497] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.656889] RSP: 0018:ffff88824874bdb0 EFLAGS: 00010246
[ 875.657608] CR2: 00000000000000d0 CR3: 000000011c26d004 CR4: 0000000000372ea0
[ 875.657899]
[ 875.665061] RAX: ffff8881807150d0 RBX: ffff8881034da230 RCX: ffff88824874bdd8
[ 875.665670] RDX: 0000000000000000 RSI: 0000000000000002 RDI: 0000000125471067
[ 875.666275] RBP: 0000000125471067 R08: ffff888248d72500 R09: ffffffff82407758
[ 875.666886] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.667495] R13: ffff888248d72500 R14: ffff888103348028 R15: ffff8881033481a0
[ 875.668102] FS: 0000000000000000(0000) GS:ffff88852cb80000(0000) knlGS:0000000000000000
[ 875.668839] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.669347] CR2: 000000012547108f CR3: 00000001805ff002 CR4: 0000000000372ea0
[ 875.669950] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.670550] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.671160] Call Trace:
[ 875.671452] <TASK>
[ 875.671724] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.672197] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.672658] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.673139] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.673571] svc_process_common+0x2b6/0x4d0
[ 875.673980] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.674397] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.674877] svc_process+0xd4/0xf0
[ 875.675237] nfsd+0xcb/0x180 [nfsd]
[ 875.675606] kthread+0xb9/0xe0
[ 875.675942] ? kthread_complete_and_exit+0x20/0x20
[ 875.676394] ret_from_fork+0x1f/0x30
[ 875.676769] </TASK>
[ 875.677044] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.681102] CR2: 000000012547108f
[ 875.681459] ---[ end trace 0000000000000000 ]---
[ 875.681460] BUG: kernel NULL pointer dereference, address: 00000000000000d0
[ 875.681689] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.682042] #PF: supervisor read access in kernel mode
[ 875.682259] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.682517] #PF: error_code(0x0000) - not-present page
[ 875.683402] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.683656] PGD 0
[ 875.683865]
[ 875.684122] P4D 0
[ 875.684233] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.684321]
[ 875.684430] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.684787] Oops: 0000 [#4] SMP
[ 875.684874] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.685226] CPU: 3 PID: 12146 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.685397] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.685748] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.686175] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.686525] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.687063] FS: 0000000000000000(0000) GS:ffff88852cb80000(0000) knlGS:0000000000000000
[ 875.687412] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.687629] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.688023] RSP: 0018:ffff88811b81fdb0 EFLAGS: 00010246
[ 875.688915] CR2: 000000012547108f CR3: 00000001805ff002 CR4: 0000000000372ea0
[ 875.689200]
[ 875.689462] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.689809] RAX: 0000000000000000 RBX: ffff88824866c230 RCX: ffff88811b81fdd8
[ 875.689896] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.690242] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560ad00
[ 875.698805] RBP: ffff88812560ad00 R08: ffff8881dd381b00 R09: ffffffff82407740
[ 875.699422] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.700031] R13: ffff8881dd381b00 R14: ffff888110e5e028 R15: ffff888110e5e1a0
[ 875.700643] FS: 0000000000000000(0000) GS:ffff88852c980000(0000) knlGS:0000000000000000
[ 875.701384] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.701892] CR2: 00000000000000d0 CR3: 0000000180622003 CR4: 0000000000372ea0
[ 875.702501] Call Trace:
[ 875.702797] <TASK>
[ 875.703065] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.703536] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.703992] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.704463] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.704897] svc_process_common+0x2b6/0x4d0
[ 875.705305] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.705717] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.706194] svc_process+0xd4/0xf0
[ 875.706549] nfsd+0xcb/0x180 [nfsd]
[ 875.706919] kthread+0xb9/0xe0
[ 875.707257] ? kthread_complete_and_exit+0x20/0x20
[ 875.707706] ret_from_fork+0x1f/0x30
[ 875.708073] </TASK>
[ 875.708348] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.721375] CR2: 00000000000000d0
[ 875.721738] ---[ end trace 0000000000000000 ]---
[ 875.721738] BUG: kernel NULL pointer dereference, address: 0000000000000028
[ 875.721972] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.722317] #PF: supervisor read access in kernel mode
[ 875.722530] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.722782] #PF: error_code(0x0000) - not-present page
[ 875.723675] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.723929] PGD 0
[ 875.724135]
[ 875.724393] P4D 0
[ 875.724499] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.724584]
[ 875.724695] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.725055] Oops: 0000 [#5] SMP
[ 875.725140] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.725485] CPU: 2 PID: 12142 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.725647] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.725992] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.726411] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.726755] RIP: 0010:vfs_setlease+0x1d/0x70
[ 875.727285] FS: 0000000000000000(0000) GS:ffff88852c980000(0000) knlGS:0000000000000000
[ 875.727631] Code: 19 37 76 00 49 89 ee e9 ac fc ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f <48> 8b 45 28 4c 89 e2 48 89 ef 48 8b 80 d0 00 00 00 48 85 c0 74 2c
[ 875.727843] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.728225] RSP: 0018:ffff88811b813db0 EFLAGS: 00010246
[ 875.729115] CR2: 00000000000000d0 CR3: 0000000180622003 CR4: 0000000000372ea0
[ 875.729396]
[ 875.735502] RAX: ffff888180715000 RBX: ffff8881034da000 RCX: ffff88811b813dd8
[ 875.736015] RDX: 0000000000000000 RSI: 0000000000000002 RDI: 0000000000000000
[ 875.736531] RBP: 0000000000000000 R08: ffff888248d73200 R09: ffffffff824065b8
[ 875.737043] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.737560] R13: ffff888248d73200 R14: ffff88824501c028 R15: ffff88824501c1a0
[ 875.738063] FS: 0000000000000000(0000) GS:ffff88852c900000(0000) knlGS:0000000000000000
[ 875.738678] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.739118] CR2: 0000000000000028 CR3: 0000000105b6e003 CR4: 0000000000372ea0
[ 875.739636] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.740138] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.740658] Call Trace:
[ 875.740907] <TASK>
[ 875.741135] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.741558] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.741941] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.742334] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.742711] svc_process_common+0x2b6/0x4d0
[ 875.743064] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.743406] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.743816] svc_process+0xd4/0xf0
[ 875.744121] nfsd+0xcb/0x180 [nfsd]
[ 875.744437] kthread+0xb9/0xe0
[ 875.744733] ? kthread_complete_and_exit+0x20/0x20
[ 875.745123] ret_from_fork+0x1f/0x30
[ 875.745434] </TASK>
[ 875.745663] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.749118] CR2: 0000000000000028
[ 875.749415] ---[ end trace 0000000000000000 ]---
[ 875.749415] BUG: kernel NULL pointer dereference, address: 0000000000000028
[ 875.749614] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.750031] #PF: supervisor read access in kernel mode
[ 875.750214] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.750521] #PF: error_code(0x0000) - not-present page
[ 875.751290] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.751596] PGD 0
[ 875.751766]
[ 875.752079] P4D 0
[ 875.752173] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.752276]
[ 875.752371] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.752797] Oops: 0000 [#6] SMP
[ 875.752874] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.753291] CPU: 4 PID: 12145 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.753429] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.753848] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.754219] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.754638] RIP: 0010:vfs_setlease+0x1d/0x70
[ 875.755100] FS: 0000000000000000(0000) GS:ffff88852c900000(0000) knlGS:0000000000000000
[ 875.755522] Code: 19 37 76 00 49 89 ee e9 ac fc ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f <48> 8b 45 28 4c 89 e2 48 89 ef 48 8b 80 d0 00 00 00 48 85 c0 74 2c
[ 875.755709] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.756176] RSP: 0018:ffff88811d127db0 EFLAGS: 00010246
[ 875.756949] CR2: 0000000000000028 CR3: 0000000105b6e003 CR4: 0000000000372ea0
[ 875.757293]
[ 875.757527] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.757946] RAX: ffff888180715068 RBX: ffff8881034da118 RCX: ffff88811d127dd8
[ 875.758023] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.758441] RDX: 0000000000000000 RSI: 0000000000000002 RDI: 0000000000000000
[ 875.768979] RBP: 0000000000000000 R08: ffff888248d72a00 R09: ffffffff824067b0
[ 875.769725] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.770466] R13: ffff888248d72a00 R14: ffff888110e5a028 R15: ffff888110e5a1a0
[ 875.771201] FS: 0000000000000000(0000) GS:ffff88852ca00000(0000) knlGS:0000000000000000
[ 875.772096] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.772725] CR2: 0000000000000028 CR3: 0000000113338003 CR4: 0000000000372ea0
[ 875.773464] Call Trace:
[ 875.773824] <TASK>
[ 875.774160] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.774737] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.775292] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.775863] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.776398] svc_process_common+0x2b6/0x4d0
[ 875.776910] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.777411] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.777994] svc_process+0xd4/0xf0
[ 875.778437] nfsd+0xcb/0x180 [nfsd]
[ 875.778894] kthread+0xb9/0xe0
[ 875.779305] ? kthread_complete_and_exit+0x20/0x20
[ 875.779859] ret_from_fork+0x1f/0x30
[ 875.780316] </TASK>
[ 875.780651] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.785657] CR2: 0000000000000028
[ 875.786087] ---[ end trace 0000000000000000 ]---
[ 875.786088] BUG: kernel NULL pointer dereference, address: 0000000000000028
[ 875.786366] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.786700] #PF: supervisor read access in kernel mode
[ 875.786962] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.787209] #PF: error_code(0x0000) - not-present page
[ 875.788286] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.788528] PGD 0
[ 875.788794]
[ 875.789047] P4D 0
[ 875.789181] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.789264]
[ 875.789396] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.789735] Oops: 0000 [#7] SMP
[ 875.789841] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.790181] CPU: 0 PID: 12141 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.790380] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.790715] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.791221] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.791561] RIP: 0010:vfs_setlease+0x1d/0x70
[ 875.792214] FS: 0000000000000000(0000) GS:ffff88852ca00000(0000) knlGS:0000000000000000
[ 875.792552] Code: 19 37 76 00 49 89 ee e9 ac fc ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f <48> 8b 45 28 4c 89 e2 48 89 ef 48 8b 80 d0 00 00 00 48 85 c0 74 2c
[ 875.792825] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.793203] RSP: 0018:ffff888244b23db0 EFLAGS: 00010246
[ 875.794279] CR2: 0000000000000028 CR3: 0000000113338003 CR4: 0000000000372ea0
[ 875.794556]
[ 875.801646] RAX: ffff8881807151a0 RBX: ffff8881034da460 RCX: ffff888244b23dd8
[ 875.802232] RDX: 0000000000000000 RSI: 0000000000000002 RDI: 0000000000000000
[ 875.802821] RBP: 0000000000000000 R08: ffff888248d72f00 R09: ffffffff824062b8
[ 875.812350] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.812947] R13: ffff888248d72f00 R14: ffff888245018028 R15: ffff8882450181a0
[ 875.813536] FS: 0000000000000000(0000) GS:ffff88852c800000(0000) knlGS:0000000000000000
[ 875.814245] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.814741] CR2: 0000000000000028 CR3: 00000001ca27d001 CR4: 0000000000372eb0
[ 875.815331] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.815919] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.816504] Call Trace:
[ 875.816801] <TASK>
[ 875.817061] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.817525] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.817968] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.818422] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.818838] svc_process_common+0x2b6/0x4d0
[ 875.819238] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.819642] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.820101] svc_process+0xd4/0xf0
[ 875.820449] nfsd+0xcb/0x180 [nfsd]
[ 875.820821] kthread+0xb9/0xe0
[ 875.821149] ? kthread_complete_and_exit+0x20/0x20
[ 875.821590] ret_from_fork+0x1f/0x30
[ 875.821946] </TASK>
[ 875.822213] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.826147] CR2: 0000000000000028
[ 875.826491] ---[ end trace 0000000000000000 ]---
[ 875.826492] BUG: kernel NULL pointer dereference, address: 00000000000000d0
[ 875.826715] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.827026] #PF: supervisor read access in kernel mode
[ 875.827237] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.827455] #PF: error_code(0x0000) - not-present page
[ 875.828299] RSP: 0018:ffff88810378bdb0 EFLAGS: 00010246
[ 875.828516] PGD 0 P4D 0
[ 875.828722]
[ 875.828943]
[ 875.829075] RAX: 0000000000000000 RBX: ffff88824866c000 RCX: ffff88810378bdd8
[ 875.829152] Oops: 0000 [#8] SMP
[ 875.829234] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560a200
[ 875.829529] CPU: 6 PID: 12144 Comm: nfsd Tainted: G D W 6.1.0-rc7_ac3a2585f018 #1
[ 875.829690] RBP: ffff88812560a200 R08: ffff8881da5ecf00 R09: ffffffff824064e0
[ 875.829993] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.13.0-0-gf21b5a4aeb02-prebuilt.qemu.org 04/01/2014
[ 875.830400] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.830699] RIP: 0010:vfs_setlease+0x27/0x70
[ 875.831221] R13: ffff8881da5ecf00 R14: ffff888110e62028 R15: ffff888110e621a0
[ 875.831516] Code: ff ff 90 0f 1f 44 00 00 41 54 49 89 d4 55 48 89 fd 48 83 ec 10 48 85 d2 74 06 48 83 fe 02 75 1f 48 8b 45 28 4c 89 e2 48 89 ef <48> 8b 80 d0 00 00 00 48 85 c0 74 2c 48 83 c4 10 5d 41 5c ff e0 48
[ 875.831723] FS: 0000000000000000(0000) GS:ffff88852c800000(0000) knlGS:0000000000000000
[ 875.832014] RSP: 0018:ffff88811467fdb0 EFLAGS: 00010246
[ 875.832874] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.833205]
[ 875.833454] CR2: 0000000000000028 CR3: 00000001ca27d001 CR4: 0000000000372eb0
[ 875.833698] RAX: 0000000000000000 RBX: ffff88824866c460 RCX: ffff88811467fdd8
[ 875.833784] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.834078] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff88812560b000
[ 875.834411] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.834706] RBP: ffff88812560b000 R08: ffff8881dd381300 R09: ffffffff824069c0
[ 875.841961] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 875.842471] R13: ffff8881dd381300 R14: ffff8881010ca028 R15: ffff8881010ca1a0
[ 875.842999] FS: 0000000000000000(0000) GS:ffff88852cb00000(0000) knlGS:0000000000000000
[ 875.843608] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 875.844056] CR2: 00000000000000d0 CR3: 0000000105b6e002 CR4: 0000000000372ea0
[ 875.844565] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 875.845081] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 875.845590] Call Trace:
[ 875.845835] <TASK>
[ 875.846072] destroy_unhashed_deleg+0x58/0xc0 [nfsd]
[ 875.846465] nfsd4_delegreturn+0x119/0x150 [nfsd]
[ 875.846850] nfsd4_proc_compound+0x282/0x5d0 [nfsd]
[ 875.847263] nfsd_dispatch+0x15d/0x250 [nfsd]
[ 875.847627] svc_process_common+0x2b6/0x4d0
[ 875.847968] ? nfsd_svc+0x330/0x330 [nfsd]
[ 875.848322] ? nfsd_shutdown_threads+0x90/0x90 [nfsd]
[ 875.848720] svc_process+0xd4/0xf0
[ 875.849025] nfsd+0xcb/0x180 [nfsd]
[ 875.849344] kthread+0xb9/0xe0
[ 875.849625] ? kthread_complete_and_exit+0x20/0x20
[ 875.850001] ret_from_fork+0x1f/0x30
[ 875.850313] </TASK>
[ 875.850539] Modules linked in: nfsd iptable_raw bonding mlx5_vfio_pci rdma_ucm ipip tunnel4 ip_gre ib_umad nf_tables vfio_pci vfio_pci_core vfio_virqfd vfio_iommu_type1 mlx5_ib vfio ib_uverbs ib_ipoib mlx5_core ip6_gre gre ip6_tunnel tunnel6 geneve nfnetlink_cttimeout openvswitch nsh vhost_net vhost vhost_iotlb tap ip6table_mangle ip6table_nat iptable_mangle ip6table_filter ip6_tables xt_conntrack xt_MASQUERADE nf_conntrack_netlink nfnetlink xt_addrtype iptable_nat nf_nat br_netfilter overlay rpcrdma ib_iser libiscsi scsi_transport_iscsi rdma_cm iw_cm ib_cm ib_core fuse [last unloaded: nf_tables]
[ 875.853944] CR2: 00000000000000d0
[ 875.854240] ---[ end trace 0000000000000000 ]---
> ---
> fs/nfsd/filecache.c | 247 ++++++++++++++++++++++----------------------
> fs/nfsd/trace.h | 1 +
> 2 files changed, 123 insertions(+), 125 deletions(-)
>
> diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
> index 0bf3727455e2..e67297ad12bf 100644
> --- a/fs/nfsd/filecache.c
> +++ b/fs/nfsd/filecache.c
> @@ -303,8 +303,7 @@ nfsd_file_alloc(struct nfsd_file_lookup_key *key, unsigned int may)
> if (key->gc)
> __set_bit(NFSD_FILE_GC, &nf->nf_flags);
> nf->nf_inode = key->inode;
> - /* nf_ref is pre-incremented for hash table */
> - refcount_set(&nf->nf_ref, 2);
> + refcount_set(&nf->nf_ref, 1);
> nf->nf_may = key->need;
> nf->nf_mark = NULL;
> }
> @@ -353,24 +352,35 @@ nfsd_file_unhash(struct nfsd_file *nf)
> return false;
> }
>
> -static bool
> +static void
> nfsd_file_free(struct nfsd_file *nf)
> {
> s64 age = ktime_to_ms(ktime_sub(ktime_get(), nf->nf_birthtime));
> - bool flush = false;
>
> trace_nfsd_file_free(nf);
>
> this_cpu_inc(nfsd_file_releases);
> this_cpu_add(nfsd_file_total_age, age);
>
> + nfsd_file_unhash(nf);
> +
> + /*
> + * We call fsync here in order to catch writeback errors. It's not
> + * strictly required by the protocol, but an nfsd_file coule get
> + * evicted from the cache before a COMMIT comes in. If another
> + * task were to open that file in the interim and scrape the error,
> + * then the client may never see it. By calling fsync here, we ensure
> + * that writeback happens before the entry is freed, and that any
> + * errors reported result in the write verifier changing.
> + */
> + nfsd_file_fsync(nf);
> +
> if (nf->nf_mark)
> nfsd_file_mark_put(nf->nf_mark);
> if (nf->nf_file) {
> get_file(nf->nf_file);
> filp_close(nf->nf_file, NULL);
> fput(nf->nf_file);
> - flush = true;
> }
>
> /*
> @@ -378,10 +388,9 @@ nfsd_file_free(struct nfsd_file *nf)
> * WARN and leak it to preserve system stability.
> */
> if (WARN_ON_ONCE(!list_empty(&nf->nf_lru)))
> - return flush;
> + return;
>
> call_rcu(&nf->nf_rcu, nfsd_file_slab_free);
> - return flush;
> }
>
> static bool
> @@ -397,17 +406,23 @@ nfsd_file_check_writeback(struct nfsd_file *nf)
> mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK);
> }
>
> -static void nfsd_file_lru_add(struct nfsd_file *nf)
> +static bool nfsd_file_lru_add(struct nfsd_file *nf)
> {
> set_bit(NFSD_FILE_REFERENCED, &nf->nf_flags);
> - if (list_lru_add(&nfsd_file_lru, &nf->nf_lru))
> + if (list_lru_add(&nfsd_file_lru, &nf->nf_lru)) {
> trace_nfsd_file_lru_add(nf);
> + return true;
> + }
> + return false;
> }
>
> -static void nfsd_file_lru_remove(struct nfsd_file *nf)
> +static bool nfsd_file_lru_remove(struct nfsd_file *nf)
> {
> - if (list_lru_del(&nfsd_file_lru, &nf->nf_lru))
> + if (list_lru_del(&nfsd_file_lru, &nf->nf_lru)) {
> trace_nfsd_file_lru_del(nf);
> + return true;
> + }
> + return false;
> }
>
> struct nfsd_file *
> @@ -418,86 +433,80 @@ nfsd_file_get(struct nfsd_file *nf)
> return NULL;
> }
>
> -static void
> +/**
> + * nfsd_file_unhash_and_queue - unhash a file and queue it to the dispose list
> + * @nf: nfsd_file to be unhashed and queued
> + * @dispose: list to which it should be queued
> + *
> + * Attempt to unhash a nfsd_file and queue it to the given list. Each file
> + * will have a reference held on behalf of the list. That reference may come
> + * from the LRU, or we may need to take one. If we can't get a reference,
> + * ignore it altogether.
> + */
> +static bool
> nfsd_file_unhash_and_queue(struct nfsd_file *nf, struct list_head *dispose)
> {
> trace_nfsd_file_unhash_and_queue(nf);
> if (nfsd_file_unhash(nf)) {
> - /* caller must call nfsd_file_dispose_list() later */
> - nfsd_file_lru_remove(nf);
> + /*
> + * If we remove it from the LRU, then just use that
> + * reference for the dispose list. Otherwise, we need
> + * to take a reference. If that fails, just ignore
> + * the file altogether.
> + */
> + if (!nfsd_file_lru_remove(nf) && !nfsd_file_get(nf))
> + return false;
> list_add(&nf->nf_lru, dispose);
> + return true;
> }
> + return false;
> }
>
> -static void
> -nfsd_file_put_noref(struct nfsd_file *nf)
> -{
> - trace_nfsd_file_put(nf);
> -
> - if (refcount_dec_and_test(&nf->nf_ref)) {
> - WARN_ON(test_bit(NFSD_FILE_HASHED, &nf->nf_flags));
> - nfsd_file_lru_remove(nf);
> - nfsd_file_free(nf);
> - }
> -}
> -
> -static void
> -nfsd_file_unhash_and_put(struct nfsd_file *nf)
> -{
> - if (nfsd_file_unhash(nf))
> - nfsd_file_put_noref(nf);
> -}
> -
> +/**
> + * nfsd_file_put - put the reference to a nfsd_file
> + * @nf: nfsd_file of which to put the reference
> + *
> + * Put a reference to a nfsd_file. In the v4 case, we just put the
> + * reference immediately. In the GC case, if the reference would be
> + * the last one, the put it on the LRU instead to be cleaned up later.
> + */
> void
> nfsd_file_put(struct nfsd_file *nf)
> {
> might_sleep();
> + trace_nfsd_file_put(nf);
>
> - if (test_bit(NFSD_FILE_GC, &nf->nf_flags))
> - nfsd_file_lru_add(nf);
> - else if (refcount_read(&nf->nf_ref) == 2)
> - nfsd_file_unhash_and_put(nf);
> -
> - if (!test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
> - nfsd_file_fsync(nf);
> - nfsd_file_put_noref(nf);
> - } else if (nf->nf_file && test_bit(NFSD_FILE_GC, &nf->nf_flags)) {
> - nfsd_file_put_noref(nf);
> - nfsd_file_schedule_laundrette();
> - } else
> - nfsd_file_put_noref(nf);
> -}
> -
> -static void
> -nfsd_file_dispose_list(struct list_head *dispose)
> -{
> - struct nfsd_file *nf;
> -
> - while(!list_empty(dispose)) {
> - nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
> - list_del_init(&nf->nf_lru);
> - nfsd_file_fsync(nf);
> - nfsd_file_put_noref(nf);
> + /*
> + * The HASHED check is racy. We may end up with the occasional
> + * unhashed entry on the LRU, but they should get cleaned up
> + * like any other.
> + */
> + if (test_bit(NFSD_FILE_GC, &nf->nf_flags) &&
> + test_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
> + /*
> + * If this is the last reference (nf_ref == 1), then transfer
> + * it to the LRU. If the add to the LRU fails, just put it as
> + * usual.
> + */
> + if (refcount_dec_not_one(&nf->nf_ref) || nfsd_file_lru_add(nf)) {
> + nfsd_file_schedule_laundrette();
> + return;
> + }
> }
> + if (refcount_dec_and_test(&nf->nf_ref))
> + nfsd_file_free(nf);
> }
>
> static void
> -nfsd_file_dispose_list_sync(struct list_head *dispose)
> +nfsd_file_dispose_list(struct list_head *dispose)
> {
> - bool flush = false;
> struct nfsd_file *nf;
>
> while(!list_empty(dispose)) {
> nf = list_first_entry(dispose, struct nfsd_file, nf_lru);
> list_del_init(&nf->nf_lru);
> - nfsd_file_fsync(nf);
> - if (!refcount_dec_and_test(&nf->nf_ref))
> - continue;
> - if (nfsd_file_free(nf))
> - flush = true;
> + nfsd_file_free(nf);
> }
> - if (flush)
> - flush_delayed_fput();
> }
>
> static void
> @@ -567,21 +576,8 @@ nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
> struct list_head *head = arg;
> struct nfsd_file *nf = list_entry(item, struct nfsd_file, nf_lru);
>
> - /*
> - * Do a lockless refcount check. The hashtable holds one reference, so
> - * we look to see if anything else has a reference, or if any have
> - * been put since the shrinker last ran. Those don't get unhashed and
> - * released.
> - *
> - * Note that in the put path, we set the flag and then decrement the
> - * counter. Here we check the counter and then test and clear the flag.
> - * That order is deliberate to ensure that we can do this locklessly.
> - */
> - if (refcount_read(&nf->nf_ref) > 1) {
> - list_lru_isolate(lru, &nf->nf_lru);
> - trace_nfsd_file_gc_in_use(nf);
> - return LRU_REMOVED;
> - }
> + /* We should only be dealing with GC entries here */
> + WARN_ON_ONCE(!test_bit(NFSD_FILE_GC, &nf->nf_flags));
>
> /*
> * Don't throw out files that are still undergoing I/O or
> @@ -592,40 +588,30 @@ nfsd_file_lru_cb(struct list_head *item, struct list_lru_one *lru,
> return LRU_SKIP;
> }
>
> + /* If it was recently added to the list, skip it */
> if (test_and_clear_bit(NFSD_FILE_REFERENCED, &nf->nf_flags)) {
> trace_nfsd_file_gc_referenced(nf);
> return LRU_ROTATE;
> }
>
> - if (!test_and_clear_bit(NFSD_FILE_HASHED, &nf->nf_flags)) {
> - trace_nfsd_file_gc_hashed(nf);
> - return LRU_SKIP;
> + /*
> + * Put the reference held on behalf of the LRU. If it wasn't the last
> + * one, then just remove it from the LRU and ignore it.
> + */
> + if (!refcount_dec_and_test(&nf->nf_ref)) {
> + trace_nfsd_file_gc_in_use(nf);
> + list_lru_isolate(lru, &nf->nf_lru);
> + return LRU_REMOVED;
> }
>
> + /* Refcount went to zero. Unhash it and queue it to the dispose list */
> + nfsd_file_unhash(nf);
> list_lru_isolate_move(lru, &nf->nf_lru, head);
> this_cpu_inc(nfsd_file_evictions);
> trace_nfsd_file_gc_disposed(nf);
> return LRU_REMOVED;
> }
>
> -/*
> - * Unhash items on @dispose immediately, then queue them on the
> - * disposal workqueue to finish releasing them in the background.
> - *
> - * cel: Note that between the time list_lru_shrink_walk runs and
> - * now, these items are in the hash table but marked unhashed.
> - * Why release these outside of lru_cb ? There's no lock ordering
> - * problem since lru_cb currently takes no lock.
> - */
> -static void nfsd_file_gc_dispose_list(struct list_head *dispose)
> -{
> - struct nfsd_file *nf;
> -
> - list_for_each_entry(nf, dispose, nf_lru)
> - nfsd_file_hash_remove(nf);
> - nfsd_file_dispose_list_delayed(dispose);
> -}
> -
> static void
> nfsd_file_gc(void)
> {
> @@ -635,7 +621,7 @@ nfsd_file_gc(void)
> ret = list_lru_walk(&nfsd_file_lru, nfsd_file_lru_cb,
> &dispose, list_lru_count(&nfsd_file_lru));
> trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru));
> - nfsd_file_gc_dispose_list(&dispose);
> + nfsd_file_dispose_list_delayed(&dispose);
> }
>
> static void
> @@ -660,7 +646,7 @@ nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc)
> ret = list_lru_shrink_walk(&nfsd_file_lru, sc,
> nfsd_file_lru_cb, &dispose);
> trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru));
> - nfsd_file_gc_dispose_list(&dispose);
> + nfsd_file_dispose_list_delayed(&dispose);
> return ret;
> }
>
> @@ -671,8 +657,11 @@ static struct shrinker nfsd_file_shrinker = {
> };
>
> /*
> - * Find all cache items across all net namespaces that match @inode and
> - * move them to @dispose. The lookup is atomic wrt nfsd_file_acquire().
> + * Find all cache items across all net namespaces that match @inode, unhash
> + * them, take references and then put them on @dispose if that was successful.
> + *
> + * The nfsd_file objects on the list will be unhashed, and each will have a
> + * reference taken.
> */
> static unsigned int
> __nfsd_file_close_inode(struct inode *inode, struct list_head *dispose)
> @@ -690,8 +679,9 @@ __nfsd_file_close_inode(struct inode *inode, struct list_head *dispose)
> nfsd_file_rhash_params);
> if (!nf)
> break;
> - nfsd_file_unhash_and_queue(nf, dispose);
> - count++;
> +
> + if (nfsd_file_unhash_and_queue(nf, dispose))
> + count++;
> } while (1);
> rcu_read_unlock();
> return count;
> @@ -703,15 +693,23 @@ __nfsd_file_close_inode(struct inode *inode, struct list_head *dispose)
> *
> * Unhash and put all cache item associated with @inode.
> */
> -static void
> +static unsigned int
> nfsd_file_close_inode(struct inode *inode)
> {
> - LIST_HEAD(dispose);
> + struct nfsd_file *nf;
> unsigned int count;
> + LIST_HEAD(dispose);
>
> count = __nfsd_file_close_inode(inode, &dispose);
> trace_nfsd_file_close_inode(inode, count);
> - nfsd_file_dispose_list_delayed(&dispose);
> + while(!list_empty(&dispose)) {
> + nf = list_first_entry(&dispose, struct nfsd_file, nf_lru);
> + list_del_init(&nf->nf_lru);
> + trace_nfsd_file_closing(nf);
> + if (refcount_dec_and_test(&nf->nf_ref))
> + nfsd_file_free(nf);
> + }
> + return count;
> }
>
> /**
> @@ -723,19 +721,15 @@ nfsd_file_close_inode(struct inode *inode)
> void
> nfsd_file_close_inode_sync(struct inode *inode)
> {
> - LIST_HEAD(dispose);
> - unsigned int count;
> -
> - count = __nfsd_file_close_inode(inode, &dispose);
> - trace_nfsd_file_close_inode_sync(inode, count);
> - nfsd_file_dispose_list_sync(&dispose);
> + if (nfsd_file_close_inode(inode))
> + flush_delayed_fput();
> }
>
> /**
> * nfsd_file_delayed_close - close unused nfsd_files
> * @work: dummy
> *
> - * Walk the LRU list and close any entries that have not been used since
> + * Walk the LRU list and destroy any entries that have not been used since
> * the last scan.
> */
> static void
> @@ -1056,8 +1050,10 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
> rcu_read_lock();
> nf = rhashtable_lookup(&nfsd_file_rhash_tbl, &key,
> nfsd_file_rhash_params);
> - if (nf)
> - nf = nfsd_file_get(nf);
> + if (nf) {
> + if (!nfsd_file_lru_remove(nf))
> + nf = nfsd_file_get(nf);
> + }
> rcu_read_unlock();
> if (nf)
> goto wait_for_construction;
> @@ -1092,11 +1088,11 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
> goto out;
> }
> open_retry = false;
> - nfsd_file_put_noref(nf);
> + if (refcount_dec_and_test(&nf->nf_ref))
> + nfsd_file_free(nf);
> goto retry;
> }
>
> - nfsd_file_lru_remove(nf);
> this_cpu_inc(nfsd_file_cache_hits);
>
> status = nfserrno(nfsd_open_break_lease(file_inode(nf->nf_file), may_flags));
> @@ -1106,7 +1102,8 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
> this_cpu_inc(nfsd_file_acquisitions);
> *pnf = nf;
> } else {
> - nfsd_file_put(nf);
> + if (refcount_dec_and_test(&nf->nf_ref))
> + nfsd_file_free(nf);
> nf = NULL;
> }
>
> @@ -1133,7 +1130,7 @@ nfsd_file_do_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
> * then unhash.
> */
> if (status != nfs_ok || key.inode->i_nlink == 0)
> - nfsd_file_unhash_and_put(nf);
> + nfsd_file_unhash(nf);
> clear_bit_unlock(NFSD_FILE_PENDING, &nf->nf_flags);
> smp_mb__after_atomic();
> wake_up_bit(&nf->nf_flags, NFSD_FILE_PENDING);
> diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
> index 940252482fd4..a44ded06af87 100644
> --- a/fs/nfsd/trace.h
> +++ b/fs/nfsd/trace.h
> @@ -906,6 +906,7 @@ DEFINE_EVENT(nfsd_file_class, name, \
> DEFINE_NFSD_FILE_EVENT(nfsd_file_free);
> DEFINE_NFSD_FILE_EVENT(nfsd_file_unhash);
> DEFINE_NFSD_FILE_EVENT(nfsd_file_put);
> +DEFINE_NFSD_FILE_EVENT(nfsd_file_closing);
> DEFINE_NFSD_FILE_EVENT(nfsd_file_unhash_and_queue);
>
> TRACE_EVENT(nfsd_file_alloc,
