Frank Sorenson <frank@xxxxxxxxxxxx> writes:
> I'm getting a 'WARNING: possible circular locking dependency detected'
> followed by a DEADLOCK when I access /proc/fs/cifs/DebugData while doing
> a lot of mounts/unmounts/IO, etc.
Could you please retry with attached patches?
I was able to reproduce an GPF with a similar test by running xfstests
from Steve's for-next branch with
'vers=3.1.1,multichannel,max_channels=2'
against Windows Server 2022.
No deadlocks or lockdep warnings. The deadlock fix from second patch
was just based on code analysis.
>From f5d9cff737b04d99510e7b824253a36652b185d9 Mon Sep 17 00:00:00 2001
From: Paulo Alcantara <pc@xxxxxxxxxxxxx>
Date: Tue, 24 Oct 2023 13:49:15 -0300
Subject: [PATCH 1/2] smb: client: fix use-after-free bug in
cifs_debug_data_proc_show()
Skip SMB sessions that are being teared down
(e.g. @ses->ses_status == SES_EXITING) in cifs_debug_data_proc_show()
to avoid use-after-free in @ses.
This fixes the following GPF when reading from /proc/fs/cifs/DebugData
while mounting and umounting
[ 816.251274] general protection fault, probably for non-canonical
address 0x6b6b6b6b6b6b6d81: 0000 [#1] PREEMPT SMP NOPTI
...
[ 816.260138] Call Trace:
[ 816.260329] <TASK>
[ 816.260499] ? die_addr+0x36/0x90
[ 816.260762] ? exc_general_protection+0x1b3/0x410
[ 816.261126] ? asm_exc_general_protection+0x26/0x30
[ 816.261502] ? cifs_debug_tcon+0xbd/0x240 [cifs]
[ 816.261878] ? cifs_debug_tcon+0xab/0x240 [cifs]
[ 816.262249] cifs_debug_data_proc_show+0x516/0xdb0 [cifs]
[ 816.262689] ? seq_read_iter+0x379/0x470
[ 816.262995] seq_read_iter+0x118/0x470
[ 816.263291] proc_reg_read_iter+0x53/0x90
[ 816.263596] ? srso_alias_return_thunk+0x5/0x7f
[ 816.263945] vfs_read+0x201/0x350
[ 816.264211] ksys_read+0x75/0x100
[ 816.264472] do_syscall_64+0x3f/0x90
[ 816.264750] entry_SYSCALL_64_after_hwframe+0x6e/0xd8
[ 816.265135] RIP: 0033:0x7fd5e669d381
Signed-off-by: Paulo Alcantara (SUSE) <pc@xxxxxxxxxxxxx>
---
fs/smb/client/cifs_debug.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c
index 76922fcc4bc6..9a0ccd87468e 100644
--- a/fs/smb/client/cifs_debug.c
+++ b/fs/smb/client/cifs_debug.c
@@ -452,6 +452,11 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
seq_printf(m, "\n\n\tSessions: ");
i = 0;
list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) {
+ spin_lock(&ses->ses_lock);
+ if (ses->ses_status == SES_EXITING) {
+ spin_unlock(&ses->ses_lock);
+ continue;
+ }
i++;
if ((ses->serverDomain == NULL) ||
(ses->serverOS == NULL) ||
@@ -472,6 +477,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v)
ses->ses_count, ses->serverOS, ses->serverNOS,
ses->capabilities, ses->ses_status);
}
+ spin_unlock(&ses->ses_lock);
seq_printf(m, "\n\tSecurity type: %s ",
get_security_type_str(server->ops->select_sectype(server, ses->sectype)));
--
2.42.0
>From 42a1b90a00bfb4315d1d30b17ca8524d5ece4f82 Mon Sep 17 00:00:00 2001
From: Paulo Alcantara <pc@xxxxxxxxxxxxx>
Date: Wed, 25 Oct 2023 14:58:35 -0300
Subject: [PATCH 2/2] smb: client: fix potential deadlock when releasing mids
All release_mid() callers seem to hold a reference of @mid so there is
no need to call kref_put(&mid->refcount, __release_mid) under
@server->mid_lock spinlock. If they don't, then an use-after-free bug
would have caused anyways.
By getting rid of such spinlock also fixes a potential deadlock as
shown below
CPU 0 CPU 1
------------------------------------------------------------------
cifs_demultiplex_thread() cifs_debug_data_proc_show()
release_mid()
spin_lock(&server->mid_lock);
spin_lock(&cifs_tcp_ses_lock)
spin_lock(&server->mid_lock)
__release_mid()
smb2_find_smb_tcon()
spin_lock(&cifs_tcp_ses_lock) *deadlock*
Signed-off-by: Paulo Alcantara (SUSE) <pc@xxxxxxxxxxxxx>
---
fs/smb/client/cifsproto.h | 7 ++++++-
fs/smb/client/smb2misc.c | 2 +-
fs/smb/client/transport.c | 11 +----------
3 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h
index 0c37eefa18a5..890ceddae07e 100644
--- a/fs/smb/client/cifsproto.h
+++ b/fs/smb/client/cifsproto.h
@@ -81,7 +81,7 @@ extern char *cifs_build_path_to_root(struct smb3_fs_context *ctx,
extern char *build_wildcard_path_from_dentry(struct dentry *direntry);
char *cifs_build_devname(char *nodename, const char *prepath);
extern void delete_mid(struct mid_q_entry *mid);
-extern void release_mid(struct mid_q_entry *mid);
+void __release_mid(struct kref *refcount);
extern void cifs_wake_up_task(struct mid_q_entry *mid);
extern int cifs_handle_standard(struct TCP_Server_Info *server,
struct mid_q_entry *mid);
@@ -740,4 +740,9 @@ static inline bool dfs_src_pathname_equal(const char *s1, const char *s2)
return true;
}
+static inline void release_mid(struct mid_q_entry *mid)
+{
+ kref_put(&mid->refcount, __release_mid);
+}
+
#endif /* _CIFSPROTO_H */
diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c
index 25f7cd6f23d6..32dfa0f7a78c 100644
--- a/fs/smb/client/smb2misc.c
+++ b/fs/smb/client/smb2misc.c
@@ -787,7 +787,7 @@ __smb2_handle_cancelled_cmd(struct cifs_tcon *tcon, __u16 cmd, __u64 mid,
{
struct close_cancelled_open *cancelled;
- cancelled = kzalloc(sizeof(*cancelled), GFP_ATOMIC);
+ cancelled = kzalloc(sizeof(*cancelled), GFP_KERNEL);
if (!cancelled)
return -ENOMEM;
diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c
index 14710afdc2a3..d553b7a54621 100644
--- a/fs/smb/client/transport.c
+++ b/fs/smb/client/transport.c
@@ -76,7 +76,7 @@ alloc_mid(const struct smb_hdr *smb_buffer, struct TCP_Server_Info *server)
return temp;
}
-static void __release_mid(struct kref *refcount)
+void __release_mid(struct kref *refcount)
{
struct mid_q_entry *midEntry =
container_of(refcount, struct mid_q_entry, refcount);
@@ -156,15 +156,6 @@ static void __release_mid(struct kref *refcount)
mempool_free(midEntry, cifs_mid_poolp);
}
-void release_mid(struct mid_q_entry *mid)
-{
- struct TCP_Server_Info *server = mid->server;
-
- spin_lock(&server->mid_lock);
- kref_put(&mid->refcount, __release_mid);
- spin_unlock(&server->mid_lock);
-}
-
void
delete_mid(struct mid_q_entry *mid)
{
--
2.42.0
