mm/dmapool.c: possible circular locking dependency detected

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



During pci hotplug tests I received this one:

[   34.735836] ======================================================
[   34.735838] [ INFO: possible circular locking dependency detected ]
[   34.735840] 3.12.0-01915-g6c86ae2-dirty #125 Not tainted
[   34.735842] -------------------------------------------------------
[   34.735843] kworker/u128:3/49 is trying to acquire lock:
[   34.735845]  (s_active#27){++++.+}, at: [<00000000003048c2>] sysfs_addrm_finish+0x52/0x8c
[   34.735854] 
[   34.735854] but task is already holding lock:
[   34.735856]  (pools_lock){+.+.+.}, at: [<0000000000262e22>] dma_pool_destroy+0x32/0x1a4
[   34.735862] 
[   34.735862] which lock already depends on the new lock.
[   34.735862] 
[   34.735864] 
[   34.735864] the existing dependency chain (in reverse order) is:
[   34.735866] 
[   34.735866] -> #1 (pools_lock){+.+.+.}:
[   34.735869]        [<00000000001aa4c0>] __lock_acquire+0xf24/0x1698
[   34.735874]        [<00000000001ab360>] lock_acquire+0xb8/0x1f8
[   34.735876]        [<0000000000610a66>] mutex_lock_nested+0x7a/0x3a4
[   34.735880]        [<0000000000262cfa>] show_pools+0x5a/0x150
[   34.735883]        [<0000000000476e80>] dev_attr_show+0x38/0x78
[   34.735886]        [<00000000003027f8>] sysfs_seq_show+0x108/0x1b4
[   34.735888]        [<00000000002a3f46>] seq_read+0x136/0x550
[   34.735891]        [<0000000000279c9e>] vfs_read+0x8e/0x160
[   34.735894]        [<0000000000279f6a>] SyS_read+0x5e/0xb0
[   34.735896]        [<00000000006158ec>] sysc_nr_ok+0x22/0x28
[   34.735900]        [<0000004963071e98>] 0x4963071e98
[   34.735902] 
[   34.735902] -> #0 (s_active#27){++++.+}:
[   34.735906]        [<00000000001a68f8>] check_prev_add+0x748/0x74c
[   34.735908]        [<00000000001aa4c0>] __lock_acquire+0xf24/0x1698
[   34.735910]        [<00000000001ab360>] lock_acquire+0xb8/0x1f8
[   34.735913]        [<0000000000303bf4>] sysfs_deactivate+0xa4/0x124
[   34.735915]        [<00000000003048c2>] sysfs_addrm_finish+0x52/0x8c
[   34.735917]        [<0000000000304ea6>] sysfs_hash_and_remove+0x66/0xb0
[   34.735919]        [<0000000000262f70>] dma_pool_destroy+0x180/0x1a4
[   34.735922]        [<000003ff805cc0b2>] mlx4_cmd_cleanup+0x2e/0xa0 [mlx4_core]
[   34.735938]        [<000003ff805d7e58>] mlx4_remove_one+0x170/0x34c [mlx4_core]
[   34.735947]        [<00000000004425be>] pci_device_remove+0x42/0x7c
[   34.735950]        [<000000000047ad96>] __device_release_driver+0x6e/0xe0
[   34.735953]        [<000000000047ae46>] device_release_driver+0x3e/0x50
[   34.735956]        [<000000000047a7a0>] bus_remove_device+0x128/0x198
[   34.735958]        [<000000000047771c>] device_del+0x13c/0x1c8
[   34.735960]        [<000000000043bd40>] pci_stop_bus_device+0xb4/0xd8
[   34.735963]        [<000000000043beec>] pci_stop_and_remove_bus_device+0x28/0x38
[   34.735965]        [<000000000044410e>] remove_callback+0x3e/0x50
[   34.735968]        [<00000000003028d4>] sysfs_schedule_callback_work+0x30/0x84
[   34.735970]        [<000000000014eea8>] process_one_work+0x200/0x614
[   34.735975]        [<0000000000150674>] worker_thread+0x11c/0x308
[   34.735977]        [<000000000015903a>] kthread+0xd2/0xdc
[   34.735981]        [<0000000000615a72>] kernel_thread_starter+0x6/0xc
[   34.735983]        [<0000000000615a6c>] kernel_thread_starter+0x0/0xc
[   34.735986] 
[   34.735986] other info that might help us debug this:
[   34.735986] 
[   34.736001]  Possible unsafe locking scenario:
[   34.736001] 
[   34.736003]        CPU0                    CPU1
[   34.736005]        ----                    ----
[   34.736006]   lock(pools_lock);
[   34.736008]                                lock(s_active#27);
[   34.736010]                                lock(pools_lock);
[   34.736013]   lock(s_active#27);
[   34.736015] 
[   34.736015]  *** DEADLOCK ***
[   34.736015] 
[   34.736018] 5 locks held by kworker/u128:3/49:
[   34.736020]  #0:  (%s#5){.+.+.+}, at: [<000000000014ee22>] process_one_work+0x17a/0x614
[   34.736025]  #1:  ((&ss->work)){+.+.+.}, at: [<000000000014ee22>] process_one_work+0x17a/0x614
[   34.736030]  #2:  (pci_remove_rescan_mutex){+.+.+.}, at: [<0000000000444102>] remove_callback+0x32/0x50
[   34.736034]  #3:  (&__lockdep_no_validate__){......}, at: [<000000000047ae3c>] device_release_driver+0x34/0x50
[   34.736039]  #4:  (pools_lock){+.+.+.}, at: [<0000000000262e22>] dma_pool_destroy+0x32/0x1a4
[   34.736043] 
[   34.736043] stack backtrace:
[   34.736046] CPU: 0 PID: 49 Comm: kworker/u128:3 Not tainted 3.12.0-01915-g6c86ae2-dirty #125
[   34.736049] Workqueue: sysfsd sysfs_schedule_callback_work
[   34.736051]        000000007c827620 000000007c827630 0000000000000002 0000000000000000 
[   34.736051]        000000007c8276c0 000000007c827638 000000007c827638 000000000011157c 
[   34.736051]        0000000000000000 000000000078a13e 00000000007a4b50 000000000000000b 
[   34.736051]        000000007c827680 000000007c827620 0000000000000000 0000000000000000 
[   34.736051]        0000000000000000 000000000011157c 000000007c827620 000000007c827680 
[   34.736067] Call Trace:
[   34.736070] ([<0000000000111464>] show_trace+0xf8/0x158)
[   34.736072]  [<000000000011152e>] show_stack+0x6a/0xe8
[   34.736074]  [<000000000060d0a6>] dump_stack+0x82/0xac
[   34.736076]  [<00000000006071bc>] print_circular_bug+0x304/0x318
[   34.736078]  [<00000000001a68f8>] check_prev_add+0x748/0x74c
[   34.736080]  [<00000000001aa4c0>] __lock_acquire+0xf24/0x1698
[   34.736082]  [<00000000001ab360>] lock_acquire+0xb8/0x1f8
[   34.736084]  [<0000000000303bf4>] sysfs_deactivate+0xa4/0x124
[   34.736086]  [<00000000003048c2>] sysfs_addrm_finish+0x52/0x8c
[   34.736088]  [<0000000000304ea6>] sysfs_hash_and_remove+0x66/0xb0
[   34.736090]  [<0000000000262f70>] dma_pool_destroy+0x180/0x1a4
[   34.736097]  [<000003ff805cc0b2>] mlx4_cmd_cleanup+0x2e/0xa0 [mlx4_core]
[   34.736105]  [<000003ff805d7e58>] mlx4_remove_one+0x170/0x34c [mlx4_core]
[   34.736107]  [<00000000004425be>] pci_device_remove+0x42/0x7c
[   34.736109]  [<000000000047ad96>] __device_release_driver+0x6e/0xe0
[   34.736111]  [<000000000047ae46>] device_release_driver+0x3e/0x50
[   34.736113]  [<000000000047a7a0>] bus_remove_device+0x128/0x198
[   34.736115]  [<000000000047771c>] device_del+0x13c/0x1c8
[   34.736117]  [<000000000043bd40>] pci_stop_bus_device+0xb4/0xd8
[   34.736119]  [<000000000043beec>] pci_stop_and_remove_bus_device+0x28/0x38
[   34.736121]  [<000000000044410e>] remove_callback+0x3e/0x50
[   34.736123]  [<00000000003028d4>] sysfs_schedule_callback_work+0x30/0x84
[   34.736125]  [<000000000014eea8>] process_one_work+0x200/0x614
[   34.736127]  [<0000000000150674>] worker_thread+0x11c/0x308
[   34.736129]  [<000000000015903a>] kthread+0xd2/0xdc
[   34.736131]  [<0000000000615a72>] kernel_thread_starter+0x6/0xc
[   34.736133]  [<0000000000615a6c>] kernel_thread_starter+0x0/0xc
[   34.736135] INFO: lockdep is turned off.

The following patch fixes this for me - but maybe someone has a better
idea to handle this.

Regards,
Sebastian
--

mm/dmapool: fix a possible deadlock

Reading from $DEV/pools could result in a deadlock when the last
dmapool is removed from this device at the same time:

[   34.735836] ======================================================
[   34.735838] [ INFO: possible circular locking dependency detected ]
[   34.735840] 3.12.0-01915-g6c86ae2-dirty #125 Not tainted
[   34.735842] -------------------------------------------------------
[   34.735843] kworker/u128:3/49 is trying to acquire lock:
[   34.735845]  (s_active#27){++++.+}, at: [<00000000003048c2>] sysfs_addrm_finish+0x52/0x8c
[   34.735854] 
[   34.735854] but task is already holding lock:
[   34.735856]  (pools_lock){+.+.+.}, at: [<0000000000262e22>] dma_pool_destroy+0x32/0x1a4
[   34.735862] 
[   34.735862] which lock already depends on the new lock.
...
[   34.736001]  Possible unsafe locking scenario:
[   34.736001] 
[   34.736003]        CPU0                    CPU1
[   34.736005]        ----                    ----
[   34.736006]   lock(pools_lock);
[   34.736008]                                lock(s_active#27);
[   34.736010]                                lock(pools_lock);
[   34.736013]   lock(s_active#27);
[   34.736015] 
[   34.736015]  *** DEADLOCK ***

Fix this by moving the pools attribute creation and removal outside
the pools_lock (which is used to protect access to the dma_pools
list). And add a new lock to serialize the pools attribute
creation/removal.

Signed-off-by: Sebastian Ott <sebott@xxxxxxxxxxxxxxxxxx>
---
 mm/dmapool.c |   39 ++++++++++++++++++++++++---------------
 1 file changed, 24 insertions(+), 15 deletions(-)

--- a/mm/dmapool.c
+++ b/mm/dmapool.c
@@ -61,7 +61,8 @@ struct dma_page {		/* cacheable header f
 	unsigned int offset;
 };
 
-static DEFINE_MUTEX(pools_lock);
+static DEFINE_MUTEX(pools_lock);     /* protect dev->dma_pools access */
+static DEFINE_MUTEX(pools_attr_lock);/* serialize pools attr creation/removal */
 
 static ssize_t
 show_pools(struct device *dev, struct device_attribute *attr, char *buf)
@@ -171,21 +172,24 @@ struct dma_pool *dma_pool_create(const c
 	retval->allocation = allocation;
 
 	if (dev) {
-		int ret;
+		bool empty;
 
+		mutex_lock(&pools_attr_lock);
 		mutex_lock(&pools_lock);
-		if (list_empty(&dev->dma_pools))
-			ret = device_create_file(dev, &dev_attr_pools);
-		else
-			ret = 0;
-		/* note:  not currently insisting "name" be unique */
-		if (!ret)
-			list_add(&retval->pools, &dev->dma_pools);
-		else {
-			kfree(retval);
-			retval = NULL;
-		}
+		empty = list_empty(&dev->dma_pools);
+		list_add(&retval->pools, &dev->dma_pools);
 		mutex_unlock(&pools_lock);
+		if (empty) {
+			if (device_create_file(dev, &dev_attr_pools)) {
+				mutex_lock(&pools_lock);
+				list_del(&retval->pools);
+				mutex_unlock(&pools_lock);
+
+				kfree(retval);
+				retval = NULL;
+			}
+		}
+		mutex_unlock(&pools_attr_lock);
 	} else
 		INIT_LIST_HEAD(&retval->pools);
 
@@ -259,11 +263,16 @@ static void pool_free_page(struct dma_po
  */
 void dma_pool_destroy(struct dma_pool *pool)
 {
+	bool empty;
+
+	mutex_lock(&pools_attr_lock);
 	mutex_lock(&pools_lock);
 	list_del(&pool->pools);
-	if (pool->dev && list_empty(&pool->dev->dma_pools))
-		device_remove_file(pool->dev, &dev_attr_pools);
+	empty = pool->dev && list_empty(&pool->dev->dma_pools);
 	mutex_unlock(&pools_lock);
+	if (empty)
+		device_remove_file(pool->dev, &dev_attr_pools);
+	mutex_unlock(&pools_attr_lock);
 
 	while (!list_empty(&pool->page_list)) {
 		struct dma_page *page;

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>




[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [ECOS]     [Asterisk Internet PBX]     [Linux API]