Version is 1.46.4, I think whether to try to release the mutex lock
in the ext2fs_close_free, such as CACHE_MTX,BOUNCE_MTX,STATS_MTX. But
you need to decide if it's the device you're checking, because I've
checked everyplace where ext2fs_close_free is called, in addition to
the call in the program end and exception branch, it is also called
when the journal device is close.
Reliable reproducer is in attachment.
-zhanchengbin.
On 2022/11/9 23:43, Theodore Ts'o wrote:
On Wed, Nov 09, 2022 at 06:40:31PM +0800, zhanchengbin wrote:
Hi Tytso,
The process is deadlocked, and an I/O error occurs when logs
are replayed. Because in the I/O error handling function, I/O
is sent again and catch the mutexlock.
What version of e2fsprogs are you using, and do you have a reliable
reproducer?
Thanks,
- Ted
.
#!/bin/bash
disk="sdb"
dir=/mnt/${disk}
mkfs.ext4 -F /dev/$disk
[ -d $dir ] || mkdir $dir
echo 1 > /sys/kernel/debug/fail_make_request/verbose
echo 5 > /sys/kernel/debug/fail_make_request/probability
echo 10 > /sys/kernel/debug/fail_make_request/interval
echo 10000000 > /sys/kernel/debug/fail_make_request/times
function set_sys()
{
local queue_dir=/sys/block/$1/queue
interval=$2
while true
do
sleep $interval
let s_num=RANDOM%4
case $s_num in
0)
scheduler=mq-deadline
;;
1)
scheduler=bfq
;;
2)
scheduler=kyber
;;
3)
scheduler=none
;;
esac
echo $scheduler > $queue_dir/scheduler
done
}
set_sys $disk 120 &>/dev/null &
i=0
while true
do
let flag=i%5
if [ $flag -le 2 ]; then
tune2fs -l /dev/$disk || exit 1
fi
mount -o errors=remount-ro /dev/$disk $dir || exit 1
tune2fs -l /dev/$disk || exit 1
fsstress -d $dir/fss -l 20 -n 500 -p 8 > /dev/null 2>&1 &
sleep $((1 + RANDOM % 3))
mount | grep $dir | grep '(ro' && exit 1
echo 1 > /sys/block/$disk/make-it-fail
sleep $((1 + RANDOM % 3))
ps -e | grep -w fsstress > /dev/null 2>&1
while [ $? -eq 0 ]
do
sleep 1
mount | grep $dir | grep '(ro' && killall -9 fsstress
ps -e | grep -w fsstress > /dev/null 2>&1
done
echo 0 > /sys/block/$disk/make-it-fail
while true
do
umount $dir && break
killall -9 fsstress > /dev/null 2>&1
sleep 0.1
done
if [ $flag -le 1 ]; then
tune2fs -l /dev/$disk || exit 1
fi
echo 1 > /sys/block/$disk/make-it-fail
echo 10 > /sys/kernel/debug/fail_make_request/probability
echo 1 > /sys/kernel/debug/fail_make_request/interval
count=100
while [ $count -ge 0 ]; do
fsck -a /dev/$disk
((count = count - 1))
done
echo 5 > /sys/kernel/debug/fail_make_request/probability
echo 10 > /sys/kernel/debug/fail_make_request/interval
echo 0 > /sys/block/$disk/make-it-fail
fsck -a /dev/$disk &> fsck-${disk}.log
ret=$?
if [ $ret -ne 0 -a $ret -ne 1 ]; then
exit 1
fi
fsck -fn /dev/$disk
ret=$?
if [ $ret -ne 0 ]; then
exit 1
fi
((i=i+1))
done
