Adds a test which runs many formats and controller resets in parallel. The intent is to expose timing holes in the controller state machine which will lead to hung task timeouts and the controller becoming unavailable. Reported by https://bugzilla.kernel.org/show_bug.cgi?id=216354 Signed-off-by: Jonathan Derrick <jonathan.derrick@xxxxxxxxx> --- v3: I noticed I couldn't rely on checking state within the loop because it was constantly on 'resetting', so I moved the last_live declaration to update when a reset completes. I updated the kernel in my QEMU instance to origin/nvme-6.2 (it was on a previous origin/nvme-6.2), and applied Klaus' format fix to QEMU. This doesn't crash my QEMU anymore, but now gets stuck in 'connecting'. tests/nvme/047 | 137 +++++++++++++++++++++++++++++++++++++++++++++ tests/nvme/047.out | 2 + 2 files changed, 139 insertions(+) create mode 100755 tests/nvme/047 create mode 100644 tests/nvme/047.out diff --git a/tests/nvme/047 b/tests/nvme/047 new file mode 100755 index 0000000..76b7fdf --- /dev/null +++ b/tests/nvme/047 @@ -0,0 +1,137 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-3.0+ +# Copyright (C) 2022 Jonathan Derrick <jonathan.derrick@xxxxxxxxx> +# +# Test nvme reset controller during admin passthru +# +# Regression for issue reported by +# https://bugzilla.kernel.org/show_bug.cgi?id=216354 +# +# Simpler form: +# for i in {1..50}; do +# nvme format -f /dev/nvme0n1 & +# echo 1 > /sys/block/nvme0n1/device/reset_controller & +# done + +. tests/nvme/rc + +#restrict test to nvme-pci only +nvme_trtype=pci + +DESCRIPTION="test nvme reset controller during admin passthru" +QUICK=1 +CAN_BE_ZONED=1 + +RUN_TIME=300 +RESET_MISSING=true +RESET_DEAD=true + +requires() { + _nvme_requires +} + +device_requires() { + _require_test_dev_is_nvme +} + +test_device() { + echo "Running ${TEST_NAME}" + + local pdev + local blkdev + local ctrldev + local sysfs + local max_timeout + local timeout + local timeleft + local start + local i + + pdev="$(_get_pci_dev_from_blkdev)" + blkdev="${TEST_DEV_SYSFS##*/}" + ctrldev="$(echo "$blkdev" | grep -Eo 'nvme[0-9]+')" + sysfs="/sys/block/$blkdev/device" + max_timeout=$(cat /proc/sys/kernel/hung_task_timeout_secs) + timeout=$((max_timeout * 3 / 4)) + + get_state() { + state=$(cat "$sysfs/state" 2> /dev/null) + if [[ -n "$state" ]]; then + echo "$state" + else + echo "unknown" + fi + } + + tmp=$(mktemp /tmp/blk_tmp_XXXXXX) + lock=$(mktemp /tmp/blk_lock_XXXXXX) + exec 47>"$lock" + update_live() { + flock -s 047 + date "+%s" > "$tmp" + flock -u 47 + } + + last_live() { + flock -s 47 + cat "$tmp" + flock -u 47 + } + + now() { + date "+%s" + } + + sleep 5 + + update_live + start=$(now) + while [[ $(($(now) - start)) -le $RUN_TIME ]]; do + # Failure case appears to stack up formats while controller is resetting/connecting + if [[ $(pgrep -cf "nvme format") -lt 100 ]]; then + for ((i=0; i<100; i++)); do + nvme format -f "$TEST_DEV" & + ( echo 1 > "$sysfs/reset_controller" && update_live; ) & + done &> /dev/null + fi + + # Might have failed probe, so reset and continue test + if [[ $(($(now) - $(last_live))) -gt 10 ]]; then + if [[ (! -c "/dev/$ctrldev" && "$RESET_MISSING" == true) || + ("$(get_state)" == "dead" && "$RESET_DEAD" == true) ]]; then + { + echo 1 > /sys/bus/pci/devices/"$pdev"/remove + echo 1 > /sys/bus/pci/rescan + } & + + timeleft=$((max_timeout - timeout)) + sleep $((timeleft < 30 ? timeleft : 30)) + if [[ ! -c "/dev/$ctrldev" ]]; then + echo "/dev/$ctrldev missing" + echo "failed to reset $ctrldev's pcie device $pdev" + break + fi + sleep 5 + continue + fi + fi + + if [[ $(($(now) - $(last_live))) -gt $timeout ]]; then + if [[ ! -c "/dev/$ctrldev" ]]; then + echo "/dev/$ctrldev missing" + break + fi + + # Assume the controller is hung and unrecoverable + echo "nvme controller hung ($(get_state))" + break + fi + done + + if [[ ! -c "/dev/$ctrldev" || "$(get_state)" != "live" ]]; then + echo "nvme still not live after $(($(now) - $(last_live))) seconds!" + fi + udevadm settle + + echo "Test complete" +} diff --git a/tests/nvme/047.out b/tests/nvme/047.out new file mode 100644 index 0000000..915d0a2 --- /dev/null +++ b/tests/nvme/047.out @@ -0,0 +1,2 @@ +Running nvme/047 +Test complete -- 2.27.0
