Test allocation strategies of the file system and validate space anomalies as reported by the system versus the allocated by the program. The test is motivated by a bug in ext4 systems where-in ENOSPC is reported by the file system even though enough space for allocations is available[1]. [1]: https://patchwork.ozlabs.org/patch/1294003 Suggested-by: Ritesh Harjani <riteshh@xxxxxxxxxxxxx> Co-authored-by: Sourabh Jain <sourabhjain@xxxxxxxxxxxxx> Signed-off-by: Sourabh Jain <sourabhjain@xxxxxxxxxxxxx> Signed-off-by: Pratik Rajesh Sampat <psampat@xxxxxxxxxxxxx> --- src/Makefile | 2 +- src/t_enospc.c | 186 ++++++++++++++++++++++++++++++++++++++++++ tests/generic/618 | 168 ++++++++++++++++++++++++++++++++++++++ tests/generic/618.out | 2 + tests/generic/group | 1 + 5 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 src/t_enospc.c create mode 100755 tests/generic/618 create mode 100644 tests/generic/618.out diff --git a/src/Makefile b/src/Makefile index 919d77c4..32940142 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,7 +17,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \ t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \ t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \ t_ofd_locks t_mmap_collision mmap-write-concurrent \ - t_get_file_time t_create_short_dirs t_create_long_dirs + t_get_file_time t_create_short_dirs t_create_long_dirs t_enospc LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \ preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \ diff --git a/src/t_enospc.c b/src/t_enospc.c new file mode 100644 index 00000000..a0a8c783 --- /dev/null +++ b/src/t_enospc.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 IBM. + * All Rights Reserved. + * + * simple mmap write multithreaded test + */ +#include <stdio.h> +#include <stdlib.h> +#include <stdbool.h> +#include <unistd.h> +#include <fcntl.h> +#include <string.h> +#include <assert.h> +#include <sys/mman.h> +#include <pthread.h> +#include <signal.h> + +#define pr_debug(fmt, ...) do { \ + if (verbose) { \ + printf (fmt, ##__VA_ARGS__); \ + } \ +} while (0) + +struct thread_s { + int id; +}; + +static float file_ratio[2] = {0.5, 0.5}; +static unsigned long fsize = 0; +static char *base_dir = "."; +static char *ratio = "1"; + +static bool use_fallocate = false; +static bool verbose = false; + +pthread_barrier_t bar; + +void handle_sigbus(int sig) +{ + pr_debug("Enospc test failed with SIGBUS\n"); + exit(7); +} + +void enospc_test(int id) +{ + int fd; + char fpath[255] = {0}; + char *addr; + unsigned long size = 0; + + if (id % 2 == 0) + size = fsize * file_ratio[0]; + else + size = fsize * file_ratio[1]; + + pthread_barrier_wait(&bar); + + sprintf(fpath, "%s/mmap-file-%d", base_dir, id); + pr_debug("Test write phase starting file %s fsize %lu, id %d\n", fpath, size, id); + + signal(SIGBUS, handle_sigbus); + + fd = open(fpath, O_RDWR | O_CREAT, 0644); + if (fd < 0) { + pr_debug("Open failed\n"); + exit(1); + } + + if (use_fallocate) + assert(fallocate(fd, 0, 0, fsize) == 0); + else + assert(ftruncate(fd, size) == 0); + + addr = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, 0); + assert(addr != MAP_FAILED); + + for (int i = 0; i < size; i++) { + addr[i] = 0xAB; + } + + assert(munmap(addr, size) != -1); + close(fd); + pr_debug("Test write phase completed...file %s, fsize %lu, id %d\n", fpath, size, id); +} + +void *spawn_test_thread(void *arg) +{ + struct thread_s *thread_info = (struct thread_s *)arg; + pthread_t tid; + + tid = pthread_self(); + pr_debug("Inside thread %lu %d\n", tid, thread_info->id); + enospc_test(thread_info->id); + return NULL; +} + +void _run_test(int threads) +{ + pthread_t tid[threads]; + + pthread_barrier_init(&bar, NULL, threads+1); + for (int i = 0; i < threads; i++) { + struct thread_s *thread_info = (struct thread_s *) malloc(sizeof(struct thread_s)); + thread_info->id = i; + assert(pthread_create(&tid[i], NULL, spawn_test_thread, thread_info) == 0); + } + + pthread_barrier_wait(&bar); + + for (int i = 0; i <threads; i++) { + assert(pthread_join(tid[i], NULL) == 0); + } + + pthread_barrier_destroy(&bar); + return; +} + +static void usage(void) +{ + printf("\nUsage: t_enospc [options]\n\n" + " -t thread count\n" + " -s size file size\n" + " -p path set base path\n" + " -f fallocate use fallocate instead of ftruncate\n" + " -v verbose print debug information\n" + " -r ratio ratio of file sizes\n"); +} + +int main(int argc, char *argv[]) +{ + int opt; + int threads = 0; + char *ratio_ptr; + + while ((opt = getopt(argc, argv, ":r:p:t:s:vf")) != -1) { + switch(opt) { + case 't': + threads = atoi(optarg); + pr_debug("Testing with (%d) threads\n", threads); + break; + case 's': + fsize = atoi(optarg); + pr_debug("size: %lu Bytes\n", fsize); + break; + case 'p': + base_dir = optarg; + break; + case 'r': + ratio = optarg; + ratio_ptr = strtok(ratio, ","); + if (ratio_ptr[0] == '1') { + file_ratio[0] = 1; + file_ratio[1] = 1; + break; + } + if (ratio_ptr != NULL) + file_ratio[0] = strtod(ratio_ptr, NULL); + ratio_ptr = strtok(NULL, ","); + if (ratio_ptr != NULL) + file_ratio[1] = strtod(ratio_ptr, NULL); + break; + case 'f': + use_fallocate = true; + break; + case 'v': + verbose = true; + break; + case '?': + usage(); + exit(1); + } + } + + /* Sanity test of parameters */ + assert(threads); + assert(fsize); + assert(file_ratio[0]); + assert(file_ratio[1]); + + pr_debug("Testing with (%d) threads\n", threads); + pr_debug("size: %lu Bytes\n", fsize); + + _run_test(threads); + return 0; +} diff --git a/tests/generic/618 b/tests/generic/618 new file mode 100755 index 00000000..57216b3d --- /dev/null +++ b/tests/generic/618 @@ -0,0 +1,168 @@ +#! /bin/bash +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2020 IBM Corporation. All Rights Reserved. +# +# FS QA Test 618 +# ENOSPC regression test in a multi-threaded scenario. Test allocation +# strategies of the file system and validate space anomalies as reported by +# the system versus the allocated by the program. +# +# With this we should be able to catch similar regressions as reported on +# ext4 on 5.7 kernel which was fixed by patch [1] +# [1]: https://patchwork.ozlabs.org/patch/1294003 +# +seq=`basename $0` +seqres=$RESULT_DIR/$seq +echo "QA output created by $seq" + +here=`pwd` +tmp=/tmp/$$ +status=1 # failure is the default! +trap "_cleanup; exit \$status" 0 1 2 3 15 + +FS_SIZE=$((240*1024*1024)) # 240MB +DEBUG=0 # set to 1 to enable debug statements in shell and c-prog +FACT=0.7 + +# Disk allocation methods +FALLOCATE=1 +FTRUNCATE=2 + +# Helps to build TEST_VECTORS +SMALL_FILE_SIZE=$((512 * 1024)) # in Bytes +BIG_FILE_SIZE=$((1536 * 1024)) # in Bytes +MIX_FILE_SIZE=$((2048 * 1024)) # (BIG + SMALL small file size) + +_cleanup() +{ + cd / + rm -f $tmp.* +} + +# get standard environment, filters and checks +. ./common/rc +. ./common/filter + +# remove previous $seqres.full before test +rm -f $seqres.full + +# Modify as appropriate. +_supported_fs generic +_require_test +_require_scratch +_require_test_program "t_enospc" + +debug() +{ + if [ $DEBUG -eq 1 ]; then + echo >&2 "func->${FUNCNAME[1]}: $1" + fi +} + +# Calculate the number of threads needed to fill the disk space +calc_thread_cnt() +{ + local file_ratio_unit=$1 + local file_ratio=$2 + local disk_saturation=$3 + local tot_avail_size + local avail_size + local thread_cnt + + IFS=',' read -ra fratio <<< $file_ratio + file_ratio_cnt=${#fratio[@]} + + tot_avail_size=$(df --block-size=1 $SCRATCH_DEV | awk 'FNR == 2 { print $4 }') + avail_size=$(echo $tot_avail_size*$disk_saturation | bc) + thread_cnt=$(echo "$file_ratio_cnt*($avail_size/$file_ratio_unit)" | bc) + + debug "Total available size: $tot_avail_size" + debug "Available size: $avail_size" + debug "Thread count: $thread_cnt" + + echo ${thread_cnt} +} + +run_testcase() +{ + IFS=':' read -ra args <<< $1 + local test_name=${args[0]} + local file_ratio_unit=${args[1]} + local file_ratio=${args[2]} + local disk_saturation=${args[3]} + local disk_alloc_method=${args[4]} + local test_iteration_cnt=${args[5]} + local extra_args="" + local thread_cnt + + if [ "$disk_alloc_method" == "$FTRUNCATE" ]; then + extra_args="$extra_args -f" + fi + + # enable the debug statements in c program + if [ "$DEBUG" -eq 1 ]; then + extra_args="$extra_args -v" + fi + + for i in $(eval echo "{1..$test_iteration_cnt}") + do + # Setup the device + _scratch_mkfs_sized $FS_SIZE >> $seqres.full 2>&1 + _scratch_mount + + thread_cnt=$(calc_thread_cnt $file_ratio_unit $file_ratio $disk_saturation) + + debug "====== Test details starts ======" + debug "Test name: $test_name" + debug "File ratio unit: $file_ratio_unit" + debug "File ratio: $file_ratio" + debug "Disk saturation $disk_saturation" + debug "Disk alloc method $disk_alloc_method" + debug "Test iteration count: $test_iteration_cnt" + debug "Extra arg: $extra_args" + debug "Thread count: $thread_cnt" + debug "============ end ===============" + + # Start the test + $here/src/t_enospc -t $thread_cnt -s $file_ratio_unit -r $file_ratio -p $SCRATCH_MNT $extra_args + + status=$(echo $?) + if [ $status -ne 0 ]; then + use_per=$(df -h | grep $SCRATCH_MNT | awk '{print substr($5, 1, length($5)-1)}' | bc) + alloc_per=$(echo "$FACT * 100" | bc) + # We are here since t_nospc failed with an error code. + # If the used filesystem space is still < available space - that means + # the test failed due to FS wrongly reported ENOSPC. + if [ $(echo "$use_per < $alloc_per" | bc) -ne 0 ]; then + if [ $status -eq 134 ]; then + echo "FAIL: Aborted assertion faliure" + elif [ $status -eq 7 ]; then + echo "FAIL: ENOSPC BUS faliure" + fi + echo "$test_name failed at iteration count: $i" + echo "$(df -h $SCRATCH_MNT)" + echo "Allocated: $alloc_per% Used: $use_per%" + exit + fi + fi + + # Make space for other tests + _scratch_unmount + done +} + +declare -a TEST_VECTORS=( +# test-name:file-ratio-unit:file-ratio:disk-saturation:disk-alloc-method:test-iteration-cnt +"Small-file-test:$SMALL_FILE_SIZE:1:$FACT:$FALLOCATE:3" +"Big-file-test:$BIG_FILE_SIZE:1:$FACT:$FALLOCATE:3" +"Mix-file-test:$MIX_FILE_SIZE:0.75,0.25:$FACT:$FALLOCATE:3" +) + +# real QA test starts here +for i in "${TEST_VECTORS[@]}"; do + run_testcase $i +done + +echo "Silence is golden" +status=0 +exit diff --git a/tests/generic/618.out b/tests/generic/618.out new file mode 100644 index 00000000..8940b72f --- /dev/null +++ b/tests/generic/618.out @@ -0,0 +1,2 @@ +QA output created by 618 +Silence is golden diff --git a/tests/generic/group b/tests/generic/group index 94e860b8..dc668c06 100644 --- a/tests/generic/group +++ b/tests/generic/group @@ -620,3 +620,4 @@ 615 auto rw 616 auto rw io_uring stress 617 auto rw io_uring stress +618 auto rw stress -- 2.28.0
