From 8b4f7e207cb2922168cfe0dadb8cf98c5270c4f7 Mon Sep 17 00:00:00 2001 From: Pratik Rajesh Sampat Date: Tue, 1 Dec 2020 18:33:28 +0530 Subject: [PATCH] generic: ENOSPC regression test in a multi-threaded scenario Test allocation strategies of the file system and validate space anomalies as reported by the system versus the allocated by the program. The test is motivated by a bug in ext4 systems where-in ENOSPC is reported by the file system even though enough space for allocations is available[1]. [1]: https://patchwork.ozlabs.org/patch/1294003 Linux kernel patch series that fixes the above regression: 53f86b170dfa ("ext4: mballoc: add blocks to PA list under same spinlock after allocating blocks") cf5e2ca6c990 ("ext4: mballoc: refactor ext4_mb_discard_preallocations()") 07b5b8e1ac40 ("ext4: mballoc: introduce pcpu seqcnt for freeing PA to improve ENOSPC handling") 8ef123fe02ca ("ext4: mballoc: refactor ext4_mb_good_group()") 993778306e79 ("ext4: mballoc: use lock for checking free blocks while retrying") Suggested-by: Ritesh Harjani Co-authored-by: Sourabh Jain Signed-off-by: Sourabh Jain Signed-off-by: Pratik Rajesh Sampat Reviewed-by: Eryu Guan Signed-off-by: Eryu Guan --- .gitignore | 1 + src/Makefile | 2 +- src/t_enospc.c | 193 +++++++++++++++++++++++++++++++++++++++ tests/generic/619 | 207 ++++++++++++++++++++++++++++++++++++++++++ tests/generic/619.out | 2 + tests/generic/group | 1 + 6 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 src/t_enospc.c create mode 100755 tests/generic/619 create mode 100644 tests/generic/619.out diff --git a/.gitignore b/.gitignore index 5f5c4a0f..041cc2d9 100644 --- a/.gitignore +++ b/.gitignore @@ -125,6 +125,7 @@ /src/t_dir_offset2 /src/t_dir_type /src/t_encrypted_d_revalidate +/src/t_enospc /src/t_ext4_dax_inline_corruption /src/t_ext4_dax_journal_corruption /src/t_futimens diff --git a/src/Makefile b/src/Makefile index 919d77c4..32940142 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,7 +17,7 @@ TARGETS = dirstress fill fill2 getpagesize holes lstat64 \ t_mmap_cow_race t_mmap_fallocate fsync-err t_mmap_write_ro \ t_ext4_dax_journal_corruption t_ext4_dax_inline_corruption \ t_ofd_locks t_mmap_collision mmap-write-concurrent \ - t_get_file_time t_create_short_dirs t_create_long_dirs + t_get_file_time t_create_short_dirs t_create_long_dirs t_enospc LINUX_TARGETS = xfsctl bstat t_mtab getdevicesize preallo_rw_pattern_reader \ preallo_rw_pattern_writer ftrunc trunc fs_perms testx looptest \ diff --git a/src/t_enospc.c b/src/t_enospc.c new file mode 100644 index 00000000..d06c6764 --- /dev/null +++ b/src/t_enospc.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020 IBM. + * All Rights Reserved. + * + * simple mmap write multithreaded test for testing enospc + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define pr_debug(fmt, ...) do { \ + if (verbose) { \ + printf (fmt, ##__VA_ARGS__); \ + } \ +} while (0) + +struct thread_s { + int id; +}; + +static float file_ratio[2] = {0.5, 0.5}; +static unsigned long fsize = 0; +static char *base_dir = "."; +static char *ratio = "1"; + +static bool use_fallocate = false; +static bool verbose = false; + +pthread_barrier_t bar; + +void handle_sigbus(int sig) +{ + pr_debug("Enospc test failed with SIGBUS\n"); + exit(7); +} + +void enospc_test(int id) +{ + int fd; + char fpath[255] = {0}; + char *addr; + unsigned long size = 0; + + /* + * Comma separated values against -r option indicates that the file + * should be divided into two small files. + * The file_ratio specifies the proportion in which the file sizes must + * be divided into. + * + * Half of the files will be divided into size of the first ratio and the + * rest of the following ratio + */ + + if (id % 2 == 0) + size = fsize * file_ratio[0]; + else + size = fsize * file_ratio[1]; + + pthread_barrier_wait(&bar); + + sprintf(fpath, "%s/mmap-file-%d", base_dir, id); + pr_debug("Test write phase starting file %s fsize %lu, id %d\n", fpath, size, id); + + signal(SIGBUS, handle_sigbus); + + fd = open(fpath, O_RDWR | O_CREAT, 0644); + if (fd < 0) { + pr_debug("Open failed\n"); + exit(1); + } + + if (use_fallocate) + assert(fallocate(fd, 0, 0, size) == 0); + else + assert(ftruncate(fd, size) == 0); + + addr = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, 0); + assert(addr != MAP_FAILED); + + for (int i = 0; i < size; i++) { + addr[i] = 0xAB; + } + + assert(munmap(addr, size) != -1); + close(fd); + pr_debug("Test write phase completed...file %s, fsize %lu, id %d\n", fpath, size, id); +} + +void *spawn_test_thread(void *arg) +{ + struct thread_s *thread_info = (struct thread_s *)arg; + + enospc_test(thread_info->id); + return NULL; +} + +void _run_test(int threads) +{ + pthread_t tid[threads]; + + pthread_barrier_init(&bar, NULL, threads+1); + for (int i = 0; i < threads; i++) { + struct thread_s *thread_info = (struct thread_s *) malloc(sizeof(struct thread_s)); + thread_info->id = i; + assert(pthread_create(&tid[i], NULL, spawn_test_thread, thread_info) == 0); + } + + pthread_barrier_wait(&bar); + + for (int i = 0; i > $seqres.full + fi +} + +# Calculate the number of threads needed to fill the disk space +# Arguments +# $1: the size of a file +# $2: ratio in which $1 file should be split into multiple files. +# $3: percentage of the disk space should be used during the test +# Calculate the number of threads needed to fill the disk space +calc_thread_cnt() +{ + local file_ratio_unit=$1 + local file_ratio=$2 + local disk_saturation=$3 + local tot_avail_size + local avail_size + local thread_cnt + + IFS=',' read -ra fratio <<< $file_ratio + file_ratio_cnt=${#fratio[@]} + + tot_avail_size=$($DF_PROG --block-size=1 $SCRATCH_DEV | awk 'FNR == 2 { print $5 }') + avail_size=$(echo $tot_avail_size*$disk_saturation | $BC_PROG) + thread_cnt=$(echo "$file_ratio_cnt*($avail_size/$file_ratio_unit)" | $BC_PROG) + + debug "Total available size: $tot_avail_size" + debug "Available size: $avail_size" + debug "Thread count: $thread_cnt" + + echo ${thread_cnt} +} + +# Arguments +# $1: a string containing test configuration separated by a colon. +# $1 is treated as an array of arguments to the function. +# Description of each array element is given below. +# +# @1: name of the test +# @2: thread in t_enospec exerciser will allocate file of @2 size +# @3: defines the proportion in which the file size defined in @2 +# should be divided into two files. +# (valid @3: more than two values are not allowed) +# values should be comma separated) +# sum of all values must be 1) +# @4: define the percentage of available memory should be used to +# during the test. +# @5: defines the disk allocation method (fallocate/ftruncate) +# @6: number of the test should run +run_testcase() +{ + IFS=':' read -ra args <<< $1 + local test_name=${args[0]} + local file_ratio_unit=${args[1]} + local file_ratio=${args[2]} + local disk_saturation=${args[3]} + local disk_alloc_method=${args[4]} + local test_iteration_cnt=${args[5]} + local extra_args="" + local thread_cnt + + if [ "$disk_alloc_method" == "$FALLOCATE" ]; then + extra_args="$extra_args -f" + fi + + # enable the debug statements in c program + if [ "$DEBUG" -eq 1 ]; then + extra_args="$extra_args -v" + fi + + debug "============ Test details start ============" + debug "Test name: $test_name" + debug "File ratio unit: $file_ratio_unit" + debug "File ratio: $file_ratio" + debug "Disk saturation $disk_saturation" + debug "Disk alloc method $disk_alloc_method" + debug "Test iteration count: $test_iteration_cnt" + debug "Extra arg: $extra_args" + + for i in $(eval echo "{1..$test_iteration_cnt}"); do + # Setup the device + _scratch_mkfs_sized $FS_SIZE >> $seqres.full 2>&1 + _scratch_mount + + debug "===== Test: $test_name iteration: $i starts =====" + thread_cnt=$(calc_thread_cnt $file_ratio_unit $file_ratio $disk_saturation) + + # Start the test + $here/src/t_enospc -t $thread_cnt -s $file_ratio_unit -r $file_ratio -p $SCRATCH_MNT $extra_args >> $seqres.full + + status=$(echo $?) + if [ $status -ne 0 ]; then + use_per=$($DF_PROG -h | grep $SCRATCH_MNT | awk '{print substr($6, 1, length($6)-1)}' | $BC_PROG) + alloc_per=$(echo "$FACT * 100" | $BC_PROG) + # We are here since t_enospc failed with an error code. + # If the used filesystem space is still < available space - that means + # the test failed due to FS wrongly reported ENOSPC. + if [ $(echo "$use_per < $alloc_per" | $BC_PROG) -ne 0 ]; then + if [ $status -eq 134 ]; then + # SIGABRT asserted exit code = 134 + echo "FAIL: Aborted assertion faliure" + elif [ $status -eq 7 ]; then + # SIGBUS asserted exit code = 7 + echo "FAIL: ENOSPC BUS faliure" + fi + echo "$test_name failed at iteration count: $i" + echo "$($DF_PROG -h $SCRATCH_MNT)" + echo "Allocated: $alloc_per% Used: $use_per%" + exit + fi + fi + + # Make space for other tests + _scratch_unmount + + debug "===== Test: $test_name iteration: $i ends =====" + done + debug "============ Test details end =============" +} + +declare -a TEST_VECTORS=( +# test-name:file-ratio-unit:file-ratio:disk-saturation:disk-alloc-method:test-iteration-cnt +"Small-file-fallocate-test:$SMALL_FILE_SIZE:1:$FACT:$FALLOCATE:3" +"Big-file-fallocate-test:$BIG_FILE_SIZE:1:$FACT:$FALLOCATE:3" +"Mix-file-fallocate-test:$MIX_FILE_SIZE:0.75,0.25:$FACT:$FALLOCATE:3" +"Small-file-ftruncate-test:$SMALL_FILE_SIZE:1:$FACT:$FTRUNCATE:3" +"Big-file-ftruncate-test:$BIG_FILE_SIZE:1:$FACT:$FTRUNCATE:3" +"Mix-file-ftruncate-test:$MIX_FILE_SIZE:0.75,0.25:$FACT:$FTRUNCATE:3" +) + +# real QA test starts here +for i in "${TEST_VECTORS[@]}"; do + run_testcase $i +done + +echo "Silence is golden" +status=0 +exit diff --git a/tests/generic/619.out b/tests/generic/619.out new file mode 100644 index 00000000..d17873db --- /dev/null +++ b/tests/generic/619.out @@ -0,0 +1,2 @@ +QA output created by 619 +Silence is golden diff --git a/tests/generic/group b/tests/generic/group index eca9d619..15a2f40e 100644 --- a/tests/generic/group +++ b/tests/generic/group @@ -621,3 +621,4 @@ 616 auto rw io_uring stress 617 auto rw io_uring stress 618 auto quick attr +619 auto rw enospc -- 2.39.5