tests/generic/019

   1 #! /bin/bash
   2 # SPDX-License-Identifier: GPL-2.0
   3
   4 #
   5 # FSQA Test No. generic/019
   6 #
   7 # Run fsstress and fio(dio/aio and mmap) and simulate disk failure
   8 # check filesystem consistency at the end.
   9 #
  10 seq=`basename $0`
  11 seqres=$RESULT_DIR/$seq
  12 echo "QA output created by $seq"
  13
  14 here=`pwd`
  15 tmp=/tmp/$$
  16 fio_config=$tmp.fio
  17 status=1        # failure is the default!
  18
  19 # get standard environment, filters and checks
  20 . ./common/rc
  21 . ./common/filter
  22 _supported_fs generic
  23 _supported_os Linux
  24 _require_scratch
  25 _require_block_device $SCRATCH_DEV
  26 _require_fail_make_request
  27
  28 SYSFS_BDEV=`_sysfs_dev $SCRATCH_DEV`
  29
  30 allow_fail_make_request()
  31 {
  32     echo "Allow global fail_make_request feature"
  33     echo 100 > $DEBUGFS_MNT/fail_make_request/probability
  34     echo 9999999 > $DEBUGFS_MNT/fail_make_request/times
  35     echo 0 >  /sys/kernel/debug/fail_make_request/verbose
  36 }
  37
  38 disallow_fail_make_request()
  39 {
  40     echo "Disallow global fail_make_request feature"
  41     echo 0 > $DEBUGFS_MNT/fail_make_request/probability
  42     echo 0 > $DEBUGFS_MNT/fail_make_request/times
  43 }
  44
  45 start_fail_scratch_dev()
  46 {
  47     echo "Force SCRATCH_DEV device failure"
  48     echo " echo 1 > $SYSFS_BDEV/make-it-fail" >> $seqres.full
  49     echo 1 > $SYSFS_BDEV/make-it-fail
  50 }
  51
  52 stop_fail_scratch_dev()
  53 {
  54     echo "Make SCRATCH_DEV device operable again"
  55     echo " echo 0 > $SYSFS_BDEV/make-it-fail" >> $seqres.full
  56     echo 0 > $SYSFS_BDEV/make-it-fail
  57 }
  58
  59 _cleanup()
  60 {
  61     poweron_scratch_dev
  62     disallow_fail_make_request
  63     rm -f $tmp.*
  64 }
  65 trap "_cleanup; exit \$status" 1 2 3 15
  66
  67 RUN_TIME=$((20+10*$TIME_FACTOR))
  68 NUM_JOBS=$((4*LOAD_FACTOR))
  69 BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
  70 FILE_SIZE=$((BLK_DEV_SIZE * 512))
  71
  72 cat >$fio_config <<EOF
  73 ###########
  74 # $seq test's fio activity
  75 # Filenames derived from jobsname and jobid like follows:
  76 # ${JOB_NAME}.${JOB_ID}.${ITERATION_ID}
  77 [global]
  78 ioengine=libaio
  79 bs=4k
  80 directory=${SCRATCH_MNT}
  81 filesize=${FILE_SIZE}
  82 size=9999T
  83 continue_on_error=write
  84 ignore_error=EIO,ENOSPC:EIO
  85 error_dump=0
  86
  87 [stress_dio_aio_activity]
  88 create_on_open=1
  89 fallocate=none
  90 iodepth=128*${LOAD_FACTOR}
  91 direct=1
  92 buffered=0
  93 numjobs=${NUM_JOBS}
  94 rw=randwrite
  95 runtime=40+${RUN_TIME}
  96 time_based
  97
  98 [stress_mmap_activity]
  99 ioengine=mmap
 100 create_on_open=0
 101 fallocate=1
 102 fdatasync=40960
 103 filesize=8M
 104 size=9999T
 105 numjobs=${NUM_JOBS}
 106 rw=randwrite
 107 runtime=40+${RUN_TIME}
 108 time_based
 109
 110 EOF
 111
 112 _require_fio $fio_config
 113
 114 # Disable all sync operations to get higher load
 115 FSSTRESS_AVOID="$FSSTRESS_AVOID -ffsync=0 -fsync=0 -ffdatasync=0 -f setattr=1"
 116
 117 _workout()
 118 {
 119         out=$SCRATCH_MNT/fsstress.$$
 120         args=`_scale_fsstress_args -p 1 -n999999999 -f setattr=0 $FSSTRESS_AVOID -d $out`
 121         echo ""
 122         echo "Start fsstress.."
 123         echo ""
 124         echo "fsstress $args" >> $seqres.full
 125         $FSSTRESS_PROG $args > /dev/null 2>&1 &
 126         fs_pid=$!
 127         echo "Start fio.."
 128         cat $fio_config >>  $seqres.full
 129         $FIO_PROG $fio_config >> $seqres.full 2>&1 &
 130         fio_pid=$!
 131
 132         # Let's it work for awhile, and force device failure
 133         sleep $RUN_TIME
 134         start_fail_scratch_dev
 135         # After device turns in to failed state filesystem may yet not know about
 136         # that so buffered write(2) may succeed, but any integrity operations
 137         # such as (sync, fsync, fdatasync, direct-io) should fail.
 138         dd if=/dev/zero of=$SCRATCH_MNT/touch_failed_filesystem count=1 bs=4k conv=fsync \
 139             >> $seqres.full 2>&1 && \
 140             _fail "failed: still able to perform integrity fsync on $SCRATCH_MNT"
 141
 142         kill $fs_pid &> /dev/null
 143         wait $fs_pid
 144         wait $fio_pid
 145
 146         # We expect that broken FS still can be umounted
 147         run_check _scratch_unmount
 148         # Once filesystem was umounted no one is able to write to block device
 149         # It is now safe to bring device back to normal state
 150         stop_fail_scratch_dev
 151
 152         # In order to check that filesystem is able to recover journal on mount(2)
 153         # perform mount/umount, after that all errors should be fixed
 154         _scratch_mount
 155         run_check _scratch_unmount
 156 }
 157
 158 # real QA test starts here
 159
 160 rm -f $seqres.full
 161 _scratch_mkfs >> $seqres.full 2>&1 || _fail "mkfs failed"
 162 _scratch_mount
 163 allow_fail_make_request
 164 _workout
 165 status=$?
 166 disallow_fail_make_request
 167 exit