tests/generic/019

   1 #! /bin/bash
   2 # FSQA Test No. generic/019
   3 #
   4 # Run fsstress and fio(dio/aio and mmap) and simulate disk failure
   5 # check filesystem consistency at the end.
   6 #
   7 #-----------------------------------------------------------------------
   8 # (c) 2013 Dmitry Monakhov
   9 #
  10 # This program is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU General Public License as
  12 # published by the Free Software Foundation.
  13 #
  14 # This program is distributed in the hope that it would be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program; if not, write the Free Software Foundation,
  21 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  22 #
  23 #-----------------------------------------------------------------------
  24 #
  25
  26 seq=`basename $0`
  27 seqres=$RESULT_DIR/$seq
  28 echo "QA output created by $seq"
  29
  30 here=`pwd`
  31 tmp=/tmp/$$
  32 fio_config=$tmp.fio
  33 status=1        # failure is the default!
  34
  35 # get standard environment, filters and checks
  36 . ./common/rc
  37 . ./common/filter
  38 _supported_fs generic
  39 _supported_os Linux
  40 _need_to_be_root
  41 _require_scratch
  42 _require_fail_make_request
  43
  44 SYSFS_BDEV=`_sysfs_dev $SCRATCH_DEV`
  45
  46 allow_fail_make_request()
  47 {
  48     echo "Allow global fail_make_request feature"
  49     echo 100 > $DEBUGFS_MNT/fail_make_request/probability
  50     echo 9999999 > $DEBUGFS_MNT/fail_make_request/times
  51     echo 0 >  /sys/kernel/debug/fail_make_request/verbose
  52 }
  53
  54 disallow_fail_make_request()
  55 {
  56     echo "Disallow global fail_make_request feature"
  57     echo 0 > $DEBUGFS_MNT/fail_make_request/probability
  58     echo 0 > $DEBUGFS_MNT/fail_make_request/times
  59 }
  60
  61 start_fail_scratch_dev()
  62 {
  63     echo "Force SCRATCH_DEV device failure"
  64     echo " echo 1 > $SYSFS_BDEV/make-it-fail" >> $seqres.full
  65     echo 1 > $SYSFS_BDEV/make-it-fail
  66 }
  67
  68 stop_fail_scratch_dev()
  69 {
  70     echo "Make SCRATCH_DEV device operable again"
  71     echo " echo 0 > $SYSFS_BDEV/make-it-fail" >> $seqres.full
  72     echo 0 > $SYSFS_BDEV/make-it-fail
  73 }
  74
  75 _cleanup()
  76 {
  77     poweron_scratch_dev
  78     disallow_fail_make_request
  79     rm -f $tmp.*
  80 }
  81 trap "_cleanup; exit \$status" 1 2 3 15
  82
  83 RUN_TIME=$((20+10*$TIME_FACTOR))
  84 NUM_JOBS=$((4*LOAD_FACTOR))
  85 BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
  86 FILE_SIZE=$((BLK_DEV_SIZE * 512))
  87
  88 cat >$fio_config <<EOF
  89 ###########
  90 # $seq test's fio activity
  91 # Filenames derived from jobsname and jobid like follows:
  92 # ${JOB_NAME}.${JOB_ID}.${ITERATION_ID}
  93 [global]
  94 ioengine=libaio
  95 bs=4k
  96 directory=${SCRATCH_MNT}
  97 filesize=${FILE_SIZE}
  98 size=9999T
  99 continue_on_error=write
 100 ignore_error=EIO,ENOSPC:EIO
 101 error_dump=0
 102
 103 [stress_dio_aio_activity]
 104 create_on_open=1
 105 fallocate=none
 106 iodepth=128*${LOAD_FACTOR}
 107 direct=1
 108 buffered=0
 109 numjobs=${NUM_JOBS}
 110 rw=randwrite
 111 runtime=40+${RUN_TIME}
 112 time_based
 113
 114 [stress_mmap_activity]
 115 ioengine=mmap
 116 create_on_open=0
 117 fallocate=1
 118 fdatasync=40960
 119 filesize=8M
 120 size=9999T
 121 numjobs=${NUM_JOBS}
 122 rw=randwrite
 123 runtime=40+${RUN_TIME}
 124 time_based
 125
 126 EOF
 127
 128 _require_fio $fio_config
 129
 130 # Disable all sync operations to get higher load
 131 FSSTRESS_AVOID="$FSSTRESS_AVOID -ffsync=0 -fsync=0 -ffdatasync=0 -f setattr=1"
 132
 133 _workout()
 134 {
 135         out=$SCRATCH_MNT/fsstress.$$
 136         args=`_scale_fsstress_args -p 1 -n999999999 -f setattr=0 $FSSTRESS_AVOID -d $out`
 137         echo ""
 138         echo "Start fsstress.."
 139         echo ""
 140         echo "fsstress $args" >> $seqres.full
 141         $FSSTRESS_PROG $args > /dev/null 2>&1 &
 142         fs_pid=$!
 143         echo "Start fio.."
 144         cat $fio_config >>  $seqres.full
 145         $FIO_PROG $fio_config >> $seqres.full 2>&1 &
 146         fio_pid=$!
 147
 148         # Let's it work for awhile, and force device failure
 149         sleep $RUN_TIME
 150         start_fail_scratch_dev
 151         # After device turns in to failed state filesystem may yet not know about
 152         # that so buffered write(2) may succeed, but any integrity operations
 153         # such as (sync, fsync, fdatasync, direct-io) should fail.
 154         dd if=/dev/zero of=$SCRATCH_MNT/touch_failed_filesystem count=1 bs=4k conv=fsync \
 155             >> $seqres.full 2>&1 && \
 156             _fail "failed: still able to perform integrity fsync on $SCRATCH_MNT"
 157
 158         kill $fs_pid
 159         wait $fs_pid
 160         wait $fio_pid
 161
 162         # We expect that broken FS still can be umounted
 163         run_check umount $SCRATCH_DEV
 164         # Once filesystem was umounted no one is able to write to block device
 165         # It is now safe to bring device back to normal state
 166         stop_fail_scratch_dev
 167
 168         # In order to check that filesystem is able to recover journal on mount(2)
 169         # perform mount/umount, after that all errors should be fixed
 170         run_check _scratch_mount
 171         run_check _scratch_unmount
 172 }
 173
 174 # real QA test starts here
 175
 176 _scratch_mkfs >> $seqres.full 2>&1 || _fail "mkfs failed"
 177 _scratch_mount || _fail "mount failed"
 178 allow_fail_make_request
 179 _workout
 180 status=$?
 181 disallow_fail_make_request
 182 exit