tests/shared/305

   1 #! /bin/bash
   2 # FSQA Test No. 305
   3 #
   4 # Run fsstress and fio(dio/aio and mmap) and simulate disk failure
   5 # check filesystem consistency at the end.
   6 #
   7 #-----------------------------------------------------------------------
   8 # (c) 2013 Dmitry Monakhov
   9 #
  10 # This program is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU General Public License as
  12 # published by the Free Software Foundation.
  13 #
  14 # This program is distributed in the hope that it would be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program; if not, write the Free Software Foundation,
  21 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  22 #
  23 #-----------------------------------------------------------------------
  24 #
  25
  26 seq=`basename $0`
  27 seqres=$RESULT_DIR/$seq
  28 seqres=$RESULT_DIR/$seq
  29 seqres=$RESULT_DIR/$seq
  30 echo "QA output created by $seq"
  31
  32 here=`pwd`
  33 tmp=/tmp/$$
  34 status=1        # failure is the default!
  35
  36 # get standard environment, filters and checks
  37 . ./common.rc
  38 . ./common.filter
  39 _supported_fs ext3 ext4 xfs btrfs reiserfs
  40 _supported_os Linux
  41 _need_to_be_root
  42 _require_scratch
  43 _require_fail_make_request
  44
  45 # TODO: Function are common enough to be moved to common.blkdev
  46 SCRATCH_REAL_DEV=`readlink -f $SCRATCH_DEV`
  47 SCRATCH_BDEV=`basename $SCRATCH_REAL_DEV`
  48
  49 allow_fail_make_request()
  50 {
  51     echo "Allow global fail_make_request feature"
  52     echo 100 > $DEBUGFS_MNT/fail_make_request/probability
  53     echo 9999999 > $DEBUGFS_MNT/fail_make_request/times
  54     echo 0 >  /sys/kernel/debug/fail_make_request/verbose
  55 }
  56
  57 disallow_fail_make_request()
  58 {
  59     echo "Disallow global fail_make_request feature"
  60     echo 0 > $DEBUGFS_MNT/fail_make_request/probability
  61     echo 0 > $DEBUGFS_MNT/fail_make_request/times
  62 }
  63
  64 start_fail_scratch_dev()
  65 {
  66     echo "Force SCRATCH_DEV device failure"
  67     echo " echo 1 > /sys/block/$SCRATCH_BDEV/make-it-fail" >> $here/$seqres.full
  68     echo 1 > /sys/block/$SCRATCH_BDEV/make-it-fail
  69
  70 }
  71
  72 stop_fail_scratch_dev()
  73 {
  74     echo "Make SCRATCH_DEV device operable again"
  75     echo " echo 0 > /sys/block/$SCRATCH_BDEV/make-it-fail" >> $here/$seqres.full
  76     echo 0 > /sys/block/$SCRATCH_BDEV/make-it-fail
  77
  78 }
  79
  80 _cleanup()
  81 {
  82     poweron_scratch_dev
  83     disallow_fail_make_request
  84 }
  85 trap "_cleanup; exit \$status" 1 2 3 15
  86
  87 RUN_TIME=$((20+10*$TIME_FACTOR))
  88 NUM_JOBS=$((4*LOAD_FACTOR))
  89 BLK_DEV_SIZE=`blockdev --getsz $SCRATCH_DEV`
  90 FILE_SIZE=$((BLK_DEV_SIZE * 512))
  91
  92 cat >$tmp-$seq.fio <<EOF
  93 ###########
  94 # $seq test's fio activity
  95 # Filenames derived from jobsname and jobid like follows:
  96 # ${JOB_NAME}.${JOB_ID}.${ITERATION_ID}
  97 [global]
  98 ioengine=libaio
  99 bs=4k
 100 directory=${SCRATCH_MNT}
 101 filesize=${FILE_SIZE}
 102 size=9999T
 103 continue_on_error=write
 104 ignore_error=EIO,ENOSPC:EIO
 105 error_dump=0
 106
 107 [stress_dio_aio_activity]
 108 create_on_open=1
 109 fallocate=none
 110 iodepth=128*${LOAD_FACTOR}
 111 direct=1
 112 buffered=0
 113 numjobs=${NUM_JOBS}
 114 rw=randwrite
 115 runtime=40+${RUN_TIME}
 116 time_based
 117
 118 [stress_mmap_activity]
 119 ioengine=mmap
 120 create_on_open=0
 121 fallocate=1
 122 fdatasync=40960
 123 filesize=8M
 124 size=9999T
 125 numjobs=${NUM_JOBS}
 126 rw=randwrite
 127 runtime=40+${RUN_TIME}
 128 time_based
 129
 130 EOF
 131
 132 _require_fio $tmp-$seq.fio
 133
 134 # Disable all sync operations to get higher load
 135 FSSTRESS_AVOID="$FSSTRESS_AVOID -ffsync=0 -fsync=0 -ffdatasync=0 -f setattr=1"
 136
 137 _workout()
 138 {
 139         out=$SCRATCH_MNT/fsstress.$$
 140         args=`_scale_fsstress_args -p 1 -n999999999 -f setattr=0 $FSSTRESS_AVOID -d $out`
 141         echo ""
 142         echo "Start fsstress.."
 143         echo ""
 144         echo "fsstress $args" >> $here/$seqres.full
 145         $FSSTRESS_PROG $args > /dev/null 2>&1 &
 146         fs_pid=$!
 147         echo "Start fio.."
 148         cat $tmp-$seq.fio >>  $seqres.full
 149         $FIO_PROG $tmp-$seq.fio >> $here/$seqres.full 2>&1 &
 150         fio_pid=$!
 151
 152         # Let's it work for awhile, and force device failure
 153         sleep $RUN_TIME
 154         start_fail_scratch_dev
 155         # After device turns in to failed state filesystem may yet not know about
 156         # that so buffered write(2) may succeed, but any integrity operations
 157         # such as (sync, fsync, fdatasync, direct-io) should fail.
 158         dd if=/dev/zero of=$SCRATCH_MNT/touch_failed_filesystem count=1 bs=4k conv=fsync \
 159             >> $here/$seqres.full 2>&1 && \
 160             _fail "failed: still able to perform integrity fsync on $SCRATCH_MNT"
 161
 162         kill $fs_pid
 163         wait $fs_pid
 164         wait $fio_pid
 165
 166         # We expect that broken FS still can be umounted
 167         run_check umount $SCRATCH_DEV
 168         # Once filesystem was umounted no one is able to write to block device
 169         # It is now safe to bring device back to normal state
 170         stop_fail_scratch_dev
 171
 172         # In order to check that filesystem is able to recover journal on mount(2)
 173         # perform mount/umount, after that all errors should be fixed
 174         run_check _scratch_mount
 175         run_check _scratch_unmount
 176         _check_scratch_fs
 177 }
 178
 179 # real QA test starts here
 180
 181 _scratch_mkfs >> $here/$seqres.full 2>&1 || _fail "mkfs failed"
 182 _scratch_mount || _fail "mount failed"
 183 allow_fail_make_request
 184 _workout
 185 status=$?
 186 disallow_fail_make_request
 187 exit