check

   1 #!/bin/bash
   2 #
   3 # Control script for QA
   4 #
   5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation.
  10 #
  11 # This program is distributed in the hope that it would be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write the Free Software Foundation,
  18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  19 #
  20 #
  21
  22 tmp=/tmp/$$
  23 status=0
  24 needwrap=true
  25 needsum=true
  26 n_try=0
  27 try=""
  28 n_bad=0
  29 sum_bad=0
  30 bad=""
  31 n_notrun=0
  32 notrun=""
  33 interrupt=true
  34 diff="diff -u"
  35 showme=false
  36 have_test_arg=false
  37 randomize=false
  38 export here=`pwd`
  39 xfile=""
  40 brief_test_summary=false
  41 do_report=false
  42 DUMP_OUTPUT=false
  43
  44 # This is a global variable used to pass test failure text to reporting gunk
  45 _err_msg=""
  46
  47 # start the initialisation work now
  48 iam=check
  49
  50 export MSGVERB="text:action"
  51 export QA_CHECK_FS=${QA_CHECK_FS:=true}
  52
  53 # number of diff lines from a failed test, 0 for whole output
  54 export DIFF_LENGTH=${DIFF_LENGTH:=10}
  55
  56 # by default don't output timestamps
  57 timestamp=${TIMESTAMP:=false}
  58
  59 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
  60
  61 SRC_GROUPS="generic shared"
  62 export SRC_DIR="tests"
  63
  64 usage()
  65 {
  66     echo "Usage: $0 [options] [testlist]"'
  67
  68 check options
  69     -nfs                test NFS
  70     -glusterfs                test GlusterFS
  71     -cifs               test CIFS
  72     -9p                 test 9p
  73     -overlay            test overlay
  74     -pvfs2          test PVFS2
  75     -tmpfs              test TMPFS
  76     -ubifs              test ubifs
  77     -l                  line mode diff
  78     -udiff              show unified diff (default)
  79     -n                  show me, do not run tests
  80     -T                  output timestamps
  81     -r                  randomize test order
  82     -d                  dump test output to stdout
  83     -b                  brief test summary
  84     -R fmt[,fmt]        generate report in formats specified. Supported format: [xunit]
  85     --large-fs          optimise scratch device for large filesystems
  86     -s section          run only specified section from config file
  87     -S section          exclude the specified section from the config file
  88
  89 testlist options
  90     -g group[,group...] include tests from these groups
  91     -x group[,group...] exclude tests from these groups
  92     -X exclude_file     exclude individual tests
  93     -E external_file    exclude individual tests
  94     [testlist]          include tests matching names in testlist
  95
  96 testlist argument is a list of tests in the form of <test dir>/<test name>.
  97
  98 <test dir> is a directory under tests that contains a group file,
  99 with a list of the names of the tests in that directory.
 100
 101 <test name> may be either a specific test file name (e.g. xfs/001) or
 102 a test file name match pattern (e.g. xfs/*).
 103
 104 group argument is either a name of a tests group to collect from all
 105 the test dirs (e.g. quick) or a name of a tests group to collect from
 106 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
 107 If you want to run all the tests in the test suite, use "-g all" to specify all
 108 groups.
 109
 110 exclude_file argument refers to a name of a file inside each test directory.
 111 for every test dir where this file is found, the listed test names are
 112 excluded from the list of tests to run from that test dir.
 113
 114 external_file argument is a path to a single file containing a list of tests
 115 to exclude in the form of <test dir>/<test name>.
 116
 117 examples:
 118  check xfs/001
 119  check -g quick
 120  check -g xfs/quick
 121  check -x stress xfs/*
 122  check -X .exclude -g auto
 123  check -E ~/.xfstests.exclude
 124 '
 125             exit 0
 126 }
 127
 128 get_sub_group_list()
 129 {
 130         local d=$1
 131         local grp=$2
 132
 133         test -s "$SRC_DIR/$d/group" || return 1
 134
 135         local grpl=$(sed -n < $SRC_DIR/$d/group \
 136                 -e 's/#.*//' \
 137                 -e 's/$/ /' \
 138                 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
 139         echo $grpl
 140 }
 141
 142 get_group_list()
 143 {
 144         local grp=$1
 145         local grpl=""
 146         local sub=$(dirname $grp)
 147
 148         if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
 149                 # group is given as <subdir>/<group> (e.g. xfs/quick)
 150                 grp=$(basename $grp)
 151                 get_sub_group_list $sub $grp
 152                 return
 153         fi
 154
 155         for d in $SRC_GROUPS $FSTYP; do
 156                 if ! test -d "$SRC_DIR/$d" ; then
 157                         continue
 158                 fi
 159                 grpl="$grpl $(get_sub_group_list $d $grp)"
 160         done
 161         echo $grpl
 162 }
 163
 164 # Find all tests, excluding files that are test metadata such as group files.
 165 # It matches test names against $VALID_TEST_NAME defined in common/rc
 166 get_all_tests()
 167 {
 168         touch $tmp.list
 169         for d in $SRC_GROUPS $FSTYP; do
 170                 if ! test -d "$SRC_DIR/$d" ; then
 171                         continue
 172                 fi
 173                 ls $SRC_DIR/$d/* | \
 174                         grep -v "\..*" | \
 175                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
 176                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
 177         done
 178 }
 179
 180 # takes the list of tests to run in $tmp.list, and removes the tests passed to
 181 # the function from that list.
 182 trim_test_list()
 183 {
 184         test_list="$*"
 185
 186         rm -f $tmp.grep
 187         numsed=0
 188         for t in $test_list
 189         do
 190             if [ $numsed -gt 100 ]; then
 191                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 192                 mv $tmp.tmp $tmp.list
 193                 numsed=0
 194                 rm -f $tmp.grep
 195             fi
 196             echo "^$t\$" >>$tmp.grep
 197             numsed=`expr $numsed + 1`
 198         done
 199         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 200         mv $tmp.tmp $tmp.list
 201         rm -f $tmp.grep
 202 }
 203
 204
 205 _wallclock()
 206 {
 207     date "+%s"
 208 }
 209
 210 _timestamp()
 211 {
 212     now=`date "+%T"`
 213     echo -n " [$now]"
 214 }
 215
 216 _prepare_test_list()
 217 {
 218         unset list
 219         # Tests specified on the command line
 220         if [ -s $tmp.arglist ]; then
 221                 cat $tmp.arglist > $tmp.list
 222         else
 223                 touch $tmp.list
 224         fi
 225
 226         # Specified groups to include
 227         # Note that the CLI processing adds a leading space to the first group
 228         # parameter, so we have to catch that here checking for "all"
 229         if ! $have_test_arg && [ "$GROUP_LIST" == " all" ]; then
 230                 # no test numbers, do everything
 231                 get_all_tests
 232         else
 233                 for group in $GROUP_LIST; do
 234                         list=$(get_group_list $group)
 235                         if [ -z "$list" ]; then
 236                                 echo "Group \"$group\" is empty or not defined?"
 237                                 exit 1
 238                         fi
 239
 240                         for t in $list; do
 241                                 grep -s "^$t\$" $tmp.list >/dev/null || \
 242                                                         echo "$t" >>$tmp.list
 243                         done
 244                 done
 245         fi
 246
 247         # Specified groups to exclude
 248         for xgroup in $XGROUP_LIST; do
 249                 list=$(get_group_list $xgroup)
 250                 if [ -z "$list" ]; then
 251                         echo "Group \"$xgroup\" is empty or not defined?"
 252                         exit 1
 253                 fi
 254
 255                 trim_test_list $list
 256         done
 257
 258         # sort the list of tests into numeric order
 259         list=`sort -n $tmp.list | uniq`
 260         rm -f $tmp.list
 261
 262         if $randomize
 263         then
 264                 list=`echo $list | awk -f randomize.awk`
 265         fi
 266 }
 267
 268 # Process command arguments first.
 269 while [ $# -gt 0 ]; do
 270         case "$1" in
 271         -\? | -h | --help) usage ;;
 272
 273         -nfs)           FSTYP=nfs ;;
 274         -glusterfs)     FSTYP=glusterfs ;;
 275         -cifs)          FSTYP=cifs ;;
 276         -9p)            FSTYP=9p ;;
 277         -overlay)       FSTYP=overlay; export OVERLAY=true ;;
 278         -pvfs2)         FSTYP=pvfs2 ;;
 279         -tmpfs)         FSTYP=tmpfs ;;
 280         -ubifs)         FSTYP=ubifs ;;
 281
 282         -g)     group=$2 ; shift ;
 283                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
 284                 ;;
 285
 286         -x)     xgroup=$2 ; shift ;
 287                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
 288                 ;;
 289
 290         -X)     xfile=$2; shift ;
 291                 for d in $SRC_GROUPS $FSTYP; do
 292                         [ -f $SRC_DIR/$d/$xfile ] || continue
 293                         for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
 294                                 echo $d/$f >> $tmp.xlist
 295                         done
 296                 done
 297                 ;;
 298         -E)     xfile=$2; shift ;
 299                 if [ -f $xfile ]; then
 300                         sed "s/#.*$//" "$xfile" >> $tmp.xlist
 301                 fi
 302                 ;;
 303         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
 304         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
 305         -l)     diff="diff" ;;
 306         -udiff) diff="$diff -u" ;;
 307
 308         -n)     showme=true ;;
 309         -r)     randomize=true ;;
 310
 311         -T)     timestamp=true ;;
 312         -d)     DUMP_OUTPUT=true ;;
 313         -b)     brief_test_summary=true;;
 314         -R)     report_fmt=$2 ; shift ;
 315                 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
 316                 do_report=true
 317                 ;;
 318         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
 319         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
 320
 321         -*)     usage ;;
 322         *)      # not an argument, we've got tests now.
 323                 have_test_arg=true ;;
 324         esac
 325
 326         # if we've found a test specification, the break out of the processing
 327         # loop before we shift the arguments so that this is the first argument
 328         # that we process in the test arg loop below.
 329         if $have_test_arg; then
 330                 break;
 331         fi
 332
 333         shift
 334 done
 335
 336 # we need common/rc, that also sources common/config. We need to source it
 337 # after processing args, overlay needs FSTYP set before sourcing common/config
 338 if ! . ./common/rc; then
 339         echo "check: failed to source common/rc"
 340         exit 1
 341 fi
 342
 343 # Process tests from command line now.
 344 if $have_test_arg; then
 345         while [ $# -gt 0 ]; do
 346                 case "$1" in
 347                 -*)     echo "Arguments before tests, please!"
 348                         status=1
 349                         exit $status
 350                         ;;
 351                 *)      # Expand test pattern (e.g. xfs/???, *fs/001)
 352                         list=$(cd $SRC_DIR; echo $1)
 353                         for t in $list; do
 354                                 test_dir=`dirname $t`
 355                                 test_dir=${test_dir#$SRC_DIR/*}
 356                                 test_name=`basename $t`
 357                                 group_file=$SRC_DIR/$test_dir/group
 358
 359                                 if egrep -q "^$test_name" $group_file; then
 360                                         # in group file ... OK
 361                                         echo $SRC_DIR/$test_dir/$test_name \
 362                                                 >>$tmp.arglist
 363                                 else
 364                                         # oops
 365                                         echo "$t - unknown test, ignored"
 366                                 fi
 367                         done
 368                         ;;
 369                 esac
 370
 371                 shift
 372         done
 373 elif [ -z "$GROUP_LIST" ]; then
 374         # default group list is the auto group. If any other group or test is
 375         # specified, we use that instead.
 376         GROUP_LIST="auto"
 377 fi
 378
 379 if [ `id -u` -ne 0 ]
 380 then
 381     echo "check: QA must be run as root"
 382     exit 1
 383 fi
 384
 385 _wipe_counters()
 386 {
 387         n_try="0"
 388         n_bad="0"
 389         n_notrun="0"
 390         unset try notrun bad
 391 }
 392
 393 _wrapup()
 394 {
 395         seq="check"
 396         check="$RESULT_BASE/check"
 397
 398         if $showme; then
 399                 if $needwrap; then
 400                         if $do_report; then
 401                                 _make_section_report
 402                         fi
 403                         needwrap=false
 404                 fi
 405         elif $needwrap; then
 406                 if [ -f $check.time -a -f $tmp.time ]; then
 407                         cat $check.time $tmp.time  \
 408                                 | $AWK_PROG '
 409                                 { t[$1] = $2 }
 410                                 END {
 411                                         if (NR > 0) {
 412                                                 for (i in t) print i " " t[i]
 413                                         }
 414                                 }' \
 415                                 | sort -n >$tmp.out
 416                         mv $tmp.out $check.time
 417                 fi
 418
 419                 echo "" >>$check.log
 420                 date >>$check.log
 421
 422                 echo "SECTION       -- $section" >>$tmp.summary
 423                 echo "=========================" >>$tmp.summary
 424                 if [ ! -z "$n_try" -a $n_try != 0 ]; then
 425                         if [ $brief_test_summary == "false" ]; then
 426                                 echo "Ran:$try"
 427                                 echo "Ran:$try" >>$tmp.summary
 428                         fi
 429                         echo "Ran:$try" >>$check.log
 430                 fi
 431
 432                 $interrupt && echo "Interrupted!" | tee -a $check.log
 433
 434                 if [ ! -z "$notrun" ]; then
 435                         if [ $brief_test_summary == "false" ]; then
 436                                 echo "Not run:$notrun"
 437                                 echo "Not run:$notrun" >>$tmp.summary
 438                         fi
 439                         echo "Not run:$notrun" >>$check.log
 440                 fi
 441
 442                 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
 443                         echo "Failures:$bad"
 444                         echo "Failed $n_bad of $n_try tests"
 445                         echo "Failures:$bad" >>$check.log
 446                         echo "Failed $n_bad of $n_try tests" >>$check.log
 447                         echo "Failures:$bad" >>$tmp.summary
 448                         echo "Failed $n_bad of $n_try tests" >>$tmp.summary
 449                 else
 450                         echo "Passed all $n_try tests"
 451                         echo "Passed all $n_try tests" >>$check.log
 452                         echo "Passed all $n_try tests" >>$tmp.summary
 453                 fi
 454                 echo "" >>$tmp.summary
 455                 if $do_report; then
 456                         _make_section_report
 457                 fi
 458                 needwrap=false
 459         fi
 460
 461         sum_bad=`expr $sum_bad + $n_bad`
 462         _wipe_counters
 463         rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
 464         if ! $OPTIONS_HAVE_SECTIONS; then
 465                 rm -f $tmp.*
 466         fi
 467 }
 468
 469 _summary()
 470 {
 471         _wrapup
 472         if $showme; then
 473                 :
 474         elif $needsum; then
 475                 count=`wc -L $tmp.summary | cut -f1 -d" "`
 476                 cat $tmp.summary
 477                 needsum=false
 478         fi
 479         rm -f $tmp.*
 480 }
 481
 482 _check_filesystems()
 483 {
 484         if [ -f ${RESULT_DIR}/require_test ]; then
 485                 _check_test_fs || err=true
 486                 rm -f ${RESULT_DIR}/require_test*
 487         else
 488                 _test_unmount 2> /dev/null
 489         fi
 490         if [ -f ${RESULT_DIR}/require_scratch ]; then
 491                 _check_scratch_fs || err=true
 492                 rm -f ${RESULT_DIR}/require_scratch*
 493         else
 494                 _scratch_unmount 2> /dev/null
 495         fi
 496 }
 497
 498 _expunge_test()
 499 {
 500         local TEST_ID="$1"
 501         if [ -s $tmp.xlist ]; then
 502                 if grep -q $TEST_ID $tmp.xlist; then
 503                         echo "       [expunged]"
 504                         return 1
 505                 fi
 506         fi
 507         return 0
 508 }
 509
 510 _init_kmemleak
 511 _prepare_test_list
 512
 513 if $OPTIONS_HAVE_SECTIONS; then
 514         trap "_summary; exit \$status" 0 1 2 3 15
 515 else
 516         trap "_wrapup; exit \$status" 0 1 2 3 15
 517 fi
 518
 519 for section in $HOST_OPTIONS_SECTIONS; do
 520         OLD_FSTYP=$FSTYP
 521         OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
 522         get_next_config $section
 523
 524         # Do we need to run only some sections ?
 525         if [ ! -z "$RUN_SECTION" ]; then
 526                 skip=true
 527                 for s in $RUN_SECTION; do
 528                         if [ $section == $s ]; then
 529                                 skip=false
 530                                 break;
 531                         fi
 532                 done
 533                 if $skip; then
 534                         continue
 535                 fi
 536         fi
 537
 538         # Did this section get excluded?
 539         if [ ! -z "$EXCLUDE_SECTION" ]; then
 540                 skip=false
 541                 for s in $EXCLUDE_SECTION; do
 542                         if [ $section == $s ]; then
 543                                 skip=true
 544                                 break;
 545                         fi
 546                 done
 547                 if $skip; then
 548                         continue
 549                 fi
 550         fi
 551
 552         mkdir -p $RESULT_BASE
 553         if [ ! -d $RESULT_BASE ]; then
 554                 echo "failed to create results directory $RESULT_BASE"
 555                 status=1
 556                 exit
 557         fi
 558
 559         if $OPTIONS_HAVE_SECTIONS; then
 560                 echo "SECTION       -- $section"
 561         fi
 562
 563         sect_start=`_wallclock`
 564         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
 565                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
 566                 _test_unmount 2> /dev/null
 567                 if ! _test_mkfs >$tmp.err 2>&1
 568                 then
 569                         echo "our local _test_mkfs routine ..."
 570                         cat $tmp.err
 571                         echo "check: failed to mkfs \$TEST_DEV using specified options"
 572                         status=1
 573                         exit
 574                 fi
 575                 if ! _test_mount
 576                 then
 577                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 578                         status=1
 579                         exit
 580                 fi
 581                 _prepare_test_list
 582         elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
 583                 _test_unmount 2> /dev/null
 584                 if ! _test_mount
 585                 then
 586                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 587                         status=1
 588                         exit
 589                 fi
 590         fi
 591
 592         init_rc
 593
 594         seq="check"
 595         check="$RESULT_BASE/check"
 596
 597         # don't leave old full output behind on a clean run
 598         rm -f $check.full
 599
 600         [ -f $check.time ] || touch $check.time
 601
 602         # print out our test configuration
 603         echo "FSTYP         -- `_full_fstyp_details`"
 604         echo "PLATFORM      -- `_full_platform_details`"
 605         if [ ! -z "$SCRATCH_DEV" ]; then
 606           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
 607           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
 608         fi
 609         echo
 610         needwrap=true
 611
 612         if [ ! -z "$SCRATCH_DEV" ]; then
 613           _scratch_unmount 2> /dev/null
 614           # call the overridden mkfs - make sure the FS is built
 615           # the same as we'll create it later.
 616
 617           if ! _scratch_mkfs >$tmp.err 2>&1
 618           then
 619               echo "our local _scratch_mkfs routine ..."
 620               cat $tmp.err
 621               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
 622               status=1
 623               exit
 624           fi
 625
 626           # call the overridden mount - make sure the FS mounts with
 627           # the same options that we'll mount with later.
 628           if ! _scratch_mount >$tmp.err 2>&1
 629           then
 630               echo "our local mount routine ..."
 631               cat $tmp.err
 632               echo "check: failed to mount \$SCRATCH_DEV using specified options"
 633               status=1
 634               exit
 635           fi
 636         fi
 637
 638         seqres="$check"
 639         _check_test_fs
 640
 641         err=false
 642         first_test=true
 643         prev_seq=""
 644         for seq in $list ; do
 645                 # Run report for previous test!
 646                 if $err ; then
 647                         bad="$bad $seqnum"
 648                         n_bad=`expr $n_bad + 1`
 649                         tc_status="fail"
 650                 fi
 651                 if $do_report && ! $first_test ; then
 652                         if [ $tc_status != "expunge" ] ; then
 653                                 _make_testcase_report "$prev_seq" "$tc_status"
 654                         fi
 655                 fi
 656                 first_test=false
 657
 658                 err=false
 659                 prev_seq="$seq"
 660                 if [ ! -f $seq ]; then
 661                         # Try to get full name in case the user supplied only
 662                         # seq id and the test has a name. A bit of hassle to
 663                         # find really the test and not its sample output or
 664                         # helping files.
 665                         bname=$(basename $seq)
 666                         full_seq=$(find $(dirname $seq) -name $bname* -executable |
 667                                 awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
 668                                      END { print shortest }')
 669                         if [ -f $full_seq ] && \
 670                            [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
 671                                 seq=$full_seq
 672                         fi
 673                 fi
 674
 675                 # the filename for the test and the name output are different.
 676                 # we don't include the tests/ directory in the name output.
 677                 export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
 678
 679                 # Similarly, the result directory needs to replace the tests/
 680                 # part of the test location.
 681                 group=`dirname $seq`
 682                 if $OPTIONS_HAVE_SECTIONS; then
 683                         export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
 684                         REPORT_DIR="$RESULT_BASE/$section"
 685                 else
 686                         export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
 687                         REPORT_DIR="$RESULT_BASE"
 688                 fi
 689                 seqres="$REPORT_DIR/$seqnum"
 690
 691                 mkdir -p $RESULT_DIR
 692                 echo -n "$seqnum"
 693
 694                 if $showme; then
 695                         _expunge_test $seqnum
 696                         if [ $? -eq 1 ]; then
 697                             tc_status="expunge"
 698                             continue
 699                         fi
 700                         echo
 701                         start=0
 702                         stop=0
 703                         tc_status="list"
 704                         n_notrun=`expr $n_notrun + 1`
 705                         continue
 706                 fi
 707
 708                 tc_status="pass"
 709                 if [ ! -f $seq ]; then
 710                         echo " - no such test?"
 711                         continue
 712                 fi
 713
 714                 # really going to try and run this one
 715                 rm -f $seqres.out.bad
 716
 717                 # check if we really should run it
 718                 _expunge_test $seqnum
 719                 if [ $? -eq 1 ]; then
 720                         tc_status="expunge"
 721                         continue
 722                 fi
 723
 724                 # record that we really tried to run this test.
 725                 try="$try $seqnum"
 726                 n_try=`expr $n_try + 1`
 727
 728                 # slashes now in names, sed barfs on them so use grep
 729                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
 730                 if [ "X$lasttime" != X ]; then
 731                         echo -n " ${lasttime}s ... "
 732                 else
 733                         echo -n "       " # prettier output with timestamps.
 734                 fi
 735                 rm -f core $seqres.notrun
 736
 737                 start=`_wallclock`
 738                 $timestamp && echo -n " ["`date "+%T"`"]"
 739                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
 740                 $LOGGER_PROG "run xfstest $seqnum"
 741                 if [ -w /dev/kmsg ]; then
 742                         export date_time=`date +"%F %T"`
 743                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
 744                         # _check_dmesg depends on this log in dmesg
 745                         touch ${RESULT_DIR}/check_dmesg
 746                 fi
 747                 if [ "$DUMP_OUTPUT" = true ]; then
 748                         ./$seq 2>&1 | tee $tmp.out
 749                         # Because $? would get tee's return code
 750                         sts=${PIPESTATUS[0]}
 751                 else
 752                         ./$seq >$tmp.out 2>&1
 753                         sts=$?
 754                 fi
 755
 756                 if [ -f core ]; then
 757                         _dump_err_cont "[dumped core]"
 758                         mv core $RESULT_BASE/$seqnum.core
 759                         err=true
 760                 fi
 761
 762                 if [ -f $seqres.notrun ]; then
 763                         $timestamp && _timestamp
 764                         stop=`_wallclock`
 765                         $timestamp || echo -n "[not run] "
 766                         $timestamp && echo " [not run]" && \
 767                                       echo -n " $seqnum -- "
 768                         cat $seqres.notrun
 769                         notrun="$notrun $seqnum"
 770                         n_notrun=`expr $n_notrun + 1`
 771                         tc_status="notrun"
 772                         continue;
 773                 fi
 774
 775                 if [ $sts -ne 0 ]; then
 776                         _dump_err_cont "[failed, exit status $sts]"
 777                         _test_unmount 2> /dev/null
 778                         _scratch_unmount 2> /dev/null
 779                         err=true
 780                 else
 781                         # the test apparently passed, so check for corruption
 782                         # and log messages that shouldn't be there.
 783                         _check_filesystems
 784                         _check_dmesg || err=true
 785                         _check_kmemleak || err=true
 786                 fi
 787
 788                 # test ends after all checks are done.
 789                 $timestamp && _timestamp
 790                 stop=`_wallclock`
 791
 792                 if [ ! -f $seq.out ]; then
 793                         _dump_err "no qualified output"
 794                         err=true
 795                         continue;
 796                 fi
 797
 798                 # coreutils 8.16+ changed quote formats in error messages
 799                 # from `foo' to 'foo'. Filter old versions to match the new
 800                 # version.
 801                 sed -i "s/\`/\'/g" $tmp.out
 802                 if diff $seq.out $tmp.out >/dev/null 2>&1 ; then
 803                         if ! $err ; then
 804                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
 805                                 echo -n " `expr $stop - $start`s"
 806                         fi
 807                         echo ""
 808                 else
 809                         _dump_err "- output mismatch (see $seqres.out.bad)"
 810                         mv $tmp.out $seqres.out.bad
 811                         $diff $seq.out $seqres.out.bad | {
 812                         if test "$DIFF_LENGTH" -le 0; then
 813                                 cat
 814                         else
 815                                 head -n "$DIFF_LENGTH"
 816                                 echo "..."
 817                                 echo "(Run '$diff $seq.out $seqres.out.bad'" \
 818                                         " to see the entire diff)"
 819                         fi; } | sed -e 's/^\(.\)/    \1/'
 820                         err=true
 821                 fi
 822         done
 823
 824         # make sure we record the status of the last test we ran.
 825         if $err ; then
 826                 bad="$bad $seqnum"
 827                 n_bad=`expr $n_bad + 1`
 828                 tc_status="fail"
 829         fi
 830         if $do_report && ! $first_test ; then
 831                 if [ $tc_status != "expunge" ] ; then
 832                         _make_testcase_report "$prev_seq" "$tc_status"
 833                 fi
 834         fi
 835
 836         sect_stop=`_wallclock`
 837         interrupt=false
 838         _wrapup
 839         interrupt=true
 840         echo
 841
 842         _test_unmount 2> /dev/null
 843         _scratch_unmount 2> /dev/null
 844 done
 845
 846 interrupt=false
 847 status=`expr $sum_bad != 0`
 848 exit