check

   1 #!/bin/bash
   2 #
   3 # Control script for QA
   4 #
   5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation.
  10 #
  11 # This program is distributed in the hope that it would be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write the Free Software Foundation,
  18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  19 #
  20 #
  21
  22 tmp=/tmp/$$
  23 status=0
  24 needwrap=true
  25 needsum=true
  26 n_try=0
  27 try=""
  28 n_bad=0
  29 sum_bad=0
  30 bad=""
  31 n_notrun=0
  32 notrun=""
  33 interrupt=true
  34 diff="diff -u"
  35 showme=false
  36 have_test_arg=false
  37 randomize=false
  38 export here=`pwd`
  39 xfile=""
  40 brief_test_summary=false
  41 err_msg=""
  42 do_report=false
  43 DUMP_OUTPUT=false
  44
  45 # start the initialisation work now
  46 iam=check
  47
  48 export MSGVERB="text:action"
  49 export QA_CHECK_FS=${QA_CHECK_FS:=true}
  50
  51 # number of diff lines from a failed test, 0 for whole output
  52 export DIFF_LENGTH=${DIFF_LENGTH:=10}
  53
  54 # by default don't output timestamps
  55 timestamp=${TIMESTAMP:=false}
  56
  57 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
  58
  59 SRC_GROUPS="generic shared"
  60 export SRC_DIR="tests"
  61
  62 usage()
  63 {
  64     echo "Usage: $0 [options] [testlist]"'
  65
  66 check options
  67     -nfs                test NFS
  68     -glusterfs                test GlusterFS
  69     -cifs               test CIFS
  70     -9p                 test 9p
  71     -overlay            test overlay
  72     -pvfs2          test PVFS2
  73     -tmpfs              test TMPFS
  74     -ubifs              test ubifs
  75     -l                  line mode diff
  76     -udiff              show unified diff (default)
  77     -n                  show me, do not run tests
  78     -T                  output timestamps
  79     -r                  randomize test order
  80     -d                  dump test output to stdout
  81     -b                  brief test summary
  82     -R fmt[,fmt]        generate report in formats specified. Supported format: [xunit]
  83     --large-fs          optimise scratch device for large filesystems
  84     -s section          run only specified section from config file
  85     -S section          exclude the specified section from the config file
  86
  87 testlist options
  88     -g group[,group...] include tests from these groups
  89     -x group[,group...] exclude tests from these groups
  90     -X exclude_file     exclude individual tests
  91     -E external_file    exclude individual tests
  92     [testlist]          include tests matching names in testlist
  93
  94 testlist argument is a list of tests in the form of <test dir>/<test name>.
  95
  96 <test dir> is a directory under tests that contains a group file,
  97 with a list of the names of the tests in that directory.
  98
  99 <test name> may be either a specific test file name (e.g. xfs/001) or
 100 a test file name match pattern (e.g. xfs/*).
 101
 102 group argument is either a name of a tests group to collect from all
 103 the test dirs (e.g. quick) or a name of a tests group to collect from
 104 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
 105
 106 exclude_file argument refers to a name of a file inside each test directory.
 107 for every test dir where this file is found, the listed test names are
 108 excluded from the list of tests to run from that test dir.
 109
 110 external_file argument is a path to a single file containing a list of tests
 111 to exclude in the form of <test dir>/<test name>.
 112
 113 examples:
 114  check xfs/001
 115  check -g quick
 116  check -g xfs/quick
 117  check -x stress xfs/*
 118  check -X .exclude -g auto
 119  check -E ~/.xfstests.exclude
 120 '
 121             exit 0
 122 }
 123
 124 get_sub_group_list()
 125 {
 126         local d=$1
 127         local grp=$2
 128
 129         test -s "$SRC_DIR/$d/group" || return 1
 130
 131         local grpl=$(sed -n < $SRC_DIR/$d/group \
 132                 -e 's/#.*//' \
 133                 -e 's/$/ /' \
 134                 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
 135         echo $grpl
 136 }
 137
 138 get_group_list()
 139 {
 140         local grp=$1
 141         local grpl=""
 142         local sub=$(dirname $grp)
 143
 144         if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
 145                 # group is given as <subdir>/<group> (e.g. xfs/quick)
 146                 grp=$(basename $grp)
 147                 get_sub_group_list $sub $grp
 148                 return
 149         fi
 150
 151         for d in $SRC_GROUPS $FSTYP; do
 152                 if ! test -d "$SRC_DIR/$d" ; then
 153                         continue
 154                 fi
 155                 grpl="$grpl $(get_sub_group_list $d $grp)"
 156         done
 157         echo $grpl
 158 }
 159
 160 # Find all tests, excluding files that are test metadata such as group files.
 161 # It matches test names against $VALID_TEST_NAME defined in common/rc
 162 get_all_tests()
 163 {
 164         touch $tmp.list
 165         for d in $SRC_GROUPS $FSTYP; do
 166                 if ! test -d "$SRC_DIR/$d" ; then
 167                         continue
 168                 fi
 169                 ls $SRC_DIR/$d/* | \
 170                         grep -v "\..*" | \
 171                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
 172                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
 173         done
 174 }
 175
 176 # takes the list of tests to run in $tmp.list, and removes the tests passed to
 177 # the function from that list.
 178 trim_test_list()
 179 {
 180         test_list="$*"
 181
 182         rm -f $tmp.grep
 183         numsed=0
 184         for t in $test_list
 185         do
 186             if [ $numsed -gt 100 ]; then
 187                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 188                 mv $tmp.tmp $tmp.list
 189                 numsed=0
 190                 rm -f $tmp.grep
 191             fi
 192             echo "^$t\$" >>$tmp.grep
 193             numsed=`expr $numsed + 1`
 194         done
 195         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 196         mv $tmp.tmp $tmp.list
 197 }
 198
 199
 200 _wallclock()
 201 {
 202     date "+%s"
 203 }
 204
 205 _timestamp()
 206 {
 207     now=`date "+%T"`
 208     echo -n " [$now]"
 209 }
 210
 211 _prepare_test_list()
 212 {
 213         unset list
 214         # Tests specified on the command line
 215         if [ -s $tmp.arglist ]; then
 216                 cat $tmp.arglist > $tmp.list
 217         else
 218                 touch $tmp.list
 219         fi
 220
 221         # Specified groups to include
 222         for group in $GROUP_LIST; do
 223                 list=$(get_group_list $group)
 224                 if [ -z "$list" ]; then
 225                         echo "Group \"$group\" is empty or not defined?"
 226                         exit 1
 227                 fi
 228
 229                 for t in $list; do
 230                         grep -s "^$t\$" $tmp.list >/dev/null || \
 231                                                         echo "$t" >>$tmp.list
 232                 done
 233         done
 234
 235         if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
 236                 # no test numbers, do everything
 237                 get_all_tests
 238         fi
 239
 240         # Specified groups to exclude
 241         for xgroup in $XGROUP_LIST; do
 242                 list=$(get_group_list $xgroup)
 243                 if [ -z "$list" ]; then
 244                         echo "Group \"$xgroup\" is empty or not defined?"
 245                         exit 1
 246                 fi
 247
 248                 trim_test_list $list
 249         done
 250
 251         # sort the list of tests into numeric order
 252         list=`sort -n $tmp.list | uniq`
 253         rm -f $tmp.list $tmp.tmp $tmp.grep
 254
 255         if $randomize
 256         then
 257                 list=`echo $list | awk -f randomize.awk`
 258         fi
 259 }
 260
 261 # Process command arguments first.
 262 while [ $# -gt 0 ]; do
 263         case "$1" in
 264         -\? | -h | --help) usage ;;
 265
 266         -nfs)           FSTYP=nfs ;;
 267         -glusterfs)     FSTYP=glusterfs ;;
 268         -cifs)          FSTYP=cifs ;;
 269         -9p)            FSTYP=9p ;;
 270         -overlay)       FSTYP=overlay; export OVERLAY=true ;;
 271         -pvfs2)         FSTYP=pvfs2 ;;
 272         -tmpfs)         FSTYP=tmpfs ;;
 273         -ubifs)         FSTYP=ubifs ;;
 274
 275         -g)     group=$2 ; shift ;
 276                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
 277                 ;;
 278
 279         -x)     xgroup=$2 ; shift ;
 280                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
 281                 ;;
 282
 283         -X)     xfile=$2; shift ;
 284                 for d in $SRC_GROUPS $FSTYP; do
 285                         [ -f $SRC_DIR/$d/$xfile ] || continue
 286                         for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
 287                                 echo $d/$f >> $tmp.xlist
 288                         done
 289                 done
 290                 ;;
 291         -E)     xfile=$2; shift ;
 292                 if [ -f $xfile ]; then
 293                         sed "s/#.*$//" "$xfile" >> $tmp.xlist
 294                 fi
 295                 ;;
 296         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
 297         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
 298         -l)     diff="diff" ;;
 299         -udiff) diff="$diff -u" ;;
 300
 301         -n)     showme=true ;;
 302         -r)     randomize=true ;;
 303
 304         -T)     timestamp=true ;;
 305         -d)     DUMP_OUTPUT=true ;;
 306         -b)     brief_test_summary=true;;
 307         -R)     report_fmt=$2 ; shift ;
 308                 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
 309                 do_report=true
 310                 ;;
 311         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
 312         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
 313
 314         -*)     usage ;;
 315         *)      # not an argument, we've got tests now.
 316                 have_test_arg=true ;;
 317         esac
 318
 319         # if we've found a test specification, the break out of the processing
 320         # loop before we shift the arguments so that this is the first argument
 321         # that we process in the test arg loop below.
 322         if $have_test_arg; then
 323                 break;
 324         fi
 325
 326         shift
 327 done
 328
 329 # we need common/config, source it after processing args, overlay needs FSTYP
 330 # set before sourcing common/config
 331 if ! . ./common/config; then
 332         echo "$iam: failed to source common/config"
 333         exit 1
 334 fi
 335
 336 # Process tests from command line now.
 337 if $have_test_arg; then
 338         while [ $# -gt 0 ]; do
 339                 case "$1" in
 340                 -*)     echo "Arguments before tests, please!"
 341                         status=1
 342                         exit $status
 343                         ;;
 344                 *)      # Expand test pattern (e.g. xfs/???, *fs/001)
 345                         list=$(cd $SRC_DIR; echo $1)
 346                         for t in $list; do
 347                                 test_dir=`dirname $t`
 348                                 test_dir=${test_dir#$SRC_DIR/*}
 349                                 test_name=`basename $t`
 350                                 group_file=$SRC_DIR/$test_dir/group
 351
 352                                 if egrep -q "^$test_name" $group_file; then
 353                                         # in group file ... OK
 354                                         echo $SRC_DIR/$test_dir/$test_name \
 355                                                 >>$tmp.arglist
 356                                 else
 357                                         # oops
 358                                         echo "$t - unknown test, ignored"
 359                                 fi
 360                         done
 361                         ;;
 362                 esac
 363
 364                 shift
 365         done
 366 fi
 367
 368 # we need common/rc
 369 if ! . ./common/rc
 370 then
 371     echo "check: failed to source common/rc"
 372     exit 1
 373 fi
 374
 375 if [ `id -u` -ne 0 ]
 376 then
 377     echo "check: QA must be run as root"
 378     exit 1
 379 fi
 380
 381 _wipe_counters()
 382 {
 383         n_try="0"
 384         n_bad="0"
 385         n_notrun="0"
 386         unset try notrun bad
 387 }
 388
 389 _wrapup()
 390 {
 391         seq="check"
 392         check="$RESULT_BASE/check"
 393
 394         if $showme; then
 395                 if $needwrap; then
 396                         if $do_report; then
 397                                 _make_section_report
 398                         fi
 399                         needwrap=false
 400                 fi
 401         elif $needwrap; then
 402                 if [ -f $check.time -a -f $tmp.time ]; then
 403                         cat $check.time $tmp.time  \
 404                                 | $AWK_PROG '
 405                                 { t[$1] = $2 }
 406                                 END {
 407                                         if (NR > 0) {
 408                                                 for (i in t) print i " " t[i]
 409                                         }
 410                                 }' \
 411                                 | sort -n >$tmp.out
 412                         mv $tmp.out $check.time
 413                 fi
 414
 415                 echo "" >>$check.log
 416                 date >>$check.log
 417
 418                 echo "SECTION       -- $section" >>$tmp.summary
 419                 echo "=========================" >>$tmp.summary
 420                 if [ ! -z "$n_try" -a $n_try != 0 ]; then
 421                         if [ $brief_test_summary == "false" ]; then
 422                                 echo "Ran:$try"
 423                                 echo "Ran:$try" >>$tmp.summary
 424                         fi
 425                         echo "Ran:$try" >>$check.log
 426                 fi
 427
 428                 $interrupt && echo "Interrupted!" | tee -a $check.log
 429
 430                 if [ ! -z "$notrun" ]; then
 431                         if [ $brief_test_summary == "false" ]; then
 432                                 echo "Not run:$notrun"
 433                                 echo "Not run:$notrun" >>$tmp.summary
 434                         fi
 435                         echo "Not run:$notrun" >>$check.log
 436                 fi
 437
 438                 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
 439                         echo "Failures:$bad"
 440                         echo "Failed $n_bad of $n_try tests"
 441                         echo "Failures:$bad" >>$check.log
 442                         echo "Failed $n_bad of $n_try tests" >>$check.log
 443                         echo "Failures:$bad" >>$tmp.summary
 444                         echo "Failed $n_bad of $n_try tests" >>$tmp.summary
 445                 else
 446                         echo "Passed all $n_try tests"
 447                         echo "Passed all $n_try tests" >>$check.log
 448                         echo "Passed all $n_try tests" >>$tmp.summary
 449                 fi
 450                 echo "" >>$tmp.summary
 451                 if $do_report; then
 452                         _make_section_report
 453                 fi
 454                 needwrap=false
 455         fi
 456
 457         sum_bad=`expr $sum_bad + $n_bad`
 458         _wipe_counters
 459         rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
 460         if ! $OPTIONS_HAVE_SECTIONS; then
 461                 rm -f $tmp.*
 462         fi
 463 }
 464
 465 _summary()
 466 {
 467         _wrapup
 468         if $showme; then
 469                 :
 470         elif $needsum; then
 471                 count=`wc -L $tmp.summary | cut -f1 -d" "`
 472                 cat $tmp.summary
 473                 needsum=false
 474         fi
 475         rm -f $tmp.*
 476 }
 477
 478 _check_filesystems()
 479 {
 480         if [ -f ${RESULT_DIR}/require_test ]; then
 481                 _check_test_fs || err=true
 482                 rm -f ${RESULT_DIR}/require_test*
 483         fi
 484         if [ -f ${RESULT_DIR}/require_scratch ]; then
 485                 _check_scratch_fs || err=true
 486                 rm -f ${RESULT_DIR}/require_scratch*
 487         fi
 488 }
 489
 490 _expunge_test()
 491 {
 492         local TEST_ID="$1"
 493         if [ -s $tmp.xlist ]; then
 494                 if grep -q $TEST_ID $tmp.xlist; then
 495                         echo "       [expunged]"
 496                         return 1
 497                 fi
 498         fi
 499         return 0
 500 }
 501
 502 _init_kmemleak
 503 _prepare_test_list
 504
 505 if $OPTIONS_HAVE_SECTIONS; then
 506         trap "_summary; exit \$status" 0 1 2 3 15
 507 else
 508         trap "_wrapup; exit \$status" 0 1 2 3 15
 509 fi
 510
 511 for section in $HOST_OPTIONS_SECTIONS; do
 512         OLD_FSTYP=$FSTYP
 513         OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
 514         get_next_config $section
 515
 516         # Do we need to run only some sections ?
 517         if [ ! -z "$RUN_SECTION" ]; then
 518                 skip=true
 519                 for s in $RUN_SECTION; do
 520                         if [ $section == $s ]; then
 521                                 skip=false
 522                                 break;
 523                         fi
 524                 done
 525                 if $skip; then
 526                         continue
 527                 fi
 528         fi
 529
 530         # Did this section get excluded?
 531         if [ ! -z "$EXCLUDE_SECTION" ]; then
 532                 skip=false
 533                 for s in $EXCLUDE_SECTION; do
 534                         if [ $section == $s ]; then
 535                                 skip=true
 536                                 break;
 537                         fi
 538                 done
 539                 if $skip; then
 540                         continue
 541                 fi
 542         fi
 543
 544         mkdir -p $RESULT_BASE
 545         if [ ! -d $RESULT_BASE ]; then
 546                 echo "failed to create results directory $RESULT_BASE"
 547                 status=1
 548                 exit
 549         fi
 550
 551         if $OPTIONS_HAVE_SECTIONS; then
 552                 echo "SECTION       -- $section"
 553         fi
 554
 555         sect_start=`_wallclock`
 556         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
 557                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
 558                 _test_unmount 2> /dev/null
 559                 if ! _test_mkfs >$tmp.err 2>&1
 560                 then
 561                         echo "our local _test_mkfs routine ..."
 562                         cat $tmp.err
 563                         echo "check: failed to mkfs \$TEST_DEV using specified options"
 564                         status=1
 565                         exit
 566                 fi
 567                 if ! _test_mount
 568                 then
 569                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 570                         status=1
 571                         exit
 572                 fi
 573                 _prepare_test_list
 574         elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
 575                 _test_unmount 2> /dev/null
 576                 if ! _test_mount
 577                 then
 578                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 579                         status=1
 580                         exit
 581                 fi
 582         fi
 583
 584         init_rc
 585
 586         seq="check"
 587         check="$RESULT_BASE/check"
 588
 589         # don't leave old full output behind on a clean run
 590         rm -f $check.full
 591
 592         [ -f $check.time ] || touch $check.time
 593
 594         # print out our test configuration
 595         echo "FSTYP         -- `_full_fstyp_details`"
 596         echo "PLATFORM      -- `_full_platform_details`"
 597         if [ ! -z "$SCRATCH_DEV" ]; then
 598           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
 599           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
 600         fi
 601         echo
 602         needwrap=true
 603
 604         if [ ! -z "$SCRATCH_DEV" ]; then
 605           _scratch_unmount 2> /dev/null
 606           # call the overridden mkfs - make sure the FS is built
 607           # the same as we'll create it later.
 608
 609           if ! _scratch_mkfs >$tmp.err 2>&1
 610           then
 611               echo "our local _scratch_mkfs routine ..."
 612               cat $tmp.err
 613               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
 614               status=1
 615               exit
 616           fi
 617
 618           # call the overridden mount - make sure the FS mounts with
 619           # the same options that we'll mount with later.
 620           if ! _scratch_mount >$tmp.err 2>&1
 621           then
 622               echo "our local mount routine ..."
 623               cat $tmp.err
 624               echo "check: failed to mount \$SCRATCH_DEV using specified options"
 625               status=1
 626               exit
 627           fi
 628         fi
 629
 630         seqres="$check"
 631         _check_test_fs
 632
 633         for seq in $list
 634         do
 635             err=false
 636             err_msg=""
 637             if [ ! -f $seq ]; then
 638                 # Try to get full name in case the user supplied only seq id
 639                 # and the test has a name. A bit of hassle to find really
 640                 # the test and not its sample output or helping files.
 641                 bname=$(basename $seq)
 642                 full_seq=$(find $(dirname $seq) -name $bname* -executable |
 643                     awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
 644                         END { print shortest }')
 645                 if [ -f $full_seq ] \
 646                     && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
 647                     seq=$full_seq
 648                 fi
 649             fi
 650
 651             # the filename for the test and the name output are different.
 652             # we don't include the tests/ directory in the name output.
 653             export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
 654
 655             # Similarly, the result directory needs to replace the tests/
 656             # part of the test location.
 657             group=`dirname $seq`
 658             if $OPTIONS_HAVE_SECTIONS; then
 659                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
 660                 REPORT_DIR="$RESULT_BASE/$section"
 661             else
 662                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
 663                 REPORT_DIR="$RESULT_BASE"
 664             fi
 665             seqres="$REPORT_DIR/$seqnum"
 666
 667             mkdir -p $RESULT_DIR
 668
 669             echo -n "$seqnum"
 670
 671             if $showme; then
 672                 _expunge_test $seqnum
 673                 if [ $? -eq 1 ]; then
 674                         continue
 675                 fi
 676                 echo
 677                 start=0
 678                 stop=0
 679                 n_notrun=`expr $n_notrun + 1`
 680                 if $do_report; then
 681                         _make_testcase_report "list"
 682                 fi
 683                 continue
 684             fi
 685             tc_status="pass"
 686             if [ ! -f $seq ]; then
 687                 echo " - no such test?"
 688             else
 689                 # really going to try and run this one
 690                 #
 691                 rm -f $seqres.out.bad
 692
 693                 # check if we really should run it
 694                 _expunge_test $seqnum
 695                 if [ $? -eq 1 ]; then
 696                         continue
 697                 fi
 698
 699                 # slashes now in names, sed barfs on them so use grep
 700                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
 701                 if [ "X$lasttime" != X ]; then
 702                         echo -n " ${lasttime}s ..."
 703                 else
 704                         echo -n "       "       # prettier output with timestamps.
 705                 fi
 706                 rm -f core $seqres.notrun
 707
 708                 start=`_wallclock`
 709                 $timestamp && echo -n " ["`date "+%T"`"]"
 710                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
 711                 $LOGGER_PROG "run xfstest $seqnum"
 712                 if [ -w /dev/kmsg ]; then
 713                         export date_time=`date +"%F %T"`
 714                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
 715                         # _check_dmesg depends on this log in dmesg
 716                         touch ${RESULT_DIR}/check_dmesg
 717                 fi
 718                 if [ "$DUMP_OUTPUT" = true ]; then
 719                         ./$seq 2>&1 | tee $tmp.rawout
 720                         # Because $? would get tee's return code
 721                         sts=${PIPESTATUS[0]}
 722                 else
 723                         ./$seq >$tmp.rawout 2>&1
 724                         sts=$?
 725                 fi
 726                 $timestamp && _timestamp
 727                 stop=`_wallclock`
 728
 729                 _fix_malloc <$tmp.rawout >$tmp.out
 730                 rm -f $tmp.rawout
 731
 732                 if [ -f core ]
 733                 then
 734                     err_msg="[dumped core]"
 735                     echo -n " $err_msg"
 736                     mv core $RESULT_BASE/$seqnum.core
 737                     err=true
 738                 fi
 739
 740                 if [ -f $seqres.notrun ]
 741                 then
 742                     $timestamp || echo -n " [not run] "
 743                     $timestamp && echo " [not run]" && echo -n "        $seqnum -- "
 744                     cat $seqres.notrun
 745                     notrun="$notrun $seqnum"
 746                     n_notrun=`expr $n_notrun + 1`
 747                     tc_status="notrun"
 748                 else
 749                     if [ $sts -ne 0 ]
 750                     then
 751                         err_msg="[failed, exit status $sts]"
 752                         echo -n " $err_msg"
 753                         err=true
 754                     fi
 755                     if [ ! -f $seq.out ]
 756                     then
 757                         _dump_err "no qualified output"
 758                         err=true
 759                     else
 760
 761                         # coreutils 8.16+ changed quote formats in error messages from
 762                         # `foo' to 'foo'. Filter old versions to match the new version.
 763                         sed -i "s/\`/\'/g" $tmp.out
 764                         if diff $seq.out $tmp.out >/dev/null 2>&1
 765                         then
 766                             if $err
 767                             then
 768                                 :
 769                             else
 770                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
 771                                 echo -n " `expr $stop - $start`s"
 772                             fi
 773                             echo ""
 774                         else
 775                             echo " - output mismatch (see $seqres.out.bad)"
 776                             mv $tmp.out $seqres.out.bad
 777                             $diff $seq.out $seqres.out.bad | {
 778                                 if test "$DIFF_LENGTH" -le 0; then
 779                                         cat
 780                                 else
 781                                         head -n "$DIFF_LENGTH"
 782                                         echo "..."
 783                                         echo "(Run '$diff $seq.out $seqres.out.bad'" \
 784                                                 " to see the entire diff)"
 785                                 fi; } | \
 786                                 sed -e 's/^\(.\)/    \1/'
 787                             err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)"
 788                             err=true
 789                         fi
 790                     fi
 791                     try="$try $seqnum"
 792                     n_try=`expr $n_try + 1`
 793                     _check_filesystems
 794                     _check_dmesg || err=true
 795                     _check_kmemleak || err=true
 796                 fi
 797
 798             fi
 799
 800             # come here for each test, except when $showme is true
 801             #
 802             if $err
 803             then
 804                 bad="$bad $seqnum"
 805                 n_bad=`expr $n_bad + 1`
 806                 quick=false
 807                 tc_status="fail"
 808             fi
 809             if $do_report; then
 810                 _make_testcase_report "$tc_status"
 811             fi
 812             seq="after_$seqnum"
 813         done
 814         sect_stop=`_wallclock`
 815         interrupt=false
 816         _wrapup
 817         interrupt=true
 818         echo
 819
 820         _test_unmount 2> /dev/null
 821         _scratch_unmount 2> /dev/null
 822 done
 823
 824 interrupt=false
 825 status=`expr $sum_bad`
 826 exit