check

   1 #!/bin/bash
   2 #
   3 # Control script for QA
   4 #
   5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation.
  10 #
  11 # This program is distributed in the hope that it would be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write the Free Software Foundation,
  18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  19 #
  20 #
  21
  22 tmp=/tmp/$$
  23 status=0
  24 needwrap=true
  25 needsum=true
  26 n_try=0
  27 try=""
  28 n_bad=0
  29 sum_bad=0
  30 bad=""
  31 n_notrun=0
  32 notrun=""
  33 interrupt=true
  34 diff="diff -u"
  35 showme=false
  36 have_test_arg=false
  37 randomize=false
  38 export here=`pwd`
  39 xfile=""
  40 brief_test_summary=false
  41 err_msg=""
  42 do_report=false
  43 DUMP_OUTPUT=false
  44
  45 # start the initialisation work now
  46 iam=check
  47
  48 export MSGVERB="text:action"
  49 export QA_CHECK_FS=${QA_CHECK_FS:=true}
  50
  51 # number of diff lines from a failed test, 0 for whole output
  52 export DIFF_LENGTH=${DIFF_LENGTH:=10}
  53
  54 # by default don't output timestamps
  55 timestamp=${TIMESTAMP:=false}
  56
  57 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
  58
  59 SRC_GROUPS="generic shared"
  60 export SRC_DIR="tests"
  61
  62 usage()
  63 {
  64     echo "Usage: $0 [options] [testlist]"'
  65
  66 check options
  67     -nfs                test NFS
  68     -glusterfs                test GlusterFS
  69     -cifs               test CIFS
  70     -9p                 test 9p
  71     -overlay            test overlay
  72     -pvfs2          test PVFS2
  73     -tmpfs              test TMPFS
  74     -ubifs              test ubifs
  75     -l                  line mode diff
  76     -udiff              show unified diff (default)
  77     -n                  show me, do not run tests
  78     -T                  output timestamps
  79     -r                  randomize test order
  80     -d                  dump test output to stdout
  81     -b                  brief test summary
  82     -R fmt[,fmt]        generate report in formats specified. Supported format: [xunit]
  83     --large-fs          optimise scratch device for large filesystems
  84     -s section          run only specified section from config file
  85     -S section          exclude the specified section from the config file
  86
  87 testlist options
  88     -g group[,group...] include tests from these groups
  89     -x group[,group...] exclude tests from these groups
  90     -X exclude_file     exclude individual tests
  91     -E external_file    exclude individual tests
  92     [testlist]          include tests matching names in testlist
  93
  94 testlist argument is a list of tests in the form of <test dir>/<test name>.
  95
  96 <test dir> is a directory under tests that contains a group file,
  97 with a list of the names of the tests in that directory.
  98
  99 <test name> may be either a specific test file name (e.g. xfs/001) or
 100 a test file name match pattern (e.g. xfs/*).
 101
 102 group argument is either a name of a tests group to collect from all
 103 the test dirs (e.g. quick) or a name of a tests group to collect from
 104 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
 105
 106 exclude_file argument refers to a name of a file inside each test directory.
 107 for every test dir where this file is found, the listed test names are
 108 excluded from the list of tests to run from that test dir.
 109
 110 external_file argument is a path to a single file containing a list of tests
 111 to exclude in the form of <test dir>/<test name>.
 112
 113 examples:
 114  check xfs/001
 115  check -g quick
 116  check -g xfs/quick
 117  check -x stress xfs/*
 118  check -X .exclude -g auto
 119  check -E ~/.xfstests.exclude
 120 '
 121             exit 0
 122 }
 123
 124 get_sub_group_list()
 125 {
 126         local d=$1
 127         local grp=$2
 128
 129         test -s "$SRC_DIR/$d/group" || return 1
 130
 131         local grpl=$(sed -n < $SRC_DIR/$d/group \
 132                 -e 's/#.*//' \
 133                 -e 's/$/ /' \
 134                 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
 135         echo $grpl
 136 }
 137
 138 get_group_list()
 139 {
 140         local grp=$1
 141         local grpl=""
 142         local sub=$(dirname $grp)
 143
 144         if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
 145                 # group is given as <subdir>/<group> (e.g. xfs/quick)
 146                 grp=$(basename $grp)
 147                 get_sub_group_list $sub $grp
 148                 return
 149         fi
 150
 151         for d in $SRC_GROUPS $FSTYP; do
 152                 if ! test -d "$SRC_DIR/$d" ; then
 153                         continue
 154                 fi
 155                 grpl="$grpl $(get_sub_group_list $d $grp)"
 156         done
 157         echo $grpl
 158 }
 159
 160 # Find all tests, excluding files that are test metadata such as group files.
 161 # It matches test names against $VALID_TEST_NAME defined in common/rc
 162 get_all_tests()
 163 {
 164         touch $tmp.list
 165         for d in $SRC_GROUPS $FSTYP; do
 166                 if ! test -d "$SRC_DIR/$d" ; then
 167                         continue
 168                 fi
 169                 ls $SRC_DIR/$d/* | \
 170                         grep -v "\..*" | \
 171                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
 172                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
 173         done
 174 }
 175
 176 # takes the list of tests to run in $tmp.list, and removes the tests passed to
 177 # the function from that list.
 178 trim_test_list()
 179 {
 180         test_list="$*"
 181
 182         rm -f $tmp.grep
 183         numsed=0
 184         for t in $test_list
 185         do
 186             if [ $numsed -gt 100 ]; then
 187                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 188                 mv $tmp.tmp $tmp.list
 189                 numsed=0
 190                 rm -f $tmp.grep
 191             fi
 192             echo "^$t\$" >>$tmp.grep
 193             numsed=`expr $numsed + 1`
 194         done
 195         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 196         mv $tmp.tmp $tmp.list
 197         rm -f $tmp.grep
 198 }
 199
 200
 201 _wallclock()
 202 {
 203     date "+%s"
 204 }
 205
 206 _timestamp()
 207 {
 208     now=`date "+%T"`
 209     echo -n " [$now]"
 210 }
 211
 212 _prepare_test_list()
 213 {
 214         unset list
 215         # Tests specified on the command line
 216         if [ -s $tmp.arglist ]; then
 217                 cat $tmp.arglist > $tmp.list
 218         else
 219                 touch $tmp.list
 220         fi
 221
 222         # Specified groups to include
 223         for group in $GROUP_LIST; do
 224                 list=$(get_group_list $group)
 225                 if [ -z "$list" ]; then
 226                         echo "Group \"$group\" is empty or not defined?"
 227                         exit 1
 228                 fi
 229
 230                 for t in $list; do
 231                         grep -s "^$t\$" $tmp.list >/dev/null || \
 232                                                         echo "$t" >>$tmp.list
 233                 done
 234         done
 235
 236         if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
 237                 # no test numbers, do everything
 238                 get_all_tests
 239         fi
 240
 241         # Specified groups to exclude
 242         for xgroup in $XGROUP_LIST; do
 243                 list=$(get_group_list $xgroup)
 244                 if [ -z "$list" ]; then
 245                         echo "Group \"$xgroup\" is empty or not defined?"
 246                         exit 1
 247                 fi
 248
 249                 trim_test_list $list
 250         done
 251
 252         # sort the list of tests into numeric order
 253         list=`sort -n $tmp.list | uniq`
 254         rm -f $tmp.list
 255
 256         if $randomize
 257         then
 258                 list=`echo $list | awk -f randomize.awk`
 259         fi
 260 }
 261
 262 # Process command arguments first.
 263 while [ $# -gt 0 ]; do
 264         case "$1" in
 265         -\? | -h | --help) usage ;;
 266
 267         -nfs)           FSTYP=nfs ;;
 268         -glusterfs)     FSTYP=glusterfs ;;
 269         -cifs)          FSTYP=cifs ;;
 270         -9p)            FSTYP=9p ;;
 271         -overlay)       FSTYP=overlay; export OVERLAY=true ;;
 272         -pvfs2)         FSTYP=pvfs2 ;;
 273         -tmpfs)         FSTYP=tmpfs ;;
 274         -ubifs)         FSTYP=ubifs ;;
 275
 276         -g)     group=$2 ; shift ;
 277                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
 278                 ;;
 279
 280         -x)     xgroup=$2 ; shift ;
 281                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
 282                 ;;
 283
 284         -X)     xfile=$2; shift ;
 285                 for d in $SRC_GROUPS $FSTYP; do
 286                         [ -f $SRC_DIR/$d/$xfile ] || continue
 287                         for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
 288                                 echo $d/$f >> $tmp.xlist
 289                         done
 290                 done
 291                 ;;
 292         -E)     xfile=$2; shift ;
 293                 if [ -f $xfile ]; then
 294                         sed "s/#.*$//" "$xfile" >> $tmp.xlist
 295                 fi
 296                 ;;
 297         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
 298         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
 299         -l)     diff="diff" ;;
 300         -udiff) diff="$diff -u" ;;
 301
 302         -n)     showme=true ;;
 303         -r)     randomize=true ;;
 304
 305         -T)     timestamp=true ;;
 306         -d)     DUMP_OUTPUT=true ;;
 307         -b)     brief_test_summary=true;;
 308         -R)     report_fmt=$2 ; shift ;
 309                 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
 310                 do_report=true
 311                 ;;
 312         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
 313         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
 314
 315         -*)     usage ;;
 316         *)      # not an argument, we've got tests now.
 317                 have_test_arg=true ;;
 318         esac
 319
 320         # if we've found a test specification, the break out of the processing
 321         # loop before we shift the arguments so that this is the first argument
 322         # that we process in the test arg loop below.
 323         if $have_test_arg; then
 324                 break;
 325         fi
 326
 327         shift
 328 done
 329
 330 # we need common/config, source it after processing args, overlay needs FSTYP
 331 # set before sourcing common/config
 332 if ! . ./common/config; then
 333         echo "$iam: failed to source common/config"
 334         exit 1
 335 fi
 336
 337 # Process tests from command line now.
 338 if $have_test_arg; then
 339         while [ $# -gt 0 ]; do
 340                 case "$1" in
 341                 -*)     echo "Arguments before tests, please!"
 342                         status=1
 343                         exit $status
 344                         ;;
 345                 *)      # Expand test pattern (e.g. xfs/???, *fs/001)
 346                         list=$(cd $SRC_DIR; echo $1)
 347                         for t in $list; do
 348                                 test_dir=`dirname $t`
 349                                 test_dir=${test_dir#$SRC_DIR/*}
 350                                 test_name=`basename $t`
 351                                 group_file=$SRC_DIR/$test_dir/group
 352
 353                                 if egrep -q "^$test_name" $group_file; then
 354                                         # in group file ... OK
 355                                         echo $SRC_DIR/$test_dir/$test_name \
 356                                                 >>$tmp.arglist
 357                                 else
 358                                         # oops
 359                                         echo "$t - unknown test, ignored"
 360                                 fi
 361                         done
 362                         ;;
 363                 esac
 364
 365                 shift
 366         done
 367 fi
 368
 369 # we need common/rc
 370 if ! . ./common/rc
 371 then
 372     echo "check: failed to source common/rc"
 373     exit 1
 374 fi
 375
 376 if [ `id -u` -ne 0 ]
 377 then
 378     echo "check: QA must be run as root"
 379     exit 1
 380 fi
 381
 382 _wipe_counters()
 383 {
 384         n_try="0"
 385         n_bad="0"
 386         n_notrun="0"
 387         unset try notrun bad
 388 }
 389
 390 _wrapup()
 391 {
 392         seq="check"
 393         check="$RESULT_BASE/check"
 394
 395         if $showme; then
 396                 if $needwrap; then
 397                         if $do_report; then
 398                                 _make_section_report
 399                         fi
 400                         needwrap=false
 401                 fi
 402         elif $needwrap; then
 403                 if [ -f $check.time -a -f $tmp.time ]; then
 404                         cat $check.time $tmp.time  \
 405                                 | $AWK_PROG '
 406                                 { t[$1] = $2 }
 407                                 END {
 408                                         if (NR > 0) {
 409                                                 for (i in t) print i " " t[i]
 410                                         }
 411                                 }' \
 412                                 | sort -n >$tmp.out
 413                         mv $tmp.out $check.time
 414                 fi
 415
 416                 echo "" >>$check.log
 417                 date >>$check.log
 418
 419                 echo "SECTION       -- $section" >>$tmp.summary
 420                 echo "=========================" >>$tmp.summary
 421                 if [ ! -z "$n_try" -a $n_try != 0 ]; then
 422                         if [ $brief_test_summary == "false" ]; then
 423                                 echo "Ran:$try"
 424                                 echo "Ran:$try" >>$tmp.summary
 425                         fi
 426                         echo "Ran:$try" >>$check.log
 427                 fi
 428
 429                 $interrupt && echo "Interrupted!" | tee -a $check.log
 430
 431                 if [ ! -z "$notrun" ]; then
 432                         if [ $brief_test_summary == "false" ]; then
 433                                 echo "Not run:$notrun"
 434                                 echo "Not run:$notrun" >>$tmp.summary
 435                         fi
 436                         echo "Not run:$notrun" >>$check.log
 437                 fi
 438
 439                 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
 440                         echo "Failures:$bad"
 441                         echo "Failed $n_bad of $n_try tests"
 442                         echo "Failures:$bad" >>$check.log
 443                         echo "Failed $n_bad of $n_try tests" >>$check.log
 444                         echo "Failures:$bad" >>$tmp.summary
 445                         echo "Failed $n_bad of $n_try tests" >>$tmp.summary
 446                 else
 447                         echo "Passed all $n_try tests"
 448                         echo "Passed all $n_try tests" >>$check.log
 449                         echo "Passed all $n_try tests" >>$tmp.summary
 450                 fi
 451                 echo "" >>$tmp.summary
 452                 if $do_report; then
 453                         _make_section_report
 454                 fi
 455                 needwrap=false
 456         fi
 457
 458         sum_bad=`expr $sum_bad + $n_bad`
 459         _wipe_counters
 460         rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
 461         if ! $OPTIONS_HAVE_SECTIONS; then
 462                 rm -f $tmp.*
 463         fi
 464 }
 465
 466 _summary()
 467 {
 468         _wrapup
 469         if $showme; then
 470                 :
 471         elif $needsum; then
 472                 count=`wc -L $tmp.summary | cut -f1 -d" "`
 473                 cat $tmp.summary
 474                 needsum=false
 475         fi
 476         rm -f $tmp.*
 477 }
 478
 479 _check_filesystems()
 480 {
 481         if [ -f ${RESULT_DIR}/require_test ]; then
 482                 _check_test_fs || err=true
 483                 rm -f ${RESULT_DIR}/require_test*
 484         fi
 485         if [ -f ${RESULT_DIR}/require_scratch ]; then
 486                 _check_scratch_fs || err=true
 487                 rm -f ${RESULT_DIR}/require_scratch*
 488         fi
 489 }
 490
 491 _expunge_test()
 492 {
 493         local TEST_ID="$1"
 494         if [ -s $tmp.xlist ]; then
 495                 if grep -q $TEST_ID $tmp.xlist; then
 496                         echo "       [expunged]"
 497                         return 1
 498                 fi
 499         fi
 500         return 0
 501 }
 502
 503 _init_kmemleak
 504 _prepare_test_list
 505
 506 if $OPTIONS_HAVE_SECTIONS; then
 507         trap "_summary; exit \$status" 0 1 2 3 15
 508 else
 509         trap "_wrapup; exit \$status" 0 1 2 3 15
 510 fi
 511
 512 for section in $HOST_OPTIONS_SECTIONS; do
 513         OLD_FSTYP=$FSTYP
 514         OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
 515         get_next_config $section
 516
 517         # Do we need to run only some sections ?
 518         if [ ! -z "$RUN_SECTION" ]; then
 519                 skip=true
 520                 for s in $RUN_SECTION; do
 521                         if [ $section == $s ]; then
 522                                 skip=false
 523                                 break;
 524                         fi
 525                 done
 526                 if $skip; then
 527                         continue
 528                 fi
 529         fi
 530
 531         # Did this section get excluded?
 532         if [ ! -z "$EXCLUDE_SECTION" ]; then
 533                 skip=false
 534                 for s in $EXCLUDE_SECTION; do
 535                         if [ $section == $s ]; then
 536                                 skip=true
 537                                 break;
 538                         fi
 539                 done
 540                 if $skip; then
 541                         continue
 542                 fi
 543         fi
 544
 545         mkdir -p $RESULT_BASE
 546         if [ ! -d $RESULT_BASE ]; then
 547                 echo "failed to create results directory $RESULT_BASE"
 548                 status=1
 549                 exit
 550         fi
 551
 552         if $OPTIONS_HAVE_SECTIONS; then
 553                 echo "SECTION       -- $section"
 554         fi
 555
 556         sect_start=`_wallclock`
 557         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
 558                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
 559                 _test_unmount 2> /dev/null
 560                 if ! _test_mkfs >$tmp.err 2>&1
 561                 then
 562                         echo "our local _test_mkfs routine ..."
 563                         cat $tmp.err
 564                         echo "check: failed to mkfs \$TEST_DEV using specified options"
 565                         status=1
 566                         exit
 567                 fi
 568                 if ! _test_mount
 569                 then
 570                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 571                         status=1
 572                         exit
 573                 fi
 574                 _prepare_test_list
 575         elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
 576                 _test_unmount 2> /dev/null
 577                 if ! _test_mount
 578                 then
 579                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 580                         status=1
 581                         exit
 582                 fi
 583         fi
 584
 585         init_rc
 586
 587         seq="check"
 588         check="$RESULT_BASE/check"
 589
 590         # don't leave old full output behind on a clean run
 591         rm -f $check.full
 592
 593         [ -f $check.time ] || touch $check.time
 594
 595         # print out our test configuration
 596         echo "FSTYP         -- `_full_fstyp_details`"
 597         echo "PLATFORM      -- `_full_platform_details`"
 598         if [ ! -z "$SCRATCH_DEV" ]; then
 599           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
 600           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
 601         fi
 602         echo
 603         needwrap=true
 604
 605         if [ ! -z "$SCRATCH_DEV" ]; then
 606           _scratch_unmount 2> /dev/null
 607           # call the overridden mkfs - make sure the FS is built
 608           # the same as we'll create it later.
 609
 610           if ! _scratch_mkfs >$tmp.err 2>&1
 611           then
 612               echo "our local _scratch_mkfs routine ..."
 613               cat $tmp.err
 614               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
 615               status=1
 616               exit
 617           fi
 618
 619           # call the overridden mount - make sure the FS mounts with
 620           # the same options that we'll mount with later.
 621           if ! _scratch_mount >$tmp.err 2>&1
 622           then
 623               echo "our local mount routine ..."
 624               cat $tmp.err
 625               echo "check: failed to mount \$SCRATCH_DEV using specified options"
 626               status=1
 627               exit
 628           fi
 629         fi
 630
 631         seqres="$check"
 632         _check_test_fs
 633
 634         for seq in $list
 635         do
 636             err=false
 637             err_msg=""
 638             if [ ! -f $seq ]; then
 639                 # Try to get full name in case the user supplied only seq id
 640                 # and the test has a name. A bit of hassle to find really
 641                 # the test and not its sample output or helping files.
 642                 bname=$(basename $seq)
 643                 full_seq=$(find $(dirname $seq) -name $bname* -executable |
 644                     awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
 645                         END { print shortest }')
 646                 if [ -f $full_seq ] \
 647                     && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
 648                     seq=$full_seq
 649                 fi
 650             fi
 651
 652             # the filename for the test and the name output are different.
 653             # we don't include the tests/ directory in the name output.
 654             export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
 655
 656             # Similarly, the result directory needs to replace the tests/
 657             # part of the test location.
 658             group=`dirname $seq`
 659             if $OPTIONS_HAVE_SECTIONS; then
 660                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
 661                 REPORT_DIR="$RESULT_BASE/$section"
 662             else
 663                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
 664                 REPORT_DIR="$RESULT_BASE"
 665             fi
 666             seqres="$REPORT_DIR/$seqnum"
 667
 668             mkdir -p $RESULT_DIR
 669
 670             echo -n "$seqnum"
 671
 672             if $showme; then
 673                 _expunge_test $seqnum
 674                 if [ $? -eq 1 ]; then
 675                         continue
 676                 fi
 677                 echo
 678                 start=0
 679                 stop=0
 680                 n_notrun=`expr $n_notrun + 1`
 681                 if $do_report; then
 682                         _make_testcase_report "list"
 683                 fi
 684                 continue
 685             fi
 686             tc_status="pass"
 687             if [ ! -f $seq ]; then
 688                 echo " - no such test?"
 689             else
 690                 # really going to try and run this one
 691                 #
 692                 rm -f $seqres.out.bad
 693
 694                 # check if we really should run it
 695                 _expunge_test $seqnum
 696                 if [ $? -eq 1 ]; then
 697                         continue
 698                 fi
 699
 700                 # slashes now in names, sed barfs on them so use grep
 701                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
 702                 if [ "X$lasttime" != X ]; then
 703                         echo -n " ${lasttime}s ..."
 704                 else
 705                         echo -n "       "       # prettier output with timestamps.
 706                 fi
 707                 rm -f core $seqres.notrun
 708
 709                 start=`_wallclock`
 710                 $timestamp && echo -n " ["`date "+%T"`"]"
 711                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
 712                 $LOGGER_PROG "run xfstest $seqnum"
 713                 if [ -w /dev/kmsg ]; then
 714                         export date_time=`date +"%F %T"`
 715                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
 716                         # _check_dmesg depends on this log in dmesg
 717                         touch ${RESULT_DIR}/check_dmesg
 718                 fi
 719                 if [ "$DUMP_OUTPUT" = true ]; then
 720                         ./$seq 2>&1 | tee $tmp.rawout
 721                         # Because $? would get tee's return code
 722                         sts=${PIPESTATUS[0]}
 723                 else
 724                         ./$seq >$tmp.rawout 2>&1
 725                         sts=$?
 726                 fi
 727                 $timestamp && _timestamp
 728                 stop=`_wallclock`
 729
 730                 _fix_malloc <$tmp.rawout >$tmp.out
 731                 rm -f $tmp.rawout
 732
 733                 if [ -f core ]
 734                 then
 735                     err_msg="[dumped core]"
 736                     echo -n " $err_msg"
 737                     mv core $RESULT_BASE/$seqnum.core
 738                     err=true
 739                 fi
 740
 741                 if [ -f $seqres.notrun ]
 742                 then
 743                     $timestamp || echo -n " [not run] "
 744                     $timestamp && echo " [not run]" && echo -n "        $seqnum -- "
 745                     cat $seqres.notrun
 746                     notrun="$notrun $seqnum"
 747                     n_notrun=`expr $n_notrun + 1`
 748                     tc_status="notrun"
 749                 else
 750                     if [ $sts -ne 0 ]
 751                     then
 752                         err_msg="[failed, exit status $sts]"
 753                         echo -n " $err_msg"
 754                         err=true
 755                     fi
 756                     if [ ! -f $seq.out ]
 757                     then
 758                         _dump_err "no qualified output"
 759                         err=true
 760                     else
 761
 762                         # coreutils 8.16+ changed quote formats in error messages from
 763                         # `foo' to 'foo'. Filter old versions to match the new version.
 764                         sed -i "s/\`/\'/g" $tmp.out
 765                         if diff $seq.out $tmp.out >/dev/null 2>&1
 766                         then
 767                             if $err
 768                             then
 769                                 :
 770                             else
 771                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
 772                                 echo -n " `expr $stop - $start`s"
 773                             fi
 774                             echo ""
 775                         else
 776                             echo " - output mismatch (see $seqres.out.bad)"
 777                             mv $tmp.out $seqres.out.bad
 778                             $diff $seq.out $seqres.out.bad | {
 779                                 if test "$DIFF_LENGTH" -le 0; then
 780                                         cat
 781                                 else
 782                                         head -n "$DIFF_LENGTH"
 783                                         echo "..."
 784                                         echo "(Run '$diff $seq.out $seqres.out.bad'" \
 785                                                 " to see the entire diff)"
 786                                 fi; } | \
 787                                 sed -e 's/^\(.\)/    \1/'
 788                             err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)"
 789                             err=true
 790                         fi
 791                     fi
 792                     try="$try $seqnum"
 793                     n_try=`expr $n_try + 1`
 794                     _check_filesystems
 795                     _check_dmesg || err=true
 796                     _check_kmemleak || err=true
 797                 fi
 798
 799             fi
 800
 801             # come here for each test, except when $showme is true
 802             #
 803             if $err
 804             then
 805                 bad="$bad $seqnum"
 806                 n_bad=`expr $n_bad + 1`
 807                 tc_status="fail"
 808             fi
 809             if $do_report; then
 810                 _make_testcase_report "$tc_status"
 811             fi
 812             seq="after_$seqnum"
 813         done
 814         sect_stop=`_wallclock`
 815         interrupt=false
 816         _wrapup
 817         interrupt=true
 818         echo
 819
 820         _test_unmount 2> /dev/null
 821         _scratch_unmount 2> /dev/null
 822 done
 823
 824 interrupt=false
 825 status=`expr $sum_bad != 0`
 826 exit