check

   1 #!/bin/bash
   2 #
   3 # Control script for QA
   4 #
   5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
   6 #
   7 # This program is free software; you can redistribute it and/or
   8 # modify it under the terms of the GNU General Public License as
   9 # published by the Free Software Foundation.
  10 #
  11 # This program is distributed in the hope that it would be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write the Free Software Foundation,
  18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  19 #
  20 #
  21
  22 tmp=/tmp/$$
  23 status=0
  24 needwrap=true
  25 needsum=true
  26 n_try=0
  27 try=""
  28 n_bad=0
  29 sum_bad=0
  30 bad=""
  31 n_notrun=0
  32 notrun=""
  33 interrupt=true
  34 diff="diff -u"
  35 showme=false
  36 have_test_arg=false
  37 randomize=false
  38 export here=`pwd`
  39 xfile=""
  40 brief_test_summary=false
  41 err_msg=""
  42
  43 DUMP_OUTPUT=false
  44
  45 # start the initialisation work now
  46 iam=check
  47
  48 export MSGVERB="text:action"
  49 export QA_CHECK_FS=${QA_CHECK_FS:=true}
  50
  51 # number of diff lines from a failed test, 0 for whole output
  52 export DIFF_LENGTH=${DIFF_LENGTH:=10}
  53
  54 # by default don't output timestamps
  55 timestamp=${TIMESTAMP:=false}
  56
  57 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist
  58
  59 SRC_GROUPS="generic shared"
  60 export SRC_DIR="tests"
  61
  62 usage()
  63 {
  64     echo "Usage: $0 [options] [testlist]"'
  65
  66 check options
  67     -nfs                test NFS
  68     -cifs               test CIFS
  69     -overlay            test overlay
  70     -tmpfs              test TMPFS
  71     -l                  line mode diff
  72     -udiff              show unified diff (default)
  73     -n                  show me, do not run tests
  74     -T                  output timestamps
  75     -r                  randomize test order
  76     -d                  dump test output to stdout
  77     -b                  brief test summary
  78     --large-fs          optimise scratch device for large filesystems
  79     -s section          run only specified section from config file
  80     -S section          exclude the specified section from the config file
  81
  82 testlist options
  83     -g group[,group...] include tests from these groups
  84     -x group[,group...] exclude tests from these groups
  85     -X exclude_file     exclude individual tests
  86     -E external_file    exclude individual tests
  87     [testlist]          include tests matching names in testlist
  88
  89 testlist argument is a list of tests in the form of <test dir>/<test name>.
  90
  91 <test dir> is a directory under tests that contains a group file,
  92 with a list of the names of the tests in that directory.
  93
  94 <test name> may be either a specific test file name (e.g. xfs/001) or
  95 a test file name match pattern (e.g. xfs/*).
  96
  97 group argument is either a name of a tests group to collect from all
  98 the test dirs (e.g. quick) or a name of a tests group to collect from
  99 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
 100
 101 exclude_file argument refers to a name of a file inside each test directory.
 102 for every test dir where this file is found, the listed test names are
 103 excluded from the list of tests to run from that test dir.
 104
 105 external_file argument is a path to a single file containing a list of tests
 106 to exclude in the form of <test dir>/<test name>.
 107
 108 examples:
 109  check xfs/001
 110  check -g quick
 111  check -g xfs/quick
 112  check -x stress xfs/*
 113  check -X .exclude -g auto
 114  check -E ~/.xfstests.exclude
 115 '
 116             exit 0
 117 }
 118
 119 get_sub_group_list()
 120 {
 121         local d=$1
 122         local grp=$2
 123
 124         test -s "$SRC_DIR/$d/group" || return 1
 125
 126         local grpl=$(sed -n < $SRC_DIR/$d/group \
 127                 -e 's/#.*//' \
 128                 -e 's/$/ /' \
 129                 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
 130         echo $grpl
 131 }
 132
 133 get_group_list()
 134 {
 135         local grp=$1
 136         local grpl=""
 137         local sub=$(dirname $grp)
 138
 139         if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
 140                 # group is given as <subdir>/<group> (e.g. xfs/quick)
 141                 grp=$(basename $grp)
 142                 get_sub_group_list $sub $grp
 143                 return
 144         fi
 145
 146         for d in $SRC_GROUPS $FSTYP; do
 147                 if ! test -d "$SRC_DIR/$d" ; then
 148                         continue
 149                 fi
 150                 grpl="$grpl $(get_sub_group_list $d $grp)"
 151         done
 152         echo $grpl
 153 }
 154
 155 # Find all tests, excluding files that are test metadata such as group files.
 156 # It matches test names against $VALID_TEST_NAME defined in common/rc
 157 get_all_tests()
 158 {
 159         touch $tmp.list
 160         for d in $SRC_GROUPS $FSTYP; do
 161                 if ! test -d "$SRC_DIR/$d" ; then
 162                         continue
 163                 fi
 164                 ls $SRC_DIR/$d/* | \
 165                         grep -v "\..*" | \
 166                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
 167                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
 168         done
 169 }
 170
 171 # takes the list of tests to run in $tmp.list, and removes the tests passed to
 172 # the function from that list.
 173 trim_test_list()
 174 {
 175         test_list="$*"
 176
 177         rm -f $tmp.grep
 178         numsed=0
 179         for t in $test_list
 180         do
 181             if [ $numsed -gt 100 ]; then
 182                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 183                 mv $tmp.tmp $tmp.list
 184                 numsed=0
 185                 rm -f $tmp.grep
 186             fi
 187             echo "^$t\$" >>$tmp.grep
 188             numsed=`expr $numsed + 1`
 189         done
 190         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
 191         mv $tmp.tmp $tmp.list
 192 }
 193
 194
 195 _wallclock()
 196 {
 197     date "+%s"
 198 }
 199
 200 _timestamp()
 201 {
 202     now=`date "+%T"`
 203     echo -n " [$now]"
 204 }
 205
 206 _prepare_test_list()
 207 {
 208         unset list
 209         # Tests specified on the command line
 210         if [ -s $tmp.arglist ]; then
 211                 cat $tmp.arglist > $tmp.list
 212         else
 213                 touch $tmp.list
 214         fi
 215
 216         # Specified groups to include
 217         for group in $GROUP_LIST; do
 218                 list=$(get_group_list $group)
 219                 if [ -z "$list" ]; then
 220                         echo "Group \"$group\" is empty or not defined?"
 221                         exit 1
 222                 fi
 223
 224                 for t in $list; do
 225                         grep -s "^$t\$" $tmp.list >/dev/null || \
 226                                                         echo "$t" >>$tmp.list
 227                 done
 228         done
 229
 230         if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
 231                 # no test numbers, do everything
 232                 get_all_tests
 233         fi
 234
 235         # Specified groups to exclude
 236         for xgroup in $XGROUP_LIST; do
 237                 list=$(get_group_list $xgroup)
 238                 if [ -z "$list" ]; then
 239                         echo "Group \"$xgroup\" is empty or not defined?"
 240                         exit 1
 241                 fi
 242
 243                 trim_test_list $list
 244         done
 245
 246         # sort the list of tests into numeric order
 247         list=`sort -n $tmp.list | uniq`
 248         rm -f $tmp.list $tmp.tmp $tmp.grep
 249
 250         if $randomize
 251         then
 252                 list=`echo $list | awk -f randomize.awk`
 253         fi
 254 }
 255
 256 # Process command arguments first.
 257 while [ $# -gt 0 ]; do
 258         case "$1" in
 259         -\? | -h | --help) usage ;;
 260
 261         -nfs)           FSTYP=nfs ;;
 262         -cifs)          FSTYP=cifs ;;
 263         -overlay)       FSTYP=overlay; export OVERLAY=true ;;
 264         -tmpfs)         FSTYP=tmpfs ;;
 265
 266         -g)     group=$2 ; shift ;
 267                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
 268                 ;;
 269
 270         -x)     xgroup=$2 ; shift ;
 271                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
 272                 ;;
 273
 274         -X)     xfile=$2; shift ;
 275                 for d in $SRC_GROUPS $FSTYP; do
 276                         [ -f $SRC_DIR/$d/$xfile ] || continue
 277                         for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
 278                                 echo $d/$f >> $tmp.xlist
 279                         done
 280                 done
 281                 ;;
 282         -E)     xfile=$2; shift ;
 283                 if [ -f $xfile ]; then
 284                         sed "s/#.*$//" "$xfile" >> $tmp.xlist
 285                 fi
 286                 ;;
 287         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
 288         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
 289         -l)     diff="diff" ;;
 290         -udiff) diff="$diff -u" ;;
 291
 292         -n)     showme=true ;;
 293         -r)     randomize=true ;;
 294
 295         -T)     timestamp=true ;;
 296         -d)     DUMP_OUTPUT=true ;;
 297         -b)     brief_test_summary=true;;
 298
 299         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
 300         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
 301
 302         -*)     usage ;;
 303         *)      # not an argument, we've got tests now.
 304                 have_test_arg=true ;;
 305         esac
 306
 307         # if we've found a test specification, the break out of the processing
 308         # loop before we shift the arguments so that this is the first argument
 309         # that we process in the test arg loop below.
 310         if $have_test_arg; then
 311                 break;
 312         fi
 313
 314         shift
 315 done
 316
 317 # we need common/config, source it after processing args, overlay needs FSTYP
 318 # set before sourcing common/config
 319 if ! . ./common/config; then
 320         echo "$iam: failed to source common/config"
 321         exit 1
 322 fi
 323
 324 # Process tests from command line now.
 325 if $have_test_arg; then
 326         while [ $# -gt 0 ]; do
 327                 case "$1" in
 328                 -*)     echo "Arguments before tests, please!"
 329                         status=1
 330                         exit $status
 331                         ;;
 332                 *)      # Expand test pattern (e.g. xfs/???, *fs/001)
 333                         list=$(cd $SRC_DIR; echo $1)
 334                         for t in $list; do
 335                                 test_dir=`dirname $t`
 336                                 test_dir=${test_dir#$SRC_DIR/*}
 337                                 test_name=`basename $t`
 338                                 group_file=$SRC_DIR/$test_dir/group
 339
 340                                 if egrep -q "^$test_name" $group_file; then
 341                                         # in group file ... OK
 342                                         echo $SRC_DIR/$test_dir/$test_name \
 343                                                 >>$tmp.arglist
 344                                 else
 345                                         # oops
 346                                         echo "$t - unknown test, ignored"
 347                                 fi
 348                         done
 349                         ;;
 350                 esac
 351
 352                 shift
 353         done
 354 fi
 355
 356 # we need common/rc
 357 if ! . ./common/rc
 358 then
 359     echo "check: failed to source common/rc"
 360     exit 1
 361 fi
 362
 363 if [ `id -u` -ne 0 ]
 364 then
 365     echo "check: QA must be run as root"
 366     exit 1
 367 fi
 368
 369 _wipe_counters()
 370 {
 371         n_try="0"
 372         n_bad="0"
 373         n_notrun="0"
 374         unset try notrun bad
 375 }
 376
 377 _wrapup()
 378 {
 379         seq="check"
 380         check="$RESULT_BASE/check"
 381
 382         if $showme; then
 383         :
 384         elif $needwrap; then
 385                 if [ -f $check.time -a -f $tmp.time ]; then
 386                         cat $check.time $tmp.time  \
 387                                 | $AWK_PROG '
 388                                 { t[$1] = $2 }
 389                                 END {
 390                                         if (NR > 0) {
 391                                                 for (i in t) print i " " t[i]
 392                                         }
 393                                 }' \
 394                                 | sort -n >$tmp.out
 395                         mv $tmp.out $check.time
 396                 fi
 397
 398                 echo "" >>$check.log
 399                 date >>$check.log
 400
 401                 echo "SECTION       -- $section" >>$tmp.summary
 402                 echo "=========================" >>$tmp.summary
 403                 if [ ! -z "$n_try" -a $n_try != 0 ]; then
 404                         if [ $brief_test_summary == "false" ]; then
 405                                 echo "Ran:$try"
 406                                 echo "Ran:$try" >>$tmp.summary
 407                         fi
 408                         echo "Ran:$try" >>$check.log
 409                 fi
 410
 411                 $interrupt && echo "Interrupted!" | tee -a $check.log
 412
 413                 if [ ! -z "$notrun" ]; then
 414                         if [ $brief_test_summary == "false" ]; then
 415                                 echo "Not run:$notrun"
 416                                 echo "Not run:$notrun" >>$tmp.summary
 417                         fi
 418                         echo "Not run:$notrun" >>$check.log
 419                 fi
 420
 421                 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
 422                         echo "Failures:$bad"
 423                         echo "Failed $n_bad of $n_try tests"
 424                         echo "Failures:$bad" >>$check.log
 425                         echo "Failed $n_bad of $n_try tests" >>$check.log
 426                         echo "Failures:$bad" >>$tmp.summary
 427                         echo "Failed $n_bad of $n_try tests" >>$tmp.summary
 428                 else
 429                         echo "Passed all $n_try tests"
 430                         echo "Passed all $n_try tests" >>$check.log
 431                         echo "Passed all $n_try tests" >>$tmp.summary
 432                 fi
 433                 echo "" >>$tmp.summary
 434                 needwrap=false
 435         fi
 436
 437         sum_bad=`expr $sum_bad + $n_bad`
 438         _wipe_counters
 439         rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
 440         if ! $OPTIONS_HAVE_SECTIONS; then
 441                 rm -f $tmp.*
 442         fi
 443 }
 444
 445 _summary()
 446 {
 447         _wrapup
 448         if $showme; then
 449                 :
 450         elif $needsum; then
 451                 count=`wc -L $tmp.summary | cut -f1 -d" "`
 452                 cat $tmp.summary
 453                 needsum=false
 454         fi
 455         rm -f $tmp.*
 456 }
 457
 458 _check_filesystems()
 459 {
 460         if [ -f ${RESULT_DIR}/require_test ]; then
 461                 _check_test_fs || err=true
 462                 rm -f ${RESULT_DIR}/require_test*
 463         fi
 464         if [ -f ${RESULT_DIR}/require_scratch ]; then
 465                 _check_scratch_fs || err=true
 466                 rm -f ${RESULT_DIR}/require_scratch*
 467         fi
 468 }
 469
 470 _prepare_test_list
 471
 472 if $OPTIONS_HAVE_SECTIONS; then
 473         trap "_summary; exit \$status" 0 1 2 3 15
 474 else
 475         trap "_wrapup; exit \$status" 0 1 2 3 15
 476 fi
 477
 478 for section in $HOST_OPTIONS_SECTIONS; do
 479         OLD_FSTYP=$FSTYP
 480         OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
 481         get_next_config $section
 482
 483         # Do we need to run only some sections ?
 484         if [ ! -z "$RUN_SECTION" ]; then
 485                 skip=true
 486                 for s in $RUN_SECTION; do
 487                         if [ $section == $s ]; then
 488                                 skip=false
 489                                 break;
 490                         fi
 491                 done
 492                 if $skip; then
 493                         continue
 494                 fi
 495         fi
 496
 497         # Did this section get excluded?
 498         if [ ! -z "$EXCLUDE_SECTION" ]; then
 499                 skip=false
 500                 for s in $EXCLUDE_SECTION; do
 501                         if [ $section == $s ]; then
 502                                 skip=true
 503                                 break;
 504                         fi
 505                 done
 506                 if $skip; then
 507                         continue
 508                 fi
 509         fi
 510
 511         mkdir -p $RESULT_BASE
 512         if [ ! -d $RESULT_BASE ]; then
 513                 echo "failed to create results directory $RESULT_BASE"
 514                 status=1
 515                 exit
 516         fi
 517
 518         if $OPTIONS_HAVE_SECTIONS; then
 519                 echo "SECTION       -- $section"
 520         fi
 521
 522         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
 523                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
 524                 _test_unmount 2> /dev/null
 525                 if ! _test_mkfs >$tmp.err 2>&1
 526                 then
 527                         echo "our local _test_mkfs routine ..."
 528                         cat $tmp.err
 529                         echo "check: failed to mkfs \$TEST_DEV using specified options"
 530                         status=1
 531                         exit
 532                 fi
 533                 if ! _test_mount
 534                 then
 535                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 536                         status=1
 537                         exit
 538                 fi
 539                 _prepare_test_list
 540         elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
 541                 _test_unmount 2> /dev/null
 542                 if ! _test_mount
 543                 then
 544                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
 545                         status=1
 546                         exit
 547                 fi
 548         fi
 549
 550         init_rc
 551
 552         seq="check"
 553         check="$RESULT_BASE/check"
 554
 555         # don't leave old full output behind on a clean run
 556         rm -f $check.full
 557
 558         [ -f $check.time ] || touch $check.time
 559
 560         # print out our test configuration
 561         echo "FSTYP         -- `_full_fstyp_details`"
 562         echo "PLATFORM      -- `_full_platform_details`"
 563         if [ ! -z "$SCRATCH_DEV" ]; then
 564           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
 565           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
 566         fi
 567         echo
 568         needwrap=true
 569
 570         if [ ! -z "$SCRATCH_DEV" ]; then
 571           _scratch_unmount 2> /dev/null
 572           # call the overridden mkfs - make sure the FS is built
 573           # the same as we'll create it later.
 574
 575           if ! _scratch_mkfs >$tmp.err 2>&1
 576           then
 577               echo "our local _scratch_mkfs routine ..."
 578               cat $tmp.err
 579               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
 580               status=1
 581               exit
 582           fi
 583
 584           # call the overridden mount - make sure the FS mounts with
 585           # the same options that we'll mount with later.
 586           if ! _scratch_mount >$tmp.err 2>&1
 587           then
 588               echo "our local mount routine ..."
 589               cat $tmp.err
 590               echo "check: failed to mount \$SCRATCH_DEV using specified options"
 591               status=1
 592               exit
 593           fi
 594         fi
 595
 596         seqres="$check"
 597         _check_test_fs
 598
 599         for seq in $list
 600         do
 601             err=false
 602             err_msg=""
 603             if [ ! -f $seq ]; then
 604                 # Try to get full name in case the user supplied only seq id
 605                 # and the test has a name. A bit of hassle to find really
 606                 # the test and not its sample output or helping files.
 607                 bname=$(basename $seq)
 608                 full_seq=$(find $(dirname $seq) -name $bname* -executable |
 609                     awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
 610                         END { print shortest }')
 611                 if [ -f $full_seq ] \
 612                     && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
 613                     seq=$full_seq
 614                 fi
 615             fi
 616
 617             # the filename for the test and the name output are different.
 618             # we don't include the tests/ directory in the name output.
 619             export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
 620
 621             # Similarly, the result directory needs to replace the tests/
 622             # part of the test location.
 623             group=`dirname $seq`
 624             if $OPTIONS_HAVE_SECTIONS; then
 625                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
 626                 seqres="$RESULT_BASE/$section/$seqnum"
 627             else
 628                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
 629                 seqres="$RESULT_BASE/$seqnum"
 630             fi
 631
 632             mkdir -p $RESULT_DIR
 633
 634             echo -n "$seqnum"
 635
 636             if $showme; then
 637                 echo
 638                 start=0
 639                 stop=0
 640                 n_notrun=`expr $n_notrun + 1`
 641                 continue
 642             fi
 643
 644             if [ ! -f $seq ]; then
 645                 echo " - no such test?"
 646             else
 647                 # really going to try and run this one
 648                 #
 649                 rm -f $seqres.out.bad
 650
 651                 # check if we really should run it
 652                 if [ -s $tmp.xlist ]; then
 653                         if grep $seqnum $tmp.xlist > /dev/null 2>&1 ; then
 654                                 echo "       [expunged]"
 655                                 continue
 656                         fi
 657                 fi
 658
 659                 # slashes now in names, sed barfs on them so use grep
 660                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
 661                 if [ "X$lasttime" != X ]; then
 662                         echo -n " ${lasttime}s ..."
 663                 else
 664                         echo -n "       "       # prettier output with timestamps.
 665                 fi
 666                 rm -f core $seqres.notrun
 667
 668                 start=`_wallclock`
 669                 $timestamp && echo -n " ["`date "+%T"`"]"
 670                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
 671                 $LOGGER_PROG "run xfstest $seqnum"
 672                 if [ -w /dev/kmsg ]; then
 673                         export date_time=`date +"%F %T"`
 674                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
 675                         # _check_dmesg depends on this log in dmesg
 676                         touch ${RESULT_DIR}/check_dmesg
 677                 fi
 678                 if [ "$DUMP_OUTPUT" = true ]; then
 679                         ./$seq 2>&1 | tee $tmp.rawout
 680                         # Because $? would get tee's return code
 681                         sts=${PIPESTATUS[0]}
 682                 else
 683                         ./$seq >$tmp.rawout 2>&1
 684                         sts=$?
 685                 fi
 686                 $timestamp && _timestamp
 687                 stop=`_wallclock`
 688
 689                 _fix_malloc <$tmp.rawout >$tmp.out
 690                 rm -f $tmp.rawout
 691
 692                 if [ -f core ]
 693                 then
 694                     err_msg="[dumped core]"
 695                     echo -n " $err_msg"
 696                     mv core $RESULT_BASE/$seqnum.core
 697                     err=true
 698                 fi
 699
 700                 if [ -f $seqres.notrun ]
 701                 then
 702                     $timestamp || echo -n " [not run] "
 703                     $timestamp && echo " [not run]" && echo -n "        $seqnum -- "
 704                     cat $seqres.notrun
 705                     notrun="$notrun $seqnum"
 706                     n_notrun=`expr $n_notrun + 1`
 707                 else
 708                     if [ $sts -ne 0 ]
 709                     then
 710                         err_msg="[failed, exit status $sts]"
 711                         echo -n " $err_msg"
 712                         err=true
 713                     fi
 714                     if [ ! -f $seq.out ]
 715                     then
 716                         _dump_err "no qualified output"
 717                         err=true
 718                     else
 719
 720                         # coreutils 8.16+ changed quote formats in error messages from
 721                         # `foo' to 'foo'. Filter old versions to match the new version.
 722                         sed -i "s/\`/\'/g" $tmp.out
 723                         if diff $seq.out $tmp.out >/dev/null 2>&1
 724                         then
 725                             if $err
 726                             then
 727                                 :
 728                             else
 729                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
 730                                 echo -n " `expr $stop - $start`s"
 731                             fi
 732                             echo ""
 733                         else
 734                             echo " - output mismatch (see $seqres.out.bad)"
 735                             mv $tmp.out $seqres.out.bad
 736                             $diff $seq.out $seqres.out.bad | {
 737                                 if test "$DIFF_LENGTH" -le 0; then
 738                                         cat
 739                                 else
 740                                         head -n "$DIFF_LENGTH"
 741                                         echo "..."
 742                                         echo "(Run '$diff $seq.out $seqres.out.bad'" \
 743                                                 " to see the entire diff)"
 744                                 fi; } | \
 745                                 sed -e 's/^\(.\)/    \1/'
 746                             err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)"
 747                             err=true
 748                         fi
 749                     fi
 750                     try="$try $seqnum"
 751                     n_try=`expr $n_try + 1`
 752                     _check_filesystems
 753                     _check_dmesg || err=true
 754                 fi
 755
 756             fi
 757
 758             # come here for each test, except when $showme is true
 759             #
 760             if $err
 761             then
 762                 bad="$bad $seqnum"
 763                 n_bad=`expr $n_bad + 1`
 764                 quick=false
 765             fi
 766
 767             seq="after_$seqnum"
 768         done
 769         interrupt=false
 770         _wrapup
 771         interrupt=true
 772         echo
 773
 774         _test_unmount 2> /dev/null
 775         _scratch_unmount 2> /dev/null
 776 done
 777
 778 interrupt=false
 779 status=`expr $sum_bad`
 780 exit