open_by_handle: add filename to error reports
[xfstests-dev.git] / check
1 #!/bin/bash
2 #
3 # Control script for QA
4 #
5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
6 #
7 # This program is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU General Public License as
9 # published by the Free Software Foundation.
10 #
11 # This program is distributed in the hope that it would be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write the Free Software Foundation,
18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 #
21
22 tmp=/tmp/$$
23 status=0
24 needwrap=true
25 needsum=true
26 n_try=0
27 try=""
28 n_bad=0
29 sum_bad=0
30 bad=""
31 n_notrun=0
32 notrun=""
33 interrupt=true
34 diff="diff -u"
35 showme=false
36 have_test_arg=false
37 randomize=false
38 export here=`pwd`
39 xfile=""
40 brief_test_summary=false
41 err_msg=""
42 do_report=false
43 DUMP_OUTPUT=false
44
45 # start the initialisation work now
46 iam=check
47
48 export MSGVERB="text:action"
49 export QA_CHECK_FS=${QA_CHECK_FS:=true}
50
51 # number of diff lines from a failed test, 0 for whole output
52 export DIFF_LENGTH=${DIFF_LENGTH:=10}
53
54 # by default don't output timestamps
55 timestamp=${TIMESTAMP:=false}
56
57 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
58
59 SRC_GROUPS="generic shared"
60 export SRC_DIR="tests"
61
62 usage()
63 {
64     echo "Usage: $0 [options] [testlist]"'
65
66 check options
67     -nfs                test NFS
68     -glusterfs                test GlusterFS
69     -cifs               test CIFS
70     -overlay            test overlay
71     -pvfs2          test PVFS2
72     -tmpfs              test TMPFS
73     -ubifs              test ubifs
74     -l                  line mode diff
75     -udiff              show unified diff (default)
76     -n                  show me, do not run tests
77     -T                  output timestamps
78     -r                  randomize test order
79     -d                  dump test output to stdout
80     -b                  brief test summary
81     -R fmt[,fmt]        generate report in formats specified. Supported format: [xunit]
82     --large-fs          optimise scratch device for large filesystems
83     -s section          run only specified section from config file
84     -S section          exclude the specified section from the config file
85
86 testlist options
87     -g group[,group...] include tests from these groups
88     -x group[,group...] exclude tests from these groups
89     -X exclude_file     exclude individual tests
90     -E external_file    exclude individual tests
91     [testlist]          include tests matching names in testlist
92
93 testlist argument is a list of tests in the form of <test dir>/<test name>.
94
95 <test dir> is a directory under tests that contains a group file,
96 with a list of the names of the tests in that directory.
97
98 <test name> may be either a specific test file name (e.g. xfs/001) or
99 a test file name match pattern (e.g. xfs/*).
100
101 group argument is either a name of a tests group to collect from all
102 the test dirs (e.g. quick) or a name of a tests group to collect from
103 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
104
105 exclude_file argument refers to a name of a file inside each test directory.
106 for every test dir where this file is found, the listed test names are
107 excluded from the list of tests to run from that test dir.
108
109 external_file argument is a path to a single file containing a list of tests
110 to exclude in the form of <test dir>/<test name>.
111
112 examples:
113  check xfs/001
114  check -g quick
115  check -g xfs/quick
116  check -x stress xfs/*
117  check -X .exclude -g auto
118  check -E ~/.xfstests.exclude
119 '
120             exit 0
121 }
122
123 get_sub_group_list()
124 {
125         local d=$1
126         local grp=$2
127
128         test -s "$SRC_DIR/$d/group" || return 1
129
130         local grpl=$(sed -n < $SRC_DIR/$d/group \
131                 -e 's/#.*//' \
132                 -e 's/$/ /' \
133                 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
134         echo $grpl
135 }
136
137 get_group_list()
138 {
139         local grp=$1
140         local grpl=""
141         local sub=$(dirname $grp)
142
143         if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
144                 # group is given as <subdir>/<group> (e.g. xfs/quick)
145                 grp=$(basename $grp)
146                 get_sub_group_list $sub $grp
147                 return
148         fi
149
150         for d in $SRC_GROUPS $FSTYP; do
151                 if ! test -d "$SRC_DIR/$d" ; then
152                         continue
153                 fi
154                 grpl="$grpl $(get_sub_group_list $d $grp)"
155         done
156         echo $grpl
157 }
158
159 # Find all tests, excluding files that are test metadata such as group files.
160 # It matches test names against $VALID_TEST_NAME defined in common/rc
161 get_all_tests()
162 {
163         touch $tmp.list
164         for d in $SRC_GROUPS $FSTYP; do
165                 if ! test -d "$SRC_DIR/$d" ; then
166                         continue
167                 fi
168                 ls $SRC_DIR/$d/* | \
169                         grep -v "\..*" | \
170                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
171                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
172         done
173 }
174
175 # takes the list of tests to run in $tmp.list, and removes the tests passed to
176 # the function from that list.
177 trim_test_list()
178 {
179         test_list="$*"
180
181         rm -f $tmp.grep
182         numsed=0
183         for t in $test_list
184         do
185             if [ $numsed -gt 100 ]; then
186                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
187                 mv $tmp.tmp $tmp.list
188                 numsed=0
189                 rm -f $tmp.grep
190             fi
191             echo "^$t\$" >>$tmp.grep
192             numsed=`expr $numsed + 1`
193         done
194         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
195         mv $tmp.tmp $tmp.list
196 }
197
198
199 _wallclock()
200 {
201     date "+%s"
202 }
203
204 _timestamp()
205 {
206     now=`date "+%T"`
207     echo -n " [$now]"
208 }
209
210 _prepare_test_list()
211 {
212         unset list
213         # Tests specified on the command line
214         if [ -s $tmp.arglist ]; then
215                 cat $tmp.arglist > $tmp.list
216         else
217                 touch $tmp.list
218         fi
219
220         # Specified groups to include
221         for group in $GROUP_LIST; do
222                 list=$(get_group_list $group)
223                 if [ -z "$list" ]; then
224                         echo "Group \"$group\" is empty or not defined?"
225                         exit 1
226                 fi
227
228                 for t in $list; do
229                         grep -s "^$t\$" $tmp.list >/dev/null || \
230                                                         echo "$t" >>$tmp.list
231                 done
232         done
233
234         if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
235                 # no test numbers, do everything
236                 get_all_tests
237         fi
238
239         # Specified groups to exclude
240         for xgroup in $XGROUP_LIST; do
241                 list=$(get_group_list $xgroup)
242                 if [ -z "$list" ]; then
243                         echo "Group \"$xgroup\" is empty or not defined?"
244                         exit 1
245                 fi
246
247                 trim_test_list $list
248         done
249
250         # sort the list of tests into numeric order
251         list=`sort -n $tmp.list | uniq`
252         rm -f $tmp.list $tmp.tmp $tmp.grep
253
254         if $randomize
255         then
256                 list=`echo $list | awk -f randomize.awk`
257         fi
258 }
259
260 # Process command arguments first.
261 while [ $# -gt 0 ]; do
262         case "$1" in
263         -\? | -h | --help) usage ;;
264
265         -nfs)           FSTYP=nfs ;;
266         -glusterfs)     FSTYP=glusterfs ;;
267         -cifs)          FSTYP=cifs ;;
268         -overlay)       FSTYP=overlay; export OVERLAY=true ;;
269         -pvfs2)         FSTYP=pvfs2 ;;
270         -tmpfs)         FSTYP=tmpfs ;;
271         -ubifs)         FSTYP=ubifs ;;
272
273         -g)     group=$2 ; shift ;
274                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
275                 ;;
276
277         -x)     xgroup=$2 ; shift ;
278                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
279                 ;;
280
281         -X)     xfile=$2; shift ;
282                 for d in $SRC_GROUPS $FSTYP; do
283                         [ -f $SRC_DIR/$d/$xfile ] || continue
284                         for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
285                                 echo $d/$f >> $tmp.xlist
286                         done
287                 done
288                 ;;
289         -E)     xfile=$2; shift ;
290                 if [ -f $xfile ]; then
291                         sed "s/#.*$//" "$xfile" >> $tmp.xlist
292                 fi
293                 ;;
294         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
295         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
296         -l)     diff="diff" ;;
297         -udiff) diff="$diff -u" ;;
298
299         -n)     showme=true ;;
300         -r)     randomize=true ;;
301
302         -T)     timestamp=true ;;
303         -d)     DUMP_OUTPUT=true ;;
304         -b)     brief_test_summary=true;;
305         -R)     report_fmt=$2 ; shift ;
306                 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
307                 do_report=true
308                 ;;
309         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
310         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
311
312         -*)     usage ;;
313         *)      # not an argument, we've got tests now.
314                 have_test_arg=true ;;
315         esac
316
317         # if we've found a test specification, the break out of the processing
318         # loop before we shift the arguments so that this is the first argument
319         # that we process in the test arg loop below.
320         if $have_test_arg; then
321                 break;
322         fi
323
324         shift
325 done
326
327 # we need common/config, source it after processing args, overlay needs FSTYP
328 # set before sourcing common/config
329 if ! . ./common/config; then
330         echo "$iam: failed to source common/config"
331         exit 1
332 fi
333
334 # Process tests from command line now.
335 if $have_test_arg; then
336         while [ $# -gt 0 ]; do
337                 case "$1" in
338                 -*)     echo "Arguments before tests, please!"
339                         status=1
340                         exit $status
341                         ;;
342                 *)      # Expand test pattern (e.g. xfs/???, *fs/001)
343                         list=$(cd $SRC_DIR; echo $1)
344                         for t in $list; do
345                                 test_dir=`dirname $t`
346                                 test_dir=${test_dir#$SRC_DIR/*}
347                                 test_name=`basename $t`
348                                 group_file=$SRC_DIR/$test_dir/group
349
350                                 if egrep -q "^$test_name" $group_file; then
351                                         # in group file ... OK
352                                         echo $SRC_DIR/$test_dir/$test_name \
353                                                 >>$tmp.arglist
354                                 else
355                                         # oops
356                                         echo "$t - unknown test, ignored"
357                                 fi
358                         done
359                         ;;
360                 esac
361
362                 shift
363         done
364 fi
365
366 # we need common/rc
367 if ! . ./common/rc
368 then
369     echo "check: failed to source common/rc"
370     exit 1
371 fi
372
373 if [ `id -u` -ne 0 ]
374 then
375     echo "check: QA must be run as root"
376     exit 1
377 fi
378
379 _wipe_counters()
380 {
381         n_try="0"
382         n_bad="0"
383         n_notrun="0"
384         unset try notrun bad
385 }
386
387 _wrapup()
388 {
389         seq="check"
390         check="$RESULT_BASE/check"
391
392         if $showme; then
393                 if $needwrap; then
394                         if $do_report; then
395                                 _make_section_report
396                         fi
397                         needwrap=false
398                 fi
399         elif $needwrap; then
400                 if [ -f $check.time -a -f $tmp.time ]; then
401                         cat $check.time $tmp.time  \
402                                 | $AWK_PROG '
403                                 { t[$1] = $2 }
404                                 END {
405                                         if (NR > 0) {
406                                                 for (i in t) print i " " t[i]
407                                         }
408                                 }' \
409                                 | sort -n >$tmp.out
410                         mv $tmp.out $check.time
411                 fi
412
413                 echo "" >>$check.log
414                 date >>$check.log
415
416                 echo "SECTION       -- $section" >>$tmp.summary
417                 echo "=========================" >>$tmp.summary
418                 if [ ! -z "$n_try" -a $n_try != 0 ]; then
419                         if [ $brief_test_summary == "false" ]; then
420                                 echo "Ran:$try"
421                                 echo "Ran:$try" >>$tmp.summary
422                         fi
423                         echo "Ran:$try" >>$check.log
424                 fi
425
426                 $interrupt && echo "Interrupted!" | tee -a $check.log
427
428                 if [ ! -z "$notrun" ]; then
429                         if [ $brief_test_summary == "false" ]; then
430                                 echo "Not run:$notrun"
431                                 echo "Not run:$notrun" >>$tmp.summary
432                         fi
433                         echo "Not run:$notrun" >>$check.log
434                 fi
435
436                 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
437                         echo "Failures:$bad"
438                         echo "Failed $n_bad of $n_try tests"
439                         echo "Failures:$bad" >>$check.log
440                         echo "Failed $n_bad of $n_try tests" >>$check.log
441                         echo "Failures:$bad" >>$tmp.summary
442                         echo "Failed $n_bad of $n_try tests" >>$tmp.summary
443                 else
444                         echo "Passed all $n_try tests"
445                         echo "Passed all $n_try tests" >>$check.log
446                         echo "Passed all $n_try tests" >>$tmp.summary
447                 fi
448                 echo "" >>$tmp.summary
449                 if $do_report; then
450                         _make_section_report
451                 fi
452                 needwrap=false
453         fi
454
455         sum_bad=`expr $sum_bad + $n_bad`
456         _wipe_counters
457         rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
458         if ! $OPTIONS_HAVE_SECTIONS; then
459                 rm -f $tmp.*
460         fi
461 }
462
463 _summary()
464 {
465         _wrapup
466         if $showme; then
467                 :
468         elif $needsum; then
469                 count=`wc -L $tmp.summary | cut -f1 -d" "`
470                 cat $tmp.summary
471                 needsum=false
472         fi
473         rm -f $tmp.*
474 }
475
476 _check_filesystems()
477 {
478         if [ -f ${RESULT_DIR}/require_test ]; then
479                 _check_test_fs || err=true
480                 rm -f ${RESULT_DIR}/require_test*
481         fi
482         if [ -f ${RESULT_DIR}/require_scratch ]; then
483                 _check_scratch_fs || err=true
484                 rm -f ${RESULT_DIR}/require_scratch*
485         fi
486 }
487
488 _prepare_test_list
489
490 if $OPTIONS_HAVE_SECTIONS; then
491         trap "_summary; exit \$status" 0 1 2 3 15
492 else
493         trap "_wrapup; exit \$status" 0 1 2 3 15
494 fi
495
496 for section in $HOST_OPTIONS_SECTIONS; do
497         OLD_FSTYP=$FSTYP
498         OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
499         get_next_config $section
500
501         # Do we need to run only some sections ?
502         if [ ! -z "$RUN_SECTION" ]; then
503                 skip=true
504                 for s in $RUN_SECTION; do
505                         if [ $section == $s ]; then
506                                 skip=false
507                                 break;
508                         fi
509                 done
510                 if $skip; then
511                         continue
512                 fi
513         fi
514
515         # Did this section get excluded?
516         if [ ! -z "$EXCLUDE_SECTION" ]; then
517                 skip=false
518                 for s in $EXCLUDE_SECTION; do
519                         if [ $section == $s ]; then
520                                 skip=true
521                                 break;
522                         fi
523                 done
524                 if $skip; then
525                         continue
526                 fi
527         fi
528
529         mkdir -p $RESULT_BASE
530         if [ ! -d $RESULT_BASE ]; then
531                 echo "failed to create results directory $RESULT_BASE"
532                 status=1
533                 exit
534         fi
535
536         if $OPTIONS_HAVE_SECTIONS; then
537                 echo "SECTION       -- $section"
538         fi
539
540         sect_start=`_wallclock`
541         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
542                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
543                 _test_unmount 2> /dev/null
544                 if ! _test_mkfs >$tmp.err 2>&1
545                 then
546                         echo "our local _test_mkfs routine ..."
547                         cat $tmp.err
548                         echo "check: failed to mkfs \$TEST_DEV using specified options"
549                         status=1
550                         exit
551                 fi
552                 if ! _test_mount
553                 then
554                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
555                         status=1
556                         exit
557                 fi
558                 _prepare_test_list
559         elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
560                 _test_unmount 2> /dev/null
561                 if ! _test_mount
562                 then
563                         echo "check: failed to mount $TEST_DEV on $TEST_DIR"
564                         status=1
565                         exit
566                 fi
567         fi
568
569         init_rc
570
571         seq="check"
572         check="$RESULT_BASE/check"
573
574         # don't leave old full output behind on a clean run
575         rm -f $check.full
576
577         [ -f $check.time ] || touch $check.time
578
579         # print out our test configuration
580         echo "FSTYP         -- `_full_fstyp_details`"
581         echo "PLATFORM      -- `_full_platform_details`"
582         if [ ! -z "$SCRATCH_DEV" ]; then
583           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
584           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
585         fi
586         echo
587         needwrap=true
588
589         if [ ! -z "$SCRATCH_DEV" ]; then
590           _scratch_unmount 2> /dev/null
591           # call the overridden mkfs - make sure the FS is built
592           # the same as we'll create it later.
593
594           if ! _scratch_mkfs >$tmp.err 2>&1
595           then
596               echo "our local _scratch_mkfs routine ..."
597               cat $tmp.err
598               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
599               status=1
600               exit
601           fi
602
603           # call the overridden mount - make sure the FS mounts with
604           # the same options that we'll mount with later.
605           if ! _scratch_mount >$tmp.err 2>&1
606           then
607               echo "our local mount routine ..."
608               cat $tmp.err
609               echo "check: failed to mount \$SCRATCH_DEV using specified options"
610               status=1
611               exit
612           fi
613         fi
614
615         seqres="$check"
616         _check_test_fs
617
618         for seq in $list
619         do
620             err=false
621             err_msg=""
622             if [ ! -f $seq ]; then
623                 # Try to get full name in case the user supplied only seq id
624                 # and the test has a name. A bit of hassle to find really
625                 # the test and not its sample output or helping files.
626                 bname=$(basename $seq)
627                 full_seq=$(find $(dirname $seq) -name $bname* -executable |
628                     awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
629                         END { print shortest }')
630                 if [ -f $full_seq ] \
631                     && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
632                     seq=$full_seq
633                 fi
634             fi
635
636             # the filename for the test and the name output are different.
637             # we don't include the tests/ directory in the name output.
638             export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
639
640             # Similarly, the result directory needs to replace the tests/
641             # part of the test location.
642             group=`dirname $seq`
643             if $OPTIONS_HAVE_SECTIONS; then
644                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
645                 REPORT_DIR="$RESULT_BASE/$section"
646             else
647                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
648                 REPORT_DIR="$RESULT_BASE"
649             fi
650             seqres="$REPORT_DIR/$seqnum"
651
652             mkdir -p $RESULT_DIR
653
654             echo -n "$seqnum"
655
656             if $showme; then
657                 echo
658                 start=0
659                 stop=0
660                 n_notrun=`expr $n_notrun + 1`
661                 if $do_report; then
662                         _make_testcase_report "list"
663                 fi
664                 continue
665             fi
666             tc_status="pass"
667             if [ ! -f $seq ]; then
668                 echo " - no such test?"
669             else
670                 # really going to try and run this one
671                 #
672                 rm -f $seqres.out.bad
673
674                 # check if we really should run it
675                 if [ -s $tmp.xlist ]; then
676                         if grep $seqnum $tmp.xlist > /dev/null 2>&1 ; then
677                                 echo "       [expunged]"
678                                 continue
679                         fi
680                 fi
681
682                 # slashes now in names, sed barfs on them so use grep
683                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
684                 if [ "X$lasttime" != X ]; then
685                         echo -n " ${lasttime}s ..."
686                 else
687                         echo -n "       "       # prettier output with timestamps.
688                 fi
689                 rm -f core $seqres.notrun
690
691                 start=`_wallclock`
692                 $timestamp && echo -n " ["`date "+%T"`"]"
693                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
694                 $LOGGER_PROG "run xfstest $seqnum"
695                 if [ -w /dev/kmsg ]; then
696                         export date_time=`date +"%F %T"`
697                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
698                         # _check_dmesg depends on this log in dmesg
699                         touch ${RESULT_DIR}/check_dmesg
700                 fi
701                 if [ "$DUMP_OUTPUT" = true ]; then
702                         ./$seq 2>&1 | tee $tmp.rawout
703                         # Because $? would get tee's return code
704                         sts=${PIPESTATUS[0]}
705                 else
706                         ./$seq >$tmp.rawout 2>&1
707                         sts=$?
708                 fi
709                 $timestamp && _timestamp
710                 stop=`_wallclock`
711
712                 _fix_malloc <$tmp.rawout >$tmp.out
713                 rm -f $tmp.rawout
714
715                 if [ -f core ]
716                 then
717                     err_msg="[dumped core]"
718                     echo -n " $err_msg"
719                     mv core $RESULT_BASE/$seqnum.core
720                     err=true
721                 fi
722
723                 if [ -f $seqres.notrun ]
724                 then
725                     $timestamp || echo -n " [not run] "
726                     $timestamp && echo " [not run]" && echo -n "        $seqnum -- "
727                     cat $seqres.notrun
728                     notrun="$notrun $seqnum"
729                     n_notrun=`expr $n_notrun + 1`
730                     tc_status="notrun"
731                 else
732                     if [ $sts -ne 0 ]
733                     then
734                         err_msg="[failed, exit status $sts]"
735                         echo -n " $err_msg"
736                         err=true
737                     fi
738                     if [ ! -f $seq.out ]
739                     then
740                         _dump_err "no qualified output"
741                         err=true
742                     else
743
744                         # coreutils 8.16+ changed quote formats in error messages from
745                         # `foo' to 'foo'. Filter old versions to match the new version.
746                         sed -i "s/\`/\'/g" $tmp.out
747                         if diff $seq.out $tmp.out >/dev/null 2>&1
748                         then
749                             if $err
750                             then
751                                 :
752                             else
753                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
754                                 echo -n " `expr $stop - $start`s"
755                             fi
756                             echo ""
757                         else
758                             echo " - output mismatch (see $seqres.out.bad)"
759                             mv $tmp.out $seqres.out.bad
760                             $diff $seq.out $seqres.out.bad | {
761                                 if test "$DIFF_LENGTH" -le 0; then
762                                         cat
763                                 else
764                                         head -n "$DIFF_LENGTH"
765                                         echo "..."
766                                         echo "(Run '$diff $seq.out $seqres.out.bad'" \
767                                                 " to see the entire diff)"
768                                 fi; } | \
769                                 sed -e 's/^\(.\)/    \1/'
770                             err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)"
771                             err=true
772                         fi
773                     fi
774                     try="$try $seqnum"
775                     n_try=`expr $n_try + 1`
776                     _check_filesystems
777                     _check_dmesg || err=true
778                 fi
779
780             fi
781
782             # come here for each test, except when $showme is true
783             #
784             if $err
785             then
786                 bad="$bad $seqnum"
787                 n_bad=`expr $n_bad + 1`
788                 quick=false
789                 tc_status="fail"
790             fi
791             if $do_report; then
792                 _make_testcase_report "$tc_status"
793             fi
794             seq="after_$seqnum"
795         done
796         sect_stop=`_wallclock`
797         interrupt=false
798         _wrapup
799         interrupt=true
800         echo
801
802         _test_unmount 2> /dev/null
803         _scratch_unmount 2> /dev/null
804 done
805
806 interrupt=false
807 status=`expr $sum_bad`
808 exit