check: avoid spurious complaints that tests/$FSTYP/group does not exist
[xfstests-dev.git] / check
1 #!/bin/bash
2 #
3 # Control script for QA
4 #
5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc.  All Rights Reserved.
6 #
7 # This program is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU General Public License as
9 # published by the Free Software Foundation.
10 #
11 # This program is distributed in the hope that it would be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 # GNU General Public License for more details.
15 #
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write the Free Software Foundation,
18 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19 #
20 #
21
22 tmp=/tmp/$$
23 status=0
24 needwrap=true
25 needsum=true
26 n_try=0
27 try=""
28 n_bad=0
29 sum_bad=0
30 bad=""
31 notrun=""
32 interrupt=true
33 diff="diff -u"
34 showme=false
35 have_test_arg=false
36 randomize=false
37 export here=`pwd`
38 xfile=""
39
40 DUMP_OUTPUT=false
41
42 # start the initialisation work now
43 iam=check
44
45 export MSGVERB="text:action"
46 export QA_CHECK_FS=${QA_CHECK_FS:=true}
47
48 # number of diff lines from a failed test, 0 for whole output
49 export DIFF_LENGTH=${DIFF_LENGTH:=10}
50
51 # by default don't output timestamps
52 timestamp=${TIMESTAMP:=false}
53
54 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist
55
56 SRC_GROUPS="generic shared"
57 export SRC_DIR="tests"
58
59 usage()
60 {
61     echo "Usage: $0 [options] [testlist]"'
62
63 check options
64     -nfs                test NFS
65     -cifs               test CIFS
66     -overlay            test overlay
67     -tmpfs              test TMPFS
68     -l                  line mode diff
69     -udiff              show unified diff (default)
70     -n                  show me, do not run tests
71     -T                  output timestamps
72     -r                  randomize test order
73     -d                  dump test output to stdout
74     --large-fs          optimise scratch device for large filesystems
75     -s section          run only specified section from config file
76     -S section          exclude the specified section from the config file
77
78 testlist options
79     -g group[,group...] include tests from these groups
80     -x group[,group...] exclude tests from these groups
81     -X file             exclude individual tests
82     -E external_file    exclude individual tests
83     [testlist]          include tests matching names in testlist
84 '
85             exit 0
86 }
87
88 get_group_list()
89 {
90         grp=$1
91
92         for d in $SRC_GROUPS $FSTYP; do
93                 if ! test -d "$SRC_DIR/$d" ; then
94                         continue
95                 fi
96                 l=$(sed -n < $SRC_DIR/$d/group \
97                         -e 's/#.*//' \
98                         -e 's/$/ /' \
99                         -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
100                 grpl="$grpl $l"
101         done
102         echo $grpl
103 }
104
105 # Find all tests, excluding files that are test metadata such as group files.
106 # It matches test names against $VALID_TEST_NAME defined in common/rc
107 get_all_tests()
108 {
109         touch $tmp.list
110         for d in $SRC_GROUPS $FSTYP; do
111                 if ! test -d "$SRC_DIR/$d" ; then
112                         continue
113                 fi
114                 ls $SRC_DIR/$d/* | \
115                         grep -v "\..*" | \
116                         grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
117                         grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
118         done
119 }
120
121 # takes the list of tests to run in $tmp.list, and removes the tests passed to
122 # the function from that list.
123 trim_test_list()
124 {
125         test_list="$*"
126
127         rm -f $tmp.grep
128         numsed=0
129         for t in $test_list
130         do
131             if [ $numsed -gt 100 ]; then
132                 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
133                 mv $tmp.tmp $tmp.list
134                 numsed=0
135                 rm -f $tmp.grep
136             fi
137             echo "^$t\$" >>$tmp.grep
138             numsed=`expr $numsed + 1`
139         done
140         grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
141         mv $tmp.tmp $tmp.list
142 }
143
144
145 _wallclock()
146 {
147     date "+%s"
148 }
149
150 _timestamp()
151 {
152     now=`date "+%T"`
153     echo -n " [$now]"
154 }
155
156 _prepare_test_list()
157 {
158         unset list
159         # Tests specified on the command line
160         if [ -s $tmp.arglist ]; then
161                 cat $tmp.arglist > $tmp.list
162         else
163                 touch $tmp.list
164         fi
165
166         # Specified groups to include
167         for group in $GROUP_LIST; do
168                 list=$(get_group_list $group)
169                 if [ -z "$list" ]; then
170                         echo "Group \"$group\" is empty or not defined?"
171                         exit 1
172                 fi
173
174                 for t in $list; do
175                         grep -s "^$t\$" $tmp.list >/dev/null || \
176                                                         echo "$t" >>$tmp.list
177                 done
178         done
179
180         if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
181                 # no test numbers, do everything
182                 get_all_tests
183         fi
184
185         # Specified groups to exclude
186         for xgroup in $XGROUP_LIST; do
187                 list=$(get_group_list $xgroup)
188                 if [ -z "$list" ]; then
189                         echo "Group \"$xgroup\" is empty or not defined?"
190                         exit 1
191                 fi
192
193                 trim_test_list $list
194         done
195
196         # sort the list of tests into numeric order
197         list=`sort -n $tmp.list | uniq`
198         rm -f $tmp.list $tmp.tmp $tmp.grep
199
200         if $randomize
201         then
202                 list=`echo $list | awk -f randomize.awk`
203         fi
204 }
205
206 # Process command arguments first.
207 while [ $# -gt 0 ]; do
208         case "$1" in
209         -\? | -h | --help) usage ;;
210
211         -nfs)           FSTYP=nfs ;;
212         -cifs)          FSTYP=cifs ;;
213         -overlay)       FSTYP=overlay ;;
214         -tmpfs)         FSTYP=tmpfs ;;
215
216         -g)     group=$2 ; shift ;
217                 GROUP_LIST="$GROUP_LIST ${group//,/ }"
218                 ;;
219
220         -x)     xgroup=$2 ; shift ;
221                 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
222                 ;;
223
224         -X)     xfile=$2; shift ;
225                 for d in $SRC_GROUPS $FSTYP; do
226                         [ -f $SRC_DIR/$d/$xfile ] || continue
227                         for f in `cat $SRC_DIR/$d/$xfile`; do
228                                 echo $d/$f >> $tmp.xlist
229                         done
230                 done
231                 ;;
232         -E)     xfile=$2; shift ;
233                 if [ -f $xfile ]; then
234                         cat "$xfile" >> $tmp.xlist
235                 fi
236                 ;;
237         -s)     RUN_SECTION="$RUN_SECTION $2"; shift ;;
238         -S)     EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
239         -l)     diff="diff" ;;
240         -udiff) diff="$diff -u" ;;
241
242         -n)     showme=true ;;
243         -r)     randomize=true ;;
244
245         -T)     timestamp=true ;;
246         -d)     DUMP_OUTPUT=true ;;
247
248         --large-fs) export LARGE_SCRATCH_DEV=yes ;;
249         --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
250
251         -*)     usage ;;
252         *)      # not an argument, we've got tests now.
253                 have_test_arg=true ;;
254         esac
255
256         # if we've found a test specification, the break out of the processing
257         # loop before we shift the arguments so that this is the first argument
258         # that we process in the test arg loop below.
259         if $have_test_arg; then
260                 break;
261         fi
262
263         shift
264 done
265
266 # we need common/config, source it after processing args, overlay needs FSTYP
267 # set before sourcing common/config
268 if ! . ./common/config; then
269         echo "$iam: failed to source common/config"
270         exit 1
271 fi
272
273 # Process tests from command line now.
274 if $have_test_arg; then
275         while [ $# -gt 0 ]; do
276                 case "$1" in
277                 -*)     echo "Arguments before tests, please!"
278                         status=1
279                         exit $status
280                         ;;
281                 *)      test_dir=`dirname $1`
282                         test_dir=${test_dir#$SRC_DIR/*}
283                         test_name=`basename $1`
284                         group_file=$SRC_DIR/$test_dir/group
285
286                         if egrep "^$test_name" $group_file >/dev/null ; then
287                                 # in group file ... OK
288                                 echo $SRC_DIR/$test_dir/$test_name >>$tmp.arglist
289                         else
290                                 # oops
291                                 echo "$1 - unknown test, ignored"
292                         fi
293                         ;;
294                 esac
295
296                 shift
297         done
298 fi
299
300 # we need common/rc
301 if ! . ./common/rc
302 then
303     echo "check: failed to source common/rc"
304     exit 1
305 fi
306
307 if [ `id -u` -ne 0 ]
308 then
309     echo "check: QA must be run as root"
310     exit 1
311 fi
312
313 _wipe_counters()
314 {
315         n_try="0"
316         n_bad="0"
317         unset try notrun bad
318 }
319
320 _wrapup()
321 {
322     seq="check"
323     check="$RESULT_BASE/check"
324
325     if $showme
326     then
327         :
328     elif $needwrap
329     then
330         if [ -f $check.time -a -f $tmp.time ]
331         then
332             cat $check.time $tmp.time \
333             | $AWK_PROG '
334         { t[$1] = $2 }
335 END     { if (NR > 0) {
336             for (i in t) print i " " t[i]
337           }
338         }' \
339             | sort -n >$tmp.out
340             mv $tmp.out $check.time
341         fi
342
343         echo "" >>$check.log
344         date >>$check.log
345         echo $list | fmt | sed -e 's/^/    /' -e "s;$SRC_DIR/;;g" >>$check.log
346         $interrupt && echo "Interrupted!" >>$check.log
347
348         echo "SECTION       -- $section" >>$tmp.summary
349         echo "=========================" >>$tmp.summary
350         if [ ! -z "$n_try" -a $n_try != 0 ]
351         then
352             echo "Ran:$try"
353             echo "Ran:$try" >>$tmp.summary
354         fi
355
356         if [ ! -z "$notrun" ]
357         then
358             echo "Not run:$notrun"
359             echo "Not run:$notrun" >>$check.log
360             echo "Not run:$notrun" >>$tmp.summary
361         fi
362
363         if [ ! -z "$n_bad" -a $n_bad != 0 ]
364         then
365             echo "Failures:$bad"
366             echo "Failed $n_bad of $n_try tests"
367             echo "Failures:$bad" | fmt >>$check.log
368             echo "Failed $n_bad of $n_try tests" >>$check.log
369             echo "Failures:$bad" >>$tmp.summary
370             echo "Failed $n_bad of $n_try tests" >>$tmp.summary
371         else
372             echo "Passed all $n_try tests"
373             echo "Passed all $n_try tests" >>$check.log
374             echo "Passed all $n_try tests" >>$tmp.summary
375         fi
376         echo "" >>$tmp.summary
377         needwrap=false
378     fi
379
380     sum_bad=`expr $sum_bad + $n_bad`
381     _wipe_counters
382     rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
383     if ! $OPTIONS_HAVE_SECTIONS; then
384         rm -f $tmp.*
385     fi
386 }
387
388 _summary()
389 {
390         _wrapup
391         if $showme; then
392                 :
393         elif $needsum; then
394                 count=`wc -L $tmp.summary | cut -f1 -d" "`
395                 cat $tmp.summary
396                 needsum=false
397         fi
398         rm -f $tmp.*
399 }
400
401 _check_filesystems()
402 {
403         if [ -f ${RESULT_DIR}/require_test ]; then
404                 _check_test_fs || err=true
405                 rm -f ${RESULT_DIR}/require_test
406         fi
407         if [ -f ${RESULT_DIR}/require_scratch ]; then
408                 _check_scratch_fs || err=true
409                 rm -f ${RESULT_DIR}/require_scratch
410         fi
411 }
412
413 _prepare_test_list
414
415 if $OPTIONS_HAVE_SECTIONS; then
416         trap "_summary; exit \$status" 0 1 2 3 15
417 else
418         trap "_wrapup; exit \$status" 0 1 2 3 15
419 fi
420
421 for section in $HOST_OPTIONS_SECTIONS; do
422         OLD_FSTYP=$FSTYP
423         OLD_MOUNT_OPTIONS=$MOUNT_OPTIONS
424         get_next_config $section
425
426         # Do we need to run only some sections ?
427         if [ ! -z "$RUN_SECTION" ]; then
428                 skip=true
429                 for s in $RUN_SECTION; do
430                         if [ $section == $s ]; then
431                                 skip=false
432                                 break;
433                         fi
434                 done
435                 if $skip; then
436                         continue
437                 fi
438         fi
439
440         # Did this section get excluded?
441         if [ ! -z "$EXCLUDE_SECTION" ]; then
442                 skip=false
443                 for s in $EXCLUDE_SECTION; do
444                         if [ $section == $s ]; then
445                                 skip=true
446                                 break;
447                         fi
448                 done
449                 if $skip; then
450                         continue
451                 fi
452         fi
453
454         mkdir -p $RESULT_BASE
455         if [ ! -d $RESULT_BASE ]; then
456                 echo "failed to create results directory $RESULT_BASE"
457                 status=1
458                 exit
459         fi
460
461         if $OPTIONS_HAVE_SECTIONS; then
462                 echo "SECTION       -- $section"
463         fi
464
465         if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
466                 echo "RECREATING    -- $FSTYP on $TEST_DEV"
467                 _test_unmount 2> /dev/null
468                 if ! _test_mkfs >$tmp.err 2>&1
469                 then
470                         echo "our local _test_mkfs routine ..."
471                         cat $tmp.err
472                         echo "check: failed to mkfs \$TEST_DEV using specified options"
473                         status=1
474                         exit
475                 fi
476                 out=`_mount_or_remount_rw "$MOUNT_OPTIONS" $TEST_DEV $TEST_DIR`
477                 if [ $? -ne 1 ]; then
478                         echo $out
479                         status=1
480                         exit
481                 fi
482                 _prepare_test_list
483         elif [ "$OLD_MOUNT_OPTIONS" != "$MOUNT_OPTIONS" ]; then
484                 _test_unmount 2> /dev/null
485                 out=`_mount_or_remount_rw "$MOUNT_OPTIONS" $TEST_DEV $TEST_DIR`
486                 if [ $? -ne 1 ]; then
487                         echo $out
488                         status=1
489                         exit
490                 fi
491         fi
492
493         init_rc
494
495         seq="check"
496         check="$RESULT_BASE/check"
497
498         # don't leave old full output behind on a clean run
499         rm -f $check.full
500
501         [ -f $check.time ] || touch $check.time
502
503         # print out our test configuration
504         echo "FSTYP         -- `_full_fstyp_details`"
505         echo "PLATFORM      -- `_full_platform_details`"
506         if [ ! -z "$SCRATCH_DEV" ]; then
507           echo "MKFS_OPTIONS  -- `_scratch_mkfs_options`"
508           echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
509         fi
510         echo
511         needwrap=true
512
513         if [ ! -z "$SCRATCH_DEV" ]; then
514           _scratch_unmount 2> /dev/null
515           # call the overridden mkfs - make sure the FS is built
516           # the same as we'll create it later.
517
518           if ! _scratch_mkfs >$tmp.err 2>&1
519           then
520               echo "our local _scratch_mkfs routine ..."
521               cat $tmp.err
522               echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
523               status=1
524               exit
525           fi
526
527           # call the overridden mount - make sure the FS mounts with
528           # the same options that we'll mount with later.
529           if ! _scratch_mount >$tmp.err 2>&1
530           then
531               echo "our local mount routine ..."
532               cat $tmp.err
533               echo "check: failed to mount \$SCRATCH_DEV using specified options"
534               status=1
535               exit
536           fi
537         fi
538
539         seqres="$check"
540         _check_test_fs
541
542         for seq in $list
543         do
544             err=false
545
546             # the filename for the test and the name output are different.
547             # we don't include the tests/ directory in the name output.
548             export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
549
550             # Similarly, the result directory needs to replace the tests/
551             # part of the test location.
552             group=`dirname $seq`
553             if $OPTIONS_HAVE_SECTIONS; then
554                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
555                 seqres="$RESULT_BASE/$section/$seqnum"
556             else
557                 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
558                 seqres="$RESULT_BASE/$seqnum"
559             fi
560
561             mkdir -p $RESULT_DIR
562
563             echo -n "$seqnum"
564
565                 if $showme; then
566                         echo
567                         continue
568                 elif [ ! -f $seq ]; then
569                         # Try to get full name in case the user supplied only seq id
570                         # and the test has a name. A bit of hassle to find really
571                         # the test and not its sample output or helping files.
572                         bname=$(basename $seq)
573                         full_seq=$(find $(dirname $seq) -name $bname* -executable |
574                                 awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
575                                         END { print shortest }')
576                         if [ -f $full_seq ] \
577                                 && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
578                                 seq=$full_seq
579                                 seqnum=${full_seq#*/}
580                         fi
581                 fi
582
583                 if [ ! -f $seq ]; then
584                         echo " - no such test?"
585                 else
586                 # really going to try and run this one
587                 #
588                 rm -f $seqres.out.bad
589
590                 # check if we really should run it
591                 if [ -s $tmp.xlist ]; then
592                         if grep $seqnum $tmp.xlist > /dev/null 2>&1 ; then
593                                 echo "       [expunged]"
594                                 continue
595                         fi
596                 fi
597
598                 # slashes now in names, sed barfs on them so use grep
599                 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
600                 if [ "X$lasttime" != X ]; then
601                         echo -n " ${lasttime}s ..."
602                 else
603                         echo -n "       "       # prettier output with timestamps.
604                 fi
605                 rm -f core $seqres.notrun
606
607                 start=`_wallclock`
608                 $timestamp && echo -n " ["`date "+%T"`"]"
609                 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
610                 $LOGGER_PROG "run xfstest $seqnum"
611                 if [ -w /dev/kmsg ]; then
612                         export date_time=`date +"%F %T"`
613                         echo "run fstests $seqnum at $date_time" > /dev/kmsg
614                         # _check_dmesg depends on this log in dmesg
615                         touch ${RESULT_DIR}/check_dmesg
616                 fi
617                 if [ "$DUMP_OUTPUT" = true ]; then
618                         ./$seq 2>&1 | tee $tmp.rawout
619                         # Because $? would get tee's return code
620                         sts=${PIPESTATUS[0]}
621                 else
622                         ./$seq >$tmp.rawout 2>&1
623                         sts=$?
624                 fi
625                 $timestamp && _timestamp
626                 stop=`_wallclock`
627
628                 _fix_malloc <$tmp.rawout >$tmp.out
629                 rm -f $tmp.rawout
630
631                 if [ -f core ]
632                 then
633                     echo -n " [dumped core]"
634                     mv core $RESULT_BASE/$seqnum.core
635                     err=true
636                 fi
637
638                 if [ -f $seqres.notrun ]
639                 then
640                     $timestamp || echo -n " [not run] "
641                     $timestamp && echo " [not run]" && echo -n "        $seqnum -- "
642                     cat $seqres.notrun
643                     notrun="$notrun $seqnum"
644                 else
645                     if [ $sts -ne 0 ]
646                     then
647                         echo -n " [failed, exit status $sts]"
648                         err=true
649                     fi
650                     if [ ! -f $seq.out ]
651                     then
652                         echo " - no qualified output"
653                         err=true
654                     else
655
656                         # coreutils 8.16+ changed quote formats in error messages from
657                         # `foo' to 'foo'. Filter old versions to match the new version.
658                         sed -i "s/\`/\'/g" $tmp.out
659                         if diff $seq.out $tmp.out >/dev/null 2>&1
660                         then
661                             if $err
662                             then
663                                 :
664                             else
665                                 echo "$seqnum `expr $stop - $start`" >>$tmp.time
666                                 echo -n " `expr $stop - $start`s"
667                             fi
668                             echo ""
669                         else
670                             echo " - output mismatch (see $seqres.out.bad)"
671                             mv $tmp.out $seqres.out.bad
672                             $diff $seq.out $seqres.out.bad | {
673                                 if test "$DIFF_LENGTH" -le 0; then
674                                         cat
675                                 else
676                                         head -n "$DIFF_LENGTH"
677                                         echo "..."
678                                         echo "(Run '$diff $seq.out $seqres.out.bad'" \
679                                                 " to see the entire diff)"
680                                 fi; } | \
681                                 sed -e 's/^\(.\)/    \1/'
682                             err=true
683                         fi
684                     fi
685                     try="$try $seqnum"
686                     n_try=`expr $n_try + 1`
687                     _check_filesystems
688                     _check_dmesg || err=true
689                 fi
690
691             fi
692
693             # come here for each test, except when $showme is true
694             #
695             if $err
696             then
697                 bad="$bad $seqnum"
698                 n_bad=`expr $n_bad + 1`
699                 quick=false
700             fi
701
702             seq="after_$seqnum"
703         done
704         _wrapup
705         echo
706
707         _test_unmount 2> /dev/null
708         _scratch_unmount 2> /dev/null
709 done
710
711 interrupt=false
712 status=`expr $sum_bad`
713 exit