3 # Control script for QA
5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc. All Rights Reserved.
7 # This program is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU General Public License as
9 # published by the Free Software Foundation.
11 # This program is distributed in the hope that it would be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write the Free Software Foundation,
18 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
40 brief_test_summary=false
44 # This is a global variable used to pass test failure text to reporting gunk
47 # start the initialisation work now
50 export MSGVERB="text:action"
51 export QA_CHECK_FS=${QA_CHECK_FS:=true}
53 # number of diff lines from a failed test, 0 for whole output
54 export DIFF_LENGTH=${DIFF_LENGTH:=10}
56 # by default don't output timestamps
57 timestamp=${TIMESTAMP:=false}
59 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
61 SRC_GROUPS="generic shared"
62 export SRC_DIR="tests"
66 echo "Usage: $0 [options] [testlist]"'
70 -glusterfs test GlusterFS
78 -udiff show unified diff (default)
79 -n show me, do not run tests
81 -r randomize test order
82 -d dump test output to stdout
84 -R fmt[,fmt] generate report in formats specified. Supported format: [xunit]
85 --large-fs optimise scratch device for large filesystems
86 -s section run only specified section from config file
87 -S section exclude the specified section from the config file
90 -g group[,group...] include tests from these groups
91 -x group[,group...] exclude tests from these groups
92 -X exclude_file exclude individual tests
93 -E external_file exclude individual tests
94 [testlist] include tests matching names in testlist
96 testlist argument is a list of tests in the form of <test dir>/<test name>.
98 <test dir> is a directory under tests that contains a group file,
99 with a list of the names of the tests in that directory.
101 <test name> may be either a specific test file name (e.g. xfs/001) or
102 a test file name match pattern (e.g. xfs/*).
104 group argument is either a name of a tests group to collect from all
105 the test dirs (e.g. quick) or a name of a tests group to collect from
106 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
107 If you want to run all the tests in the test suite, use "-g all" to specify all
110 exclude_file argument refers to a name of a file inside each test directory.
111 for every test dir where this file is found, the listed test names are
112 excluded from the list of tests to run from that test dir.
114 external_file argument is a path to a single file containing a list of tests
115 to exclude in the form of <test dir>/<test name>.
121 check -x stress xfs/*
122 check -X .exclude -g auto
123 check -E ~/.xfstests.exclude
133 test -s "$SRC_DIR/$d/group" || return 1
135 local grpl=$(sed -n < $SRC_DIR/$d/group \
138 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
146 local sub=$(dirname $grp)
148 if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
149 # group is given as <subdir>/<group> (e.g. xfs/quick)
151 get_sub_group_list $sub $grp
155 for d in $SRC_GROUPS $FSTYP; do
156 if ! test -d "$SRC_DIR/$d" ; then
159 grpl="$grpl $(get_sub_group_list $d $grp)"
164 # Find all tests, excluding files that are test metadata such as group files.
165 # It matches test names against $VALID_TEST_NAME defined in common/rc
169 for d in $SRC_GROUPS $FSTYP; do
170 if ! test -d "$SRC_DIR/$d" ; then
175 grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
176 grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
180 # takes the list of tests to run in $tmp.list, and removes the tests passed to
181 # the function from that list.
190 if [ $numsed -gt 100 ]; then
191 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
192 mv $tmp.tmp $tmp.list
196 echo "^$t\$" >>$tmp.grep
197 numsed=`expr $numsed + 1`
199 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
200 mv $tmp.tmp $tmp.list
219 # Tests specified on the command line
220 if [ -s $tmp.arglist ]; then
221 cat $tmp.arglist > $tmp.list
226 # Specified groups to include
227 # Note that the CLI processing adds a leading space to the first group
228 # parameter, so we have to catch that here checking for "all"
229 if ! $have_test_arg && [ "$GROUP_LIST" == " all" ]; then
230 # no test numbers, do everything
233 for group in $GROUP_LIST; do
234 list=$(get_group_list $group)
235 if [ -z "$list" ]; then
236 echo "Group \"$group\" is empty or not defined?"
241 grep -s "^$t\$" $tmp.list >/dev/null || \
242 echo "$t" >>$tmp.list
247 # Specified groups to exclude
248 for xgroup in $XGROUP_LIST; do
249 list=$(get_group_list $xgroup)
250 if [ -z "$list" ]; then
251 echo "Group \"$xgroup\" is empty or not defined?"
258 # sort the list of tests into numeric order
259 list=`sort -n $tmp.list | uniq`
264 list=`echo $list | awk -f randomize.awk`
268 # Process command arguments first.
269 while [ $# -gt 0 ]; do
271 -\? | -h | --help) usage ;;
274 -glusterfs) FSTYP=glusterfs ;;
277 -overlay) FSTYP=overlay; export OVERLAY=true ;;
278 -pvfs2) FSTYP=pvfs2 ;;
279 -tmpfs) FSTYP=tmpfs ;;
280 -ubifs) FSTYP=ubifs ;;
282 -g) group=$2 ; shift ;
283 GROUP_LIST="$GROUP_LIST ${group//,/ }"
286 -x) xgroup=$2 ; shift ;
287 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
290 -X) xfile=$2; shift ;
291 for d in $SRC_GROUPS $FSTYP; do
292 [ -f $SRC_DIR/$d/$xfile ] || continue
293 for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
294 echo $d/$f >> $tmp.xlist
298 -E) xfile=$2; shift ;
299 if [ -f $xfile ]; then
300 sed "s/#.*$//" "$xfile" >> $tmp.xlist
303 -s) RUN_SECTION="$RUN_SECTION $2"; shift ;;
304 -S) EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
306 -udiff) diff="$diff -u" ;;
309 -r) randomize=true ;;
311 -T) timestamp=true ;;
312 -d) DUMP_OUTPUT=true ;;
313 -b) brief_test_summary=true;;
314 -R) report_fmt=$2 ; shift ;
315 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
318 --large-fs) export LARGE_SCRATCH_DEV=yes ;;
319 --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
322 *) # not an argument, we've got tests now.
323 have_test_arg=true ;;
326 # if we've found a test specification, the break out of the processing
327 # loop before we shift the arguments so that this is the first argument
328 # that we process in the test arg loop below.
329 if $have_test_arg; then
336 # we need common/rc, that also sources common/config. We need to source it
337 # after processing args, overlay needs FSTYP set before sourcing common/config
338 if ! . ./common/rc; then
339 echo "check: failed to source common/rc"
343 # Process tests from command line now.
344 if $have_test_arg; then
345 while [ $# -gt 0 ]; do
347 -*) echo "Arguments before tests, please!"
351 *) # Expand test pattern (e.g. xfs/???, *fs/001)
352 list=$(cd $SRC_DIR; echo $1)
354 test_dir=`dirname $t`
355 test_dir=${test_dir#$SRC_DIR/*}
356 test_name=`basename $t`
357 group_file=$SRC_DIR/$test_dir/group
359 if egrep -q "^$test_name" $group_file; then
360 # in group file ... OK
361 echo $SRC_DIR/$test_dir/$test_name \
365 echo "$t - unknown test, ignored"
373 elif [ -z "$GROUP_LIST" ]; then
374 # default group list is the auto group. If any other group or test is
375 # specified, we use that instead.
381 echo "check: QA must be run as root"
396 check="$RESULT_BASE/check"
406 if [ -f $check.time -a -f $tmp.time ]; then
407 cat $check.time $tmp.time \
412 for (i in t) print i " " t[i]
416 mv $tmp.out $check.time
422 echo "SECTION -- $section" >>$tmp.summary
423 echo "=========================" >>$tmp.summary
424 if [ ! -z "$n_try" -a $n_try != 0 ]; then
425 if [ $brief_test_summary == "false" ]; then
427 echo "Ran:$try" >>$tmp.summary
429 echo "Ran:$try" >>$check.log
432 $interrupt && echo "Interrupted!" | tee -a $check.log
434 if [ ! -z "$notrun" ]; then
435 if [ $brief_test_summary == "false" ]; then
436 echo "Not run:$notrun"
437 echo "Not run:$notrun" >>$tmp.summary
439 echo "Not run:$notrun" >>$check.log
442 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
444 echo "Failed $n_bad of $n_try tests"
445 echo "Failures:$bad" >>$check.log
446 echo "Failed $n_bad of $n_try tests" >>$check.log
447 echo "Failures:$bad" >>$tmp.summary
448 echo "Failed $n_bad of $n_try tests" >>$tmp.summary
450 echo "Passed all $n_try tests"
451 echo "Passed all $n_try tests" >>$check.log
452 echo "Passed all $n_try tests" >>$tmp.summary
454 echo "" >>$tmp.summary
461 sum_bad=`expr $sum_bad + $n_bad`
463 rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
464 if ! $OPTIONS_HAVE_SECTIONS; then
475 count=`wc -L $tmp.summary | cut -f1 -d" "`
484 if [ -f ${RESULT_DIR}/require_test ]; then
485 _check_test_fs || err=true
486 rm -f ${RESULT_DIR}/require_test*
488 _test_unmount 2> /dev/null
490 if [ -f ${RESULT_DIR}/require_scratch ]; then
491 _check_scratch_fs || err=true
492 rm -f ${RESULT_DIR}/require_scratch*
494 _scratch_unmount 2> /dev/null
501 if [ -s $tmp.xlist ]; then
502 if grep -q $TEST_ID $tmp.xlist; then
513 if $OPTIONS_HAVE_SECTIONS; then
514 trap "_summary; exit \$status" 0 1 2 3 15
516 trap "_wrapup; exit \$status" 0 1 2 3 15
519 for section in $HOST_OPTIONS_SECTIONS; do
521 OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
522 get_next_config $section
524 # Do we need to run only some sections ?
525 if [ ! -z "$RUN_SECTION" ]; then
527 for s in $RUN_SECTION; do
528 if [ $section == $s ]; then
538 # Did this section get excluded?
539 if [ ! -z "$EXCLUDE_SECTION" ]; then
541 for s in $EXCLUDE_SECTION; do
542 if [ $section == $s ]; then
552 mkdir -p $RESULT_BASE
553 if [ ! -d $RESULT_BASE ]; then
554 echo "failed to create results directory $RESULT_BASE"
559 if $OPTIONS_HAVE_SECTIONS; then
560 echo "SECTION -- $section"
563 sect_start=`_wallclock`
564 if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
565 echo "RECREATING -- $FSTYP on $TEST_DEV"
566 _test_unmount 2> /dev/null
567 if ! _test_mkfs >$tmp.err 2>&1
569 echo "our local _test_mkfs routine ..."
571 echo "check: failed to mkfs \$TEST_DEV using specified options"
577 echo "check: failed to mount $TEST_DEV on $TEST_DIR"
582 elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
583 _test_unmount 2> /dev/null
586 echo "check: failed to mount $TEST_DEV on $TEST_DIR"
595 check="$RESULT_BASE/check"
597 # don't leave old full output behind on a clean run
600 [ -f $check.time ] || touch $check.time
602 # print out our test configuration
603 echo "FSTYP -- `_full_fstyp_details`"
604 echo "PLATFORM -- `_full_platform_details`"
605 if [ ! -z "$SCRATCH_DEV" ]; then
606 echo "MKFS_OPTIONS -- `_scratch_mkfs_options`"
607 echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
612 if [ ! -z "$SCRATCH_DEV" ]; then
613 _scratch_unmount 2> /dev/null
614 # call the overridden mkfs - make sure the FS is built
615 # the same as we'll create it later.
617 if ! _scratch_mkfs >$tmp.err 2>&1
619 echo "our local _scratch_mkfs routine ..."
621 echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
626 # call the overridden mount - make sure the FS mounts with
627 # the same options that we'll mount with later.
628 if ! _scratch_mount >$tmp.err 2>&1
630 echo "our local mount routine ..."
632 echo "check: failed to mount \$SCRATCH_DEV using specified options"
644 for seq in $list ; do
645 # Run report for previous test!
648 n_bad=`expr $n_bad + 1`
651 if $do_report && ! $first_test ; then
652 if [ $tc_status != "expunge" ] ; then
653 _make_testcase_report "$prev_seq" "$tc_status"
660 if [ ! -f $seq ]; then
661 # Try to get full name in case the user supplied only
662 # seq id and the test has a name. A bit of hassle to
663 # find really the test and not its sample output or
665 bname=$(basename $seq)
666 full_seq=$(find $(dirname $seq) -name $bname* -executable |
667 awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
668 END { print shortest }')
669 if [ -f $full_seq ] && \
670 [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
675 # the filename for the test and the name output are different.
676 # we don't include the tests/ directory in the name output.
677 export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
679 # Similarly, the result directory needs to replace the tests/
680 # part of the test location.
682 if $OPTIONS_HAVE_SECTIONS; then
683 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
684 REPORT_DIR="$RESULT_BASE/$section"
686 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
687 REPORT_DIR="$RESULT_BASE"
689 seqres="$REPORT_DIR/$seqnum"
695 _expunge_test $seqnum
696 if [ $? -eq 1 ]; then
704 n_notrun=`expr $n_notrun + 1`
709 if [ ! -f $seq ]; then
710 echo " - no such test?"
714 # really going to try and run this one
715 rm -f $seqres.out.bad
717 # check if we really should run it
718 _expunge_test $seqnum
719 if [ $? -eq 1 ]; then
724 # record that we really tried to run this test.
726 n_try=`expr $n_try + 1`
728 # slashes now in names, sed barfs on them so use grep
729 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
730 if [ "X$lasttime" != X ]; then
731 echo -n " ${lasttime}s ... "
733 echo -n " " # prettier output with timestamps.
735 rm -f core $seqres.notrun
738 $timestamp && echo -n " ["`date "+%T"`"]"
739 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
740 $LOGGER_PROG "run xfstest $seqnum"
741 if [ -w /dev/kmsg ]; then
742 export date_time=`date +"%F %T"`
743 echo "run fstests $seqnum at $date_time" > /dev/kmsg
744 # _check_dmesg depends on this log in dmesg
745 touch ${RESULT_DIR}/check_dmesg
747 if [ "$DUMP_OUTPUT" = true ]; then
748 ./$seq 2>&1 | tee $tmp.out
749 # Because $? would get tee's return code
752 ./$seq >$tmp.out 2>&1
757 _dump_err_cont "[dumped core]"
758 mv core $RESULT_BASE/$seqnum.core
762 if [ -f $seqres.notrun ]; then
763 $timestamp && _timestamp
765 $timestamp || echo -n "[not run] "
766 $timestamp && echo " [not run]" && \
767 echo -n " $seqnum -- "
769 notrun="$notrun $seqnum"
770 n_notrun=`expr $n_notrun + 1`
775 if [ $sts -ne 0 ]; then
776 _dump_err_cont "[failed, exit status $sts]"
777 _test_unmount 2> /dev/null
778 _scratch_unmount 2> /dev/null
781 # the test apparently passed, so check for corruption
782 # and log messages that shouldn't be there.
784 _check_dmesg || err=true
785 _check_kmemleak || err=true
788 # test ends after all checks are done.
789 $timestamp && _timestamp
792 if [ ! -f $seq.out ]; then
793 _dump_err "no qualified output"
798 # coreutils 8.16+ changed quote formats in error messages
799 # from `foo' to 'foo'. Filter old versions to match the new
801 sed -i "s/\`/\'/g" $tmp.out
802 if diff $seq.out $tmp.out >/dev/null 2>&1 ; then
804 echo "$seqnum `expr $stop - $start`" >>$tmp.time
805 echo -n " `expr $stop - $start`s"
809 _dump_err "- output mismatch (see $seqres.out.bad)"
810 mv $tmp.out $seqres.out.bad
811 $diff $seq.out $seqres.out.bad | {
812 if test "$DIFF_LENGTH" -le 0; then
815 head -n "$DIFF_LENGTH"
817 echo "(Run '$diff $seq.out $seqres.out.bad'" \
818 " to see the entire diff)"
819 fi; } | sed -e 's/^\(.\)/ \1/'
824 # make sure we record the status of the last test we ran.
827 n_bad=`expr $n_bad + 1`
830 if $do_report && ! $first_test ; then
831 if [ $tc_status != "expunge" ] ; then
832 _make_testcase_report "$prev_seq" "$tc_status"
836 sect_stop=`_wallclock`
842 _test_unmount 2> /dev/null
843 _scratch_unmount 2> /dev/null
847 status=`expr $sum_bad != 0`