3 # Control script for QA
5 # Copyright (c) 2000-2002,2006 Silicon Graphics, Inc. All Rights Reserved.
7 # This program is free software; you can redistribute it and/or
8 # modify it under the terms of the GNU General Public License as
9 # published by the Free Software Foundation.
11 # This program is distributed in the hope that it would be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write the Free Software Foundation,
18 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
40 brief_test_summary=false
45 # start the initialisation work now
48 export MSGVERB="text:action"
49 export QA_CHECK_FS=${QA_CHECK_FS:=true}
51 # number of diff lines from a failed test, 0 for whole output
52 export DIFF_LENGTH=${DIFF_LENGTH:=10}
54 # by default don't output timestamps
55 timestamp=${TIMESTAMP:=false}
57 rm -f $tmp.list $tmp.tmp $tmp.grep $here/$iam.out $tmp.xlist $tmp.report.*
59 SRC_GROUPS="generic shared"
60 export SRC_DIR="tests"
64 echo "Usage: $0 [options] [testlist]"'
72 -udiff show unified diff (default)
73 -n show me, do not run tests
75 -r randomize test order
76 -d dump test output to stdout
78 -R fmt[,fmt] generate report in formats specified. Supported format: [xunit]
79 --large-fs optimise scratch device for large filesystems
80 -s section run only specified section from config file
81 -S section exclude the specified section from the config file
84 -g group[,group...] include tests from these groups
85 -x group[,group...] exclude tests from these groups
86 -X exclude_file exclude individual tests
87 -E external_file exclude individual tests
88 [testlist] include tests matching names in testlist
90 testlist argument is a list of tests in the form of <test dir>/<test name>.
92 <test dir> is a directory under tests that contains a group file,
93 with a list of the names of the tests in that directory.
95 <test name> may be either a specific test file name (e.g. xfs/001) or
96 a test file name match pattern (e.g. xfs/*).
98 group argument is either a name of a tests group to collect from all
99 the test dirs (e.g. quick) or a name of a tests group to collect from
100 a specific tests dir in the form of <test dir>/<group name> (e.g. xfs/quick).
102 exclude_file argument refers to a name of a file inside each test directory.
103 for every test dir where this file is found, the listed test names are
104 excluded from the list of tests to run from that test dir.
106 external_file argument is a path to a single file containing a list of tests
107 to exclude in the form of <test dir>/<test name>.
113 check -x stress xfs/*
114 check -X .exclude -g auto
115 check -E ~/.xfstests.exclude
125 test -s "$SRC_DIR/$d/group" || return 1
127 local grpl=$(sed -n < $SRC_DIR/$d/group \
130 -e "s;^\($VALID_TEST_NAME\).* $grp .*;$SRC_DIR/$d/\1;p")
138 local sub=$(dirname $grp)
140 if [ -n "$sub" -a "$sub" != "." -a -d "$SRC_DIR/$sub" ]; then
141 # group is given as <subdir>/<group> (e.g. xfs/quick)
143 get_sub_group_list $sub $grp
147 for d in $SRC_GROUPS $FSTYP; do
148 if ! test -d "$SRC_DIR/$d" ; then
151 grpl="$grpl $(get_sub_group_list $d $grp)"
156 # Find all tests, excluding files that are test metadata such as group files.
157 # It matches test names against $VALID_TEST_NAME defined in common/rc
161 for d in $SRC_GROUPS $FSTYP; do
162 if ! test -d "$SRC_DIR/$d" ; then
167 grep "^$SRC_DIR/$d/$VALID_TEST_NAME"| \
168 grep -v "group\|Makefile" >> $tmp.list 2>/dev/null
172 # takes the list of tests to run in $tmp.list, and removes the tests passed to
173 # the function from that list.
182 if [ $numsed -gt 100 ]; then
183 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
184 mv $tmp.tmp $tmp.list
188 echo "^$t\$" >>$tmp.grep
189 numsed=`expr $numsed + 1`
191 grep -v -f $tmp.grep <$tmp.list >$tmp.tmp
192 mv $tmp.tmp $tmp.list
210 # Tests specified on the command line
211 if [ -s $tmp.arglist ]; then
212 cat $tmp.arglist > $tmp.list
217 # Specified groups to include
218 for group in $GROUP_LIST; do
219 list=$(get_group_list $group)
220 if [ -z "$list" ]; then
221 echo "Group \"$group\" is empty or not defined?"
226 grep -s "^$t\$" $tmp.list >/dev/null || \
227 echo "$t" >>$tmp.list
231 if ! $have_test_arg && [ -z "$GROUP_LIST" ]; then
232 # no test numbers, do everything
236 # Specified groups to exclude
237 for xgroup in $XGROUP_LIST; do
238 list=$(get_group_list $xgroup)
239 if [ -z "$list" ]; then
240 echo "Group \"$xgroup\" is empty or not defined?"
247 # sort the list of tests into numeric order
248 list=`sort -n $tmp.list | uniq`
249 rm -f $tmp.list $tmp.tmp $tmp.grep
253 list=`echo $list | awk -f randomize.awk`
257 # Process command arguments first.
258 while [ $# -gt 0 ]; do
260 -\? | -h | --help) usage ;;
264 -overlay) FSTYP=overlay; export OVERLAY=true ;;
265 -tmpfs) FSTYP=tmpfs ;;
267 -g) group=$2 ; shift ;
268 GROUP_LIST="$GROUP_LIST ${group//,/ }"
271 -x) xgroup=$2 ; shift ;
272 XGROUP_LIST="$XGROUP_LIST ${xgroup//,/ }"
275 -X) xfile=$2; shift ;
276 for d in $SRC_GROUPS $FSTYP; do
277 [ -f $SRC_DIR/$d/$xfile ] || continue
278 for f in `sed "s/#.*$//" $SRC_DIR/$d/$xfile`; do
279 echo $d/$f >> $tmp.xlist
283 -E) xfile=$2; shift ;
284 if [ -f $xfile ]; then
285 sed "s/#.*$//" "$xfile" >> $tmp.xlist
288 -s) RUN_SECTION="$RUN_SECTION $2"; shift ;;
289 -S) EXCLUDE_SECTION="$EXCLUDE_SECTION $2"; shift ;;
291 -udiff) diff="$diff -u" ;;
294 -r) randomize=true ;;
296 -T) timestamp=true ;;
297 -d) DUMP_OUTPUT=true ;;
298 -b) brief_test_summary=true;;
299 -R) report_fmt=$2 ; shift ;
300 REPORT_LIST="$REPORT_LIST ${report_fmt//,/ }"
303 --large-fs) export LARGE_SCRATCH_DEV=yes ;;
304 --extra-space=*) export SCRATCH_DEV_EMPTY_SPACE=${r#*=} ;;
307 *) # not an argument, we've got tests now.
308 have_test_arg=true ;;
311 # if we've found a test specification, the break out of the processing
312 # loop before we shift the arguments so that this is the first argument
313 # that we process in the test arg loop below.
314 if $have_test_arg; then
321 # we need common/config, source it after processing args, overlay needs FSTYP
322 # set before sourcing common/config
323 if ! . ./common/config; then
324 echo "$iam: failed to source common/config"
328 # Process tests from command line now.
329 if $have_test_arg; then
330 while [ $# -gt 0 ]; do
332 -*) echo "Arguments before tests, please!"
336 *) # Expand test pattern (e.g. xfs/???, *fs/001)
337 list=$(cd $SRC_DIR; echo $1)
339 test_dir=`dirname $t`
340 test_dir=${test_dir#$SRC_DIR/*}
341 test_name=`basename $t`
342 group_file=$SRC_DIR/$test_dir/group
344 if egrep -q "^$test_name" $group_file; then
345 # in group file ... OK
346 echo $SRC_DIR/$test_dir/$test_name \
350 echo "$t - unknown test, ignored"
363 echo "check: failed to source common/rc"
369 echo "check: QA must be run as root"
384 check="$RESULT_BASE/check"
394 if [ -f $check.time -a -f $tmp.time ]; then
395 cat $check.time $tmp.time \
400 for (i in t) print i " " t[i]
404 mv $tmp.out $check.time
410 echo "SECTION -- $section" >>$tmp.summary
411 echo "=========================" >>$tmp.summary
412 if [ ! -z "$n_try" -a $n_try != 0 ]; then
413 if [ $brief_test_summary == "false" ]; then
415 echo "Ran:$try" >>$tmp.summary
417 echo "Ran:$try" >>$check.log
420 $interrupt && echo "Interrupted!" | tee -a $check.log
422 if [ ! -z "$notrun" ]; then
423 if [ $brief_test_summary == "false" ]; then
424 echo "Not run:$notrun"
425 echo "Not run:$notrun" >>$tmp.summary
427 echo "Not run:$notrun" >>$check.log
430 if [ ! -z "$n_bad" -a $n_bad != 0 ]; then
432 echo "Failed $n_bad of $n_try tests"
433 echo "Failures:$bad" >>$check.log
434 echo "Failed $n_bad of $n_try tests" >>$check.log
435 echo "Failures:$bad" >>$tmp.summary
436 echo "Failed $n_bad of $n_try tests" >>$tmp.summary
438 echo "Passed all $n_try tests"
439 echo "Passed all $n_try tests" >>$check.log
440 echo "Passed all $n_try tests" >>$tmp.summary
442 echo "" >>$tmp.summary
449 sum_bad=`expr $sum_bad + $n_bad`
451 rm -f /tmp/*.rawout /tmp/*.out /tmp/*.err /tmp/*.time
452 if ! $OPTIONS_HAVE_SECTIONS; then
463 count=`wc -L $tmp.summary | cut -f1 -d" "`
472 if [ -f ${RESULT_DIR}/require_test ]; then
473 _check_test_fs || err=true
474 rm -f ${RESULT_DIR}/require_test*
476 if [ -f ${RESULT_DIR}/require_scratch ]; then
477 _check_scratch_fs || err=true
478 rm -f ${RESULT_DIR}/require_scratch*
484 if $OPTIONS_HAVE_SECTIONS; then
485 trap "_summary; exit \$status" 0 1 2 3 15
487 trap "_wrapup; exit \$status" 0 1 2 3 15
490 for section in $HOST_OPTIONS_SECTIONS; do
492 OLD_TEST_FS_MOUNT_OPTS=$TEST_FS_MOUNT_OPTS
493 get_next_config $section
495 # Do we need to run only some sections ?
496 if [ ! -z "$RUN_SECTION" ]; then
498 for s in $RUN_SECTION; do
499 if [ $section == $s ]; then
509 # Did this section get excluded?
510 if [ ! -z "$EXCLUDE_SECTION" ]; then
512 for s in $EXCLUDE_SECTION; do
513 if [ $section == $s ]; then
523 mkdir -p $RESULT_BASE
524 if [ ! -d $RESULT_BASE ]; then
525 echo "failed to create results directory $RESULT_BASE"
530 if $OPTIONS_HAVE_SECTIONS; then
531 echo "SECTION -- $section"
534 sect_start=`_wallclock`
535 if $RECREATE_TEST_DEV || [ "$OLD_FSTYP" != "$FSTYP" ]; then
536 echo "RECREATING -- $FSTYP on $TEST_DEV"
537 _test_unmount 2> /dev/null
538 if ! _test_mkfs >$tmp.err 2>&1
540 echo "our local _test_mkfs routine ..."
542 echo "check: failed to mkfs \$TEST_DEV using specified options"
548 echo "check: failed to mount $TEST_DEV on $TEST_DIR"
553 elif [ "$OLD_TEST_FS_MOUNT_OPTS" != "$TEST_FS_MOUNT_OPTS" ]; then
554 _test_unmount 2> /dev/null
557 echo "check: failed to mount $TEST_DEV on $TEST_DIR"
566 check="$RESULT_BASE/check"
568 # don't leave old full output behind on a clean run
571 [ -f $check.time ] || touch $check.time
573 # print out our test configuration
574 echo "FSTYP -- `_full_fstyp_details`"
575 echo "PLATFORM -- `_full_platform_details`"
576 if [ ! -z "$SCRATCH_DEV" ]; then
577 echo "MKFS_OPTIONS -- `_scratch_mkfs_options`"
578 echo "MOUNT_OPTIONS -- `_scratch_mount_options`"
583 if [ ! -z "$SCRATCH_DEV" ]; then
584 _scratch_unmount 2> /dev/null
585 # call the overridden mkfs - make sure the FS is built
586 # the same as we'll create it later.
588 if ! _scratch_mkfs >$tmp.err 2>&1
590 echo "our local _scratch_mkfs routine ..."
592 echo "check: failed to mkfs \$SCRATCH_DEV using specified options"
597 # call the overridden mount - make sure the FS mounts with
598 # the same options that we'll mount with later.
599 if ! _scratch_mount >$tmp.err 2>&1
601 echo "our local mount routine ..."
603 echo "check: failed to mount \$SCRATCH_DEV using specified options"
616 if [ ! -f $seq ]; then
617 # Try to get full name in case the user supplied only seq id
618 # and the test has a name. A bit of hassle to find really
619 # the test and not its sample output or helping files.
620 bname=$(basename $seq)
621 full_seq=$(find $(dirname $seq) -name $bname* -executable |
622 awk '(NR == 1 || length < length(shortest)) { shortest = $0 }\
623 END { print shortest }')
624 if [ -f $full_seq ] \
625 && [ x$(echo $bname | grep -o "^$VALID_TEST_ID") != x ]; then
630 # the filename for the test and the name output are different.
631 # we don't include the tests/ directory in the name output.
632 export seqnum=`echo $seq | sed -e "s;$SRC_DIR/;;"`
634 # Similarly, the result directory needs to replace the tests/
635 # part of the test location.
637 if $OPTIONS_HAVE_SECTIONS; then
638 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;${RESULT_BASE}/$section;"`
639 REPORT_DIR="$RESULT_BASE/$section"
641 export RESULT_DIR=`echo $group | sed -e "s;$SRC_DIR;$RESULT_BASE;"`
642 REPORT_DIR="$RESULT_BASE"
644 seqres="$REPORT_DIR/$seqnum"
654 n_notrun=`expr $n_notrun + 1`
656 _make_testcase_report "list"
661 if [ ! -f $seq ]; then
662 echo " - no such test?"
664 # really going to try and run this one
666 rm -f $seqres.out.bad
668 # check if we really should run it
669 if [ -s $tmp.xlist ]; then
670 if grep $seqnum $tmp.xlist > /dev/null 2>&1 ; then
676 # slashes now in names, sed barfs on them so use grep
677 lasttime=`grep -w ^$seqnum $check.time | awk '// {print $2}'`
678 if [ "X$lasttime" != X ]; then
679 echo -n " ${lasttime}s ..."
681 echo -n " " # prettier output with timestamps.
683 rm -f core $seqres.notrun
686 $timestamp && echo -n " ["`date "+%T"`"]"
687 [ ! -x $seq ] && chmod u+x $seq # ensure we can run it
688 $LOGGER_PROG "run xfstest $seqnum"
689 if [ -w /dev/kmsg ]; then
690 export date_time=`date +"%F %T"`
691 echo "run fstests $seqnum at $date_time" > /dev/kmsg
692 # _check_dmesg depends on this log in dmesg
693 touch ${RESULT_DIR}/check_dmesg
695 if [ "$DUMP_OUTPUT" = true ]; then
696 ./$seq 2>&1 | tee $tmp.rawout
697 # Because $? would get tee's return code
700 ./$seq >$tmp.rawout 2>&1
703 $timestamp && _timestamp
706 _fix_malloc <$tmp.rawout >$tmp.out
711 err_msg="[dumped core]"
713 mv core $RESULT_BASE/$seqnum.core
717 if [ -f $seqres.notrun ]
719 $timestamp || echo -n " [not run] "
720 $timestamp && echo " [not run]" && echo -n " $seqnum -- "
722 notrun="$notrun $seqnum"
723 n_notrun=`expr $n_notrun + 1`
728 err_msg="[failed, exit status $sts]"
734 _dump_err "no qualified output"
738 # coreutils 8.16+ changed quote formats in error messages from
739 # `foo' to 'foo'. Filter old versions to match the new version.
740 sed -i "s/\`/\'/g" $tmp.out
741 if diff $seq.out $tmp.out >/dev/null 2>&1
747 echo "$seqnum `expr $stop - $start`" >>$tmp.time
748 echo -n " `expr $stop - $start`s"
752 echo " - output mismatch (see $seqres.out.bad)"
753 mv $tmp.out $seqres.out.bad
754 $diff $seq.out $seqres.out.bad | {
755 if test "$DIFF_LENGTH" -le 0; then
758 head -n "$DIFF_LENGTH"
760 echo "(Run '$diff $seq.out $seqres.out.bad'" \
761 " to see the entire diff)"
763 sed -e 's/^\(.\)/ \1/'
764 err_msg="output mismatch (see $diff $seq.out $seqres.out.bad)"
769 n_try=`expr $n_try + 1`
771 _check_dmesg || err=true
776 # come here for each test, except when $showme is true
781 n_bad=`expr $n_bad + 1`
786 _make_testcase_report "$tc_status"
790 sect_stop=`_wallclock`
796 _test_unmount 2> /dev/null
797 _scratch_unmount 2> /dev/null
801 status=`expr $sum_bad`