tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of version 2 of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 #
  13 # Further, this software is distributed without any warranty that it is
  14 # free of the rightful claim of any third person regarding infringement
  15 # or the like.  Any license provided herein, whether implied or
  16 # otherwise, applies only to this software file.  Patent licenses, if
  17 # any, provided herein do not apply to combinations of this program with
  18 # other software, or any other product whatsoever.
  19 #
  20 # You should have received a copy of the GNU General Public License along
  21 # with this program; if not, write the Free Software Foundation, Inc., 59
  22 # Temple Place - Suite 330, Boston MA 02111-1307, USA.
  23 #
  24 # Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  25 # Mountain View, CA  94043, or:
  26 #
  27 # http://www.sgi.com
  28 #
  29 # For further information regarding this notice, see:
  30 #
  31 # http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  32 #
  33
  34 # automatic qa system. 31/08/00 dxm@sgi.com
  35
  36 # configuration (you could tune this)
  37
  38 _log()
  39 {
  40     echo "$*" >&2
  41     echo "$*" >> $LOG
  42     sync
  43 }
  44
  45 _fail()
  46 {
  47     if [ "$started" = "1" ]
  48     then
  49         echo "auto-qa stopped" | wall
  50         started=0
  51     fi
  52
  53     _log "$*"
  54
  55     # send special email if a cron'd qa run fails
  56     case $state
  57     in
  58         cron*)
  59             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  60         ;;
  61     esac
  62
  63     status=1
  64     exit 1
  65 }
  66
  67 _get_kernel_version()
  68 {
  69     [ -x "$KWORKAREA" ] \
  70         || _fail "can't access kernel workarea $KWORKAREA"
  71     [ -r "$KWORKAREA/Makefile" ] \
  72         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  73
  74     eval `awk '
  75         BEGIN { FS = "[ \t=]+" }
  76         /^VERSION =/ { a=$2 }
  77         /^PATCHLEVEL =/ { b=$2 }
  78         /^SUBLEVEL =/ { c=$2 }
  79         /^EXTRAVERSION =/ { d=$2 }
  80         END {
  81             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  82         }
  83    ' < $KWORKAREA/Makefile`
  84 }
  85
  86 # this should be constant
  87
  88 ROOT="$HOME/qa"
  89 HOST=`hostname -s`
  90 if [ ! -z "$CVSROOT" ]; then
  91     export WORKAREA="$ROOT/xfs-cmds"
  92 else
  93     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  94 fi
  95
  96
  97 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  98 STATE=$ROOT/qa.state
  99 QADIR="$WORKAREA/xfstests"
 100 SUDO="su -c"
 101 CONFIG="$ROOT/$HOST.config"
 102 COMMON_CONFIG="$WORKAREA/xfstests/common.config"
 103 SH="/bin/sh"
 104 LOG="$ROOT/qa.log"
 105
 106 # do some cleanup on exit
 107
 108 _cleanup()
 109 {
 110     umount $SCRATCH_DEV &> /dev/null
 111     umount $TEST_DEV &> /dev/null
 112     if [ "$started" = 1 ]
 113     then
 114         echo "auto-qa stopped" | wall
 115         started=0
 116     fi
 117 }
 118 status=1
 119 trap "_cleanup; exit \$status" 0 1 2 3 15
 120
 121 # clean exit
 122
 123 _success()
 124 {
 125     status=0
 126     exit 0
 127 }
 128
 129 _get_state()
 130 {
 131     state=`cat $STATE`
 132 }
 133
 134 _set_state()
 135 {
 136     echo $1 > $STATE
 137     _get_state
 138 }
 139
 140 _change_state()
 141 {
 142     new=$1
 143
 144     case $state
 145     in
 146         *-*)
 147             case $new
 148             in
 149                 *-*)
 150                     _set_state $new
 151                     ;;
 152                 *)
 153                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 154                     ;;
 155             esac
 156             ;;
 157         *)
 158             _set_state $new
 159             ;;
 160     esac
 161 }
 162
 163 _sudo()
 164 {
 165     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 166 }
 167
 168 _restart()
 169 {
 170     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 171 }
 172
 173 _update_autoqa_file()
 174 {
 175     SELF="$ROOT/auto-qa"
 176     SELF_UPDATE="xfstests/tools/auto-qa"
 177     if [ -z "$CVSROOT" ]; then
 178             cmd="p_tupdate $SELF_UPDATE"
 179     else
 180             cmd="cvs -z3 update $SELF_UPDATE"
 181     fi
 182     exec $SH -c "cd $WORKAREA ; $cmd; chmod +x $SELF_UPDATE ; exec $SELF"
 183 }
 184
 185 _update_workarea()
 186 {
 187     if [ -z "$CVSROOT" ]; then
 188         _log "  *** p_tupdate"
 189         cd "$1"
 190         WORKAREA="$1" p_tupdate 2>&1 \
 191                         || _fail "          !!! p_tupdate failed"
 192
 193         _log "  *** p_check/p_purge"
 194         cd "$1"
 195         WORKAREA="$1" p_check -s | p_purge -yiu 2>&1 \
 196                         || _fail "          !!! p_check/p_purge failed"
 197
 198         _log "  *** non-trunk files"
 199         cd "$1"
 200         WORKAREA="$1" p_list -c 2>&1 \
 201                         || _fail "          !!! p_list failed"
 202     else
 203         _log "  *** cvs update"
 204         cd "$1"
 205         cvs -z3 update -d
 206     fi
 207 }
 208
 209 _test_mkfs_xfs()
 210 {
 211     TEST_OPTIONS=""
 212     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 213         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 214     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 215         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 216     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 217     mkfs_status=$?
 218     if [ "$USE_BIG_LOOPFS" = yes ]; then
 219         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 220         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 221     fi
 222     return $mkfs_status
 223 }
 224
 225 _test_mount()
 226 {
 227     TEST_OPTIONS=""
 228     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 229         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 230     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 231         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 232     _sudo mount -t xfs $TEST_OPTIONS $* $TEST_DEV $TEST_DIR
 233 }
 234
 235 _i386_install()
 236 {
 237     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 238                 || _fail "          !!! install kernel failed"
 239     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 240                 || _fail "          !!! install kernel failed"
 241     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 242         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 243                 || _fail "          !!! install modules failed"
 244     fi
 245
 246     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 247         _log "  *** reinit lilo"
 248         _sudo /sbin/lilo 2>&1 \
 249                 || _fail "          !!! reinit lilo failed"
 250     fi
 251 }
 252
 253 _i386_restart()
 254 {
 255     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 256         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 257                 || _fail "          !!! lilo failed"
 258     fi
 259 }
 260
 261 _ia64_install()
 262 {
 263     echo not yet implemented
 264 }
 265
 266 _ia64_restart()
 267 {
 268     echo not yet implemented
 269 }
 270
 271
 272 _log "*** XFS QA (`date`)"
 273
 274 _get_state
 275
 276 # check preconditions for starting state
 277 case $1
 278 in
 279     cron-init)
 280         case $state
 281         in
 282             *done)
 283                 ;;
 284             *)
 285                 _fail "    !!! cron-init while not in \"*done\" state"
 286                 ;;
 287         esac
 288         ;;
 289     cron-restarted)
 290         # we don't auto restart after reboot, but cron the restart
 291         # to happen a bit later - it's much easier and safer that way
 292         if [ "$state" != "cron-restarted" ]
 293         then
 294             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 295         fi
 296         ;;
 297 esac
 298
 299 if [ "$1" != "" ]
 300 then
 301     _set_state $1
 302 fi
 303
 304 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 305 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 306 [ -d "$KWORKAREA" ]     || _fail "    !!! QA kernel \"$KWORKAREA\" not found"
 307 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 308 [ -r "$CONFIG" ]        || _fail "    !!! Can't read config file $CONFIG"
 309 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 310
 311 _get_kernel_version
 312 IMAGE="$BOOT/vmlinuz-$EXTRA"
 313 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 314 MODULES="/lib/modules/$SVERSION"
 315
 316 cd $ROOT
 317
 318 started=1
 319 echo "auto-qa started" | wall
 320
 321 while true
 322 do
 323     _get_state
 324
 325     _log "    *** state $state start (`date`)"
 326     _log "      (user=$USER, host=$HOST)"
 327     new_state=""
 328
 329     case $state
 330     in
 331         *init)
 332             echo "" > $ROOT/qa.log
 333             echo "" > $ROOT/qa.full
 334             _log "******************************************************"
 335             _log "QA init $VERSION (`date`)"
 336             _log "******************************************************"
 337             _log "--- kernel ($IMAGE)"
 338             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 339                 _log "--- modules ($MODULES)"
 340
 341             _change_state "inited"
 342             _update_autoqa_file
 343             ;;
 344
 345         *inited)
 346             _log "      *** QA initialized"
 347             new_state="update"
 348             ;;
 349
 350         *update)
 351             _update_workarea "$KWORKAREA"
 352             _update_workarea "$WORKAREA"
 353             new_state="cleantools"
 354             ;;
 355
 356         *cleantools)
 357             # we need to configure or else we might fail to clean
 358             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 359             do
 360                 [ -d $WORKAREA/$pkg ] || continue
 361                 cd $WORKAREA/$pkg
 362                 _log "  *** clean $pkg tools"
 363                 make realclean 2>&1 \
 364                         || _fail "          !!! clean $pkg failed"
 365             done
 366             new_state="buildtools"
 367             ;;
 368
 369         *buildtools)
 370             _log "      *** build and install tools"
 371             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 372             do
 373                 [ -d $WORKAREA/$pkg ] || continue
 374                 cd $WORKAREA/$pkg
 375
 376                 # use e-fence - but this will only take effect on configure
 377                 make configure 2>&1 \
 378                         || _fail "          !!! configure $pkg failed"
 379                 make default 2>&1 \
 380                         || _fail "          !!! build $pkg failed"
 381
 382                 _sudo make install install-dev 2>&1 \
 383                         || _fail "          !!! install $pkg failed"
 384
 385                 # attr and acl now have install-lib targets as well
 386                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 387                 _sudo make install-lib 2>&1 \
 388                         || _fail "          !!! install $pkg failed"
 389             done
 390
 391             new_state="cleankernel"
 392             ;;
 393
 394         *cleankernel)
 395             _log "      *** clean kernel"
 396             cd "$KWORKAREA"
 397             make mrproper 2>&1 \
 398                         || _fail "          !!! clean kernel failed"
 399
 400             _log "      *** install configuration file"
 401             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 402                         || _fail "          !!! failed to install config"
 403
 404             _log "      *** remove version file"
 405             rm -f include/linux/version.h 2>&1 \
 406                         || _fail "          !!! failed to clean version"
 407
 408             new_state="reconfig"
 409             ;;
 410
 411         *reconfig)
 412
 413             _log "      *** reconfig kernel"
 414
 415             # we better start from scratch if this fails
 416             _change_state "cleankernel"
 417
 418             cd "$KWORKAREA"
 419             # we want to use default options for any new config options.
 420             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 421                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 422                         || _fail "          !!! reconfig oldconfig failed"
 423             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 424                         || _fail "          !!! reconfig dep failed"
 425
 426             new_state="buildkernel"
 427             ;;
 428
 429         *buildkernel)
 430             _log "      *** build kernel"
 431
 432             _change_state "clean" ; # we better start from scratch if this fails
 433
 434             cd "$KWORKAREA"
 435             [ -z "$KTARGET" ] && KTARGET=bzImage
 436             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 437                 KTARGET="$KTARGET modules"
 438             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 439                         || _fail "          !!! build $KTARGET failed"
 440             new_state="install"
 441             ;;
 442
 443         *install)
 444             _log "      *** blat old modules"
 445             _sudo rm -rf $MODULES
 446
 447             _log "      *** install kernel"
 448             cd "$KWORKAREA"
 449             case `uname -m`
 450             in
 451                 i386|i686)      _i386_install ;;
 452                 ia64)           _ia64_install ;;
 453             esac
 454
 455             if [ -z "$KRESTART" -o "$KRESTART" = yes ]; then
 456                 new_state="restart"
 457             else
 458                 new_state="done"
 459             fi
 460             ;;
 461
 462         *restart)
 463             _log "          *** select qa kernel"
 464             case `uname -m`
 465             in
 466                 i386|i686)      _i386_restart ;;
 467                 ia64)           _ia64_restart ;;
 468             esac
 469
 470             _log "          *** prepare to restart"
 471             _change_state "restarted"
 472
 473             _log "          *** restarting"
 474
 475             _restart # doesn't return
 476             ;;
 477
 478         *restarted)
 479             _log "          *** QA reentered after restart"
 480
 481             new_state="check"
 482             ;;
 483
 484         *check)
 485             uname=`uname -a`
 486             _log "          *** uname $uname"
 487             _log "          *** kernel"
 488             [ -n "/boot/*$EXTRA*" ] && \
 489                 ls -l /boot/*$EXTRA*  2>&1
 490             _log "          *** kernel modules"
 491             [ -d /lib/modules/$SVERSION-$EXTRA/kernel/fs/xfs ] && \
 492                 ls -l /lib/modules/$SVERSION-$EXTRA/kernel/fs/xfs/*
 493
 494             if [ "$MODULAR" -eq 0 ]; then
 495                 new_state="reset"
 496             else
 497                 new_state="probe"
 498             fi
 499             ;;
 500
 501         *probe)
 502             _log "          *** modules dependencies"
 503             _sudo depmod -a  2>&1 \
 504                         || _fail "          !!! failed to depmod -a"
 505
 506             _log "          *** unmounting XFS mounts"
 507             _sudo umount -a -t xfs 2>&1
 508
 509             _log "          *** removing modules"
 510             for m in xfsidbg xfs kdbm_pg kdbm_vm
 511             do
 512                 _sudo rmmod $m 2> /dev/null
 513             done
 514
 515             _log "          *** installing modules"
 516             _sudo modprobe xfs 2>&1 \
 517                         || _fail "          !!! failed to modprobe xfs"
 518
 519             new_state="reset"
 520             ;;
 521
 522         *reset)
 523
 524             _log "          *** unmounting TEST_DEV"
 525             _sudo umount $TEST_DEV 2>&1
 526
 527             _log "          *** unmounting SCRATCH_DEV"
 528             _sudo umount $SCRATCH_DEV 2>&1
 529
 530             _log "          *** clean TEST_DEV"
 531             _test_mkfs_xfs 2>&1 \
 532                         || _fail "          !!! failed to mkfs TEST_DEV"
 533
 534             _log "          *** mounting TEST_DEV"
 535             _test_mount 2>&1 \
 536                         || _fail "          !!! failed to mount"
 537
 538             new_state="run"
 539             ;;
 540
 541         soak-run)
 542             cd $QADIR
 543
 544             _log "          *** run soak test"
 545             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 546                         || _fail "          !!! failed to run soak test"
 547
 548             new_state="done"
 549             ;;
 550
 551         bench-run)
 552             cd $QADIR
 553
 554             # $BENCHMARK is typically unset, which equates to "all"
 555             #
 556             _log "          *** run benchmarks"
 557             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 558                         || _fail "          !!! failed to run benchmarks"
 559
 560             _log ""
 561             _log "          *** send results mail"
 562             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 563
 564             new_state="done"
 565             ;;
 566
 567         *run)
 568             cd $QADIR
 569
 570             _log "          *** run tests ($CHECK_OPTIONS)"
 571             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 572
 573             _log ""
 574             _log "          *** send status mail"
 575             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 576
 577             new_state="done"
 578             ;;
 579
 580         *done)
 581             _log "*** QA run complete"
 582
 583             _success
 584             ;;
 585
 586         *nothing)
 587             new_state="done"
 588             _log "    *** do nothing"
 589             ;;
 590
 591         *)
 592             _fail "        !!! unknown state $state"
 593             ;;
 594     esac
 595
 596     _log "    *** state $state done (`date`)"
 597     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 598     _change_state $new_state
 599
 600 done