tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of version 2 of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 #
  13 # Further, this software is distributed without any warranty that it is
  14 # free of the rightful claim of any third person regarding infringement
  15 # or the like.  Any license provided herein, whether implied or
  16 # otherwise, applies only to this software file.  Patent licenses, if
  17 # any, provided herein do not apply to combinations of this program with
  18 # other software, or any other product whatsoever.
  19 #
  20 # You should have received a copy of the GNU General Public License along
  21 # with this program; if not, write the Free Software Foundation, Inc., 59
  22 # Temple Place - Suite 330, Boston MA 02111-1307, USA.
  23 #
  24 # Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  25 # Mountain View, CA  94043, or:
  26 #
  27 # http://www.sgi.com
  28 #
  29 # For further information regarding this notice, see:
  30 #
  31 # http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  32 #
  33
  34 # automatic qa system. 31/08/00 dxm@sgi.com
  35 #
  36 # Usage: auto_qa start-state [stop-state]
  37 # Do auto_qa from start-state up to stop-state inclusive
  38 # or if no stop-state given then do it until reach "done" state.
  39 #
  40
  41 _log()
  42 {
  43     echo "$*" >&2
  44     echo "$*" >> $LOG
  45     sync
  46 }
  47
  48 _fail()
  49 {
  50     if [ "$started" = "1" ]
  51     then
  52         echo "auto-qa stopped" | wall
  53         started=0
  54     fi
  55
  56     _log "$*"
  57
  58     # send special email if a cron'd qa run fails
  59     case $state
  60     in
  61         cron*)
  62             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  63         ;;
  64     esac
  65
  66     status=1
  67     exit 1
  68 }
  69
  70 _get_kernel_version()
  71 {
  72     [ -x "$KWORKAREA" ] || return
  73     [ -r "$KWORKAREA/Makefile" ] \
  74         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  75
  76     eval `awk '
  77         BEGIN { FS = "[ \t=]+" }
  78         /^VERSION =/ { a=$2 }
  79         /^PATCHLEVEL =/ { b=$2 }
  80         /^SUBLEVEL =/ { c=$2 }
  81         /^EXTRAVERSION =/ { d=$2 }
  82         END {
  83             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  84         }
  85    ' < $KWORKAREA/Makefile`
  86 }
  87
  88 # this should be constant
  89
  90 ROOT="$HOME/qa"
  91 HOST=`hostname -s`
  92 if [ ! -z "$CVSROOT" ]; then
  93     export WORKAREA="$ROOT/xfs-cmds"
  94 else
  95     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  96 fi
  97
  98
  99 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
 100 STATE=$ROOT/qa.state
 101 QADIR="$WORKAREA/xfstests"
 102 SUDO="su -c"
 103 CONFIG="$ROOT/$HOST.config"
 104 COMMON_CONFIG="$QADIR/common.config"
 105 SH="/bin/sh"
 106 LOG="$ROOT/qa.log"
 107
 108 # do some cleanup on exit
 109
 110 _cleanup()
 111 {
 112     umount $SCRATCH_DEV &> /dev/null
 113     umount $TEST_DEV &> /dev/null
 114     if [ "$started" = 1 ]
 115     then
 116         echo "auto-qa stopped" | wall
 117         started=0
 118     fi
 119 }
 120 status=1
 121 trap "_cleanup; exit \$status" 0 1 2 3 15
 122
 123 # clean exit
 124
 125 _success()
 126 {
 127     status=0
 128     exit 0
 129 }
 130
 131 _get_state()
 132 {
 133     state=`cat $STATE`
 134 }
 135
 136 _set_state()
 137 {
 138     echo $1 > $STATE
 139     _get_state
 140 }
 141
 142 _change_state()
 143 {
 144     new=$1
 145
 146     # if have state XXXX-state then preserve XXXX-newstate
 147
 148     case $state
 149     in
 150         *-*)
 151             case $new
 152             in
 153                 *-*)
 154                     _set_state $new
 155                     ;;
 156                 *)
 157                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 158                     ;;
 159             esac
 160             ;;
 161         *)
 162             _set_state $new
 163             ;;
 164     esac
 165 }
 166
 167 _sudo()
 168 {
 169     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 170 }
 171
 172 _restart()
 173 {
 174     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 175 }
 176
 177 _update_workarea()
 178 {
 179     if [ -z "$CVSROOT" ]; then
 180         _log "  *** p_tupdate"
 181         cd "$1"
 182         WORKAREA="$1" p_tupdate 2>&1 \
 183                         || _fail "          !!! p_tupdate failed"
 184
 185         _log "  *** non-trunk files"
 186         cd "$1"
 187         WORKAREA="$1" p_list -c 2>&1 \
 188                         || _fail "          !!! p_list failed"
 189     else
 190         _log "  *** cvs update"
 191         cd "$1"
 192         cvs -z3 update -d
 193     fi
 194 }
 195
 196 _test_mkfs_xfs()
 197 {
 198     TEST_OPTIONS=""
 199     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 200         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 201     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 202         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 203     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 204     mkfs_status=$?
 205     if [ "$USE_BIG_LOOPFS" = yes ]; then
 206         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 207         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 208     fi
 209     return $mkfs_status
 210 }
 211
 212 _test_mount()
 213 {
 214     TEST_OPTIONS=""
 215     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 216         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 217     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 218         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 219     _sudo mount -t xfs $TEST_OPTIONS $* $TEST_DEV $TEST_DIR
 220 }
 221
 222 _i386_install()
 223 {
 224     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 225                 || _fail "          !!! install kernel failed"
 226     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 227                 || _fail "          !!! install kernel failed"
 228     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 229         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 230                 || _fail "          !!! install modules failed"
 231     fi
 232
 233
 234     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 235         _log "  *** reinit lilo"
 236         _sudo /sbin/lilo 2>&1 \
 237                 || _fail "          !!! reinit lilo failed"
 238     fi
 239 }
 240
 241 _i386_restart()
 242 {
 243     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 244         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 245                 || _fail "          !!! lilo failed"
 246     fi
 247 }
 248
 249 _ia64_install()
 250 {
 251     echo not yet implemented
 252 }
 253
 254 _ia64_restart()
 255 {
 256     echo not yet implemented
 257 }
 258
 259 _check_kernel()
 260 {
 261     [ -d "$KWORKAREA" ] || _fail "    !!! QA kernel workarea \"$KWORKAREA\" not found"
 262     [ -r "$CONFIG" ]    || _fail "    !!! Can't read config file $CONFIG"
 263 }
 264
 265
 266 _log "*** XFS QA (`date`)"
 267
 268 _get_state
 269
 270 # check preconditions for starting state
 271 case $1
 272 in
 273     cron-init)
 274         case $state
 275         in
 276             *done)
 277                 ;;
 278             *)
 279                 _fail "    !!! cron-init while not in \"*done\" state"
 280                 ;;
 281         esac
 282         ;;
 283     cron-restarted)
 284         # we don't auto restart after reboot, but cron the restart
 285         # to happen a bit later - it's much easier and safer that way
 286         if [ "$state" != "cron-restarted" ]
 287         then
 288             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 289         fi
 290         ;;
 291 esac
 292
 293 [ -n "$1" ] && _set_state $1
 294 [ -n "$2" ] && stop_state=$2
 295
 296 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 297 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 298 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 299 cd $QADIR
 300 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 301
 302 _get_kernel_version
 303 IMAGE="$BOOT/vmlinuz-$EXTRA"
 304 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 305 MODULES="/lib/modules/$SVERSION"
 306
 307 cd $ROOT
 308
 309 started=1
 310 echo "auto-qa started" | wall
 311
 312 while true
 313 do
 314     _get_state
 315
 316     _log "    *** state $state start (`date`)"
 317     _log "      (user=$USER, host=$HOST)"
 318     new_state=""
 319
 320     start_state=$state
 321
 322     case $state
 323     in
 324         *init)
 325             echo "" > $ROOT/qa.log
 326             echo "" > $ROOT/qa.full
 327             _log "******************************************************"
 328             _log "QA init $VERSION (`date`)"
 329             _log "******************************************************"
 330             new_state="updatetools"
 331             ;;
 332
 333         *updatetools)
 334             _update_workarea "$WORKAREA"
 335             new_state="cleantools"
 336             ;;
 337
 338         *cleantools)
 339             # we need to configure or else we might fail to clean
 340             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 341             do
 342                 [ -d $WORKAREA/$pkg ] || continue
 343                 cd $WORKAREA/$pkg
 344                 _log "  *** clean $pkg tools"
 345                 make realclean 2>&1 \
 346                         || _fail "          !!! clean $pkg failed"
 347             done
 348             new_state="buildtools"
 349             ;;
 350
 351         *buildtools)
 352             _log "      *** build and install tools"
 353             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 354             do
 355                 [ -d $WORKAREA/$pkg ] || continue
 356                 cd $WORKAREA/$pkg
 357
 358                 # use e-fence - but this will only take effect on configure
 359                 make configure 2>&1 \
 360                         || _fail "          !!! configure $pkg failed"
 361                 make default 2>&1 \
 362                         || _fail "          !!! build $pkg failed"
 363
 364                 _sudo make install install-dev 2>&1 \
 365                         || _fail "          !!! install $pkg failed"
 366
 367                 # attr and acl now have install-lib targets as well
 368                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 369                 _sudo make install-lib 2>&1 \
 370                         || _fail "          !!! install $pkg failed"
 371             done
 372
 373             new_state="updatekernel"
 374             ;;
 375
 376         *updatekernel)
 377             _check_kernel
 378             _update_workarea "$KWORKAREA"
 379             new_state="cleankernel"
 380             ;;
 381
 382         *cleankernel)
 383             _check_kernel
 384             _log "      *** clean kernel"
 385             cd "$KWORKAREA"
 386             make mrproper 2>&1 \
 387                         || _fail "          !!! clean kernel failed"
 388
 389             _log "      *** install configuration file"
 390             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 391                         || _fail "          !!! failed to install config"
 392
 393             _log "      *** remove version file"
 394             rm -f include/linux/version.h 2>&1 \
 395                         || _fail "          !!! failed to clean version"
 396
 397             new_state="reconfig"
 398             ;;
 399
 400         *reconfig)
 401             _check_kernel
 402             _log "      *** reconfig kernel"
 403
 404             # we better start from scratch if this fails
 405             _change_state "cleankernel"
 406
 407             cd "$KWORKAREA"
 408             # we want to use default options for any new config options.
 409             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 410                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 411                         || _fail "          !!! reconfig oldconfig failed"
 412             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 413                         || _fail "          !!! reconfig dep failed"
 414
 415             new_state="buildkernel"
 416             ;;
 417
 418         *buildkernel)
 419             _check_kernel
 420             _log "      *** build kernel"
 421             _log "          --- kernel ($IMAGE)"
 422             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 423             _log "          --- modules ($MODULES)"
 424
 425             _change_state "cleankernel" ; # we better start from scratch if this fails
 426
 427             cd "$KWORKAREA"
 428             [ -z "$KTARGET" ] && KTARGET=bzImage
 429             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 430                 KTARGET="$KTARGET modules"
 431             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 432                         || _fail "          !!! build $KTARGET failed"
 433             new_state="install"
 434             ;;
 435
 436         *install)
 437             _check_kernel
 438             _log "      *** blat old modules"
 439             _sudo rm -rf $MODULES
 440
 441             _log "      *** install kernel"
 442             cd "$KWORKAREA"
 443             case `uname -m`
 444             in
 445                 i386|i686)      _i386_install ;;
 446                 ia64)           _ia64_install ;;
 447             esac
 448
 449             new_state="restart"
 450             ;;
 451
 452         *restart)
 453             _log "          *** select qa kernel"
 454             case `uname -m`
 455             in
 456                 i386|i686)      _i386_restart ;;
 457                 ia64)           _ia64_restart ;;
 458             esac
 459
 460             _log "          *** prepare to restart"
 461             _change_state "restarted"
 462
 463             _log "          *** restarting"
 464
 465             _restart # doesn't return
 466             ;;
 467
 468         *restarted)
 469             _log "          *** QA reentered after restart"
 470
 471             new_state="check"
 472             ;;
 473
 474         *check)
 475             uname=`uname -a`
 476             _log "          *** uname $uname"
 477
 478             if [ "$MODULAR" -eq 0 ]; then
 479                 new_state="reset"
 480             else
 481                 new_state="probe"
 482             fi
 483             ;;
 484
 485         *probe)
 486             _log "          *** modules dependencies"
 487             _sudo depmod -a  2>&1 \
 488                         || _fail "          !!! failed to depmod -a"
 489
 490             _log "          *** unmounting XFS mounts"
 491             _sudo umount -a -t xfs 2>&1
 492
 493             _log "          *** removing modules"
 494             for m in xfsidbg xfs kdbm_pg kdbm_vm
 495             do
 496                 _sudo rmmod $m 2> /dev/null
 497             done
 498
 499             _log "          *** installing modules"
 500             _sudo modprobe xfs 2>&1 \
 501                         || _fail "          !!! failed to modprobe xfs"
 502
 503             new_state="reset"
 504             ;;
 505
 506         *reset)
 507
 508             _log "          *** unmounting TEST_DEV"
 509             _sudo umount $TEST_DEV 2>&1
 510
 511             _log "          *** unmounting SCRATCH_DEV"
 512             _sudo umount $SCRATCH_DEV 2>&1
 513
 514             _log "          *** clean TEST_DEV"
 515             _test_mkfs_xfs 2>&1 \
 516                         || _fail "          !!! failed to mkfs TEST_DEV"
 517
 518             _log "          *** mounting TEST_DEV"
 519             _test_mount 2>&1 \
 520                         || _fail "          !!! failed to mount"
 521
 522             new_state="run"
 523             ;;
 524
 525         soak-run)
 526             cd $QADIR
 527
 528             _log "          *** run soak test"
 529             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 530                         || _fail "          !!! failed to run soak test"
 531
 532             new_state="done"
 533             ;;
 534
 535         bench-run)
 536             cd $QADIR
 537
 538             # $BENCHMARK is typically unset, which equates to "all"
 539             #
 540             _log "          *** run benchmarks"
 541             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 542                         || _fail "          !!! failed to run benchmarks"
 543
 544             _log ""
 545             _log "          *** send results mail"
 546             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 547
 548             new_state="done"
 549             ;;
 550
 551         *run)
 552             cd $QADIR
 553
 554             _log "          *** run tests ($CHECK_OPTIONS)"
 555             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 556
 557             _log ""
 558             _log "          *** send status mail"
 559             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 560
 561             new_state="done"
 562             ;;
 563
 564         *done)
 565             _log "*** requested QA state transitions complete"
 566
 567             _success
 568             ;;
 569
 570         *nothing)
 571             new_state="done"
 572             _log "    *** do nothing"
 573             ;;
 574
 575         *)
 576             _fail "        !!! unknown state $state"
 577             ;;
 578     esac
 579
 580     _log "    *** state $state done (`date`)"
 581     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 582
 583     if [ -n "$stop_state" ]
 584     then
 585         # remove hyphen prefixes
 586         s1=`echo $start_state | sed 's/.*-//'`
 587         s2=`echo $stop_state | sed 's/.*-//'`
 588
 589         if [ $s1 = $s2 ]
 590         then
 591             # we have been requested to stop here and not go on
 592             new_state="done"
 593         fi
 594     fi
 595
 596     _change_state $new_state
 597
 598 done