tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # automatic qa system. 31/08/00 dxm@sgi.com
   6 #
   7 # Usage: auto_qa start-state [stop-state]
   8 # Do auto_qa from start-state up to stop-state inclusive
   9 # or if no stop-state given then do it until reach "done" state.
  10 #
  11
  12 _log()
  13 {
  14     echo "$*" >&2
  15     echo "$*" >> $LOG
  16     sync
  17 }
  18
  19 _fail()
  20 {
  21     if [ "$started" = "1" ]
  22     then
  23         echo "auto-qa stopped" | wall
  24         started=0
  25     fi
  26
  27     _log "$*"
  28
  29     # send special email if a cron'd qa run fails
  30     case $state
  31     in
  32         cron*)
  33             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  34         ;;
  35     esac
  36
  37     status=1
  38     exit 1
  39 }
  40
  41 _get_kernel_version()
  42 {
  43     [ -x "$KWORKAREA" ] || return
  44     [ -r "$KWORKAREA/Makefile" ] \
  45         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  46
  47     eval `awk '
  48         BEGIN { FS = "[ \t=]+" }
  49         /^VERSION =/ { a=$2 }
  50         /^PATCHLEVEL =/ { b=$2 }
  51         /^SUBLEVEL =/ { c=$2 }
  52         /^EXTRAVERSION =/ { d=$2 }
  53         END {
  54             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  55         }
  56    ' < $KWORKAREA/Makefile`
  57 }
  58
  59 # this should be constant
  60
  61 ROOT="$HOME/qa"
  62 HOST=`hostname -s`
  63 if [ ! -z "$CVSROOT" ]; then
  64     export WORKAREA="$ROOT/xfs-cmds"
  65 else
  66     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  67 fi
  68
  69
  70 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  71 STATE=$ROOT/qa.state
  72 QADIR="$WORKAREA/xfstests"
  73 SUDO="su -c"
  74 CONFIG="$ROOT/$HOST.config"
  75 COMMON_CONFIG="$QADIR/common.config"
  76 SH="/bin/sh"
  77 LOG="$ROOT/qa.log"
  78
  79 # do some cleanup on exit
  80
  81 _cleanup()
  82 {
  83     umount $SCRATCH_DEV &> /dev/null
  84     umount $TEST_DEV &> /dev/null
  85     if [ "$started" = 1 ]
  86     then
  87         echo "auto-qa stopped" | wall
  88         started=0
  89     fi
  90 }
  91 status=1
  92 trap "_cleanup; exit \$status" 0 1 2 3 15
  93
  94 # clean exit
  95
  96 _success()
  97 {
  98     status=0
  99     exit 0
 100 }
 101
 102 _get_state()
 103 {
 104     state=`cat $STATE`
 105 }
 106
 107 _set_state()
 108 {
 109     echo $1 > $STATE
 110     _get_state
 111 }
 112
 113 _change_state()
 114 {
 115     new=$1
 116
 117     # if have state XXXX-state then preserve XXXX-newstate
 118
 119     case $state
 120     in
 121         *-*)
 122             case $new
 123             in
 124                 *-*)
 125                     _set_state $new
 126                     ;;
 127                 *)
 128                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 129                     ;;
 130             esac
 131             ;;
 132         *)
 133             _set_state $new
 134             ;;
 135     esac
 136 }
 137
 138 _sudo()
 139 {
 140     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 141 }
 142
 143 _restart()
 144 {
 145     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 146 }
 147
 148 _update_workarea()
 149 {
 150     if [ -z "$CVSROOT" ]; then
 151         _log "  *** p_tupdate"
 152         cd "$1"
 153         WORKAREA="$1" p_tupdate 2>&1 \
 154                         || _fail "          !!! p_tupdate failed"
 155
 156         _log "  *** non-trunk files"
 157         cd "$1"
 158         WORKAREA="$1" p_list -c 2>&1 \
 159                         || _fail "          !!! p_list failed"
 160     else
 161         _log "  *** cvs update"
 162         cd "$1"
 163         cvs -z3 update -d
 164     fi
 165 }
 166
 167 _test_mkfs_xfs()
 168 {
 169     TEST_OPTIONS=""
 170     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 171         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 172     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 173         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 174     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 175     mkfs_status=$?
 176     if [ "$USE_BIG_LOOPFS" = yes ]; then
 177         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 178         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 179     fi
 180     return $mkfs_status
 181 }
 182
 183 _test_mount()
 184 {
 185     TEST_OPTIONS=""
 186     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 187         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 188     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 189         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 190     _sudo mount -t xfs $TEST_OPTIONS $* $TEST_DEV $TEST_DIR
 191 }
 192
 193 _i386_install()
 194 {
 195     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 196                 || _fail "          !!! install kernel failed"
 197     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 198                 || _fail "          !!! install kernel failed"
 199     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 200         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 201                 || _fail "          !!! install modules failed"
 202     fi
 203
 204
 205     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 206         _log "  *** reinit lilo"
 207         _sudo /sbin/lilo 2>&1 \
 208                 || _fail "          !!! reinit lilo failed"
 209     fi
 210 }
 211
 212 _i386_restart()
 213 {
 214     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 215         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 216                 || _fail "          !!! lilo failed"
 217     fi
 218 }
 219
 220 _ia64_install()
 221 {
 222     echo not yet implemented
 223 }
 224
 225 _ia64_restart()
 226 {
 227     echo not yet implemented
 228 }
 229
 230 _check_kernel()
 231 {
 232     [ -d "$KWORKAREA" ] || _fail "    !!! QA kernel workarea \"$KWORKAREA\" not found"
 233     [ -r "$CONFIG" ]    || _fail "    !!! Can't read config file $CONFIG"
 234 }
 235
 236
 237 _log "*** XFS QA (`date`)"
 238
 239 _get_state
 240
 241 # check preconditions for starting state
 242 case $1
 243 in
 244     cron-init)
 245         case $state
 246         in
 247             *done)
 248                 ;;
 249             *)
 250                 _fail "    !!! cron-init while not in \"*done\" state"
 251                 ;;
 252         esac
 253         ;;
 254     cron-restarted)
 255         # we don't auto restart after reboot, but cron the restart
 256         # to happen a bit later - it's much easier and safer that way
 257         if [ "$state" != "cron-restarted" ]
 258         then
 259             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 260         fi
 261         ;;
 262 esac
 263
 264 [ -n "$1" ] && _set_state $1
 265 [ -n "$2" ] && stop_state=$2
 266
 267 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 268 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 269 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 270 cd $QADIR
 271 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 272
 273 _get_kernel_version
 274 IMAGE="$BOOT/vmlinuz-$EXTRA"
 275 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 276 MODULES="/lib/modules/$SVERSION"
 277
 278 cd $ROOT
 279
 280 started=1
 281 echo "auto-qa started" | wall
 282
 283 while true
 284 do
 285     _get_state
 286
 287     _log "    *** state $state start (`date`)"
 288     _log "      (user=$USER, host=$HOST)"
 289     new_state=""
 290
 291     start_state=$state
 292
 293     case $state
 294     in
 295         *init)
 296             echo "" > $ROOT/qa.log
 297             echo "" > $ROOT/qa.full
 298             _log "******************************************************"
 299             _log "QA init $VERSION (`date`)"
 300             _log "******************************************************"
 301             new_state="updatetools"
 302             ;;
 303
 304         *updatetools)
 305             _update_workarea "$WORKAREA"
 306             new_state="cleantools"
 307             ;;
 308
 309         *cleantools)
 310             # we need to configure or else we might fail to clean
 311             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 312             do
 313                 [ -d $WORKAREA/$pkg ] || continue
 314                 cd $WORKAREA/$pkg
 315                 _log "  *** clean $pkg tools"
 316                 make realclean 2>&1 \
 317                         || _fail "          !!! clean $pkg failed"
 318             done
 319             new_state="buildtools"
 320             ;;
 321
 322         *buildtools)
 323             _log "      *** build and install tools"
 324             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 325             do
 326                 [ -d $WORKAREA/$pkg ] || continue
 327                 cd $WORKAREA/$pkg
 328
 329                 # use e-fence - but this will only take effect on configure
 330                 make configure 2>&1 \
 331                         || _fail "          !!! configure $pkg failed"
 332                 make default 2>&1 \
 333                         || _fail "          !!! build $pkg failed"
 334
 335                 _sudo make install install-dev 2>&1 \
 336                         || _fail "          !!! install $pkg failed"
 337
 338                 # attr and acl now have install-lib targets as well
 339                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 340                 _sudo make install-lib 2>&1 \
 341                         || _fail "          !!! install $pkg failed"
 342             done
 343
 344             new_state="updatekernel"
 345             ;;
 346
 347         *updatekernel)
 348             _check_kernel
 349             _update_workarea "$KWORKAREA"
 350             new_state="cleankernel"
 351             ;;
 352
 353         *cleankernel)
 354             _check_kernel
 355             _log "      *** clean kernel"
 356             cd "$KWORKAREA"
 357             make mrproper 2>&1 \
 358                         || _fail "          !!! clean kernel failed"
 359
 360             _log "      *** install configuration file"
 361             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 362                         || _fail "          !!! failed to install config"
 363
 364             _log "      *** remove version file"
 365             rm -f include/linux/version.h 2>&1 \
 366                         || _fail "          !!! failed to clean version"
 367
 368             new_state="reconfig"
 369             ;;
 370
 371         *reconfig)
 372             _check_kernel
 373             _log "      *** reconfig kernel"
 374
 375             # we better start from scratch if this fails
 376             _change_state "cleankernel"
 377
 378             cd "$KWORKAREA"
 379             # we want to use default options for any new config options.
 380             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 381                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 382                         || _fail "          !!! reconfig oldconfig failed"
 383             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 384                         || _fail "          !!! reconfig dep failed"
 385
 386             new_state="buildkernel"
 387             ;;
 388
 389         *buildkernel)
 390             _check_kernel
 391             _log "      *** build kernel"
 392             _log "          --- kernel ($IMAGE)"
 393             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 394             _log "          --- modules ($MODULES)"
 395
 396             _change_state "cleankernel" ; # we better start from scratch if this fails
 397
 398             cd "$KWORKAREA"
 399             [ -z "$KTARGET" ] && KTARGET=bzImage
 400             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 401                 KTARGET="$KTARGET modules"
 402             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 403                         || _fail "          !!! build $KTARGET failed"
 404             new_state="install"
 405             ;;
 406
 407         *install)
 408             _check_kernel
 409             _log "      *** blat old modules"
 410             _sudo rm -rf $MODULES
 411
 412             _log "      *** install kernel"
 413             cd "$KWORKAREA"
 414             case `uname -m`
 415             in
 416                 i386|i686)      _i386_install ;;
 417                 ia64)           _ia64_install ;;
 418             esac
 419
 420             new_state="restart"
 421             ;;
 422
 423         *restart)
 424             _log "          *** select qa kernel"
 425             case `uname -m`
 426             in
 427                 i386|i686)      _i386_restart ;;
 428                 ia64)           _ia64_restart ;;
 429             esac
 430
 431             _log "          *** prepare to restart"
 432             _change_state "restarted"
 433
 434             _log "          *** restarting"
 435
 436             _restart # doesn't return
 437             ;;
 438
 439         *restarted)
 440             _log "          *** QA reentered after restart"
 441
 442             new_state="check"
 443             ;;
 444
 445         *check)
 446             uname=`uname -a`
 447             _log "          *** uname $uname"
 448
 449             if [ "$MODULAR" -eq 0 ]; then
 450                 new_state="reset"
 451             else
 452                 new_state="probe"
 453             fi
 454             ;;
 455
 456         *probe)
 457             _log "          *** modules dependencies"
 458             _sudo depmod -a  2>&1 \
 459                         || _fail "          !!! failed to depmod -a"
 460
 461             _log "          *** unmounting XFS mounts"
 462             _sudo umount -a -t xfs 2>&1
 463
 464             _log "          *** removing modules"
 465             for m in xfsidbg xfs kdbm_pg kdbm_vm
 466             do
 467                 _sudo rmmod $m 2> /dev/null
 468             done
 469
 470             _log "          *** installing modules"
 471             _sudo modprobe xfs 2>&1 \
 472                         || _fail "          !!! failed to modprobe xfs"
 473
 474             new_state="reset"
 475             ;;
 476
 477         *reset)
 478
 479             _log "          *** unmounting TEST_DEV"
 480             _sudo umount $TEST_DEV 2>&1
 481
 482             _log "          *** unmounting SCRATCH_DEV"
 483             _sudo umount $SCRATCH_DEV 2>&1
 484
 485             _log "          *** clean TEST_DEV"
 486             _test_mkfs_xfs 2>&1 \
 487                         || _fail "          !!! failed to mkfs TEST_DEV"
 488
 489             _log "          *** mounting TEST_DEV"
 490             _test_mount 2>&1 \
 491                         || _fail "          !!! failed to mount"
 492
 493             new_state="run"
 494             ;;
 495
 496         soak-run)
 497             cd $QADIR
 498
 499             _log "          *** run soak test"
 500             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 501                         || _fail "          !!! failed to run soak test"
 502
 503             new_state="done"
 504             ;;
 505
 506         bench-run)
 507             cd $QADIR
 508
 509             # $BENCHMARK is typically unset, which equates to "all"
 510             #
 511             _log "          *** run benchmarks"
 512             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 513                         || _fail "          !!! failed to run benchmarks"
 514
 515             _log ""
 516             _log "          *** send results mail"
 517             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 518
 519             new_state="done"
 520             ;;
 521
 522         *run)
 523             cd $QADIR
 524
 525             _log "          *** run tests ($CHECK_OPTIONS)"
 526             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 527
 528             _log ""
 529             _log "          *** send status mail"
 530             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 531
 532             new_state="done"
 533             ;;
 534
 535         *done)
 536             _log "*** requested QA state transitions complete"
 537
 538             _success
 539             ;;
 540
 541         *nothing)
 542             new_state="done"
 543             _log "    *** do nothing"
 544             ;;
 545
 546         *)
 547             _fail "        !!! unknown state $state"
 548             ;;
 549     esac
 550
 551     _log "    *** state $state done (`date`)"
 552     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 553
 554     if [ -n "$stop_state" ]
 555     then
 556         # remove hyphen prefixes
 557         s1=`echo $start_state | sed 's/.*-//'`
 558         s2=`echo $stop_state | sed 's/.*-//'`
 559
 560         if [ $s1 = $s2 ]
 561         then
 562             # we have been requested to stop here and not go on
 563             new_state="done"
 564         fi
 565     fi
 566
 567     _change_state $new_state
 568
 569 done