tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program; if not, write the Free Software Foundation,
  16 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 #
  18 #
  19 # automatic qa system. 31/08/00 dxm@sgi.com
  20 #
  21 # Usage: auto_qa start-state [stop-state]
  22 # Do auto_qa from start-state up to stop-state inclusive
  23 # or if no stop-state given then do it until reach "done" state.
  24 #
  25
  26 _log()
  27 {
  28     echo "$*" >&2
  29     echo "$*" >> $LOG
  30     sync
  31 }
  32
  33 _fail()
  34 {
  35     if [ "$started" = "1" ]
  36     then
  37         echo "auto-qa stopped" | wall
  38         started=0
  39     fi
  40
  41     _log "$*"
  42
  43     # send special email if a cron'd qa run fails
  44     case $state
  45     in
  46         cron*)
  47             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  48         ;;
  49     esac
  50
  51     status=1
  52     exit 1
  53 }
  54
  55 _get_kernel_version()
  56 {
  57     [ -x "$KWORKAREA" ] || return
  58     [ -r "$KWORKAREA/Makefile" ] \
  59         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  60
  61     eval `awk '
  62         BEGIN { FS = "[ \t=]+" }
  63         /^VERSION =/ { a=$2 }
  64         /^PATCHLEVEL =/ { b=$2 }
  65         /^SUBLEVEL =/ { c=$2 }
  66         /^EXTRAVERSION =/ { d=$2 }
  67         END {
  68             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  69         }
  70    ' < $KWORKAREA/Makefile`
  71 }
  72
  73 # this should be constant
  74
  75 ROOT="$HOME/qa"
  76 HOST=`hostname -s`
  77 if [ ! -z "$CVSROOT" ]; then
  78     export WORKAREA="$ROOT/xfs-cmds"
  79 else
  80     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  81 fi
  82
  83
  84 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  85 STATE=$ROOT/qa.state
  86 QADIR="$WORKAREA/xfstests"
  87 SUDO="su -c"
  88 CONFIG="$ROOT/$HOST.config"
  89 COMMON_CONFIG="$QADIR/common/config"
  90 SH="/bin/sh"
  91 LOG="$ROOT/qa.log"
  92
  93 # do some cleanup on exit
  94
  95 _cleanup()
  96 {
  97     umount $SCRATCH_DEV &> /dev/null
  98     umount $TEST_DEV &> /dev/null
  99     if [ "$started" = 1 ]
 100     then
 101         echo "auto-qa stopped" | wall
 102         started=0
 103     fi
 104 }
 105 status=1
 106 trap "_cleanup; exit \$status" 0 1 2 3 15
 107
 108 # clean exit
 109
 110 _success()
 111 {
 112     status=0
 113     exit 0
 114 }
 115
 116 _get_state()
 117 {
 118     state=`cat $STATE`
 119 }
 120
 121 _set_state()
 122 {
 123     echo $1 > $STATE
 124     _get_state
 125 }
 126
 127 _change_state()
 128 {
 129     new=$1
 130
 131     # if have state XXXX-state then preserve XXXX-newstate
 132
 133     case $state
 134     in
 135         *-*)
 136             case $new
 137             in
 138                 *-*)
 139                     _set_state $new
 140                     ;;
 141                 *)
 142                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 143                     ;;
 144             esac
 145             ;;
 146         *)
 147             _set_state $new
 148             ;;
 149     esac
 150 }
 151
 152 _sudo()
 153 {
 154     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 155 }
 156
 157 _restart()
 158 {
 159     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 160 }
 161
 162 _update_workarea()
 163 {
 164     if [ -z "$CVSROOT" ]; then
 165         _log "  *** p_tupdate"
 166         cd "$1"
 167         WORKAREA="$1" p_tupdate 2>&1 \
 168                         || _fail "          !!! p_tupdate failed"
 169
 170         _log "  *** non-trunk files"
 171         cd "$1"
 172         WORKAREA="$1" p_list -c 2>&1 \
 173                         || _fail "          !!! p_list failed"
 174     else
 175         _log "  *** cvs update"
 176         cd "$1"
 177         cvs -z3 update -d
 178     fi
 179 }
 180
 181 _test_mkfs_xfs()
 182 {
 183     TEST_OPTIONS=""
 184     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 185         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 186     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 187         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 188     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 189     mkfs_status=$?
 190     if [ "$USE_BIG_LOOPFS" = yes ]; then
 191         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 192         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 193     fi
 194     return $mkfs_status
 195 }
 196
 197 _test_mount()
 198 {
 199     TEST_OPTIONS=""
 200     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 201         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 202     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 203         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 204     _sudo mount -t xfs $TEST_OPTIONS $MOUNT_OPTIONS $* $TEST_DEV $TEST_DIR
 205 }
 206
 207 _i386_install()
 208 {
 209     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 210                 || _fail "          !!! install kernel failed"
 211     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 212                 || _fail "          !!! install kernel failed"
 213     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 214         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 215                 || _fail "          !!! install modules failed"
 216     fi
 217
 218
 219     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 220         _log "  *** reinit lilo"
 221         _sudo /sbin/lilo 2>&1 \
 222                 || _fail "          !!! reinit lilo failed"
 223     fi
 224 }
 225
 226 _i386_restart()
 227 {
 228     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 229         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 230                 || _fail "          !!! lilo failed"
 231     fi
 232 }
 233
 234 _ia64_install()
 235 {
 236     echo not yet implemented
 237 }
 238
 239 _ia64_restart()
 240 {
 241     echo not yet implemented
 242 }
 243
 244 _check_kernel()
 245 {
 246     [ -d "$KWORKAREA" ] || _fail "    !!! QA kernel workarea \"$KWORKAREA\" not found"
 247     [ -r "$CONFIG" ]    || _fail "    !!! Can't read config file $CONFIG"
 248 }
 249
 250
 251 _log "*** XFS QA (`date`)"
 252
 253 _get_state
 254
 255 # check preconditions for starting state
 256 case $1
 257 in
 258     cron-init)
 259         case $state
 260         in
 261             *done)
 262                 ;;
 263             *)
 264                 _fail "    !!! cron-init while not in \"*done\" state"
 265                 ;;
 266         esac
 267         ;;
 268     cron-restarted)
 269         # we don't auto restart after reboot, but cron the restart
 270         # to happen a bit later - it's much easier and safer that way
 271         if [ "$state" != "cron-restarted" ]
 272         then
 273             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 274         fi
 275         ;;
 276 esac
 277
 278 [ -n "$1" ] && _set_state $1
 279 [ -n "$2" ] && stop_state=$2
 280
 281 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 282 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 283 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 284 cd $QADIR
 285 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 286
 287 _get_kernel_version
 288 IMAGE="$BOOT/vmlinuz-$EXTRA"
 289 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 290 MODULES="/lib/modules/$SVERSION"
 291
 292 cd $ROOT
 293
 294 started=1
 295 echo "auto-qa started" | wall
 296
 297 while true
 298 do
 299     _get_state
 300
 301     _log "    *** state $state start (`date`)"
 302     _log "      (user=$USER, host=$HOST)"
 303     new_state=""
 304
 305     start_state=$state
 306
 307     case $state
 308     in
 309         *init)
 310             echo "" > $ROOT/qa.log
 311             echo "" > $ROOT/qa.full
 312             _log "******************************************************"
 313             _log "QA init $VERSION (`date`)"
 314             _log "******************************************************"
 315             new_state="updatetools"
 316             ;;
 317
 318         *updatetools)
 319             _update_workarea "$WORKAREA"
 320             new_state="cleantools"
 321             ;;
 322
 323         *cleantools)
 324             # we need to configure or else we might fail to clean
 325             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 326             do
 327                 [ -d $WORKAREA/$pkg ] || continue
 328                 cd $WORKAREA/$pkg
 329                 _log "  *** clean $pkg tools"
 330                 make realclean 2>&1 \
 331                         || _fail "          !!! clean $pkg failed"
 332             done
 333             new_state="buildtools"
 334             ;;
 335
 336         *buildtools)
 337             _log "      *** build and install tools"
 338             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 339             do
 340                 [ -d $WORKAREA/$pkg ] || continue
 341                 cd $WORKAREA/$pkg
 342
 343                 # use e-fence - but this will only take effect on configure
 344                 make configure 2>&1 \
 345                         || _fail "          !!! configure $pkg failed"
 346                 make default 2>&1 \
 347                         || _fail "          !!! build $pkg failed"
 348
 349                 _sudo make install install-dev 2>&1 \
 350                         || _fail "          !!! install $pkg failed"
 351
 352                 # attr and acl now have install-lib targets as well
 353                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 354                 _sudo make install-lib 2>&1 \
 355                         || _fail "          !!! install $pkg failed"
 356             done
 357
 358             new_state="updatekernel"
 359             ;;
 360
 361         *updatekernel)
 362             _check_kernel
 363             _update_workarea "$KWORKAREA"
 364             new_state="cleankernel"
 365             ;;
 366
 367         *cleankernel)
 368             _check_kernel
 369             _log "      *** clean kernel"
 370             cd "$KWORKAREA"
 371             make mrproper 2>&1 \
 372                         || _fail "          !!! clean kernel failed"
 373
 374             _log "      *** install configuration file"
 375             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 376                         || _fail "          !!! failed to install config"
 377
 378             _log "      *** remove version file"
 379             rm -f include/linux/version.h 2>&1 \
 380                         || _fail "          !!! failed to clean version"
 381
 382             new_state="reconfig"
 383             ;;
 384
 385         *reconfig)
 386             _check_kernel
 387             _log "      *** reconfig kernel"
 388
 389             # we better start from scratch if this fails
 390             _change_state "cleankernel"
 391
 392             cd "$KWORKAREA"
 393             # we want to use default options for any new config options.
 394             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 395                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 396                         || _fail "          !!! reconfig oldconfig failed"
 397             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 398                         || _fail "          !!! reconfig dep failed"
 399
 400             new_state="buildkernel"
 401             ;;
 402
 403         *buildkernel)
 404             _check_kernel
 405             _log "      *** build kernel"
 406             _log "          --- kernel ($IMAGE)"
 407             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 408             _log "          --- modules ($MODULES)"
 409
 410             _change_state "cleankernel" ; # we better start from scratch if this fails
 411
 412             cd "$KWORKAREA"
 413             [ -z "$KTARGET" ] && KTARGET=bzImage
 414             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 415                 KTARGET="$KTARGET modules"
 416             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 417                         || _fail "          !!! build $KTARGET failed"
 418             new_state="install"
 419             ;;
 420
 421         *install)
 422             _check_kernel
 423             _log "      *** blat old modules"
 424             _sudo rm -rf $MODULES
 425
 426             _log "      *** install kernel"
 427             cd "$KWORKAREA"
 428             case `uname -m`
 429             in
 430                 i386|i686)      _i386_install ;;
 431                 ia64)           _ia64_install ;;
 432             esac
 433
 434             new_state="restart"
 435             ;;
 436
 437         *restart)
 438             _log "          *** select qa kernel"
 439             case `uname -m`
 440             in
 441                 i386|i686)      _i386_restart ;;
 442                 ia64)           _ia64_restart ;;
 443             esac
 444
 445             _log "          *** prepare to restart"
 446             _change_state "restarted"
 447
 448             _log "          *** restarting"
 449
 450             _restart # doesn't return
 451             ;;
 452
 453         *restarted)
 454             _log "          *** QA reentered after restart"
 455
 456             new_state="check"
 457             ;;
 458
 459         *check)
 460             uname=`uname -a`
 461             _log "          *** uname $uname"
 462
 463             if [ "$MODULAR" -eq 0 ]; then
 464                 new_state="reset"
 465             else
 466                 new_state="probe"
 467             fi
 468             ;;
 469
 470         *probe)
 471             _log "          *** modules dependencies"
 472             _sudo depmod -a  2>&1 \
 473                         || _fail "          !!! failed to depmod -a"
 474
 475             _log "          *** unmounting XFS mounts"
 476             _sudo umount -a -t xfs 2>&1
 477
 478             _log "          *** removing modules"
 479             for m in xfsidbg xfs kdbm_pg kdbm_vm
 480             do
 481                 _sudo rmmod $m 2> /dev/null
 482             done
 483
 484             _log "          *** installing modules"
 485             _sudo modprobe xfs 2>&1 \
 486                         || _fail "          !!! failed to modprobe xfs"
 487
 488             new_state="reset"
 489             ;;
 490
 491         *reset)
 492
 493             _log "          *** unmounting TEST_DEV"
 494             _sudo umount $TEST_DEV 2>&1
 495
 496             _log "          *** unmounting SCRATCH_DEV"
 497             _sudo umount $SCRATCH_DEV 2>&1
 498
 499             _log "          *** clean TEST_DEV"
 500             _test_mkfs_xfs 2>&1 \
 501                         || _fail "          !!! failed to mkfs TEST_DEV"
 502
 503             _log "          *** mounting TEST_DEV"
 504             _test_mount 2>&1 \
 505                         || _fail "          !!! failed to mount"
 506
 507             new_state="run"
 508             ;;
 509
 510         soak-run)
 511             cd $QADIR
 512
 513             _log "          *** run soak test"
 514             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 515                         || _fail "          !!! failed to run soak test"
 516
 517             new_state="done"
 518             ;;
 519
 520         bench-run)
 521             cd $QADIR
 522
 523             # $BENCHMARK is typically unset, which equates to "all"
 524             #
 525             _log "          *** run benchmarks"
 526             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 527                         || _fail "          !!! failed to run benchmarks"
 528
 529             _log ""
 530             _log "          *** send results mail"
 531             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 532
 533             new_state="done"
 534             ;;
 535
 536         *run)
 537             cd $QADIR
 538
 539             _log "          *** run tests ($CHECK_OPTIONS)"
 540             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 541
 542             _log ""
 543             _log "          *** send status mail"
 544             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 545
 546             new_state="done"
 547             ;;
 548
 549         *done)
 550             _log "*** requested QA state transitions complete"
 551
 552             _success
 553             ;;
 554
 555         *nothing)
 556             new_state="done"
 557             _log "    *** do nothing"
 558             ;;
 559
 560         *)
 561             _fail "        !!! unknown state $state"
 562             ;;
 563     esac
 564
 565     _log "    *** state $state done (`date`)"
 566     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 567
 568     if [ -n "$stop_state" ]
 569     then
 570         # remove hyphen prefixes
 571         s1=`echo $start_state | sed 's/.*-//'`
 572         s2=`echo $stop_state | sed 's/.*-//'`
 573
 574         if [ $s1 = $s2 ]
 575         then
 576             # we have been requested to stop here and not go on
 577             new_state="done"
 578         fi
 579     fi
 580
 581     _change_state $new_state
 582
 583 done