tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of version 2 of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 #
  13 # Further, this software is distributed without any warranty that it is
  14 # free of the rightful claim of any third person regarding infringement
  15 # or the like.  Any license provided herein, whether implied or
  16 # otherwise, applies only to this software file.  Patent licenses, if
  17 # any, provided herein do not apply to combinations of this program with
  18 # other software, or any other product whatsoever.
  19 #
  20 # You should have received a copy of the GNU General Public License along
  21 # with this program; if not, write the Free Software Foundation, Inc., 59
  22 # Temple Place - Suite 330, Boston MA 02111-1307, USA.
  23 #
  24 # Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  25 # Mountain View, CA  94043, or:
  26 #
  27 # http://www.sgi.com
  28 #
  29 # For further information regarding this notice, see:
  30 #
  31 # http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  32 #
  33
  34 # automatic qa system. 31/08/00 dxm@sgi.com
  35
  36 # configuration (you could tune this)
  37
  38 _log()
  39 {
  40     echo "$*" >&2
  41     echo "$*" >> $LOG
  42     sync
  43 }
  44
  45 _fail()
  46 {
  47     if [ "$started" = "1" ]
  48     then
  49         echo "auto-qa stopped" | wall
  50         started=0
  51     fi
  52
  53     _log "$*"
  54
  55     # send special email if a cron'd qa run fails
  56     case $state
  57     in
  58         cron*)
  59             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  60         ;;
  61     esac
  62
  63     status=1
  64     exit 1
  65 }
  66
  67 _get_kernel_version()
  68 {
  69     [ -x "$KWORKAREA" ] || return
  70     [ -r "$KWORKAREA/Makefile" ] \
  71         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  72
  73     eval `awk '
  74         BEGIN { FS = "[ \t=]+" }
  75         /^VERSION =/ { a=$2 }
  76         /^PATCHLEVEL =/ { b=$2 }
  77         /^SUBLEVEL =/ { c=$2 }
  78         /^EXTRAVERSION =/ { d=$2 }
  79         END {
  80             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  81         }
  82    ' < $KWORKAREA/Makefile`
  83 }
  84
  85 # this should be constant
  86
  87 ROOT="$HOME/qa"
  88 HOST=`hostname -s`
  89 if [ ! -z "$CVSROOT" ]; then
  90     export WORKAREA="$ROOT/xfs-cmds"
  91 else
  92     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  93 fi
  94
  95
  96 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  97 STATE=$ROOT/qa.state
  98 QADIR="$WORKAREA/xfstests"
  99 SUDO="su -c"
 100 CONFIG="$ROOT/$HOST.config"
 101 COMMON_CONFIG="$WORKAREA/xfstests/common.config"
 102 SH="/bin/sh"
 103 LOG="$ROOT/qa.log"
 104
 105 # do some cleanup on exit
 106
 107 _cleanup()
 108 {
 109     umount $SCRATCH_DEV &> /dev/null
 110     umount $TEST_DEV &> /dev/null
 111     if [ "$started" = 1 ]
 112     then
 113         echo "auto-qa stopped" | wall
 114         started=0
 115     fi
 116 }
 117 status=1
 118 trap "_cleanup; exit \$status" 0 1 2 3 15
 119
 120 # clean exit
 121
 122 _success()
 123 {
 124     status=0
 125     exit 0
 126 }
 127
 128 _get_state()
 129 {
 130     state=`cat $STATE`
 131 }
 132
 133 _set_state()
 134 {
 135     echo $1 > $STATE
 136     _get_state
 137 }
 138
 139 _change_state()
 140 {
 141     new=$1
 142
 143     case $state
 144     in
 145         *-*)
 146             case $new
 147             in
 148                 *-*)
 149                     _set_state $new
 150                     ;;
 151                 *)
 152                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 153                     ;;
 154             esac
 155             ;;
 156         *)
 157             _set_state $new
 158             ;;
 159     esac
 160 }
 161
 162 _sudo()
 163 {
 164     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 165 }
 166
 167 _restart()
 168 {
 169     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 170 }
 171
 172 _update_autoqa_file()
 173 {
 174     SELF="$ROOT/auto-qa"
 175     SELF_UPDATE="xfstests/tools/auto-qa"
 176     if [ -z "$CVSROOT" ]; then
 177             cmd="p_tupdate $SELF_UPDATE"
 178     else
 179             cmd="cvs -z3 update $SELF_UPDATE"
 180     fi
 181     exec $SH -c "cd $WORKAREA ; $cmd; chmod +x $SELF_UPDATE ; exec $SELF"
 182 }
 183
 184 _update_workarea()
 185 {
 186     if [ -z "$CVSROOT" ]; then
 187         _log "  *** p_tupdate"
 188         cd "$1"
 189         WORKAREA="$1" p_tupdate 2>&1 \
 190                         || _fail "          !!! p_tupdate failed"
 191
 192         _log "  *** non-trunk files"
 193         cd "$1"
 194         WORKAREA="$1" p_list -c 2>&1 \
 195                         || _fail "          !!! p_list failed"
 196     else
 197         _log "  *** cvs update"
 198         cd "$1"
 199         cvs -z3 update -d
 200     fi
 201 }
 202
 203 _test_mkfs_xfs()
 204 {
 205     TEST_OPTIONS=""
 206     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 207         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 208     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 209         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 210     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 211     mkfs_status=$?
 212     if [ "$USE_BIG_LOOPFS" = yes ]; then
 213         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 214         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 215     fi
 216     return $mkfs_status
 217 }
 218
 219 _test_mount()
 220 {
 221     TEST_OPTIONS=""
 222     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 223         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 224     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 225         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 226     _sudo mount -t xfs $TEST_OPTIONS $* $TEST_DEV $TEST_DIR
 227 }
 228
 229 _i386_install()
 230 {
 231     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 232                 || _fail "          !!! install kernel failed"
 233     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 234                 || _fail "          !!! install kernel failed"
 235     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 236         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 237                 || _fail "          !!! install modules failed"
 238     fi
 239
 240     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 241         _log "  *** reinit lilo"
 242         _sudo /sbin/lilo 2>&1 \
 243                 || _fail "          !!! reinit lilo failed"
 244     fi
 245 }
 246
 247 _i386_restart()
 248 {
 249     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 250         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 251                 || _fail "          !!! lilo failed"
 252     fi
 253 }
 254
 255 _ia64_install()
 256 {
 257     echo not yet implemented
 258 }
 259
 260 _ia64_restart()
 261 {
 262     echo not yet implemented
 263 }
 264
 265
 266 _log "*** XFS QA (`date`)"
 267
 268 _get_state
 269
 270 # check preconditions for starting state
 271 case $1
 272 in
 273     cron-init)
 274         case $state
 275         in
 276             *done)
 277                 ;;
 278             *)
 279                 _fail "    !!! cron-init while not in \"*done\" state"
 280                 ;;
 281         esac
 282         ;;
 283     cron-restarted)
 284         # we don't auto restart after reboot, but cron the restart
 285         # to happen a bit later - it's much easier and safer that way
 286         if [ "$state" != "cron-restarted" ]
 287         then
 288             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 289         fi
 290         ;;
 291 esac
 292
 293 if [ "$1" != "" ]
 294 then
 295     _set_state $1
 296 fi
 297
 298 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 299 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 300 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 301 [ -r "$CONFIG" ]        || _fail "    !!! Can't read config file $CONFIG"
 302 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 303
 304 _get_kernel_version
 305 IMAGE="$BOOT/vmlinuz-$EXTRA"
 306 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 307 MODULES="/lib/modules/$SVERSION"
 308
 309 cd $ROOT
 310
 311 started=1
 312 echo "auto-qa started" | wall
 313
 314 while true
 315 do
 316     _get_state
 317
 318     _log "    *** state $state start (`date`)"
 319     _log "      (user=$USER, host=$HOST)"
 320     new_state=""
 321
 322     case $state
 323     in
 324         *init)
 325             echo "" > $ROOT/qa.log
 326             echo "" > $ROOT/qa.full
 327             _log "******************************************************"
 328             _log "QA init $VERSION (`date`)"
 329             _log "******************************************************"
 330             _log "--- kernel ($IMAGE)"
 331             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 332                 _log "--- modules ($MODULES)"
 333
 334             _change_state "inited"
 335             _update_autoqa_file
 336             ;;
 337
 338         *inited)
 339             _log "      *** QA initialized"
 340             new_state="update"
 341             ;;
 342
 343         *update)
 344             _update_workarea "$KWORKAREA"
 345             _update_workarea "$WORKAREA"
 346             new_state="cleantools"
 347             ;;
 348
 349         *cleantools)
 350             # we need to configure or else we might fail to clean
 351             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 352             do
 353                 [ -d $WORKAREA/$pkg ] || continue
 354                 cd $WORKAREA/$pkg
 355                 _log "  *** clean $pkg tools"
 356                 make realclean 2>&1 \
 357                         || _fail "          !!! clean $pkg failed"
 358             done
 359             new_state="buildtools"
 360             ;;
 361
 362         *buildtools)
 363             _log "      *** build and install tools"
 364             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 365             do
 366                 [ -d $WORKAREA/$pkg ] || continue
 367                 cd $WORKAREA/$pkg
 368
 369                 # use e-fence - but this will only take effect on configure
 370                 make configure 2>&1 \
 371                         || _fail "          !!! configure $pkg failed"
 372                 make default 2>&1 \
 373                         || _fail "          !!! build $pkg failed"
 374
 375                 _sudo make install install-dev 2>&1 \
 376                         || _fail "          !!! install $pkg failed"
 377
 378                 # attr and acl now have install-lib targets as well
 379                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 380                 _sudo make install-lib 2>&1 \
 381                         || _fail "          !!! install $pkg failed"
 382             done
 383
 384             new_state="cleankernel"
 385             ;;
 386
 387         *cleankernel)
 388             _log "      *** clean kernel"
 389             cd "$KWORKAREA"
 390             make mrproper 2>&1 \
 391                         || _fail "          !!! clean kernel failed"
 392
 393             _log "      *** install configuration file"
 394             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 395                         || _fail "          !!! failed to install config"
 396
 397             _log "      *** remove version file"
 398             rm -f include/linux/version.h 2>&1 \
 399                         || _fail "          !!! failed to clean version"
 400
 401             new_state="reconfig"
 402             ;;
 403
 404         *reconfig)
 405
 406             _log "      *** reconfig kernel"
 407
 408             # we better start from scratch if this fails
 409             _change_state "cleankernel"
 410
 411             cd "$KWORKAREA"
 412             # we want to use default options for any new config options.
 413             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 414                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 415                         || _fail "          !!! reconfig oldconfig failed"
 416             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 417                         || _fail "          !!! reconfig dep failed"
 418
 419             new_state="buildkernel"
 420             ;;
 421
 422         *buildkernel)
 423             _log "      *** build kernel"
 424
 425             _change_state "cleankernel" ; # we better start from scratch if this fails
 426
 427             cd "$KWORKAREA"
 428             [ -z "$KTARGET" ] && KTARGET=bzImage
 429             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 430                 KTARGET="$KTARGET modules"
 431             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 432                         || _fail "          !!! build $KTARGET failed"
 433             new_state="install"
 434             ;;
 435
 436         *install)
 437             _log "      *** blat old modules"
 438             _sudo rm -rf $MODULES
 439
 440             _log "      *** install kernel"
 441             cd "$KWORKAREA"
 442             case `uname -m`
 443             in
 444                 i386|i686)      _i386_install ;;
 445                 ia64)           _ia64_install ;;
 446             esac
 447
 448             if [ -z "$KRESTART" -o "$KRESTART" = yes ]; then
 449                 new_state="restart"
 450             else
 451                 new_state="done"
 452             fi
 453             ;;
 454
 455         *restart)
 456             _log "          *** select qa kernel"
 457             case `uname -m`
 458             in
 459                 i386|i686)      _i386_restart ;;
 460                 ia64)           _ia64_restart ;;
 461             esac
 462
 463             _log "          *** prepare to restart"
 464             _change_state "restarted"
 465
 466             _log "          *** restarting"
 467
 468             _restart # doesn't return
 469             ;;
 470
 471         *restarted)
 472             _log "          *** QA reentered after restart"
 473
 474             new_state="check"
 475             ;;
 476
 477         *check)
 478             uname=`uname -a`
 479             _log "          *** uname $uname"
 480
 481             if [ "$MODULAR" -eq 0 ]; then
 482                 new_state="reset"
 483             else
 484                 new_state="probe"
 485             fi
 486             ;;
 487
 488         *probe)
 489             _log "          *** modules dependencies"
 490             _sudo depmod -a  2>&1 \
 491                         || _fail "          !!! failed to depmod -a"
 492
 493             _log "          *** unmounting XFS mounts"
 494             _sudo umount -a -t xfs 2>&1
 495
 496             _log "          *** removing modules"
 497             for m in xfsidbg xfs kdbm_pg kdbm_vm
 498             do
 499                 _sudo rmmod $m 2> /dev/null
 500             done
 501
 502             _log "          *** installing modules"
 503             _sudo modprobe xfs 2>&1 \
 504                         || _fail "          !!! failed to modprobe xfs"
 505
 506             new_state="reset"
 507             ;;
 508
 509         *reset)
 510
 511             _log "          *** unmounting TEST_DEV"
 512             _sudo umount $TEST_DEV 2>&1
 513
 514             _log "          *** unmounting SCRATCH_DEV"
 515             _sudo umount $SCRATCH_DEV 2>&1
 516
 517             _log "          *** clean TEST_DEV"
 518             _test_mkfs_xfs 2>&1 \
 519                         || _fail "          !!! failed to mkfs TEST_DEV"
 520
 521             _log "          *** mounting TEST_DEV"
 522             _test_mount 2>&1 \
 523                         || _fail "          !!! failed to mount"
 524
 525             new_state="run"
 526             ;;
 527
 528         soak-run)
 529             cd $QADIR
 530
 531             _log "          *** run soak test"
 532             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 533                         || _fail "          !!! failed to run soak test"
 534
 535             new_state="done"
 536             ;;
 537
 538         bench-run)
 539             cd $QADIR
 540
 541             # $BENCHMARK is typically unset, which equates to "all"
 542             #
 543             _log "          *** run benchmarks"
 544             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 545                         || _fail "          !!! failed to run benchmarks"
 546
 547             _log ""
 548             _log "          *** send results mail"
 549             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 550
 551             new_state="done"
 552             ;;
 553
 554         *run)
 555             cd $QADIR
 556
 557             _log "          *** run tests ($CHECK_OPTIONS)"
 558             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 559
 560             _log ""
 561             _log "          *** send status mail"
 562             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 563
 564             new_state="done"
 565             ;;
 566
 567         *done)
 568             _log "*** QA run complete"
 569
 570             _success
 571             ;;
 572
 573         *nothing)
 574             new_state="done"
 575             _log "    *** do nothing"
 576             ;;
 577
 578         *)
 579             _fail "        !!! unknown state $state"
 580             ;;
 581     esac
 582
 583     _log "    *** state $state done (`date`)"
 584     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 585     _change_state $new_state
 586
 587 done