tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2003 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program; if not, write the Free Software Foundation,
  16 # Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17 #
  18 #
  19 # automatic qa system. 31/08/00 dxm@sgi.com
  20 #
  21 # Usage: auto_qa start-state [stop-state]
  22 # Do auto_qa from start-state up to stop-state inclusive
  23 # or if no stop-state given then do it until reach "done" state.
  24 #
  25
  26 _log()
  27 {
  28     echo "$*" >&2
  29     echo "$*" >> $LOG
  30     sync
  31 }
  32
  33 _fail()
  34 {
  35     if [ "$started" = "1" ]
  36     then
  37         echo "auto-qa stopped" | wall
  38         started=0
  39     fi
  40
  41     _log "$*"
  42
  43     # send special email if a cron'd qa run fails
  44     case $state
  45     in
  46         cron*)
  47             mail -s "XFS QA status report" $EMAIL < $LOG 2>&1
  48         ;;
  49     esac
  50
  51     status=1
  52     exit 1
  53 }
  54
  55 _get_kernel_version()
  56 {
  57     [ -x "$KWORKAREA" ] || return
  58     [ -r "$KWORKAREA/Makefile" ] \
  59         || _fail "can't read kernel makefile $KWORKAREA/Makefile"
  60
  61     eval `awk '
  62         BEGIN { FS = "[ \t=]+" }
  63         /^VERSION =/ { a=$2 }
  64         /^PATCHLEVEL =/ { b=$2 }
  65         /^SUBLEVEL =/ { c=$2 }
  66         /^EXTRAVERSION =/ { d=$2 }
  67         END {
  68             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  69         }
  70    ' < $KWORKAREA/Makefile`
  71 }
  72
  73 # this should be constant
  74
  75 ROOT="$HOME/qa"
  76 HOST=`hostname -s`
  77 if [ ! -z "$CVSROOT" ]; then
  78     export WORKAREA="$ROOT/xfs-cmds"
  79 else
  80     [ -z "$WORKAREA" ] && export WORKAREA="$ROOT/xfs-cmds"
  81 fi
  82
  83
  84 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  85 STATE=$ROOT/qa.state
  86 QADIR="$WORKAREA/xfstests"
  87 SUDO="su -c"
  88 CONFIG="$ROOT/$HOST.config"
  89 COMMON_CONFIG="$QADIR/common/config"
  90 SH="/bin/sh"
  91 LOG="$ROOT/qa.log"
  92
  93 # do some cleanup on exit
  94
  95 _cleanup()
  96 {
  97     umount $SCRATCH_DEV &> /dev/null
  98     umount $TEST_DEV &> /dev/null
  99     if [ "$started" = 1 ]
 100     then
 101         echo "auto-qa stopped" | wall
 102         started=0
 103     fi
 104 }
 105 status=1
 106 trap "_cleanup; exit \$status" 0 1 2 3 15
 107
 108 # clean exit
 109
 110 _success()
 111 {
 112     status=0
 113     exit 0
 114 }
 115
 116 _get_state()
 117 {
 118     state=`cat $STATE`
 119 }
 120
 121 _set_state()
 122 {
 123     echo $1 > $STATE
 124     _get_state
 125 }
 126
 127 _change_state()
 128 {
 129     new=$1
 130
 131     # if have state XXXX-state then preserve XXXX-newstate
 132
 133     case $state
 134     in
 135         *-*)
 136             case $new
 137             in
 138                 *-*)
 139                     _set_state $new
 140                     ;;
 141                 *)
 142                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 143                     ;;
 144             esac
 145             ;;
 146         *)
 147             _set_state $new
 148             ;;
 149     esac
 150 }
 151
 152 _sudo()
 153 {
 154     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 155 }
 156
 157 _restart()
 158 {
 159     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 160 }
 161
 162 _update_workarea()
 163 {
 164     if [ -z "$CVSROOT" ]; then
 165         _log "  *** p_tupdate"
 166         cd "$1"
 167         WORKAREA="$1" p_tupdate 2>&1 \
 168                         || _fail "          !!! p_tupdate failed"
 169
 170         _log "  *** non-trunk files"
 171         cd "$1"
 172         WORKAREA="$1" p_list -c 2>&1 \
 173                         || _fail "          !!! p_list failed"
 174     else
 175         _log "  *** cvs update"
 176         cd "$1"
 177         cvs -z3 update -d
 178     fi
 179 }
 180
 181 _test_mkfs_xfs()
 182 {
 183     TEST_OPTIONS=""
 184     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 185         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 186     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 187         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 188     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 189     mkfs_status=$?
 190     if [ "$USE_BIG_LOOPFS" = yes ]; then
 191         [ -z "$RETAIN_AG_BYTES" ] && RETAIN_AG_BYTES=0
 192         _sudo $QADIR/tools/ag-wipe -q -r $RETAIN_AG_BYTES $TEST_DEV
 193     fi
 194     return $mkfs_status
 195 }
 196
 197 _test_mount()
 198 {
 199     TEST_OPTIONS=""
 200     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 201         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 202     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 203         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 204     _sudo mount -t xfs $TEST_OPTIONS $MOUNT_OPTIONS $* $TEST_DEV $TEST_DIR
 205 }
 206
 207 _i386_install()
 208 {
 209     _sudo cp -f "$KWORKAREA/arch/i386/boot/bzImage" "$IMAGE" 2>&1 \
 210                 || _fail "          !!! install kernel failed"
 211     _sudo cp -f "$KWORKAREA/System.map" "$SYSTEMMAP" 2>&1 \
 212                 || _fail "          !!! install kernel failed"
 213     if [ -z "$KMODULES" -o "$KMODULES" = yes ]; then
 214         _sudo make EXTRAVERSION=-$EXTRA modules_install 2>&1 \
 215                 || _fail "          !!! install modules failed"
 216     fi
 217
 218
 219     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 220         _log "  *** reinit lilo"
 221         _sudo /sbin/lilo 2>&1 \
 222                 || _fail "          !!! reinit lilo failed"
 223     fi
 224 }
 225
 226 _i386_restart()
 227 {
 228     if [ -z "$KINSTALL" -o "$KINSTALL" = lilo ]; then
 229         _sudo /sbin/lilo -R $EXTRA $KERNEL_OPTIONS 2>&1 \
 230                 || _fail "          !!! lilo failed"
 231     fi
 232 }
 233
 234 _ia64_install()
 235 {
 236     echo not yet implemented
 237 }
 238
 239 _ia64_restart()
 240 {
 241     echo not yet implemented
 242 }
 243
 244 _check_kernel()
 245 {
 246     [ -d "$KWORKAREA" ] || _fail "    !!! QA kernel workarea \"$KWORKAREA\" not found"
 247     [ -r "$CONFIG" ]    || _fail "    !!! Can't read config file $CONFIG"
 248 }
 249
 250
 251 _log "*** XFS QA (`date`)"
 252
 253 _get_state
 254
 255 # check preconditions for starting state
 256 case $1
 257 in
 258     cron-init)
 259         case $state
 260         in
 261             *done)
 262                 ;;
 263             *)
 264                 _fail "    !!! cron-init while not in \"*done\" state"
 265                 ;;
 266         esac
 267         ;;
 268     cron-restarted)
 269         # we don't auto restart after reboot, but cron the restart
 270         # to happen a bit later - it's much easier and safer that way
 271         if [ "$state" != "cron-restarted" ]
 272         then
 273             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 274         fi
 275         ;;
 276 esac
 277
 278 [ -n "$1" ] && _set_state $1
 279 [ -n "$2" ] && stop_state=$2
 280
 281 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 282 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 283 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 284 cd $QADIR
 285 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 286
 287 _get_kernel_version
 288 IMAGE="$BOOT/vmlinuz-$EXTRA"
 289 SYSTEMMAP="$BOOT/System.map-$VERSION-$EXTRA"
 290 MODULES="/lib/modules/$SVERSION"
 291
 292 cd $ROOT
 293
 294 started=1
 295 echo "auto-qa started" | wall
 296
 297 while true
 298 do
 299     _get_state
 300
 301     _log "    *** state $state start (`date`)"
 302     _log "      (user=$USER, host=$HOST)"
 303     new_state=""
 304
 305     start_state=$state
 306
 307     case $state
 308     in
 309         *init)
 310             echo "" > $ROOT/qa.log
 311             echo "" > $ROOT/qa.full
 312             _log "******************************************************"
 313             _log "QA init $VERSION (`date`)"
 314             _log "******************************************************"
 315             new_state="updatetools"
 316             ;;
 317
 318         *updatetools)
 319             _update_workarea "$WORKAREA"
 320             new_state="cleantools"
 321             ;;
 322
 323         *cleantools)
 324             # we need to configure or else we might fail to clean
 325             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 326             do
 327                 [ -d $WORKAREA/$pkg ] || continue
 328                 cd $WORKAREA/$pkg
 329                 _log "  *** clean $pkg tools"
 330                 make realclean 2>&1 \
 331                         || _fail "          !!! clean $pkg failed"
 332             done
 333             new_state="buildtools"
 334             ;;
 335
 336         *buildtools)
 337             _log "      *** build and install tools"
 338             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 339             do
 340                 [ -d $WORKAREA/$pkg ] || continue
 341                 cd $WORKAREA/$pkg
 342
 343                 make configure 2>&1 \
 344                         || _fail "          !!! configure $pkg failed"
 345                 make default 2>&1 \
 346                         || _fail "          !!! build $pkg failed"
 347
 348                 _sudo make install install-dev 2>&1 \
 349                         || _fail "          !!! install $pkg failed"
 350
 351                 # attr and acl now have install-lib targets as well
 352                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 353                 _sudo make install-lib 2>&1 \
 354                         || _fail "          !!! install $pkg failed"
 355             done
 356
 357             new_state="updatekernel"
 358             ;;
 359
 360         *updatekernel)
 361             _check_kernel
 362             _update_workarea "$KWORKAREA"
 363             new_state="cleankernel"
 364             ;;
 365
 366         *cleankernel)
 367             _check_kernel
 368             _log "      *** clean kernel"
 369             cd "$KWORKAREA"
 370             make mrproper 2>&1 \
 371                         || _fail "          !!! clean kernel failed"
 372
 373             _log "      *** install configuration file"
 374             cp -f $CONFIG "$KWORKAREA/.config" 2>&1 \
 375                         || _fail "          !!! failed to install config"
 376
 377             _log "      *** remove version file"
 378             rm -f include/linux/version.h 2>&1 \
 379                         || _fail "          !!! failed to clean version"
 380
 381             new_state="reconfig"
 382             ;;
 383
 384         *reconfig)
 385             _check_kernel
 386             _log "      *** reconfig kernel"
 387
 388             # we better start from scratch if this fails
 389             _change_state "cleankernel"
 390
 391             cd "$KWORKAREA"
 392             # we want to use default options for any new config options.
 393             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 394                 make EXTRAVERSION=-$EXTRA oldconfig 2>&1 \
 395                         || _fail "          !!! reconfig oldconfig failed"
 396             make EXTRAVERSION=-$EXTRA dep 2>&1 \
 397                         || _fail "          !!! reconfig dep failed"
 398
 399             new_state="buildkernel"
 400             ;;
 401
 402         *buildkernel)
 403             _check_kernel
 404             _log "      *** build kernel"
 405             _log "          --- kernel ($IMAGE)"
 406             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 407             _log "          --- modules ($MODULES)"
 408
 409             _change_state "cleankernel" ; # we better start from scratch if this fails
 410
 411             cd "$KWORKAREA"
 412             [ -z "$KTARGET" ] && KTARGET=bzImage
 413             [ -z "$KMODULES" -o "$KMODULES" = yes ] && \
 414                 KTARGET="$KTARGET modules"
 415             make -j4 EXTRAVERSION=-$EXTRA $KTARGET 2>&1 \
 416                         || _fail "          !!! build $KTARGET failed"
 417             new_state="install"
 418             ;;
 419
 420         *install)
 421             _check_kernel
 422             _log "      *** blat old modules"
 423             _sudo rm -rf $MODULES
 424
 425             _log "      *** install kernel"
 426             cd "$KWORKAREA"
 427             case `uname -m`
 428             in
 429                 i386|i686)      _i386_install ;;
 430                 ia64)           _ia64_install ;;
 431             esac
 432
 433             new_state="restart"
 434             ;;
 435
 436         *restart)
 437             _log "          *** select qa kernel"
 438             case `uname -m`
 439             in
 440                 i386|i686)      _i386_restart ;;
 441                 ia64)           _ia64_restart ;;
 442             esac
 443
 444             _log "          *** prepare to restart"
 445             _change_state "restarted"
 446
 447             _log "          *** restarting"
 448
 449             _restart # doesn't return
 450             ;;
 451
 452         *restarted)
 453             _log "          *** QA reentered after restart"
 454
 455             new_state="check"
 456             ;;
 457
 458         *check)
 459             uname=`uname -a`
 460             _log "          *** uname $uname"
 461
 462             if [ "$MODULAR" -eq 0 ]; then
 463                 new_state="reset"
 464             else
 465                 new_state="probe"
 466             fi
 467             ;;
 468
 469         *probe)
 470             _log "          *** modules dependencies"
 471             _sudo depmod -a  2>&1 \
 472                         || _fail "          !!! failed to depmod -a"
 473
 474             _log "          *** unmounting XFS mounts"
 475             _sudo umount -a -t xfs 2>&1
 476
 477             _log "          *** removing modules"
 478             for m in xfsidbg xfs kdbm_pg kdbm_vm
 479             do
 480                 _sudo rmmod $m 2> /dev/null
 481             done
 482
 483             _log "          *** installing modules"
 484             _sudo modprobe xfs 2>&1 \
 485                         || _fail "          !!! failed to modprobe xfs"
 486
 487             new_state="reset"
 488             ;;
 489
 490         *reset)
 491
 492             _log "          *** unmounting TEST_DEV"
 493             _sudo umount $TEST_DEV 2>&1
 494
 495             _log "          *** unmounting SCRATCH_DEV"
 496             _sudo umount $SCRATCH_DEV 2>&1
 497
 498             _log "          *** clean TEST_DEV"
 499             _test_mkfs_xfs 2>&1 \
 500                         || _fail "          !!! failed to mkfs TEST_DEV"
 501
 502             _log "          *** mounting TEST_DEV"
 503             _test_mount 2>&1 \
 504                         || _fail "          !!! failed to mount"
 505
 506             new_state="run"
 507             ;;
 508
 509         soak-run)
 510             cd $QADIR
 511
 512             _log "          *** run soak test"
 513             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 514                         || _fail "          !!! failed to run soak test"
 515
 516             new_state="done"
 517             ;;
 518
 519         bench-run)
 520             cd $QADIR
 521
 522             # $BENCHMARK is typically unset, which equates to "all"
 523             #
 524             _log "          *** run benchmarks"
 525             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 526                         || _fail "          !!! failed to run benchmarks"
 527
 528             _log ""
 529             _log "          *** send results mail"
 530             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 531
 532             new_state="done"
 533             ;;
 534
 535         *run)
 536             cd $QADIR
 537
 538             _log "          *** run tests ($CHECK_OPTIONS)"
 539             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 540
 541             _log ""
 542             _log "          *** send status mail"
 543             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 544
 545             new_state="done"
 546             ;;
 547
 548         *done)
 549             _log "*** requested QA state transitions complete"
 550
 551             _success
 552             ;;
 553
 554         *nothing)
 555             new_state="done"
 556             _log "    *** do nothing"
 557             ;;
 558
 559         *)
 560             _fail "        !!! unknown state $state"
 561             ;;
 562     esac
 563
 564     _log "    *** state $state done (`date`)"
 565     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 566
 567     if [ -n "$stop_state" ]
 568     then
 569         # remove hyphen prefixes
 570         s1=`echo $start_state | sed 's/.*-//'`
 571         s2=`echo $stop_state | sed 's/.*-//'`
 572
 573         if [ $s1 = $s2 ]
 574         then
 575             # we have been requested to stop here and not go on
 576             new_state="done"
 577         fi
 578     fi
 579
 580     _change_state $new_state
 581
 582 done