tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of version 2 of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 #
  13 # Further, this software is distributed without any warranty that it is
  14 # free of the rightful claim of any third person regarding infringement
  15 # or the like.  Any license provided herein, whether implied or
  16 # otherwise, applies only to this software file.  Patent licenses, if
  17 # any, provided herein do not apply to combinations of this program with
  18 # other software, or any other product whatsoever.
  19 #
  20 # You should have received a copy of the GNU General Public License along
  21 # with this program; if not, write the Free Software Foundation, Inc., 59
  22 # Temple Place - Suite 330, Boston MA 02111-1307, USA.
  23 #
  24 # Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  25 # Mountain View, CA  94043, or:
  26 #
  27 # http://www.sgi.com
  28 #
  29 # For further information regarding this notice, see:
  30 #
  31 # http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  32 #
  33
  34 # automatic qa system. 31/08/00 dxm@sgi.com
  35
  36 _log()
  37 {
  38     echo "$*" >&2
  39     echo "$*" >> $LOG
  40 }
  41
  42 _fail()
  43 {
  44     if [ "$started" = "1" ]
  45     then
  46         echo "auto-qa stopped" | wall
  47         started=0
  48     fi
  49
  50     _log "$*"
  51
  52     # send special email if a cron'd qa run fails
  53     case $state
  54     in
  55         cron*)
  56             mail -s "xfs qa status report" $ADMINEMAIL \
  57                 < $LOG 2>&1
  58         ;;
  59     esac
  60
  61     status=1
  62     exit 1
  63 }
  64
  65 # configuration (you could tune this)
  66
  67 EXTRA="-xfs-qa"
  68 VERSION="2.4.0"
  69 BOOT="/boot"
  70 SOAK_PASSES="-1"
  71 SOAK_STRESS="10000"
  72 SOAK_PROC="3"
  73
  74 # this should be constant
  75
  76 ROOT="$HOME/qa"
  77 HOST=`hostname -s`
  78 export WORKAREA="$ROOT/linux-xfs"
  79
  80 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
  81 STATE=$ROOT/qa.state
  82 QADIR="$WORKAREA/cmd/xfs/stress"
  83 SUDO="su -c"
  84 IMAGE="$BOOT/vmlinuz-$VERSION$EXTRA"
  85 SYSTEMMAP="$BOOT/System.map-$VERSION$EXTRA"
  86 CONFIG="$ROOT/$HOST-$VERSION$EXTRA.config"
  87 MODULES="/lib/modules/$VERSION$EXTRA"
  88 SELF="$ROOT/auto-qa"
  89 SELF_UPDATE="cmd/xfs/tools/auto-qa"
  90 COMMON_CONFIG="$WORKAREA/cmd/xfs/stress/common.config"
  91 SH="/bin/sh"
  92 LOG="$ROOT/qa.log"
  93
  94 # need to add auto-qa hosts here
  95
  96 case $HOST
  97 in
  98     fuzzy)
  99         EMAIL="dxm@larry"
 100         ADMINEMAIL="dxm@larry"
 101         MODULAR=1
 102         ;;
 103     bruce)
 104         EMAIL="dxm@larry"
 105         ADMINEMAIL="dxm@larry"
 106         MODULAR=1
 107         ;;
 108     sagan)
 109         EMAIL="dxm@larry"
 110         ADMINEMAIL="dxm@larry"
 111         MODULAR=0
 112         ;;
 113     troppo)
 114         EMAIL="nathans@larry"
 115         ADMINEMAIL="nathans@larry"
 116         MODULAR=0
 117         ;;
 118     *)
 119         _fail "auto-qa: no configuration information for host '$HOST'"
 120         ;;
 121 esac
 122
 123 # do some cleanup on exit
 124
 125 _cleanup()
 126 {
 127     umount $SCRATCH_DEV &> /dev/null
 128     umount $TEST_DEV &> /dev/null
 129     if [ "$started" = 1 ]
 130     then
 131         echo "auto-qa stopped" | wall
 132         started=0
 133     fi
 134 }
 135 status=1
 136 trap "_cleanup; exit \$status" 0 1 2 3 15
 137
 138 # clean exit
 139
 140 _success()
 141 {
 142     status=0
 143     exit 0
 144 }
 145
 146 _get_state()
 147 {
 148     state=`cat $STATE`
 149 }
 150
 151 _set_state()
 152 {
 153     echo $1 > $STATE
 154     _get_state
 155 }
 156
 157 _change_state()
 158 {
 159     new=$1
 160
 161     case $state
 162     in
 163         *-*)
 164             case $new
 165             in
 166                 *-*)
 167                     _set_state $new
 168                     ;;
 169                 *)
 170                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 171                     ;;
 172             esac
 173             ;;
 174         *)
 175             _set_state $new
 176             ;;
 177     esac
 178 }
 179
 180 _sudo()
 181 {
 182     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 183 }
 184
 185 _restart()
 186 {
 187     exec $ROOT/su -c "shutdown -r 2 \"auto-qa rebooting\" &" < /dev/null
 188 }
 189
 190 _log "*** linux-xfs QA (`date`)"
 191
 192 _get_state
 193
 194 # check preconditions for starting state
 195 case $1
 196 in
 197     cron-init)
 198         case $state
 199         in
 200             *done)
 201                 ;;
 202             *)
 203                 _fail "    !!! cron-init while not in \"*done\" state"
 204                 ;;
 205         esac
 206         ;;
 207     cron-restarted)
 208         # we don't auto restart after reboot, but cron the restart
 209         # to happen a bit later - it's much easier and safer that way
 210         if [ $state != "cron-restarted" ]
 211         then
 212             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 213         fi
 214         ;;
 215 esac
 216
 217 if [ "$1" != "" ]
 218 then
 219     _set_state $1
 220 fi
 221
 222 [ $UID -eq 0 ]          && _fail "    !!! QA most be run as a normal user"
 223 [ -d $ROOT ]            || _fail "    !!! QA root \"$ROOT\" not found"
 224 [ -d $WORKAREA ]        || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 225 [ -r $CONFIG ]          || _fail "    !!! Can't read config file $CONFIG"
 226 . $COMMON_CONFIG        || _fail "    !!! Couldn't source $COMMON_CONFIG"
 227
 228 cd $ROOT
 229
 230 started=1
 231 echo "auto-qa started" | wall
 232
 233 while true
 234 do
 235     _get_state
 236
 237     _log "    *** state $state start (`date`)"
 238     _log "        (user=$USER, host=$HOST)"
 239     new_state=""
 240
 241     case $state
 242     in
 243         *init)
 244             echo "" > $ROOT/qa.log
 245             echo "" > $ROOT/qa.full
 246             _log "******************************************"
 247             _log "QA init (`date`)"
 248             _log "******************************************"
 249
 250             _change_state "inited"
 251             exec $SH -c "cd $WORKAREA ; p_tupdate $SELF_UPDATE ; chmod +x $SELF_UPDATE ; exec $SELF"
 252             ;;
 253
 254         *inited)
 255             _log "        *** QA initialized"
 256             new_state="update"
 257             ;;
 258
 259         *update)
 260             _log "        *** p_tupdate"
 261             cd $WORKAREA
 262             p_tupdate 2>&1 \
 263                                     || _fail "            !!! p_tupdate failed"
 264
 265             _log "        *** p_check/p_purge"
 266             cd $WORKAREA
 267             p_check -s | p_purge -yiu 2>&1 \
 268                                     || _fail "            !!! p_check/p_purge failed"
 269
 270             _log "        *** non-trunk files"
 271             cd $WORKAREA
 272             p_list -c 2>&1 \
 273                                     || _fail "            !!! p_list failed"
 274
 275             new_state="clean"
 276             ;;
 277
 278         *clean)
 279             # we need to configure or else we might fail to clean stress/src
 280             _log "        *** configure for clean"
 281             cd $WORKAREA/cmd/xfs
 282             make configure 2>&1 \
 283                                     || _fail "            !!! configure for clean failed"
 284
 285             _log "        *** clean stress/src"
 286             cd $WORKAREA/cmd/xfs/stress/src
 287             make clobber 2>&1 \
 288                                     || _fail "            !!! clean stress/src failed"
 289
 290             _log "        *** clean xfs tools"
 291             cd $WORKAREA/cmd/xfs
 292             make realclean 2>&1 \
 293                                     || _fail "            !!! clean tools failed"
 294
 295             _log "        *** clean quota tools"
 296             cd $WORKAREA/cmd/quota
 297             ( rm -f configure ; autoconf ; sh configure ; make clean ) 2>&1 \
 298                                     || _fail "            !!! clean quota failed"
 299
 300             _log "        *** clean qa"
 301             cd $WORKAREA/cmd/xfs/stress
 302             rm -f *.full *.bad *.log *.time *.core core 2>&1 \
 303                                     || _fail "            !!! clean qa failed"
 304
 305             _log "        *** clean linux"
 306             cd $WORKAREA/linux
 307             make mrproper 2>&1 \
 308                                     || _fail "            !!! clean linux failed"
 309
 310             _log "        *** install configuration file"
 311             cp -f $CONFIG $WORKAREA/linux/.config 2>&1 \
 312                                     || _fail "            !!! failed to install config"
 313
 314             _log "        *** remove version file"
 315             rm -f include/linux/version.h 2>&1 \
 316                                     || _fail "            !!! failed to clean version"
 317
 318             new_state="reconfig"
 319             ;;
 320
 321         *reconfig)
 322
 323             _log "        *** reconfig kernel"
 324
 325             _change_state "clean" ;# if this fails, we'd better start from scratch
 326
 327             cd $WORKAREA/linux
 328             # we want to use default options for any new config options.
 329             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 330                 make EXTRAVERSION=$EXTRA oldconfig 2>&1 \
 331                                     || _fail "            !!! reconfig oldconfig failed"
 332             make EXTRAVERSION=$EXTRA dep 2>&1 \
 333                                     || _fail "            !!! reconfig dep failed"
 334
 335             new_state="build"
 336             ;;
 337
 338         *build)
 339             _log "        *** build kernel"
 340
 341             _change_state "clean" ;# if this fails, we'd better start from scratch
 342
 343             cd $WORKAREA/linux
 344             make -j2 EXTRAVERSION=$EXTRA bzImage 2>&1 \
 345                                     || _fail "            !!! build bzImage failed"
 346             make -j2 EXTRAVERSION=$EXTRA modules 2>&1 \
 347                                     || _fail "            !!! build modules failed"
 348
 349             _log "        *** build tools"
 350             cd $WORKAREA/cmd/xfs
 351
 352             # use e-fence - but this will only take effect on configure
 353             export MALLOCLIB=/usr/lib/libefence.a
 354             make configure 2>&1 \
 355                                     || _fail "            !!! configure tools failed"
 356             make default 2>&1 \
 357                                     || _fail "            !!! build tools failed"
 358
 359             _log "        *** build quota tools"
 360             cd $WORKAREA/cmd/quota
 361
 362             # use e-fence - but this will only take effect on configure
 363             export MALLOCLIB=/usr/lib/libefence.a
 364             ( rm -f configure ; autoconf ; sh configure ; make all ) 2>&1 \
 365                                     || _fail "            !!! build quota failed"
 366
 367             _log "        *** build stress/src"
 368             cd $WORKAREA/cmd/xfs/stress/src
 369
 370             make default 2>&1 \
 371                                     || _fail "            !!! build stress/src failed"
 372
 373             new_state="install"
 374             ;;
 375
 376         *install)
 377             _log "        *** blat old modules"
 378
 379             _sudo rm -rf $MODULES
 380
 381             _log "        *** install kernel"
 382             cd $WORKAREA/linux
 383             _sudo cp -f $WORKAREA/linux/arch/i386/boot/bzImage $IMAGE 2>&1 \
 384                                     || _fail "            !!! install kernel failed"
 385             _sudo cp -f $WORKAREA/linux/System.map $SYSTEMMAP 2>&1 \
 386                                     || _fail "            !!! install kernel failed"
 387             _sudo make EXTRAVERSION=$EXTRA modules_install 2>&1 \
 388                                     || _fail "            !!! install modules failed"
 389
 390             _log "        *** install xfs tools"
 391             cd $WORKAREA/cmd/xfs
 392             _sudo make install 2>&1 \
 393                                     || _fail "            !!! install tools failed"
 394
 395             _log "        *** install quota tools"
 396             cd $WORKAREA/cmd/quota
 397             _sudo make install 2>&1 \
 398                                     || _fail "            !!! install quota failed"
 399
 400             _log "        *** reinit lilo"
 401             _sudo /sbin/lilo 2>&1 \
 402                                     || _fail "            !!! reinit lilo failed"
 403             new_state="restart"
 404             ;;
 405
 406         *restart)
 407             _log "            *** select qa kernel"
 408             _sudo /sbin/lilo -R linux-xfs-qa 2>&1 \
 409                                     || _fail "            !!! lilo failed"
 410
 411             _log "            *** prepare to restart"
 412             _change_state "restarted"
 413
 414             _log "            *** restarting"
 415
 416             _restart # doesn't return
 417             ;;
 418
 419         *restarted)
 420             _log "            *** QA reentered after restart"
 421
 422             new_state="check"
 423             ;;
 424
 425         *check)
 426             uname=`uname -ar`
 427             _log "            *** uname $uname"
 428             _log "            *** user tools"
 429             ls -l /sbin/*xfs* /usr/sbin/*xfs* 2>&1
 430             _log "            *** kernel"
 431             ls -l /boot/*$EXTRA*  2>&1
 432             _log "            *** kernel modules"
 433             ls -l /lib/modules/$VERSION$EXTRA/kernel/fs/pagebuf/* \
 434                   /lib/modules/$VERSION$EXTRA/kernel/fs/xfs/*     \
 435                   /lib/modules/$VERSION$EXTRA/kernel/fs/xfs/support/*
 436
 437             if [ $MODULAR -eq 0 ]
 438             then
 439                 new_state="reset"
 440             else
 441                 new_state="probe"
 442             fi
 443             ;;
 444
 445         *probe)
 446             _log "            *** modules dependencies"
 447
 448             _sudo depmod -a  2>&1 \
 449                                     || _fail "            !!! failed to depmod -a"
 450
 451             _log "            *** unmounting XFS mounts"
 452
 453             _sudo umount -a -t xfs 2>&1
 454
 455             _log "            *** removing modules"
 456
 457             for m in xfsidbg xfs pagebuf kdbm_pg kdbm_vm
 458             do
 459                 _sudo rmmod $m 2> /dev/null
 460             done
 461
 462             _log "            *** installing modules"
 463
 464             _sudo modprobe xfs 2>&1 \
 465                                     || _fail "            !!! failed to modprobe xfs"
 466
 467             new_state="reset"
 468             ;;
 469
 470         *reset)
 471
 472             _log "            *** unmounting TEST_DEV"
 473
 474             _sudo umount $TEST_DEV 2>&1
 475
 476             _log "            *** unmounting SCRATCH_DEV"
 477
 478             _sudo umount $SCRATCH_DEV 2>&1
 479
 480             _log "            *** clean TEST_DEV"
 481
 482             _sudo mkfs -t xfs -f $TEST_DEV 2>&1 \
 483                                     || _fail "            !!! failed to mkfs TEST_DEV"
 484
 485             _log "            *** mounting TEST_DEV"
 486
 487             _sudo mount -t xfs $TEST_DEV $TEST_DIR 2>&1 \
 488                                     || _fail "            !!! failed to mount"
 489
 490             new_state="run"
 491             ;;
 492
 493
 494         soak-run)
 495             cd $QADIR
 496
 497             _log "            *** run soak test"
 498             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC\
 499                                     || _fail "            !!! failed to run soak test"
 500
 501             new_state="done"
 502             ;;
 503
 504         *run)
 505             cd $QADIR
 506
 507             _log "            *** run tests"
 508             _sudo ./check -l -g auto 2>&1 | tee $ROOT/qa.out
 509
 510             _log ""
 511             _log "            *** send status mail"
 512             mail -s "xfs qa status report" $EMAIL < $ROOT/qa.out 2>&1
 513
 514             new_state="done"
 515             ;;
 516
 517         *done)
 518             _log "*** QA run complete"
 519
 520             _success
 521             ;;
 522
 523         *nothing)
 524             new_state="done"
 525             _log "    *** do nothing"
 526             ;;
 527
 528         *)
 529             _fail "           !!! unknown state $state"
 530             ;;
 531     esac
 532
 533     _log "    *** state $state done (`date`)"
 534     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 535     _change_state $new_state
 536
 537 done