tools/auto-qa

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2000-2002 Silicon Graphics, Inc.  All Rights Reserved.
   4 #
   5 # This program is free software; you can redistribute it and/or modify it
   6 # under the terms of version 2 of the GNU General Public License as
   7 # published by the Free Software Foundation.
   8 #
   9 # This program is distributed in the hope that it would be useful, but
  10 # WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 #
  13 # Further, this software is distributed without any warranty that it is
  14 # free of the rightful claim of any third person regarding infringement
  15 # or the like.  Any license provided herein, whether implied or
  16 # otherwise, applies only to this software file.  Patent licenses, if
  17 # any, provided herein do not apply to combinations of this program with
  18 # other software, or any other product whatsoever.
  19 #
  20 # You should have received a copy of the GNU General Public License along
  21 # with this program; if not, write the Free Software Foundation, Inc., 59
  22 # Temple Place - Suite 330, Boston MA 02111-1307, USA.
  23 #
  24 # Contact information: Silicon Graphics, Inc., 1600 Amphitheatre Pkwy,
  25 # Mountain View, CA  94043, or:
  26 #
  27 # http://www.sgi.com
  28 #
  29 # For further information regarding this notice, see:
  30 #
  31 # http://oss.sgi.com/projects/GenInfo/SGIGPLNoticeExplan/
  32 #
  33
  34 # automatic qa system. 31/08/00 dxm@sgi.com
  35
  36 # configuration (you could tune this)
  37
  38 _log()
  39 {
  40     echo "$*" >&2
  41     echo "$*" >> $LOG
  42     sync
  43 }
  44
  45 _fail()
  46 {
  47     if [ "$started" = "1" ]
  48     then
  49         echo "auto-qa stopped" | wall
  50         started=0
  51     fi
  52
  53     _log "$*"
  54
  55     # send special email if a cron'd qa run fails
  56     case $state
  57     in
  58         cron*)
  59             mail -s "XFS QA status report" $EMAIL \
  60                 < $LOG 2>&1
  61         ;;
  62     esac
  63
  64     status=1
  65     exit 1
  66 }
  67
  68 _get_kernel_version()
  69 {
  70     [ -x "$WORKAREA" ] \
  71         || _fail "can't access workarea $WORKAREA"
  72     [ -r "$WORKAREA/linux/Makefile" ] \
  73         || _fail "can't read makefile $WORKAREA/linux/Makefile"
  74
  75     eval `awk '
  76         BEGIN { FS = "[ \t=]+" }
  77         /^VERSION =/ { a=$2 }
  78         /^PATCHLEVEL =/ { b=$2 }
  79         /^SUBLEVEL =/ { c=$2 }
  80         /^EXTRAVERSION =/ { d=$2 }
  81         END {
  82             print "VERSION=" a "." b "." c d " ; SVERSION=" a "." b "." c
  83         }
  84    ' < $WORKAREA/linux/Makefile`
  85 }
  86
  87 # this should be constant
  88
  89 ROOT="$HOME/qa"
  90 HOST=`hostname -s`
  91 if [ ! -z "$CVSROOT" ]; then
  92     WORKAREA="$ROOT/linux-2.4-xfs"
  93 else
  94     [ -z "$WORKAREA" ] && WORKAREA="$ROOT/linux-xfs"
  95 fi
  96 export WORKAREA
  97
  98
  99 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin/ptools:/usr/local/bin"
 100 STATE=$ROOT/qa.state
 101 QADIR="$WORKAREA/cmd/xfstests"
 102 SUDO="su -c"
 103 CONFIG="$ROOT/$HOST.config"
 104 COMMON_CONFIG="$WORKAREA/cmd/xfstests/common.config"
 105 SH="/bin/sh"
 106 LOG="$ROOT/qa.log"
 107
 108 # do some cleanup on exit
 109
 110 _cleanup()
 111 {
 112     umount $SCRATCH_DEV &> /dev/null
 113     umount $TEST_DEV &> /dev/null
 114     if [ "$started" = 1 ]
 115     then
 116         echo "auto-qa stopped" | wall
 117         started=0
 118     fi
 119 }
 120 status=1
 121 trap "_cleanup; exit \$status" 0 1 2 3 15
 122
 123 # clean exit
 124
 125 _success()
 126 {
 127     status=0
 128     exit 0
 129 }
 130
 131 _get_state()
 132 {
 133     state=`cat $STATE`
 134 }
 135
 136 _set_state()
 137 {
 138     echo $1 > $STATE
 139     _get_state
 140 }
 141
 142 _change_state()
 143 {
 144     new=$1
 145
 146     case $state
 147     in
 148         *-*)
 149             case $new
 150             in
 151                 *-*)
 152                     _set_state $new
 153                     ;;
 154                 *)
 155                     _set_state `echo $state | sed "s/-.*$/-$new/"`
 156                     ;;
 157             esac
 158             ;;
 159         *)
 160             _set_state $new
 161             ;;
 162     esac
 163 }
 164
 165 _sudo()
 166 {
 167     $ROOT/su -c "$*" < /dev/null ;# HACK - we need a hacked su at the mo
 168 }
 169
 170 _restart()
 171 {
 172     exec $ROOT/su -c "(shutdown -r now \"auto-qa rebooting\" )&" < /dev/null
 173 }
 174
 175 _update_autoqa_file()
 176 {
 177     SELF="$ROOT/auto-qa"
 178     SELF_UPDATE="cmd/xfstests/tools/auto-qa"
 179     if [ -z "$CVSROOT" ]; then
 180             cmd="p_tupdate $SELF_UPDATE"
 181     else
 182             cmd="cvs -z3 update $SELF_UPDATE"
 183     fi
 184     exec $SH -c "cd $WORKAREA ; $cmd; chmod +x $SELF_UPDATE ; exec $SELF"
 185 }
 186
 187 _update_workarea()
 188 {
 189     if [ -z "$CVSROOT" ]; then
 190         _log "  *** p_tupdate"
 191         cd $WORKAREA
 192         p_tupdate 2>&1 \
 193                         || _fail "          !!! p_tupdate failed"
 194
 195         _log "  *** p_check/p_purge"
 196         cd $WORKAREA
 197         p_check -s | p_purge -yiu 2>&1 \
 198                         || _fail "          !!! p_check/p_purge failed"
 199
 200         _log "  *** non-trunk files"
 201         cd $WORKAREA
 202         p_list -c 2>&1 \
 203                         || _fail "          !!! p_list failed"
 204     else
 205         _log "  *** cvs update"
 206         cd $WORKAREA
 207         cvs -z3 update -d
 208     fi
 209 }
 210
 211 _test_mkfs_xfs()
 212 {
 213     TEST_OPTIONS=""
 214     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 215         TEST_OPTIONS="$TEST_OPTIONS -rrtdev=$TEST_RTDEV"
 216     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 217         TEST_OPTIONS="$TEST_OPTIONS -llogdev=$TEST_LOGDEV"
 218     _sudo /sbin/mkfs.xfs -f $TEST_OPTIONS $MKFS_OPTIONS $* $TEST_DEV
 219     mkfs_status=$?
 220     [ "$USE_BIG_LOOPFS" = yes ] && \
 221         _sudo $QADIR/tools/ag-wipe -c $TEST_DEV >/dev/null
 222     return $mkfs_status
 223 }
 224
 225 _test_mount()
 226 {
 227     TEST_OPTIONS=""
 228     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_RTDEV" ] && \
 229         TEST_OPTIONS="$TEST_OPTIONS -ortdev=$TEST_RTDEV"
 230     [ "$USE_EXTERNAL" = yes -a ! -z "$TEST_LOGDEV" ] && \
 231         TEST_OPTIONS="$TEST_OPTIONS -ologdev=$TEST_LOGDEV"
 232     _sudo mount -t xfs $TEST_OPTIONS $* $TEST_DEV $TEST_DIR
 233 }
 234
 235
 236 _log "*** linux-xfs QA (`date`)"
 237
 238 _get_state
 239
 240 # check preconditions for starting state
 241 case $1
 242 in
 243     cron-init)
 244         case $state
 245         in
 246             *done)
 247                 ;;
 248             *)
 249                 _fail "    !!! cron-init while not in \"*done\" state"
 250                 ;;
 251         esac
 252         ;;
 253     cron-restarted)
 254         # we don't auto restart after reboot, but cron the restart
 255         # to happen a bit later - it's much easier and safer that way
 256         if [ "$state" != "cron-restarted" ]
 257         then
 258             _fail "    !!! cron-restarted while not in \"cron-restarted\" state"
 259         fi
 260         ;;
 261 esac
 262
 263 if [ "$1" != "" ]
 264 then
 265     _set_state $1
 266 fi
 267
 268 [ "$UID" -eq 0 ]        && _fail "    !!! QA most be run as a normal user"
 269 [ -d "$ROOT" ]          || _fail "    !!! QA root \"$ROOT\" not found"
 270 [ -d "$WORKAREA" ]      || _fail "    !!! QA workarea \"$WORKAREA\" not found"
 271 [ -r "$CONFIG" ]        || _fail "    !!! Can't read config file $CONFIG"
 272 . "$COMMON_CONFIG"      || _fail "    !!! Couldn't source $COMMON_CONFIG"
 273
 274 _get_kernel_version
 275 IMAGE="$BOOT/vmlinuz$EXTRA"
 276 SYSTEMMAP="$BOOT/System.map-$VERSION$EXTRA"
 277 MODULES="/lib/modules/$SVERSION"
 278
 279 cd $ROOT
 280
 281 started=1
 282 echo "auto-qa started" | wall
 283
 284 while true
 285 do
 286     _get_state
 287
 288     _log "    *** state $state start (`date`)"
 289     _log "      (user=$USER, host=$HOST)"
 290     new_state=""
 291
 292     case $state
 293     in
 294         *init)
 295             echo "" > $ROOT/qa.log
 296             echo "" > $ROOT/qa.full
 297             _log "******************************************************"
 298             _log "QA init $VERSION (`date`)"
 299             _log "******************************************************"
 300             _log "--- kernel ($IMAGE)"
 301             _log "--- modules ($MODULES)"
 302
 303             _change_state "inited"
 304             _update_autoqa_file
 305             ;;
 306
 307         *inited)
 308             _log "      *** QA initialized"
 309             new_state="update"
 310             ;;
 311
 312         *update)
 313             _update_workarea
 314             new_state="clean"
 315             ;;
 316
 317         *clean)
 318             # we need to configure or else we might fail to clean
 319             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 320             do
 321                 cd $WORKAREA/cmd/$pkg
 322                 _log "  *** clean $pkg tools"
 323                 make realclean 2>&1 \
 324                         || _fail "          !!! clean $pkg failed"
 325             done
 326
 327             _log "      *** clean linux"
 328             cd $WORKAREA/linux
 329             make mrproper 2>&1 \
 330                         || _fail "          !!! clean linux failed"
 331
 332             _log "      *** install configuration file"
 333             cp -f $CONFIG $WORKAREA/linux/.config 2>&1 \
 334                         || _fail "          !!! failed to install config"
 335
 336             _log "      *** remove version file"
 337             rm -f include/linux/version.h 2>&1 \
 338                         || _fail "          !!! failed to clean version"
 339
 340             new_state="reconfig"
 341             ;;
 342
 343         *reconfig)
 344
 345             _log "      *** reconfig kernel"
 346
 347             _change_state "clean" ; # we better start from scratch if this fails
 348
 349             cd $WORKAREA/linux
 350             # we want to use default options for any new config options.
 351             echo -e "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" | \
 352                 make EXTRAVERSION=$EXTRA oldconfig 2>&1 \
 353                         || _fail "          !!! reconfig oldconfig failed"
 354             make EXTRAVERSION=$EXTRA dep 2>&1 \
 355                         || _fail "          !!! reconfig dep failed"
 356
 357             new_state="buildtools"
 358             ;;
 359
 360         *buildtools)
 361             _log "      *** build and install tools"
 362             for pkg in attr acl xfsprogs dmapi xfsdump xfstests
 363             do
 364                 cd $WORKAREA/cmd/$pkg
 365
 366                 # use e-fence - but this will only take effect on configure
 367                 make configure 2>&1 \
 368                         || _fail "          !!! configure $pkg failed"
 369                 make default 2>&1 \
 370                         || _fail "          !!! build $pkg failed"
 371
 372                 _sudo make install install-dev 2>&1 \
 373                         || _fail "          !!! install $pkg failed"
 374
 375                 # attr and acl now have install-lib targets as well
 376                 [ "$pkg" = "attr" -o "$pkg" = "acl" ] || continue
 377                 _sudo make install-lib 2>&1 \
 378                         || _fail "          !!! install $pkg failed"
 379             done
 380
 381             new_state="buildkernel"
 382             ;;
 383
 384         *buildkernel)
 385             _log "      *** build kernel"
 386
 387             _change_state "clean" ; # we better start from scratch if this fails
 388
 389             cd $WORKAREA/linux
 390             make -j4 EXTRAVERSION=$EXTRA bzImage modules 2>&1 \
 391                         || _fail "          !!! build bzImage/modules failed"
 392             new_state="install"
 393             ;;
 394
 395         *install)
 396             _log "      *** blat old modules"
 397
 398             _sudo rm -rf $MODULES
 399
 400             _log "      *** install kernel"
 401             cd $WORKAREA/linux
 402             _sudo cp -f $WORKAREA/linux/arch/i386/boot/bzImage $IMAGE 2>&1 \
 403                         || _fail "          !!! install kernel failed"
 404             _sudo cp -f $WORKAREA/linux/System.map $SYSTEMMAP 2>&1 \
 405                         || _fail "          !!! install kernel failed"
 406             _sudo make EXTRAVERSION=$EXTRA modules_install 2>&1 \
 407                         || _fail "          !!! install modules failed"
 408
 409             _log "      *** reinit lilo"
 410             _sudo /sbin/lilo 2>&1 \
 411                         || _fail "          !!! reinit lilo failed"
 412             new_state="restart"
 413             ;;
 414
 415         *restart)
 416             _log "          *** select qa kernel"
 417             _sudo /sbin/lilo -R linux-xfs-qa $KERNEL_OPTIONS 2>&1 \
 418                         || _fail "          !!! lilo failed"
 419
 420             _log "          *** prepare to restart"
 421             _change_state "restarted"
 422
 423             _log "          *** restarting"
 424
 425             _restart # doesn't return
 426             ;;
 427
 428         *restarted)
 429             _log "          *** QA reentered after restart"
 430
 431             new_state="check"
 432             ;;
 433
 434         *check)
 435             uname=`uname -ar`
 436             _log "          *** uname $uname"
 437             _log "          *** kernel"
 438             ls -l /boot/*$EXTRA*  2>&1
 439             _log "          *** kernel modules"
 440             [ -d /lib/modules/$SVERSION$EXTRA/kernel/fs/xfs ] && \
 441                 ls -l /lib/modules/$SVERSION$EXTRA/kernel/fs/xfs/*
 442
 443             if [ "$MODULAR" -eq 0 ]
 444             then
 445                 new_state="reset"
 446             else
 447                 new_state="probe"
 448             fi
 449             ;;
 450
 451         *probe)
 452             _log "          *** modules dependencies"
 453
 454             _sudo depmod -a  2>&1 \
 455                         || _fail "          !!! failed to depmod -a"
 456
 457             _log "          *** unmounting XFS mounts"
 458
 459             _sudo umount -a -t xfs 2>&1
 460
 461             _log "          *** removing modules"
 462
 463             for m in xfsidbg xfs kdbm_pg kdbm_vm
 464             do
 465                 _sudo rmmod $m 2> /dev/null
 466             done
 467
 468             _log "          *** installing modules"
 469
 470             _sudo modprobe xfs 2>&1 \
 471                         || _fail "          !!! failed to modprobe xfs"
 472
 473             new_state="reset"
 474             ;;
 475
 476         *reset)
 477
 478             _log "          *** unmounting TEST_DEV"
 479
 480             _sudo umount $TEST_DEV 2>&1
 481
 482             _log "          *** unmounting SCRATCH_DEV"
 483
 484             _sudo umount $SCRATCH_DEV 2>&1
 485
 486             _log "          *** clean TEST_DEV"
 487
 488             _test_mkfs_xfs 2>&1 \
 489                         || _fail "          !!! failed to mkfs TEST_DEV"
 490
 491             _log "          *** mounting TEST_DEV"
 492
 493             _test_mount 2>&1 \
 494                         || _fail "          !!! failed to mount"
 495
 496             new_state="run"
 497             ;;
 498
 499         soak-run)
 500             cd $QADIR
 501
 502             _log "          *** run soak test"
 503             _sudo ./soak $SOAK_PASSES $SOAK_STRESS $SOAK_PROC \
 504                         || _fail "          !!! failed to run soak test"
 505
 506             new_state="done"
 507             ;;
 508
 509         bench-run)
 510             cd $QADIR
 511
 512             # $BENCHMARK is typically unset, which equates to "all"
 513             #
 514             _log "          *** run benchmarks"
 515             _sudo ./bench $BENCH_PASSES `id -nu && id -ng` $BENCHMARK \
 516                         || _fail "          !!! failed to run benchmarks"
 517
 518             _log ""
 519             _log "          *** send results mail"
 520             mail -s "XFS QA benchmark results" $EMAIL < $QADIR/bench.out 2>&1
 521
 522             new_state="done"
 523             ;;
 524
 525         *run)
 526             cd $QADIR
 527
 528             _log "          *** run tests ($CHECK_OPTIONS)"
 529             _sudo ./check -l $CHECK_OPTIONS 2>&1 | tee $ROOT/qa.out
 530
 531             _log ""
 532             _log "          *** send status mail"
 533             mail -s "XFS QA status report" $EMAIL < $ROOT/qa.out 2>&1
 534
 535             new_state="done"
 536             ;;
 537
 538         *done)
 539             _log "*** QA run complete"
 540
 541             _success
 542             ;;
 543
 544         *nothing)
 545             new_state="done"
 546             _log "    *** do nothing"
 547             ;;
 548
 549         *)
 550             _fail "        !!! unknown state $state"
 551             ;;
 552     esac
 553
 554     _log "    *** state $state done (`date`)"
 555     [ "$new_state" = "" ] && _fail "    !!! no new state set"
 556     _change_state $new_state
 557
 558 done