From 710fef96eab2ab8f864f055694f4f6ee468d3536 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 31 Jul 2019 17:28:18 -0500 Subject: [PATCH] qa/standalone/mon/health-mutes: add tests Make sure mute and unmute work. Make sure stick is sticky. Mkae sure counts can go down bupt if they go upt hte mute clears. Signed-off-by: Sage Weil --- qa/standalone/mon/health-mute.sh | 96 ++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100755 qa/standalone/mon/health-mute.sh diff --git a/qa/standalone/mon/health-mute.sh b/qa/standalone/mon/health-mute.sh new file mode 100755 index 0000000000000..e781b47bf13ac --- /dev/null +++ b/qa/standalone/mon/health-mute.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:7143" # git grep '\<714\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none --mon-pg-warn-min-per-osd 0 --mon-max-pg-per-osd 1000 " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} + +function TEST_mute() { + local dir=$1 + setup $dir || return 1 + + set -o pipefail + + run_mon $dir a || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_osd $dir 1 || return 1 + run_osd $dir 2 || return 1 + + ceph osd pool create foo 10 + ceph osd pool application enable foo rbd --yes-i-really-mean-it + wait_for_clean || return 1 + + ceph -s + ceph health | grep HEALTH_OK || return 1 + ceph osd set noup + ceph -s + ceph health detail | grep OSDMAP_FLAGS || return 1 + ceph osd down 0 + ceph -s + ceph health detail | grep OSD_DOWN || return 1 + ceph health detail | grep HEALTH_WARN || return 1 + + ceph health mute OSD_DOWN + ceph health mute OSDMAP_FLAGS + ceph -s + ceph health | grep HEALTH_OK | grep OSD_DOWN | grep OSDMAP_FLAGS || return 1 + ceph health unmute OSD_DOWN + ceph -s + ceph health | grep HEALTH_WARN || return 1 + + # ttl + ceph health mute OSD_DOWN 10s + ceph -s + ceph health | grep HEALTH_OK || return 1 + sleep 15 + ceph -s + ceph health | grep HEALTH_WARN || return 1 + + # sticky + ceph health mute OSDMAP_FLAGS --sticky + ceph osd unset noup + sleep 5 + ceph -s + ceph health | grep OSDMAP_FLAGS || return 1 + ceph osd set noup + ceph -s + ceph health | grep HEALTH_OK || return 1 + + # rachet down on OSD_DOWN count + ceph osd down 0 1 + ceph -s + ceph health detail | grep OSD_DOWN || return 1 + ceph health mute OSD_DOWN + kill daemons $dir TERM osd.0 + ceph osd unset noup + ceph -s + ceph health detail | grep OSD_DOWN || return 1 + ceph health | grep HEALTH_OK || return 1 + sleep 10 + ceph osd set noup + ceph health mute OSDMAP_FLAGS + ceph -s + ceph health | grep HEALTH_OK || return 1 + ceph osd down 1 + ceph -s + ceph health | grep HEALTH_WARN || return 1 + + teardown $dir || return 1 +} + +main health-mute "$@" -- 2.39.5