From: Kamoltat Date: Thu, 28 Jul 2022 19:05:30 +0000 (+0000) Subject: qa/standalone/mon: init mon-stretched-cluster.sh X-Git-Tag: v16.2.11~63^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=025d3fa6b928d6fe2a794011e81db0c52d0c9cc0;p=ceph.git qa/standalone/mon: init mon-stretched-cluster.sh Added bug reproducer for https://bugzilla.redhat.com/show_bug.cgi?id=2104207 Added more logs in MON. Signed-off-by: Kamoltat (cherry picked from commit 62fe3cb8b9cd5b03a9751fc116d37573795c209b) --- diff --git a/qa/standalone/mon/mon-stretched-cluster.sh b/qa/standalone/mon/mon-stretched-cluster.sh new file mode 100755 index 000000000000..b530f7294221 --- /dev/null +++ b/qa/standalone/mon/mon-stretched-cluster.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + +source $CEPH_ROOT/qa/standalone/ceph-helpers.sh +function run() { + local dir=$1 + shift + + export CEPH_MON_A="127.0.0.1:7139" # git grep '\<7139\>' : there must be only one + export CEPH_MON_B="127.0.0.1:7141" # git grep '\<7141\>' : there must be only one + export CEPH_MON_C="127.0.0.1:7142" # git grep '\<7142\>' : there must be only one + export CEPH_MON_D="127.0.0.1:7143" # git grep '\<7143\>' : there must be only one + export CEPH_MON_E="127.0.0.1:7144" # git grep '\<7144\>' : there must be only one + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + + export BASE_CEPH_ARGS=$CEPH_ARGS + CEPH_ARGS+="--mon-host=$CEPH_MON_A" + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + setup $dir || return 1 + $func $dir || return 1 + teardown $dir || return 1 + done +} +TEST_stretched_cluster_failover_add_three_osds(){ + local dir=$1 + local OSDS=8 + setup $dir || return 1 + + run_mon $dir a --public-addr $CEPH_MON_A || return 1 + wait_for_quorum 300 1 || return 1 + + run_mon $dir b --public-addr $CEPH_MON_B || return 1 + CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B" + wait_for_quorum 300 2 || return 1 + + run_mon $dir c --public-addr $CEPH_MON_C || return 1 + CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C" + wait_for_quorum 300 3 || return 1 + + run_mon $dir d --public-addr $CEPH_MON_D || return 1 + CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C,$CEPH_MON_D" + wait_for_quorum 300 4 || return 1 + + run_mon $dir e --public-addr $CEPH_MON_E || return 1 + CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C,$CEPH_MON_D,$CEPH_MON_E" + wait_for_quorum 300 5 || return 1 + + ceph mon set election_strategy connectivity + ceph mon add disallowed_leader e + + run_mgr $dir x || return 1 + run_mgr $dir y || return 1 + run_mgr $dir z || return 1 + + for osd in $(seq 0 $(expr $OSDS - 1)) + do + run_osd $dir $osd || return 1 + done + + for zone in iris pze + do + ceph osd crush add-bucket $zone zone + ceph osd crush move $zone root=default + done + + + ceph osd crush add-bucket node-2 host + ceph osd crush add-bucket node-3 host + ceph osd crush add-bucket node-4 host + ceph osd crush add-bucket node-5 host + + ceph osd crush move node-2 zone=iris + ceph osd crush move node-3 zone=iris + ceph osd crush move node-4 zone=pze + ceph osd crush move node-5 zone=pze + + ceph osd crush move osd.0 host=node-2 + ceph osd crush move osd.1 host=node-2 + ceph osd crush move osd.2 host=node-3 + ceph osd crush move osd.3 host=node-3 + ceph osd crush move osd.4 host=node-4 + ceph osd crush move osd.5 host=node-4 + ceph osd crush move osd.6 host=node-5 + ceph osd crush move osd.7 host=node-5 + + ceph mon set_location a zone=iris host=node-2 + ceph mon set_location b zone=iris host=node-3 + ceph mon set_location c zone=pze host=node-4 + ceph mon set_location d zone=pze host=node-5 + + hostname=$(hostname -s) + ceph osd crush remove $hostname || return 1 + ceph osd getcrushmap > crushmap || return 1 + crushtool --decompile crushmap > crushmap.txt || return 1 + sed 's/^# end crush map$//' crushmap.txt > crushmap_modified.txt || return 1 + cat >> crushmap_modified.txt << EOF +rule stretch_rule { + id 1 + type replicated + min_size 1 + max_size 10 + step take iris + step chooseleaf firstn 2 type host + step emit + step take pze + step chooseleaf firstn 2 type host + step emit +} + +# end crush map +EOF + + crushtool --compile crushmap_modified.txt -o crushmap.bin || return 1 + ceph osd setcrushmap -i crushmap.bin || return 1 + local stretched_poolname=stretched_rbdpool + ceph osd pool create $stretched_poolname 32 32 stretch_rule || return 1 + ceph osd pool set $stretched_poolname size 4 || return 1 + + sleep 3 + + ceph mon set_location e zone=arbiter host=node-1 + ceph mon enable_stretch_mode e stretch_rule zone + + kill_daemons $dir KILL mon.c || return 1 + kill_daemons $dir KILL mon.d || return 1 + + kill_daemons $dir KILL osd.4 || return 1 + kill_daemons $dir KILL osd.5 || return 1 + kill_daemons $dir KILL osd.6 || return 1 + kill_daemons $dir KILL osd.7 || return 1 + + ceph -s + + sleep 3 + + run_osd $dir 8 || return 1 + run_osd $dir 9 || return 1 + run_osd $dir 10 || return 1 + + ceph -s + + sleep 3 + + teardown $dir || return 1 + +} +main mon-stretched-cluster "$@" \ No newline at end of file diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index e5f304d10b21..418436de2ccb 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -6582,6 +6582,7 @@ void Monitor::notify_new_monmap(bool can_change_external_state) if (is_stretch_mode()) { if (!monmap->stretch_marked_down_mons.empty()) { + dout(20) << __func__ << " stretch_marked_down_mons: " << monmap->stretch_marked_down_mons << dendl; set_degraded_stretch_mode(); } } @@ -6687,10 +6688,14 @@ struct CMonGoRecovery : public Context { void Monitor::go_recovery_stretch_mode() { dout(20) << __func__ << dendl; + dout(20) << "is_leader(): " << is_leader() << dendl; if (!is_leader()) return; + dout(20) << "is_degraded_stretch_mode(): " << is_degraded_stretch_mode() << dendl; if (!is_degraded_stretch_mode()) return; + dout(20) << "is_recovering_stretch_mode(): " << is_recovering_stretch_mode() << dendl; if (is_recovering_stretch_mode()) return; - + dout(20) << "dead_mon_buckets.size(): " << dead_mon_buckets.size() << dendl; + dout(20) << "dead_mon_buckets: " << dead_mon_buckets << dendl; if (dead_mon_buckets.size()) { ceph_assert( 0 == "how did we try and do stretch recovery while we have dead monitor buckets?"); // we can't recover if we are missing monitors in a zone! @@ -6771,6 +6776,7 @@ void Monitor::trigger_degraded_stretch_mode(const set& dead_mons, void Monitor::set_degraded_stretch_mode() { + dout(20) << __func__ << dendl; degraded_stretch_mode = true; recovering_stretch_mode = false; osdmon()->set_degraded_stretch_mode(); diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 3191ed5bf18b..97034994546a 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -954,6 +954,10 @@ void OSDMonitor::update_from_paxos(bool *need_bootstrap) dout(20) << "Degraded stretch mode set in this map" << dendl; if (!osdmap.recovering_stretch_mode) { mon.set_degraded_stretch_mode(); + dout(20) << "prev_num_up_osd: " << prev_num_up_osd << dendl; + dout(20) << "osdmap.num_up_osd: " << osdmap.num_up_osd << dendl; + dout(20) << "osdmap.num_osd: " << osdmap.num_osd << dendl; + dout(20) << "mon_stretch_cluster_recovery_ratio: " << cct->_conf.get_val("mon_stretch_cluster_recovery_ratio") << dendl; if (prev_num_up_osd < osdmap.num_up_osd && (osdmap.num_up_osd / (double)osdmap.num_osd) > cct->_conf.get_val("mon_stretch_cluster_recovery_ratio")) {