+++ /dev/null
-#!/usr/bin/env bash
-
-source $CEPH_ROOT/qa/standalone/ceph-helpers.sh
-function run() {
- local dir=$1
- shift
-
- export CEPH_MON_A="127.0.0.1:7139" # git grep '\<7139\>' : there must be only one
- export CEPH_MON_B="127.0.0.1:7141" # git grep '\<7141\>' : there must be only one
- export CEPH_MON_C="127.0.0.1:7142" # git grep '\<7142\>' : there must be only one
- export CEPH_MON_D="127.0.0.1:7143" # git grep '\<7143\>' : there must be only one
- export CEPH_MON_E="127.0.0.1:7144" # git grep '\<7144\>' : there must be only one
- export CEPH_ARGS
- CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
-
- export BASE_CEPH_ARGS=$CEPH_ARGS
- CEPH_ARGS+="--mon-host=$CEPH_MON_A"
-
- local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
- for func in $funcs ; do
- setup $dir || return 1
- $func $dir || return 1
- teardown $dir || return 1
- done
-}
-TEST_stretched_cluster_failover_add_three_osds(){
- local dir=$1
- local OSDS=8
- setup $dir || return 1
-
- run_mon $dir a --public-addr $CEPH_MON_A || return 1
- wait_for_quorum 300 1 || return 1
-
- run_mon $dir b --public-addr $CEPH_MON_B || return 1
- CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B"
- wait_for_quorum 300 2 || return 1
-
- run_mon $dir c --public-addr $CEPH_MON_C || return 1
- CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C"
- wait_for_quorum 300 3 || return 1
-
- run_mon $dir d --public-addr $CEPH_MON_D || return 1
- CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C,$CEPH_MON_D"
- wait_for_quorum 300 4 || return 1
-
- run_mon $dir e --public-addr $CEPH_MON_E || return 1
- CEPH_ARGS="$BASE_CEPH_ARGS --mon-host=$CEPH_MON_A,$CEPH_MON_B,$CEPH_MON_C,$CEPH_MON_D,$CEPH_MON_E"
- wait_for_quorum 300 5 || return 1
-
- ceph mon set election_strategy connectivity
- ceph mon add disallowed_leader e
-
- run_mgr $dir x || return 1
- run_mgr $dir y || return 1
- run_mgr $dir z || return 1
-
- for osd in $(seq 0 $(expr $OSDS - 1))
- do
- run_osd $dir $osd || return 1
- done
-
- for zone in iris pze
- do
- ceph osd crush add-bucket $zone zone
- ceph osd crush move $zone root=default
- done
-
-
- ceph osd crush add-bucket node-2 host
- ceph osd crush add-bucket node-3 host
- ceph osd crush add-bucket node-4 host
- ceph osd crush add-bucket node-5 host
-
- ceph osd crush move node-2 zone=iris
- ceph osd crush move node-3 zone=iris
- ceph osd crush move node-4 zone=pze
- ceph osd crush move node-5 zone=pze
-
- ceph osd crush move osd.0 host=node-2
- ceph osd crush move osd.1 host=node-2
- ceph osd crush move osd.2 host=node-3
- ceph osd crush move osd.3 host=node-3
- ceph osd crush move osd.4 host=node-4
- ceph osd crush move osd.5 host=node-4
- ceph osd crush move osd.6 host=node-5
- ceph osd crush move osd.7 host=node-5
-
- ceph mon set_location a zone=iris host=node-2
- ceph mon set_location b zone=iris host=node-3
- ceph mon set_location c zone=pze host=node-4
- ceph mon set_location d zone=pze host=node-5
-
- hostname=$(hostname -s)
- ceph osd crush remove $hostname || return 1
- ceph osd getcrushmap > crushmap || return 1
- crushtool --decompile crushmap > crushmap.txt || return 1
- sed 's/^# end crush map$//' crushmap.txt > crushmap_modified.txt || return 1
- cat >> crushmap_modified.txt << EOF
-rule stretch_rule {
- id 1
- type replicated
- min_size 1
- max_size 10
- step take iris
- step chooseleaf firstn 2 type host
- step emit
- step take pze
- step chooseleaf firstn 2 type host
- step emit
-}
-
-# end crush map
-EOF
-
- crushtool --compile crushmap_modified.txt -o crushmap.bin || return 1
- ceph osd setcrushmap -i crushmap.bin || return 1
- local stretched_poolname=stretched_rbdpool
- ceph osd pool create $stretched_poolname 32 32 stretch_rule || return 1
- ceph osd pool set $stretched_poolname size 4 || return 1
-
- sleep 3
-
- ceph mon set_location e zone=arbiter host=node-1
- ceph mon enable_stretch_mode e stretch_rule zone
-
- kill_daemons $dir KILL mon.c || return 1
- kill_daemons $dir KILL mon.d || return 1
-
- kill_daemons $dir KILL osd.4 || return 1
- kill_daemons $dir KILL osd.5 || return 1
- kill_daemons $dir KILL osd.6 || return 1
- kill_daemons $dir KILL osd.7 || return 1
-
- ceph -s
-
- sleep 3
-
- run_osd $dir 8 || return 1
- run_osd $dir 9 || return 1
- run_osd $dir 10 || return 1
-
- ceph -s
-
- sleep 3
-
- teardown $dir || return 1
-
-}
-main mon-stretched-cluster "$@"
\ No newline at end of file
if (is_stretch_mode()) {
if (!monmap->stretch_marked_down_mons.empty()) {
- dout(20) << __func__ << " stretch_marked_down_mons: " << monmap->stretch_marked_down_mons << dendl;
set_degraded_stretch_mode();
}
}
void Monitor::go_recovery_stretch_mode()
{
dout(20) << __func__ << dendl;
- dout(20) << "is_leader(): " << is_leader() << dendl;
if (!is_leader()) return;
- dout(20) << "is_degraded_stretch_mode(): " << is_degraded_stretch_mode() << dendl;
if (!is_degraded_stretch_mode()) return;
- dout(20) << "is_recovering_stretch_mode(): " << is_recovering_stretch_mode() << dendl;
if (is_recovering_stretch_mode()) return;
- dout(20) << "dead_mon_buckets.size(): " << dead_mon_buckets.size() << dendl;
- dout(20) << "dead_mon_buckets: " << dead_mon_buckets << dendl;
+
if (dead_mon_buckets.size()) {
ceph_assert( 0 == "how did we try and do stretch recovery while we have dead monitor buckets?");
// we can't recover if we are missing monitors in a zone!
void Monitor::set_degraded_stretch_mode()
{
- dout(20) << __func__ << dendl;
degraded_stretch_mode = true;
recovering_stretch_mode = false;
osdmon()->set_degraded_stretch_mode();