From: Sage Weil Date: Sat, 23 Mar 2019 21:49:33 +0000 (-0500) Subject: mon/MDSMonitor: add 'mds ok-to-stop' command X-Git-Tag: v15.0.0~2^2~1 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=aa33a26e32d270286ee110f57cfcf63df63fa913;p=ceph-ci.git mon/MDSMonitor: add 'mds ok-to-stop' command Signed-off-by: Sage Weil --- diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh index 152396f6c02..83ca1e2511e 100755 --- a/qa/standalone/ceph-helpers.sh +++ b/qa/standalone/ceph-helpers.sh @@ -574,6 +574,28 @@ function run_mgr() { "$@" || return 1 } +function run_mds() { + local dir=$1 + shift + local id=$1 + shift + local data=$dir/$id + + ceph-mds \ + --id $id \ + $EXTRA_OPTS \ + --debug-mds 20 \ + --debug-objecter 20 \ + --debug-ms 20 \ + --chdir= \ + --mds-data=$data \ + --log-file=$dir/\$name.log \ + --admin-socket=$(get_asok_path) \ + --run-dir=$dir \ + --pid-file=$dir/\$name.pid \ + "$@" || return 1 +} + ####################################################################### ## diff --git a/qa/standalone/misc/ok-to-stop.sh b/qa/standalone/misc/ok-to-stop.sh index 9894d3bf4bf..e53b28d1eb2 100755 --- a/qa/standalone/misc/ok-to-stop.sh +++ b/qa/standalone/misc/ok-to-stop.sh @@ -204,4 +204,38 @@ function TEST_5_mons_checks() { ceph mon ok-to-rm e || return 1 } +function TEST_0_mds() { + local dir=$1 + + CEPH_ARGS="$ORIG_CEPH_ARGS --mon-host=$CEPH_MON_A " + + run_mon $dir a --public-addr=$CEPH_MON_A || return 1 + run_mgr $dir x || return 1 + run_osd $dir 0 || return 1 + run_mds $dir a || return 1 + + ceph osd pool create meta 1 || return 1 + ceph osd pool create data 1 || return 1 + ceph fs new myfs meta data || return 1 + sleep 5 + + ! ceph mds ok-to-stop a || return 1 + ! ceph mds ok-to-stop a dne || return 1 + ceph mds ok-to-stop dne || return 1 + + run_mds $dir b || return 1 + sleep 5 + + ceph mds ok-to-stop a || return 1 + ceph mds ok-to-stop b || return 1 + ! ceph mds ok-to-stop a b || return 1 + ceph mds ok-to-stop a dne1 dne2 || return 1 + ceph mds ok-to-stop b dne || return 1 + ! ceph mds ok-to-stop a b dne || return 1 + ceph mds ok-to-stop dne1 dne2 || return 1 + + kill_daemons $dir KILL mds.a +} + + main ok-to-stop "$@" diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h index ca286b290c4..2e7028a4c05 100644 --- a/src/mds/FSMap.h +++ b/src/mds/FSMap.h @@ -65,15 +65,27 @@ public: * STANDBY_REPLAY for the gid `who` */ bool has_standby_replay(mds_gid_t who) const + { + return get_standby_replay(who) != MDS_GID_NONE; + } + mds_gid_t get_standby_replay(mds_gid_t who) const { for (const auto &i : mds_map.mds_info) { const auto &info = i.second; if (info.state == MDSMap::STATE_STANDBY_REPLAY && info.rank == mds_map.mds_info.at(who).rank) { - return true; + return info.global_id; } } - + return MDS_GID_NONE; + } + bool is_standby_replay(mds_gid_t who) const + { + auto p = mds_map.mds_info.find(who); + if (p != mds_map.mds_info.end() && + p->second.state == MDSMap::STATE_STANDBY_REPLAY) { + return true; + } return false; } @@ -173,6 +185,19 @@ public: return legacy_client_fscid; } + size_t get_num_standby() const { + return standby_daemons.size(); + } + + bool is_any_degraded() const { + for (auto& i : filesystems) { + if (i.second->mds_map.is_degraded()) { + return true; + } + } + return false; + } + /** * Get state of all daemons (for all filesystems, including all standbys) */ @@ -370,6 +395,16 @@ public: } } + bool is_standby_replay(mds_gid_t who) const + { + return filesystems.at(mds_roles.at(who))->is_standby_replay(who); + } + + mds_gid_t get_standby_replay(mds_gid_t who) const + { + return filesystems.at(mds_roles.at(who))->get_standby_replay(who); + } + /** * A daemon has told us it's compat, and it's too new * for the one we had previously. Impose the new one diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index 19bfdcd5642..6258c20f861 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -897,6 +897,57 @@ bool MDSMonitor::preprocess_command(MonOpRequestRef op) ds << fsmap; } r = 0; + } else if (prefix == "mds ok-to-stop") { + vector ids; + if (!cmd_getval(g_ceph_context, cmdmap, "ids", ids)) { + r = -EINVAL; + ss << "must specify mds id"; + goto out; + } + if (fsmap.is_any_degraded()) { + ss << "one or more filesystems is currently degraded"; + r = -EBUSY; + goto out; + } + set stopping; + for (auto& id : ids) { + ostringstream ess; + mds_gid_t gid = gid_from_arg(fsmap, id, ess); + if (gid == MDS_GID_NONE) { + // the mds doesn't exist, but no file systems are unhappy, so losing it + // can't have any effect. + continue; + } + stopping.insert(gid); + } + set active; + set standby; + for (auto gid : stopping) { + if (fsmap.gid_has_rank(gid)) { + // ignore standby-replay daemons (at this level) + if (!fsmap.is_standby_replay(gid)) { + auto standby = fsmap.get_standby_replay(gid); + if (standby == MDS_GID_NONE || + stopping.count(standby)) { + // no standby-replay, or we're also stopping the standby-replay + // for this mds + active.insert(gid); + } + } + } else { + // net loss of a standby + standby.insert(gid); + } + } + if (fsmap.get_num_standby() - standby.size() < active.size()) { + r = -EBUSY; + ss << "insufficent standby MDS daemons to stop active gids " + << stringify(active) + << " and/or standby gids " << stringify(standby);; + goto out; + } + r = 0; + ss << "should be safe to stop " << ids; } else if (prefix == "fs dump") { int64_t epocharg; epoch_t epoch; diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 7394564fe0a..b6b93293625 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -325,6 +325,9 @@ COMMAND_WITH_FLAG("mds stop name=role,type=CephString", "stop mds", \ COMMAND_WITH_FLAG("mds deactivate name=role,type=CephString", "clean up specified MDS rank (use with `set max_mds` to shrink cluster)", \ "mds", "rw", FLAG(OBSOLETE)) +COMMAND("mds ok-to-stop name=ids,type=CephString,n=N", + "check whether stopping the specified MDS would reduce immediate availability", + "mds", "r") COMMAND_WITH_FLAG("mds set_max_mds " \ "name=maxmds,type=CephInt,range=0", \ "set max MDS index", "mds", "rw", FLAG(OBSOLETE))