mon/MDSMonitor: add 'mds ok-to-stop' command

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/standalone/ceph-helpers.sh b/qa/standalone/ceph-helpers.sh
index 152396f6c02ee12b7a490c563bc8ec626f159f9c..83ca1e2511e23c71fa9ca859065e60a331d31bb0 100755 (executable)
--- a/qa/standalone/ceph-helpers.sh
+++ b/qa/standalone/ceph-helpers.sh
@@ -574,6 +574,28 @@ function run_mgr() {
         "$@" || return 1
 }
 
+function run_mds() {
+    local dir=$1
+    shift
+    local id=$1
+    shift
+    local data=$dir/$id
+
+    ceph-mds \
+        --id $id \
+        $EXTRA_OPTS \
+       --debug-mds 20 \
+       --debug-objecter 20 \
+        --debug-ms 20 \
+        --chdir= \
+        --mds-data=$data \
+        --log-file=$dir/\$name.log \
+        --admin-socket=$(get_asok_path) \
+        --run-dir=$dir \
+        --pid-file=$dir/\$name.pid \
+        "$@" || return 1
+}
+
 #######################################################################
 
 ##

diff --git a/qa/standalone/misc/ok-to-stop.sh b/qa/standalone/misc/ok-to-stop.sh
index 9894d3bf4bfd72a6cdea89af356e3fb5929447fe..e53b28d1eb28b0ca6baeb0df54ef4aea0ba03714 100755 (executable)
--- a/qa/standalone/misc/ok-to-stop.sh
+++ b/qa/standalone/misc/ok-to-stop.sh
@@ -204,4 +204,38 @@ function TEST_5_mons_checks() {
     ceph mon ok-to-rm e || return 1
 }
 
+function TEST_0_mds() {
+    local dir=$1
+
+    CEPH_ARGS="$ORIG_CEPH_ARGS --mon-host=$CEPH_MON_A "
+
+    run_mon $dir a --public-addr=$CEPH_MON_A || return 1
+    run_mgr $dir x || return 1
+    run_osd $dir 0 || return 1
+    run_mds $dir a || return 1
+
+    ceph osd pool create meta 1 || return 1
+    ceph osd pool create data 1 || return 1
+    ceph fs new myfs meta data || return 1
+    sleep 5
+
+    ! ceph mds ok-to-stop a || return 1
+    ! ceph mds ok-to-stop a dne || return 1
+    ceph mds ok-to-stop dne || return 1
+
+    run_mds $dir b || return 1
+    sleep 5
+
+    ceph mds ok-to-stop a || return 1
+    ceph mds ok-to-stop b || return 1
+    ! ceph mds ok-to-stop a b || return 1
+    ceph mds ok-to-stop a dne1 dne2 || return 1
+    ceph mds ok-to-stop b dne || return 1
+    ! ceph mds ok-to-stop a b dne || return 1
+    ceph mds ok-to-stop dne1 dne2 || return 1
+
+    kill_daemons $dir KILL mds.a
+}
+
+
 main ok-to-stop "$@"

diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h
index ca286b290c45eaaefa354ab9f72bc1a9f2d97d21..2e7028a4c05fd9b0467e346985f846594e7bbf79 100644 (file)
--- a/src/mds/FSMap.h
+++ b/src/mds/FSMap.h
@@ -65,15 +65,27 @@ public:
    * STANDBY_REPLAY for the gid `who`
    */
   bool has_standby_replay(mds_gid_t who) const
+  {
+    return get_standby_replay(who) != MDS_GID_NONE;
+  }
+  mds_gid_t get_standby_replay(mds_gid_t who) const
   {
     for (const auto &i : mds_map.mds_info) {
       const auto &info = i.second;
       if (info.state == MDSMap::STATE_STANDBY_REPLAY
           && info.rank == mds_map.mds_info.at(who).rank) {
-        return true;
+        return info.global_id;
       }
     }
-
+    return MDS_GID_NONE;
+  }
+  bool is_standby_replay(mds_gid_t who) const
+  {
+    auto p = mds_map.mds_info.find(who);
+    if (p != mds_map.mds_info.end() &&
+       p->second.state == MDSMap::STATE_STANDBY_REPLAY) {
+      return true;
+    }
     return false;
   }
 
@@ -173,6 +185,19 @@ public:
     return legacy_client_fscid;
   }
 
+  size_t get_num_standby() const {
+    return standby_daemons.size();
+  }
+
+  bool is_any_degraded() const {
+    for (auto& i : filesystems) {
+      if (i.second->mds_map.is_degraded()) {
+       return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Get state of all daemons (for all filesystems, including all standbys)
    */
@@ -370,6 +395,16 @@ public:
     }
   }
 
+  bool is_standby_replay(mds_gid_t who) const
+  {
+    return filesystems.at(mds_roles.at(who))->is_standby_replay(who);
+  }
+
+  mds_gid_t get_standby_replay(mds_gid_t who) const
+  {
+    return filesystems.at(mds_roles.at(who))->get_standby_replay(who);
+  }
+
   /**
    * A daemon has told us it's compat, and it's too new
    * for the one we had previously.  Impose the new one

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 19bfdcd56423ebf98a13a1d9d7312f417c7539de..6258c20f861e8c8cbb8368ffe80c6a83a234a073 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -897,6 +897,57 @@ bool MDSMonitor::preprocess_command(MonOpRequestRef op)
       ds << fsmap;
     }
     r = 0;
+  } else if (prefix == "mds ok-to-stop") {
+    vector<string> ids;
+    if (!cmd_getval(g_ceph_context, cmdmap, "ids", ids)) {
+      r = -EINVAL;
+      ss << "must specify mds id";
+      goto out;
+    }
+    if (fsmap.is_any_degraded()) {
+      ss << "one or more filesystems is currently degraded";
+      r = -EBUSY;
+      goto out;
+    }
+    set<mds_gid_t> stopping;
+    for (auto& id : ids) {
+      ostringstream ess;
+      mds_gid_t gid = gid_from_arg(fsmap, id, ess);
+      if (gid == MDS_GID_NONE) {
+       // the mds doesn't exist, but no file systems are unhappy, so losing it
+       // can't have any effect.
+       continue;
+      }
+      stopping.insert(gid);
+    }
+    set<mds_gid_t> active;
+    set<mds_gid_t> standby;
+    for (auto gid : stopping) {
+      if (fsmap.gid_has_rank(gid)) {
+       // ignore standby-replay daemons (at this level)
+       if (!fsmap.is_standby_replay(gid)) {
+         auto standby = fsmap.get_standby_replay(gid);
+         if (standby == MDS_GID_NONE ||
+             stopping.count(standby)) {
+           // no standby-replay, or we're also stopping the standby-replay
+           // for this mds
+           active.insert(gid);
+         }
+       }
+      } else {
+       // net loss of a standby
+       standby.insert(gid);
+      }
+    }
+    if (fsmap.get_num_standby() - standby.size() < active.size()) {
+      r = -EBUSY;
+      ss << "insufficent standby MDS daemons to stop active gids "
+        << stringify(active)
+        << " and/or standby gids " << stringify(standby);;
+      goto out;
+    }
+    r = 0;
+    ss << "should be safe to stop " << ids;
   } else if (prefix == "fs dump") {
     int64_t epocharg;
     epoch_t epoch;

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 7394564fe0aa8bef883b03629bc73a384c25258c..b6b93293625ccd7e4ab8ee872e8fa2a564e83c4f 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -325,6 +325,9 @@ COMMAND_WITH_FLAG("mds stop name=role,type=CephString", "stop mds", \
 COMMAND_WITH_FLAG("mds deactivate name=role,type=CephString",
         "clean up specified MDS rank (use with `set max_mds` to shrink cluster)", \
        "mds", "rw", FLAG(OBSOLETE))
+COMMAND("mds ok-to-stop name=ids,type=CephString,n=N",
+       "check whether stopping the specified MDS would reduce immediate availability",
+       "mds", "r")
 COMMAND_WITH_FLAG("mds set_max_mds " \
        "name=maxmds,type=CephInt,range=0", \
        "set max MDS index", "mds", "rw", FLAG(OBSOLETE))