mds: automaticly allow multi-active MDS after removing all old snapshots

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/MDSMap.h b/src/mds/MDSMap.h
index 1c30686310ebb92823ef3640f366fbef1328f7af..b1b52e672f40f4f16b17051498269bc42c43f704 100644 (file)
--- a/src/mds/MDSMap.h
+++ b/src/mds/MDSMap.h
@@ -283,6 +283,12 @@ public:
   bool allows_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_SNAPS); }
   bool was_snaps_ever_allowed() const { return ever_allowed_features & CEPH_MDSMAP_ALLOW_SNAPS; }
 
+  void set_multimds_snaps_allowed() {
+    set_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS);
+    ever_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS;
+    explicitly_allowed_features |= CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS;
+  }
+  void clear_multimds_snaps_allowed() { clear_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); }
   bool allows_multimds_snaps() const { return test_flag(CEPH_MDSMAP_ALLOW_MULTIMDS_SNAPS); }
 
   epoch_t get_epoch() const { return epoch; }

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index ca60ea602854d2d162813381910c0b0e4d4e498b..b26c9063e31206c4c1d147f336feb29d1ad1395c 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -240,6 +240,47 @@ void MDSRank::hit_export_target(utime_t now, mds_rank_t rank, double amount)
   em.first->second.hit(now, amount);
 }
 
+class C_MDS_MonCommand : public MDSInternalContext {
+  std::string cmd;
+public:
+  std::string outs;
+  C_MDS_MonCommand(MDSRank *m, std::string_view c)
+    : MDSInternalContext(m), cmd(c) {}
+  void finish(int r) override {
+    mds->_mon_command_finish(r, cmd, outs);
+  }
+};
+
+void MDSRank::_mon_command_finish(int r, std::string_view cmd, std::string_view outs)
+{
+  if (r < 0) {
+    dout(0) << __func__ << ": mon command " << cmd << " failed with errno " << r
+           << " (" << outs << ")" << dendl;
+  } else {
+    dout(1) << __func__ << ": mon command " << cmd << " succeed" << dendl;
+  }
+}
+
+void MDSRank::set_mdsmap_multimds_snaps_allowed()
+{
+  static bool already_sent = false;
+  if (already_sent)
+    return;
+
+  stringstream ss;
+  ss << "{\"prefix\":\"fs set\", \"fs_name\":\"" <<  mdsmap->get_fs_name() << "\", ";
+  ss << "\"var\":\"allow_multimds_snaps\", \"val\":\"true\", ";
+  ss << "\"confirm\":\"--yes-i-am-really-a-mds\"}";
+  std::vector<std::string> cmd = {ss.str()};
+
+  dout(0) << __func__ << ": sending mon command: " << cmd[0] << dendl;
+
+  C_MDS_MonCommand *fin = new C_MDS_MonCommand(this, cmd[0]);
+  monc->start_mon_command(cmd, {}, nullptr, &fin->outs, new C_IO_Wrapper(this, fin));
+
+  already_sent = true;
+}
+
 void MDSRankDispatcher::tick()
 {
   heartbeat_reset();
@@ -285,8 +326,16 @@ void MDSRankDispatcher::tick()
     balancer->tick();
     mdcache->find_stale_fragment_freeze();
     mdcache->migrator->find_stale_export_freeze();
-    if (snapserver)
+
+    if (mdsmap->get_tableserver() == whoami) {
       snapserver->check_osd_map(false);
+      // Filesystem was created by pre-mimic mds. Allow multi-active mds after
+      // all old snapshots are deleted.
+      if (!mdsmap->allows_multimds_snaps() &&
+         snapserver->can_allow_multimds_snaps()) {
+       set_mdsmap_multimds_snaps_allowed();
+      }
+    }
   }
 
   if (is_active() || is_stopping()) {
@@ -2755,7 +2804,8 @@ void MDSRank::check_ops_in_flight()
 
 void MDSRankDispatcher::handle_osd_map()
 {
-  if (is_active() && snapserver) {
+  if (is_active() &&
+      mdsmap->get_tableserver() == whoami) {
     snapserver->check_osd_map(true);
   }
 

diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index ef77333dff65e4bc744fdcb2971d9884879e86c9..983a6c3c7d62c2e39c79f04824c22899cc28906a 100644 (file)
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -217,6 +217,7 @@ class MDSRank {
     bool is_any_replay() const { return (is_replay() || is_standby_replay()); }
     bool is_stopped() const { return mdsmap->is_stopped(whoami); }
     bool is_cluster_degraded() const { return cluster_degraded; }
+    bool allows_multimds_snaps() const { return mdsmap->allows_multimds_snaps(); }
 
     void handle_write_error(int err);
 
@@ -523,6 +524,9 @@ class MDSRank {
     /* Update MDSMap export_targets for this rank. Called on ::tick(). */
     void update_targets(utime_t now);
 
+    friend class C_MDS_MonCommand;
+    void _mon_command_finish(int r, std::string_view cmd, std::string_view outs);
+    void set_mdsmap_multimds_snaps_allowed();
 private:
     mono_time starttime = mono_clock::zero();
 };

diff --git a/src/mds/SnapRealm.cc b/src/mds/SnapRealm.cc
index 935880a2449d9ad13231200ff331ede8316ba083..4a64415ac6f42bfdb420ec3c3c38e0492321ca59 100644 (file)
--- a/src/mds/SnapRealm.cc
+++ b/src/mds/SnapRealm.cc
@@ -154,6 +154,13 @@ bool SnapRealm::_open_parents(MDSInternalContextBase *finish, snapid_t first, sn
   if (!srnode.past_parent_snaps.empty())
     assert(mdcache->mds->snapclient->get_cached_version() > 0);
 
+  if (!srnode.past_parents.empty() &&
+      mdcache->mds->allows_multimds_snaps()) {
+    dout(10) << " skip non-empty past_parents since multimds_snaps is allowed" << dendl;
+    open = true;
+    return true;
+  }
+
   // and my past parents too!
   assert(srnode.past_parents.size() >= num_open_past_parents);
   if (srnode.past_parents.size() > num_open_past_parents) {
@@ -205,6 +212,13 @@ bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last) const
   if (!srnode.past_parent_snaps.empty())
     assert(mdcache->mds->snapclient->get_cached_version() > 0);
 
+  if (!srnode.past_parents.empty() &&
+      mdcache->mds->allows_multimds_snaps()) {
+    dout(10) << " skip non-empty past_parents since multimds_snaps is allowed" << dendl;
+    open = true;
+    return true;
+  }
+
   for (auto p = srnode.past_parents.lower_bound(first);
        p != srnode.past_parents.end();
        ++p) {

diff --git a/src/mds/SnapServer.cc b/src/mds/SnapServer.cc
index 7e97ab8b0ef864d970478ede7c80d32c0b9b5694..b7ae4519a162da269fe1df45e8df3f086b4580c7 100644 (file)
--- a/src/mds/SnapServer.cc
+++ b/src/mds/SnapServer.cc
@@ -60,6 +60,7 @@ void SnapServer::reset_state()
   }
   last_created = last_snap;
   last_destroyed = last_snap;
+  snaprealm_v2_since = last_snap + 1;
   version++;
 }
 

diff --git a/src/mds/SnapServer.h b/src/mds/SnapServer.h
index 94f7a5bfa11670cb473f880367265faf2cc02fc2..3475099d35015c869e5762d2db784447b7f52130 100644 (file)
--- a/src/mds/SnapServer.h
+++ b/src/mds/SnapServer.h
@@ -26,6 +26,7 @@ protected:
   MonClient *mon_client = nullptr;
   snapid_t last_snap;
   snapid_t last_created, last_destroyed;
+  snapid_t snaprealm_v2_since;
   map<snapid_t, SnapInfo> snaps;
   map<int, set<snapid_t> > need_to_purge;
   
@@ -36,7 +37,7 @@ protected:
   version_t last_checked_osdmap;
 
   void encode_server_state(bufferlist& bl) const override {
-    ENCODE_START(4, 3, bl);
+    ENCODE_START(5, 3, bl);
     encode(last_snap, bl);
     encode(snaps, bl);
     encode(need_to_purge, bl);
@@ -45,10 +46,11 @@ protected:
     encode(pending_noop, bl);
     encode(last_created, bl);
     encode(last_destroyed, bl);
+    encode(snaprealm_v2_since, bl);
     ENCODE_FINISH(bl);
   }
   void decode_server_state(bufferlist::iterator& bl) override {
-    DECODE_START_LEGACY_COMPAT_LEN(4, 3, 3, bl);
+    DECODE_START_LEGACY_COMPAT_LEN(5, 3, 3, bl);
     decode(last_snap, bl);
     decode(snaps, bl);
     decode(need_to_purge, bl);
@@ -69,6 +71,11 @@ protected:
       last_created = last_snap;
       last_destroyed = last_snap;
     }
+    if (struct_v >= 5)
+      decode(snaprealm_v2_since, bl);
+    else
+      snaprealm_v2_since = last_snap + 1;
+
     DECODE_FINISH(bl);
   }
 
@@ -90,6 +97,10 @@ public:
 
   void check_osd_map(bool force);
 
+  bool can_allow_multimds_snaps() const {
+    return snaps.empty() || snaps.begin()->first >= snaprealm_v2_since;
+  }
+
   void encode(bufferlist& bl) const {
     encode_server_state(bl);
   }

diff --git a/src/mon/FSCommands.cc b/src/mon/FSCommands.cc
index 25792b97610a8d6342ddbb5dbf03665968ab188b..054dd8beb88c04a2fd5f407613b1cb3632c96d5c 100644 (file)
--- a/src/mon/FSCommands.cc
+++ b/src/mon/FSCommands.cc
@@ -380,6 +380,37 @@ public:
            << " parameter to control the number of active MDSs"
            << " allowed. This command is DEPRECATED and will be"
            << " REMOVED from future releases.";
+    } else if (var == "allow_multimds_snaps") {
+      bool enable = false;
+      int r = parse_bool(val, &enable, ss);
+      if (r != 0) {
+        return r;
+      }
+
+      string confirm;
+      if (!cmd_getval(g_ceph_context, cmdmap, "confirm", confirm) ||
+         confirm != "--yes-i-am-really-a-mds") {
+       ss << "Warning! This command is for MDS only. Do not run it manually";
+       return -EPERM;
+      }
+
+      if (enable) {
+       ss << "enabled multimds with snapshot";
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+         fs->mds_map.set_multimds_snaps_allowed();
+        });
+      } else {
+       ss << "disabled multimds with snapshot";
+        fsmap.modify_filesystem(
+            fs->fscid,
+            [](std::shared_ptr<Filesystem> fs)
+        {
+         fs->mds_map.clear_multimds_snaps_allowed();
+        });
+      }
     } else if (var == "allow_dirfrags") {
         ss << "Directory fragmentation is now permanently enabled."
            << " This command is DEPRECATED and will be REMOVED from future releases.";

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 9e0079e3371e179d29cdad0f00ba9ef5cd519e43..0c7d18f0a7d97010bec1538dc6fac5485521b2cd 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -315,8 +315,8 @@ COMMAND_WITH_FLAG("mds set_max_mds " \
        "name=maxmds,type=CephInt,range=0", \
        "set max MDS index", "mds", "rw", "cli,rest", FLAG(OBSOLETE))
 COMMAND_WITH_FLAG("mds set " \
-       "name=var,type=CephChoices,strings=max_mds|max_file_size"
-       "|allow_new_snaps|inline_data|allow_multimds|allow_dirfrags " \
+       "name=var,type=CephChoices,strings=max_mds|max_file_size|inline_data|"
+       "allow_new_snaps|allow_multimds|allow_multimds_snaps|allow_dirfrags " \
        "name=val,type=CephString "                                     \
        "name=confirm,type=CephString,req=false",                       \
        "set mds parameter <var> to <val>", "mds", "rw", "cli,rest", FLAG(OBSOLETE))