mon/MDSMonitor: propose if FSMap struct_v is too old

To flush older versions which may still be an empty MDSMap (for clusters
that have never used CephFS), we need to force a proposal so older
versions of the struct are trimmed.

This is the main fix of this branch. We removed code which processed old
encodings of the MDSMap in the mon store via 60bc524. That broke old
ceph clusters which never used CephFS (see cited ticket below).  This is
because the initial epoch is an empty MDSMap (back in Infernalis/Hammer)
that is never updated. So, the fix here is to just do proposals
periodically until all of the old structs are automatically trimmed by
the mons.

Fixes: 60bc524827bac072658203e56b1fa3dede9641c5
Fixes: https://tracker.ceph.com/issues/51673
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit 56c3fc802ee8848ba85da4300adcc2ee8bd95416)

diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc
index cdd2a8922151eb6816093c46f01774dc9ba4a47a..802124f9bc445248d1ed4b004c1e3a2277784a14 100644 (file)
--- a/src/mds/FSMap.cc
+++ b/src/mds/FSMap.cc
@@ -622,7 +622,7 @@ void FSMap::update_compat(const CompatSet &c)
 
 void FSMap::encode(bufferlist& bl, uint64_t features) const
 {
-  ENCODE_START(7, 6, bl);
+  ENCODE_START(STRUCT_VERSION, 6, bl);
   encode(epoch, bl);
   encode(next_filesystem_id, bl);
   encode(legacy_client_fscid, bl);
@@ -643,8 +643,9 @@ void FSMap::encode(bufferlist& bl, uint64_t features) const
 
 void FSMap::decode(bufferlist::const_iterator& p)
 {
-  DECODE_START(7, p);
+  DECODE_START(STRUCT_VERSION, p);
   DECODE_OLDEST(7);
+  struct_version = struct_v;
   decode(epoch, p);
   decode(next_filesystem_id, p);
   decode(legacy_client_fscid, p);

diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h
index 8a3a6b69cb4e2c316c3aabb76a81cb00df8db07f..1bd5ca7989582b99d7e33ba95ebc34636b032475 100644 (file)
--- a/src/mds/FSMap.h
+++ b/src/mds/FSMap.h
@@ -219,6 +219,9 @@ public:
   friend class PaxosFSMap;
   using mds_info_t = MDSMap::mds_info_t;
 
+  static const version_t STRUCT_VERSION = 7;
+  static const version_t STRUCT_VERSION_TRIM_TO = 7;
+
   FSMap() : compat(MDSMap::get_compat_set_default()) {}
 
   FSMap(const FSMap &rhs)
@@ -231,7 +234,8 @@ public:
       ever_enabled_multiple(rhs.ever_enabled_multiple),
       mds_roles(rhs.mds_roles),
       standby_daemons(rhs.standby_daemons),
-      standby_epochs(rhs.standby_epochs)
+      standby_epochs(rhs.standby_epochs),
+      struct_version(rhs.struct_version)
   {
     filesystems.clear();
     for (const auto &i : rhs.filesystems) {
@@ -512,6 +516,11 @@ public:
   epoch_t get_epoch() const { return epoch; }
   void inc_epoch() { epoch++; }
 
+  version_t get_struct_version() const { return struct_version; }
+  bool is_struct_old() const {
+    return struct_version < STRUCT_VERSION_TRIM_TO;
+  }
+
   size_t filesystem_count() const {return filesystems.size();}
   bool filesystem_exists(fs_cluster_id_t fscid) const {return filesystems.count(fscid) > 0;}
   Filesystem::const_ref get_filesystem(fs_cluster_id_t fscid) const {return std::const_pointer_cast<const Filesystem>(filesystems.at(fscid));}
@@ -590,6 +599,9 @@ protected:
   // For MDS daemons not yet assigned to a Filesystem
   std::map<mds_gid_t, mds_info_t> standby_daemons;
   std::map<mds_gid_t, epoch_t> standby_epochs;
+
+private:
+  epoch_t struct_version = 0;
 };
 WRITE_CLASS_ENCODER_FEATURES(FSMap)
 

diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc
index 404e5932bcd75b5a1ade42b3c6a0e08fa32d1253..fca12925240f0b793bd466030dff493dff5bbb7a 100644 (file)
--- a/src/mon/MDSMonitor.cc
+++ b/src/mon/MDSMonitor.cc
@@ -2284,6 +2284,34 @@ void MDSMonitor::tick()
   bool do_propose = false;
   bool propose_osdmap = false;
 
+  if (check_fsmap_struct_version) {
+    /* Allow time for trimming otherwise PaxosService::is_writeable will always
+     * be false.
+     */
+
+    auto now = clock::now();
+    auto elapsed = now - last_fsmap_struct_flush;
+    if (elapsed > std::chrono::seconds(30)) {
+      FSMap fsmap;
+      bufferlist bl;
+      auto v = get_first_committed();
+      int err = get_version(v, bl);
+      if (err) {
+        derr << "could not get version " << v << dendl;
+        ceph_abort();
+      }
+      fsmap.decode(bl);
+      if (fsmap.is_struct_old()) {
+        dout(5) << "fsmap struct is too old; proposing to flush out old versions" << dendl;
+        do_propose = true;
+        last_fsmap_struct_flush = now;
+      } else {
+        dout(20) << "struct is recent" << dendl;
+        check_fsmap_struct_version = false;
+      }
+    }
+  }
+
   do_propose |= pending.check_health();
 
   /* Check health and affinity of ranks */

diff --git a/src/mon/MDSMonitor.h b/src/mon/MDSMonitor.h
index d54434b4975e3fee999458ae0e7cd1f491deff82..c70814996fdc4e04cf4349b690a3894e8487a2c6 100644 (file)
--- a/src/mon/MDSMonitor.h
+++ b/src/mon/MDSMonitor.h
@@ -32,6 +32,9 @@ class FileSystemCommandHandler;
 
 class MDSMonitor : public PaxosService, public PaxosFSMap, protected CommandHandler {
  public:
+  using clock = ceph::coarse_mono_clock;
+  using time = ceph::coarse_mono_time;
+
   MDSMonitor(Monitor &mn, Paxos &p, std::string service_name);
 
   // service methods
@@ -146,6 +149,10 @@ protected:
   // when the mon was not updating us for some period (e.g. during slow
   // election) to reset last_beacon timeouts
   ceph::mono_time last_tick = ceph::mono_clock::zero();
+
+private:
+  time last_fsmap_struct_flush = clock::zero();
+  bool check_fsmap_struct_version = true;
 };
 
 #endif