mon: add a STRETCH_MODE feature and check it

We look at the up OSDs when enabling stretch mode to make sure the running
ones can handle it. OSDs require their peers to support stretch mode
when connecting, via OSDMap::get_features(). Monitors reject booting OSDs
if stretch mode is enabled and the OSD doesn't support it.

The monitors are already validated and gated by having CONNECTIVITY
election mode enabled.

Signed-off-by: Greg Farnum <gfarnum@redhat.com>

diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 88813e4f440fc0759653c73dc1a168b8a283520f..df2d6e1e47d23f4571674f527fb7e6acbfca201f 100644 (file)
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -115,7 +115,7 @@ DEFINE_CEPH_FEATURE(21, 2, RADOS_BACKOFF)    // overlap
 DEFINE_CEPH_FEATURE(21, 2, OSDMAP_PG_UPMAP)  // overlap
 DEFINE_CEPH_FEATURE(21, 2, CRUSH_CHOOSE_ARGS) // overlap
 DEFINE_CEPH_FEATURE_RETIRED(22, 1, BACKFILL_RESERVATION, JEWEL, LUMINOUS)
-// available
+DEFINE_CEPH_FEATURE(22, 3, STRETCH_MODE)
 DEFINE_CEPH_FEATURE(23, 1, MSG_AUTH)         // 3.19 req (unless nocephx_require_signatures)
 DEFINE_CEPH_FEATURE_RETIRED(24, 1, RECOVERY_RESERVATION, JEWEL, LUMINOUS)
 DEFINE_CEPH_FEATURE(24, 2, RECOVERY_RESERVATION_2)
@@ -244,6 +244,7 @@ DEFINE_CEPH_FEATURE_DEPRECATED(63, 1, RESERVED_BROKEN, LUMINOUS) // client-facin
         CEPH_FEATURE_CEPHX_V2 | \
         CEPH_FEATURE_OSD_PGLOG_HARDLIMIT | \
         CEPH_FEATUREMASK_SERVER_OCTOPUS | \
+        CEPH_FEATUREMASK_STRETCH_MODE | \
         CEPH_FEATUREMASK_OSD_REPOP_MLCOD | \
         CEPH_FEATUREMASK_SERVER_PACIFIC | \
         0ULL)

diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc
index b2159c2ef1b29b5e0996073a439d13865e30e9e9..7f47516335557bce5ae7b2ac3a11a1ebf84f643d 100644 (file)
--- a/src/mon/MonmapMonitor.cc
+++ b/src/mon/MonmapMonitor.cc
@@ -1019,11 +1019,6 @@ bool MonmapMonitor::prepare_command(MonOpRequestRef op)
       return false;
     }
     {
-      struct Plugger {
-       Paxos *p;
-       Plugger(Paxos *p) : p(p) { p->plug(); }
-       ~Plugger() { p->unplug(); }
-      } plugger(paxos);
       if (monmap.stretch_mode_enabled) {
        ss << "stretch mode is already engaged";
        err = -EINVAL;
@@ -1052,10 +1047,20 @@ bool MonmapMonitor::prepare_command(MonOpRequestRef op)
        err = -EINVAL;
        goto reply;
       }
+      //okay, initial arguments make sense, check pools and cluster state
+      err = mon->osdmon()->check_cluster_features(CEPH_FEATUREMASK_STRETCH_MODE, ss);
+      if (err)
+       goto reply;
+      struct Plugger {
+       Paxos *p;
+       Plugger(Paxos *p) : p(p) { p->plug(); }
+       ~Plugger() { p->unplug(); }
+      } plugger(paxos);
+
       set<pg_pool_t*> pools;
       bool okay = false;
       int errcode = 0;
-      //okay, initial arguments make sense, check pools and cluster state
+
       mon->osdmon()->try_enable_stretch_mode_pools(ss, &okay, &errcode,
                                                   &pools, new_crush_rule);
       if (!okay) {

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index ed13719fe4a08adec71e1b7137bf86650d78191c..02ed3692ffb8de44d199b44ba29d6bab836b36b4 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -3506,6 +3506,14 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
     goto ignore;
   }
 
+  if (osdmap.stretch_mode_enabled &&
+      !(m->osd_features & CEPH_FEATUREMASK_STRETCH_MODE)) {
+    mon->clog->info() << "disallowing boot of OSD "
+                     << m->get_orig_source_inst()
+                     << " because stretch mode is on and OSD lacks support";
+    goto ignore;
+  }
+
   // already booted?
   if (osdmap.is_up(from) &&
       osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs()) &&

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index 5008da80dd61a7444c11803640c7f0ddc454d70f..fc5e41586897fd52f2a09f93f7add80a18bb3bbe 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -356,7 +356,6 @@ private:
   void encode_trim_extra(MonitorDBStore::TransactionRef tx, version_t first) override;
 
   void update_msgr_features();
-  int check_cluster_features(uint64_t features, std::stringstream &ss);
   /**
    * check if the cluster supports the features required by the
    * given crush map. Outputs the daemons which don't support it
@@ -412,6 +411,12 @@ private:
   void send_full(MonOpRequestRef op);
   void send_incremental(MonOpRequestRef op, epoch_t first);
 public:
+  /**
+   * Make sure the existing (up) OSDs support the given features
+   * @return 0 on success, or an error code if any OSDs re missing features.
+   * @param ss Filled in with ane explanation of failure, if any
+   */
+  int check_cluster_features(uint64_t features, std::stringstream &ss);
   // @param req an optional op request, if the osdmaps are replies to it. so
   //            @c Monitor::send_reply() can mark_event with it.
   void send_incremental(epoch_t first, MonSession *session, bool onetime,

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index f795900db570e5541b0cdf635000db2326df7ec7..5aa14d2b0c80b3f261fa6dc0a81bedc2b1ea0bc6 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1654,6 +1654,11 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
       features |= kraken_features;
     }
     mask |= kraken_features;
+
+    if (stretch_mode_enabled) {
+      features |= CEPH_FEATUREMASK_STRETCH_MODE;
+      mask |= CEPH_FEATUREMASK_STRETCH_MODE;
+    }
   }
 
   if (require_min_compat_client >= ceph_release_t::nautilus) {

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 612f91f122a9528c18439e1ad2d20f436890f1bf..2e9dbcc8158e1da23e9832010db7676b78f5492e 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -553,7 +553,6 @@ private:
     CEPH_FEATUREMASK_SERVER_MIMIC |
     CEPH_FEATUREMASK_SERVER_NAUTILUS |
     CEPH_FEATUREMASK_SERVER_OCTOPUS;
-  // TODO: add a SIGNIFICANT_FEATURE for stretch mode...
 
   struct addrs_s {
     mempool::osdmap::vector<std::shared_ptr<entity_addrvec_t> > client_addrs;