mon: add NVMEOF_BEACON_DIFF to mon_feature_t and mon CompatSet

NOPE NOPE
In order for the client to safely send BEACON_DIFF messages, it
needs to be the case that the leader at the time of receipt will
support BEACON_DIFF.

Simply using the connection features for the MonClient's target mon is
insufficient, because it might be a peon.  If the peon supports
BEACON_DIFF and the leader does not the leader will either crash or
interpret it as a full BEACON.  Neither outcome is acceptable.

Instead, we need to wire up a feature bit to the MonMap mon_feature_t
members and the CompatSet.

Adding FEATURE_BEACON_DIFF to ceph::features::mon get_supported()
and get_persistent() ensures that once all monitors in the quorum
support it, MonMap::get_required_features() will include it.
See Elector::propose_to_peers, Monitor::(win|lose)_election,
MonmapMonitor::apply_mon_features.

Once FEATURE_BEACON_DIFF is present in MonMap::get_required_features():
- Monitor::apply_monmap_to_compatset_features() will prevent
  downgrades of the monitors by updating the CompatSet to include
  CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF
- Monitor::calc_quorum_requirements() will set
  Monitor::required_features to require the NVMEOF_BEACON_DIFF
  for any monitor peers.
- MonClient::get_monmap_required_features() will eventually include
  ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF.

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 902dd1a6c2397eff14b52e8bc11fa8fc4ebe8b5e..1055463d013f21e2703fb5eae5aaf16a3bfac613 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -564,6 +564,9 @@ CompatSet Monitor::get_supported_features()
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_REEF);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SQUID);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_TENTACLE);
+
+  // Release-independent features
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF);
   return compat;
 }
 
@@ -2573,6 +2576,16 @@ void Monitor::apply_monmap_to_compatset_features()
     new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_TENTACLE);
   }
 
+
+  // Release-independent features
+  if (monmap_features.contains_all(ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF)) {
+    ceph_assert(ceph::features::mon::get_persistent().contains_all(
+           ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF));
+    // this feature should only ever be set if the quorum supports it.
+    ceph_assert(HAVE_FEATURE(quorum_con_features, NVMEOF_BEACON_DIFF));
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF);
+  }
+
   dout(5) << __func__ << dendl;
   _apply_compatset_features(new_features);
 }
@@ -2635,6 +2648,13 @@ void Monitor::calc_quorum_requirements()
     required_features |= CEPH_FEATUREMASK_SERVER_NAUTILUS |
       CEPH_FEATUREMASK_CEPHX_V2;
   }
+
+  // Release-independent features
+  if (monmap->get_required_features().contains_all(
+       ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF)) {
+    required_features |= CEPH_FEATUREMASK_NVMEOF_BEACON_DIFF;
+  }
+
   dout(10) << __func__ << " required_features " << required_features << dendl;
 }
 

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 93c758dc5019a59bd6fdaaaf4087e86ccb2f3cec..ac27e5394b6c8ec79e7e102865f444bc46863786 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -1153,6 +1153,9 @@ private:
 #define CEPH_MON_FEATURE_INCOMPAT_REEF CompatSet::Feature(15, "reef ondisk layout")
 #define CEPH_MON_FEATURE_INCOMPAT_SQUID CompatSet::Feature(16, "squid ondisk layout")
 #define CEPH_MON_FEATURE_INCOMPAT_TENTACLE CompatSet::Feature(17, "tentacle ondisk layout")
+
+// Release-independent features
+#define CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF CompatSet::Feature(32, "nvmeof beacon diff")
 // make sure you add your feature to Monitor::get_supported_features
 
 

diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h
index d51847c2b02d561d6285b38d9ac6fb5f5d646708..273e13467081b49530b99259addff11a0d20d185 100644 (file)
--- a/src/mon/mon_types.h
+++ b/src/mon/mon_types.h
@@ -536,6 +536,10 @@ namespace ceph {
       constexpr mon_feature_t FEATURE_SQUID(      (1ULL << 10));
       constexpr mon_feature_t FEATURE_TENTACLE(   (1ULL << 11));
 
+
+      // Release-independent features
+      constexpr mon_feature_t FEATURE_NVMEOF_BEACON_DIFF(   (1ULL << 32));
+
       constexpr mon_feature_t FEATURE_RESERVED(   (1ULL << 63));
       constexpr mon_feature_t FEATURE_NONE(       (0ULL));
 
@@ -558,6 +562,10 @@ namespace ceph {
          FEATURE_REEF |
          FEATURE_SQUID |
          FEATURE_TENTACLE |
+
+         // Release-independent features
+         FEATURE_NVMEOF_BEACON_DIFF |
+
          FEATURE_NONE
          );
       }
@@ -585,6 +593,10 @@ namespace ceph {
          FEATURE_REEF |
          FEATURE_SQUID |
          FEATURE_TENTACLE |
+
+         // Release-independent features
+         FEATURE_NVMEOF_BEACON_DIFF |
+
          FEATURE_NONE
          );
       }
@@ -663,6 +675,9 @@ static inline const char *ceph::features::mon::get_feature_name(uint64_t b) {
     return "squid";
   } else if (f == FEATURE_TENTACLE) {
     return "tentacle";
+  // Release-independent features
+  } else if (f == FEATURE_NVMEOF_BEACON_DIFF) {
+    return "nvmeof_beacon_diff";
   } else if (f == FEATURE_RESERVED) {
     return "reserved";
   }
@@ -695,6 +710,9 @@ inline mon_feature_t ceph::features::mon::get_feature_by_name(const std::string
     return FEATURE_SQUID;
   } else if (n == "tentacle") {
     return FEATURE_TENTACLE;
+  // Release-independent features
+  } else if (n == "nvmeof_beacon_diff") {
+    return FEATURE_NVMEOF_BEACON_DIFF;
   } else if (n == "reserved") {
     return FEATURE_RESERVED;
   }

diff --git a/src/test/cli/monmaptool/feature-set-unset-list.t b/src/test/cli/monmaptool/feature-set-unset-list.t
index 9622af0f8e2611ca7bac43a758ac5a91af56116d..65841b7600ef1acbd71a2615ab6213bb82e368c0 100644 (file)
--- a/src/test/cli/monmaptool/feature-set-unset-list.t
+++ b/src/test/cli/monmaptool/feature-set-unset-list.t
@@ -12,21 +12,21 @@
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
   MONMAP FEATURES:
       persistent: [none]
       optional:   [none]
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
   monmap:persistent:[none]
   monmap:optional:[none]
   monmap:required:[none]
-  available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-  available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+  available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+  available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
 
   $ monmaptool --feature-set foo /tmp/test.monmap.1234
   unknown features name 'foo' or unable to parse value: Expected option value to be integer, got 'foo'
@@ -45,8 +45,8 @@
       required:   [kraken(1),octopus(32),unknown(4096)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
 
   $ monmaptool --feature-unset 32 --optional --feature-list /tmp/test.monmap.1234
   monmaptool: monmap file /tmp/test.monmap.1234
@@ -56,8 +56,8 @@
       required:   [kraken(1),octopus(32),unknown(4096)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ monmaptool --feature-unset 32 --persistent --feature-unset 4096 --optional --feature-list /tmp/test.monmap.1234
@@ -68,8 +68,8 @@
       required:   [kraken(1)]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ monmaptool --feature-unset kraken --feature-list /tmp/test.monmap.1234
@@ -80,8 +80,8 @@
       required:   [none]
   
   AVAILABLE FEATURES:
-      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
-      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)]
+      supported:  [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
+      persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)]
   monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors)
 
   $ rm /tmp/test.monmap.1234