From: Samuel Just Date: Thu, 6 Nov 2025 23:54:50 +0000 (+0000) Subject: mon: add NVMEOF_BEACON_DIFF to mon_feature_t and mon CompatSet X-Git-Tag: testing/wip-vshankar-testing-20260120.085915-debug~25^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e55ad7bce2fb85096cd31ff9846403f9dbd01e85;p=ceph-ci.git mon: add NVMEOF_BEACON_DIFF to mon_feature_t and mon CompatSet NOPE NOPE In order for the client to safely send BEACON_DIFF messages, it needs to be the case that the leader at the time of receipt will support BEACON_DIFF. Simply using the connection features for the MonClient's target mon is insufficient, because it might be a peon. If the peon supports BEACON_DIFF and the leader does not the leader will either crash or interpret it as a full BEACON. Neither outcome is acceptable. Instead, we need to wire up a feature bit to the MonMap mon_feature_t members and the CompatSet. Adding FEATURE_BEACON_DIFF to ceph::features::mon get_supported() and get_persistent() ensures that once all monitors in the quorum support it, MonMap::get_required_features() will include it. See Elector::propose_to_peers, Monitor::(win|lose)_election, MonmapMonitor::apply_mon_features. Once FEATURE_BEACON_DIFF is present in MonMap::get_required_features(): - Monitor::apply_monmap_to_compatset_features() will prevent downgrades of the monitors by updating the CompatSet to include CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF - Monitor::calc_quorum_requirements() will set Monitor::required_features to require the NVMEOF_BEACON_DIFF for any monitor peers. - MonClient::get_monmap_required_features() will eventually include ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF. Signed-off-by: Samuel Just --- diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 902dd1a6c23..1055463d013 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -564,6 +564,9 @@ CompatSet Monitor::get_supported_features() compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_REEF); compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SQUID); compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_TENTACLE); + + // Release-independent features + compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF); return compat; } @@ -2573,6 +2576,16 @@ void Monitor::apply_monmap_to_compatset_features() new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_TENTACLE); } + + // Release-independent features + if (monmap_features.contains_all(ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF)) { + ceph_assert(ceph::features::mon::get_persistent().contains_all( + ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF)); + // this feature should only ever be set if the quorum supports it. + ceph_assert(HAVE_FEATURE(quorum_con_features, NVMEOF_BEACON_DIFF)); + new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF); + } + dout(5) << __func__ << dendl; _apply_compatset_features(new_features); } @@ -2635,6 +2648,13 @@ void Monitor::calc_quorum_requirements() required_features |= CEPH_FEATUREMASK_SERVER_NAUTILUS | CEPH_FEATUREMASK_CEPHX_V2; } + + // Release-independent features + if (monmap->get_required_features().contains_all( + ceph::features::mon::FEATURE_NVMEOF_BEACON_DIFF)) { + required_features |= CEPH_FEATUREMASK_NVMEOF_BEACON_DIFF; + } + dout(10) << __func__ << " required_features " << required_features << dendl; } diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 93c758dc501..ac27e5394b6 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -1153,6 +1153,9 @@ private: #define CEPH_MON_FEATURE_INCOMPAT_REEF CompatSet::Feature(15, "reef ondisk layout") #define CEPH_MON_FEATURE_INCOMPAT_SQUID CompatSet::Feature(16, "squid ondisk layout") #define CEPH_MON_FEATURE_INCOMPAT_TENTACLE CompatSet::Feature(17, "tentacle ondisk layout") + +// Release-independent features +#define CEPH_MON_FEATURE_INCOMPAT_NVMEOF_BEACON_DIFF CompatSet::Feature(32, "nvmeof beacon diff") // make sure you add your feature to Monitor::get_supported_features diff --git a/src/mon/mon_types.h b/src/mon/mon_types.h index d51847c2b02..273e1346708 100644 --- a/src/mon/mon_types.h +++ b/src/mon/mon_types.h @@ -536,6 +536,10 @@ namespace ceph { constexpr mon_feature_t FEATURE_SQUID( (1ULL << 10)); constexpr mon_feature_t FEATURE_TENTACLE( (1ULL << 11)); + + // Release-independent features + constexpr mon_feature_t FEATURE_NVMEOF_BEACON_DIFF( (1ULL << 32)); + constexpr mon_feature_t FEATURE_RESERVED( (1ULL << 63)); constexpr mon_feature_t FEATURE_NONE( (0ULL)); @@ -558,6 +562,10 @@ namespace ceph { FEATURE_REEF | FEATURE_SQUID | FEATURE_TENTACLE | + + // Release-independent features + FEATURE_NVMEOF_BEACON_DIFF | + FEATURE_NONE ); } @@ -585,6 +593,10 @@ namespace ceph { FEATURE_REEF | FEATURE_SQUID | FEATURE_TENTACLE | + + // Release-independent features + FEATURE_NVMEOF_BEACON_DIFF | + FEATURE_NONE ); } @@ -663,6 +675,9 @@ static inline const char *ceph::features::mon::get_feature_name(uint64_t b) { return "squid"; } else if (f == FEATURE_TENTACLE) { return "tentacle"; + // Release-independent features + } else if (f == FEATURE_NVMEOF_BEACON_DIFF) { + return "nvmeof_beacon_diff"; } else if (f == FEATURE_RESERVED) { return "reserved"; } @@ -695,6 +710,9 @@ inline mon_feature_t ceph::features::mon::get_feature_by_name(const std::string return FEATURE_SQUID; } else if (n == "tentacle") { return FEATURE_TENTACLE; + // Release-independent features + } else if (n == "nvmeof_beacon_diff") { + return FEATURE_NVMEOF_BEACON_DIFF; } else if (n == "reserved") { return FEATURE_RESERVED; } diff --git a/src/test/cli/monmaptool/feature-set-unset-list.t b/src/test/cli/monmaptool/feature-set-unset-list.t index 9622af0f8e2..65841b7600e 100644 --- a/src/test/cli/monmaptool/feature-set-unset-list.t +++ b/src/test/cli/monmaptool/feature-set-unset-list.t @@ -12,21 +12,21 @@ required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] MONMAP FEATURES: persistent: [none] optional: [none] required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] monmap:persistent:[none] monmap:optional:[none] monmap:required:[none] - available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + available:supported:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + available:persistent:[kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] $ monmaptool --feature-set foo /tmp/test.monmap.1234 unknown features name 'foo' or unable to parse value: Expected option value to be integer, got 'foo' @@ -45,8 +45,8 @@ required: [kraken(1),octopus(32),unknown(4096)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] $ monmaptool --feature-unset 32 --optional --feature-list /tmp/test.monmap.1234 monmaptool: monmap file /tmp/test.monmap.1234 @@ -56,8 +56,8 @@ required: [kraken(1),octopus(32),unknown(4096)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ monmaptool --feature-unset 32 --persistent --feature-unset 4096 --optional --feature-list /tmp/test.monmap.1234 @@ -68,8 +68,8 @@ required: [kraken(1)] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ monmaptool --feature-unset kraken --feature-list /tmp/test.monmap.1234 @@ -80,8 +80,8 @@ required: [none] AVAILABLE FEATURES: - supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] - persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048)] + supported: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] + persistent: [kraken(1),luminous(2),mimic(4),osdmap-prune(8),nautilus(16),octopus(32),pacific(64),elector-pinging(128),quincy(256),reef(512),squid(1024),tentacle(2048),nvmeof_beacon_diff(4294967296)] monmaptool: writing epoch 0 to /tmp/test.monmap.1234 (1 monitors) $ rm /tmp/test.monmap.1234