From: Greg Farnum Date: Thu, 2 Jul 2020 18:52:12 +0000 (+0000) Subject: mon: add a STRETCH_MODE feature and check it X-Git-Tag: v16.1.0~1053^2~11 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4cb4f2130bf2e4f28d7b9037d46786999ccd85f3;p=ceph.git mon: add a STRETCH_MODE feature and check it We look at the up OSDs when enabling stretch mode to make sure the running ones can handle it. OSDs require their peers to support stretch mode when connecting, via OSDMap::get_features(). Monitors reject booting OSDs if stretch mode is enabled and the OSD doesn't support it. The monitors are already validated and gated by having CONNECTIVITY election mode enabled. Signed-off-by: Greg Farnum --- diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h index 88813e4f440..df2d6e1e47d 100644 --- a/src/include/ceph_features.h +++ b/src/include/ceph_features.h @@ -115,7 +115,7 @@ DEFINE_CEPH_FEATURE(21, 2, RADOS_BACKOFF) // overlap DEFINE_CEPH_FEATURE(21, 2, OSDMAP_PG_UPMAP) // overlap DEFINE_CEPH_FEATURE(21, 2, CRUSH_CHOOSE_ARGS) // overlap DEFINE_CEPH_FEATURE_RETIRED(22, 1, BACKFILL_RESERVATION, JEWEL, LUMINOUS) -// available +DEFINE_CEPH_FEATURE(22, 3, STRETCH_MODE) DEFINE_CEPH_FEATURE(23, 1, MSG_AUTH) // 3.19 req (unless nocephx_require_signatures) DEFINE_CEPH_FEATURE_RETIRED(24, 1, RECOVERY_RESERVATION, JEWEL, LUMINOUS) DEFINE_CEPH_FEATURE(24, 2, RECOVERY_RESERVATION_2) @@ -244,6 +244,7 @@ DEFINE_CEPH_FEATURE_DEPRECATED(63, 1, RESERVED_BROKEN, LUMINOUS) // client-facin CEPH_FEATURE_CEPHX_V2 | \ CEPH_FEATURE_OSD_PGLOG_HARDLIMIT | \ CEPH_FEATUREMASK_SERVER_OCTOPUS | \ + CEPH_FEATUREMASK_STRETCH_MODE | \ CEPH_FEATUREMASK_OSD_REPOP_MLCOD | \ CEPH_FEATUREMASK_SERVER_PACIFIC | \ 0ULL) diff --git a/src/mon/MonmapMonitor.cc b/src/mon/MonmapMonitor.cc index b2159c2ef1b..7f475163355 100644 --- a/src/mon/MonmapMonitor.cc +++ b/src/mon/MonmapMonitor.cc @@ -1019,11 +1019,6 @@ bool MonmapMonitor::prepare_command(MonOpRequestRef op) return false; } { - struct Plugger { - Paxos *p; - Plugger(Paxos *p) : p(p) { p->plug(); } - ~Plugger() { p->unplug(); } - } plugger(paxos); if (monmap.stretch_mode_enabled) { ss << "stretch mode is already engaged"; err = -EINVAL; @@ -1052,10 +1047,20 @@ bool MonmapMonitor::prepare_command(MonOpRequestRef op) err = -EINVAL; goto reply; } + //okay, initial arguments make sense, check pools and cluster state + err = mon->osdmon()->check_cluster_features(CEPH_FEATUREMASK_STRETCH_MODE, ss); + if (err) + goto reply; + struct Plugger { + Paxos *p; + Plugger(Paxos *p) : p(p) { p->plug(); } + ~Plugger() { p->unplug(); } + } plugger(paxos); + set pools; bool okay = false; int errcode = 0; - //okay, initial arguments make sense, check pools and cluster state + mon->osdmon()->try_enable_stretch_mode_pools(ss, &okay, &errcode, &pools, new_crush_rule); if (!okay) { diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index ed13719fe4a..02ed3692ffb 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -3506,6 +3506,14 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op) goto ignore; } + if (osdmap.stretch_mode_enabled && + !(m->osd_features & CEPH_FEATUREMASK_STRETCH_MODE)) { + mon->clog->info() << "disallowing boot of OSD " + << m->get_orig_source_inst() + << " because stretch mode is on and OSD lacks support"; + goto ignore; + } + // already booted? if (osdmap.is_up(from) && osdmap.get_addrs(from).legacy_equals(m->get_orig_source_addrs()) && diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index 5008da80dd6..fc5e4158689 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -356,7 +356,6 @@ private: void encode_trim_extra(MonitorDBStore::TransactionRef tx, version_t first) override; void update_msgr_features(); - int check_cluster_features(uint64_t features, std::stringstream &ss); /** * check if the cluster supports the features required by the * given crush map. Outputs the daemons which don't support it @@ -412,6 +411,12 @@ private: void send_full(MonOpRequestRef op); void send_incremental(MonOpRequestRef op, epoch_t first); public: + /** + * Make sure the existing (up) OSDs support the given features + * @return 0 on success, or an error code if any OSDs re missing features. + * @param ss Filled in with ane explanation of failure, if any + */ + int check_cluster_features(uint64_t features, std::stringstream &ss); // @param req an optional op request, if the osdmaps are replies to it. so // @c Monitor::send_reply() can mark_event with it. void send_incremental(epoch_t first, MonSession *session, bool onetime, diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index f795900db57..5aa14d2b0c8 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -1654,6 +1654,11 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const features |= kraken_features; } mask |= kraken_features; + + if (stretch_mode_enabled) { + features |= CEPH_FEATUREMASK_STRETCH_MODE; + mask |= CEPH_FEATUREMASK_STRETCH_MODE; + } } if (require_min_compat_client >= ceph_release_t::nautilus) { diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index 612f91f122a..2e9dbcc8158 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -553,7 +553,6 @@ private: CEPH_FEATUREMASK_SERVER_MIMIC | CEPH_FEATUREMASK_SERVER_NAUTILUS | CEPH_FEATUREMASK_SERVER_OCTOPUS; - // TODO: add a SIGNIFICANT_FEATURE for stretch mode... struct addrs_s { mempool::osdmap::vector > client_addrs;