From: Sage Weil Date: Fri, 12 May 2017 15:42:52 +0000 (-0400) Subject: osd: replace require_*_osds flags with require_osd_release field X-Git-Tag: v12.1.0~57^2~6^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3118d9a15434e3eeb481cb91377feae2f649a940;p=ceph.git osd: replace require_*_osds flags with require_osd_release field - OSDMap encode and decode translate between the flags and int representations. - OSDMap::Incremental only does decode; we do not expect to ever encode an incremental osdmap for an old osd that sets any of these flags. - the 'osd set' command still lets you set the jewel and kraken flags, but not luminous. - OSDMap::apply_incremental handles the conversion of legacy require flags to the new field if the jewel or kraken flags have to be set before starting the osd upgrade. - clear out the legacy flags when we make the luminous transition only; until then we keep using the old flag in the encoded and decoded version (although the require_osd_release field will be accurate in memory in all cases). Signed-off-by: Sage Weil --- diff --git a/qa/releases/luminous-with-mgr.yaml b/qa/releases/luminous-with-mgr.yaml index 880df732b98..ea313076870 100644 --- a/qa/releases/luminous-with-mgr.yaml +++ b/qa/releases/luminous-with-mgr.yaml @@ -1,7 +1,7 @@ tasks: - exec: osd.0: - - ceph osd set require_luminous_osds + - ceph osd require-osd-release luminous - ceph osd set-require-min-compat-client luminous - ceph.healthy: overrides: diff --git a/qa/releases/luminous.yaml b/qa/releases/luminous.yaml index 22bb78b2d67..9ed76715a96 100644 --- a/qa/releases/luminous.yaml +++ b/qa/releases/luminous.yaml @@ -9,7 +9,7 @@ tasks: wait-for-healthy: false - exec: osd.0: - - ceph osd set require_luminous_osds + - ceph osd require-osd-release luminous - ceph osd set-require-min-compat-client luminous - ceph.healthy: overrides: diff --git a/qa/suites/rados/thrash/d-require-luminous/at-end.yaml b/qa/suites/rados/thrash/d-require-luminous/at-end.yaml index f9711a126e2..bb1d7073ccc 100644 --- a/qa/suites/rados/thrash/d-require-luminous/at-end.yaml +++ b/qa/suites/rados/thrash/d-require-luminous/at-end.yaml @@ -5,7 +5,7 @@ tasks: - full_sequential_finally: - exec: mon.a: - - ceph osd set require_luminous_osds + - ceph osd require-osd-release luminous # make sure osds have latest map - rados -p rbd bench 5 write -b 4096 - ceph.healthy: diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh index afb6dbc9e12..8c128f127c7 100755 --- a/qa/workunits/cephtool/test.sh +++ b/qa/workunits/cephtool/test.sh @@ -1181,10 +1181,13 @@ function test_mon_osd() expect_false ceph osd unset sortbitwise # cannot be unset expect_false ceph osd set bogus expect_false ceph osd unset bogus + ceph osd require-osd-release luminous + # can't lower (or use new command for anything but jewel) + expect_false ceph osd require-osd-release jewel + # these are no-ops but should succeed. ceph osd set require_jewel_osds - expect_false ceph osd unset require_jewel_osds ceph osd set require_kraken_osds - expect_false ceph osd unset require_kraken_osds + expect_false ceph osd unset require_jewel_osds ceph osd set noup ceph osd down 0 diff --git a/src/include/rados.h b/src/include/rados.h index 50323943bee..8e4aedf4b51 100644 --- a/src/include/rados.h +++ b/src/include/rados.h @@ -156,6 +156,9 @@ extern const char *ceph_osd_state_name(int s); CEPH_OSDMAP_REQUIRE_KRAKEN | \ CEPH_OSDMAP_REQUIRE_LUMINOUS | \ CEPH_OSDMAP_SORTBITWISE) +#define CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS (CEPH_OSDMAP_REQUIRE_JEWEL | \ + CEPH_OSDMAP_REQUIRE_KRAKEN | \ + CEPH_OSDMAP_REQUIRE_LUMINOUS) /* * major ceph release numbers diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 7469b223323..7ccbe632967 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -632,7 +632,7 @@ COMMAND("osd erasure-code-profile ls", \ "list all erasure code profiles", \ "osd", "r", "cli,rest") COMMAND("osd set " \ - "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|require_jewel_osds|require_kraken_osds|require_luminous_osds", \ + "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|require_jewel_osds|require_kraken_osds", \ "set ", "osd", "rw", "cli,rest") COMMAND("osd unset " \ "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent", \ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index fb25d5dbb44..0d3b3b7dd08 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -220,10 +220,10 @@ void OSDMonitor::create_initial() newmap.set_flag(CEPH_OSDMAP_SORTBITWISE); // new cluster should require latest by default - newmap.set_flag(CEPH_OSDMAP_REQUIRE_JEWEL); - newmap.set_flag(CEPH_OSDMAP_REQUIRE_KRAKEN); - if (!g_conf->mon_debug_no_require_luminous) { - newmap.set_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS); + if (g_conf->mon_debug_no_require_luminous) { + newmap.require_osd_release = CEPH_RELEASE_KRAKEN; + } else { + newmap.require_osd_release = CEPH_RELEASE_LUMINOUS; newmap.full_ratio = g_conf->mon_osd_full_ratio; if (newmap.full_ratio > 1.0) newmap.full_ratio /= 100; newmap.backfillfull_ratio = g_conf->mon_osd_backfillfull_ratio; @@ -7519,11 +7519,17 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, } else { ss << "not all up OSDs have OSD_BITWISE_HOBJ_SORT feature"; err = -EPERM; + goto reply; } } else if (key == "require_jewel_osds") { if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) { ss << "the sortbitwise flag must be set before require_jewel_osds"; err = -EPERM; + goto reply; + } else if (osdmap.require_osd_release >= CEPH_RELEASE_JEWEL) { + ss << "require_osd_release is already >= jewel"; + err = 0; + goto reply; } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_JEWEL)) { return prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_JEWEL); } else { @@ -7534,6 +7540,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) { ss << "the sortbitwise flag must be set before require_kraken_osds"; err = -EPERM; + goto reply; + } else if (osdmap.require_osd_release >= CEPH_RELEASE_KRAKEN) { + ss << "require_osd_release is already >= kraken"; + err = 0; + goto reply; } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_KRAKEN)) { bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_KRAKEN); // ensure JEWEL is also set @@ -7543,20 +7554,6 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, ss << "not all up OSDs have CEPH_FEATURE_SERVER_KRAKEN feature"; err = -EPERM; } - } else if (key == "require_luminous_osds") { - if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) { - ss << "the sortbitwise flag must be set before require_luminous_osds"; - err = -EPERM; - } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_LUMINOUS)) { - bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_LUMINOUS); - // ensure JEWEL and KRAKEN are also set - pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_JEWEL; - pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_KRAKEN; - return r; - } else { - ss << "not all up OSDs have CEPH_FEATURE_SERVER_LUMINOUS feature"; - err = -EPERM; - } } else { ss << "unrecognized flag '" << key << "'"; err = -EINVAL; diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index dd110e5faf6..d80723c74c9 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -724,7 +724,17 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl) if (struct_v >= 6) { ::decode(new_require_osd_release, bl); } else { - new_require_osd_release = -1; + if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_LUMINOUS)) { + // only for compat with post-kraken pre-luminous test clusters + new_require_osd_release = CEPH_RELEASE_LUMINOUS; + new_flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS); + } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_KRAKEN)) { + new_require_osd_release = CEPH_RELEASE_KRAKEN; + } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_JEWEL)) { + new_require_osd_release = CEPH_RELEASE_JEWEL; + } else { + new_require_osd_release = -1; + } } DECODE_FINISH(bl); // osd-only data } @@ -1505,8 +1515,22 @@ int OSDMap::apply_incremental(const Incremental &inc) } // nope, incremental. - if (inc.new_flags >= 0) + if (inc.new_flags >= 0) { flags = inc.new_flags; + // the below is just to cover a newly-upgraded luminous mon + // cluster that has to set require_jewel_osds or + // require_kraken_osds before the osds can be upgraded to + // luminous. + if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) { + if (require_osd_release < CEPH_RELEASE_KRAKEN) { + require_osd_release = CEPH_RELEASE_KRAKEN; + } + } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) { + if (require_osd_release < CEPH_RELEASE_JEWEL) { + require_osd_release = CEPH_RELEASE_JEWEL; + } + } + } if (inc.new_max_osd >= 0) set_max_osd(inc.new_max_osd); @@ -1684,6 +1708,9 @@ int OSDMap::apply_incremental(const Incremental &inc) } if (inc.new_require_osd_release >= 0) { require_osd_release = inc.new_require_osd_release; + if (require_osd_release >= CEPH_RELEASE_LUMINOUS) { + flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS); + } } // do new crush map last (after up/down stuff) @@ -2203,7 +2230,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const { uint8_t v = 4; - if (!HAVE_FEATURE(features, OSDMAP_PG_UPMAP)) { + if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) { v = 3; } ENCODE_START(v, 1, bl); // client-usable data @@ -2217,7 +2244,18 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const ::encode(pool_name, bl); ::encode(pool_max, bl); - ::encode(flags, bl); + if (v < 4) { + decltype(flags) f = flags; + if (require_osd_release >= CEPH_RELEASE_LUMINOUS) + f |= CEPH_OSDMAP_REQUIRE_LUMINOUS; + else if (require_osd_release == CEPH_RELEASE_KRAKEN) + f |= CEPH_OSDMAP_REQUIRE_KRAKEN; + else if (require_osd_release == CEPH_RELEASE_JEWEL) + f |= CEPH_OSDMAP_REQUIRE_JEWEL; + ::encode(f, bl); + } else { + ::encode(flags, bl); + } ::encode(max_osd, bl); ::encode(osd_state, bl); @@ -2525,8 +2563,21 @@ void OSDMap::decode(bufferlist::iterator& bl) } if (struct_v >= 5) { ::decode(require_osd_release, bl); + if (require_osd_release >= CEPH_RELEASE_LUMINOUS) { + flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS); + } } else { - require_osd_release = -1; + if (flags & CEPH_OSDMAP_REQUIRE_LUMINOUS) { + // only for compat with post-kraken pre-luminous test clusters + require_osd_release = CEPH_RELEASE_LUMINOUS; + flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS); + } else if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) { + require_osd_release = CEPH_RELEASE_KRAKEN; + } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) { + require_osd_release = CEPH_RELEASE_JEWEL; + } else { + require_osd_release = 0; + } } DECODE_FINISH(bl); // osd-only data } diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h index ac5df7dd069..a35a77fa4ca 100644 --- a/src/osd/OSDMap.h +++ b/src/osd/OSDMap.h @@ -276,9 +276,11 @@ private: /// min compat client we want to support string require_min_compat_client; +public: /// require osds to run at least this release uint8_t require_osd_release = 0; +private: mutable uint64_t cached_up_osd_features; mutable bool crc_defined;