osd: replace require_*_osds flags with require_osd_release field

- OSDMap encode and decode translate between the flags and int
representations.
- OSDMap::Incremental only does decode; we do not expect to ever encode
an incremental osdmap for an old osd that sets any of these flags.
- the 'osd set' command still lets you set the jewel and kraken flags,
but not luminous.
- OSDMap::apply_incremental handles the conversion of legacy require flags
to the new field if the jewel or kraken flags have to be set before
starting the osd upgrade.
- clear out the legacy flags when we make the luminous transition only;
until then we keep using the old flag in the encoded and decoded version
(although the require_osd_release field will be accurate in memory in all
cases).

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/qa/releases/luminous-with-mgr.yaml b/qa/releases/luminous-with-mgr.yaml
index 880df732b98edd56a943d502d6fcc0712d9ab785..ea3130768705eda898274a890089816d7865e5bc 100644 (file)
--- a/qa/releases/luminous-with-mgr.yaml
+++ b/qa/releases/luminous-with-mgr.yaml
@@ -1,7 +1,7 @@
 tasks:
 - exec:
     osd.0:
-      - ceph osd set require_luminous_osds
+      - ceph osd require-osd-release luminous
       - ceph osd set-require-min-compat-client luminous
 - ceph.healthy:
 overrides:

diff --git a/qa/releases/luminous.yaml b/qa/releases/luminous.yaml
index 22bb78b2d675652f2db601a1ae6b1bb96242c92c..9ed76715a9603e0332567d96226ebbedc19756fc 100644 (file)
--- a/qa/releases/luminous.yaml
+++ b/qa/releases/luminous.yaml
@@ -9,7 +9,7 @@ tasks:
     wait-for-healthy: false
 - exec:
     osd.0:
-      - ceph osd set require_luminous_osds
+      - ceph osd require-osd-release luminous
       - ceph osd set-require-min-compat-client luminous
 - ceph.healthy:
 overrides:

diff --git a/qa/suites/rados/thrash/d-require-luminous/at-end.yaml b/qa/suites/rados/thrash/d-require-luminous/at-end.yaml
index f9711a126e20244d125886e577355c73e26cecf4..bb1d7073ccc2daf2736e84c5f8dafb5460cea5b9 100644 (file)
--- a/qa/suites/rados/thrash/d-require-luminous/at-end.yaml
+++ b/qa/suites/rados/thrash/d-require-luminous/at-end.yaml
@@ -5,7 +5,7 @@ tasks:
 - full_sequential_finally:
   - exec:
       mon.a:
-        - ceph osd set require_luminous_osds
+        - ceph osd require-osd-release luminous
 # make sure osds have latest map
         - rados -p rbd bench 5 write -b 4096
   - ceph.healthy:

diff --git a/qa/workunits/cephtool/test.sh b/qa/workunits/cephtool/test.sh
index afb6dbc9e124fe01626209051e244026c288b1b6..8c128f127c7f6aef541d902782199ebba1214ef7 100755 (executable)
--- a/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@ -1181,10 +1181,13 @@ function test_mon_osd()
   expect_false ceph osd unset sortbitwise  # cannot be unset
   expect_false ceph osd set bogus
   expect_false ceph osd unset bogus
+  ceph osd require-osd-release luminous
+  # can't lower (or use new command for anything but jewel)
+  expect_false ceph osd require-osd-release jewel
+  # these are no-ops but should succeed.
   ceph osd set require_jewel_osds
-  expect_false ceph osd unset require_jewel_osds
   ceph osd set require_kraken_osds
-  expect_false ceph osd unset require_kraken_osds
+  expect_false ceph osd unset require_jewel_osds
 
   ceph osd set noup
   ceph osd down 0

diff --git a/src/include/rados.h b/src/include/rados.h
index 50323943bee3ddbf6cd1dc7bfa07c3ced6256362..8e4aedf4b5118b4cb9ee3ee6a18fdae14e64fa83 100644 (file)
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -156,6 +156,9 @@ extern const char *ceph_osd_state_name(int s);
                                      CEPH_OSDMAP_REQUIRE_KRAKEN |      \
                                      CEPH_OSDMAP_REQUIRE_LUMINOUS |    \
                                      CEPH_OSDMAP_SORTBITWISE)
+#define CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS (CEPH_OSDMAP_REQUIRE_JEWEL |  \
+                                         CEPH_OSDMAP_REQUIRE_KRAKEN |  \
+                                         CEPH_OSDMAP_REQUIRE_LUMINOUS)
 
 /*
  * major ceph release numbers

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index 7469b223323fadebf5af548e0c0489430c3439dd..7ccbe63296706e335ef39246c0c843b9a6d3c243 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -632,7 +632,7 @@ COMMAND("osd erasure-code-profile ls", \
        "list all erasure code profiles", \
        "osd", "r", "cli,rest")
 COMMAND("osd set " \
-       "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|require_jewel_osds|require_kraken_osds|require_luminous_osds", \
+       "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|require_jewel_osds|require_kraken_osds", \
        "set <key>", "osd", "rw", "cli,rest")
 COMMAND("osd unset " \
        "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent", \

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index fb25d5dbb4410d687b866e2036af2527d831292f..0d3b3b7dd08a22272d57ddfa976fcbd4bf09bb51 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -220,10 +220,10 @@ void OSDMonitor::create_initial()
   newmap.set_flag(CEPH_OSDMAP_SORTBITWISE);
 
   // new cluster should require latest by default
-  newmap.set_flag(CEPH_OSDMAP_REQUIRE_JEWEL);
-  newmap.set_flag(CEPH_OSDMAP_REQUIRE_KRAKEN);
-  if (!g_conf->mon_debug_no_require_luminous) {
-    newmap.set_flag(CEPH_OSDMAP_REQUIRE_LUMINOUS);
+  if (g_conf->mon_debug_no_require_luminous) {
+    newmap.require_osd_release = CEPH_RELEASE_KRAKEN;
+  } else {
+    newmap.require_osd_release = CEPH_RELEASE_LUMINOUS;
     newmap.full_ratio = g_conf->mon_osd_full_ratio;
     if (newmap.full_ratio > 1.0) newmap.full_ratio /= 100;
     newmap.backfillfull_ratio = g_conf->mon_osd_backfillfull_ratio;
@@ -7519,11 +7519,17 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       } else {
        ss << "not all up OSDs have OSD_BITWISE_HOBJ_SORT feature";
        err = -EPERM;
+       goto reply;
       }
     } else if (key == "require_jewel_osds") {
       if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
        ss << "the sortbitwise flag must be set before require_jewel_osds";
        err = -EPERM;
+       goto reply;
+      } else if (osdmap.require_osd_release >= CEPH_RELEASE_JEWEL) {
+       ss << "require_osd_release is already >= jewel";
+       err = 0;
+       goto reply;
       } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_JEWEL)) {
        return prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_JEWEL);
       } else {
@@ -7534,6 +7540,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
       if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
        ss << "the sortbitwise flag must be set before require_kraken_osds";
        err = -EPERM;
+       goto reply;
+      } else if (osdmap.require_osd_release >= CEPH_RELEASE_KRAKEN) {
+       ss << "require_osd_release is already >= kraken";
+       err = 0;
+       goto reply;
       } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_KRAKEN)) {
        bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_KRAKEN);
        // ensure JEWEL is also set
@@ -7543,20 +7554,6 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
        ss << "not all up OSDs have CEPH_FEATURE_SERVER_KRAKEN feature";
        err = -EPERM;
       }
-    } else if (key == "require_luminous_osds") {
-      if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
-       ss << "the sortbitwise flag must be set before require_luminous_osds";
-       err = -EPERM;
-      } else if (HAVE_FEATURE(osdmap.get_up_osd_features(), SERVER_LUMINOUS)) {
-       bool r = prepare_set_flag(op, CEPH_OSDMAP_REQUIRE_LUMINOUS);
-       // ensure JEWEL and KRAKEN are also set
-       pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_JEWEL;
-       pending_inc.new_flags |= CEPH_OSDMAP_REQUIRE_KRAKEN;
-       return r;
-      } else {
-       ss << "not all up OSDs have CEPH_FEATURE_SERVER_LUMINOUS feature";
-       err = -EPERM;
-      }
     } else {
       ss << "unrecognized flag '" << key << "'";
       err = -EINVAL;

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index dd110e5faf6bc95e484c75744ce42b19318ef997..d80723c74c98dc80fed3df9358db0bc72b4ff23e 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -724,7 +724,17 @@ void OSDMap::Incremental::decode(bufferlist::iterator& bl)
     if (struct_v >= 6) {
       ::decode(new_require_osd_release, bl);
     } else {
-      new_require_osd_release = -1;
+      if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_LUMINOUS)) {
+       // only for compat with post-kraken pre-luminous test clusters
+       new_require_osd_release = CEPH_RELEASE_LUMINOUS;
+       new_flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_KRAKEN)) {
+       new_require_osd_release = CEPH_RELEASE_KRAKEN;
+      } else if (new_flags >= 0 && (new_flags & CEPH_OSDMAP_REQUIRE_JEWEL)) {
+       new_require_osd_release = CEPH_RELEASE_JEWEL;
+      } else {
+       new_require_osd_release = -1;
+      }
     }
     DECODE_FINISH(bl); // osd-only data
   }
@@ -1505,8 +1515,22 @@ int OSDMap::apply_incremental(const Incremental &inc)
   }
 
   // nope, incremental.
-  if (inc.new_flags >= 0)
+  if (inc.new_flags >= 0) {
     flags = inc.new_flags;
+    // the below is just to cover a newly-upgraded luminous mon
+    // cluster that has to set require_jewel_osds or
+    // require_kraken_osds before the osds can be upgraded to
+    // luminous.
+    if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) {
+      if (require_osd_release < CEPH_RELEASE_KRAKEN) {
+       require_osd_release = CEPH_RELEASE_KRAKEN;
+      }
+    } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) {
+      if (require_osd_release < CEPH_RELEASE_JEWEL) {
+       require_osd_release = CEPH_RELEASE_JEWEL;
+      }
+    }
+  }
 
   if (inc.new_max_osd >= 0)
     set_max_osd(inc.new_max_osd);
@@ -1684,6 +1708,9 @@ int OSDMap::apply_incremental(const Incremental &inc)
   }
   if (inc.new_require_osd_release >= 0) {
     require_osd_release = inc.new_require_osd_release;
+    if (require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+      flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+    }
   }
 
   // do new crush map last (after up/down stuff)
@@ -2203,7 +2230,7 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
 
   {
     uint8_t v = 4;
-    if (!HAVE_FEATURE(features, OSDMAP_PG_UPMAP)) {
+    if (!HAVE_FEATURE(features, SERVER_LUMINOUS)) {
       v = 3;
     }
     ENCODE_START(v, 1, bl); // client-usable data
@@ -2217,7 +2244,18 @@ void OSDMap::encode(bufferlist& bl, uint64_t features) const
     ::encode(pool_name, bl);
     ::encode(pool_max, bl);
 
-    ::encode(flags, bl);
+    if (v < 4) {
+      decltype(flags) f = flags;
+      if (require_osd_release >= CEPH_RELEASE_LUMINOUS)
+       f |= CEPH_OSDMAP_REQUIRE_LUMINOUS;
+      else if (require_osd_release == CEPH_RELEASE_KRAKEN)
+       f |= CEPH_OSDMAP_REQUIRE_KRAKEN;
+      else if (require_osd_release == CEPH_RELEASE_JEWEL)
+       f |= CEPH_OSDMAP_REQUIRE_JEWEL;
+      ::encode(f, bl);
+    } else {
+      ::encode(flags, bl);
+    }
 
     ::encode(max_osd, bl);
     ::encode(osd_state, bl);
@@ -2525,8 +2563,21 @@ void OSDMap::decode(bufferlist::iterator& bl)
     }
     if (struct_v >= 5) {
       ::decode(require_osd_release, bl);
+      if (require_osd_release >= CEPH_RELEASE_LUMINOUS) {
+       flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      }
     } else {
-      require_osd_release = -1;
+      if (flags & CEPH_OSDMAP_REQUIRE_LUMINOUS) {
+       // only for compat with post-kraken pre-luminous test clusters
+       require_osd_release = CEPH_RELEASE_LUMINOUS;
+       flags &= ~(CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS);
+      } else if (flags & CEPH_OSDMAP_REQUIRE_KRAKEN) {
+       require_osd_release = CEPH_RELEASE_KRAKEN;
+      } else if (flags & CEPH_OSDMAP_REQUIRE_JEWEL) {
+       require_osd_release = CEPH_RELEASE_JEWEL;
+      } else {
+       require_osd_release = 0;
+      }
     }
     DECODE_FINISH(bl); // osd-only data
   }

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index ac5df7dd06966c58c87c4586877b7f52f3d184e8..a35a77fa4ca56231827349ce5bf5ab2da8f81ce7 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -276,9 +276,11 @@ private:
   /// min compat client we want to support
   string require_min_compat_client;
 
+public:
   /// require osds to run at least this release
   uint8_t require_osd_release = 0;
 
+private:
   mutable uint64_t cached_up_osd_features;
 
   mutable bool crc_defined;