OSDMap, OSDMonitor: add flag for all osds supporting recovery deletes

Just like sortbitwise, this can only be toggled on, and once on osds
that do not support it are not allowed to boot.

Signed-off-by: Josh Durgin <jdurgin@redhat.com>

diff --git a/src/include/rados.h b/src/include/rados.h
index 2ee2dcca3827df9a0e1aa2594f390cac3c276e4b..0a242b72b1520392ec749ddd634ef6a358e758b8 100644 (file)
--- a/src/include/rados.h
+++ b/src/include/rados.h
@@ -155,11 +155,13 @@ extern const char *ceph_osd_state_name(int s);
 #define CEPH_OSDMAP_REQUIRE_JEWEL    (1<<16) /* require jewel for booting osds */
 #define CEPH_OSDMAP_REQUIRE_KRAKEN   (1<<17) /* require kraken for booting osds */
 #define CEPH_OSDMAP_REQUIRE_LUMINOUS (1<<18) /* require l for booting osds */
+#define CEPH_OSDMAP_RECOVERY_DELETES (1<<19) /* deletes performed during recovery instead of peering */
 
 /* these are hidden in 'ceph status' view */
 #define CEPH_OSDMAP_SEMIHIDDEN_FLAGS (CEPH_OSDMAP_REQUIRE_JEWEL|       \
                                      CEPH_OSDMAP_REQUIRE_KRAKEN |      \
                                      CEPH_OSDMAP_REQUIRE_LUMINOUS |    \
+                                     CEPH_OSDMAP_RECOVERY_DELETES |    \
                                      CEPH_OSDMAP_SORTBITWISE)
 #define CEPH_OSDMAP_LEGACY_REQUIRE_FLAGS (CEPH_OSDMAP_REQUIRE_JEWEL |  \
                                          CEPH_OSDMAP_REQUIRE_KRAKEN |  \

diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h
index bd255fe5b2e5470774f7876893f801ee825f4a69..514baf4d79c5b4d63bf40c4b5f8315185132debd 100644 (file)
--- a/src/mon/MonCommands.h
+++ b/src/mon/MonCommands.h
@@ -683,7 +683,7 @@ COMMAND("osd erasure-code-profile ls", \
        "list all erasure code profiles", \
        "osd", "r", "cli,rest")
 COMMAND("osd set " \
-       "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|require_jewel_osds|require_kraken_osds", \
+       "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent|sortbitwise|recovery_deletes|require_jewel_osds|require_kraken_osds", \
        "set <key>", "osd", "rw", "cli,rest")
 COMMAND("osd unset " \
        "name=key,type=CephChoices,strings=full|pause|noup|nodown|noout|noin|nobackfill|norebalance|norecover|noscrub|nodeep-scrub|notieragent", \

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 4a613ae8843bc85ac0d7c8db72d620c2ff0d82a5..d7aab1e71f86a8a3674cca4de774fb9a7fefd6d4 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -2068,6 +2068,14 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
     goto ignore;
   }
 
+  if (osdmap.test_flag(CEPH_OSDMAP_RECOVERY_DELETES) &&
+      !(m->osd_features & CEPH_FEATURE_OSD_RECOVERY_DELETES)) {
+    mon->clog->info() << "disallowing boot of OSD "
+                     << m->get_orig_source_inst()
+                     << " because 'recovery_deletes' osdmap flag is set and OSD lacks the OSD_RECOVERY_DELETES feature";
+    goto ignore;
+  }
+
   if (any_of(osdmap.get_pools().begin(),
             osdmap.get_pools().end(),
             [](const std::pair<int64_t,pg_pool_t>& pool)
@@ -8275,6 +8283,14 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
        err = -EPERM;
        goto reply;
       }
+    } else if (key == "recovery_deletes") {
+      if (HAVE_FEATURE(osdmap.get_up_osd_features(), OSD_RECOVERY_DELETES)) {
+       return prepare_set_flag(op, CEPH_OSDMAP_RECOVERY_DELETES);
+      } else {
+       ss << "not all up OSDs have OSD_RECOVERY_DELETES feature";
+       err = -EPERM;
+       goto reply;
+      }
     } else if (key == "require_jewel_osds") {
       if (!osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE)) {
        ss << "the sortbitwise flag must be set before require_jewel_osds";

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index ea43ae773228f7a156a18e799a17da291cee7d23..d4422af7acddc883c9e5785f6519316f01c400b2 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -2951,6 +2951,8 @@ string OSDMap::get_flag_string(unsigned f)
     s += ",require_kraken_osds";
   if (f & CEPH_OSDMAP_REQUIRE_LUMINOUS)
     s += ",require_luminous_osds";
+  if (f & CEPH_OSDMAP_RECOVERY_DELETES)
+    s += ",recovery_deletes";
   if (s.length())
     s.erase(0, 1);
   return s;

diff --git a/src/osd/PG.h b/src/osd/PG.h
index b6316e496e6598adafe4fa95aff0c3987da06aa6..c5f5e0b0309077abfb746004a7eecd78d6122ff2 100644 (file)
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -2226,6 +2226,9 @@ public:
 
   uint64_t get_min_acting_features() const { return acting_features; }
   uint64_t get_min_upacting_features() const { return upacting_features; }
+  bool perform_deletes_during_peering() const {
+    return !(get_osdmap()->test_flag(CEPH_OSDMAP_RECOVERY_DELETES));
+  }
 
   void init_primary_up_acting(
     const vector<int> &newup,