erasure-code: shec plugin feature

There is one new plugin (shec). When upgrading a cluster, there
must be a protection against the following scenario:

   * the mon are upgraded but not the osd
   * a new pool is created using plugin shec
   * the osd fail to load the shec plugin because they have not been
     upgraded

A feature bit is added : PLUGINS_V3. The monitor will only agree to
create an erasure code profile for the shec plugin if all OSDs
supports PLUGINS_V3. Once such an erasure code profile is stored in the
OSDMap, an OSD can only boot if it supports the PLUGINS_V3 feature,
which means it is able to load the shec plugin.

The monitors will only activate the PLUGINS_V3 feature if all monitors
in the quorum support it. It protects against the following scenario:

   * the leader is upgraded the peons are not upgraded
   * the leader creates a pool with plugin=shec because all OSD have
     the PLUGINS_V3 feature
   * the leader goes down and a non upgraded peon becomes the leader
   * an old OSD tries to join the cluster
   * the new leader will let the OSD boot because it does not contain
     the logic that would excluded it
   * the old OSD will fail when required to load the plugin shec

This is going to be needed each time new plugins are added, which is
impractical. A more generic plugin upgrade support should be added
instead, as described in http://tracker.ceph.com/issues/7291.

See also 9687150ceac9cc7e506bc227f430d4207a6d7489 for the PLUGINS_V2
implementation.

http://tracker.ceph.com/issues/10887 Fixes: #10887

Signed-off-by: Loic Dachary <ldachary@redhat.com>

diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 8ac0f4d598c3ecfe0300c14b9666dcab85f9d230..0cfc20add7aeb184529fd1f453edb22a52d34792 100644 (file)
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -66,6 +66,7 @@
 #define CEPH_FEATURE_MON_METADATA (1ULL<<50)
 #define CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT (1ULL<<51) /* can sort objs bitwise */
 #define CEPH_FEATURE_OSD_PROXY_WRITE_FEATURES (1ULL<<52)
+#define CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3 (1ULL<<53)
 
 #define CEPH_FEATURE_RESERVED2 (1ULL<<61)  /* slow down, we are almost out... */
 #define CEPH_FEATURE_RESERVED  (1ULL<<62)  /* DO NOT USE THIS ... last bit! */
@@ -153,6 +154,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
          CEPH_FEATURE_OSD_MIN_SIZE_RECOVERY |           \
         CEPH_FEATURE_MON_METADATA |                     \
         CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT |            \
+         CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3 |   \
          CEPH_FEATURE_OSD_PROXY_WRITE_FEATURES |         \
         0ULL)
 

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index f570733036eae5eb9c2028ef072976325078f054..08420265d3650acc3a02d16470fb2112fa70738c 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -377,6 +377,7 @@ CompatSet Monitor::get_supported_features()
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
   compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
+  compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3);
   return compat;
 }
 
@@ -1941,6 +1942,9 @@ void Monitor::apply_quorum_to_compatset_features()
   if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) {
     new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
   }
+  if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3) {
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3);
+  }
   if (new_features.compare(features) != 0) {
     CompatSet diff = features.unsupported(new_features);
     dout(1) << __func__ << " enabling new quorum features: " << diff << dendl;
@@ -1966,6 +1970,9 @@ void Monitor::apply_compatset_features_to_quorum_requirements()
   if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2)) {
     required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2;
   }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3)) {
+    required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3;
+  }
   dout(10) << __func__ << " required_features " << required_features << dendl;
 }
 

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index 8eae20f1510d71be7131eeb5710d5ff078e4c5a5..929e8a82dd773c0e958135769ec319a89655b25c 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -978,6 +978,7 @@ public:
 #define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools")
 #define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding")
 #define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2 CompatSet::Feature(6, "support isa/lrc erasure code")
+#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3 CompatSet::Feature(7, "support shec erasure code")
 // make sure you add your feature to Monitor::get_supported_features
 
 long parse_pos_long(const char *s, ostream *pss = NULL);

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 6c2893d0f503eb225ed00cabc14ba57bb67c12ed..95d57dbf5242474daea2a68bdd7a4d48c8d2f7b0 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -1830,6 +1830,15 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op)
     goto ignore;
   }
 
+  if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+       CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3) &&
+      !(m->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3)) {
+    dout(0) << __func__ << " osdmap requires erasure code plugins v3 but osd at "
+            << m->get_orig_source_inst()
+            << " doesn't announce support -- ignore" << dendl;
+    goto ignore;
+  }
+
   if (osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE) &&
       !(m->osd_features & CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT)) {
     mon->clog->info() << "disallowing boot of OSD "
@@ -5573,10 +5582,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op,
        if (err)
          goto reply;
       } else if (plugin == "shec") {
-       if (!g_ceph_context->check_experimental_feature_enabled("shec", &ss)) {
-         err = -EINVAL;
+       err = check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3, ss);
+       if (err == -EAGAIN)
+         goto wait;
+       if (err)
          goto reply;
-       }
       }
       err = normalize_profile(profile_map, &ss);
       if (err)

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 516796a10e2f664b52aef242f494c9320ca8038f..4d565594df54d3b163f55663c493aff4be9f7166 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -1066,9 +1066,12 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
         ++p) {
       const map<string,string> &profile = p->second;
       map<string,string>::const_iterator plugin = profile.find("plugin");
-      if (plugin != profile.end() && (plugin->second == "isa" ||
-                                     plugin->second == "lrc"))
-       features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2;
+      if (plugin != profile.end()) {
+       if (plugin->second == "isa" || plugin->second == "lrc")
+         features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2;
+       if (plugin->second == "shec")
+         features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3;
+      }
     }
   }
   mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;

diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh
index 8d99bf760564a3f92f08ad6595e2e3b9693a3090..28405538fef016296550ce354a13b503f878e199 100755 (executable)
--- a/src/test/erasure-code/test-erasure-code.sh
+++ b/src/test/erasure-code/test-erasure-code.sh
@@ -25,7 +25,6 @@ function run() {
     export CEPH_MON="127.0.0.1:7101"
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
-    CEPH_ARGS+="--enable-experimental-unrecoverable-data-corrupting-features=shec "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
 
     setup $dir || return 1

diff --git a/src/test/erasure-code/test-erasure-eio.sh b/src/test/erasure-code/test-erasure-eio.sh
index 570e1afa468c146d636ec605bdf98f23e70b9d5d..3c93338bbd7e53d84c89ec4a5e4c0c155f6f2753 100755 (executable)
--- a/src/test/erasure-code/test-erasure-eio.sh
+++ b/src/test/erasure-code/test-erasure-eio.sh
@@ -25,7 +25,6 @@ function run() {
     export CEPH_MON="127.0.0.1:7112"
     export CEPH_ARGS
     CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none "
-    CEPH_ARGS+="--enable-experimental-unrecoverable-data-corrupting-features=shec "
     CEPH_ARGS+="--mon-host=$CEPH_MON "
 
     local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}

diff --git a/src/test/mon/osd-erasure-code-profile.sh b/src/test/mon/osd-erasure-code-profile.sh
index 38444041eb61b95cfb308bf43b0e2138650b1374..27be3464fe81982bdf51f637cee9fa64953b5585 100755 (executable)
--- a/src/test/mon/osd-erasure-code-profile.sh
+++ b/src/test/mon/osd-erasure-code-profile.sh
@@ -122,19 +122,6 @@ function TEST_get() {
     grep -q "unknown erasure code profile 'WRONG'" $dir/out || return 1
 }
 
-function TEST_experimental_shec() {
-    local dir=$1
-    local id=$2
-
-    run_mon $dir a || return 1
-
-    local profile=shec-profile
-
-    ! ./ceph osd erasure-code-profile set $profile plugin=shec > $dir/out 2>&1 || return 1
-    grep "experimental feature 'shec'" $dir/out || return 1
-    ! ./ceph osd erasure-code-profile ls | grep $profile || return 1
-}
-
 function TEST_set_idempotent() {
     local dir=$1
     local id=$2