erasure-code: isa/lrc plugin feature

There are two new plugins (isa and lrc). When upgrading a cluster, there
must be a protection against the following scenario:

  * the mon are upgraded but not the osd
  * a new pool is created using plugin isa
  * the osd fail to load the isa plugin because they have not been
    upgraded

A feature bit is added : PLUGINS_V2. The monitor will only agree to
create an erasure code profile for the isa or lrc plugin if all OSDs
supports PLUGINS_V2. Once such an erasure code profile is stored in the
OSDMap, an OSD can only boot if it supports the PLUGINS_V2 feature,
which means it is able to load the isa and lrc plugins.

The monitors will only activate the PLUGINS_V2 feature if all monitors
in the quorum support it. It protects against the following scenario:

  * the leader is upgraded the peons are not upgraded
  * the leader creates a pool with plugin=lrc because all OSD have
    the PLUGINS_V2 feature
  * the leader goes down and a non upgraded peon becomes the leader
  * an old OSD tries to join the cluster
  * the new leader will let the OSD boot because it does not contain
    the logic that would excluded it
  * the old OSD will fail when required to load the plugin lrc

This is going to be needed each time new plugins are added, which is
impractical. A more generic plugin upgrade support should be added
instead, as described in http://tracker.ceph.com/issues/7291.

http://tracker.ceph.com/issues/9343 Refs: #9343

Signed-off-by: Loic Dachary <loic-201408@dachary.org>

diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h
index 6b2a5fb50532b23e09953a4fbb262cfeeb63e968..3ee73fbab7d08251278599901291667813045f45 100644 (file)
--- a/src/include/ceph_features.h
+++ b/src/include/ceph_features.h
@@ -52,6 +52,7 @@
 #define CEPH_FEATURE_OSD_PRIMARY_AFFINITY (1ULL<<41)  /* overlap w/ tunables3 */
 #define CEPH_FEATURE_MSGR_KEEPALIVE2   (1ULL<<42)
 #define CEPH_FEATURE_OSD_POOLRESEND    (1ULL<<43)
+#define CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2 (1ULL<<44)
 
 /*
  * The introduction of CEPH_FEATURE_OSD_SNAPMAPPER caused the feature
@@ -124,6 +125,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) {
         CEPH_FEATURE_OSD_PRIMARY_AFFINITY |    \
         CEPH_FEATURE_MSGR_KEEPALIVE2 | \
         CEPH_FEATURE_OSD_POOLRESEND |  \
+         CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2 |   \
         0ULL)
 
 #define CEPH_FEATURES_SUPPORTED_DEFAULT  CEPH_FEATURES_ALL

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 0a535ef7decb009d44675e5b39b5e041a5eea9c4..69493f7f71befa4e4c153629e050190091c6f612 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -347,6 +347,7 @@ CompatSet Monitor::get_supported_features()
   ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS);
   ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES);
   ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
+  ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
   return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat,
                   ceph_mon_feature_incompat);
 }
@@ -1943,7 +1944,9 @@ void Monitor::apply_quorum_to_compatset_features()
   if (quorum_features & CEPH_FEATURE_OSDMAP_ENC) {
     new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC);
   }
-
+  if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) {
+    new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2);
+  }
   if (new_features.compare(features) != 0) {
     CompatSet diff = features.unsupported(new_features);
     dout(1) << __func__ << " enabling new quorum features: " << diff << dendl;
@@ -1966,6 +1969,9 @@ void Monitor::apply_compatset_features_to_quorum_requirements()
   if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC)) {
     required_features |= CEPH_FEATURE_OSDMAP_ENC;
   }
+  if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2)) {
+    required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2;
+  }
   dout(10) << __func__ << " required_features " << required_features << dendl;
 }
 

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index b5b04db5d14d0611dee254bfe66fe88a67e5b615..d413dca1e66909095a51cb524a8af49eccdf0d25 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -956,6 +956,7 @@ public:
 #define CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS CompatSet::Feature (3, "single paxos with k/v store (v0.\?)")
 #define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools")
 #define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding")
+#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2 CompatSet::Feature(6, "support isa/lrc erasure code")
 // make sure you add your feature to Monitor::get_supported_features
 
 long parse_pos_long(const char *s, ostream *pss = NULL);

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 2a73e77f8f316eca38ca161b2db0b783856088d0..2ae6aab13ce9f7e8e0a592468f31d1b2050b8fff 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -5,6 +5,7 @@
  *
  * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
  * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
  *
  * Author: Loic Dachary <loic@dachary.org>
  *
@@ -1259,7 +1260,16 @@ bool OSDMonitor::preprocess_boot(MOSDBoot *m)
   if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
        CEPH_FEATURE_OSD_ERASURE_CODES) &&
       !(m->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) {
-    dout(0) << __func__ << " osdmap requires Erasure Codes but osd at "
+    dout(0) << __func__ << " osdmap requires erasure code but osd at "
+            << m->get_orig_source_inst()
+            << " doesn't announce support -- ignore" << dendl;
+    goto ignore;
+  }
+
+  if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) &
+       CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) &&
+      !(m->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2)) {
+    dout(0) << __func__ << " osdmap requires erasure code plugins v2 but osd at "
             << m->get_orig_source_inst()
             << " doesn't announce support -- ignore" << dendl;
     goto ignore;
@@ -4500,6 +4510,13 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m,
     err = parse_erasure_code_profile(profile, &profile_map, ss);
     if (err)
       goto reply;
+    if (profile_map.find("plugin") == profile_map.end()) {
+      ss << "erasure-code-profile " << profile_map
+        << " must contain a plugin entry" << std::endl;
+      err = -EINVAL;
+      goto reply;
+    }
+    string plugin = profile_map["plugin"];
 
     if (osdmap.has_erasure_code_profile(name)) {
       if (osdmap.get_erasure_code_profile(name) == profile_map) {
@@ -4517,6 +4534,13 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m,
       dout(20) << "erasure code profile " << name << " try again" << dendl;
       goto wait;
     } else {
+      if (plugin == "isa" || plugin == "lrc") {
+       err = check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2, ss);
+       if (err == -EAGAIN)
+         goto wait;
+       if (err)
+         goto reply;
+      }
       dout(20) << "erasure code profile " << name << " set" << dendl;
       pending_inc.set_erasure_code_profile(name, profile_map);
     }

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index fb55a8150d9dc56f426378c06cea1c45389d777e..1014a81e684b7c15acd2fd16f0e2ba945d51ad8b 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -993,6 +993,17 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const
        features |= CEPH_FEATURE_CRUSH_TUNABLES3;
     }
   }
+  if (entity_type == CEPH_ENTITY_TYPE_OSD) {
+    for (map<string,map<string,string> >::const_iterator p = erasure_code_profiles.begin();
+        p != erasure_code_profiles.end();
+        p++) {
+      const map<string,string> &profile = p->second;
+      map<string,string>::const_iterator plugin = profile.find("plugin");
+      if (plugin != profile.end() && (plugin->second == "isa" ||
+                                     plugin->second == "lrc"))
+       features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2;
+    }
+  }
   mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL;
   if (entity_type != CEPH_ENTITY_TYPE_CLIENT)
     mask |= CEPH_FEATURE_OSD_ERASURE_CODES;