From 9687150ceac9cc7e506bc227f430d4207a6d7489 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Thu, 4 Sep 2014 17:37:45 +0200 Subject: [PATCH] erasure-code: isa/lrc plugin feature There are two new plugins (isa and lrc). When upgrading a cluster, there must be a protection against the following scenario: * the mon are upgraded but not the osd * a new pool is created using plugin isa * the osd fail to load the isa plugin because they have not been upgraded A feature bit is added : PLUGINS_V2. The monitor will only agree to create an erasure code profile for the isa or lrc plugin if all OSDs supports PLUGINS_V2. Once such an erasure code profile is stored in the OSDMap, an OSD can only boot if it supports the PLUGINS_V2 feature, which means it is able to load the isa and lrc plugins. The monitors will only activate the PLUGINS_V2 feature if all monitors in the quorum support it. It protects against the following scenario: * the leader is upgraded the peons are not upgraded * the leader creates a pool with plugin=lrc because all OSD have the PLUGINS_V2 feature * the leader goes down and a non upgraded peon becomes the leader * an old OSD tries to join the cluster * the new leader will let the OSD boot because it does not contain the logic that would excluded it * the old OSD will fail when required to load the plugin lrc This is going to be needed each time new plugins are added, which is impractical. A more generic plugin upgrade support should be added instead, as described in http://tracker.ceph.com/issues/7291. http://tracker.ceph.com/issues/9343 Refs: #9343 Signed-off-by: Loic Dachary --- src/include/ceph_features.h | 2 ++ src/mon/Monitor.cc | 8 +++++++- src/mon/Monitor.h | 1 + src/mon/OSDMonitor.cc | 26 +++++++++++++++++++++++++- src/osd/OSDMap.cc | 11 +++++++++++ 5 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h index 6b2a5fb50532b..3ee73fbab7d08 100644 --- a/src/include/ceph_features.h +++ b/src/include/ceph_features.h @@ -52,6 +52,7 @@ #define CEPH_FEATURE_OSD_PRIMARY_AFFINITY (1ULL<<41) /* overlap w/ tunables3 */ #define CEPH_FEATURE_MSGR_KEEPALIVE2 (1ULL<<42) #define CEPH_FEATURE_OSD_POOLRESEND (1ULL<<43) +#define CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2 (1ULL<<44) /* * The introduction of CEPH_FEATURE_OSD_SNAPMAPPER caused the feature @@ -124,6 +125,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) { CEPH_FEATURE_OSD_PRIMARY_AFFINITY | \ CEPH_FEATURE_MSGR_KEEPALIVE2 | \ CEPH_FEATURE_OSD_POOLRESEND | \ + CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2 | \ 0ULL) #define CEPH_FEATURES_SUPPORTED_DEFAULT CEPH_FEATURES_ALL diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index 0a535ef7decb0..69493f7f71bef 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -347,6 +347,7 @@ CompatSet Monitor::get_supported_features() ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS); ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES); ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC); + ceph_mon_feature_incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2); return CompatSet(ceph_mon_feature_compat, ceph_mon_feature_ro_compat, ceph_mon_feature_incompat); } @@ -1943,7 +1944,9 @@ void Monitor::apply_quorum_to_compatset_features() if (quorum_features & CEPH_FEATURE_OSDMAP_ENC) { new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC); } - + if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) { + new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2); + } if (new_features.compare(features) != 0) { CompatSet diff = features.unsupported(new_features); dout(1) << __func__ << " enabling new quorum features: " << diff << dendl; @@ -1966,6 +1969,9 @@ void Monitor::apply_compatset_features_to_quorum_requirements() if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC)) { required_features |= CEPH_FEATURE_OSDMAP_ENC; } + if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2)) { + required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2; + } dout(10) << __func__ << " required_features " << required_features << dendl; } diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index b5b04db5d14d0..d413dca1e6690 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -956,6 +956,7 @@ public: #define CEPH_MON_FEATURE_INCOMPAT_SINGLE_PAXOS CompatSet::Feature (3, "single paxos with k/v store (v0.\?)") #define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools") #define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding") +#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2 CompatSet::Feature(6, "support isa/lrc erasure code") // make sure you add your feature to Monitor::get_supported_features long parse_pos_long(const char *s, ostream *pss = NULL); diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 2a73e77f8f316..2ae6aab13ce9f 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -5,6 +5,7 @@ * * Copyright (C) 2004-2006 Sage Weil * Copyright (C) 2013,2014 Cloudwatt + * Copyright (C) 2014 Red Hat * * Author: Loic Dachary * @@ -1259,7 +1260,16 @@ bool OSDMonitor::preprocess_boot(MOSDBoot *m) if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) & CEPH_FEATURE_OSD_ERASURE_CODES) && !(m->get_connection()->get_features() & CEPH_FEATURE_OSD_ERASURE_CODES)) { - dout(0) << __func__ << " osdmap requires Erasure Codes but osd at " + dout(0) << __func__ << " osdmap requires erasure code but osd at " + << m->get_orig_source_inst() + << " doesn't announce support -- ignore" << dendl; + goto ignore; + } + + if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) & + CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) && + !(m->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2)) { + dout(0) << __func__ << " osdmap requires erasure code plugins v2 but osd at " << m->get_orig_source_inst() << " doesn't announce support -- ignore" << dendl; goto ignore; @@ -4500,6 +4510,13 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m, err = parse_erasure_code_profile(profile, &profile_map, ss); if (err) goto reply; + if (profile_map.find("plugin") == profile_map.end()) { + ss << "erasure-code-profile " << profile_map + << " must contain a plugin entry" << std::endl; + err = -EINVAL; + goto reply; + } + string plugin = profile_map["plugin"]; if (osdmap.has_erasure_code_profile(name)) { if (osdmap.get_erasure_code_profile(name) == profile_map) { @@ -4517,6 +4534,13 @@ bool OSDMonitor::prepare_command_impl(MMonCommand *m, dout(20) << "erasure code profile " << name << " try again" << dendl; goto wait; } else { + if (plugin == "isa" || plugin == "lrc") { + err = check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2, ss); + if (err == -EAGAIN) + goto wait; + if (err) + goto reply; + } dout(20) << "erasure code profile " << name << " set" << dendl; pending_inc.set_erasure_code_profile(name, profile_map); } diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index fb55a8150d9dc..1014a81e684b7 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -993,6 +993,17 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const features |= CEPH_FEATURE_CRUSH_TUNABLES3; } } + if (entity_type == CEPH_ENTITY_TYPE_OSD) { + for (map >::const_iterator p = erasure_code_profiles.begin(); + p != erasure_code_profiles.end(); + p++) { + const map &profile = p->second; + map::const_iterator plugin = profile.find("plugin"); + if (plugin != profile.end() && (plugin->second == "isa" || + plugin->second == "lrc")) + features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2; + } + } mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL; if (entity_type != CEPH_ENTITY_TYPE_CLIENT) mask |= CEPH_FEATURE_OSD_ERASURE_CODES; -- 2.39.5