From 271513f8f091f8eb31cee40307f330416c8c601e Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Thu, 6 Aug 2015 15:02:38 +0200 Subject: [PATCH] erasure-code: shec plugin feature There is one new plugin (shec). When upgrading a cluster, there must be a protection against the following scenario: * the mon are upgraded but not the osd * a new pool is created using plugin shec * the osd fail to load the shec plugin because they have not been upgraded A feature bit is added : PLUGINS_V3. The monitor will only agree to create an erasure code profile for the shec plugin if all OSDs supports PLUGINS_V3. Once such an erasure code profile is stored in the OSDMap, an OSD can only boot if it supports the PLUGINS_V3 feature, which means it is able to load the shec plugin. The monitors will only activate the PLUGINS_V3 feature if all monitors in the quorum support it. It protects against the following scenario: * the leader is upgraded the peons are not upgraded * the leader creates a pool with plugin=shec because all OSD have the PLUGINS_V3 feature * the leader goes down and a non upgraded peon becomes the leader * an old OSD tries to join the cluster * the new leader will let the OSD boot because it does not contain the logic that would excluded it * the old OSD will fail when required to load the plugin shec This is going to be needed each time new plugins are added, which is impractical. A more generic plugin upgrade support should be added instead, as described in http://tracker.ceph.com/issues/7291. See also 9687150ceac9cc7e506bc227f430d4207a6d7489 for the PLUGINS_V2 implementation. http://tracker.ceph.com/issues/10887 Fixes: #10887 Signed-off-by: Loic Dachary --- src/include/ceph_features.h | 2 ++ src/mon/Monitor.cc | 7 +++++++ src/mon/Monitor.h | 1 + src/mon/OSDMonitor.cc | 16 +++++++++++++--- src/osd/OSDMap.cc | 9 ++++++--- src/test/erasure-code/test-erasure-code.sh | 1 - src/test/erasure-code/test-erasure-eio.sh | 1 - src/test/mon/osd-erasure-code-profile.sh | 13 ------------- 8 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/include/ceph_features.h b/src/include/ceph_features.h index 8ac0f4d598c3e..0cfc20add7aeb 100644 --- a/src/include/ceph_features.h +++ b/src/include/ceph_features.h @@ -66,6 +66,7 @@ #define CEPH_FEATURE_MON_METADATA (1ULL<<50) #define CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT (1ULL<<51) /* can sort objs bitwise */ #define CEPH_FEATURE_OSD_PROXY_WRITE_FEATURES (1ULL<<52) +#define CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3 (1ULL<<53) #define CEPH_FEATURE_RESERVED2 (1ULL<<61) /* slow down, we are almost out... */ #define CEPH_FEATURE_RESERVED (1ULL<<62) /* DO NOT USE THIS ... last bit! */ @@ -153,6 +154,7 @@ static inline unsigned long long ceph_sanitize_features(unsigned long long f) { CEPH_FEATURE_OSD_MIN_SIZE_RECOVERY | \ CEPH_FEATURE_MON_METADATA | \ CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT | \ + CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3 | \ CEPH_FEATURE_OSD_PROXY_WRITE_FEATURES | \ 0ULL) diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index f570733036eae..08420265d3650 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -377,6 +377,7 @@ CompatSet Monitor::get_supported_features() compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES); compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC); compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2); + compat.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3); return compat; } @@ -1941,6 +1942,9 @@ void Monitor::apply_quorum_to_compatset_features() if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2) { new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2); } + if (quorum_features & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3) { + new_features.incompat.insert(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3); + } if (new_features.compare(features) != 0) { CompatSet diff = features.unsupported(new_features); dout(1) << __func__ << " enabling new quorum features: " << diff << dendl; @@ -1966,6 +1970,9 @@ void Monitor::apply_compatset_features_to_quorum_requirements() if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2)) { required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2; } + if (features.incompat.contains(CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3)) { + required_features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3; + } dout(10) << __func__ << " required_features " << required_features << dendl; } diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 8eae20f1510d7..929e8a82dd773 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -978,6 +978,7 @@ public: #define CEPH_MON_FEATURE_INCOMPAT_OSD_ERASURE_CODES CompatSet::Feature(4, "support erasure code pools") #define CEPH_MON_FEATURE_INCOMPAT_OSDMAP_ENC CompatSet::Feature(5, "new-style osdmap encoding") #define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V2 CompatSet::Feature(6, "support isa/lrc erasure code") +#define CEPH_MON_FEATURE_INCOMPAT_ERASURE_CODE_PLUGINS_V3 CompatSet::Feature(7, "support shec erasure code") // make sure you add your feature to Monitor::get_supported_features long parse_pos_long(const char *s, ostream *pss = NULL); diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 6c2893d0f503e..95d57dbf52424 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -1830,6 +1830,15 @@ bool OSDMonitor::preprocess_boot(MonOpRequestRef op) goto ignore; } + if ((osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL) & + CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3) && + !(m->get_connection()->get_features() & CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3)) { + dout(0) << __func__ << " osdmap requires erasure code plugins v3 but osd at " + << m->get_orig_source_inst() + << " doesn't announce support -- ignore" << dendl; + goto ignore; + } + if (osdmap.test_flag(CEPH_OSDMAP_SORTBITWISE) && !(m->osd_features & CEPH_FEATURE_OSD_BITWISE_HOBJ_SORT)) { mon->clog->info() << "disallowing boot of OSD " @@ -5573,10 +5582,11 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, if (err) goto reply; } else if (plugin == "shec") { - if (!g_ceph_context->check_experimental_feature_enabled("shec", &ss)) { - err = -EINVAL; + err = check_cluster_features(CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3, ss); + if (err == -EAGAIN) + goto wait; + if (err) goto reply; - } } err = normalize_profile(profile_map, &ss); if (err) diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 516796a10e2f6..4d565594df54d 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -1066,9 +1066,12 @@ uint64_t OSDMap::get_features(int entity_type, uint64_t *pmask) const ++p) { const map &profile = p->second; map::const_iterator plugin = profile.find("plugin"); - if (plugin != profile.end() && (plugin->second == "isa" || - plugin->second == "lrc")) - features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2; + if (plugin != profile.end()) { + if (plugin->second == "isa" || plugin->second == "lrc") + features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V2; + if (plugin->second == "shec") + features |= CEPH_FEATURE_ERASURE_CODE_PLUGINS_V3; + } } } mask |= CEPH_FEATURE_OSDHASHPSPOOL | CEPH_FEATURE_OSD_CACHEPOOL; diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh index 8d99bf760564a..28405538fef01 100755 --- a/src/test/erasure-code/test-erasure-code.sh +++ b/src/test/erasure-code/test-erasure-code.sh @@ -25,7 +25,6 @@ function run() { export CEPH_MON="127.0.0.1:7101" export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--enable-experimental-unrecoverable-data-corrupting-features=shec " CEPH_ARGS+="--mon-host=$CEPH_MON " setup $dir || return 1 diff --git a/src/test/erasure-code/test-erasure-eio.sh b/src/test/erasure-code/test-erasure-eio.sh index 570e1afa468c1..3c93338bbd7e5 100755 --- a/src/test/erasure-code/test-erasure-eio.sh +++ b/src/test/erasure-code/test-erasure-eio.sh @@ -25,7 +25,6 @@ function run() { export CEPH_MON="127.0.0.1:7112" export CEPH_ARGS CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " - CEPH_ARGS+="--enable-experimental-unrecoverable-data-corrupting-features=shec " CEPH_ARGS+="--mon-host=$CEPH_MON " local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} diff --git a/src/test/mon/osd-erasure-code-profile.sh b/src/test/mon/osd-erasure-code-profile.sh index 38444041eb61b..27be3464fe819 100755 --- a/src/test/mon/osd-erasure-code-profile.sh +++ b/src/test/mon/osd-erasure-code-profile.sh @@ -122,19 +122,6 @@ function TEST_get() { grep -q "unknown erasure code profile 'WRONG'" $dir/out || return 1 } -function TEST_experimental_shec() { - local dir=$1 - local id=$2 - - run_mon $dir a || return 1 - - local profile=shec-profile - - ! ./ceph osd erasure-code-profile set $profile plugin=shec > $dir/out 2>&1 || return 1 - grep "experimental feature 'shec'" $dir/out || return 1 - ! ./ceph osd erasure-code-profile ls | grep $profile || return 1 -} - function TEST_set_idempotent() { local dir=$1 local id=$2 -- 2.47.3