From: Bassam Tabbara Date: Wed, 21 Sep 2016 14:00:01 +0000 (-0700) Subject: erasure-code: Backward compatibility with legacy EC plugins X-Git-Tag: v11.0.1~44^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e7e0b1bc1e159e35b151eeb9d5cc64bc57cd1b76;p=ceph.git erasure-code: Backward compatibility with legacy EC plugins Resurrected jerasure_generic, jerasure_sse3, jerasure_sse4, jerasure_neon, shec_generic, shec_sse3, shec_sse4 and shec_neon. These all are exact copies of the new jerasure and shec plugins that support SIMD detection. Moved EC preload code in ceph-mon and ceph-osd to a central location, added warning when preloading legacy plugins. OSMonitor::get_erasure_code and OSDMonitor:normalize_profile will now check if legacy EC plugins are used and log a warning. Added tests to check that warnings make it to the log. Signed-off-by: Bassam Tabbara --- diff --git a/cmake/modules/SIMDExt.cmake b/cmake/modules/SIMDExt.cmake index 378e2f843c45..c531d8ca2f25 100644 --- a/cmake/modules/SIMDExt.cmake +++ b/cmake/modules/SIMDExt.cmake @@ -15,6 +15,7 @@ # if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") + set(HAVE_ARM 1) CHECK_C_COMPILER_FLAG(-march=armv8-a+crc HAVE_ARMV8_CRC) if(HAVE_ARMV8_CRC) set(ARM_CRC_FLAGS "-march=armv8-a+crc") @@ -24,11 +25,13 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64") set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -march=armv8-a+simd") endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm|ARM") + set(HAVE_ARM 1) CHECK_C_COMPILER_FLAG(-mfpu=neon HAVE_ARM_NEON) if(HAVE_ARM_NEON) set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -mfpu=neon") endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "i386|i686|amd64|x86_64|AMD64") + set(HAVE_INTEL 1) CHECK_C_COMPILER_FLAG(-msse HAVE_INTEL_SSE) if(HAVE_INTEL_SSE) set(SIMD_COMPILE_FLAGS "${SIMD_COMPILE_FLAGS} -msse") diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index 5755827f3ad1..de0447606c28 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -45,8 +45,6 @@ using namespace std; #include "include/assert.h" -#include "erasure-code/ErasureCodePlugin.h" - #define dout_subsys ceph_subsys_mon Monitor *mon = NULL; @@ -181,21 +179,6 @@ void usage() generic_server_usage(); } -int preload_erasure_code() -{ - string plugins = g_conf->osd_erasure_code_plugins; - stringstream ss; - int r = ErasureCodePluginRegistry::instance().preload( - plugins, - g_conf->erasure_code_dir, - &ss); - if (r) - derr << ss.str() << dendl; - else - dout(10) << ss.str() << dendl; - return r; -} - int main(int argc, const char **argv) { int err; @@ -507,7 +490,7 @@ int main(int argc, const char **argv) } common_init_finish(g_ceph_context); global_init_chdir(g_ceph_context); - if (preload_erasure_code() < 0) + if (global_init_preload_erasure_code(g_ceph_context) < 0) prefork.exit(1); } diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 33fafe1a3830..521d9e87c6a1 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -47,8 +47,6 @@ using namespace std; #include "include/assert.h" -#include "erasure-code/ErasureCodePlugin.h" - #define dout_subsys ceph_subsys_osd namespace { @@ -92,21 +90,6 @@ void usage() generic_server_usage(); } -int preload_erasure_code() -{ - string plugins = g_conf->osd_erasure_code_plugins; - stringstream ss; - int r = ErasureCodePluginRegistry::instance().preload( - plugins, - g_conf->erasure_code_dir, - &ss); - if (r) - derr << ss.str() << dendl; - else - dout(10) << ss.str() << dendl; - return r; -} - int main(int argc, const char **argv) { vector args; @@ -575,7 +558,7 @@ int main(int argc, const char **argv) return -1; global_init_chdir(g_ceph_context); - if (preload_erasure_code() < 0) + if (global_init_preload_erasure_code(g_ceph_context) < 0) return -1; osd = new OSD(g_ceph_context, diff --git a/src/erasure-code/CMakeLists.txt b/src/erasure-code/CMakeLists.txt index 943dea187f26..1b4870403a36 100644 --- a/src/erasure-code/CMakeLists.txt +++ b/src/erasure-code/CMakeLists.txt @@ -7,6 +7,16 @@ include_directories(jerasure/jerasure/include) include_directories(jerasure/gf-complete/include) include_directories(jerasure) +# legacy jerasure flavors. these are left here for backward compatibility +# and should be removed in future versions +set(jerasure_legacy_flavors generic) +if(HAVE_ARM) + list(APPEND jerasure_legacy_flavors neon) +endif() +if(HAVE_INTEL) + list(APPEND jerasure_legacy_flavors sse3 sse4) +endif() + add_subdirectory(jerasure) add_subdirectory(lrc) add_subdirectory(shec) @@ -25,4 +35,5 @@ add_library(erasure_code_objs OBJECT ErasureCode.cc) add_custom_target(erasure_code_plugins DEPENDS ${EC_ISA_LIB} ec_lrc - ec_jerasure) + ec_jerasure + ec_shec) diff --git a/src/erasure-code/jerasure/CMakeLists.txt b/src/erasure-code/jerasure/CMakeLists.txt index a17f5c392382..3225b6b99e11 100644 --- a/src/erasure-code/jerasure/CMakeLists.txt +++ b/src/erasure-code/jerasure/CMakeLists.txt @@ -72,10 +72,21 @@ set(jerasure_srcs jerasure_init.cc) add_library(jerasure_objs OBJECT ${jerasure_srcs}) -add_library(ec_jerasure SHARED +set(ec_jerasure_objs $ $ $ $) + +add_library(ec_jerasure SHARED ${ec_jerasure_objs}) target_link_libraries(ec_jerasure ${EXTRALIBS}) install(TARGETS ec_jerasure DESTINATION ${erasure_plugin_dir}) + +# legacy libraries +foreach(flavor ${jerasure_legacy_flavors}) + set(plugin_name "ec_jerasure_${flavor}") + add_library(${plugin_name} SHARED ${ec_jerasure_objs}) + install(TARGETS ${plugin_name} DESTINATION ${erasure_plugin_dir}) + add_dependencies(ec_jerasure ${plugin_name}) +endforeach() + diff --git a/src/erasure-code/shec/CMakeLists.txt b/src/erasure-code/shec/CMakeLists.txt index 4c5eaf408b85..64cac19465fe 100644 --- a/src/erasure-code/shec/CMakeLists.txt +++ b/src/erasure-code/shec/CMakeLists.txt @@ -10,10 +10,20 @@ add_library(shec_utils OBJECT determinant.c) add_dependencies(shec_utils ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) -add_library(ec_shec SHARED +set(ec_shec_objs $ $ $) + +add_library(ec_shec SHARED ${ec_shec_objs}) add_dependencies(ec_shec ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) target_link_libraries(ec_shec ${EXTRALIBS}) install(TARGETS ec_shec DESTINATION ${erasure_plugin_dir}) + +# legacy libraries +foreach(flavor ${jerasure_legacy_flavors}) + set(plugin_name "ec_shec_${flavor}") + add_library(${plugin_name} SHARED ${ec_shec_objs}) + install(TARGETS ${plugin_name} DESTINATION ${erasure_plugin_dir}) + add_dependencies(ec_shec ${plugin_name}) +endforeach() diff --git a/src/global/global_init.cc b/src/global/global_init.cc index 033516240b6c..3a905c0b5763 100644 --- a/src/global/global_init.cc +++ b/src/global/global_init.cc @@ -19,11 +19,13 @@ #include "common/errno.h" #include "common/signal.h" #include "common/version.h" +#include "erasure-code/ErasureCodePlugin.h" #include "global/global_context.h" #include "global/global_init.h" #include "global/pidfile.h" #include "global/signal_handler.h" #include "include/compat.h" +#include "include/str_list.h" #include #include @@ -454,3 +456,48 @@ int global_init_shutdown_stderr(CephContext *cct) return 0; } +int global_init_preload_erasure_code(const CephContext *cct) +{ + const md_config_t *conf = cct->_conf; + string plugins = conf->osd_erasure_code_plugins; + + // validate that this is a not a legacy plugin + list plugins_list; + get_str_list(plugins, plugins_list); + for (list::iterator i = plugins_list.begin(); + i != plugins_list.end(); + ++i) { + string plugin_name = *i; + string replacement = ""; + + if (plugin_name == "jerasure_generic" || + plugin_name == "jerasure_sse3" || + plugin_name == "jerasure_sse4" || + plugin_name == "jerasure_neon") { + replacement = "jerasure"; + } + else if (plugin_name == "shec_generic" || + plugin_name == "shec_sse3" || + plugin_name == "shec_sse4" || + plugin_name == "shec_neon") { + replacement = "shec"; + } + + if (replacement != "") { + dout(0) << "WARNING: osd_erasure_code_plugins contains plugin " + << plugin_name << " that is now deprecated. Please modify the value " + << "for osd_erasure_code_plugins to use " << replacement << " instead." << dendl; + } + } + + stringstream ss; + int r = ErasureCodePluginRegistry::instance().preload( + plugins, + conf->erasure_code_dir, + &ss); + if (r) + derr << ss.str() << dendl; + else + dout(10) << ss.str() << dendl; + return r; +} diff --git a/src/global/global_init.h b/src/global/global_init.h index ad3fd6286a35..1fe877d9cf4c 100644 --- a/src/global/global_init.h +++ b/src/global/global_init.h @@ -88,6 +88,12 @@ void global_init_chdir(const CephContext *cct); */ int global_init_shutdown_stderr(CephContext *cct); +/* + * Preload the erasure coding libraries to detect early issues with + * configuration. + */ +int global_init_preload_erasure_code(const CephContext *cct); + /** * print daemon startup banner/warning */ diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc index 9196a283f9a5..81cd181daed9 100644 --- a/src/mon/OSDMonitor.cc +++ b/src/mon/OSDMonitor.cc @@ -67,7 +67,7 @@ #define dout_subsys ceph_subsys_mon #undef dout_prefix #define dout_prefix _prefix(_dout, mon, osdmap) -static ostream& _prefix(std::ostream *_dout, Monitor *mon, OSDMap& osdmap) { +static ostream& _prefix(std::ostream *_dout, Monitor *mon, const OSDMap& osdmap) { return *_dout << "mon." << mon->name << "@" << mon->rank << "(" << mon->get_state_name() << ").osd e" << osdmap.get_epoch() << " "; @@ -4418,20 +4418,47 @@ int OSDMonitor::crush_rename_bucket(const string& srcname, pending_inc.crush.clear(); newcrush.encode(pending_inc.crush); - *ss << "renamed bucket " << srcname << " into " << dstname; + *ss << "renamed bucket " << srcname << " into " << dstname; return 0; } -int OSDMonitor::normalize_profile(ErasureCodeProfile &profile, ostream *ss) +void OSDMonitor::check_legacy_ec_plugin(const string& plugin, const string& profile) const +{ + string replacement = ""; + + if (plugin == "jerasure_generic" || + plugin == "jerasure_sse3" || + plugin == "jerasure_sse4" || + plugin == "jerasure_neon") { + replacement = "jerasure"; + } else if (plugin == "shec_generic" || + plugin == "shec_sse3" || + plugin == "shec_sse4" || + plugin == "shec_neon") { + replacement = "shec"; + } + + if (replacement != "") { + dout(0) << "WARNING: erasure coding profile " << profile << " uses plugin " + << plugin << " that has been deprecated. Please use " + << replacement << " instead." << dendl; + } +} + +int OSDMonitor::normalize_profile(const string& profilename, + ErasureCodeProfile &profile, + ostream *ss) { ErasureCodeInterfaceRef erasure_code; ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); ErasureCodeProfile::const_iterator plugin = profile.find("plugin"); + check_legacy_ec_plugin(plugin->second, profilename); int err = instance.factory(plugin->second, g_conf->erasure_code_dir, profile, &erasure_code, ss); if (err) return err; + return erasure_code->init(profile, ss); } @@ -4488,6 +4515,7 @@ int OSDMonitor::get_erasure_code(const string &erasure_code_profile, << profile << std::endl; return -EINVAL; } + check_legacy_ec_plugin(plugin->second, erasure_code_profile); ErasureCodePluginRegistry &instance = ErasureCodePluginRegistry::instance(); return instance.factory(plugin->second, g_conf->erasure_code_dir, @@ -6158,14 +6186,14 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, if (err) goto reply; } - err = normalize_profile(profile_map, &ss); + err = normalize_profile(name, profile_map, &ss); if (err) goto reply; if (osdmap.has_erasure_code_profile(name)) { ErasureCodeProfile existing_profile_map = osdmap.get_erasure_code_profile(name); - err = normalize_profile(existing_profile_map, &ss); + err = normalize_profile(name, existing_profile_map, &ss); if (err) goto reply; @@ -6219,7 +6247,7 @@ bool OSDMonitor::prepare_command_impl(MonOpRequestRef op, &ss); if (err) goto reply; - err = normalize_profile(profile_map, &ss); + err = normalize_profile(name, profile_map, &ss); if (err) goto reply; dout(20) << "erasure code profile set " << profile << "=" diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h index c1d603175535..8e55bf778d6f 100644 --- a/src/mon/OSDMonitor.h +++ b/src/mon/OSDMonitor.h @@ -278,7 +278,11 @@ private: int crush_rename_bucket(const string& srcname, const string& dstname, ostream *ss); - int normalize_profile(ErasureCodeProfile &profile, ostream *ss); + void check_legacy_ec_plugin(const string& plugin, + const string& profile) const; + int normalize_profile(const string& profilename, + ErasureCodeProfile &profile, + ostream *ss); int crush_ruleset_create_erasure(const string &name, const string &profile, int *ruleset, diff --git a/src/test/erasure-code/CMakeLists.txt b/src/test/erasure-code/CMakeLists.txt index 4426d88710e8..2d8b3e3bb08f 100644 --- a/src/test/erasure-code/CMakeLists.txt +++ b/src/test/erasure-code/CMakeLists.txt @@ -1,3 +1,4 @@ +add_ceph_test(test-erasure-code-plugins.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-erasure-code-plugins.sh) add_executable(ceph_erasure_code_benchmark ${CMAKE_SOURCE_DIR}/src/erasure-code/ErasureCode.cc diff --git a/src/test/erasure-code/test-erasure-code-plugins.sh b/src/test/erasure-code/test-erasure-code-plugins.sh new file mode 100755 index 000000000000..3fa49c9a2010 --- /dev/null +++ b/src/test/erasure-code/test-erasure-code-plugins.sh @@ -0,0 +1,109 @@ +#!/bin/bash -x + +source $(dirname $0)/../detect-build-env-vars.sh +source $CEPH_ROOT/qa/workunits/ceph-helpers.sh + +arch=$(uname -p) + +case $arch in + i[[3456]]86*|x86_64*|amd64*) + legacy_jerasure_plugins=(jerasure_generic jerasure_sse3 jerasure_sse4) + legacy_shec_plugins=(shec_generic shec_sse3 shec_sse4) + ;; + aarch64*|arm*) + legacy_jerasure_plugins=(jerasure_generic jerasure_neon) + legacy_shec_plugins=(shec_generic shec_neon) + ;; +esac + +plugins=(jerasure shec lrc isa) + +function run() { + local dir=$1 + shift + + export CEPH_MON="127.0.0.1:17109" + export CEPH_ARGS + CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " + CEPH_ARGS+="--mon-host=$CEPH_MON " + + local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} + for func in $funcs ; do + $func $dir || return 1 + done +} + +function TEST_preload_warning() { + local dir=$1 + + for plugin in ${legacy_jerasure_plugins[*]} ${legacy_shec_plugins[*]}; do + setup $dir || return 1 + run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 + grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/mon.a.log || return 1 + grep "WARNING: osd_erasure_code_plugins contains plugin ${plugin}" $dir/osd.0.log || return 1 + teardown $dir || return 1 + done + return 0 +} + +function TEST_preload_no_warning() { + local dir=$1 + + for plugin in ${plugins[*]}; do + setup $dir || return 1 + run_mon $dir a --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + run_osd $dir 0 --osd_erasure_code_plugins="${plugin}" || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 + ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/mon.a.log || return 1 + ! grep "WARNING: osd_erasure_code_plugins contains plugin" $dir/osd.0.log || return 1 + teardown $dir || return 1 + done + + return 0 +} + +function TEST_preload_no_warning_default() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + run_osd $dir 0 || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 + ! grep "WARNING: osd_erasure_code_plugins" $dir/mon.a.log || return 1 + ! grep "WARNING: osd_erasure_code_plugins" $dir/osd.0.log || return 1 + teardown $dir || return 1 + + return 0 +} + +function TEST_ec_profile_warning() { + local dir=$1 + + setup $dir || return 1 + run_mon $dir a || return 1 + for id in $(seq 0 2) ; do + run_osd $dir $id || return 1 + done + wait_for_clean || return 1 + + for plugin in ${legacy_jerasure_plugins[*]}; do + ceph osd erasure-code-profile set prof-${plugin} ruleset-failure-domain=osd technique=reed_sol_van plugin=${plugin} || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 + done + + for plugin in ${legacy_shec_plugins[*]}; do + ceph osd erasure-code-profile set prof-${plugin} ruleset-failure-domain=osd plugin=${plugin} || return 1 + CEPH_ARGS='' ceph --admin-daemon $dir/ceph-mon.a.asok log flush || return 1 + grep "WARNING: erasure coding profile prof-${plugin} uses plugin ${plugin}" $dir/mon.a.log || return 1 + done + + teardown $dir || return 1 +} + +main test-erasure-code-plugins "$@"