From: Loic Dachary Date: Mon, 18 Aug 2014 23:30:15 +0000 (+0200) Subject: erasure-code: preload the jerasure plugin X-Git-Tag: v0.80.6~43^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=164f1a1959a863848319585fa752250c7b261381;p=ceph.git erasure-code: preload the jerasure plugin Load the jerasure plugin when ceph-osd starts to avoid the following scenario: * ceph-osd-v1 is running but did not load jerasure * ceph-osd-v2 is installed being installed but takes time : the files are installed before ceph-osd is restarted * ceph-osd-v1 is required to handle an erasure coded placement group and loads jerasure (the v2 version which is not API compatible) * ceph-osd-v1 calls the v2 jerasure plugin and does not reference the expected part of the code and crashes Although this problem shows in the context of teuthology, it is unlikely to happen on a real cluster because it involves upgrading immediately after installing and running an OSD. Once it is backported to firefly, it will not even happen in teuthology tests because the upgrade from firefly to master will use the firefly version including this fix. While it would be possible to walk the plugin directory and preload whatever it contains, that would not work for plugins such as jerasure that load other plugins depending on the CPU features, or even plugins such as isa which only work on specific CPU. http://tracker.ceph.com/issues/9153 Fixes: #9153 Backport: firefly Signed-off-by: Loic Dachary (cherry picked from commit 9b802701f78288ba4f706c65b853415c69002d27) Conflicts: src/test/erasure-code/test-erasure-code.sh src/common/config_opts.h --- diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 029ef28c405..a2f454206c9 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -48,6 +48,8 @@ using namespace std; #include "include/assert.h" +#include "erasure-code/ErasureCodePlugin.h" + #define dout_subsys ceph_subsys_osd OSD *osd = NULL; @@ -66,6 +68,21 @@ void usage() generic_server_usage(); } +int preload_erasure_code() +{ + string directory = g_conf->osd_pool_default_erasure_code_directory; + string plugins = g_conf->osd_erasure_code_plugins; + stringstream ss; + int r = ErasureCodePluginRegistry::instance().preload(plugins, + directory, + ss); + if (r) + derr << ss.str() << dendl; + else + dout(10) << ss.str() << dendl; + return r; +} + int main(int argc, const char **argv) { vector args; @@ -451,6 +468,9 @@ int main(int argc, const char **argv) return -1; global_init_chdir(g_ceph_context); + if (preload_erasure_code() < -1) + return -1; + osd = new OSD(g_ceph_context, store, whoami, diff --git a/src/common/config_opts.h b/src/common/config_opts.h index fdd35575dfe..f5fbede9636 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -435,6 +435,7 @@ OPTION(osd_pool_default_erasure_code_profile, "k=2 " "m=1 " ) // default properties of osd pool create +OPTION(osd_erasure_code_plugins, OPT_STR, "jerasure") // list of erasure code plugins OPTION(osd_pool_default_flags, OPT_INT, 0) // default flags for new pools OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true) // use new pg hashing to prevent pool/pg overlap OPTION(osd_pool_default_hit_set_bloom_fpp, OPT_FLOAT, .05) diff --git a/src/erasure-code/ErasureCodePlugin.cc b/src/erasure-code/ErasureCodePlugin.cc index da075d22c7b..f77041eff94 100644 --- a/src/erasure-code/ErasureCodePlugin.cc +++ b/src/erasure-code/ErasureCodePlugin.cc @@ -4,6 +4,7 @@ * Ceph - scalable distributed file system * * Copyright (C) 2013,2014 Cloudwatt + * Copyright (C) 2014 Red Hat * * Author: Loic Dachary * @@ -19,6 +20,7 @@ #include "ErasureCodePlugin.h" #include "common/errno.h" +#include "include/str_list.h" #define PLUGIN_PREFIX "libec_" #define PLUGIN_SUFFIX ".so" @@ -130,6 +132,26 @@ int ErasureCodePluginRegistry::load(const std::string &plugin_name, (*plugin)->library = library; + ss << __func__ << ": " << plugin_name << " "; + return 0; } +int ErasureCodePluginRegistry::preload(const std::string &plugins, + const std::string &directory, + ostream &ss) +{ + map profile; + profile["directory"] = directory; + list plugins_list; + get_str_list(plugins, plugins_list); + for (list::iterator i = plugins_list.begin(); + i != plugins_list.end(); + i++) { + ErasureCodePlugin *plugin; + int r = load(*i, profile, &plugin, ss); + if (r) + return r; + } + return 0; +} diff --git a/src/erasure-code/ErasureCodePlugin.h b/src/erasure-code/ErasureCodePlugin.h index e891079c7f3..7f0b1e993be 100644 --- a/src/erasure-code/ErasureCodePlugin.h +++ b/src/erasure-code/ErasureCodePlugin.h @@ -67,6 +67,9 @@ namespace ceph { ErasureCodePlugin **plugin, ostream &ss); + int preload(const std::string &plugins, + const std::string &directory, + ostream &ss); }; } diff --git a/src/test/ceph-disk.sh b/src/test/ceph-disk.sh index 35e27026696..4ae4bf99662 100755 --- a/src/test/ceph-disk.sh +++ b/src/test/ceph-disk.sh @@ -29,6 +29,7 @@ CEPH_ARGS+=" --run-dir=$DIR" CEPH_ARGS+=" --mon-host=$MONA" CEPH_ARGS+=" --log-file=$DIR/\$name.log" CEPH_ARGS+=" --pid-file=$DIR/\$name.pidfile" +CEPH_ARGS+=" --osd-pool-default-erasure-code-directory=.libs" CEPH_ARGS+=" --auth-supported=none" CEPH_DISK_ARGS= CEPH_DISK_ARGS+=" --statedir=$DIR" diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh index 852c5c14847..63663e0a384 100755 --- a/src/test/erasure-code/test-erasure-code.sh +++ b/src/test/erasure-code/test-erasure-code.sh @@ -30,6 +30,9 @@ function run() { for id in $(seq 0 4) ; do run_osd $dir $id || return 1 done + # check that erasure code plugins are preloaded + CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 + grep 'load: jerasure' $dir/osd-0.log || return 1 create_erasure_coded_pool || return 1 FUNCTIONS=${FUNCTIONS:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for TEST_function in $FUNCTIONS ; do diff --git a/src/test/osd/osd-test-helpers.sh b/src/test/osd/osd-test-helpers.sh index 5117ae38b50..1ea17dd12a9 100644 --- a/src/test/osd/osd-test-helpers.sh +++ b/src/test/osd/osd-test-helpers.sh @@ -37,6 +37,7 @@ function run_osd() { ceph_args+=" --osd-journal-size=100" ceph_args+=" --osd-data=$osd_data" ceph_args+=" --chdir=" + ceph_args+=" --osd-pool-default-erasure-code-directory=.libs" ceph_args+=" --run-dir=$dir" ceph_args+=" --debug-osd=20" ceph_args+=" --log-file=$dir/osd-\$id.log"