From 9b802701f78288ba4f706c65b853415c69002d27 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Tue, 19 Aug 2014 01:30:15 +0200 Subject: [PATCH] erasure-code: preload the jerasure plugin Load the jerasure plugin when ceph-osd starts to avoid the following scenario: * ceph-osd-v1 is running but did not load jerasure * ceph-osd-v2 is installed being installed but takes time : the files are installed before ceph-osd is restarted * ceph-osd-v1 is required to handle an erasure coded placement group and loads jerasure (the v2 version which is not API compatible) * ceph-osd-v1 calls the v2 jerasure plugin and does not reference the expected part of the code and crashes Although this problem shows in the context of teuthology, it is unlikely to happen on a real cluster because it involves upgrading immediately after installing and running an OSD. Once it is backported to firefly, it will not even happen in teuthology tests because the upgrade from firefly to master will use the firefly version including this fix. While it would be possible to walk the plugin directory and preload whatever it contains, that would not work for plugins such as jerasure that load other plugins depending on the CPU features, or even plugins such as isa which only work on specific CPU. http://tracker.ceph.com/issues/9153 Fixes: #9153 Backport: firefly Signed-off-by: Loic Dachary --- src/ceph_osd.cc | 20 ++++++++++++++++++++ src/common/config_opts.h | 1 + src/erasure-code/ErasureCodePlugin.cc | 22 ++++++++++++++++++++++ src/erasure-code/ErasureCodePlugin.h | 3 +++ src/test/ceph-disk.sh | 1 + src/test/erasure-code/test-erasure-code.sh | 3 +++ src/test/osd/osd-test-helpers.sh | 1 + 7 files changed, 51 insertions(+) diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc index 24587add0e12..ba8f2c5f0e5d 100644 --- a/src/ceph_osd.cc +++ b/src/ceph_osd.cc @@ -48,6 +48,8 @@ using namespace std; #include "include/assert.h" +#include "erasure-code/ErasureCodePlugin.h" + #define dout_subsys ceph_subsys_osd OSD *osd = NULL; @@ -66,6 +68,21 @@ void usage() generic_server_usage(); } +int preload_erasure_code() +{ + string directory = g_conf->osd_pool_default_erasure_code_directory; + string plugins = g_conf->osd_erasure_code_plugins; + stringstream ss; + int r = ErasureCodePluginRegistry::instance().preload(plugins, + directory, + ss); + if (r) + derr << ss.str() << dendl; + else + dout(10) << ss.str() << dendl; + return r; +} + int main(int argc, const char **argv) { vector args; @@ -456,6 +473,9 @@ int main(int argc, const char **argv) return -1; global_init_chdir(g_ceph_context); + if (preload_erasure_code() < -1) + return -1; + osd = new OSD(g_ceph_context, store, whoami, diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 8b4bf9d369a3..4479d73a18a3 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -442,6 +442,7 @@ OPTION(osd_pool_default_erasure_code_profile, "k=2 " "m=1 " ) // default properties of osd pool create +OPTION(osd_erasure_code_plugins, OPT_STR, "jerasure isa") // list of erasure code plugins OPTION(osd_pool_default_flags, OPT_INT, 0) // default flags for new pools OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true) // use new pg hashing to prevent pool/pg overlap OPTION(osd_pool_default_hit_set_bloom_fpp, OPT_FLOAT, .05) diff --git a/src/erasure-code/ErasureCodePlugin.cc b/src/erasure-code/ErasureCodePlugin.cc index 2b709b4c59d3..b35beb0b2973 100644 --- a/src/erasure-code/ErasureCodePlugin.cc +++ b/src/erasure-code/ErasureCodePlugin.cc @@ -4,6 +4,7 @@ * Ceph distributed storage system * * Copyright (C) 2013,2014 Cloudwatt + * Copyright (C) 2014 Red Hat * * Author: Loic Dachary * @@ -19,6 +20,7 @@ #include "ErasureCodePlugin.h" #include "common/errno.h" +#include "include/str_list.h" #define PLUGIN_PREFIX "libec_" #define PLUGIN_SUFFIX ".so" @@ -130,6 +132,26 @@ int ErasureCodePluginRegistry::load(const std::string &plugin_name, (*plugin)->library = library; + ss << __func__ << ": " << plugin_name << " "; + return 0; } +int ErasureCodePluginRegistry::preload(const std::string &plugins, + const std::string &directory, + ostream &ss) +{ + map profile; + profile["directory"] = directory; + list plugins_list; + get_str_list(plugins, plugins_list); + for (list::iterator i = plugins_list.begin(); + i != plugins_list.end(); + i++) { + ErasureCodePlugin *plugin; + int r = load(*i, profile, &plugin, ss); + if (r) + return r; + } + return 0; +} diff --git a/src/erasure-code/ErasureCodePlugin.h b/src/erasure-code/ErasureCodePlugin.h index 4beed987de42..3fa7c477534c 100644 --- a/src/erasure-code/ErasureCodePlugin.h +++ b/src/erasure-code/ErasureCodePlugin.h @@ -67,6 +67,9 @@ namespace ceph { ErasureCodePlugin **plugin, ostream &ss); + int preload(const std::string &plugins, + const std::string &directory, + ostream &ss); }; } diff --git a/src/test/ceph-disk.sh b/src/test/ceph-disk.sh index 60074ad90e84..49ede4821ab6 100755 --- a/src/test/ceph-disk.sh +++ b/src/test/ceph-disk.sh @@ -30,6 +30,7 @@ CEPH_ARGS+=" --run-dir=$DIR" CEPH_ARGS+=" --mon-host=$MONA" CEPH_ARGS+=" --log-file=$DIR/\$name.log" CEPH_ARGS+=" --pid-file=$DIR/\$name.pidfile" +CEPH_ARGS+=" --osd-pool-default-erasure-code-directory=.libs" CEPH_ARGS+=" --auth-supported=none" CEPH_DISK_ARGS= CEPH_DISK_ARGS+=" --statedir=$DIR" diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh index e27c5e5232e1..7c2c17c9a783 100755 --- a/src/test/erasure-code/test-erasure-code.sh +++ b/src/test/erasure-code/test-erasure-code.sh @@ -30,6 +30,9 @@ function run() { for id in $(seq 0 4) ; do run_osd $dir $id || return 1 done + # check that erasure code plugins are preloaded + CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1 + grep 'load: jerasure' $dir/osd-0.log || return 1 create_erasure_coded_pool ecpool || return 1 FUNCTIONS=${FUNCTIONS:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for TEST_function in $FUNCTIONS ; do diff --git a/src/test/osd/osd-test-helpers.sh b/src/test/osd/osd-test-helpers.sh index 37a398eea537..a1b073ae4518 100644 --- a/src/test/osd/osd-test-helpers.sh +++ b/src/test/osd/osd-test-helpers.sh @@ -38,6 +38,7 @@ function run_osd() { ceph_args+=" --osd-journal-size=100" ceph_args+=" --osd-data=$osd_data" ceph_args+=" --chdir=" + ceph_args+=" --osd-pool-default-erasure-code-directory=.libs" ceph_args+=" --run-dir=$dir" ceph_args+=" --debug-osd=20" ceph_args+=" --log-file=$dir/osd-\$id.log" -- 2.47.3