erasure-code: preload the jerasure plugin

Load the jerasure plugin when ceph-osd starts to avoid the following
scenario:

* ceph-osd-v1 is running but did not load jerasure

* ceph-osd-v2 is installed being installed but takes time : the files
  are installed before ceph-osd is restarted

* ceph-osd-v1 is required to handle an erasure coded placement group and
  loads jerasure (the v2 version which is not API compatible)

* ceph-osd-v1 calls the v2 jerasure plugin and does not reference the
  expected part of the code and crashes

Although this problem shows in the context of teuthology, it is unlikely
to happen on a real cluster because it involves upgrading immediately
after installing and running an OSD. Once it is backported to firefly,
it will not even happen in teuthology tests because the upgrade from
firefly to master will use the firefly version including this fix.

While it would be possible to walk the plugin directory and preload
whatever it contains, that would not work for plugins such as jerasure
that load other plugins depending on the CPU features, or even plugins
such as isa which only work on specific CPU.

http://tracker.ceph.com/issues/9153 Fixes: #9153

Backport: firefly
Signed-off-by: Loic Dachary <loic-201408@dachary.org>
(cherry picked from commit 9b802701f78288ba4f706c65b853415c69002d27)

Conflicts:
	src/test/erasure-code/test-erasure-code.sh
	src/common/config_opts.h

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 029ef28c40565b09334d77c77b505e10b85b6dd0..a2f454206c9349c1bf7b28332b8292419eb3528f 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -48,6 +48,8 @@ using namespace std;
 
 #include "include/assert.h"
 
+#include "erasure-code/ErasureCodePlugin.h"
+
 #define dout_subsys ceph_subsys_osd
 
 OSD *osd = NULL;
@@ -66,6 +68,21 @@ void usage()
   generic_server_usage();
 }
 
+int preload_erasure_code()
+{
+  string directory = g_conf->osd_pool_default_erasure_code_directory;
+  string plugins = g_conf->osd_erasure_code_plugins;
+  stringstream ss;
+  int r = ErasureCodePluginRegistry::instance().preload(plugins,
+                                                       directory,
+                                                       ss);
+  if (r)
+    derr << ss.str() << dendl;
+  else
+    dout(10) << ss.str() << dendl;
+  return r;
+}
+
 int main(int argc, const char **argv) 
 {
   vector<const char*> args;
@@ -451,6 +468,9 @@ int main(int argc, const char **argv)
     return -1;
   global_init_chdir(g_ceph_context);
 
+  if (preload_erasure_code() < -1)
+    return -1;
+
   osd = new OSD(g_ceph_context,
                store,
                whoami,

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index fdd35575dfe9a0dc5f5fadf72d3f2f257353847f..f5fbede963683deea900d0c063b033119751980c 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -435,6 +435,7 @@ OPTION(osd_pool_default_erasure_code_profile,
        "k=2 "
        "m=1 "
        ) // default properties of osd pool create
+OPTION(osd_erasure_code_plugins, OPT_STR, "jerasure") // list of erasure code plugins
 OPTION(osd_pool_default_flags, OPT_INT, 0)   // default flags for new pools
 OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true)   // use new pg hashing to prevent pool/pg overlap
 OPTION(osd_pool_default_hit_set_bloom_fpp, OPT_FLOAT, .05)

diff --git a/src/erasure-code/ErasureCodePlugin.cc b/src/erasure-code/ErasureCodePlugin.cc
index da075d22c7b679322f6f305d48acc14398833aed..f77041eff941f47e13a4bf5c108a3b2bc0b1cb60 100644 (file)
--- a/src/erasure-code/ErasureCodePlugin.cc
+++ b/src/erasure-code/ErasureCodePlugin.cc
@@ -4,6 +4,7 @@
  * Ceph - scalable distributed file system
  *
  * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
  *
  * Author: Loic Dachary <loic@dachary.org>
  *
@@ -19,6 +20,7 @@
 
 #include "ErasureCodePlugin.h"
 #include "common/errno.h"
+#include "include/str_list.h"
 
 #define PLUGIN_PREFIX "libec_"
 #define PLUGIN_SUFFIX ".so"
@@ -130,6 +132,26 @@ int ErasureCodePluginRegistry::load(const std::string &plugin_name,
 
   (*plugin)->library = library;
 
+  ss << __func__ << ": " << plugin_name << " ";
+
   return 0;
 }
 
+int ErasureCodePluginRegistry::preload(const std::string &plugins,
+                                      const std::string &directory,
+                                      ostream &ss)
+{
+  map<string,string> profile;
+  profile["directory"] = directory;
+  list<string> plugins_list;
+  get_str_list(plugins, plugins_list);
+  for (list<string>::iterator i = plugins_list.begin();
+       i != plugins_list.end();
+       i++) {
+    ErasureCodePlugin *plugin;
+    int r = load(*i, profile, &plugin, ss);
+    if (r)
+      return r;
+  }
+  return 0;
+}

diff --git a/src/erasure-code/ErasureCodePlugin.h b/src/erasure-code/ErasureCodePlugin.h
index e891079c7f3b262965a9e62660188a555dddf953..7f0b1e993be8083b64a6782bfa851427653de188 100644 (file)
--- a/src/erasure-code/ErasureCodePlugin.h
+++ b/src/erasure-code/ErasureCodePlugin.h
@@ -67,6 +67,9 @@ namespace ceph {
             ErasureCodePlugin **plugin,
             ostream &ss);
 
+    int preload(const std::string &plugins,
+               const std::string &directory,
+               ostream &ss);
   };
 }
 

diff --git a/src/test/ceph-disk.sh b/src/test/ceph-disk.sh
index 35e2702669681ee9e52cb127fefae872b33e0a96..4ae4bf996625d97f59f326c107fff6a0e02d3b0a 100755 (executable)
--- a/src/test/ceph-disk.sh
+++ b/src/test/ceph-disk.sh
@@ -29,6 +29,7 @@ CEPH_ARGS+=" --run-dir=$DIR"
 CEPH_ARGS+=" --mon-host=$MONA"
 CEPH_ARGS+=" --log-file=$DIR/\$name.log"
 CEPH_ARGS+=" --pid-file=$DIR/\$name.pidfile"
+CEPH_ARGS+=" --osd-pool-default-erasure-code-directory=.libs"
 CEPH_ARGS+=" --auth-supported=none"
 CEPH_DISK_ARGS=
 CEPH_DISK_ARGS+=" --statedir=$DIR"

diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh
index 852c5c148470a4d5c86e70d039aaf55fdd1ad527..63663e0a38448af73733d3996f6d16f23bd55a9a 100755 (executable)
--- a/src/test/erasure-code/test-erasure-code.sh
+++ b/src/test/erasure-code/test-erasure-code.sh
@@ -30,6 +30,9 @@ function run() {
     for id in $(seq 0 4) ; do
         run_osd $dir $id || return 1
     done
+    # check that erasure code plugins are preloaded
+    CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1
+    grep 'load: jerasure' $dir/osd-0.log || return 1
     create_erasure_coded_pool || return 1
     FUNCTIONS=${FUNCTIONS:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for TEST_function in $FUNCTIONS ; do

diff --git a/src/test/osd/osd-test-helpers.sh b/src/test/osd/osd-test-helpers.sh
index 5117ae38b50aee3a3b01aa8dd02bd33f0953e272..1ea17dd12a93a181b981334560ff5bce97d34645 100644 (file)
--- a/src/test/osd/osd-test-helpers.sh
+++ b/src/test/osd/osd-test-helpers.sh
@@ -37,6 +37,7 @@ function run_osd() {
     ceph_args+=" --osd-journal-size=100"
     ceph_args+=" --osd-data=$osd_data"
     ceph_args+=" --chdir="
+    ceph_args+=" --osd-pool-default-erasure-code-directory=.libs"
     ceph_args+=" --run-dir=$dir"
     ceph_args+=" --debug-osd=20"
     ceph_args+=" --log-file=$dir/osd-\$id.log"