erasure-code: preload the jerasure plugin

Load the jerasure plugin when ceph-osd starts to avoid the following
scenario:

* ceph-osd-v1 is running but did not load jerasure

* ceph-osd-v2 is installed being installed but takes time : the files
  are installed before ceph-osd is restarted

* ceph-osd-v1 is required to handle an erasure coded placement group and
  loads jerasure (the v2 version which is not API compatible)

* ceph-osd-v1 calls the v2 jerasure plugin and does not reference the
  expected part of the code and crashes

Although this problem shows in the context of teuthology, it is unlikely
to happen on a real cluster because it involves upgrading immediately
after installing and running an OSD. Once it is backported to firefly,
it will not even happen in teuthology tests because the upgrade from
firefly to master will use the firefly version including this fix.

While it would be possible to walk the plugin directory and preload
whatever it contains, that would not work for plugins such as jerasure
that load other plugins depending on the CPU features, or even plugins
such as isa which only work on specific CPU.

http://tracker.ceph.com/issues/9153 Fixes: #9153

Backport: firefly
Signed-off-by: Loic Dachary <loic-201408@dachary.org>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index 24587add0e12696f39e9a3d3aec469d0787ed4e2..ba8f2c5f0e5d9434b9b61f1cbee82414256589d8 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -48,6 +48,8 @@ using namespace std;
 
 #include "include/assert.h"
 
+#include "erasure-code/ErasureCodePlugin.h"
+
 #define dout_subsys ceph_subsys_osd
 
 OSD *osd = NULL;
@@ -66,6 +68,21 @@ void usage()
   generic_server_usage();
 }
 
+int preload_erasure_code()
+{
+  string directory = g_conf->osd_pool_default_erasure_code_directory;
+  string plugins = g_conf->osd_erasure_code_plugins;
+  stringstream ss;
+  int r = ErasureCodePluginRegistry::instance().preload(plugins,
+                                                       directory,
+                                                       ss);
+  if (r)
+    derr << ss.str() << dendl;
+  else
+    dout(10) << ss.str() << dendl;
+  return r;
+}
+
 int main(int argc, const char **argv) 
 {
   vector<const char*> args;
@@ -456,6 +473,9 @@ int main(int argc, const char **argv)
     return -1;
   global_init_chdir(g_ceph_context);
 
+  if (preload_erasure_code() < -1)
+    return -1;
+
   osd = new OSD(g_ceph_context,
                store,
                whoami,

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 8b4bf9d369a3ba1097807edc8f6f43577cf5999d..4479d73a18a38255f798172537f1af7c538f51eb 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -442,6 +442,7 @@ OPTION(osd_pool_default_erasure_code_profile,
        "k=2 "
        "m=1 "
        ) // default properties of osd pool create
+OPTION(osd_erasure_code_plugins, OPT_STR, "jerasure isa") // list of erasure code plugins
 OPTION(osd_pool_default_flags, OPT_INT, 0)   // default flags for new pools
 OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true)   // use new pg hashing to prevent pool/pg overlap
 OPTION(osd_pool_default_hit_set_bloom_fpp, OPT_FLOAT, .05)

diff --git a/src/erasure-code/ErasureCodePlugin.cc b/src/erasure-code/ErasureCodePlugin.cc
index 2b709b4c59d3955e6f2282edc2bb66619f11f9cf..b35beb0b2973aa65285e09f2af36b7794a0178e8 100644 (file)
--- a/src/erasure-code/ErasureCodePlugin.cc
+++ b/src/erasure-code/ErasureCodePlugin.cc
@@ -4,6 +4,7 @@
  * Ceph distributed storage system
  *
  * Copyright (C) 2013,2014 Cloudwatt <libre.licensing@cloudwatt.com>
+ * Copyright (C) 2014 Red Hat <contact@redhat.com>
  *
  * Author: Loic Dachary <loic@dachary.org>
  *
@@ -19,6 +20,7 @@
 
 #include "ErasureCodePlugin.h"
 #include "common/errno.h"
+#include "include/str_list.h"
 
 #define PLUGIN_PREFIX "libec_"
 #define PLUGIN_SUFFIX ".so"
@@ -130,6 +132,26 @@ int ErasureCodePluginRegistry::load(const std::string &plugin_name,
 
   (*plugin)->library = library;
 
+  ss << __func__ << ": " << plugin_name << " ";
+
   return 0;
 }
 
+int ErasureCodePluginRegistry::preload(const std::string &plugins,
+                                      const std::string &directory,
+                                      ostream &ss)
+{
+  map<string,string> profile;
+  profile["directory"] = directory;
+  list<string> plugins_list;
+  get_str_list(plugins, plugins_list);
+  for (list<string>::iterator i = plugins_list.begin();
+       i != plugins_list.end();
+       i++) {
+    ErasureCodePlugin *plugin;
+    int r = load(*i, profile, &plugin, ss);
+    if (r)
+      return r;
+  }
+  return 0;
+}

diff --git a/src/erasure-code/ErasureCodePlugin.h b/src/erasure-code/ErasureCodePlugin.h
index 4beed987de426271196301cbdca9f0d6ae4d728c..3fa7c477534c1e45d9404d118327a4ba4eca12d0 100644 (file)
--- a/src/erasure-code/ErasureCodePlugin.h
+++ b/src/erasure-code/ErasureCodePlugin.h
@@ -67,6 +67,9 @@ namespace ceph {
             ErasureCodePlugin **plugin,
             ostream &ss);
 
+    int preload(const std::string &plugins,
+               const std::string &directory,
+               ostream &ss);
   };
 }
 

diff --git a/src/test/ceph-disk.sh b/src/test/ceph-disk.sh
index 60074ad90e84dcda8d653f58074d4d2b174da425..49ede4821ab69340f75321a7f3575b008f033c82 100755 (executable)
--- a/src/test/ceph-disk.sh
+++ b/src/test/ceph-disk.sh
@@ -30,6 +30,7 @@ CEPH_ARGS+=" --run-dir=$DIR"
 CEPH_ARGS+=" --mon-host=$MONA"
 CEPH_ARGS+=" --log-file=$DIR/\$name.log"
 CEPH_ARGS+=" --pid-file=$DIR/\$name.pidfile"
+CEPH_ARGS+=" --osd-pool-default-erasure-code-directory=.libs"
 CEPH_ARGS+=" --auth-supported=none"
 CEPH_DISK_ARGS=
 CEPH_DISK_ARGS+=" --statedir=$DIR"

diff --git a/src/test/erasure-code/test-erasure-code.sh b/src/test/erasure-code/test-erasure-code.sh
index e27c5e5232e10c580bef1c63242500ca87ff39d0..7c2c17c9a783a081570986946bfb0dd3f2adc17b 100755 (executable)
--- a/src/test/erasure-code/test-erasure-code.sh
+++ b/src/test/erasure-code/test-erasure-code.sh
@@ -30,6 +30,9 @@ function run() {
     for id in $(seq 0 4) ; do
         run_osd $dir $id || return 1
     done
+    # check that erasure code plugins are preloaded
+    CEPH_ARGS='' ./ceph --admin-daemon $dir/ceph-osd.0.asok log flush || return 1
+    grep 'load: jerasure' $dir/osd-0.log || return 1
     create_erasure_coded_pool ecpool || return 1
     FUNCTIONS=${FUNCTIONS:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
     for TEST_function in $FUNCTIONS ; do

diff --git a/src/test/osd/osd-test-helpers.sh b/src/test/osd/osd-test-helpers.sh
index 37a398eea537a63921ca1c515991c2bd8879cb9f..a1b073ae45185262a1a96889fc33239c2df7b52c 100644 (file)
--- a/src/test/osd/osd-test-helpers.sh
+++ b/src/test/osd/osd-test-helpers.sh
@@ -38,6 +38,7 @@ function run_osd() {
     ceph_args+=" --osd-journal-size=100"
     ceph_args+=" --osd-data=$osd_data"
     ceph_args+=" --chdir="
+    ceph_args+=" --osd-pool-default-erasure-code-directory=.libs"
     ceph_args+=" --run-dir=$dir"
     ceph_args+=" --debug-osd=20"
     ceph_args+=" --log-file=$dir/osd-\$id.log"