os/bluestore: Add config bluestore_use_ebd

When EBD(extblkdev) plugin is in use usually it needs to present all the time.
For bluestore deployed with EBD plugin it makes it an error if bluestore tries
to mount and EBD plugin is not present.

Preload of extblkdev plugins was misplaced.
Moved loading plugins into BlueStore.
This way both OSD and tools can load plugins.

Plugins are now loaded only:
- before mkfs
- when extblkdev plugin is signalled in label meta

Signed-off-by: Adam Kupczyk <akupczyk@ibm.com>

diff --git a/src/ceph_osd.cc b/src/ceph_osd.cc
index da09b7bae5a7c273b7cabc46e4bc5e231918cd09..baf1b4f91854da21d3f7880c2b1d23723ec1d604 100644 (file)
--- a/src/ceph_osd.cc
+++ b/src/ceph_osd.cc
@@ -28,7 +28,6 @@
 #include "mon/MonClient.h"
 #include "include/ceph_features.h"
 #include "common/config.h"
-#include "extblkdev/ExtBlkDevPlugin.h"
 
 #include "mon/MonMap.h"
 
@@ -493,14 +492,6 @@ flushjournal_out:
     }
     forker.exit(0);
   }
-  
-  {
-    int r = extblkdev::preload(g_ceph_context);
-    if (r < 0) {
-      derr << "Failed preloading extblkdev plugins, error code: " << r << dendl;
-      forker.exit(1);
-    }
-  }
 
   string magic;
   uuid_d cluster_fsid, osd_fsid;

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index c06cd8ea6e3ea603fa9d1674c74120fc5c73bc73..e54e42ab92299d0ae344b27af1452d6a0ff9f3db 100644 (file)
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -4515,6 +4515,18 @@ options:
   flags:
   - create
   with_legacy: true
+- name: bluestore_use_ebd
+  type: bool
+  level: advanced
+  desc: EBD plugin used during mkfs is required for mounts.
+  long_desc: The flag has two effects, on mkfs and on mount.
+    If set for mkfs and EBD plugin did attach to the device then save plugin name to OSD metadata.
+    If set when mounting and plugin name is stored in OSD metadata then refuse to mount.
+    The flag has no meaning if the device has no associated EBD plugin.
+  default: true
+  see_also:
+  - osd_extblkdev_plugins
+  with_legacy: true
 - name: bluestore_bdev_label_multi
   type: bool
   level: advanced

diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc
index 6ce95c827cd5c805438a98adb5690295758f8396..c401ed0ae7e3b52469d59d73cf841a051bc80568 100644 (file)
--- a/src/os/bluestore/BlueStore.cc
+++ b/src/os/bluestore/BlueStore.cc
@@ -62,6 +62,7 @@
 #include "Writer.h"
 #include "Compression.h"
 #include "BlueAdmin.h"
+#include "extblkdev/ExtBlkDevPlugin.h"
 
 #if defined(WITH_LTTNG)
 #define TRACEPOINT_DEFINE
@@ -7152,7 +7153,15 @@ int BlueStore::_open_bdev(bool create)
   ceph_assert(bdev == NULL);
   string p = path + "/block";
   bdev = BlockDevice::create(cct, p, aio_cb, static_cast<void*>(this), discard_cb, static_cast<void*>(this), "bluestore");
-  int r = bdev->open(p);
+  int r = 0;
+  int plugin_preload_r = 0;
+  if (cct->_conf->bluestore_use_ebd) {
+    //load plugins
+    plugin_preload_r = extblkdev::preload(cct);
+    // do not complain yet. wait until we check "extblkdev" meta.
+  }
+
+  r = bdev->open(p);
   if (r < 0)
     goto fail;
 
@@ -7165,6 +7174,31 @@ int BlueStore::_open_bdev(bool create)
     dout(5) << __func__ << " trimmed device:" << p << dendl;
   }
 
+  if (!create && cct->_conf->bluestore_use_ebd) {
+    // for regular bdev opens check if it was deployed with plugin
+    string meta_plugin_id;
+    r = read_meta("extblkdev", &meta_plugin_id);
+    if (r == 0) {
+      // plugin selection fixed to meta, plugins must be loaded
+      if (plugin_preload_r != 0) {
+        // we will complain twice - once generally about not loading plugins,
+        // and later that specific plugin is not ready
+        derr << "Failed preloading extblkdev plugins, error code: " << plugin_preload_r << dendl;
+      }
+      string bdev_plugin_id;
+      r = bdev->get_ebd_id(bdev_plugin_id);
+      if (r != 0) {
+        derr << __func__ << " plugin " << meta_plugin_id << " not loaded" << dendl;
+        goto fail_close;
+      }
+      if (meta_plugin_id != bdev_plugin_id) {
+        derr << __func__ << " plugin '" << meta_plugin_id << "' used on mkfs, "
+          << "but now uses plugin '" << bdev_plugin_id << "'" << dendl;
+        goto fail_close;
+      }
+    }
+  }
+
   if (bdev->supported_bdev_label()) {
     if (create) {
       r = _set_main_bdev_label();
@@ -8624,6 +8658,20 @@ int BlueStore::mkfs()
         return r;
     }
   }
+  if (cct->_conf->bluestore_use_ebd) {
+    // check if EBD plugin is enabled
+    string plugin_id;
+    r = bdev->get_ebd_id(plugin_id);
+    if (r == 0) {
+      // retrieved name, save plugin into bdev metadata
+      r = write_meta("extblkdev", plugin_id);
+      if (r < 0)
+        return r;
+    } else {
+      // Non zero result is not a problem, it just means we do not have EBD plugin.
+      r = 0;
+    }
+  }
 
   freelist_type = "bitmap";
   dout(10) << " freelist_type " << freelist_type << dendl;