From 89accffc588d02c601a9af396e4e2d29fb1a5a89 Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Wed, 20 May 2020 14:36:49 -0400
Subject: [PATCH] librbd: initial plugin registry implementation

The plugin registry will attempt to dynamically load the specified
librbd plugins at image open time.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 src/common/options.cc           |  5 ++++
 src/librbd/CMakeLists.txt       |  5 ++++
 src/librbd/ImageCtx.cc          |  6 +++-
 src/librbd/ImageCtx.h           |  3 ++
 src/librbd/PluginRegistry.cc    | 52 +++++++++++++++++++++++++++++++++
 src/librbd/PluginRegistry.h     | 38 ++++++++++++++++++++++++
 src/librbd/image/OpenRequest.cc | 30 +++++++++++++++++++
 src/librbd/image/OpenRequest.h  |  6 ++++
 src/librbd/plugin/Types.h       | 31 ++++++++++++++++++++
 9 files changed, 175 insertions(+), 1 deletion(-)
 create mode 100644 src/librbd/PluginRegistry.cc
 create mode 100644 src/librbd/PluginRegistry.h
 create mode 100644 src/librbd/plugin/Types.h

diff --git a/src/common/options.cc b/src/common/options.cc
index c39095bfde89..99aad045c946 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7513,6 +7513,11 @@ static std::vector<Option> get_rbd_options() {
     .set_default(10)
     .set_min(1)
     .set_description("the number of quiesce notification attempts"),
+
+    Option("rbd_plugins", Option::TYPE_STR, Option::LEVEL_ADVANCED)
+    .set_default("")
+    .set_description("comma-delimited list of librbd plugins to enable"),
+
   });
 }
 
diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt
index 5deb68071bc9..57823cdbf678 100644
--- a/src/librbd/CMakeLists.txt
+++ b/src/librbd/CMakeLists.txt
@@ -20,6 +20,7 @@ set(librbd_internal_srcs
   MirroringWatcher.cc
   ObjectMap.cc
   Operations.cc
+  PluginRegistry.cc
   TrashWatcher.cc
   Utils.cc
   Watcher.cc
@@ -166,6 +167,9 @@ set(librbd_internal_srcs
   watcher/RewatchRequest.cc
   ${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc)
 
+add_custom_target(librbd_plugins)
+set(librbd_plugins_dir ${CEPH_INSTALL_PKGLIBDIR}/librbd)
+
 if(WITH_EVENTTRACE)
   list(APPEND librbd_internal_srcs ../common/EventTrace.cc)
 endif()
@@ -211,6 +215,7 @@ add_library(librbd ${CEPH_SHARED}
 if(WITH_LTTNG)
   add_dependencies(librbd librbd-tp)
 endif()
+add_dependencies(librbd librbd_plugins)
 
 target_link_libraries(librbd PRIVATE
   rbd_internal
diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index b1798c4680e3..3c85fede4852 100644
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -21,7 +21,7 @@
 #include "librbd/LibrbdAdminSocketHook.h"
 #include "librbd/ObjectMap.h"
 #include "librbd/Operations.h"
-#include "librbd/operation/ResizeRequest.h"
+#include "librbd/PluginRegistry.h"
 #include "librbd/Types.h"
 #include "librbd/Utils.h"
 #include "librbd/exclusive_lock/AutomaticPolicy.h"
@@ -32,6 +32,7 @@
 #include "librbd/io/ObjectDispatcher.h"
 #include "librbd/io/QosImageDispatch.h"
 #include "librbd/journal/StandardPolicy.h"
+#include "librbd/operation/ResizeRequest.h"
 
 #include "osdc/Striper.h"
 #include <boost/bind.hpp>
@@ -123,6 +124,7 @@ public:
       operations(new Operations<>(*this)),
       exclusive_lock(nullptr), object_map(nullptr),
       op_work_queue(nullptr),
+      plugin_registry(new PluginRegistry<ImageCtx>(this)),
       external_callback_completions(32),
       event_socket_completions(32),
       asok_hook(nullptr),
@@ -179,6 +181,8 @@ public:
     delete exclusive_lock_policy;
     delete operations;
     delete state;
+
+    delete plugin_registry;
   }
 
   void ImageCtx::init() {
diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index ed47d43a5569..d1c2ef135cd9 100644
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -49,6 +49,7 @@ namespace librbd {
   class LibrbdAdminSocketHook;
   template <typename> class ObjectMap;
   template <typename> class Operations;
+  template <typename> class PluginRegistry;
 
   namespace cache {
   template <typename> class ImageCache;
@@ -185,6 +186,8 @@ namespace librbd {
 
     ContextWQ *op_work_queue;
 
+    PluginRegistry<ImageCtx>* plugin_registry;
+
     typedef boost::lockfree::queue<
       io::AioCompletion*,
       boost::lockfree::allocator<ceph::allocator<void>>> Completions;
diff --git a/src/librbd/PluginRegistry.cc b/src/librbd/PluginRegistry.cc
new file mode 100644
index 000000000000..c600c860ffdb
--- /dev/null
+++ b/src/librbd/PluginRegistry.cc
@@ -0,0 +1,52 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/PluginRegistry.h"
+#include "include/Context.h"
+#include "common/dout.h"
+#include "librbd/ImageCtx.h"
+#include <boost/tokenizer.hpp>
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::PluginRegistry: " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+
+template <typename I>
+PluginRegistry<I>::PluginRegistry(I* image_ctx) : m_image_ctx(image_ctx) {
+}
+
+template <typename I>
+void PluginRegistry<I>::init(const std::string& plugins, Context* on_finish) {
+  auto cct = m_image_ctx->cct;
+  auto plugin_registry = cct->get_plugin_registry();
+
+  auto gather_ctx = new C_Gather(cct, on_finish);
+
+  boost::tokenizer<boost::escaped_list_separator<char>> tokenizer(plugins);
+  for (auto token : tokenizer) {
+    ldout(cct, 5) << "attempting to load plugin: " << token << dendl;
+
+    auto ctx = gather_ctx->new_sub();
+
+    auto plugin = dynamic_cast<plugin::Interface<I>*>(
+      plugin_registry->get_with_load("librbd", "librbd_" + token));
+    if (plugin == nullptr) {
+      lderr(cct) << "failed to load plugin: " << token << dendl;
+      ctx->complete(-ENOENT);
+      break;
+    }
+
+    m_plugin_hook_points.emplace_back();
+    auto hook_points = &m_plugin_hook_points.back();
+    plugin->init(m_image_ctx, hook_points, ctx);
+  }
+
+  gather_ctx->activate();
+}
+
+} // namespace librbd
+
+template class librbd::PluginRegistry<librbd::ImageCtx>;
diff --git a/src/librbd/PluginRegistry.h b/src/librbd/PluginRegistry.h
new file mode 100644
index 000000000000..2e975f89d6b7
--- /dev/null
+++ b/src/librbd/PluginRegistry.h
@@ -0,0 +1,38 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_PLUGIN_REGISTRY_H
+#define CEPH_LIBRBD_PLUGIN_REGISTRY_H
+
+#include "librbd/plugin/Types.h"
+#include <string>
+#include <list>
+
+struct Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+template <typename ImageCtxT>
+class PluginRegistry {
+public:
+  PluginRegistry(ImageCtxT* image_ctx);
+
+  void init(const std::string& plugins, Context* on_finish);
+
+private:
+  typedef std::list<plugin::HookPoints> PluginHookPoints;
+
+  ImageCtxT* m_image_ctx;
+  std::string m_plugins;
+
+  PluginHookPoints m_plugin_hook_points;
+
+};
+
+} // namespace librbd
+
+extern template class librbd::PluginRegistry<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_PLUGIN_REGISTRY_H
diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc
index 88d5d62e0e5a..f9e7696ec2aa 100644
--- a/src/librbd/image/OpenRequest.cc
+++ b/src/librbd/image/OpenRequest.cc
@@ -6,6 +6,7 @@
 #include "common/errno.h"
 #include "cls/rbd/cls_rbd_client.h"
 #include "librbd/ImageCtx.h"
+#include "librbd/PluginRegistry.h"
 #include "librbd/Utils.h"
 #include "librbd/cache/ObjectCacherObjectDispatch.h"
 #include "librbd/cache/WriteAroundObjectDispatch.h"
@@ -519,6 +520,35 @@ Context *OpenRequest<I>::handle_refresh(int *result) {
     return nullptr;
   }
 
+  send_init_plugin_registry();
+  return nullptr;
+}
+
+template <typename I>
+void OpenRequest<I>::send_init_plugin_registry() {
+  CephContext *cct = m_image_ctx->cct;
+
+  auto plugins = m_image_ctx->config.template get_val<std::string>(
+    "rbd_plugins");
+  ldout(cct, 10) << __func__ << ": plugins=" << plugins << dendl;
+
+  auto ctx = create_context_callback<
+    OpenRequest<I>, &OpenRequest<I>::handle_init_plugin_registry>(this);
+  m_image_ctx->plugin_registry->init(plugins, ctx);
+}
+
+template <typename I>
+Context* OpenRequest<I>::handle_init_plugin_registry(int *result) {
+  CephContext *cct = m_image_ctx->cct;
+  ldout(cct, 10) << __func__ << ": r=" << *result << dendl;
+
+  if (*result < 0) {
+    lderr(cct) << "failed to initialize plugin registry: "
+               << cpp_strerror(*result) << dendl;
+    send_close_image(*result);
+    return nullptr;
+  }
+
   return send_parent_cache(result);
 }
 
diff --git a/src/librbd/image/OpenRequest.h b/src/librbd/image/OpenRequest.h
index 460839bb3d85..9241ef0b64d4 100644
--- a/src/librbd/image/OpenRequest.h
+++ b/src/librbd/image/OpenRequest.h
@@ -61,6 +61,9 @@ private:
    *            V2_GET_DATA_POOL --------------> REFRESH
    *                                                |
    *                                                v
+   *                                             INIT_PLUGIN_REGISTRY
+   *                                                |
+   *                                                v
    *                                             INIT_PARENT_CACHE(skip if
    *                                                |               disable)
    *                                                v
@@ -123,6 +126,9 @@ private:
   void send_refresh();
   Context *handle_refresh(int *result);
 
+  void send_init_plugin_registry();
+  Context* handle_init_plugin_registry(int *result);
+
   Context* send_parent_cache(int *result);
   Context* handle_parent_cache(int *result);
 
diff --git a/src/librbd/plugin/Types.h b/src/librbd/plugin/Types.h
new file mode 100644
index 000000000000..73f45099ff9e
--- /dev/null
+++ b/src/librbd/plugin/Types.h
@@ -0,0 +1,31 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_PLUGIN_TYPES_H
+#define CEPH_LIBRBD_PLUGIN_TYPES_H
+
+#include "common/PluginRegistry.h"
+
+struct CephContext;
+struct Context;
+
+namespace librbd {
+namespace plugin {
+
+struct HookPoints {
+  // TODO later commits will add support for exclusive-lock hook points
+};
+
+template <typename ImageCtxT>
+struct Interface : public ceph::Plugin {
+  Interface(CephContext* cct) : Plugin(cct) {
+  }
+
+  virtual void init(ImageCtxT* image_ctx, HookPoints* hook_points,
+                    Context* on_finish) = 0;
+};
+
+} // namespace plugin
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_PLUGIN_TYPES_H
-- 
2.47.3