librbd: utilize a singleton ContextWQ instead of per-image instance

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 4d5dd3ae18c5d76c12d621305181b5b5c21c9f34..0dcd7880b86bd11ca28289516ea139d4c89fa267 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -54,12 +54,20 @@ namespace {
 
 class ThreadPoolSingleton : public ThreadPool {
 public:
+  ContextWQ *op_work_queue;
+
   explicit ThreadPoolSingleton(CephContext *cct)
     : ThreadPool(cct, "librbd::thread_pool", "tp_librbd", 1,
-                 "rbd_op_threads") {
+                 "rbd_op_threads"),
+      op_work_queue(new ContextWQ("librbd::op_work_queue",
+                                  cct->_conf->rbd_op_thread_timeout,
+                                  this)) {
     start();
   }
   ~ThreadPoolSingleton() override {
+    op_work_queue->drain();
+    delete op_work_queue;
+
     stop();
   }
 };
@@ -199,13 +207,11 @@ struct C_InvalidateCache : public Context {
 
     memset(&header, 0, sizeof(header));
 
-    ThreadPool *thread_pool_singleton = get_thread_pool_instance(cct);
+    ThreadPool *thread_pool;
+    get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
     io_work_queue = new io::ImageRequestWQ(
       this, "librbd::io_work_queue", cct->_conf->rbd_op_thread_timeout,
-      thread_pool_singleton);
-    op_work_queue = new ContextWQ("librbd::op_work_queue",
-                                  cct->_conf->rbd_op_thread_timeout,
-                                  thread_pool_singleton);
+      thread_pool);
 
     if (cct->_conf->rbd_auto_exclusive_lock_until_manual_request) {
       exclusive_lock_policy = new exclusive_lock::AutomaticPolicy(this);
@@ -241,12 +247,10 @@ struct C_InvalidateCache : public Context {
 
     md_ctx.aio_flush();
     data_ctx.aio_flush();
-    op_work_queue->drain();
     io_work_queue->drain();
 
     delete journal_policy;
     delete exclusive_lock_policy;
-    delete op_work_queue;
     delete io_work_queue;
     delete operations;
     delete state;
@@ -1092,11 +1096,14 @@ struct C_InvalidateCache : public Context {
     journal_policy = policy;
   }
 
-  ThreadPool *ImageCtx::get_thread_pool_instance(CephContext *cct) {
+  void ImageCtx::get_thread_pool_instance(CephContext *cct,
+                                          ThreadPool **thread_pool,
+                                          ContextWQ **op_work_queue) {
     ThreadPoolSingleton *thread_pool_singleton;
     cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
       thread_pool_singleton, "librbd::thread_pool");
-    return thread_pool_singleton;
+    *thread_pool = thread_pool_singleton;
+    *op_work_queue = thread_pool_singleton->op_work_queue;
   }
 
   void ImageCtx::get_timer_instance(CephContext *cct, SafeTimer **timer,

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 76f38efb929ebe79b43a12ea2067c96db4b38250..bd9465dabb0e8452f9fd43015abdd7088e32778b 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -317,7 +317,9 @@ namespace librbd {
     journal::Policy *get_journal_policy() const;
     void set_journal_policy(journal::Policy *policy);
 
-    static ThreadPool *get_thread_pool_instance(CephContext *cct);
+    static void get_thread_pool_instance(CephContext *cct,
+                                         ThreadPool **thread_pool,
+                                         ContextWQ **op_work_queue);
     static void get_timer_instance(CephContext *cct, SafeTimer **timer,
                                    Mutex **timer_lock);
   };

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index b959bbd72816b5f6b54f5ffc06b9bd6f671cf36d..e75f3bfc75cc833c9c17fb752f61557f6ed93c71 100644 (file)
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -344,20 +344,18 @@ int Journal<I>::create(librados::IoCtx &io_ctx, const std::string &image_id,
   CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
   ldout(cct, 5) << __func__ << ": image=" << image_id << dendl;
 
+  ThreadPool *thread_pool;
+  ContextWQ *op_work_queue;
+  ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
+
   C_SaferCond cond;
   journal::TagData tag_data(LOCAL_MIRROR_UUID);
-  ContextWQ op_work_queue("librbd::op_work_queue",
-                          cct->_conf->rbd_op_thread_timeout,
-                          ImageCtx::get_thread_pool_instance(cct));
   journal::CreateRequest<I> *req = journal::CreateRequest<I>::create(
     io_ctx, image_id, order, splay_width, object_pool, cls::journal::Tag::TAG_CLASS_NEW,
-    tag_data, IMAGE_CLIENT_ID, &op_work_queue, &cond);
+    tag_data, IMAGE_CLIENT_ID, op_work_queue, &cond);
   req->send();
 
-  int r = cond.wait();
-  op_work_queue.drain();
-
-  return r;
+  return cond.wait();
 }
 
 template <typename I>
@@ -365,18 +363,16 @@ int Journal<I>::remove(librados::IoCtx &io_ctx, const std::string &image_id) {
   CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
   ldout(cct, 5) << __func__ << ": image=" << image_id << dendl;
 
+  ThreadPool *thread_pool;
+  ContextWQ *op_work_queue;
+  ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
+
   C_SaferCond cond;
-  ContextWQ op_work_queue("librbd::op_work_queue",
-                          cct->_conf->rbd_op_thread_timeout,
-                          ImageCtx::get_thread_pool_instance(cct));
   journal::RemoveRequest<I> *req = journal::RemoveRequest<I>::create(
-    io_ctx, image_id, IMAGE_CLIENT_ID, &op_work_queue, &cond);
+    io_ctx, image_id, IMAGE_CLIENT_ID, op_work_queue, &cond);
   req->send();
 
-  int r = cond.wait();
-  op_work_queue.drain();
-
-  return r;
+  return cond.wait();
 }
 
 template <typename I>

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 175fc862ba3936220af323666c87e204258574eb..082784a140992654f454baef4a0a20494d8b7347 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -988,19 +988,18 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force,
     if (old_format) {
       r = create_v1(io_ctx, imgname, size, order);
     } else {
-      C_SaferCond cond;
-      ContextWQ op_work_queue("librbd::op_work_queue",
-                              cct->_conf->rbd_op_thread_timeout,
-                              ImageCtx::get_thread_pool_instance(cct));
+      ThreadPool *thread_pool;
+      ContextWQ *op_work_queue;
+      ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
 
+      C_SaferCond cond;
       std::string id = util::generate_image_id(io_ctx);
       image::CreateRequest<> *req = image::CreateRequest<>::create(
         io_ctx, imgname, id, size, opts, non_primary_global_image_id,
-        primary_mirror_uuid, skip_mirror_enable, &op_work_queue, &cond);
+        primary_mirror_uuid, skip_mirror_enable, op_work_queue, &cond);
       req->send();
 
       r = cond.wait();
-      op_work_queue.drain();
     }
 
     int r1 = opts.set(RBD_IMAGE_OPTION_ORDER, order);
@@ -1621,18 +1620,16 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force,
     ldout(cct, 20) << "remove " << &io_ctx << " "
                    << (image_id.empty() ? image_name : image_id) << dendl;
 
+    ThreadPool *thread_pool;
+    ContextWQ *op_work_queue;
+    ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
+
     C_SaferCond cond;
-    ContextWQ op_work_queue("librbd::op_work_queue",
-                            cct->_conf->rbd_op_thread_timeout,
-                            ImageCtx::get_thread_pool_instance(cct));
-    librbd::image::RemoveRequest<> *req = librbd::image::RemoveRequest<>::create(
-      io_ctx, image_name, image_id, force, prog_ctx, &op_work_queue, &cond);
+    auto req = librbd::image::RemoveRequest<>::create(
+      io_ctx, image_name, image_id, force, prog_ctx, op_work_queue, &cond);
     req->send();
 
-    int r = cond.wait();
-    op_work_queue.drain();
-
-    return r;
+    return cond.wait();
   }
 
   int snap_list(ImageCtx *ictx, vector<snap_info_t>& snaps)