librbd: add task pool / work queue for AIO requests

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
(cherry picked from commit afb896d91f886b647baf38f7ec94cc3739f6d2a9)

Conflicts:
	src/common/config_opts.h: trivial resolution
	src/librbd/ImageCtx.cc: trivial resolution
	src/librbd/ImageCtx.h: trivial resolution
	src/librbd/internal.cc: trivial resolution

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index d8ecdc70357c764a78cff6045c45df1a3059a264..2290c569fbf1a291f70523232f7f24ff12e3da4a 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -727,6 +727,8 @@ OPTION(journal_ignore_corruption, OPT_BOOL, false) // assume journal is not corr
 OPTION(rados_mon_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a response from the monitor before returning an error from a rados operation. 0 means on limit.
 OPTION(rados_osd_op_timeout, OPT_DOUBLE, 0) // how many seconds to wait for a response from osds before returning an error from a rados operation. 0 means no limit.
 
+OPTION(rbd_op_threads, OPT_INT, 1)
+OPTION(rbd_op_thread_timeout, OPT_INT, 60)
 OPTION(rbd_cache, OPT_BOOL, false) // whether to enable caching (writeback unless rbd_cache_max_dirty is 0)
 OPTION(rbd_cache_writethrough_until_flush, OPT_BOOL, false) // whether to make writeback caching writethrough until flush is called, to be sure the user of librbd will send flushs so that writeback is safe
 OPTION(rbd_cache_size, OPT_LONGLONG, 32<<20)         // cache size in bytes

diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc
index 1295d421c23533965062f9d60b93dc6a4efe1d55..942ab78a565cc37d8b64098942de1434076dbc35 100644 (file)
--- a/src/librbd/ImageCtx.cc
+++ b/src/librbd/ImageCtx.cc
@@ -6,6 +6,7 @@
 #include "common/dout.h"
 #include "common/errno.h"
 #include "common/perf_counters.h"
+#include "common/WorkQueue.h"
 
 #include "librbd/internal.h"
 #include "librbd/WatchCtx.h"
@@ -27,6 +28,23 @@ using librados::snap_t;
 using librados::IoCtx;
 
 namespace librbd {
+
+namespace {
+
+class ThreadPoolSingleton : public ThreadPool {
+public:
+  ThreadPoolSingleton(CephContext *cct)
+    : ThreadPool(cct, "librbd::thread_pool", cct->_conf->rbd_op_threads,
+                 "rbd_op_threads") {
+    start();
+  }
+  virtual ~ThreadPoolSingleton() {
+    stop();
+  }
+};
+
+} // anonymous namespace
+
   ImageCtx::ImageCtx(const string &image_name, const string &image_id,
                     const char *snap, IoCtx& p, bool ro)
     : cct((CephContext*)p.cct()),
@@ -53,7 +71,7 @@ namespace librbd {
       id(image_id), parent(NULL),
       stripe_unit(0), stripe_count(0),
       object_cacher(NULL), writeback_handler(NULL), object_set(NULL),
-      pending_aio(0)
+      pending_aio(0), aio_work_queue(NULL)
   {
     md_ctx.dup(p);
     data_ctx.dup(p);
@@ -98,6 +116,13 @@ namespace librbd {
       object_set->return_enoent = true;
       object_cacher->start();
     }
+
+    ThreadPoolSingleton *thread_pool_singleton;
+    cct->lookup_or_create_singleton_object<ThreadPoolSingleton>(
+      thread_pool_singleton, "librbd::thread_pool");
+    aio_work_queue = new ContextWQ("librbd::aio_work_queue",
+                                   cct->_conf->rbd_op_thread_timeout,
+                                   thread_pool_singleton);
   }
 
   ImageCtx::~ImageCtx() {
@@ -115,6 +140,8 @@ namespace librbd {
       object_set = NULL;
     }
     delete[] format_string;
+
+    delete aio_work_queue;
   }
 
   int ImageCtx::init() {

diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h
index 5a0d63752e6165c3051258747be8b005b3dc0959..406192b57531f49488674ba84a58a587954e28b5 100644 (file)
--- a/src/librbd/ImageCtx.h
+++ b/src/librbd/ImageCtx.h
@@ -26,6 +26,7 @@
 #include "librbd/parent_types.h"
 
 class CephContext;
+class ContextWQ;
 class PerfCounters;
 
 namespace librbd {
@@ -95,6 +96,8 @@ namespace librbd {
     Cond pending_aio_cond;
     uint64_t pending_aio;
 
+    ContextWQ *aio_work_queue;
+
     /**
      * Either image_name or image_id must be set.
      * If id is not known, pass the empty std::string,

diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc
index 1456012460aee2c50ac4b1a53169bc3b23a9c42f..76f3e403644dd8cb6690f050834e426ca166c904 100644 (file)
--- a/src/librbd/internal.cc
+++ b/src/librbd/internal.cc
@@ -9,6 +9,7 @@
 #include "common/dout.h"
 #include "common/errno.h"
 #include "common/Throttle.h"
+#include "common/WorkQueue.h"
 #include "cls/lock/cls_lock_client.h"
 #include "include/stringify.h"
 
@@ -2151,6 +2152,9 @@ reprotect_and_return_err:
   void close_image(ImageCtx *ictx)
   {
     ldout(ictx->cct, 20) << "close_image " << ictx << dendl;
+
+    ictx->aio_work_queue->drain();
+
     if (ictx->object_cacher) {
       ictx->shutdown_cache(); // implicitly flushes
     } else {