From: Jason Dillaman Date: Tue, 28 Feb 2017 14:24:53 +0000 (-0500) Subject: librbd: utilize a singleton ContextWQ instead of per-image instance X-Git-Tag: v12.0.1~203^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=0d66a4013083128cf86e1ff2634e71ce587d25a7;p=ceph.git librbd: utilize a singleton ContextWQ instead of per-image instance Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index 4d5dd3ae18c5..0dcd7880b86b 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -54,12 +54,20 @@ namespace { class ThreadPoolSingleton : public ThreadPool { public: + ContextWQ *op_work_queue; + explicit ThreadPoolSingleton(CephContext *cct) : ThreadPool(cct, "librbd::thread_pool", "tp_librbd", 1, - "rbd_op_threads") { + "rbd_op_threads"), + op_work_queue(new ContextWQ("librbd::op_work_queue", + cct->_conf->rbd_op_thread_timeout, + this)) { start(); } ~ThreadPoolSingleton() override { + op_work_queue->drain(); + delete op_work_queue; + stop(); } }; @@ -199,13 +207,11 @@ struct C_InvalidateCache : public Context { memset(&header, 0, sizeof(header)); - ThreadPool *thread_pool_singleton = get_thread_pool_instance(cct); + ThreadPool *thread_pool; + get_thread_pool_instance(cct, &thread_pool, &op_work_queue); io_work_queue = new io::ImageRequestWQ( this, "librbd::io_work_queue", cct->_conf->rbd_op_thread_timeout, - thread_pool_singleton); - op_work_queue = new ContextWQ("librbd::op_work_queue", - cct->_conf->rbd_op_thread_timeout, - thread_pool_singleton); + thread_pool); if (cct->_conf->rbd_auto_exclusive_lock_until_manual_request) { exclusive_lock_policy = new exclusive_lock::AutomaticPolicy(this); @@ -241,12 +247,10 @@ struct C_InvalidateCache : public Context { md_ctx.aio_flush(); data_ctx.aio_flush(); - op_work_queue->drain(); io_work_queue->drain(); delete journal_policy; delete exclusive_lock_policy; - delete op_work_queue; delete io_work_queue; delete operations; delete state; @@ -1092,11 +1096,14 @@ struct C_InvalidateCache : public Context { journal_policy = policy; } - ThreadPool *ImageCtx::get_thread_pool_instance(CephContext *cct) { + void ImageCtx::get_thread_pool_instance(CephContext *cct, + ThreadPool **thread_pool, + ContextWQ **op_work_queue) { ThreadPoolSingleton *thread_pool_singleton; cct->lookup_or_create_singleton_object( thread_pool_singleton, "librbd::thread_pool"); - return thread_pool_singleton; + *thread_pool = thread_pool_singleton; + *op_work_queue = thread_pool_singleton->op_work_queue; } void ImageCtx::get_timer_instance(CephContext *cct, SafeTimer **timer, diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 76f38efb929e..bd9465dabb0e 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -317,7 +317,9 @@ namespace librbd { journal::Policy *get_journal_policy() const; void set_journal_policy(journal::Policy *policy); - static ThreadPool *get_thread_pool_instance(CephContext *cct); + static void get_thread_pool_instance(CephContext *cct, + ThreadPool **thread_pool, + ContextWQ **op_work_queue); static void get_timer_instance(CephContext *cct, SafeTimer **timer, Mutex **timer_lock); }; diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc index b959bbd72816..e75f3bfc75cc 100644 --- a/src/librbd/Journal.cc +++ b/src/librbd/Journal.cc @@ -344,20 +344,18 @@ int Journal::create(librados::IoCtx &io_ctx, const std::string &image_id, CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 5) << __func__ << ": image=" << image_id << dendl; + ThreadPool *thread_pool; + ContextWQ *op_work_queue; + ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue); + C_SaferCond cond; journal::TagData tag_data(LOCAL_MIRROR_UUID); - ContextWQ op_work_queue("librbd::op_work_queue", - cct->_conf->rbd_op_thread_timeout, - ImageCtx::get_thread_pool_instance(cct)); journal::CreateRequest *req = journal::CreateRequest::create( io_ctx, image_id, order, splay_width, object_pool, cls::journal::Tag::TAG_CLASS_NEW, - tag_data, IMAGE_CLIENT_ID, &op_work_queue, &cond); + tag_data, IMAGE_CLIENT_ID, op_work_queue, &cond); req->send(); - int r = cond.wait(); - op_work_queue.drain(); - - return r; + return cond.wait(); } template @@ -365,18 +363,16 @@ int Journal::remove(librados::IoCtx &io_ctx, const std::string &image_id) { CephContext *cct = reinterpret_cast(io_ctx.cct()); ldout(cct, 5) << __func__ << ": image=" << image_id << dendl; + ThreadPool *thread_pool; + ContextWQ *op_work_queue; + ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue); + C_SaferCond cond; - ContextWQ op_work_queue("librbd::op_work_queue", - cct->_conf->rbd_op_thread_timeout, - ImageCtx::get_thread_pool_instance(cct)); journal::RemoveRequest *req = journal::RemoveRequest::create( - io_ctx, image_id, IMAGE_CLIENT_ID, &op_work_queue, &cond); + io_ctx, image_id, IMAGE_CLIENT_ID, op_work_queue, &cond); req->send(); - int r = cond.wait(); - op_work_queue.drain(); - - return r; + return cond.wait(); } template diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index 175fc862ba39..082784a14099 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -988,19 +988,18 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, if (old_format) { r = create_v1(io_ctx, imgname, size, order); } else { - C_SaferCond cond; - ContextWQ op_work_queue("librbd::op_work_queue", - cct->_conf->rbd_op_thread_timeout, - ImageCtx::get_thread_pool_instance(cct)); + ThreadPool *thread_pool; + ContextWQ *op_work_queue; + ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue); + C_SaferCond cond; std::string id = util::generate_image_id(io_ctx); image::CreateRequest<> *req = image::CreateRequest<>::create( io_ctx, imgname, id, size, opts, non_primary_global_image_id, - primary_mirror_uuid, skip_mirror_enable, &op_work_queue, &cond); + primary_mirror_uuid, skip_mirror_enable, op_work_queue, &cond); req->send(); r = cond.wait(); - op_work_queue.drain(); } int r1 = opts.set(RBD_IMAGE_OPTION_ORDER, order); @@ -1621,18 +1620,16 @@ int mirror_image_disable_internal(ImageCtx *ictx, bool force, ldout(cct, 20) << "remove " << &io_ctx << " " << (image_id.empty() ? image_name : image_id) << dendl; + ThreadPool *thread_pool; + ContextWQ *op_work_queue; + ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue); + C_SaferCond cond; - ContextWQ op_work_queue("librbd::op_work_queue", - cct->_conf->rbd_op_thread_timeout, - ImageCtx::get_thread_pool_instance(cct)); - librbd::image::RemoveRequest<> *req = librbd::image::RemoveRequest<>::create( - io_ctx, image_name, image_id, force, prog_ctx, &op_work_queue, &cond); + auto req = librbd::image::RemoveRequest<>::create( + io_ctx, image_name, image_id, force, prog_ctx, op_work_queue, &cond); req->send(); - int r = cond.wait(); - op_work_queue.drain(); - - return r; + return cond.wait(); } int snap_list(ImageCtx *ictx, vector& snaps)