From 34dd39c0343dfc2405cee32594e9e5d30f6e4483 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Sun, 28 Feb 2016 15:22:09 -0500 Subject: [PATCH] librbd: use aio watch/unwatch during open/close state machines Removed the place holders in the open/close async state machines with the newly added aio watch/unwatch methods. Signed-off-by: Jason Dillaman --- src/librbd/ImageCtx.cc | 16 ++----- src/librbd/ImageCtx.h | 3 +- src/librbd/ImageWatcher.cc | 75 +++++++++++++++++++++++++------- src/librbd/ImageWatcher.h | 18 +++++++- src/librbd/image/CloseRequest.cc | 12 +++-- src/librbd/image/OpenRequest.cc | 28 ++---------- 6 files changed, 91 insertions(+), 61 deletions(-) diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index e91000357a05..65d1d9e37aad 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -287,9 +287,8 @@ struct C_InvalidateCache : public Context { } void ImageCtx::shutdown() { - if (image_watcher != nullptr) { - unregister_watch(); - } + delete image_watcher; + image_watcher = nullptr; delete asok_hook; asok_hook = nullptr; @@ -780,17 +779,10 @@ struct C_InvalidateCache : public Context { object_cacher->clear_nonexistence(object_set); } - int ImageCtx::register_watch() { + void ImageCtx::register_watch(Context *on_finish) { assert(image_watcher == NULL); image_watcher = new ImageWatcher(*this); - return image_watcher->register_watch(); - } - - void ImageCtx::unregister_watch() { - assert(image_watcher != NULL); - image_watcher->unregister_watch(); - delete image_watcher; - image_watcher = NULL; + image_watcher->register_watch(on_finish); } uint64_t ImageCtx::prune_parent_extents(vector >& objectx, diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 0d84f0caad37..e914491d1439 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -255,8 +255,7 @@ namespace librbd { int invalidate_cache(bool purge_on_error=false); void invalidate_cache(Context *on_finish); void clear_nonexistence_cache(); - int register_watch(); - void unregister_watch(); + void register_watch(Context *on_finish); uint64_t prune_parent_extents(vector >& objectx, uint64_t overlap); diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc index 3510028e281d..2bd3c0a61cb7 100644 --- a/src/librbd/ImageWatcher.cc +++ b/src/librbd/ImageWatcher.cc @@ -30,6 +30,43 @@ namespace librbd { using namespace image_watcher; using namespace watch_notify; using util::create_context_callback; +using util::create_rados_safe_callback; + +namespace { + +struct C_UnwatchAndFlush : public Context { + librados::Rados rados; + Context *on_finish; + bool flushing = false; + int ret_val = 0; + + C_UnwatchAndFlush(librados::IoCtx &io_ctx, Context *on_finish) + : rados(io_ctx), on_finish(on_finish) { + } + + virtual void complete(int r) override { + if (ret_val == 0 && r < 0) { + ret_val = r; + } + + if (!flushing) { + flushing = true; + + librados::AioCompletion *aio_comp = create_rados_safe_callback(this); + r = rados.aio_watch_flush(aio_comp); + assert(r == 0); + aio_comp->release(); + } else { + Context::complete(ret_val); + } + } + + virtual void finish(int r) override { + on_finish->complete(r); + } +}; + +} // anonymous namespace static const double RETRY_DELAY_SECONDS = 1.0; @@ -54,40 +91,46 @@ ImageWatcher::~ImageWatcher() } } -int ImageWatcher::register_watch() { +void ImageWatcher::register_watch(Context *on_finish) { ldout(m_image_ctx.cct, 10) << this << " registering image watcher" << dendl; - RWLock::WLocker l(m_watch_lock); + RWLock::RLocker watch_locker(m_watch_lock); + assert(m_watch_state == WATCH_STATE_UNREGISTERED); + librados::AioCompletion *aio_comp = create_rados_safe_callback( + new C_RegisterWatch(this, on_finish)); + int r = m_image_ctx.md_ctx.aio_watch(m_image_ctx.header_oid, aio_comp, + &m_watch_handle, &m_watch_ctx); + assert(r == 0); + aio_comp->release(); +} + +void ImageWatcher::handle_register_watch(int r) { + RWLock::WLocker watch_locker(m_watch_lock); assert(m_watch_state == WATCH_STATE_UNREGISTERED); - int r = m_image_ctx.md_ctx.watch2(m_image_ctx.header_oid, - &m_watch_handle, - &m_watch_ctx); if (r < 0) { - return r; + m_watch_handle = 0; + } else if (r >= 0) { + m_watch_state = WATCH_STATE_REGISTERED; } - - m_watch_state = WATCH_STATE_REGISTERED; - return 0; } -int ImageWatcher::unregister_watch() { +void ImageWatcher::unregister_watch(Context *on_finish) { ldout(m_image_ctx.cct, 10) << this << " unregistering image watcher" << dendl; cancel_async_requests(); m_task_finisher->cancel_all(); - int r = 0; { RWLock::WLocker l(m_watch_lock); if (m_watch_state == WATCH_STATE_REGISTERED) { - r = m_image_ctx.md_ctx.unwatch2(m_watch_handle); + librados::AioCompletion *aio_comp = create_rados_safe_callback( + new C_UnwatchAndFlush(m_image_ctx.md_ctx, on_finish)); + int r = m_image_ctx.md_ctx.aio_unwatch(m_watch_handle, aio_comp); + assert(r == 0); + aio_comp->release(); } m_watch_state = WATCH_STATE_UNREGISTERED; } - - librados::Rados rados(m_image_ctx.md_ctx); - rados.watch_flush(); - return r; } void ImageWatcher::flush(Context *on_finish) { diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h index 26738a6b728c..89edb35df2c6 100644 --- a/src/librbd/ImageWatcher.h +++ b/src/librbd/ImageWatcher.h @@ -30,8 +30,8 @@ public: ImageWatcher(ImageCtx& image_ctx); ~ImageWatcher(); - int register_watch(); - int unregister_watch(); + void register_watch(Context *on_finish); + void unregister_watch(Context *on_finish); void flush(Context *on_finish); int notify_flatten(uint64_t request_id, ProgressContext &prog_ctx); @@ -149,6 +149,18 @@ private: ProgressContext *m_prog_ctx; }; + struct C_RegisterWatch : public Context { + ImageWatcher *image_watcher; + Context *on_finish; + + C_RegisterWatch(ImageWatcher *image_watcher, Context *on_finish) + : image_watcher(image_watcher), on_finish(on_finish) { + } + virtual void finish(int r) override { + image_watcher->handle_register_watch(r); + on_finish->complete(r); + } + }; struct C_NotifyAck : public Context { ImageWatcher *image_watcher; uint64_t notify_id; @@ -224,6 +236,8 @@ private: image_watcher::Notifier m_notifier; + void handle_register_watch(int r); + void schedule_cancel_async_requests(); void cancel_async_requests(); diff --git a/src/librbd/image/CloseRequest.cc b/src/librbd/image/CloseRequest.cc index 19916396a071..fa75e9426204 100644 --- a/src/librbd/image/CloseRequest.cc +++ b/src/librbd/image/CloseRequest.cc @@ -33,18 +33,22 @@ CloseRequest::CloseRequest(I *image_ctx, Context *on_finish) template void CloseRequest::send() { - // TODO - send_shut_down_aio_queue(); - //send_unregister_image_watcher(); + send_unregister_image_watcher(); } template void CloseRequest::send_unregister_image_watcher() { + if (m_image_ctx->image_watcher == nullptr) { + send_shut_down_aio_queue(); + return; + } + CephContext *cct = m_image_ctx->cct; ldout(cct, 10) << this << " " << __func__ << dendl; // prevent incoming requests from our peers - + m_image_ctx->image_watcher->unregister_watch(create_context_callback< + CloseRequest, &CloseRequest::handle_unregister_image_watcher>(this)); } template diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc index 68abbba620ab..fa1311c7863a 100644 --- a/src/librbd/image/OpenRequest.cc +++ b/src/librbd/image/OpenRequest.cc @@ -19,26 +19,6 @@ namespace librbd { namespace image { -namespace { - -template -class C_RegisterWatch : public Context { -public: - I &image_ctx; - Context *on_finish; - - C_RegisterWatch(I &image_ctx, Context *on_finish) - : image_ctx(image_ctx), on_finish(on_finish) { - } - - virtual void finish(int r) { - assert(r == 0); - on_finish->complete(image_ctx.register_watch()); - } -}; - -} // anonymous namespace - using util::create_context_callback; using util::create_rados_ack_callback; @@ -291,12 +271,10 @@ void OpenRequest::send_register_watch() { CephContext *cct = m_image_ctx->cct; ldout(cct, 10) << this << " " << __func__ << dendl; - // no librados async version of watch using klass = OpenRequest; - Context *ctx = new C_RegisterWatch( - *m_image_ctx, - create_context_callback(this)); - m_image_ctx->op_work_queue->queue(ctx); + Context *ctx = create_context_callback< + klass, &klass::handle_register_watch>(this); + m_image_ctx->register_watch(ctx); } else { send_refresh(); } -- 2.47.3