From 6dda2b9384d53b107985289a4355f8ed3c7f5b69 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Sat, 1 Oct 2016 22:23:48 +0300 Subject: [PATCH] rbd-mirror HA: move librbd::image_watcher::Notifier to librbd::object_watcher Fixes: http://tracker.ceph.com/issues/17017 Signed-off-by: Mykola Golub --- src/librbd/CMakeLists.txt | 2 +- src/librbd/ImageWatcher.cc | 5 ++- src/librbd/ImageWatcher.h | 4 +-- src/librbd/image_watcher/NotifyLockOwner.cc | 5 +-- src/librbd/image_watcher/NotifyLockOwner.h | 13 +++---- .../Notifier.cc | 35 +++++++++---------- .../Notifier.h | 22 +++++++----- 7 files changed, 45 insertions(+), 41 deletions(-) rename src/librbd/{image_watcher => object_watcher}/Notifier.cc (63%) rename src/librbd/{image_watcher => object_watcher}/Notifier.h (68%) diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 4f46c1f2bacd..68f3f461c7e1 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -41,7 +41,6 @@ set(librbd_internal_srcs image/RefreshRequest.cc image/SetFlagsRequest.cc image/SetSnapRequest.cc - image_watcher/Notifier.cc image_watcher/NotifyLockOwner.cc image_watcher/RewatchRequest.cc journal/RemoveRequest.cc @@ -62,6 +61,7 @@ set(librbd_internal_srcs object_map/SnapshotRollbackRequest.cc object_map/UnlockRequest.cc object_map/UpdateRequest.cc + object_watcher/Notifier.cc operation/DisableFeaturesRequest.cc operation/EnableFeaturesRequest.cc operation/FlattenRequest.cc diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc index ea875736b405..d5fd6fed8466 100644 --- a/src/librbd/ImageWatcher.cc +++ b/src/librbd/ImageWatcher.cc @@ -11,7 +11,6 @@ #include "librbd/TaskFinisher.h" #include "librbd/Utils.h" #include "librbd/exclusive_lock/Policy.h" -#include "librbd/image_watcher/Notifier.h" #include "librbd/image_watcher/NotifyLockOwner.h" #include "librbd/image_watcher/RewatchRequest.h" #include "include/encoding.h" @@ -85,7 +84,7 @@ ImageWatcher::ImageWatcher(I &image_ctx) m_task_finisher(new TaskFinisher(*m_image_ctx.cct)), m_async_request_lock(util::unique_lock_name("librbd::ImageWatcher::m_async_request_lock", this)), m_owner_client_id_lock(util::unique_lock_name("librbd::ImageWatcher::m_owner_client_id_lock", this)), - m_notifier(image_ctx) + m_notifier(image_ctx.op_work_queue, image_ctx.md_ctx, image_ctx.header_oid) { } @@ -377,7 +376,7 @@ void ImageWatcher::notify_header_update(librados::IoCtx &io_ctx, // supports legacy (empty buffer) clients bufferlist bl; ::encode(NotifyMessage(HeaderUpdatePayload()), bl); - io_ctx.notify2(oid, bl, image_watcher::Notifier::NOTIFY_TIMEOUT, nullptr); + io_ctx.notify2(oid, bl, object_watcher::Notifier::NOTIFY_TIMEOUT, nullptr); } template diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h index db52a600422e..cac2d9d0e346 100644 --- a/src/librbd/ImageWatcher.h +++ b/src/librbd/ImageWatcher.h @@ -8,7 +8,7 @@ #include "common/RWLock.h" #include "include/Context.h" #include "include/rbd/librbd.hpp" -#include "librbd/image_watcher/Notifier.h" +#include "librbd/object_watcher/Notifier.h" #include "librbd/WatchNotifyTypes.h" #include #include @@ -241,7 +241,7 @@ private: Mutex m_owner_client_id_lock; watch_notify::ClientId m_owner_client_id; - image_watcher::Notifier m_notifier; + object_watcher::Notifier m_notifier; void handle_register_watch(int r); diff --git a/src/librbd/image_watcher/NotifyLockOwner.cc b/src/librbd/image_watcher/NotifyLockOwner.cc index 09a3e43e8da9..34be88a58051 100644 --- a/src/librbd/image_watcher/NotifyLockOwner.cc +++ b/src/librbd/image_watcher/NotifyLockOwner.cc @@ -6,7 +6,7 @@ #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "librbd/WatchNotifyTypes.h" -#include "librbd/image_watcher/Notifier.h" +#include "librbd/object_watcher/Notifier.h" #include #define dout_subsys ceph_subsys_rbd @@ -20,7 +20,8 @@ namespace image_watcher { using namespace watch_notify; using util::create_context_callback; -NotifyLockOwner::NotifyLockOwner(ImageCtx &image_ctx, Notifier ¬ifier, +NotifyLockOwner::NotifyLockOwner(ImageCtx &image_ctx, + object_watcher::Notifier ¬ifier, bufferlist &&bl, Context *on_finish) : m_image_ctx(image_ctx), m_notifier(notifier), m_bl(std::move(bl)), m_on_finish(on_finish) { diff --git a/src/librbd/image_watcher/NotifyLockOwner.h b/src/librbd/image_watcher/NotifyLockOwner.h index 0b85097b73f6..95dac6b5709e 100644 --- a/src/librbd/image_watcher/NotifyLockOwner.h +++ b/src/librbd/image_watcher/NotifyLockOwner.h @@ -12,25 +12,26 @@ namespace librbd { struct ImageCtx; -namespace image_watcher { +namespace object_watcher { class Notifier; } -class Notifier; +namespace image_watcher { class NotifyLockOwner { public: - static NotifyLockOwner *create(ImageCtx &image_ctx, Notifier ¬ifier, + static NotifyLockOwner *create(ImageCtx &image_ctx, + object_watcher::Notifier ¬ifier, bufferlist &&bl, Context *on_finish) { return new NotifyLockOwner(image_ctx, notifier, std::move(bl), on_finish); } - NotifyLockOwner(ImageCtx &image_ctx, Notifier ¬ifier, bufferlist &&bl, - Context *on_finish); + NotifyLockOwner(ImageCtx &image_ctx, object_watcher::Notifier ¬ifier, + bufferlist &&bl, Context *on_finish); void send(); private: ImageCtx &m_image_ctx; - Notifier &m_notifier; + object_watcher::Notifier &m_notifier; bufferlist m_bl; bufferlist m_out_bl; diff --git a/src/librbd/image_watcher/Notifier.cc b/src/librbd/object_watcher/Notifier.cc similarity index 63% rename from src/librbd/image_watcher/Notifier.cc rename to src/librbd/object_watcher/Notifier.cc index 7569f7572c1f..019c786ee62c 100644 --- a/src/librbd/image_watcher/Notifier.cc +++ b/src/librbd/object_watcher/Notifier.cc @@ -1,24 +1,26 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "librbd/image_watcher/Notifier.h" +#include "librbd/object_watcher/Notifier.h" #include "common/WorkQueue.h" #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix -#define dout_prefix *_dout << "librbd::image_watcher::Notifier: " +#define dout_prefix *_dout << "librbd::object_watcher::Notifier: " namespace librbd { -namespace image_watcher { +namespace object_watcher { const uint64_t Notifier::NOTIFY_TIMEOUT = 5000; -Notifier::Notifier(ImageCtx &image_ctx) - : m_image_ctx(image_ctx), +Notifier::Notifier(ContextWQ *work_queue, IoCtx &ioctx, const std::string &oid) + : m_work_queue(work_queue), m_oid(oid), m_aio_notify_lock(util::unique_lock_name( - "librbd::image_watcher::Notifier::m_aio_notify_lock", this)) { + "librbd::object_watcher::Notifier::m_aio_notify_lock", this)) { + m_ioctx.dup(ioctx); + m_cct = reinterpret_cast(m_ioctx.cct()); } Notifier::~Notifier() { @@ -29,7 +31,7 @@ Notifier::~Notifier() { void Notifier::flush(Context *on_finish) { Mutex::Locker aio_notify_locker(m_aio_notify_lock); if (m_pending_aio_notifies == 0) { - m_image_ctx.op_work_queue->queue(on_finish, 0); + m_work_queue->queue(on_finish, 0); return; } @@ -41,38 +43,35 @@ void Notifier::notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish) { Mutex::Locker aio_notify_locker(m_aio_notify_lock); ++m_pending_aio_notifies; - CephContext *cct = m_image_ctx.cct; - ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies - << dendl; + ldout(m_cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; } C_AioNotify *ctx = new C_AioNotify(this, on_finish); librados::AioCompletion *comp = util::create_rados_ack_callback(ctx); - int r = m_image_ctx.md_ctx.aio_notify(m_image_ctx.header_oid, comp, bl, - NOTIFY_TIMEOUT, out_bl); + int r = m_ioctx.aio_notify(m_oid, comp, bl, NOTIFY_TIMEOUT, out_bl); assert(r == 0); comp->release(); } void Notifier::handle_notify(int r, Context *on_finish) { if (on_finish != nullptr) { - m_image_ctx.op_work_queue->queue(on_finish, r); + m_work_queue->queue(on_finish, r); } Mutex::Locker aio_notify_locker(m_aio_notify_lock); assert(m_pending_aio_notifies > 0); --m_pending_aio_notifies; - CephContext *cct = m_image_ctx.cct; - ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies - << dendl; + ldout(m_cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; if (m_pending_aio_notifies == 0) { for (auto ctx : m_aio_notify_flush_ctxs) { - m_image_ctx.op_work_queue->queue(ctx, 0); + m_work_queue->queue(ctx, 0); } m_aio_notify_flush_ctxs.clear(); } } -} // namespace image_watcher +} // namespace object_watcher } // namespace librbd diff --git a/src/librbd/image_watcher/Notifier.h b/src/librbd/object_watcher/Notifier.h similarity index 68% rename from src/librbd/image_watcher/Notifier.h rename to src/librbd/object_watcher/Notifier.h index 5470c185a8db..bfe996eee74d 100644 --- a/src/librbd/image_watcher/Notifier.h +++ b/src/librbd/object_watcher/Notifier.h @@ -1,26 +1,27 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#ifndef CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H -#define CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H +#ifndef CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H +#define CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H #include "include/int_types.h" #include "include/buffer_fwd.h" #include "include/Context.h" +#include "include/rados/librados.hpp" #include "common/Mutex.h" +#include "common/WorkQueue.h" #include namespace librbd { -struct ImageCtx; - -namespace image_watcher { +namespace object_watcher { class Notifier { public: static const uint64_t NOTIFY_TIMEOUT; - Notifier(ImageCtx &image_ctx); + Notifier(ContextWQ *work_queue, librados::IoCtx &ioctx, + const std::string &oid); ~Notifier(); void flush(Context *on_finish); @@ -41,7 +42,10 @@ private: } }; - ImageCtx &m_image_ctx; + ContextWQ *m_work_queue; + librados::IoCtx m_ioctx; + CephContext *m_cct; + std::string m_oid; Mutex m_aio_notify_lock; size_t m_pending_aio_notifies = 0; @@ -51,7 +55,7 @@ private: }; -} // namespace image_watcher +} // namespace object_watcher } // namespace librbd -#endif // CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H +#endif // CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H -- 2.47.3