From: Mykola Golub Date: Sat, 1 Oct 2016 19:23:48 +0000 (+0300) Subject: rbd-mirror HA: move librbd::image_watcher::Notifier to librbd::object_watcher X-Git-Tag: v11.1.0~685^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=6dda2b9384d53b107985289a4355f8ed3c7f5b69;p=ceph.git rbd-mirror HA: move librbd::image_watcher::Notifier to librbd::object_watcher Fixes: http://tracker.ceph.com/issues/17017 Signed-off-by: Mykola Golub --- diff --git a/src/librbd/CMakeLists.txt b/src/librbd/CMakeLists.txt index 4f46c1f2bacd..68f3f461c7e1 100644 --- a/src/librbd/CMakeLists.txt +++ b/src/librbd/CMakeLists.txt @@ -41,7 +41,6 @@ set(librbd_internal_srcs image/RefreshRequest.cc image/SetFlagsRequest.cc image/SetSnapRequest.cc - image_watcher/Notifier.cc image_watcher/NotifyLockOwner.cc image_watcher/RewatchRequest.cc journal/RemoveRequest.cc @@ -62,6 +61,7 @@ set(librbd_internal_srcs object_map/SnapshotRollbackRequest.cc object_map/UnlockRequest.cc object_map/UpdateRequest.cc + object_watcher/Notifier.cc operation/DisableFeaturesRequest.cc operation/EnableFeaturesRequest.cc operation/FlattenRequest.cc diff --git a/src/librbd/ImageWatcher.cc b/src/librbd/ImageWatcher.cc index ea875736b405..d5fd6fed8466 100644 --- a/src/librbd/ImageWatcher.cc +++ b/src/librbd/ImageWatcher.cc @@ -11,7 +11,6 @@ #include "librbd/TaskFinisher.h" #include "librbd/Utils.h" #include "librbd/exclusive_lock/Policy.h" -#include "librbd/image_watcher/Notifier.h" #include "librbd/image_watcher/NotifyLockOwner.h" #include "librbd/image_watcher/RewatchRequest.h" #include "include/encoding.h" @@ -85,7 +84,7 @@ ImageWatcher::ImageWatcher(I &image_ctx) m_task_finisher(new TaskFinisher(*m_image_ctx.cct)), m_async_request_lock(util::unique_lock_name("librbd::ImageWatcher::m_async_request_lock", this)), m_owner_client_id_lock(util::unique_lock_name("librbd::ImageWatcher::m_owner_client_id_lock", this)), - m_notifier(image_ctx) + m_notifier(image_ctx.op_work_queue, image_ctx.md_ctx, image_ctx.header_oid) { } @@ -377,7 +376,7 @@ void ImageWatcher::notify_header_update(librados::IoCtx &io_ctx, // supports legacy (empty buffer) clients bufferlist bl; ::encode(NotifyMessage(HeaderUpdatePayload()), bl); - io_ctx.notify2(oid, bl, image_watcher::Notifier::NOTIFY_TIMEOUT, nullptr); + io_ctx.notify2(oid, bl, object_watcher::Notifier::NOTIFY_TIMEOUT, nullptr); } template diff --git a/src/librbd/ImageWatcher.h b/src/librbd/ImageWatcher.h index db52a600422e..cac2d9d0e346 100644 --- a/src/librbd/ImageWatcher.h +++ b/src/librbd/ImageWatcher.h @@ -8,7 +8,7 @@ #include "common/RWLock.h" #include "include/Context.h" #include "include/rbd/librbd.hpp" -#include "librbd/image_watcher/Notifier.h" +#include "librbd/object_watcher/Notifier.h" #include "librbd/WatchNotifyTypes.h" #include #include @@ -241,7 +241,7 @@ private: Mutex m_owner_client_id_lock; watch_notify::ClientId m_owner_client_id; - image_watcher::Notifier m_notifier; + object_watcher::Notifier m_notifier; void handle_register_watch(int r); diff --git a/src/librbd/image_watcher/Notifier.cc b/src/librbd/image_watcher/Notifier.cc deleted file mode 100644 index 7569f7572c1f..000000000000 --- a/src/librbd/image_watcher/Notifier.cc +++ /dev/null @@ -1,78 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab - -#include "librbd/image_watcher/Notifier.h" -#include "common/WorkQueue.h" -#include "librbd/ImageCtx.h" -#include "librbd/Utils.h" - -#define dout_subsys ceph_subsys_rbd -#undef dout_prefix -#define dout_prefix *_dout << "librbd::image_watcher::Notifier: " - -namespace librbd { -namespace image_watcher { - -const uint64_t Notifier::NOTIFY_TIMEOUT = 5000; - -Notifier::Notifier(ImageCtx &image_ctx) - : m_image_ctx(image_ctx), - m_aio_notify_lock(util::unique_lock_name( - "librbd::image_watcher::Notifier::m_aio_notify_lock", this)) { -} - -Notifier::~Notifier() { - Mutex::Locker aio_notify_locker(m_aio_notify_lock); - assert(m_pending_aio_notifies == 0); -} - -void Notifier::flush(Context *on_finish) { - Mutex::Locker aio_notify_locker(m_aio_notify_lock); - if (m_pending_aio_notifies == 0) { - m_image_ctx.op_work_queue->queue(on_finish, 0); - return; - } - - m_aio_notify_flush_ctxs.push_back(on_finish); -} - -void Notifier::notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish) { - { - Mutex::Locker aio_notify_locker(m_aio_notify_lock); - ++m_pending_aio_notifies; - - CephContext *cct = m_image_ctx.cct; - ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies - << dendl; - } - - C_AioNotify *ctx = new C_AioNotify(this, on_finish); - librados::AioCompletion *comp = util::create_rados_ack_callback(ctx); - int r = m_image_ctx.md_ctx.aio_notify(m_image_ctx.header_oid, comp, bl, - NOTIFY_TIMEOUT, out_bl); - assert(r == 0); - comp->release(); -} - -void Notifier::handle_notify(int r, Context *on_finish) { - if (on_finish != nullptr) { - m_image_ctx.op_work_queue->queue(on_finish, r); - } - - Mutex::Locker aio_notify_locker(m_aio_notify_lock); - assert(m_pending_aio_notifies > 0); - --m_pending_aio_notifies; - - CephContext *cct = m_image_ctx.cct; - ldout(cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies - << dendl; - if (m_pending_aio_notifies == 0) { - for (auto ctx : m_aio_notify_flush_ctxs) { - m_image_ctx.op_work_queue->queue(ctx, 0); - } - m_aio_notify_flush_ctxs.clear(); - } -} - -} // namespace image_watcher -} // namespace librbd diff --git a/src/librbd/image_watcher/Notifier.h b/src/librbd/image_watcher/Notifier.h deleted file mode 100644 index 5470c185a8db..000000000000 --- a/src/librbd/image_watcher/Notifier.h +++ /dev/null @@ -1,57 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab - -#ifndef CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H -#define CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H - -#include "include/int_types.h" -#include "include/buffer_fwd.h" -#include "include/Context.h" -#include "common/Mutex.h" -#include - -namespace librbd { - -struct ImageCtx; - -namespace image_watcher { - -class Notifier { -public: - static const uint64_t NOTIFY_TIMEOUT; - - Notifier(ImageCtx &image_ctx); - ~Notifier(); - - void flush(Context *on_finish); - void notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish); - -private: - typedef std::list Contexts; - - struct C_AioNotify : public Context { - Notifier *notifier; - Context *on_finish; - - C_AioNotify(Notifier *notifier, Context *on_finish) - : notifier(notifier), on_finish(on_finish) { - } - virtual void finish(int r) override { - notifier->handle_notify(r, on_finish); - } - }; - - ImageCtx &m_image_ctx; - - Mutex m_aio_notify_lock; - size_t m_pending_aio_notifies = 0; - Contexts m_aio_notify_flush_ctxs; - - void handle_notify(int r, Context *on_finish); - -}; - -} // namespace image_watcher -} // namespace librbd - -#endif // CEPH_LIBRBD_IMAGE_WATCHER_NOTIFIER_H diff --git a/src/librbd/image_watcher/NotifyLockOwner.cc b/src/librbd/image_watcher/NotifyLockOwner.cc index 09a3e43e8da9..34be88a58051 100644 --- a/src/librbd/image_watcher/NotifyLockOwner.cc +++ b/src/librbd/image_watcher/NotifyLockOwner.cc @@ -6,7 +6,7 @@ #include "librbd/ImageCtx.h" #include "librbd/Utils.h" #include "librbd/WatchNotifyTypes.h" -#include "librbd/image_watcher/Notifier.h" +#include "librbd/object_watcher/Notifier.h" #include #define dout_subsys ceph_subsys_rbd @@ -20,7 +20,8 @@ namespace image_watcher { using namespace watch_notify; using util::create_context_callback; -NotifyLockOwner::NotifyLockOwner(ImageCtx &image_ctx, Notifier ¬ifier, +NotifyLockOwner::NotifyLockOwner(ImageCtx &image_ctx, + object_watcher::Notifier ¬ifier, bufferlist &&bl, Context *on_finish) : m_image_ctx(image_ctx), m_notifier(notifier), m_bl(std::move(bl)), m_on_finish(on_finish) { diff --git a/src/librbd/image_watcher/NotifyLockOwner.h b/src/librbd/image_watcher/NotifyLockOwner.h index 0b85097b73f6..95dac6b5709e 100644 --- a/src/librbd/image_watcher/NotifyLockOwner.h +++ b/src/librbd/image_watcher/NotifyLockOwner.h @@ -12,25 +12,26 @@ namespace librbd { struct ImageCtx; -namespace image_watcher { +namespace object_watcher { class Notifier; } -class Notifier; +namespace image_watcher { class NotifyLockOwner { public: - static NotifyLockOwner *create(ImageCtx &image_ctx, Notifier ¬ifier, + static NotifyLockOwner *create(ImageCtx &image_ctx, + object_watcher::Notifier ¬ifier, bufferlist &&bl, Context *on_finish) { return new NotifyLockOwner(image_ctx, notifier, std::move(bl), on_finish); } - NotifyLockOwner(ImageCtx &image_ctx, Notifier ¬ifier, bufferlist &&bl, - Context *on_finish); + NotifyLockOwner(ImageCtx &image_ctx, object_watcher::Notifier ¬ifier, + bufferlist &&bl, Context *on_finish); void send(); private: ImageCtx &m_image_ctx; - Notifier &m_notifier; + object_watcher::Notifier &m_notifier; bufferlist m_bl; bufferlist m_out_bl; diff --git a/src/librbd/object_watcher/Notifier.cc b/src/librbd/object_watcher/Notifier.cc new file mode 100644 index 000000000000..019c786ee62c --- /dev/null +++ b/src/librbd/object_watcher/Notifier.cc @@ -0,0 +1,77 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "librbd/object_watcher/Notifier.h" +#include "common/WorkQueue.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" + +#define dout_subsys ceph_subsys_rbd +#undef dout_prefix +#define dout_prefix *_dout << "librbd::object_watcher::Notifier: " + +namespace librbd { +namespace object_watcher { + +const uint64_t Notifier::NOTIFY_TIMEOUT = 5000; + +Notifier::Notifier(ContextWQ *work_queue, IoCtx &ioctx, const std::string &oid) + : m_work_queue(work_queue), m_oid(oid), + m_aio_notify_lock(util::unique_lock_name( + "librbd::object_watcher::Notifier::m_aio_notify_lock", this)) { + m_ioctx.dup(ioctx); + m_cct = reinterpret_cast(m_ioctx.cct()); +} + +Notifier::~Notifier() { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + assert(m_pending_aio_notifies == 0); +} + +void Notifier::flush(Context *on_finish) { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + if (m_pending_aio_notifies == 0) { + m_work_queue->queue(on_finish, 0); + return; + } + + m_aio_notify_flush_ctxs.push_back(on_finish); +} + +void Notifier::notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish) { + { + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + ++m_pending_aio_notifies; + + ldout(m_cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; + } + + C_AioNotify *ctx = new C_AioNotify(this, on_finish); + librados::AioCompletion *comp = util::create_rados_ack_callback(ctx); + int r = m_ioctx.aio_notify(m_oid, comp, bl, NOTIFY_TIMEOUT, out_bl); + assert(r == 0); + comp->release(); +} + +void Notifier::handle_notify(int r, Context *on_finish) { + if (on_finish != nullptr) { + m_work_queue->queue(on_finish, r); + } + + Mutex::Locker aio_notify_locker(m_aio_notify_lock); + assert(m_pending_aio_notifies > 0); + --m_pending_aio_notifies; + + ldout(m_cct, 20) << __func__ << ": pending=" << m_pending_aio_notifies + << dendl; + if (m_pending_aio_notifies == 0) { + for (auto ctx : m_aio_notify_flush_ctxs) { + m_work_queue->queue(ctx, 0); + } + m_aio_notify_flush_ctxs.clear(); + } +} + +} // namespace object_watcher +} // namespace librbd diff --git a/src/librbd/object_watcher/Notifier.h b/src/librbd/object_watcher/Notifier.h new file mode 100644 index 000000000000..bfe996eee74d --- /dev/null +++ b/src/librbd/object_watcher/Notifier.h @@ -0,0 +1,61 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H +#define CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H + +#include "include/int_types.h" +#include "include/buffer_fwd.h" +#include "include/Context.h" +#include "include/rados/librados.hpp" +#include "common/Mutex.h" +#include "common/WorkQueue.h" +#include + +namespace librbd { + +namespace object_watcher { + +class Notifier { +public: + static const uint64_t NOTIFY_TIMEOUT; + + Notifier(ContextWQ *work_queue, librados::IoCtx &ioctx, + const std::string &oid); + ~Notifier(); + + void flush(Context *on_finish); + void notify(bufferlist &bl, bufferlist *out_bl, Context *on_finish); + +private: + typedef std::list Contexts; + + struct C_AioNotify : public Context { + Notifier *notifier; + Context *on_finish; + + C_AioNotify(Notifier *notifier, Context *on_finish) + : notifier(notifier), on_finish(on_finish) { + } + virtual void finish(int r) override { + notifier->handle_notify(r, on_finish); + } + }; + + ContextWQ *m_work_queue; + librados::IoCtx m_ioctx; + CephContext *m_cct; + std::string m_oid; + + Mutex m_aio_notify_lock; + size_t m_pending_aio_notifies = 0; + Contexts m_aio_notify_flush_ctxs; + + void handle_notify(int r, Context *on_finish); + +}; + +} // namespace object_watcher +} // namespace librbd + +#endif // CEPH_LIBRBD_OBJECT_WATCHER_NOTIFIER_H