From: Jason Dillaman Date: Wed, 15 Jan 2020 20:03:18 +0000 (-0500) Subject: rbd-mirror: periodic remote pool metadata poller X-Git-Tag: v15.1.0~175^2~5 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=e08500268fd8d103617c029253a09d953814f0f0;p=ceph.git rbd-mirror: periodic remote pool metadata poller The mirror uuid and mirror peer uuid should be periodically retrieved from the remote peer. The peer ping logic can also be moved to this helper class. Signed-off-by: Jason Dillaman --- diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 60dff58144ac1..2412108f5b1b1 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -19,6 +19,7 @@ set(rbd_mirror_internal NamespaceReplayer.cc PoolReplayer.cc PoolWatcher.cc + RemotePoolPoller.cc ServiceDaemon.cc Threads.cc Throttler.cc diff --git a/src/tools/rbd_mirror/RemotePoolPoller.cc b/src/tools/rbd_mirror/RemotePoolPoller.cc new file mode 100644 index 0000000000000..a4514f684e7d6 --- /dev/null +++ b/src/tools/rbd_mirror/RemotePoolPoller.cc @@ -0,0 +1,267 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "RemotePoolPoller.h" +#include "include/ceph_assert.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Timer.h" +#include "common/WorkQueue.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::RemotePollPoller: " << this << " " \ + << __func__ << ": " + +namespace rbd { +namespace mirror { + +static const double POLL_INTERVAL_SECONDS = 30; + +using librbd::util::create_rados_callback; + +template +RemotePoolPoller::~RemotePoolPoller() { + ceph_assert(m_timer_task == nullptr); +} + +template +void RemotePoolPoller::init(Context* on_finish) { + dout(10) << dendl; + + ceph_assert(m_state == STATE_INITIALIZING); + ceph_assert(m_on_finish == nullptr); + m_on_finish = on_finish; + + get_mirror_uuid(); +} + +template +void RemotePoolPoller::shut_down(Context* on_finish) { + dout(10) << dendl; + + std::unique_lock locker(m_threads->timer_lock); + ceph_assert(m_state == STATE_POLLING); + m_state = STATE_SHUTTING_DOWN; + + if (m_timer_task == nullptr) { + // currently executing a poll + ceph_assert(m_on_finish == nullptr); + m_on_finish = on_finish; + return; + } + + m_threads->timer->cancel_event(m_timer_task); + m_timer_task = nullptr; + m_threads->work_queue->queue(on_finish, 0); +} + +template +void RemotePoolPoller::get_mirror_uuid() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_uuid_get_start(&op); + + auto aio_comp = create_rados_callback< + RemotePoolPoller, &RemotePoolPoller::handle_get_mirror_uuid>(this); + m_out_bl.clear(); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void RemotePoolPoller::handle_get_mirror_uuid(int r) { + dout(10) << "r=" << r << dendl; + std::string remote_mirror_uuid; + if (r >= 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_uuid_get_finish(&it, &remote_mirror_uuid); + if (r >= 0 && remote_mirror_uuid.empty()) { + r = -ENOENT; + } + } + + if (r < 0) { + if (r == -ENOENT) { + dout(5) << "remote mirror uuid missing" << dendl; + } else { + derr << "failed to retrieve remote mirror uuid: " << cpp_strerror(r) + << dendl; + } + + m_remote_pool_meta.mirror_uuid = ""; + } + + // if we have the mirror uuid, we will poll until shut down + if (m_state == STATE_INITIALIZING) { + if (r < 0) { + schedule_task(r); + return; + } + + m_state = STATE_POLLING; + } + + dout(10) << "remote_mirror_uuid=" << remote_mirror_uuid << dendl; + if (m_remote_pool_meta.mirror_uuid != remote_mirror_uuid) { + m_remote_pool_meta.mirror_uuid = remote_mirror_uuid; + m_updated = true; + } + + mirror_peer_ping(); +} + +template +void RemotePoolPoller::mirror_peer_ping() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_peer_ping(&op, m_site_name, m_local_fsid); + + auto aio_comp = create_rados_callback< + RemotePoolPoller, &RemotePoolPoller::handle_mirror_peer_ping>(this); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void RemotePoolPoller::handle_mirror_peer_ping(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -EOPNOTSUPP) { + // older OSD that doesn't support snaphot-based mirroring, so no need + // to query remote peers + dout(10) << "remote peer does not support snapshot-based mirroring" + << dendl; + notify_listener(); + return; + } else if (r < 0) { + // we can still see if we can perform a peer list and find outselves + derr << "failed to ping remote mirror peer: " << cpp_strerror(r) << dendl; + } + + mirror_peer_list(); +} + +template +void RemotePoolPoller::mirror_peer_list() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_peer_list_start(&op); + + auto aio_comp = create_rados_callback< + RemotePoolPoller, &RemotePoolPoller::handle_mirror_peer_list>(this); + m_out_bl.clear(); + int r = m_remote_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void RemotePoolPoller::handle_mirror_peer_list(int r) { + dout(10) << "r=" << r << dendl; + + std::vector peers; + if (r == 0) { + auto iter = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_peer_list_finish(&iter, &peers); + } + + if (r < 0) { + derr << "failed to retrieve mirror peers: " << cpp_strerror(r) << dendl; + } + + cls::rbd::MirrorPeer* matched_peer = nullptr; + for (auto& peer : peers) { + if (peer.mirror_peer_direction == cls::rbd::MIRROR_PEER_DIRECTION_TX) { + continue; + } + + if (peer.fsid == m_local_fsid) { + matched_peer = &peer; + break; + } else if (peer.site_name == m_site_name) { + // keep searching in case we hit an exact match by fsid + matched_peer = &peer; + } + } + + // older OSDs don't support peer ping so we might fail to find a match, + // which will prevent snapshot mirroring from functioning + std::string remote_mirror_peer_uuid; + if (matched_peer != nullptr) { + remote_mirror_peer_uuid = matched_peer->uuid; + } + + dout(10) << "remote_mirror_peer_uuid=" << remote_mirror_peer_uuid << dendl; + if (m_remote_pool_meta.mirror_peer_uuid != remote_mirror_peer_uuid) { + m_remote_pool_meta.mirror_peer_uuid = remote_mirror_peer_uuid; + m_updated = true; + } + + notify_listener(); +} + +template +void RemotePoolPoller::notify_listener() { + bool updated = false; + std::swap(updated, m_updated); + if (updated) { + dout(10) << dendl; + m_listener.handle_updated(m_remote_pool_meta); + } + + schedule_task(0); +} + +template +void RemotePoolPoller::schedule_task(int r) { + std::unique_lock locker{m_threads->timer_lock}; + + if (m_state == STATE_POLLING) { + dout(10) << dendl; + + ceph_assert(m_timer_task == nullptr); + m_timer_task = new LambdaContext([this](int) { + handle_task(); + }); + + m_threads->timer->add_event_after(POLL_INTERVAL_SECONDS, m_timer_task); + } + + // finish init or shut down callback + if (m_on_finish != nullptr) { + locker.unlock(); + Context* on_finish = nullptr; + std::swap(on_finish, m_on_finish); + on_finish->complete(m_state == STATE_SHUTTING_DOWN ? 0 : r); + } +} + +template +void RemotePoolPoller::handle_task() { + dout(10) << dendl; + + ceph_assert(ceph_mutex_is_locked_by_me(m_threads->timer_lock)); + m_timer_task = nullptr; + + auto ctx = new LambdaContext([this](int) { + get_mirror_uuid(); + }); + m_threads->work_queue->queue(ctx); +} + +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::RemotePoolPoller; diff --git a/src/tools/rbd_mirror/RemotePoolPoller.h b/src/tools/rbd_mirror/RemotePoolPoller.h new file mode 100644 index 0000000000000..a0f77ff90a06c --- /dev/null +++ b/src/tools/rbd_mirror/RemotePoolPoller.h @@ -0,0 +1,133 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_REMOTE_POOL_POLLER_H +#define CEPH_RBD_MIRROR_REMOTE_POOL_POLLER_H + +#include "include/rados/librados.hpp" +#include "tools/rbd_mirror/Types.h" +#include + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +template struct Threads; + +namespace remote_pool_poller { + +struct Listener { + virtual ~Listener() {} + + virtual void handle_updated(const RemotePoolMeta& remote_pool_meta) = 0; +}; + +}; // namespace remote_pool_poller + +template +class RemotePoolPoller { +public: + static RemotePoolPoller* create( + Threads* threads, + librados::IoCtx& remote_io_ctx, + const std::string& site_name, + const std::string& local_fsid, + remote_pool_poller::Listener& listener) { + return new RemotePoolPoller(threads, remote_io_ctx, site_name, local_fsid, + listener); + } + + RemotePoolPoller( + Threads* threads, + librados::IoCtx& remote_io_ctx, + const std::string& site_name, + const std::string& local_fsid, + remote_pool_poller::Listener& listener) + : m_threads(threads), + m_remote_io_ctx(remote_io_ctx), + m_site_name(site_name), + m_local_fsid(local_fsid), + m_listener(listener) { + } + ~RemotePoolPoller(); + + void init(Context* on_finish); + void shut_down(Context* on_finish); + +private: + /** + * @verbatim + * + * + * | + * |/----------------------------\ + * | | + * v | + * MIRROR_UUID_GET | + * | | + * v | + * MIRROR_PEER_PING | + * | | + * v | + * MIRROR_PEER_LIST | + * | | + * v | + * MIRROR_UUID_GET | + * | | + * v (skip if no changes) | + * NOTIFY_LISTENER | + * | | + * | (repeat periodically) | + * |\----------------------------/ + * | + * v + * + * + * @endverbatim + */ + + enum State { + STATE_INITIALIZING, + STATE_POLLING, + STATE_SHUTTING_DOWN + }; + + Threads* m_threads; + librados::IoCtx& m_remote_io_ctx; + std::string m_site_name; + std::string m_local_fsid; + remote_pool_poller::Listener& m_listener; + + bufferlist m_out_bl; + + RemotePoolMeta m_remote_pool_meta; + bool m_updated = false; + + State m_state = STATE_INITIALIZING; + Context* m_timer_task = nullptr; + Context* m_on_finish = nullptr; + + void get_mirror_uuid(); + void handle_get_mirror_uuid(int r); + + void mirror_peer_ping(); + void handle_mirror_peer_ping(int r); + + void mirror_peer_list(); + void handle_mirror_peer_list(int r); + + void notify_listener(); + + void schedule_task(int r); + void handle_task(); + +}; + +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::RemotePoolPoller; + +#endif // CEPH_RBD_MIRROR_REMOTE_POOL_POLLER_H diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index ab3146eb420e1..09f18d7ac6f7a 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -53,6 +53,18 @@ std::ostream &operator<<(std::ostream &, const ImageId &image_id); typedef std::set ImageIds; +struct RemotePoolMeta { + RemotePoolMeta() {} + RemotePoolMeta(const std::string& remote_mirror_uuid, + const std::string& remote_mirror_peer_uuid) + : mirror_uuid(remote_mirror_uuid), + mirror_peer_uuid(remote_mirror_peer_uuid) { + } + + std::string mirror_uuid; + std::string mirror_peer_uuid; +}; + template struct Peer { std::string peer_uuid;