From ec6c26f7456579e5772bc8c461f1b6f3e901f033 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 15 Jan 2020 15:09:05 -0500 Subject: [PATCH] rbd-mirror: pool replayer should instantiate the remote pool poller Let the poller pull the metadata from the remote before advancing to the leader watcher initialization. If the remote metadata changes during runtime, stop the pool replayer so that it can be re-initialized. Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 70 +++++++++++++++++++ src/tools/rbd_mirror/PoolReplayer.cc | 66 +++++++++++++++++ src/tools/rbd_mirror/PoolReplayer.h | 12 ++++ src/tools/rbd_mirror/Types.cc | 6 ++ src/tools/rbd_mirror/Types.h | 3 + 5 files changed, 157 insertions(+) diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 756d3906490..040ce633a6a 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -14,6 +14,7 @@ #include "tools/rbd_mirror/LeaderWatcher.h" #include "tools/rbd_mirror/NamespaceReplayer.h" #include "tools/rbd_mirror/PoolReplayer.h" +#include "tools/rbd_mirror/RemotePoolPoller.h" #include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/Threads.h" #include "common/Formatter.h" @@ -200,6 +201,33 @@ struct LeaderWatcher { LeaderWatcher* LeaderWatcher::s_instance = nullptr; +template<> +struct RemotePoolPoller { + static RemotePoolPoller* s_instance; + + remote_pool_poller::Listener* listener = nullptr; + + static RemotePoolPoller* create( + Threads* threads, + librados::IoCtx& remote_io_ctx, + const std::string& local_site_name, + const std::string& local_fsid, + remote_pool_poller::Listener& listener) { + ceph_assert(s_instance != nullptr); + s_instance->listener = &listener; + return s_instance; + } + + MOCK_METHOD1(init, void(Context*)); + MOCK_METHOD1(shut_down, void(Context*)); + + RemotePoolPoller() { + s_instance = this; + } +}; + +RemotePoolPoller* RemotePoolPoller::s_instance = nullptr; + template<> struct ServiceDaemon { MOCK_METHOD2(add_namespace, void(int64_t, const std::string &)); @@ -260,6 +288,7 @@ public: typedef PoolReplayer MockPoolReplayer; typedef Throttler MockThrottler; typedef NamespaceReplayer MockNamespaceReplayer; + typedef RemotePoolPoller MockRemotePoolPoller; typedef LeaderWatcher MockLeaderWatcher; typedef ServiceDaemon MockServiceDaemon; typedef Threads MockThreads; @@ -353,6 +382,31 @@ public: .Times(AtLeast(0)); } + void expect_remote_pool_poller_init( + MockRemotePoolPoller& mock_remote_pool_poller, + const RemotePoolMeta& remote_pool_meta, int r) { + EXPECT_CALL(mock_remote_pool_poller, init(_)) + .WillOnce(Invoke( + [this, &mock_remote_pool_poller, remote_pool_meta, r] + (Context* ctx) { + if (r >= 0) { + mock_remote_pool_poller.listener->handle_updated( + remote_pool_meta); + } + + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_remote_pool_poller_shut_down( + MockRemotePoolPoller& mock_remote_pool_poller, int r) { + EXPECT_CALL(mock_remote_pool_poller, shut_down(_)) + .WillOnce(Invoke( + [this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + void expect_namespace_replayer_is_blacklisted( MockNamespaceReplayer &mock_namespace_replayer, bool blacklisted) { @@ -504,6 +558,9 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); @@ -523,6 +580,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { expect_leader_watcher_shut_down(*mock_leader_watcher); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); } @@ -560,6 +618,9 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); @@ -592,6 +653,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { expect_leader_watcher_shut_down(*mock_leader_watcher); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); } @@ -642,6 +704,9 @@ TEST_F(TestMockPoolReplayer, Namespaces) { nullptr); expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); @@ -707,6 +772,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer); expect_leader_watcher_shut_down(*mock_leader_watcher); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); } @@ -754,6 +820,9 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { nullptr); expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx); expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0); + auto mock_remote_pool_poller = new MockRemotePoolPoller(); + expect_remote_pool_poller_init(*mock_remote_pool_poller, + {"remote mirror uuid", ""}, 0); expect_namespace_replayer_init(*mock_default_namespace_replayer, 0); expect_leader_watcher_init(*mock_leader_watcher, 0); @@ -837,6 +906,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { expect_leader_watcher_shut_down(*mock_leader_watcher); expect_namespace_replayer_shut_down(*mock_default_namespace_replayer); + expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0); pool_replayer.shut_down(); } diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index b99070bd5b9..55a52280092 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -15,6 +15,7 @@ #include "global/global_context.h" #include "librbd/api/Config.h" #include "librbd/api/Namespace.h" +#include "RemotePoolPoller.h" #include "ServiceDaemon.h" #include "Threads.h" @@ -205,6 +206,21 @@ private: } // anonymous namespace +template +struct PoolReplayer::RemotePoolPollerListener + : public remote_pool_poller::Listener { + + PoolReplayer* m_pool_replayer; + + RemotePoolPollerListener(PoolReplayer* pool_replayer) + : m_pool_replayer(pool_replayer) { + } + + void handle_updated(const RemotePoolMeta& remote_pool_meta) override { + m_pool_replayer->handle_remote_pool_meta_updated(remote_pool_meta); + } +}; + template PoolReplayer::PoolReplayer( Threads *threads, ServiceDaemon *service_daemon, @@ -319,6 +335,32 @@ void PoolReplayer::init(const std::string& site_name) { m_image_deletion_throttler.reset( Throttler::create(cct, "rbd_mirror_concurrent_image_deletions")); + std::string local_fsid; + r = m_local_rados->cluster_fsid(&local_fsid); + if (r < 0) { + derr << "failed to retrieve local fsid: " << cpp_strerror(r) << dendl; + return; + } + + m_remote_pool_poller_listener.reset(new RemotePoolPollerListener(this)); + m_remote_pool_poller.reset(RemotePoolPoller::create( + m_threads, m_remote_io_ctx, m_site_name, local_fsid, + *m_remote_pool_poller_listener)); + + C_SaferCond on_pool_poller_init; + m_remote_pool_poller->init(&on_pool_poller_init); + r = on_pool_poller_init.wait(); + if (r < 0) { + derr << "failed to initialize remote pool poller: " << cpp_strerror(r) + << dendl; + m_callout_id = m_service_daemon->add_or_update_callout( + m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR, + "unable to initialize remote pool poller"); + m_remote_pool_poller.reset(); + return; + } + ceph_assert(!m_remote_pool_meta.mirror_uuid.empty()); + m_default_namespace_replayer.reset(NamespaceReplayer::create( "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, m_site_name, m_threads, m_image_sync_throttler.get(), @@ -385,6 +427,14 @@ void PoolReplayer::shut_down() { } m_default_namespace_replayer.reset(); + if (m_remote_pool_poller) { + C_SaferCond ctx; + m_remote_pool_poller->shut_down(&ctx); + ctx.wait(); + } + m_remote_pool_poller.reset(); + m_remote_pool_poller_listener.reset(); + m_image_sync_throttler.reset(); m_image_deletion_throttler.reset(); @@ -1024,6 +1074,22 @@ void PoolReplayer::handle_instances_removed( } } +template +void PoolReplayer::handle_remote_pool_meta_updated( + const RemotePoolMeta& remote_pool_meta) { + dout(5) << "remote_pool_meta=" << remote_pool_meta << dendl; + + if (!m_default_namespace_replayer) { + m_remote_pool_meta = remote_pool_meta; + return; + } + + derr << "remote pool metadata updated unexpectedly" << dendl; + std::unique_lock locker{m_lock}; + m_stopping = true; + m_cond.notify_all(); +} + } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 9f295f4788c..1948eea6034 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -31,9 +31,13 @@ namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { +template class RemotePoolPoller; +namespace remote_pool_poller { struct Listener; } + template class ServiceDaemon; template struct Threads; + /** * Controls mirroring for a single remote cluster. */ @@ -91,6 +95,8 @@ private: * @endverbatim */ + struct RemotePoolPollerListener; + int init_rados(const std::string &cluster_name, const std::string &client_name, const std::string &mon_host, @@ -182,6 +188,8 @@ private: m_threads->work_queue->queue(on_lock); } + void handle_remote_pool_meta_updated(const RemotePoolMeta& remote_pool_meta); + Threads *m_threads; ServiceDaemon *m_service_daemon; journal::CacheManagerHandler *m_cache_manager_handler; @@ -204,6 +212,10 @@ private: std::string m_local_mirror_uuid; + RemotePoolMeta m_remote_pool_meta; + std::unique_ptr m_remote_pool_poller_listener; + std::unique_ptr> m_remote_pool_poller; + std::unique_ptr> m_default_namespace_replayer; std::map *> m_namespace_replayers; diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc index 74fe318ebca..5377e7aa192 100644 --- a/src/tools/rbd_mirror/Types.cc +++ b/src/tools/rbd_mirror/Types.cc @@ -11,6 +11,12 @@ std::ostream &operator<<(std::ostream &os, const ImageId &image_id) { << "id=" << image_id.id; } +std::ostream& operator<<(std::ostream& lhs, + const RemotePoolMeta& rhs) { + return lhs << "mirror_uuid=" << rhs.mirror_uuid << ", " + "mirror_pool_uuid=" << rhs.mirror_peer_uuid; +} + std::ostream& operator<<(std::ostream& lhs, const PeerSpec &peer) { return lhs << "uuid: " << peer.uuid << " cluster: " << peer.cluster_name diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index 09f18d7ac6f..63e19586417 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -65,6 +65,9 @@ struct RemotePoolMeta { std::string mirror_peer_uuid; }; +std::ostream& operator<<(std::ostream& lhs, + const RemotePoolMeta& remote_pool_meta); + template struct Peer { std::string peer_uuid; -- 2.39.5