From 3d3714b590ac26136ec7e7ec9264068219483e83 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Tue, 28 Jan 2020 15:24:09 -0500 Subject: [PATCH] rbd-mirror: cache local and remote pool meta properties for all pools When replicating cloned images across pools, the replayer will need access to the meta from the parent image pool. By caching the data, it avoids the need to have the image replayer query the parent pools. Signed-off-by: Jason Dillaman --- .../test_mock_BootstrapRequest.cc | 2 +- src/test/rbd_mirror/test_ImageReplayer.cc | 5 +- .../rbd_mirror/test_mock_ImageReplayer.cc | 4 +- .../rbd_mirror/test_mock_InstanceReplayer.cc | 12 ++- .../rbd_mirror/test_mock_NamespaceReplayer.cc | 15 ++-- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 10 ++- src/tools/rbd_mirror/CMakeLists.txt | 1 + src/tools/rbd_mirror/ImageReplayer.cc | 7 +- src/tools/rbd_mirror/ImageReplayer.h | 10 ++- src/tools/rbd_mirror/InstanceReplayer.cc | 6 +- src/tools/rbd_mirror/InstanceReplayer.h | 10 ++- src/tools/rbd_mirror/Mirror.cc | 5 +- src/tools/rbd_mirror/Mirror.h | 2 + src/tools/rbd_mirror/NamespaceReplayer.cc | 13 ++- src/tools/rbd_mirror/NamespaceReplayer.h | 11 ++- src/tools/rbd_mirror/PoolMetaCache.cc | 83 +++++++++++++++++++ src/tools/rbd_mirror/PoolMetaCache.h | 49 +++++++++++ src/tools/rbd_mirror/PoolReplayer.cc | 16 +++- src/tools/rbd_mirror/PoolReplayer.h | 3 + src/tools/rbd_mirror/Types.cc | 5 ++ src/tools/rbd_mirror/Types.h | 12 +++ .../image_replayer/BootstrapRequest.cc | 2 + .../image_replayer/BootstrapRequest.h | 8 +- 23 files changed, 253 insertions(+), 38 deletions(-) create mode 100644 src/tools/rbd_mirror/PoolMetaCache.cc create mode 100644 src/tools/rbd_mirror/PoolMetaCache.h diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc index 6928b426f0c..e0dbb1c47df 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc @@ -539,7 +539,7 @@ public: local_mirror_uuid, {"remote mirror uuid", "remote mirror peer uuid"}, - nullptr, nullptr, + nullptr, nullptr, nullptr, &m_mock_state_builder, &m_do_resync, on_finish); } diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index 0a2456566be..fc7a3f86e6e 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -37,6 +37,7 @@ #include "tools/rbd_mirror/ImageReplayer.h" #include "tools/rbd_mirror/InstanceWatcher.h" #include "tools/rbd_mirror/MirrorStatusUpdater.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/Throttler.h" #include "tools/rbd_mirror/Types.h" @@ -161,7 +162,7 @@ public: m_replayer = new ImageReplayerT(m_local_ioctx, m_local_mirror_uuid, m_global_image_id, m_threads.get(), m_instance_watcher, m_local_status_updater, - nullptr); + nullptr, &m_pool_meta_cache); m_replayer->add_peer({"peer uuid", m_remote_ioctx, {}, nullptr}); } @@ -391,6 +392,8 @@ public: static int _image_number; + rbd::mirror::PoolMetaCache m_pool_meta_cache{g_ceph_context}; + std::shared_ptr m_local_cluster; std::unique_ptr> m_threads; std::unique_ptr> m_image_sync_throttler; diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index f2612d7f94c..f09375f217d 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -107,6 +107,7 @@ struct BootstrapRequest { const std::string &local_mirror_uuid, const RemotePoolMeta& remote_pool_meta, ::journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache, rbd::mirror::ProgressContext *progress_ctx, StateBuilder** state_builder, bool *do_resync, Context *on_finish) { @@ -354,7 +355,8 @@ public: void create_image_replayer(MockThreads &mock_threads) { m_image_replayer = new MockImageReplayer( m_local_io_ctx, "local_mirror_uuid", "global image id", - &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr); + &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr, + nullptr); m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx, {"remote mirror uuid", "remote mirror peer uuid"}, diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc index 002786108bd..574e3d5507a 100644 --- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc @@ -70,7 +70,8 @@ struct ImageReplayer { Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { ceph_assert(s_instance != nullptr); s_instance->global_image_id = global_image_id; return s_instance; @@ -179,7 +180,8 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) { MockImageReplayer mock_image_replayer; MockInstanceReplayer instance_replayer( m_local_io_ctx, "local_mirror_uuid", - &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr); + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); std::string global_image_id("global_image_id"); EXPECT_CALL(mock_image_replayer, get_global_image_id()) @@ -248,7 +250,8 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) { MockImageReplayer mock_image_replayer; MockInstanceReplayer instance_replayer( m_local_io_ctx, "local_mirror_uuid", - &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr); + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); std::string global_image_id("global_image_id"); EXPECT_CALL(mock_image_replayer, get_global_image_id()) @@ -321,7 +324,8 @@ TEST_F(TestMockInstanceReplayer, Reacquire) { MockImageReplayer mock_image_replayer; MockInstanceReplayer instance_replayer( m_local_io_ctx, "local_mirror_uuid", - &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr); + &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr, + nullptr); std::string global_image_id("global_image_id"); EXPECT_CALL(mock_image_replayer, get_global_image_id()) diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index 53729598d7b..d31b30c07d1 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -99,7 +99,8 @@ struct InstanceReplayer { Threads *threads, ServiceDaemon *service_daemon, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { ceph_assert(s_instance != nullptr); return s_instance; } @@ -400,7 +401,7 @@ TEST_F(TestMockNamespaceReplayer, Init_LocalMirrorStatusUpdaterError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -422,7 +423,7 @@ TEST_F(TestMockNamespaceReplayer, Init_RemoteMirrorStatusUpdaterError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -447,7 +448,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -477,7 +478,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, nullptr, nullptr); + nullptr, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -504,7 +505,7 @@ TEST_F(TestMockNamespaceReplayer, Init) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, &mock_service_daemon, nullptr); + nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -543,7 +544,7 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads, - nullptr, nullptr, &mock_service_daemon, nullptr); + nullptr, nullptr, &mock_service_daemon, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index 2c46c1af5f9..6a29056e307 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -13,6 +13,7 @@ #include "tools/rbd_mirror/Throttler.h" #include "tools/rbd_mirror/LeaderWatcher.h" #include "tools/rbd_mirror/NamespaceReplayer.h" +#include "tools/rbd_mirror/PoolMetaCache.h" #include "tools/rbd_mirror/PoolReplayer.h" #include "tools/rbd_mirror/RemotePoolPoller.h" #include "tools/rbd_mirror/ServiceDaemon.h" @@ -138,7 +139,8 @@ struct NamespaceReplayer { Throttler *image_sync_throttler, Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { ceph_assert(s_instances.count(name)); auto namespace_replayer = s_instances[name]; s_instances.erase(name); @@ -523,6 +525,8 @@ public: expect_service_daemon_add_or_update_attribute( mock_service_daemon, "instance_id", {instance_id}); } + + PoolMetaCache m_pool_meta_cache{g_ceph_context}; }; TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { @@ -570,6 +574,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) { mock_service_daemon, instance_id); MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -630,6 +635,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) { mock_service_daemon, instance_id); MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -716,6 +722,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) { mock_service_daemon, instance_id); MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); @@ -832,6 +839,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) { mock_service_daemon, instance_id); MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr, + &m_pool_meta_cache, m_local_io_ctx.get_id(), peer_spec, {}); pool_replayer.init("siteA"); diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 3c35e85a76e..9211718d12e 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -17,6 +17,7 @@ set(rbd_mirror_internal MirrorStatusUpdater.cc MirrorStatusWatcher.cc NamespaceReplayer.cc + PoolMetaCache.cc PoolReplayer.cc PoolWatcher.cc RemotePoolPoller.cc diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index dfd895491a6..a6c4cb6575e 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -217,12 +217,14 @@ ImageReplayer::ImageReplayer( const std::string &global_image_id, Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) : + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) : m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid), m_global_image_id(global_image_id), m_threads(threads), m_instance_watcher(instance_watcher), m_local_status_updater(local_status_updater), m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), m_local_image_name(global_image_id), m_lock(ceph::make_mutex("rbd::mirror::ImageReplayer " + stringify(local_io_ctx.get_id()) + " " + global_image_id)), @@ -352,7 +354,8 @@ void ImageReplayer::bootstrap() { m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_instance_watcher, m_global_image_id, m_local_mirror_uuid, m_remote_image_peer.remote_pool_meta, m_cache_manager_handler, - &m_progress_cxt, &m_state_builder, &m_resync_requested, ctx); + m_pool_meta_cache, &m_progress_cxt, &m_state_builder, &m_resync_requested, + ctx); request->get(); m_bootstrap_request = request; diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index bff9e2af9e1..c9718e06b9c 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -24,6 +24,7 @@ namespace mirror { template struct InstanceWatcher; template struct MirrorStatusUpdater; +struct PoolMetaCache; template struct Threads; namespace image_replayer { @@ -45,10 +46,11 @@ public: const std::string &global_image_id, Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { return new ImageReplayer(local_io_ctx, local_mirror_uuid, global_image_id, threads, instance_watcher, local_status_updater, - cache_manager_handler); + cache_manager_handler, pool_meta_cache); } void destroy() { delete this; @@ -60,7 +62,8 @@ public: Threads *threads, InstanceWatcher *instance_watcher, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler); + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache); virtual ~ImageReplayer(); ImageReplayer(const ImageReplayer&) = delete; ImageReplayer& operator=(const ImageReplayer&) = delete; @@ -181,6 +184,7 @@ private: InstanceWatcher *m_instance_watcher; MirrorStatusUpdater* m_local_status_updater; journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; Peers m_peers; Peer m_remote_image_peer; diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc index 21ff8da7257..2b9b47a0e01 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.cc +++ b/src/tools/rbd_mirror/InstanceReplayer.cc @@ -38,11 +38,13 @@ InstanceReplayer::InstanceReplayer( librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, Threads *threads, ServiceDaemon* service_daemon, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) : m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid), m_threads(threads), m_service_daemon(service_daemon), m_local_status_updater(local_status_updater), m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), m_lock(ceph::make_mutex("rbd::mirror::InstanceReplayer " + stringify(local_io_ctx.get_id()))) { } @@ -148,7 +150,7 @@ void InstanceReplayer::acquire_image(InstanceWatcher *instance_watcher, auto image_replayer = ImageReplayer::create( m_local_io_ctx, m_local_mirror_uuid, global_image_id, m_threads, instance_watcher, m_local_status_updater, - m_cache_manager_handler); + m_cache_manager_handler, m_pool_meta_cache); dout(10) << global_image_id << ": creating replayer " << image_replayer << dendl; diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h index 2bd8d11f59d..3119fd3ebda 100644 --- a/src/tools/rbd_mirror/InstanceReplayer.h +++ b/src/tools/rbd_mirror/InstanceReplayer.h @@ -22,6 +22,7 @@ namespace mirror { template class ImageReplayer; template class InstanceWatcher; template class MirrorStatusUpdater; +struct PoolMetaCache; template class ServiceDaemon; template struct Threads; @@ -32,10 +33,11 @@ public: librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid, Threads *threads, ServiceDaemon *service_daemon, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { return new InstanceReplayer(local_io_ctx, local_mirror_uuid, threads, service_daemon, local_status_updater, - cache_manager_handler); + cache_manager_handler, pool_meta_cache); } void destroy() { delete this; @@ -46,7 +48,8 @@ public: Threads *threads, ServiceDaemon *service_daemon, MirrorStatusUpdater* local_status_updater, - journal::CacheManagerHandler *cache_manager_handler); + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache); ~InstanceReplayer(); int init(); @@ -97,6 +100,7 @@ private: ServiceDaemon *m_service_daemon; MirrorStatusUpdater* m_local_status_updater; journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; ceph::mutex m_lock; AsyncOpTracker m_async_op_tracker; diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc index 787c6980ecf..e24a78ae805 100644 --- a/src/tools/rbd_mirror/Mirror.cc +++ b/src/tools/rbd_mirror/Mirror.cc @@ -14,6 +14,7 @@ #include "librbd/ImageCtx.h" #include "perfglue/heap_profiler.h" #include "Mirror.h" +#include "PoolMetaCache.h" #include "ServiceDaemon.h" #include "Threads.h" @@ -485,6 +486,7 @@ Mirror::Mirror(CephContext *cct, const std::vector &args) : m_args(args), m_local(new librados::Rados()), m_cache_manager_handler(new CacheManagerHandler(cct)), + m_pool_meta_cache(new PoolMetaCache(cct)), m_asok_hook(new MirrorAdminSocketHook(cct, this)) { m_threads = @@ -727,7 +729,8 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers, dout(20) << "starting pool replayer for " << peer << dendl; unique_ptr> pool_replayer( new PoolReplayer<>(m_threads, m_service_daemon.get(), - m_cache_manager_handler.get(), kv.first, peer, + m_cache_manager_handler.get(), + m_pool_meta_cache.get(), kv.first, peer, m_args)); // TODO: make async diff --git a/src/tools/rbd_mirror/Mirror.h b/src/tools/rbd_mirror/Mirror.h index 28f71c54dc2..f92a63b6877 100644 --- a/src/tools/rbd_mirror/Mirror.h +++ b/src/tools/rbd_mirror/Mirror.h @@ -28,6 +28,7 @@ template struct ServiceDaemon; template struct Threads; class CacheManagerHandler; class MirrorAdminSocketHook; +class PoolMetaCache; /** * Contains the main loop and overall state for rbd-mirror. @@ -74,6 +75,7 @@ private: // monitor local cluster for config changes in peers std::unique_ptr m_local_cluster_watcher; std::unique_ptr m_cache_manager_handler; + std::unique_ptr m_pool_meta_cache; std::map>> m_pool_replayers; std::atomic m_stopping = { false }; bool m_manual_stop = false; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 3d86ff1fe92..5cc7d1b2f09 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -40,10 +40,13 @@ NamespaceReplayer::NamespaceReplayer( librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, const std::string &local_mirror_uuid, const std::string& local_mirror_peer_uuid, - const RemotePoolMeta& remote_pool_meta, Threads *threads, - Throttler *image_sync_throttler, Throttler *image_deletion_throttler, + const RemotePoolMeta& remote_pool_meta, + Threads *threads, + Throttler *image_sync_throttler, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, - journal::CacheManagerHandler *cache_manager_handler) : + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) : m_namespace_name(name), m_local_mirror_uuid(local_mirror_uuid), m_local_mirror_peer_uuid(local_mirror_peer_uuid), @@ -52,6 +55,7 @@ NamespaceReplayer::NamespaceReplayer( m_image_deletion_throttler(image_deletion_throttler), m_service_daemon(service_daemon), m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), m_lock(ceph::make_mutex(librbd::util::unique_lock_name( "rbd::mirror::NamespaceReplayer " + name, this))), m_local_pool_watcher_listener(this, true), @@ -345,7 +349,8 @@ void NamespaceReplayer::init_instance_replayer() { m_instance_replayer.reset(InstanceReplayer::create( m_local_io_ctx, m_local_mirror_uuid, m_threads, m_service_daemon, - m_local_status_updater.get(), m_cache_manager_handler)); + m_local_status_updater.get(), m_cache_manager_handler, + m_pool_meta_cache)); auto ctx = create_context_callback, &NamespaceReplayer::handle_init_instance_replayer>(this); diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h index 0d5e0a95c27..4fbf4c53bec 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.h +++ b/src/tools/rbd_mirror/NamespaceReplayer.h @@ -32,6 +32,7 @@ namespace librbd { class ImageCtx; } namespace rbd { namespace mirror { +struct PoolMetaCache; template class ServiceDaemon; template class Throttler; template struct Threads; @@ -53,12 +54,14 @@ public: Throttler *image_sync_throttler, Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, - journal::CacheManagerHandler *cache_manager_handler) { + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache) { return new NamespaceReplayer(name, local_ioctx, remote_ioctx, local_mirror_uuid, local_mirror_peer_uuid, remote_pool_meta, threads, image_sync_throttler, image_deletion_throttler, - service_daemon, cache_manager_handler); + service_daemon, cache_manager_handler, + pool_meta_cache); } NamespaceReplayer(const std::string &name, @@ -71,7 +74,8 @@ public: Throttler *image_sync_throttler, Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, - journal::CacheManagerHandler *cache_manager_handler); + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache); NamespaceReplayer(const NamespaceReplayer&) = delete; NamespaceReplayer& operator=(const NamespaceReplayer&) = delete; @@ -271,6 +275,7 @@ private: Throttler *m_image_deletion_throttler; ServiceDaemon *m_service_daemon; journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; mutable ceph::mutex m_lock; diff --git a/src/tools/rbd_mirror/PoolMetaCache.cc b/src/tools/rbd_mirror/PoolMetaCache.cc new file mode 100644 index 00000000000..261802a55c5 --- /dev/null +++ b/src/tools/rbd_mirror/PoolMetaCache.cc @@ -0,0 +1,83 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "common/debug.h" +#include "common/dout.h" +#include "tools/rbd_mirror/PoolMetaCache.h" +#include + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::PoolMetaCache: " \ + << this << " " << __func__ << ": " + +namespace rbd { +namespace mirror { + +int PoolMetaCache::get_local_pool_meta( + int64_t pool_id, + LocalPoolMeta* local_pool_meta) const { + dout(15) << "pool_id=" << pool_id << dendl; + + std::shared_lock locker{m_lock}; + auto it = m_local_pool_metas.find(pool_id); + if (it == m_local_pool_metas.end()) { + return -ENOENT; + } + + *local_pool_meta = it->second; + return 0; +} + +void PoolMetaCache::set_local_pool_meta( + int64_t pool_id, + const LocalPoolMeta& local_pool_meta) { + dout(15) << "pool_id=" << pool_id << ", " + << "local_pool_meta=" << local_pool_meta << dendl; + + std::unique_lock locker(m_lock); + m_local_pool_metas[pool_id] = local_pool_meta; +} + +void PoolMetaCache::remove_local_pool_meta(int64_t pool_id) { + dout(15) << "pool_id=" << pool_id << dendl; + + std::unique_lock locker(m_lock); + m_local_pool_metas.erase(pool_id); +} + +int PoolMetaCache::get_remote_pool_meta( + int64_t pool_id, + RemotePoolMeta* remote_pool_meta) const { + dout(15) << "pool_id=" << pool_id << dendl; + + std::shared_lock locker{m_lock}; + auto it = m_remote_pool_metas.find(pool_id); + if (it == m_remote_pool_metas.end()) { + return -ENOENT; + } + + *remote_pool_meta = it->second; + return 0; +} + +void PoolMetaCache::set_remote_pool_meta( + int64_t pool_id, + const RemotePoolMeta& remote_pool_meta) { + dout(15) << "pool_id=" << pool_id << ", " + << "remote_pool_meta=" << remote_pool_meta << dendl; + + std::unique_lock locker(m_lock); + m_remote_pool_metas[pool_id] = remote_pool_meta; +} + +void PoolMetaCache::remove_remote_pool_meta(int64_t pool_id) { + dout(15) << "pool_id=" << pool_id << dendl; + + std::unique_lock locker(m_lock); + m_remote_pool_metas.erase(pool_id); +} + +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/PoolMetaCache.h b/src/tools/rbd_mirror/PoolMetaCache.h new file mode 100644 index 00000000000..cef4a689a7f --- /dev/null +++ b/src/tools/rbd_mirror/PoolMetaCache.h @@ -0,0 +1,49 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_POOL_META_CACHE_H +#define CEPH_RBD_MIRROR_POOL_META_CACHE_H + +#include "include/int_types.h" +#include "common/ceph_mutex.h" +#include "tools/rbd_mirror/Types.h" +#include + +struct CephContext; + +namespace rbd { +namespace mirror { + +class PoolMetaCache { +public: + PoolMetaCache(CephContext* cct) + : m_cct(cct) { + } + PoolMetaCache(const PoolMetaCache&) = delete; + PoolMetaCache& operator=(const PoolMetaCache&) = delete; + + int get_local_pool_meta(int64_t pool_id, + LocalPoolMeta* local_pool_meta) const; + void set_local_pool_meta(int64_t pool_id, + const LocalPoolMeta& local_pool_meta); + void remove_local_pool_meta(int64_t pool_id); + + int get_remote_pool_meta(int64_t pool_id, + RemotePoolMeta* remote_pool_meta) const; + void set_remote_pool_meta(int64_t pool_id, + const RemotePoolMeta& remote_pool_meta); + void remove_remote_pool_meta(int64_t pool_id); + +private: + CephContext* m_cct; + + mutable ceph::shared_mutex m_lock = + ceph::make_shared_mutex("rbd::mirror::PoolMetaCache::m_lock"); + std::map m_local_pool_metas; + std::map m_remote_pool_metas; +}; + +} // namespace mirror +} // namespace rbd + +#endif // CEPH_RBD_MIRROR_POOL_META_CACHE_H diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 11623e2a3c4..416ca4ba9f7 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -15,6 +15,7 @@ #include "global/global_context.h" #include "librbd/api/Config.h" #include "librbd/api/Namespace.h" +#include "PoolMetaCache.h" #include "RemotePoolPoller.h" #include "ServiceDaemon.h" #include "Threads.h" @@ -224,11 +225,13 @@ struct PoolReplayer::RemotePoolPollerListener template PoolReplayer::PoolReplayer( Threads *threads, ServiceDaemon *service_daemon, - journal::CacheManagerHandler *cache_manager_handler, int64_t local_pool_id, + journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache, int64_t local_pool_id, const PeerSpec &peer, const std::vector &args) : m_threads(threads), m_service_daemon(service_daemon), m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), m_local_pool_id(local_pool_id), m_peer(peer), m_args(args), @@ -360,12 +363,16 @@ void PoolReplayer::init(const std::string& site_name) { return; } ceph_assert(!m_remote_pool_meta.mirror_uuid.empty()); + m_pool_meta_cache->set_remote_pool_meta( + m_remote_io_ctx.get_id(), m_remote_pool_meta); + m_pool_meta_cache->set_local_pool_meta( + m_local_io_ctx.get_id(), {m_local_mirror_uuid}); m_default_namespace_replayer.reset(NamespaceReplayer::create( "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, m_remote_pool_meta, m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), m_service_daemon, - m_cache_manager_handler)); + m_cache_manager_handler, m_pool_meta_cache)); C_SaferCond on_init; m_default_namespace_replayer->init(&on_init); @@ -431,6 +438,9 @@ void PoolReplayer::shut_down() { C_SaferCond ctx; m_remote_pool_poller->shut_down(&ctx); ctx.wait(); + + m_pool_meta_cache->remove_remote_pool_meta(m_remote_io_ctx.get_id()); + m_pool_meta_cache->remove_local_pool_meta(m_local_io_ctx.get_id()); } m_remote_pool_poller.reset(); m_remote_pool_poller_listener.reset(); @@ -657,7 +667,7 @@ void PoolReplayer::update_namespace_replayers() { name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, m_remote_pool_meta, m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), m_service_daemon, - m_cache_manager_handler); + m_cache_manager_handler, m_pool_meta_cache); auto on_init = new LambdaContext( [this, namespace_replayer, name, &mirroring_namespaces, ctx=gather_ctx->new_sub()](int r) { diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 1948eea6034..5ac84dd2645 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -34,6 +34,7 @@ namespace mirror { template class RemotePoolPoller; namespace remote_pool_poller { struct Listener; } +struct PoolMetaCache; template class ServiceDaemon; template struct Threads; @@ -47,6 +48,7 @@ public: PoolReplayer(Threads *threads, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler, + PoolMetaCache* pool_meta_cache, int64_t local_pool_id, const PeerSpec &peer, const std::vector &args); ~PoolReplayer(); @@ -193,6 +195,7 @@ private: Threads *m_threads; ServiceDaemon *m_service_daemon; journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; int64_t m_local_pool_id = -1; PeerSpec m_peer; std::vector m_args; diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc index 5377e7aa192..aa9bc7c3fe5 100644 --- a/src/tools/rbd_mirror/Types.cc +++ b/src/tools/rbd_mirror/Types.cc @@ -11,6 +11,11 @@ std::ostream &operator<<(std::ostream &os, const ImageId &image_id) { << "id=" << image_id.id; } +std::ostream& operator<<(std::ostream& lhs, + const LocalPoolMeta& rhs) { + return lhs << "mirror_uuid=" << rhs.mirror_uuid; +} + std::ostream& operator<<(std::ostream& lhs, const RemotePoolMeta& rhs) { return lhs << "mirror_uuid=" << rhs.mirror_uuid << ", " diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h index f2981713f93..c299aabe314 100644 --- a/src/tools/rbd_mirror/Types.h +++ b/src/tools/rbd_mirror/Types.h @@ -53,6 +53,18 @@ std::ostream &operator<<(std::ostream &, const ImageId &image_id); typedef std::set ImageIds; +struct LocalPoolMeta { + LocalPoolMeta() {} + LocalPoolMeta(const std::string& mirror_uuid) + : mirror_uuid(mirror_uuid) { + } + + std::string mirror_uuid; +}; + +std::ostream& operator<<(std::ostream& lhs, + const LocalPoolMeta& local_pool_meta); + struct RemotePoolMeta { RemotePoolMeta() {} RemotePoolMeta(const std::string& mirror_uuid, diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 334759ae19d..2265f706892 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -54,6 +54,7 @@ BootstrapRequest::BootstrapRequest( const std::string& local_mirror_uuid, const RemotePoolMeta& remote_pool_meta, ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder** state_builder, bool* do_resync, @@ -69,6 +70,7 @@ BootstrapRequest::BootstrapRequest( m_local_mirror_uuid(local_mirror_uuid), m_remote_pool_meta(remote_pool_meta), m_cache_manager_handler(cache_manager_handler), + m_pool_meta_cache(pool_meta_cache), m_progress_ctx(progress_ctx), m_state_builder(state_builder), m_do_resync(do_resync), diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h index 42010b6e91a..bfad0b0942d 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h @@ -27,6 +27,7 @@ class ProgressContext; template class ImageSync; template class InstanceWatcher; +struct PoolMetaCache; template struct Threads; namespace image_replayer { @@ -47,14 +48,15 @@ public: const std::string& local_mirror_uuid, const RemotePoolMeta& remote_pool_meta, ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder** state_builder, bool* do_resync, Context* on_finish) { return new BootstrapRequest( threads, local_io_ctx, remote_io_ctx, instance_watcher, global_image_id, - local_mirror_uuid, remote_pool_meta, cache_manager_handler, progress_ctx, - state_builder, do_resync, on_finish); + local_mirror_uuid, remote_pool_meta, cache_manager_handler, + pool_meta_cache, progress_ctx, state_builder, do_resync, on_finish); } BootstrapRequest( @@ -66,6 +68,7 @@ public: const std::string& local_mirror_uuid, const RemotePoolMeta& remote_pool_meta, ::journal::CacheManagerHandler* cache_manager_handler, + PoolMetaCache* pool_meta_cache, ProgressContext* progress_ctx, StateBuilder** state_builder, bool* do_resync, @@ -131,6 +134,7 @@ private: std::string m_local_mirror_uuid; RemotePoolMeta m_remote_pool_meta; ::journal::CacheManagerHandler *m_cache_manager_handler; + PoolMetaCache* m_pool_meta_cache; ProgressContext *m_progress_ctx; StateBuilder** m_state_builder; bool *m_do_resync; -- 2.39.5