rbd-mirror: cache local and remote pool meta properties for all pools

When replicating cloned images across pools, the replayer will
need access to the meta from the parent image pool. By caching
the data, it avoids the need to have the image replayer query
the parent pools.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
index 6928b426f0c196e409a948adf502d6d5e6c6a369..e0dbb1c47df0113e1e1aec81fb01dff6a43ccc0f 100644 (file)
--- a/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
+++ b/src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
@@ -539,7 +539,7 @@ public:
                                     local_mirror_uuid,
                                     {"remote mirror uuid",
                                      "remote mirror peer uuid"},
-                                    nullptr, nullptr,
+                                    nullptr, nullptr, nullptr,
                                     &m_mock_state_builder,
                                     &m_do_resync, on_finish);
   }

diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index 0a2456566bedbd2d3ac2a72a6b9f33b59c62e4fa..fc7a3f86e6e0110791d3ee4b3c5c0970683743f9 100644 (file)
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -37,6 +37,7 @@
 #include "tools/rbd_mirror/ImageReplayer.h"
 #include "tools/rbd_mirror/InstanceWatcher.h"
 #include "tools/rbd_mirror/MirrorStatusUpdater.h"
+#include "tools/rbd_mirror/PoolMetaCache.h"
 #include "tools/rbd_mirror/Threads.h"
 #include "tools/rbd_mirror/Throttler.h"
 #include "tools/rbd_mirror/Types.h"
@@ -161,7 +162,7 @@ public:
     m_replayer = new ImageReplayerT(m_local_ioctx, m_local_mirror_uuid,
                                     m_global_image_id, m_threads.get(),
                                     m_instance_watcher, m_local_status_updater,
-                                    nullptr);
+                                    nullptr, &m_pool_meta_cache);
     m_replayer->add_peer({"peer uuid", m_remote_ioctx, {}, nullptr});
   }
 
@@ -391,6 +392,8 @@ public:
 
   static int _image_number;
 
+  rbd::mirror::PoolMetaCache m_pool_meta_cache{g_ceph_context};
+
   std::shared_ptr<librados::Rados> m_local_cluster;
   std::unique_ptr<rbd::mirror::Threads<>> m_threads;
   std::unique_ptr<rbd::mirror::Throttler<>> m_image_sync_throttler;

diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
index f2612d7f94cac7febc38fc5a135dd65031010e71..f09375f217d10cac5a709612b9c134737b80c8b1 100644 (file)
--- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
@@ -107,6 +107,7 @@ struct BootstrapRequest<librbd::MockTestImageCtx> {
       const std::string &local_mirror_uuid,
       const RemotePoolMeta& remote_pool_meta,
       ::journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache,
       rbd::mirror::ProgressContext *progress_ctx,
       StateBuilder<librbd::MockTestImageCtx>** state_builder,
       bool *do_resync, Context *on_finish) {
@@ -354,7 +355,8 @@ public:
   void create_image_replayer(MockThreads &mock_threads) {
     m_image_replayer = new MockImageReplayer(
         m_local_io_ctx, "local_mirror_uuid", "global image id",
-        &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr);
+        &mock_threads, &m_instance_watcher, &m_local_status_updater, nullptr,
+        nullptr);
     m_image_replayer->add_peer({"peer_uuid", m_remote_io_ctx,
                                 {"remote mirror uuid",
                                  "remote mirror peer uuid"},

diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
index 002786108bd90fcd365392abbf850c4c1897ab18..574e3d5507ad486c75e53f495e2e08bbcfc0b4fe 100644 (file)
--- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
@@ -70,7 +70,8 @@ struct ImageReplayer<librbd::MockTestImageCtx> {
       Threads<librbd::MockTestImageCtx> *threads,
       InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
       MirrorStatusUpdater<librbd::MockTestImageCtx>* local_status_updater,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     ceph_assert(s_instance != nullptr);
     s_instance->global_image_id = global_image_id;
     return s_instance;
@@ -179,7 +180,8 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) {
   MockImageReplayer mock_image_replayer;
   MockInstanceReplayer instance_replayer(
       m_local_io_ctx, "local_mirror_uuid",
-      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr);
+      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr,
+      nullptr);
   std::string global_image_id("global_image_id");
 
   EXPECT_CALL(mock_image_replayer, get_global_image_id())
@@ -248,7 +250,8 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) {
   MockImageReplayer mock_image_replayer;
   MockInstanceReplayer instance_replayer(
       m_local_io_ctx, "local_mirror_uuid",
-      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr);
+      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr,
+      nullptr);
   std::string global_image_id("global_image_id");
 
   EXPECT_CALL(mock_image_replayer, get_global_image_id())
@@ -321,7 +324,8 @@ TEST_F(TestMockInstanceReplayer, Reacquire) {
   MockImageReplayer mock_image_replayer;
   MockInstanceReplayer instance_replayer(
       m_local_io_ctx, "local_mirror_uuid",
-      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr);
+      &mock_threads, &mock_service_daemon, &mock_status_updater, nullptr,
+      nullptr);
   std::string global_image_id("global_image_id");
 
   EXPECT_CALL(mock_image_replayer, get_global_image_id())

diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
index 53729598d7be35a1e5a0471d99ac73bbe21a4afc..d31b30c07d185685babf6d1ad375014c12d3ec07 100644 (file)
--- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
@@ -99,7 +99,8 @@ struct InstanceReplayer<librbd::MockTestImageCtx> {
       Threads<librbd::MockTestImageCtx> *threads,
       ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
       MirrorStatusUpdater<librbd::MockTestImageCtx>* local_status_updater,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     ceph_assert(s_instance != nullptr);
     return s_instance;
   }
@@ -400,7 +401,7 @@ TEST_F(TestMockNamespaceReplayer, Init_LocalMirrorStatusUpdaterError) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, nullptr, nullptr);
+      nullptr, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -422,7 +423,7 @@ TEST_F(TestMockNamespaceReplayer, Init_RemoteMirrorStatusUpdaterError) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, nullptr, nullptr);
+      nullptr, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -447,7 +448,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, nullptr, nullptr);
+      nullptr, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -477,7 +478,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, nullptr, nullptr);
+      nullptr, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -504,7 +505,7 @@ TEST_F(TestMockNamespaceReplayer, Init) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, &mock_service_daemon, nullptr);
+      nullptr, nullptr, &mock_service_daemon, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -543,7 +544,7 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) {
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
       "local peer uuid", {"remote mirror uuid", ""}, m_mock_threads,
-      nullptr, nullptr, &mock_service_daemon, nullptr);
+      nullptr, nullptr, &mock_service_daemon, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);

diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
index 2c46c1af5f9c76be520fcaf05af505bd5fa4bc01..6a29056e3073828e4dc7d6bbf011b4206b4674f2 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
@@ -13,6 +13,7 @@
 #include "tools/rbd_mirror/Throttler.h"
 #include "tools/rbd_mirror/LeaderWatcher.h"
 #include "tools/rbd_mirror/NamespaceReplayer.h"
+#include "tools/rbd_mirror/PoolMetaCache.h"
 #include "tools/rbd_mirror/PoolReplayer.h"
 #include "tools/rbd_mirror/RemotePoolPoller.h"
 #include "tools/rbd_mirror/ServiceDaemon.h"
@@ -138,7 +139,8 @@ struct NamespaceReplayer<librbd::MockTestImageCtx> {
       Throttler<librbd::MockTestImageCtx> *image_sync_throttler,
       Throttler<librbd::MockTestImageCtx> *image_deletion_throttler,
       ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     ceph_assert(s_instances.count(name));
     auto namespace_replayer = s_instances[name];
     s_instances.erase(name);
@@ -523,6 +525,8 @@ public:
     expect_service_daemon_add_or_update_attribute(
         mock_service_daemon, "instance_id", {instance_id});
   }
+
+  PoolMetaCache m_pool_meta_cache{g_ceph_context};
 };
 
 TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
@@ -570,6 +574,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
     mock_service_daemon, instance_id);
 
   MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+                                 &m_pool_meta_cache,
                                  m_local_io_ctx.get_id(), peer_spec, {});
   pool_replayer.init("siteA");
 
@@ -630,6 +635,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
     mock_service_daemon, instance_id);
 
   MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+                                 &m_pool_meta_cache,
                                  m_local_io_ctx.get_id(), peer_spec, {});
   pool_replayer.init("siteA");
 
@@ -716,6 +722,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
     mock_service_daemon, instance_id);
 
   MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+                                 &m_pool_meta_cache,
                                  m_local_io_ctx.get_id(), peer_spec, {});
   pool_replayer.init("siteA");
 
@@ -832,6 +839,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
     mock_service_daemon, instance_id);
 
   MockPoolReplayer pool_replayer(&mock_threads, &mock_service_daemon, nullptr,
+                                 &m_pool_meta_cache,
                                  m_local_io_ctx.get_id(), peer_spec, {});
   pool_replayer.init("siteA");
 

diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt
index 3c35e85a76e71bd4c1618ef9d97ffbfb0f5f74b1..9211718d12e64ad8ae7eed4394e06a1260607f55 100644 (file)
--- a/src/tools/rbd_mirror/CMakeLists.txt
+++ b/src/tools/rbd_mirror/CMakeLists.txt
@@ -17,6 +17,7 @@ set(rbd_mirror_internal
   MirrorStatusUpdater.cc
   MirrorStatusWatcher.cc
   NamespaceReplayer.cc
+  PoolMetaCache.cc
   PoolReplayer.cc
   PoolWatcher.cc
   RemotePoolPoller.cc

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index dfd895491a6d8c38c4698a5ab73919bbdb580bac..a6c4cb6575ed24b6006a65b0fbf2a92f05f74689 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -217,12 +217,14 @@ ImageReplayer<I>::ImageReplayer(
     const std::string &global_image_id, Threads<I> *threads,
     InstanceWatcher<I> *instance_watcher,
     MirrorStatusUpdater<I>* local_status_updater,
-    journal::CacheManagerHandler *cache_manager_handler) :
+    journal::CacheManagerHandler *cache_manager_handler,
+    PoolMetaCache* pool_meta_cache) :
   m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid),
   m_global_image_id(global_image_id), m_threads(threads),
   m_instance_watcher(instance_watcher),
   m_local_status_updater(local_status_updater),
   m_cache_manager_handler(cache_manager_handler),
+  m_pool_meta_cache(pool_meta_cache),
   m_local_image_name(global_image_id),
   m_lock(ceph::make_mutex("rbd::mirror::ImageReplayer " +
       stringify(local_io_ctx.get_id()) + " " + global_image_id)),
@@ -352,7 +354,8 @@ void ImageReplayer<I>::bootstrap() {
       m_threads, m_local_io_ctx, m_remote_image_peer.io_ctx, m_instance_watcher,
       m_global_image_id, m_local_mirror_uuid,
       m_remote_image_peer.remote_pool_meta, m_cache_manager_handler,
-      &m_progress_cxt, &m_state_builder, &m_resync_requested, ctx);
+      m_pool_meta_cache, &m_progress_cxt, &m_state_builder, &m_resync_requested,
+      ctx);
 
   request->get();
   m_bootstrap_request = request;

diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h
index bff9e2af9e13ac7bd4102b87130b90641158e8e8..c9718e06b9c209dcd4716155f092ad0f42c3eb60 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.h
+++ b/src/tools/rbd_mirror/ImageReplayer.h
@@ -24,6 +24,7 @@ namespace mirror {
 
 template <typename> struct InstanceWatcher;
 template <typename> struct MirrorStatusUpdater;
+struct PoolMetaCache;
 template <typename> struct Threads;
 
 namespace image_replayer {
@@ -45,10 +46,11 @@ public:
       const std::string &global_image_id, Threads<ImageCtxT> *threads,
       InstanceWatcher<ImageCtxT> *instance_watcher,
       MirrorStatusUpdater<ImageCtxT>* local_status_updater,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     return new ImageReplayer(local_io_ctx, local_mirror_uuid, global_image_id,
                              threads, instance_watcher, local_status_updater,
-                             cache_manager_handler);
+                             cache_manager_handler, pool_meta_cache);
   }
   void destroy() {
     delete this;
@@ -60,7 +62,8 @@ public:
                 Threads<ImageCtxT> *threads,
                 InstanceWatcher<ImageCtxT> *instance_watcher,
                 MirrorStatusUpdater<ImageCtxT>* local_status_updater,
-                journal::CacheManagerHandler *cache_manager_handler);
+                journal::CacheManagerHandler *cache_manager_handler,
+                PoolMetaCache* pool_meta_cache);
   virtual ~ImageReplayer();
   ImageReplayer(const ImageReplayer&) = delete;
   ImageReplayer& operator=(const ImageReplayer&) = delete;
@@ -181,6 +184,7 @@ private:
   InstanceWatcher<ImageCtxT> *m_instance_watcher;
   MirrorStatusUpdater<ImageCtxT>* m_local_status_updater;
   journal::CacheManagerHandler *m_cache_manager_handler;
+  PoolMetaCache* m_pool_meta_cache;
 
   Peers m_peers;
   Peer<ImageCtxT> m_remote_image_peer;

diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index 21ff8da725704cd20e49863270e233b50a802373..2b9b47a0e012206fd31d2c18bb0d32394ac4b711 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -38,11 +38,13 @@ InstanceReplayer<I>::InstanceReplayer(
     librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
     Threads<I> *threads, ServiceDaemon<I>* service_daemon,
     MirrorStatusUpdater<I>* local_status_updater,
-    journal::CacheManagerHandler *cache_manager_handler)
+    journal::CacheManagerHandler *cache_manager_handler,
+    PoolMetaCache* pool_meta_cache)
   : m_local_io_ctx(local_io_ctx), m_local_mirror_uuid(local_mirror_uuid),
     m_threads(threads), m_service_daemon(service_daemon),
     m_local_status_updater(local_status_updater),
     m_cache_manager_handler(cache_manager_handler),
+    m_pool_meta_cache(pool_meta_cache),
     m_lock(ceph::make_mutex("rbd::mirror::InstanceReplayer " +
         stringify(local_io_ctx.get_id()))) {
 }
@@ -148,7 +150,7 @@ void InstanceReplayer<I>::acquire_image(InstanceWatcher<I> *instance_watcher,
     auto image_replayer = ImageReplayer<I>::create(
         m_local_io_ctx, m_local_mirror_uuid, global_image_id,
         m_threads, instance_watcher, m_local_status_updater,
-        m_cache_manager_handler);
+        m_cache_manager_handler, m_pool_meta_cache);
 
     dout(10) << global_image_id << ": creating replayer " << image_replayer
              << dendl;

diff --git a/src/tools/rbd_mirror/InstanceReplayer.h b/src/tools/rbd_mirror/InstanceReplayer.h
index 2bd8d11f59d51aaffbad51f4dfa4029074558e2b..3119fd3ebdaa25668602b3946b439b9d904c0dfa 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.h
+++ b/src/tools/rbd_mirror/InstanceReplayer.h
@@ -22,6 +22,7 @@ namespace mirror {
 template <typename> class ImageReplayer;
 template <typename> class InstanceWatcher;
 template <typename> class MirrorStatusUpdater;
+struct PoolMetaCache;
 template <typename> class ServiceDaemon;
 template <typename> struct Threads;
 
@@ -32,10 +33,11 @@ public:
       librados::IoCtx &local_io_ctx, const std::string &local_mirror_uuid,
       Threads<ImageCtxT> *threads, ServiceDaemon<ImageCtxT> *service_daemon,
       MirrorStatusUpdater<ImageCtxT>* local_status_updater,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     return new InstanceReplayer(local_io_ctx, local_mirror_uuid, threads,
                                 service_daemon, local_status_updater,
-                                cache_manager_handler);
+                                cache_manager_handler, pool_meta_cache);
   }
   void destroy() {
     delete this;
@@ -46,7 +48,8 @@ public:
                    Threads<ImageCtxT> *threads,
                    ServiceDaemon<ImageCtxT> *service_daemon,
                    MirrorStatusUpdater<ImageCtxT>* local_status_updater,
-                   journal::CacheManagerHandler *cache_manager_handler);
+                   journal::CacheManagerHandler *cache_manager_handler,
+                   PoolMetaCache* pool_meta_cache);
   ~InstanceReplayer();
 
   int init();
@@ -97,6 +100,7 @@ private:
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   MirrorStatusUpdater<ImageCtxT>* m_local_status_updater;
   journal::CacheManagerHandler *m_cache_manager_handler;
+  PoolMetaCache* m_pool_meta_cache;
 
   ceph::mutex m_lock;
   AsyncOpTracker m_async_op_tracker;

diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc
index 787c6980ecfce88ff9d2b801f3c67812dd8666b1..e24a78ae805af7b2869101aacbaa8a5707c6761d 100644 (file)
--- a/src/tools/rbd_mirror/Mirror.cc
+++ b/src/tools/rbd_mirror/Mirror.cc
@@ -14,6 +14,7 @@
 #include "librbd/ImageCtx.h"
 #include "perfglue/heap_profiler.h"
 #include "Mirror.h"
+#include "PoolMetaCache.h"
 #include "ServiceDaemon.h"
 #include "Threads.h"
 
@@ -485,6 +486,7 @@ Mirror::Mirror(CephContext *cct, const std::vector<const char*> &args) :
   m_args(args),
   m_local(new librados::Rados()),
   m_cache_manager_handler(new CacheManagerHandler(cct)),
+  m_pool_meta_cache(new PoolMetaCache(cct)),
   m_asok_hook(new MirrorAdminSocketHook(cct, this))
 {
   m_threads =
@@ -727,7 +729,8 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers,
         dout(20) << "starting pool replayer for " << peer << dendl;
         unique_ptr<PoolReplayer<>> pool_replayer(
             new PoolReplayer<>(m_threads, m_service_daemon.get(),
-                               m_cache_manager_handler.get(), kv.first, peer,
+                               m_cache_manager_handler.get(),
+                               m_pool_meta_cache.get(), kv.first, peer,
                                m_args));
 
         // TODO: make async

diff --git a/src/tools/rbd_mirror/Mirror.h b/src/tools/rbd_mirror/Mirror.h
index 28f71c54dc2dd5e35c404dd3e0defc72d8b528a9..f92a63b687762474e939e14c6231f5a27c3e82c5 100644 (file)
--- a/src/tools/rbd_mirror/Mirror.h
+++ b/src/tools/rbd_mirror/Mirror.h
@@ -28,6 +28,7 @@ template <typename> struct ServiceDaemon;
 template <typename> struct Threads;
 class CacheManagerHandler;
 class MirrorAdminSocketHook;
+class PoolMetaCache;
 
 /**
  * Contains the main loop and overall state for rbd-mirror.
@@ -74,6 +75,7 @@ private:
   // monitor local cluster for config changes in peers
   std::unique_ptr<ClusterWatcher> m_local_cluster_watcher;
   std::unique_ptr<CacheManagerHandler> m_cache_manager_handler;
+  std::unique_ptr<PoolMetaCache> m_pool_meta_cache;
   std::map<PoolPeer, std::unique_ptr<PoolReplayer<>>> m_pool_replayers;
   std::atomic<bool> m_stopping = { false };
   bool m_manual_stop = false;

diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc
index 3d86ff1fe92949e4569b34f45962c282374ded6b..5cc7d1b2f09c333ab28669b508f3b5f5715c766d 100644 (file)
--- a/src/tools/rbd_mirror/NamespaceReplayer.cc
+++ b/src/tools/rbd_mirror/NamespaceReplayer.cc
@@ -40,10 +40,13 @@ NamespaceReplayer<I>::NamespaceReplayer(
     librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
     const std::string &local_mirror_uuid,
     const std::string& local_mirror_peer_uuid,
-    const RemotePoolMeta& remote_pool_meta, Threads<I> *threads,
-    Throttler<I> *image_sync_throttler, Throttler<I> *image_deletion_throttler,
+    const RemotePoolMeta& remote_pool_meta,
+    Threads<I> *threads,
+    Throttler<I> *image_sync_throttler,
+    Throttler<I> *image_deletion_throttler,
     ServiceDaemon<I> *service_daemon,
-    journal::CacheManagerHandler *cache_manager_handler) :
+    journal::CacheManagerHandler *cache_manager_handler,
+    PoolMetaCache* pool_meta_cache) :
   m_namespace_name(name),
   m_local_mirror_uuid(local_mirror_uuid),
   m_local_mirror_peer_uuid(local_mirror_peer_uuid),
@@ -52,6 +55,7 @@ NamespaceReplayer<I>::NamespaceReplayer(
   m_image_deletion_throttler(image_deletion_throttler),
   m_service_daemon(service_daemon),
   m_cache_manager_handler(cache_manager_handler),
+  m_pool_meta_cache(pool_meta_cache),
   m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
       "rbd::mirror::NamespaceReplayer " + name, this))),
   m_local_pool_watcher_listener(this, true),
@@ -345,7 +349,8 @@ void NamespaceReplayer<I>::init_instance_replayer() {
 
   m_instance_replayer.reset(InstanceReplayer<I>::create(
       m_local_io_ctx, m_local_mirror_uuid, m_threads, m_service_daemon,
-      m_local_status_updater.get(), m_cache_manager_handler));
+      m_local_status_updater.get(), m_cache_manager_handler,
+      m_pool_meta_cache));
   auto ctx = create_context_callback<NamespaceReplayer<I>,
       &NamespaceReplayer<I>::handle_init_instance_replayer>(this);
 

diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h
index 0d5e0a95c273b7494b2253ab67ef67461bf21e16..4fbf4c53bec4dd6a1654befc5e96e643ab7a7407 100644 (file)
--- a/src/tools/rbd_mirror/NamespaceReplayer.h
+++ b/src/tools/rbd_mirror/NamespaceReplayer.h
@@ -32,6 +32,7 @@ namespace librbd { class ImageCtx; }
 namespace rbd {
 namespace mirror {
 
+struct PoolMetaCache;
 template <typename> class ServiceDaemon;
 template <typename> class Throttler;
 template <typename> struct Threads;
@@ -53,12 +54,14 @@ public:
       Throttler<ImageCtxT> *image_sync_throttler,
       Throttler<ImageCtxT> *image_deletion_throttler,
       ServiceDaemon<ImageCtxT> *service_daemon,
-      journal::CacheManagerHandler *cache_manager_handler) {
+      journal::CacheManagerHandler *cache_manager_handler,
+      PoolMetaCache* pool_meta_cache) {
     return new NamespaceReplayer(name, local_ioctx, remote_ioctx,
                                  local_mirror_uuid, local_mirror_peer_uuid,
                                  remote_pool_meta, threads,
                                  image_sync_throttler, image_deletion_throttler,
-                                 service_daemon, cache_manager_handler);
+                                 service_daemon, cache_manager_handler,
+                                 pool_meta_cache);
   }
 
   NamespaceReplayer(const std::string &name,
@@ -71,7 +74,8 @@ public:
                     Throttler<ImageCtxT> *image_sync_throttler,
                     Throttler<ImageCtxT> *image_deletion_throttler,
                     ServiceDaemon<ImageCtxT> *service_daemon,
-                    journal::CacheManagerHandler *cache_manager_handler);
+                    journal::CacheManagerHandler *cache_manager_handler,
+                    PoolMetaCache* pool_meta_cache);
   NamespaceReplayer(const NamespaceReplayer&) = delete;
   NamespaceReplayer& operator=(const NamespaceReplayer&) = delete;
 
@@ -271,6 +275,7 @@ private:
   Throttler<ImageCtxT> *m_image_deletion_throttler;
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   journal::CacheManagerHandler *m_cache_manager_handler;
+  PoolMetaCache* m_pool_meta_cache;
 
   mutable ceph::mutex m_lock;
 

diff --git a/src/tools/rbd_mirror/PoolMetaCache.cc b/src/tools/rbd_mirror/PoolMetaCache.cc
new file mode 100644 (file)
index 0000000..261802a
--- /dev/null
+++ b/src/tools/rbd_mirror/PoolMetaCache.cc
@@ -0,0 +1,83 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "common/debug.h"
+#include "common/dout.h"
+#include "tools/rbd_mirror/PoolMetaCache.h"
+#include <shared_mutex>
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd_mirror
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::mirror::PoolMetaCache: " \
+                           << this << " " << __func__ << ": "
+
+namespace rbd {
+namespace mirror {
+
+int PoolMetaCache::get_local_pool_meta(
+    int64_t pool_id,
+    LocalPoolMeta* local_pool_meta) const {
+  dout(15) << "pool_id=" << pool_id << dendl;
+
+  std::shared_lock locker{m_lock};
+  auto it = m_local_pool_metas.find(pool_id);
+  if (it == m_local_pool_metas.end()) {
+    return -ENOENT;
+  }
+
+  *local_pool_meta = it->second;
+  return 0;
+}
+
+void PoolMetaCache::set_local_pool_meta(
+    int64_t pool_id,
+    const LocalPoolMeta& local_pool_meta) {
+  dout(15) << "pool_id=" << pool_id << ", "
+           << "local_pool_meta=" << local_pool_meta << dendl;
+
+  std::unique_lock locker(m_lock);
+  m_local_pool_metas[pool_id] = local_pool_meta;
+}
+
+void PoolMetaCache::remove_local_pool_meta(int64_t pool_id) {
+  dout(15) << "pool_id=" << pool_id << dendl;
+
+  std::unique_lock locker(m_lock);
+  m_local_pool_metas.erase(pool_id);
+}
+
+int PoolMetaCache::get_remote_pool_meta(
+    int64_t pool_id,
+    RemotePoolMeta* remote_pool_meta) const {
+  dout(15) << "pool_id=" << pool_id << dendl;
+
+  std::shared_lock locker{m_lock};
+  auto it = m_remote_pool_metas.find(pool_id);
+  if (it == m_remote_pool_metas.end()) {
+    return -ENOENT;
+  }
+
+  *remote_pool_meta = it->second;
+  return 0;
+}
+
+void PoolMetaCache::set_remote_pool_meta(
+    int64_t pool_id,
+    const RemotePoolMeta& remote_pool_meta) {
+  dout(15) << "pool_id=" << pool_id << ", "
+           << "remote_pool_meta=" << remote_pool_meta << dendl;
+
+  std::unique_lock locker(m_lock);
+  m_remote_pool_metas[pool_id] = remote_pool_meta;
+}
+
+void PoolMetaCache::remove_remote_pool_meta(int64_t pool_id) {
+  dout(15) << "pool_id=" << pool_id << dendl;
+
+  std::unique_lock locker(m_lock);
+  m_remote_pool_metas.erase(pool_id);
+}
+
+} // namespace mirror
+} // namespace rbd

diff --git a/src/tools/rbd_mirror/PoolMetaCache.h b/src/tools/rbd_mirror/PoolMetaCache.h
new file mode 100644 (file)
index 0000000..cef4a68
--- /dev/null
+++ b/src/tools/rbd_mirror/PoolMetaCache.h
@@ -0,0 +1,49 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_RBD_MIRROR_POOL_META_CACHE_H
+#define CEPH_RBD_MIRROR_POOL_META_CACHE_H
+
+#include "include/int_types.h"
+#include "common/ceph_mutex.h"
+#include "tools/rbd_mirror/Types.h"
+#include <map>
+
+struct CephContext;
+
+namespace rbd {
+namespace mirror {
+
+class PoolMetaCache {
+public:
+  PoolMetaCache(CephContext* cct)
+    : m_cct(cct) {
+  }
+  PoolMetaCache(const PoolMetaCache&) = delete;
+  PoolMetaCache& operator=(const PoolMetaCache&) = delete;
+
+  int get_local_pool_meta(int64_t pool_id,
+                          LocalPoolMeta* local_pool_meta) const;
+  void set_local_pool_meta(int64_t pool_id,
+                           const LocalPoolMeta& local_pool_meta);
+  void remove_local_pool_meta(int64_t pool_id);
+
+  int get_remote_pool_meta(int64_t pool_id,
+                           RemotePoolMeta* remote_pool_meta) const;
+  void set_remote_pool_meta(int64_t pool_id,
+                            const RemotePoolMeta& remote_pool_meta);
+  void remove_remote_pool_meta(int64_t pool_id);
+
+private:
+  CephContext* m_cct;
+
+  mutable ceph::shared_mutex m_lock =
+    ceph::make_shared_mutex("rbd::mirror::PoolMetaCache::m_lock");
+  std::map<int64_t, LocalPoolMeta> m_local_pool_metas;
+  std::map<int64_t, RemotePoolMeta> m_remote_pool_metas;
+};
+
+} // namespace mirror
+} // namespace rbd
+
+#endif // CEPH_RBD_MIRROR_POOL_META_CACHE_H

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index 11623e2a3c49a38bff5d72aa107537f4f5f05746..416ca4ba9f70cd1f87e4b0594718614189035f9e 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -15,6 +15,7 @@
 #include "global/global_context.h"
 #include "librbd/api/Config.h"
 #include "librbd/api/Namespace.h"
+#include "PoolMetaCache.h"
 #include "RemotePoolPoller.h"
 #include "ServiceDaemon.h"
 #include "Threads.h"
@@ -224,11 +225,13 @@ struct PoolReplayer<I>::RemotePoolPollerListener
 template <typename I>
 PoolReplayer<I>::PoolReplayer(
     Threads<I> *threads, ServiceDaemon<I> *service_daemon,
-    journal::CacheManagerHandler *cache_manager_handler, int64_t local_pool_id,
+    journal::CacheManagerHandler *cache_manager_handler,
+    PoolMetaCache* pool_meta_cache, int64_t local_pool_id,
     const PeerSpec &peer, const std::vector<const char*> &args) :
   m_threads(threads),
   m_service_daemon(service_daemon),
   m_cache_manager_handler(cache_manager_handler),
+  m_pool_meta_cache(pool_meta_cache),
   m_local_pool_id(local_pool_id),
   m_peer(peer),
   m_args(args),
@@ -360,12 +363,16 @@ void PoolReplayer<I>::init(const std::string& site_name) {
     return;
   }
   ceph_assert(!m_remote_pool_meta.mirror_uuid.empty());
+  m_pool_meta_cache->set_remote_pool_meta(
+    m_remote_io_ctx.get_id(), m_remote_pool_meta);
+  m_pool_meta_cache->set_local_pool_meta(
+    m_local_io_ctx.get_id(), {m_local_mirror_uuid});
 
   m_default_namespace_replayer.reset(NamespaceReplayer<I>::create(
       "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
       m_remote_pool_meta, m_threads, m_image_sync_throttler.get(),
       m_image_deletion_throttler.get(), m_service_daemon,
-      m_cache_manager_handler));
+      m_cache_manager_handler, m_pool_meta_cache));
 
   C_SaferCond on_init;
   m_default_namespace_replayer->init(&on_init);
@@ -431,6 +438,9 @@ void PoolReplayer<I>::shut_down() {
     C_SaferCond ctx;
     m_remote_pool_poller->shut_down(&ctx);
     ctx.wait();
+
+    m_pool_meta_cache->remove_remote_pool_meta(m_remote_io_ctx.get_id());
+    m_pool_meta_cache->remove_local_pool_meta(m_local_io_ctx.get_id());
   }
   m_remote_pool_poller.reset();
   m_remote_pool_poller_listener.reset();
@@ -657,7 +667,7 @@ void PoolReplayer<I>::update_namespace_replayers() {
         name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
         m_remote_pool_meta, m_threads, m_image_sync_throttler.get(),
         m_image_deletion_throttler.get(), m_service_daemon,
-        m_cache_manager_handler);
+        m_cache_manager_handler, m_pool_meta_cache);
     auto on_init = new LambdaContext(
         [this, namespace_replayer, name, &mirroring_namespaces,
          ctx=gather_ctx->new_sub()](int r) {

diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h
index 1948eea603420842d64e3144224509f3978dce3c..5ac84dd264569f7da15f570cba49f1f8309947c8 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.h
+++ b/src/tools/rbd_mirror/PoolReplayer.h
@@ -34,6 +34,7 @@ namespace mirror {
 template <typename> class RemotePoolPoller;
 namespace remote_pool_poller { struct Listener; }
 
+struct PoolMetaCache;
 template <typename> class ServiceDaemon;
 template <typename> struct Threads;
 
@@ -47,6 +48,7 @@ public:
   PoolReplayer(Threads<ImageCtxT> *threads,
                ServiceDaemon<ImageCtxT> *service_daemon,
                journal::CacheManagerHandler *cache_manager_handler,
+               PoolMetaCache* pool_meta_cache,
               int64_t local_pool_id, const PeerSpec &peer,
               const std::vector<const char*> &args);
   ~PoolReplayer();
@@ -193,6 +195,7 @@ private:
   Threads<ImageCtxT> *m_threads;
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   journal::CacheManagerHandler *m_cache_manager_handler;
+  PoolMetaCache* m_pool_meta_cache;
   int64_t m_local_pool_id = -1;
   PeerSpec m_peer;
   std::vector<const char*> m_args;

diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc
index 5377e7aa192f6dd611205fa5beea0d64885ae585..aa9bc7c3fe55f556d0e27563f79f49b7799514ca 100644 (file)
--- a/src/tools/rbd_mirror/Types.cc
+++ b/src/tools/rbd_mirror/Types.cc
@@ -11,6 +11,11 @@ std::ostream &operator<<(std::ostream &os, const ImageId &image_id) {
             << "id=" << image_id.id;
 }
 
+std::ostream& operator<<(std::ostream& lhs,
+                         const LocalPoolMeta& rhs) {
+  return lhs << "mirror_uuid=" << rhs.mirror_uuid;
+}
+
 std::ostream& operator<<(std::ostream& lhs,
                          const RemotePoolMeta& rhs) {
   return lhs << "mirror_uuid=" << rhs.mirror_uuid << ", "

diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h
index f2981713f9346e210202771ac841f6158e9d2939..c299aabe3145b4583d31350d143a7191d5e502ee 100644 (file)
--- a/src/tools/rbd_mirror/Types.h
+++ b/src/tools/rbd_mirror/Types.h
@@ -53,6 +53,18 @@ std::ostream &operator<<(std::ostream &, const ImageId &image_id);
 
 typedef std::set<ImageId> ImageIds;
 
+struct LocalPoolMeta {
+  LocalPoolMeta() {}
+  LocalPoolMeta(const std::string& mirror_uuid)
+    : mirror_uuid(mirror_uuid) {
+  }
+
+  std::string mirror_uuid;
+};
+
+std::ostream& operator<<(std::ostream& lhs,
+                         const LocalPoolMeta& local_pool_meta);
+
 struct RemotePoolMeta {
   RemotePoolMeta() {}
   RemotePoolMeta(const std::string& mirror_uuid,

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
index 334759ae19da2ff20eed1f4208613b1ac529a010..2265f706892599fc469fa1a522783c194548650a 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
@@ -54,6 +54,7 @@ BootstrapRequest<I>::BootstrapRequest(
     const std::string& local_mirror_uuid,
     const RemotePoolMeta& remote_pool_meta,
     ::journal::CacheManagerHandler* cache_manager_handler,
+    PoolMetaCache* pool_meta_cache,
     ProgressContext* progress_ctx,
     StateBuilder<I>** state_builder,
     bool* do_resync,
@@ -69,6 +70,7 @@ BootstrapRequest<I>::BootstrapRequest(
     m_local_mirror_uuid(local_mirror_uuid),
     m_remote_pool_meta(remote_pool_meta),
     m_cache_manager_handler(cache_manager_handler),
+    m_pool_meta_cache(pool_meta_cache),
     m_progress_ctx(progress_ctx),
     m_state_builder(state_builder),
     m_do_resync(do_resync),

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
index 42010b6e91ae87e4730fbbe572f20c4869a0d5ff..bfad0b0942dc59b743df294588508750862a639c 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
@@ -27,6 +27,7 @@ class ProgressContext;
 
 template <typename> class ImageSync;
 template <typename> class InstanceWatcher;
+struct PoolMetaCache;
 template <typename> struct Threads;
 
 namespace image_replayer {
@@ -47,14 +48,15 @@ public:
       const std::string& local_mirror_uuid,
       const RemotePoolMeta& remote_pool_meta,
       ::journal::CacheManagerHandler* cache_manager_handler,
+      PoolMetaCache* pool_meta_cache,
       ProgressContext* progress_ctx,
       StateBuilder<ImageCtxT>** state_builder,
       bool* do_resync,
       Context* on_finish) {
     return new BootstrapRequest(
       threads, local_io_ctx, remote_io_ctx, instance_watcher, global_image_id,
-      local_mirror_uuid, remote_pool_meta, cache_manager_handler, progress_ctx,
-      state_builder, do_resync, on_finish);
+      local_mirror_uuid, remote_pool_meta, cache_manager_handler,
+      pool_meta_cache, progress_ctx, state_builder, do_resync, on_finish);
   }
 
   BootstrapRequest(
@@ -66,6 +68,7 @@ public:
       const std::string& local_mirror_uuid,
       const RemotePoolMeta& remote_pool_meta,
       ::journal::CacheManagerHandler* cache_manager_handler,
+      PoolMetaCache* pool_meta_cache,
       ProgressContext* progress_ctx,
       StateBuilder<ImageCtxT>** state_builder,
       bool* do_resync,
@@ -131,6 +134,7 @@ private:
   std::string m_local_mirror_uuid;
   RemotePoolMeta m_remote_pool_meta;
   ::journal::CacheManagerHandler *m_cache_manager_handler;
+  PoolMetaCache* m_pool_meta_cache;
   ProgressContext *m_progress_ctx;
   StateBuilder<ImageCtxT>** m_state_builder;
   bool *m_do_resync;