rbd-mirror: pool replayer should instantiate the remote pool poller

Let the poller pull the metadata from the remote before advancing to
the leader watcher initialization. If the remote metadata changes
during runtime, stop the pool replayer so that it can be
re-initialized.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
index 756d3906490f32b4c6d8f5ba1e7f064a62428c25..040ce633a6a126846b78275ee45d840d70fa200e 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
@@ -14,6 +14,7 @@
 #include "tools/rbd_mirror/LeaderWatcher.h"
 #include "tools/rbd_mirror/NamespaceReplayer.h"
 #include "tools/rbd_mirror/PoolReplayer.h"
+#include "tools/rbd_mirror/RemotePoolPoller.h"
 #include "tools/rbd_mirror/ServiceDaemon.h"
 #include "tools/rbd_mirror/Threads.h"
 #include "common/Formatter.h"
@@ -200,6 +201,33 @@ struct LeaderWatcher<librbd::MockTestImageCtx> {
 
 LeaderWatcher<librbd::MockTestImageCtx>* LeaderWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
 
+template<>
+struct RemotePoolPoller<librbd::MockTestImageCtx> {
+  static RemotePoolPoller* s_instance;
+
+  remote_pool_poller::Listener* listener = nullptr;
+
+  static RemotePoolPoller* create(
+      Threads<librbd::MockTestImageCtx>* threads,
+      librados::IoCtx& remote_io_ctx,
+      const std::string& local_site_name,
+      const std::string& local_fsid,
+      remote_pool_poller::Listener& listener) {
+    ceph_assert(s_instance != nullptr);
+    s_instance->listener = &listener;
+    return s_instance;
+  }
+
+  MOCK_METHOD1(init, void(Context*));
+  MOCK_METHOD1(shut_down, void(Context*));
+
+  RemotePoolPoller() {
+    s_instance = this;
+  }
+};
+
+RemotePoolPoller<librbd::MockTestImageCtx>* RemotePoolPoller<librbd::MockTestImageCtx>::s_instance = nullptr;
+
 template<>
 struct ServiceDaemon<librbd::MockTestImageCtx> {
   MOCK_METHOD2(add_namespace, void(int64_t, const std::string &));
@@ -260,6 +288,7 @@ public:
   typedef PoolReplayer<librbd::MockTestImageCtx> MockPoolReplayer;
   typedef Throttler<librbd::MockTestImageCtx> MockThrottler;
   typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer;
+  typedef RemotePoolPoller<librbd::MockTestImageCtx> MockRemotePoolPoller;
   typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
   typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon;
   typedef Threads<librbd::MockTestImageCtx> MockThreads;
@@ -353,6 +382,31 @@ public:
       .Times(AtLeast(0));
   }
 
+  void expect_remote_pool_poller_init(
+      MockRemotePoolPoller& mock_remote_pool_poller,
+      const RemotePoolMeta& remote_pool_meta, int r) {
+    EXPECT_CALL(mock_remote_pool_poller, init(_))
+      .WillOnce(Invoke(
+                  [this, &mock_remote_pool_poller, remote_pool_meta, r]
+                  (Context* ctx) {
+                    if (r >= 0) {
+                      mock_remote_pool_poller.listener->handle_updated(
+                        remote_pool_meta);
+                    }
+
+                    m_threads->work_queue->queue(ctx, r);
+                }));
+  }
+
+  void expect_remote_pool_poller_shut_down(
+      MockRemotePoolPoller& mock_remote_pool_poller, int r) {
+    EXPECT_CALL(mock_remote_pool_poller, shut_down(_))
+      .WillOnce(Invoke(
+                  [this, r](Context* ctx) {
+                    m_threads->work_queue->queue(ctx, r);
+                }));
+  }
+
   void expect_namespace_replayer_is_blacklisted(
       MockNamespaceReplayer &mock_namespace_replayer,
       bool blacklisted) {
@@ -504,6 +558,9 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
 
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -523,6 +580,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -560,6 +618,9 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
 
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -592,6 +653,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -642,6 +704,9 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
                  nullptr);
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -707,6 +772,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
   expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer);
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -754,6 +820,9 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
                  nullptr);
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -837,6 +906,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index b99070bd5b96b874201219b9ef2b1366b8e85212..55a522800921aeca99fea8191ae35905531286c4 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -15,6 +15,7 @@
 #include "global/global_context.h"
 #include "librbd/api/Config.h"
 #include "librbd/api/Namespace.h"
+#include "RemotePoolPoller.h"
 #include "ServiceDaemon.h"
 #include "Threads.h"
 
@@ -205,6 +206,21 @@ private:
 
 } // anonymous namespace
 
+template <typename I>
+struct PoolReplayer<I>::RemotePoolPollerListener
+  : public remote_pool_poller::Listener {
+
+  PoolReplayer<I>* m_pool_replayer;
+
+  RemotePoolPollerListener(PoolReplayer<I>* pool_replayer)
+    : m_pool_replayer(pool_replayer) {
+  }
+
+  void handle_updated(const RemotePoolMeta& remote_pool_meta) override {
+    m_pool_replayer->handle_remote_pool_meta_updated(remote_pool_meta);
+  }
+};
+
 template <typename I>
 PoolReplayer<I>::PoolReplayer(
     Threads<I> *threads, ServiceDaemon<I> *service_daemon,
@@ -319,6 +335,32 @@ void PoolReplayer<I>::init(const std::string& site_name) {
   m_image_deletion_throttler.reset(
       Throttler<I>::create(cct, "rbd_mirror_concurrent_image_deletions"));
 
+  std::string local_fsid;
+  r = m_local_rados->cluster_fsid(&local_fsid);
+  if (r < 0) {
+    derr << "failed to retrieve local fsid: " << cpp_strerror(r) << dendl;
+    return;
+  }
+
+  m_remote_pool_poller_listener.reset(new RemotePoolPollerListener(this));
+  m_remote_pool_poller.reset(RemotePoolPoller<I>::create(
+    m_threads, m_remote_io_ctx, m_site_name, local_fsid,
+    *m_remote_pool_poller_listener));
+
+  C_SaferCond on_pool_poller_init;
+  m_remote_pool_poller->init(&on_pool_poller_init);
+  r = on_pool_poller_init.wait();
+  if (r < 0) {
+    derr << "failed to initialize remote pool poller: " << cpp_strerror(r)
+         << dendl;
+    m_callout_id = m_service_daemon->add_or_update_callout(
+      m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR,
+      "unable to initialize remote pool poller");
+    m_remote_pool_poller.reset();
+    return;
+  }
+  ceph_assert(!m_remote_pool_meta.mirror_uuid.empty());
+
   m_default_namespace_replayer.reset(NamespaceReplayer<I>::create(
       "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
       m_site_name, m_threads, m_image_sync_throttler.get(),
@@ -385,6 +427,14 @@ void PoolReplayer<I>::shut_down() {
   }
   m_default_namespace_replayer.reset();
 
+  if (m_remote_pool_poller) {
+    C_SaferCond ctx;
+    m_remote_pool_poller->shut_down(&ctx);
+    ctx.wait();
+  }
+  m_remote_pool_poller.reset();
+  m_remote_pool_poller_listener.reset();
+
   m_image_sync_throttler.reset();
   m_image_deletion_throttler.reset();
 
@@ -1024,6 +1074,22 @@ void PoolReplayer<I>::handle_instances_removed(
   }
 }
 
+template <typename I>
+void PoolReplayer<I>::handle_remote_pool_meta_updated(
+    const RemotePoolMeta& remote_pool_meta) {
+  dout(5) << "remote_pool_meta=" << remote_pool_meta << dendl;
+
+  if (!m_default_namespace_replayer) {
+    m_remote_pool_meta = remote_pool_meta;
+    return;
+  }
+
+  derr << "remote pool metadata updated unexpectedly" << dendl;
+  std::unique_lock locker{m_lock};
+  m_stopping = true;
+  m_cond.notify_all();
+}
+
 } // namespace mirror
 } // namespace rbd
 

diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h
index 9f295f4788caffd4e3ba65e3919c9f85d9369615..1948eea603420842d64e3144224509f3978dce3c 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.h
+++ b/src/tools/rbd_mirror/PoolReplayer.h
@@ -31,9 +31,13 @@ namespace librbd { class ImageCtx; }
 namespace rbd {
 namespace mirror {
 
+template <typename> class RemotePoolPoller;
+namespace remote_pool_poller { struct Listener; }
+
 template <typename> class ServiceDaemon;
 template <typename> struct Threads;
 
+
 /**
  * Controls mirroring for a single remote cluster.
  */
@@ -91,6 +95,8 @@ private:
    * @endverbatim
    */
 
+  struct RemotePoolPollerListener;
+
   int init_rados(const std::string &cluster_name,
                  const std::string &client_name,
                  const std::string &mon_host,
@@ -182,6 +188,8 @@ private:
     m_threads->work_queue->queue(on_lock);
   }
 
+  void handle_remote_pool_meta_updated(const RemotePoolMeta& remote_pool_meta);
+
   Threads<ImageCtxT> *m_threads;
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   journal::CacheManagerHandler *m_cache_manager_handler;
@@ -204,6 +212,10 @@ private:
 
   std::string m_local_mirror_uuid;
 
+  RemotePoolMeta m_remote_pool_meta;
+  std::unique_ptr<remote_pool_poller::Listener> m_remote_pool_poller_listener;
+  std::unique_ptr<RemotePoolPoller<ImageCtxT>> m_remote_pool_poller;
+
   std::unique_ptr<NamespaceReplayer<ImageCtxT>> m_default_namespace_replayer;
   std::map<std::string, NamespaceReplayer<ImageCtxT> *> m_namespace_replayers;
 

diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc
index 74fe318ebcae51b33f442399a2ca62c001d45462..5377e7aa192f6dd611205fa5beea0d64885ae585 100644 (file)
--- a/src/tools/rbd_mirror/Types.cc
+++ b/src/tools/rbd_mirror/Types.cc
@@ -11,6 +11,12 @@ std::ostream &operator<<(std::ostream &os, const ImageId &image_id) {
             << "id=" << image_id.id;
 }
 
+std::ostream& operator<<(std::ostream& lhs,
+                         const RemotePoolMeta& rhs) {
+  return lhs << "mirror_uuid=" << rhs.mirror_uuid << ", "
+                "mirror_pool_uuid=" << rhs.mirror_peer_uuid;
+}
+
 std::ostream& operator<<(std::ostream& lhs, const PeerSpec &peer) {
   return lhs << "uuid: " << peer.uuid
             << " cluster: " << peer.cluster_name

diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h
index 09f18d7ac6f7a4925959868c7e2c050595d6c9d5..63e19586417e419718b384c2e0829f7cab98e8ed 100644 (file)
--- a/src/tools/rbd_mirror/Types.h
+++ b/src/tools/rbd_mirror/Types.h
@@ -65,6 +65,9 @@ struct RemotePoolMeta {
   std::string mirror_peer_uuid;
 };
 
+std::ostream& operator<<(std::ostream& lhs,
+                         const RemotePoolMeta& remote_pool_meta);
+
 template <typename I>
 struct Peer {
   std::string peer_uuid;