From: Jason Dillaman <dillaman@redhat.com>
Date: Wed, 15 Jan 2020 20:09:05 +0000 (-0500)
Subject: rbd-mirror: pool replayer should instantiate the remote pool poller
X-Git-Tag: v15.1.0~175^2~4
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ec6c26f7456579e5772bc8c461f1b6f3e901f033;p=ceph.git

rbd-mirror: pool replayer should instantiate the remote pool poller

Let the poller pull the metadata from the remote before advancing to
the leader watcher initialization. If the remote metadata changes
during runtime, stop the pool replayer so that it can be
re-initialized.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
index 756d3906490f..040ce633a6a1 100644
--- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
@@ -14,6 +14,7 @@
 #include "tools/rbd_mirror/LeaderWatcher.h"
 #include "tools/rbd_mirror/NamespaceReplayer.h"
 #include "tools/rbd_mirror/PoolReplayer.h"
+#include "tools/rbd_mirror/RemotePoolPoller.h"
 #include "tools/rbd_mirror/ServiceDaemon.h"
 #include "tools/rbd_mirror/Threads.h"
 #include "common/Formatter.h"
@@ -200,6 +201,33 @@ struct LeaderWatcher<librbd::MockTestImageCtx> {
 
 LeaderWatcher<librbd::MockTestImageCtx>* LeaderWatcher<librbd::MockTestImageCtx>::s_instance = nullptr;
 
+template<>
+struct RemotePoolPoller<librbd::MockTestImageCtx> {
+  static RemotePoolPoller* s_instance;
+
+  remote_pool_poller::Listener* listener = nullptr;
+
+  static RemotePoolPoller* create(
+      Threads<librbd::MockTestImageCtx>* threads,
+      librados::IoCtx& remote_io_ctx,
+      const std::string& local_site_name,
+      const std::string& local_fsid,
+      remote_pool_poller::Listener& listener) {
+    ceph_assert(s_instance != nullptr);
+    s_instance->listener = &listener;
+    return s_instance;
+  }
+
+  MOCK_METHOD1(init, void(Context*));
+  MOCK_METHOD1(shut_down, void(Context*));
+
+  RemotePoolPoller() {
+    s_instance = this;
+  }
+};
+
+RemotePoolPoller<librbd::MockTestImageCtx>* RemotePoolPoller<librbd::MockTestImageCtx>::s_instance = nullptr;
+
 template<>
 struct ServiceDaemon<librbd::MockTestImageCtx> {
   MOCK_METHOD2(add_namespace, void(int64_t, const std::string &));
@@ -260,6 +288,7 @@ public:
   typedef PoolReplayer<librbd::MockTestImageCtx> MockPoolReplayer;
   typedef Throttler<librbd::MockTestImageCtx> MockThrottler;
   typedef NamespaceReplayer<librbd::MockTestImageCtx> MockNamespaceReplayer;
+  typedef RemotePoolPoller<librbd::MockTestImageCtx> MockRemotePoolPoller;
   typedef LeaderWatcher<librbd::MockTestImageCtx> MockLeaderWatcher;
   typedef ServiceDaemon<librbd::MockTestImageCtx> MockServiceDaemon;
   typedef Threads<librbd::MockTestImageCtx> MockThreads;
@@ -353,6 +382,31 @@ public:
       .Times(AtLeast(0));
   }
 
+  void expect_remote_pool_poller_init(
+      MockRemotePoolPoller& mock_remote_pool_poller,
+      const RemotePoolMeta& remote_pool_meta, int r) {
+    EXPECT_CALL(mock_remote_pool_poller, init(_))
+      .WillOnce(Invoke(
+                  [this, &mock_remote_pool_poller, remote_pool_meta, r]
+                  (Context* ctx) {
+                    if (r >= 0) {
+                      mock_remote_pool_poller.listener->handle_updated(
+                        remote_pool_meta);
+                    }
+
+                    m_threads->work_queue->queue(ctx, r);
+                }));
+  }
+
+  void expect_remote_pool_poller_shut_down(
+      MockRemotePoolPoller& mock_remote_pool_poller, int r) {
+    EXPECT_CALL(mock_remote_pool_poller, shut_down(_))
+      .WillOnce(Invoke(
+                  [this, r](Context* ctx) {
+                    m_threads->work_queue->queue(ctx, r);
+                }));
+  }
+
   void expect_namespace_replayer_is_blacklisted(
       MockNamespaceReplayer &mock_namespace_replayer,
       bool blacklisted) {
@@ -504,6 +558,9 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
 
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -523,6 +580,7 @@ TEST_F(TestMockPoolReplayer, ConfigKeyOverride) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -560,6 +618,9 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
 
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -592,6 +653,7 @@ TEST_F(TestMockPoolReplayer, AcquireReleaseLeader) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -642,6 +704,9 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
                  nullptr);
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -707,6 +772,7 @@ TEST_F(TestMockPoolReplayer, Namespaces) {
   expect_namespace_replayer_shut_down(*mock_ns1_namespace_replayer);
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
@@ -754,6 +820,9 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
                  nullptr);
   expect_create_ioctx(mock_local_rados_client, mock_local_io_ctx);
   expect_mirror_uuid_get(mock_local_io_ctx, "uuid", 0);
+  auto mock_remote_pool_poller = new MockRemotePoolPoller();
+  expect_remote_pool_poller_init(*mock_remote_pool_poller,
+                                 {"remote mirror uuid", ""}, 0);
   expect_namespace_replayer_init(*mock_default_namespace_replayer, 0);
   expect_leader_watcher_init(*mock_leader_watcher, 0);
 
@@ -837,6 +906,7 @@ TEST_F(TestMockPoolReplayer, NamespacesError) {
 
   expect_leader_watcher_shut_down(*mock_leader_watcher);
   expect_namespace_replayer_shut_down(*mock_default_namespace_replayer);
+  expect_remote_pool_poller_shut_down(*mock_remote_pool_poller, 0);
 
   pool_replayer.shut_down();
 }
diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index b99070bd5b96..55a522800921 100644
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -15,6 +15,7 @@
 #include "global/global_context.h"
 #include "librbd/api/Config.h"
 #include "librbd/api/Namespace.h"
+#include "RemotePoolPoller.h"
 #include "ServiceDaemon.h"
 #include "Threads.h"
 
@@ -205,6 +206,21 @@ private:
 
 } // anonymous namespace
 
+template <typename I>
+struct PoolReplayer<I>::RemotePoolPollerListener
+  : public remote_pool_poller::Listener {
+
+  PoolReplayer<I>* m_pool_replayer;
+
+  RemotePoolPollerListener(PoolReplayer<I>* pool_replayer)
+    : m_pool_replayer(pool_replayer) {
+  }
+
+  void handle_updated(const RemotePoolMeta& remote_pool_meta) override {
+    m_pool_replayer->handle_remote_pool_meta_updated(remote_pool_meta);
+  }
+};
+
 template <typename I>
 PoolReplayer<I>::PoolReplayer(
     Threads<I> *threads, ServiceDaemon<I> *service_daemon,
@@ -319,6 +335,32 @@ void PoolReplayer<I>::init(const std::string& site_name) {
   m_image_deletion_throttler.reset(
       Throttler<I>::create(cct, "rbd_mirror_concurrent_image_deletions"));
 
+  std::string local_fsid;
+  r = m_local_rados->cluster_fsid(&local_fsid);
+  if (r < 0) {
+    derr << "failed to retrieve local fsid: " << cpp_strerror(r) << dendl;
+    return;
+  }
+
+  m_remote_pool_poller_listener.reset(new RemotePoolPollerListener(this));
+  m_remote_pool_poller.reset(RemotePoolPoller<I>::create(
+    m_threads, m_remote_io_ctx, m_site_name, local_fsid,
+    *m_remote_pool_poller_listener));
+
+  C_SaferCond on_pool_poller_init;
+  m_remote_pool_poller->init(&on_pool_poller_init);
+  r = on_pool_poller_init.wait();
+  if (r < 0) {
+    derr << "failed to initialize remote pool poller: " << cpp_strerror(r)
+         << dendl;
+    m_callout_id = m_service_daemon->add_or_update_callout(
+      m_local_pool_id, m_callout_id, service_daemon::CALLOUT_LEVEL_ERROR,
+      "unable to initialize remote pool poller");
+    m_remote_pool_poller.reset();
+    return;
+  }
+  ceph_assert(!m_remote_pool_meta.mirror_uuid.empty());
+
   m_default_namespace_replayer.reset(NamespaceReplayer<I>::create(
       "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
       m_site_name, m_threads, m_image_sync_throttler.get(),
@@ -385,6 +427,14 @@ void PoolReplayer<I>::shut_down() {
   }
   m_default_namespace_replayer.reset();
 
+  if (m_remote_pool_poller) {
+    C_SaferCond ctx;
+    m_remote_pool_poller->shut_down(&ctx);
+    ctx.wait();
+  }
+  m_remote_pool_poller.reset();
+  m_remote_pool_poller_listener.reset();
+
   m_image_sync_throttler.reset();
   m_image_deletion_throttler.reset();
 
@@ -1024,6 +1074,22 @@ void PoolReplayer<I>::handle_instances_removed(
   }
 }
 
+template <typename I>
+void PoolReplayer<I>::handle_remote_pool_meta_updated(
+    const RemotePoolMeta& remote_pool_meta) {
+  dout(5) << "remote_pool_meta=" << remote_pool_meta << dendl;
+
+  if (!m_default_namespace_replayer) {
+    m_remote_pool_meta = remote_pool_meta;
+    return;
+  }
+
+  derr << "remote pool metadata updated unexpectedly" << dendl;
+  std::unique_lock locker{m_lock};
+  m_stopping = true;
+  m_cond.notify_all();
+}
+
 } // namespace mirror
 } // namespace rbd
 
diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h
index 9f295f4788ca..1948eea60342 100644
--- a/src/tools/rbd_mirror/PoolReplayer.h
+++ b/src/tools/rbd_mirror/PoolReplayer.h
@@ -31,9 +31,13 @@ namespace librbd { class ImageCtx; }
 namespace rbd {
 namespace mirror {
 
+template <typename> class RemotePoolPoller;
+namespace remote_pool_poller { struct Listener; }
+
 template <typename> class ServiceDaemon;
 template <typename> struct Threads;
 
+
 /**
  * Controls mirroring for a single remote cluster.
  */
@@ -91,6 +95,8 @@ private:
    * @endverbatim
    */
 
+  struct RemotePoolPollerListener;
+
   int init_rados(const std::string &cluster_name,
                  const std::string &client_name,
                  const std::string &mon_host,
@@ -182,6 +188,8 @@ private:
     m_threads->work_queue->queue(on_lock);
   }
 
+  void handle_remote_pool_meta_updated(const RemotePoolMeta& remote_pool_meta);
+
   Threads<ImageCtxT> *m_threads;
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   journal::CacheManagerHandler *m_cache_manager_handler;
@@ -204,6 +212,10 @@ private:
 
   std::string m_local_mirror_uuid;
 
+  RemotePoolMeta m_remote_pool_meta;
+  std::unique_ptr<remote_pool_poller::Listener> m_remote_pool_poller_listener;
+  std::unique_ptr<RemotePoolPoller<ImageCtxT>> m_remote_pool_poller;
+
   std::unique_ptr<NamespaceReplayer<ImageCtxT>> m_default_namespace_replayer;
   std::map<std::string, NamespaceReplayer<ImageCtxT> *> m_namespace_replayers;
 
diff --git a/src/tools/rbd_mirror/Types.cc b/src/tools/rbd_mirror/Types.cc
index 74fe318ebcae..5377e7aa192f 100644
--- a/src/tools/rbd_mirror/Types.cc
+++ b/src/tools/rbd_mirror/Types.cc
@@ -11,6 +11,12 @@ std::ostream &operator<<(std::ostream &os, const ImageId &image_id) {
             << "id=" << image_id.id;
 }
 
+std::ostream& operator<<(std::ostream& lhs,
+                         const RemotePoolMeta& rhs) {
+  return lhs << "mirror_uuid=" << rhs.mirror_uuid << ", "
+                "mirror_pool_uuid=" << rhs.mirror_peer_uuid;
+}
+
 std::ostream& operator<<(std::ostream& lhs, const PeerSpec &peer) {
   return lhs << "uuid: " << peer.uuid
 	     << " cluster: " << peer.cluster_name
diff --git a/src/tools/rbd_mirror/Types.h b/src/tools/rbd_mirror/Types.h
index 09f18d7ac6f7..63e19586417e 100644
--- a/src/tools/rbd_mirror/Types.h
+++ b/src/tools/rbd_mirror/Types.h
@@ -65,6 +65,9 @@ struct RemotePoolMeta {
   std::string mirror_peer_uuid;
 };
 
+std::ostream& operator<<(std::ostream& lhs,
+                         const RemotePoolMeta& remote_pool_meta);
+
 template <typename I>
 struct Peer {
   std::string peer_uuid;