rbd-mirror: mirror status updater optionally pings peer

This will register the local site as a peer in the remote for
TX-only scenarios.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index aa2abfd64ef73091b7794bc678814061b7eaaaaa..8bf1724f6daf4ef3f1dca10318df6465f52f50c1 100644 (file)
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -132,7 +132,7 @@ public:
     EXPECT_EQ(0, m_local_ioctx.create(RBD_MIRRORING, false));
 
     m_local_status_updater = rbd::mirror::MirrorStatusUpdater<>::create(
-      m_local_ioctx, m_threads.get());
+      m_local_ioctx, m_threads.get(), "");
     C_SaferCond status_updater_ctx;
     m_local_status_updater->init(&status_updater_ctx);
     EXPECT_EQ(0, status_updater_ctx.wait());

diff --git a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc
index b9dd01b5da1256bb399c473fa2aabf14a5013793..ae379b60bb2e0644fd5e7cf4029f992f696f6c08 100644 (file)
--- a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc
+++ b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc
@@ -167,7 +167,8 @@ public:
   }
 
   void expect_mirror_status_update(
-      const MirrorImageSiteStatuses& mirror_image_site_statuses, int r) {
+      const MirrorImageSiteStatuses& mirror_image_site_statuses,
+      const std::string& site_name, const std::string& fsid, int r) {
     EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _))
       .WillOnce(Invoke([this](auto&&... args) {
           int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...);
@@ -175,8 +176,21 @@ public:
           return r;
         }));
 
-    for (auto& [global_image_id, mirror_image_status] :
+    if (!site_name.empty()) {
+      // status updates to remote site include ping
+      bufferlist in_bl;
+      encode(site_name, in_bl);
+      encode(fsid, in_bl);
+      encode(static_cast<uint8_t>(cls::rbd::MIRROR_PEER_DIRECTION_TX), in_bl);
+      EXPECT_CALL(*m_mock_local_io_ctx,
+                  exec(RBD_MIRRORING, _, StrEq("rbd"),
+                       StrEq("mirror_peer_ping"), ContentsEqual(in_bl), _, _))
+        .WillOnce(Return(0));
+    }
+
+    for (auto [global_image_id, mirror_image_status] :
            mirror_image_site_statuses) {
+      mirror_image_status.fsid = fsid;
       expect_mirror_status_update(global_image_id, mirror_image_status, r);
       if (r < 0) {
         break;
@@ -233,7 +247,7 @@ public:
 
 TEST_F(TestMockMirrorStatusUpdater, InitShutDown) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -247,7 +261,7 @@ TEST_F(TestMockMirrorStatusUpdater, InitShutDown) {
 
 TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -264,7 +278,7 @@ TEST_F(TestMockMirrorStatusUpdater, InitStatusWatcherError) {
 
 TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -283,7 +297,7 @@ TEST_F(TestMockMirrorStatusUpdater, ShutDownStatusWatcherError) {
 
 TEST_F(TestMockMirrorStatusUpdater, SmallBatch) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -305,7 +319,7 @@ TEST_F(TestMockMirrorStatusUpdater, SmallBatch) {
   Context* update_task = nullptr;
   fire_timer_event(&timer_event, &update_task);
 
-  expect_mirror_status_update(mirror_image_site_statuses, 0);
+  expect_mirror_status_update(mirror_image_site_statuses, "", "", 0);
   update_task->complete(0);
 
   shut_down_mirror_status_updater(mock_mirror_status_updater,
@@ -314,7 +328,7 @@ TEST_F(TestMockMirrorStatusUpdater, SmallBatch) {
 
 TEST_F(TestMockMirrorStatusUpdater, LargeBatch) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -345,8 +359,8 @@ TEST_F(TestMockMirrorStatusUpdater, LargeBatch) {
   Context* update_task = nullptr;
   fire_timer_event(&timer_event, &update_task);
 
-  expect_mirror_status_update(mirror_image_site_statuses_1, 0);
-  expect_mirror_status_update(mirror_image_site_statuses_2, 0);
+  expect_mirror_status_update(mirror_image_site_statuses_1, "", "", 0);
+  expect_mirror_status_update(mirror_image_site_statuses_2, "", "", 0);
   update_task->complete(0);
 
   shut_down_mirror_status_updater(mock_mirror_status_updater,
@@ -355,7 +369,7 @@ TEST_F(TestMockMirrorStatusUpdater, LargeBatch) {
 
 TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -375,7 +389,8 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) {
 
   expect_mirror_status_update(
     {{"1", cls::rbd::MirrorImageSiteStatus{
-        "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, 0);
+        "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}},
+    "", "", 0);
   update_task->complete(0);
 
   shut_down_mirror_status_updater(mock_mirror_status_updater,
@@ -384,7 +399,7 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) {
 
 TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -414,7 +429,8 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) {
   expect_work_queue(false);
   expect_mirror_status_update(
     {{"1", cls::rbd::MirrorImageSiteStatus{
-        "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, 0);
+        "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}},
+    "", "", 0);
 
   update_task->complete(0);
 
@@ -424,7 +440,7 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) {
 
 TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -435,7 +451,8 @@ TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) {
                              *mock_mirror_status_watcher, &timer_event);
 
   expect_work_queue(false);
-  expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}}, 0);
+  expect_mirror_status_update({{"1", cls::rbd::MirrorImageSiteStatus{}}},
+                              "", "", 0);
   mock_mirror_status_updater.set_mirror_image_status("1", {}, true);
 
   shut_down_mirror_status_updater(mock_mirror_status_updater,
@@ -444,7 +461,7 @@ TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) {
 
 TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -467,7 +484,7 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) {
 
 TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -501,7 +518,7 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) {
 
 TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) {
   MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
-                                                     m_mock_threads);
+                                                     m_mock_threads, "");
   MockMirrorStatusWatcher* mock_mirror_status_watcher =
     new MockMirrorStatusWatcher();
 
@@ -535,5 +552,39 @@ TEST_F(TestMockMirrorStatusUpdater, ShutDownWhileUpdating) {
   ASSERT_EQ(0, on_shutdown.wait());
 }
 
+TEST_F(TestMockMirrorStatusUpdater, MirrorPeerSitePing) {
+  std::string fsid;
+  ASSERT_EQ(0, _rados->cluster_fsid(&fsid));
+
+  MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx,
+                                                     m_mock_threads, "siteA");
+  MockMirrorStatusWatcher* mock_mirror_status_watcher =
+    new MockMirrorStatusWatcher();
+
+  InSequence seq;
+
+  Context* timer_event = nullptr;
+  init_mirror_status_updater(mock_mirror_status_updater,
+                             *mock_mirror_status_watcher, &timer_event);
+
+  MirrorImageSiteStatuses mirror_image_site_statuses;
+  for (auto i = 0; i < 100; ++i) {
+    auto pair = mirror_image_site_statuses.emplace(
+      stringify(i), cls::rbd::MirrorImageSiteStatus{});
+    mock_mirror_status_updater.set_mirror_image_status(pair.first->first,
+                                                       pair.first->second,
+                                                       false);
+  }
+
+  Context* update_task = nullptr;
+  fire_timer_event(&timer_event, &update_task);
+
+  expect_mirror_status_update(mirror_image_site_statuses, "siteA", fsid, 0);
+  update_task->complete(0);
+
+  shut_down_mirror_status_updater(mock_mirror_status_updater,
+                                  *mock_mirror_status_watcher);
+}
+
 } // namespace mirror
 } // namespace rbd

diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
index bd52f9218145c95585c47fcce0bebdff2b952709..1dcf61abb453be45c6357d1ddb970319f9ed812f 100644 (file)
--- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
@@ -174,7 +174,8 @@ struct MirrorStatusUpdater<librbd::MockTestImageCtx> {
   static MirrorStatusUpdater* s_instance;
 
   static MirrorStatusUpdater *create(librados::IoCtx &io_ctx,
-                                     Threads<librbd::MockTestImageCtx> *threads) {
+                                     Threads<librbd::MockTestImageCtx> *threads,
+                                     const std::string& site_name) {
     ceph_assert(s_instance != nullptr);
     return s_instance;
   }

diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc
index 43bfdf4ca97ebaf6df3eb2b88d88b67e84569014..0fd042e828dcdab5e34d819a614ac76420476e3c 100644 (file)
--- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc
+++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc
@@ -30,11 +30,13 @@ using librbd::util::create_rados_callback;
 
 template <typename I>
 MirrorStatusUpdater<I>::MirrorStatusUpdater(
-    librados::IoCtx& io_ctx, Threads<I> *threads)
-  : m_io_ctx(io_ctx), m_threads(threads),
+    librados::IoCtx& io_ctx, Threads<I> *threads,
+    const std::string& site_name)
+  : m_io_ctx(io_ctx), m_threads(threads), m_site_name(site_name),
     m_lock(ceph::make_mutex("rbd::mirror::MirrorStatusUpdater " +
                             stringify(m_io_ctx.get_id()))) {
-  dout(10) << "pool_id=" << m_io_ctx.get_id() << dendl;
+  dout(10) << "site_name=" << site_name << ", "
+           << "pool_id=" << m_io_ctx.get_id() << dendl;
 }
 
 template <typename I>
@@ -47,6 +49,16 @@ template <typename I>
 void MirrorStatusUpdater<I>::init(Context* on_finish) {
   dout(10) << dendl;
 
+  if (!m_site_name.empty()) {
+    librados::Rados rados(m_io_ctx);
+    int r = rados.cluster_fsid(&m_fsid);
+    if (r < 0) {
+      derr << "failed to retrieve fsid: " << cpp_strerror(r) << dendl;
+      m_threads->work_queue->queue(on_finish, r);
+      return;
+    }
+  }
+
   ceph_assert(!m_initialized);
   m_initialized = true;
 
@@ -299,6 +311,12 @@ void MirrorStatusUpdater<I>::update_task(int r) {
     librados::ObjectWriteOperation op;
     uint32_t op_count = 0;
 
+    if (!m_site_name.empty()) {
+      // updates to remote sites should include local site name
+      // to ensure status includes this peer
+      librbd::cls_client::mirror_peer_ping(&op, m_site_name, m_fsid);
+    }
+
     while (it != updating_global_image_ids.end() &&
            op_count < MAX_UPDATES_PER_OP) {
       auto& global_image_id = *it;
@@ -309,6 +327,7 @@ void MirrorStatusUpdater<I>::update_task(int r) {
         continue;
       }
 
+      status_it->second.fsid = m_fsid;
       librbd::cls_client::mirror_image_status_set(&op, global_image_id,
                                                   status_it->second);
       ++op_count;

diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h
index 2f23a15fde03b3ef3e6771e9df6bc27d9e908ea7..90e4697953c19c447512c5837a5cf986fc3ad15a 100644 (file)
--- a/src/tools/rbd_mirror/MirrorStatusUpdater.h
+++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h
@@ -26,11 +26,13 @@ class MirrorStatusUpdater {
 public:
 
   static MirrorStatusUpdater* create(librados::IoCtx& io_ctx,
-                                     Threads<ImageCtxT> *threads) {
-    return new MirrorStatusUpdater(io_ctx, threads);
+                                     Threads<ImageCtxT> *threads,
+                                     const std::string& site_name) {
+    return new MirrorStatusUpdater(io_ctx, threads, site_name);
   }
 
-  MirrorStatusUpdater(librados::IoCtx& io_ctx, Threads<ImageCtxT> *threads);
+  MirrorStatusUpdater(librados::IoCtx& io_ctx, Threads<ImageCtxT> *threads,
+                      const std::string& site_name);
   ~MirrorStatusUpdater();
 
   void init(Context* on_finish);
@@ -68,6 +70,8 @@ private:
 
   librados::IoCtx m_io_ctx;
   Threads<ImageCtxT>* m_threads;
+  std::string m_site_name;
+  std::string m_fsid;
 
   Context* m_timer_task = nullptr;
 

diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc
index 8919945cad1345d1616c99c3814cb74b60f1a78f..d3910c105b19ec52435635c5906282553f04b537 100644 (file)
--- a/src/tools/rbd_mirror/NamespaceReplayer.cc
+++ b/src/tools/rbd_mirror/NamespaceReplayer.cc
@@ -265,7 +265,7 @@ void NamespaceReplayer<I>::init_local_status_updater() {
   ceph_assert(!m_local_status_updater);
 
   m_local_status_updater.reset(MirrorStatusUpdater<I>::create(
-    m_local_io_ctx, m_threads));
+    m_local_io_ctx, m_threads, ""));
   auto ctx = create_context_callback<
     NamespaceReplayer<I>,
     &NamespaceReplayer<I>::handle_init_local_status_updater>(this);