From 238e284a614bdb37496c45e3d5e9336ce283b0eb Mon Sep 17 00:00:00 2001
From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 27 Feb 2020 15:58:59 -0500
Subject: [PATCH] rbd-mirror: simple snapshot split-brain detection

If the local image has a primary demotion and the remote does not
have a matching non-primary demotion snapshot, we should throw a
split-brain error.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---
 .../snapshot/test_mock_Replayer.cc            | 62 +++++++++++++++++++
 .../image_replayer/snapshot/Replayer.cc       | 10 +++
 2 files changed, 72 insertions(+)

diff --git a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
index 49d4b3884ce4e..47e232e9f064c 100644
--- a/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
+++ b/src/test/rbd_mirror/image_replayer/snapshot/test_mock_Replayer.cc
@@ -1759,6 +1759,68 @@ TEST_F(TestMockImageReplayerSnapshotReplayer, UnlinkPeerError) {
                                         mock_remote_image_ctx));
 }
 
+TEST_F(TestMockImageReplayerSnapshotReplayer, SplitBrain) {
+  librbd::MockTestImageCtx mock_local_image_ctx{*m_local_image_ctx};
+  librbd::MockTestImageCtx mock_remote_image_ctx{*m_remote_image_ctx};
+
+  MockThreads mock_threads(m_threads);
+  expect_work_queue_repeatedly(mock_threads);
+
+  MockReplayerListener mock_replayer_listener;
+  expect_notification(mock_threads, mock_replayer_listener);
+
+  InSequence seq;
+
+  MockImageMeta mock_image_meta;
+  MockStateBuilder mock_state_builder(mock_local_image_ctx,
+                                      mock_remote_image_ctx,
+                                      mock_image_meta);
+  MockReplayer mock_replayer{&mock_threads, "local mirror uuid",
+                             &m_pool_meta_cache, &mock_state_builder,
+                             &mock_replayer_listener};
+  m_pool_meta_cache.set_remote_pool_meta(
+    m_remote_io_ctx.get_id(),
+    {"remote mirror uuid", "remote mirror peer uuid"});
+
+  librbd::UpdateWatchCtx* update_watch_ctx = nullptr;
+  ASSERT_EQ(0, init_entry_replayer(mock_replayer, mock_threads,
+                                   mock_local_image_ctx,
+                                   mock_remote_image_ctx,
+                                   mock_replayer_listener,
+                                   mock_image_meta,
+                                   &update_watch_ctx));
+
+  // inject a primary demote to local image
+  mock_remote_image_ctx.snap_info = {
+    {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
+       cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY, {"remote mirror peer uuid"},
+       "", 0U, true, 0, {}},
+     0, {}, 0, 0, {}}}};
+  mock_local_image_ctx.snap_info = {
+    {1U, librbd::SnapInfo{"snap1", cls::rbd::MirrorSnapshotNamespace{
+       cls::rbd::MIRROR_SNAPSHOT_STATE_PRIMARY_DEMOTED, {}, "", 0U, true, 0,
+       {}},
+     0, {}, 0, 0, {}}}};
+
+  // detect split-brain
+  expect_load_image_meta(mock_image_meta, false, 0);
+  expect_is_refresh_required(mock_local_image_ctx, false);
+  expect_is_refresh_required(mock_remote_image_ctx, false);
+
+  // wake-up replayer
+  update_watch_ctx->handle_notify();
+
+  // wait for sync to complete and expect replay complete
+  ASSERT_EQ(0, wait_for_notification(1));
+  ASSERT_FALSE(mock_replayer.is_replaying());
+  ASSERT_EQ(-EEXIST, mock_replayer.get_error_code());
+  ASSERT_EQ(std::string{"split-brain"}, mock_replayer.get_error_description());
+
+  ASSERT_EQ(0, shut_down_entry_replayer(mock_replayer, mock_threads,
+                                        mock_local_image_ctx,
+                                        mock_remote_image_ctx));
+}
+
 } // namespace snapshot
 } // namespace image_replayer
 } // namespace mirror
diff --git a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
index 5b5c038df4e82..3d5003a0c1b52 100644
--- a/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
+++ b/src/tools/rbd_mirror/image_replayer/snapshot/Replayer.cc
@@ -464,6 +464,7 @@ void Replayer<I>::scan_remote_mirror_snapshots(
   // reset state in case new snapshot is added while we are scanning
   m_image_updated = false;
 
+  bool split_brain = false;
   bool remote_demoted = false;
   auto remote_image_ctx = m_state_builder->remote_image_ctx;
   std::shared_lock image_locker{remote_image_ctx->image_lock};
@@ -522,11 +523,13 @@ void Replayer<I>::scan_remote_mirror_snapshots(
                    << "remote_snap_id=" << remote_snap_id << ", "
                    << "local_snap_id=" << m_local_snap_id_start << dendl;
           m_remote_snap_id_start = remote_snap_id;
+          split_brain = false;
           continue;
         } else if (m_remote_snap_id_start == 0) {
           // still looking for our matching demotion snapshot
           dout(15) << "skipping remote snapshot " << remote_snap_id << " "
                    << "while searching for demotion" << dendl;
+          split_brain = true;
           continue;
         }
       } else {
@@ -590,6 +593,13 @@ void Replayer<I>::scan_remote_mirror_snapshots(
 
   if (is_replay_interrupted(locker)) {
     return;
+  } else if (split_brain) {
+    derr << "split-brain detected: failed to find matching non-primary "
+         << "snapshot in remote image: "
+         << "local_snap_id_start=" << m_local_snap_id_start << ", "
+         << "local_snap_ns=" << m_local_mirror_snap_ns << dendl;
+    handle_replay_complete(locker, -EEXIST, "split-brain");
+    return;
   } else if (remote_demoted) {
     dout(10) << "remote image demoted" << dendl;
     handle_replay_complete(locker, -EREMOTEIO, "remote image demoted");
-- 
2.39.5