From: Jason Dillaman <dillaman@redhat.com>
Date: Wed, 9 May 2018 14:33:03 +0000 (-0400)
Subject: rbd-mirror: propagate image delete if leader was offline
X-Git-Tag: v14.0.0~95^2~17
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=b3acc569837985590fad2705ae2dcd7f148667a5;p=ceph-ci.git

rbd-mirror: propagate image delete if leader was offline

In an active/active scenario, if the leader was offline while mirroring for
a remote image was disabled, the assigned replayer instance may not detect
the image removal.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
index 7b57b9a5ac6..68692f1f75b 100644
--- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
@@ -189,7 +189,6 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) {
 
   C_SaferCond on_acquire;
   EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
-  EXPECT_CALL(mock_image_replayer, set_finished(false));
   EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
   EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
   EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
@@ -259,7 +258,6 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) {
 
   C_SaferCond on_acquire;
   EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
-  EXPECT_CALL(mock_image_replayer, set_finished(false));
   EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
   EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
   EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
@@ -306,5 +304,63 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) {
   ASSERT_TRUE(timer_ctx2 != nullptr);
   delete timer_ctx2;
 }
+
+TEST_F(TestMockInstanceReplayer, Reacquire) {
+  MockThreads mock_threads(m_threads);
+  MockServiceDaemon mock_service_daemon;
+  MockInstanceWatcher mock_instance_watcher;
+  MockImageReplayer mock_image_replayer;
+  MockInstanceReplayer instance_replayer(
+    &mock_threads, &mock_service_daemon,
+    rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
+    "local_mirror_uuid", m_local_io_ctx.get_id());
+  std::string global_image_id("global_image_id");
+
+  EXPECT_CALL(mock_image_replayer, get_global_image_id())
+    .WillRepeatedly(ReturnRef(global_image_id));
+
+  InSequence seq;
+  expect_work_queue(mock_threads);
+  Context *timer_ctx = nullptr;
+  expect_add_event_after(mock_threads, &timer_ctx);
+  instance_replayer.init();
+  instance_replayer.add_peer("peer_uuid", m_remote_io_ctx);
+
+  // Acquire
+
+  EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
+  EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
+  EXPECT_CALL(mock_image_replayer, start(nullptr, false));
+  expect_work_queue(mock_threads);
+
+  C_SaferCond on_acquire1;
+  instance_replayer.acquire_image(&mock_instance_watcher, global_image_id,
+                                  &on_acquire1);
+  ASSERT_EQ(0, on_acquire1.wait());
+
+  // Re-acquire
+  EXPECT_CALL(mock_image_replayer, set_finished(false));
+  EXPECT_CALL(mock_image_replayer, restart());
+  expect_work_queue(mock_threads);
+
+  C_SaferCond on_acquire2;
+  instance_replayer.acquire_image(&mock_instance_watcher, global_image_id,
+                                  &on_acquire2);
+  ASSERT_EQ(0, on_acquire2.wait());
+
+  expect_work_queue(mock_threads);
+  expect_cancel_event(mock_threads, true);
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  expect_work_queue(mock_threads);
+  expect_work_queue(mock_threads);
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  EXPECT_CALL(mock_image_replayer, destroy());
+  instance_replayer.shut_down();
+  ASSERT_TRUE(timer_ctx != nullptr);
+  delete timer_ctx;
+}
+
 } // namespace mirror
 } // namespace rbd
diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index 32ac77e4330..0e21bf5f97a 100644
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -155,13 +155,16 @@ void InstanceReplayer<I>::acquire_image(InstanceWatcher<I> *instance_watcher,
     assert(m_peers.size() == 1);
     auto peer = *m_peers.begin();
     image_replayer->add_peer(peer.peer_uuid, peer.io_ctx);
+    start_image_replayer(image_replayer);
+  } else {
+    // A duplicate acquire notification implies (1) connection hiccup or
+    // (2) new leader election. For the second case, restart the replayer to
+    // detect if the image has been deleted while the leader was offline
+    auto& image_replayer = it->second;
+    image_replayer->set_finished(false);
+    image_replayer->restart();
   }
 
-  auto& image_replayer = it->second;
-  // TODO temporary until policy integrated
-  image_replayer->set_finished(false);
-
-  start_image_replayer(image_replayer);
   m_threads->work_queue->queue(on_finish, 0);
 }