rbd-mirror: propagate image delete if leader was offline

In an active/active scenario, if the leader was offline while mirroring for
a remote image was disabled, the assigned replayer instance may not detect
the image removal.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
index 7b57b9a5ac64ca13df47f1cf506b36c6e00d6f9d..68692f1f75b8484b017b432ee13179d0eeed9494 100644 (file)
--- a/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_InstanceReplayer.cc
@@ -189,7 +189,6 @@ TEST_F(TestMockInstanceReplayer, AcquireReleaseImage) {
 
   C_SaferCond on_acquire;
   EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
-  EXPECT_CALL(mock_image_replayer, set_finished(false));
   EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
   EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
   EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
@@ -259,7 +258,6 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) {
 
   C_SaferCond on_acquire;
   EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
-  EXPECT_CALL(mock_image_replayer, set_finished(false));
   EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
   EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
   EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
@@ -306,5 +304,63 @@ TEST_F(TestMockInstanceReplayer, RemoveFinishedImage) {
   ASSERT_TRUE(timer_ctx2 != nullptr);
   delete timer_ctx2;
 }
+
+TEST_F(TestMockInstanceReplayer, Reacquire) {
+  MockThreads mock_threads(m_threads);
+  MockServiceDaemon mock_service_daemon;
+  MockInstanceWatcher mock_instance_watcher;
+  MockImageReplayer mock_image_replayer;
+  MockInstanceReplayer instance_replayer(
+    &mock_threads, &mock_service_daemon,
+    rbd::mirror::RadosRef(new librados::Rados(m_local_io_ctx)),
+    "local_mirror_uuid", m_local_io_ctx.get_id());
+  std::string global_image_id("global_image_id");
+
+  EXPECT_CALL(mock_image_replayer, get_global_image_id())
+    .WillRepeatedly(ReturnRef(global_image_id));
+
+  InSequence seq;
+  expect_work_queue(mock_threads);
+  Context *timer_ctx = nullptr;
+  expect_add_event_after(mock_threads, &timer_ctx);
+  instance_replayer.init();
+  instance_replayer.add_peer("peer_uuid", m_remote_io_ctx);
+
+  // Acquire
+
+  EXPECT_CALL(mock_image_replayer, add_peer("peer_uuid", _));
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  EXPECT_CALL(mock_image_replayer, is_blacklisted()).WillOnce(Return(false));
+  EXPECT_CALL(mock_image_replayer, is_finished()).WillOnce(Return(false));
+  EXPECT_CALL(mock_image_replayer, start(nullptr, false));
+  expect_work_queue(mock_threads);
+
+  C_SaferCond on_acquire1;
+  instance_replayer.acquire_image(&mock_instance_watcher, global_image_id,
+                                  &on_acquire1);
+  ASSERT_EQ(0, on_acquire1.wait());
+
+  // Re-acquire
+  EXPECT_CALL(mock_image_replayer, set_finished(false));
+  EXPECT_CALL(mock_image_replayer, restart());
+  expect_work_queue(mock_threads);
+
+  C_SaferCond on_acquire2;
+  instance_replayer.acquire_image(&mock_instance_watcher, global_image_id,
+                                  &on_acquire2);
+  ASSERT_EQ(0, on_acquire2.wait());
+
+  expect_work_queue(mock_threads);
+  expect_cancel_event(mock_threads, true);
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  expect_work_queue(mock_threads);
+  expect_work_queue(mock_threads);
+  EXPECT_CALL(mock_image_replayer, is_stopped()).WillOnce(Return(true));
+  EXPECT_CALL(mock_image_replayer, destroy());
+  instance_replayer.shut_down();
+  ASSERT_TRUE(timer_ctx != nullptr);
+  delete timer_ctx;
+}
+
 } // namespace mirror
 } // namespace rbd

diff --git a/src/tools/rbd_mirror/InstanceReplayer.cc b/src/tools/rbd_mirror/InstanceReplayer.cc
index 32ac77e4330c55012580ef4bcdf6a205cee95c72..0e21bf5f97a356d11c290abde7ec0b665cd978e7 100644 (file)
--- a/src/tools/rbd_mirror/InstanceReplayer.cc
+++ b/src/tools/rbd_mirror/InstanceReplayer.cc
@@ -155,13 +155,16 @@ void InstanceReplayer<I>::acquire_image(InstanceWatcher<I> *instance_watcher,
     assert(m_peers.size() == 1);
     auto peer = *m_peers.begin();
     image_replayer->add_peer(peer.peer_uuid, peer.io_ctx);
+    start_image_replayer(image_replayer);
+  } else {
+    // A duplicate acquire notification implies (1) connection hiccup or
+    // (2) new leader election. For the second case, restart the replayer to
+    // detect if the image has been deleted while the leader was offline
+    auto& image_replayer = it->second;
+    image_replayer->set_finished(false);
+    image_replayer->restart();
   }
 
-  auto& image_replayer = it->second;
-  // TODO temporary until policy integrated
-  image_replayer->set_finished(false);
-
-  start_image_replayer(image_replayer);
   m_threads->work_queue->queue(on_finish, 0);
 }