librbd: guard journal replay from losing exclusive lock

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc
index 3b5f041892109444257762b28eb6c35096d0914c..5d942980dfbcdd8c09a4aa6a432a35f6f8651738 100644 (file)
--- a/src/librbd/ExclusiveLock.cc
+++ b/src/librbd/ExclusiveLock.cc
@@ -54,7 +54,6 @@ template <typename I>
 bool ExclusiveLock<I>::accept_ops() const {
   Mutex::Locker locker(ML<I>::m_lock);
   bool accept_ops = (!ML<I>::is_state_shutdown() &&
-                     !ML<I>::is_state_pre_releasing() &&
                      (ML<I>::is_state_locked() ||
                       ML<I>::is_state_post_acquiring()));
 

diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc
index c19a6a80c50022f9b218f3a2e9625d5024052c32..aa73cdeddac73c45ffd15307a5c5430694aca329 100644 (file)
--- a/src/librbd/Journal.cc
+++ b/src/librbd/Journal.cc
@@ -1332,9 +1332,11 @@ void Journal<I>::handle_replay_process_safe(ReplayEntry replay_entry, int r) {
 
   ldout(cct, 20) << this << " " << __func__ << ": r=" << r << dendl;
   if (r < 0) {
-    lderr(cct) << this << " " << __func__ << ": "
-               << "failed to commit journal event to disk: " << cpp_strerror(r)
-               << dendl;
+    if (r != -ECANCELED) {
+      lderr(cct) << this << " " << __func__ << ": "
+                 << "failed to commit journal event to disk: "
+                 << cpp_strerror(r) << dendl;
+    }
 
     if (m_state == STATE_REPLAYING) {
       // abort the replay if we have an error

diff --git a/src/librbd/journal/Replay.cc b/src/librbd/journal/Replay.cc
index d0b585922980cd89d78d0eef10cd76d1ab458bcf..29289ac580a4885ca53e0355ba652e145df5ef2b 100644 (file)
--- a/src/librbd/journal/Replay.cc
+++ b/src/librbd/journal/Replay.cc
@@ -5,6 +5,7 @@
 #include "common/dout.h"
 #include "common/errno.h"
 #include "common/WorkQueue.h"
+#include "librbd/ExclusiveLock.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
 #include "librbd/internal.h"
@@ -118,6 +119,14 @@ struct ExecuteOp : public Context {
 
     ldout(cct, 20) << ": ExecuteOp::" << __func__ << dendl;
     RWLock::RLocker owner_locker(image_ctx.owner_lock);
+
+    if (image_ctx.exclusive_lock == nullptr ||
+        !image_ctx.exclusive_lock->accept_ops()) {
+      ldout(cct, 5) << ": lost exclusive lock -- skipping op" << dendl;
+      on_op_complete->complete(-ECANCELED);
+      return;
+    }
+
     execute(event);
   }
 };
@@ -197,6 +206,14 @@ void Replay<I>::process(const EventEntry &event_entry,
   on_ready = util::create_async_context_callback(m_image_ctx, on_ready);
 
   RWLock::RLocker owner_lock(m_image_ctx.owner_lock);
+  if (m_image_ctx.exclusive_lock == nullptr ||
+      !m_image_ctx.exclusive_lock->accept_ops()) {
+    ldout(cct, 5) << ": lost exclusive lock -- skipping event" << dendl;
+    m_image_ctx.op_work_queue->queue(on_safe, -ECANCELED);
+    on_ready->complete(0);
+    return;
+  }
+
   boost::apply_visitor(EventVisitor(this, on_ready, on_safe),
                        event_entry.event);
 }

diff --git a/src/test/librbd/journal/test_mock_Replay.cc b/src/test/librbd/journal/test_mock_Replay.cc
index 14d2a6a4b79efefeeb3d097f3f5aba6a4c3f21d6..1cc67ceac4da91de2c0fae6c77b873ebb153f99a 100644 (file)
--- a/src/test/librbd/journal/test_mock_Replay.cc
+++ b/src/test/librbd/journal/test_mock_Replay.cc
@@ -128,6 +128,11 @@ public:
   TestMockJournalReplay() : m_invoke_lock("m_invoke_lock") {
   }
 
+  void expect_accept_ops(MockExclusiveLock &mock_exclusive_lock, bool accept) {
+    EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillRepeatedly(
+      Return(accept));
+  }
+
   void expect_aio_discard(MockIoImageRequest &mock_io_image_request,
                           io::AioCompletion **aio_comp, uint64_t off,
                           uint64_t len, bool skip_partial_discard) {
@@ -333,6 +338,11 @@ TEST_F(TestMockJournalReplay, AioDiscard) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -361,6 +371,11 @@ TEST_F(TestMockJournalReplay, AioWrite) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -389,6 +404,11 @@ TEST_F(TestMockJournalReplay, AioFlush) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -415,6 +435,11 @@ TEST_F(TestMockJournalReplay, AioWriteSame) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -443,6 +468,11 @@ TEST_F(TestMockJournalReplay, IOError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -471,6 +501,11 @@ TEST_F(TestMockJournalReplay, SoftFlushIO) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -510,6 +545,11 @@ TEST_F(TestMockJournalReplay, PauseIO) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -551,6 +591,11 @@ TEST_F(TestMockJournalReplay, Flush) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -579,6 +624,11 @@ TEST_F(TestMockJournalReplay, OpFinishError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -610,6 +660,11 @@ TEST_F(TestMockJournalReplay, BlockedOpFinishError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -649,6 +704,11 @@ TEST_F(TestMockJournalReplay, MissingOpFinishEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -703,6 +763,11 @@ TEST_F(TestMockJournalReplay, MissingOpFinishEventCancelOps) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -750,6 +815,11 @@ TEST_F(TestMockJournalReplay, UnknownOpFinishEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -770,6 +840,11 @@ TEST_F(TestMockJournalReplay, OpEventError) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -806,6 +881,11 @@ TEST_F(TestMockJournalReplay, SnapCreateEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -847,6 +927,11 @@ TEST_F(TestMockJournalReplay, SnapCreateEventExists) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -884,6 +969,11 @@ TEST_F(TestMockJournalReplay, SnapRemoveEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -920,6 +1010,11 @@ TEST_F(TestMockJournalReplay, SnapRemoveEventDNE) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -956,6 +1051,11 @@ TEST_F(TestMockJournalReplay, SnapRenameEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -989,6 +1089,11 @@ TEST_F(TestMockJournalReplay, SnapRenameEventExists) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1022,6 +1127,11 @@ TEST_F(TestMockJournalReplay, SnapProtectEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1058,6 +1168,11 @@ TEST_F(TestMockJournalReplay, SnapProtectEventBusy) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1094,6 +1209,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1130,6 +1250,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectOpFinishBusy) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1162,6 +1287,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectEventInvalid) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1198,6 +1328,11 @@ TEST_F(TestMockJournalReplay, SnapRollbackEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1234,6 +1369,11 @@ TEST_F(TestMockJournalReplay, RenameEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1266,6 +1406,11 @@ TEST_F(TestMockJournalReplay, RenameEventExists) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1298,6 +1443,11 @@ TEST_F(TestMockJournalReplay, ResizeEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1335,6 +1485,11 @@ TEST_F(TestMockJournalReplay, FlattenEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1367,6 +1522,11 @@ TEST_F(TestMockJournalReplay, FlattenEventInvalid) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1402,6 +1562,11 @@ TEST_F(TestMockJournalReplay, UpdateFeaturesEvent) {
   bool enabled = !ictx->test_features(features);
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1440,6 +1605,11 @@ TEST_F(TestMockJournalReplay, MetadataSetEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1472,6 +1642,11 @@ TEST_F(TestMockJournalReplay, MetadataRemoveEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1504,6 +1679,11 @@ TEST_F(TestMockJournalReplay, MetadataRemoveEventDNE) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1536,6 +1716,11 @@ TEST_F(TestMockJournalReplay, UnknownEvent) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1562,6 +1747,11 @@ TEST_F(TestMockJournalReplay, RefreshImageBeforeOpStart) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   expect_op_work_queue(mock_image_ctx);
 
@@ -1599,6 +1789,11 @@ TEST_F(TestMockJournalReplay, FlushEventAfterShutDown) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -1620,6 +1815,11 @@ TEST_F(TestMockJournalReplay, ModifyEventAfterShutDown) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -1642,6 +1842,11 @@ TEST_F(TestMockJournalReplay, OpEventAfterShutDown) {
   ASSERT_EQ(0, open_image(m_image_name, &ictx));
 
   MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, true);
+
   MockJournalReplay mock_journal_replay(mock_image_ctx);
   MockIoImageRequest mock_io_image_request;
   expect_op_work_queue(mock_image_ctx);
@@ -1656,5 +1861,68 @@ TEST_F(TestMockJournalReplay, OpEventAfterShutDown) {
   ASSERT_EQ(-ESHUTDOWN, on_safe.wait());
 }
 
+TEST_F(TestMockJournalReplay, LockLostBeforeProcess) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, false);
+
+  MockJournalReplay mock_journal_replay(mock_image_ctx);
+  MockIoImageRequest mock_io_image_request;
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  C_SaferCond on_ready;
+  C_SaferCond on_safe;
+  when_process(mock_journal_replay, EventEntry{AioFlushEvent()},
+               &on_ready, &on_safe);
+  ASSERT_EQ(0, on_ready.wait());
+  ASSERT_EQ(-ECANCELED, on_safe.wait());
+
+  ASSERT_EQ(0, when_shut_down(mock_journal_replay, false));
+}
+
+TEST_F(TestMockJournalReplay, LockLostBeforeExecuteOp) {
+  REQUIRE_FEATURE(RBD_FEATURE_JOURNALING);
+
+  librbd::ImageCtx *ictx;
+  ASSERT_EQ(0, open_image(m_image_name, &ictx));
+
+  MockReplayImageCtx mock_image_ctx(*ictx);
+
+  MockExclusiveLock mock_exclusive_lock;
+  mock_image_ctx.exclusive_lock = &mock_exclusive_lock;
+  expect_accept_ops(mock_exclusive_lock, false);
+
+  MockJournalReplay mock_journal_replay(mock_image_ctx);
+  expect_op_work_queue(mock_image_ctx);
+
+  InSequence seq;
+  EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillOnce(Return(true));
+  EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillOnce(Return(true));
+  expect_refresh_image(mock_image_ctx, false, 0);
+
+  C_SaferCond on_start_ready;
+  C_SaferCond on_start_safe;
+  when_process(mock_journal_replay, EventEntry{RenameEvent(123, "image")},
+               &on_start_ready, &on_start_safe);
+  ASSERT_EQ(0, on_start_ready.wait());
+
+  C_SaferCond on_finish_ready;
+  C_SaferCond on_finish_safe;
+  when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)},
+               &on_finish_ready, &on_finish_safe);
+
+  ASSERT_EQ(-ECANCELED, on_start_safe.wait());
+  ASSERT_EQ(0, on_finish_ready.wait());
+  ASSERT_EQ(-ECANCELED, on_finish_safe.wait());
+}
+
 } // namespace journal
 } // namespace librbd

diff --git a/src/test/librbd/mock/MockExclusiveLock.h b/src/test/librbd/mock/MockExclusiveLock.h
index 023cd5edb5f03936e588e199ca7e8d9c1d0cba9d..58ae19edd10b8a76f0d09b94de7fbf516587cb47 100644 (file)
--- a/src/test/librbd/mock/MockExclusiveLock.h
+++ b/src/test/librbd/mock/MockExclusiveLock.h
@@ -26,6 +26,9 @@ struct MockExclusiveLock {
 
   MOCK_METHOD1(acquire_lock, void(Context *));
   MOCK_METHOD1(release_lock, void(Context *));
+
+  MOCK_METHOD0(accept_requests, bool());
+  MOCK_METHOD0(accept_ops, bool());
 };
 
 } // namespace librbd

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 7e4211b02ee6a7c6f6d4e6b004259a52f73909ab..25611f60e326f704813df646a050a19a32b38820 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -1168,9 +1168,14 @@ void ImageReplayer<I>::handle_preprocess_entry_safe(int r) {
   dout(20) << "r=" << r << dendl;
 
   if (r < 0) {
-    derr << "failed to preprocess journal event" << dendl;
     m_event_replay_tracker.finish_op();
-    handle_replay_complete(r, "failed to preprocess journal event");
+
+    if (r == -ECANCELED) {
+      handle_replay_complete(0, "lost exclusive lock");
+    } else {
+      derr << "failed to preprocess journal event" << dendl;
+      handle_replay_complete(r, "failed to preprocess journal event");
+    }
     return;
   }