From: Jason Dillaman Date: Fri, 26 May 2017 20:19:35 +0000 (-0400) Subject: librbd: guard journal replay from losing exclusive lock X-Git-Tag: ses5-milestone6~8^2~5^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5f1f51238b7989cbf73475d4324a12da50ce1cc0;p=ceph.git librbd: guard journal replay from losing exclusive lock Signed-off-by: Jason Dillaman --- diff --git a/src/librbd/ExclusiveLock.cc b/src/librbd/ExclusiveLock.cc index 3b5f04189210..5d942980dfbc 100644 --- a/src/librbd/ExclusiveLock.cc +++ b/src/librbd/ExclusiveLock.cc @@ -54,7 +54,6 @@ template bool ExclusiveLock::accept_ops() const { Mutex::Locker locker(ML::m_lock); bool accept_ops = (!ML::is_state_shutdown() && - !ML::is_state_pre_releasing() && (ML::is_state_locked() || ML::is_state_post_acquiring())); diff --git a/src/librbd/Journal.cc b/src/librbd/Journal.cc index c19a6a80c500..aa73cdeddac7 100644 --- a/src/librbd/Journal.cc +++ b/src/librbd/Journal.cc @@ -1332,9 +1332,11 @@ void Journal::handle_replay_process_safe(ReplayEntry replay_entry, int r) { ldout(cct, 20) << this << " " << __func__ << ": r=" << r << dendl; if (r < 0) { - lderr(cct) << this << " " << __func__ << ": " - << "failed to commit journal event to disk: " << cpp_strerror(r) - << dendl; + if (r != -ECANCELED) { + lderr(cct) << this << " " << __func__ << ": " + << "failed to commit journal event to disk: " + << cpp_strerror(r) << dendl; + } if (m_state == STATE_REPLAYING) { // abort the replay if we have an error diff --git a/src/librbd/journal/Replay.cc b/src/librbd/journal/Replay.cc index d0b585922980..29289ac580a4 100644 --- a/src/librbd/journal/Replay.cc +++ b/src/librbd/journal/Replay.cc @@ -5,6 +5,7 @@ #include "common/dout.h" #include "common/errno.h" #include "common/WorkQueue.h" +#include "librbd/ExclusiveLock.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" #include "librbd/internal.h" @@ -118,6 +119,14 @@ struct ExecuteOp : public Context { ldout(cct, 20) << ": ExecuteOp::" << __func__ << dendl; RWLock::RLocker owner_locker(image_ctx.owner_lock); + + if (image_ctx.exclusive_lock == nullptr || + !image_ctx.exclusive_lock->accept_ops()) { + ldout(cct, 5) << ": lost exclusive lock -- skipping op" << dendl; + on_op_complete->complete(-ECANCELED); + return; + } + execute(event); } }; @@ -197,6 +206,14 @@ void Replay::process(const EventEntry &event_entry, on_ready = util::create_async_context_callback(m_image_ctx, on_ready); RWLock::RLocker owner_lock(m_image_ctx.owner_lock); + if (m_image_ctx.exclusive_lock == nullptr || + !m_image_ctx.exclusive_lock->accept_ops()) { + ldout(cct, 5) << ": lost exclusive lock -- skipping event" << dendl; + m_image_ctx.op_work_queue->queue(on_safe, -ECANCELED); + on_ready->complete(0); + return; + } + boost::apply_visitor(EventVisitor(this, on_ready, on_safe), event_entry.event); } diff --git a/src/test/librbd/journal/test_mock_Replay.cc b/src/test/librbd/journal/test_mock_Replay.cc index 14d2a6a4b79e..1cc67ceac4da 100644 --- a/src/test/librbd/journal/test_mock_Replay.cc +++ b/src/test/librbd/journal/test_mock_Replay.cc @@ -128,6 +128,11 @@ public: TestMockJournalReplay() : m_invoke_lock("m_invoke_lock") { } + void expect_accept_ops(MockExclusiveLock &mock_exclusive_lock, bool accept) { + EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillRepeatedly( + Return(accept)); + } + void expect_aio_discard(MockIoImageRequest &mock_io_image_request, io::AioCompletion **aio_comp, uint64_t off, uint64_t len, bool skip_partial_discard) { @@ -333,6 +338,11 @@ TEST_F(TestMockJournalReplay, AioDiscard) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -361,6 +371,11 @@ TEST_F(TestMockJournalReplay, AioWrite) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -389,6 +404,11 @@ TEST_F(TestMockJournalReplay, AioFlush) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -415,6 +435,11 @@ TEST_F(TestMockJournalReplay, AioWriteSame) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -443,6 +468,11 @@ TEST_F(TestMockJournalReplay, IOError) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -471,6 +501,11 @@ TEST_F(TestMockJournalReplay, SoftFlushIO) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -510,6 +545,11 @@ TEST_F(TestMockJournalReplay, PauseIO) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -551,6 +591,11 @@ TEST_F(TestMockJournalReplay, Flush) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -579,6 +624,11 @@ TEST_F(TestMockJournalReplay, OpFinishError) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -610,6 +660,11 @@ TEST_F(TestMockJournalReplay, BlockedOpFinishError) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -649,6 +704,11 @@ TEST_F(TestMockJournalReplay, MissingOpFinishEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -703,6 +763,11 @@ TEST_F(TestMockJournalReplay, MissingOpFinishEventCancelOps) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -750,6 +815,11 @@ TEST_F(TestMockJournalReplay, UnknownOpFinishEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -770,6 +840,11 @@ TEST_F(TestMockJournalReplay, OpEventError) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -806,6 +881,11 @@ TEST_F(TestMockJournalReplay, SnapCreateEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -847,6 +927,11 @@ TEST_F(TestMockJournalReplay, SnapCreateEventExists) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -884,6 +969,11 @@ TEST_F(TestMockJournalReplay, SnapRemoveEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -920,6 +1010,11 @@ TEST_F(TestMockJournalReplay, SnapRemoveEventDNE) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -956,6 +1051,11 @@ TEST_F(TestMockJournalReplay, SnapRenameEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -989,6 +1089,11 @@ TEST_F(TestMockJournalReplay, SnapRenameEventExists) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1022,6 +1127,11 @@ TEST_F(TestMockJournalReplay, SnapProtectEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1058,6 +1168,11 @@ TEST_F(TestMockJournalReplay, SnapProtectEventBusy) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1094,6 +1209,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1130,6 +1250,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectOpFinishBusy) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1162,6 +1287,11 @@ TEST_F(TestMockJournalReplay, SnapUnprotectEventInvalid) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1198,6 +1328,11 @@ TEST_F(TestMockJournalReplay, SnapRollbackEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1234,6 +1369,11 @@ TEST_F(TestMockJournalReplay, RenameEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1266,6 +1406,11 @@ TEST_F(TestMockJournalReplay, RenameEventExists) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1298,6 +1443,11 @@ TEST_F(TestMockJournalReplay, ResizeEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1335,6 +1485,11 @@ TEST_F(TestMockJournalReplay, FlattenEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1367,6 +1522,11 @@ TEST_F(TestMockJournalReplay, FlattenEventInvalid) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1402,6 +1562,11 @@ TEST_F(TestMockJournalReplay, UpdateFeaturesEvent) { bool enabled = !ictx->test_features(features); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1440,6 +1605,11 @@ TEST_F(TestMockJournalReplay, MetadataSetEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1472,6 +1642,11 @@ TEST_F(TestMockJournalReplay, MetadataRemoveEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1504,6 +1679,11 @@ TEST_F(TestMockJournalReplay, MetadataRemoveEventDNE) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1536,6 +1716,11 @@ TEST_F(TestMockJournalReplay, UnknownEvent) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1562,6 +1747,11 @@ TEST_F(TestMockJournalReplay, RefreshImageBeforeOpStart) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); expect_op_work_queue(mock_image_ctx); @@ -1599,6 +1789,11 @@ TEST_F(TestMockJournalReplay, FlushEventAfterShutDown) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -1620,6 +1815,11 @@ TEST_F(TestMockJournalReplay, ModifyEventAfterShutDown) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -1642,6 +1842,11 @@ TEST_F(TestMockJournalReplay, OpEventAfterShutDown) { ASSERT_EQ(0, open_image(m_image_name, &ictx)); MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, true); + MockJournalReplay mock_journal_replay(mock_image_ctx); MockIoImageRequest mock_io_image_request; expect_op_work_queue(mock_image_ctx); @@ -1656,5 +1861,68 @@ TEST_F(TestMockJournalReplay, OpEventAfterShutDown) { ASSERT_EQ(-ESHUTDOWN, on_safe.wait()); } +TEST_F(TestMockJournalReplay, LockLostBeforeProcess) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, false); + + MockJournalReplay mock_journal_replay(mock_image_ctx); + MockIoImageRequest mock_io_image_request; + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + C_SaferCond on_ready; + C_SaferCond on_safe; + when_process(mock_journal_replay, EventEntry{AioFlushEvent()}, + &on_ready, &on_safe); + ASSERT_EQ(0, on_ready.wait()); + ASSERT_EQ(-ECANCELED, on_safe.wait()); + + ASSERT_EQ(0, when_shut_down(mock_journal_replay, false)); +} + +TEST_F(TestMockJournalReplay, LockLostBeforeExecuteOp) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockReplayImageCtx mock_image_ctx(*ictx); + + MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + expect_accept_ops(mock_exclusive_lock, false); + + MockJournalReplay mock_journal_replay(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillOnce(Return(true)); + EXPECT_CALL(mock_exclusive_lock, accept_ops()).WillOnce(Return(true)); + expect_refresh_image(mock_image_ctx, false, 0); + + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; + when_process(mock_journal_replay, EventEntry{RenameEvent(123, "image")}, + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); + + ASSERT_EQ(-ECANCELED, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(-ECANCELED, on_finish_safe.wait()); +} + } // namespace journal } // namespace librbd diff --git a/src/test/librbd/mock/MockExclusiveLock.h b/src/test/librbd/mock/MockExclusiveLock.h index 023cd5edb5f0..58ae19edd10b 100644 --- a/src/test/librbd/mock/MockExclusiveLock.h +++ b/src/test/librbd/mock/MockExclusiveLock.h @@ -26,6 +26,9 @@ struct MockExclusiveLock { MOCK_METHOD1(acquire_lock, void(Context *)); MOCK_METHOD1(release_lock, void(Context *)); + + MOCK_METHOD0(accept_requests, bool()); + MOCK_METHOD0(accept_ops, bool()); }; } // namespace librbd diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 7e4211b02ee6..25611f60e326 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -1168,9 +1168,14 @@ void ImageReplayer::handle_preprocess_entry_safe(int r) { dout(20) << "r=" << r << dendl; if (r < 0) { - derr << "failed to preprocess journal event" << dendl; m_event_replay_tracker.finish_op(); - handle_replay_complete(r, "failed to preprocess journal event"); + + if (r == -ECANCELED) { + handle_replay_complete(0, "lost exclusive lock"); + } else { + derr << "failed to preprocess journal event" << dendl; + handle_replay_complete(r, "failed to preprocess journal event"); + } return; }