From: Jason Dillaman Date: Tue, 22 Dec 2015 20:55:19 +0000 (-0500) Subject: tests: update tests to support replaying snap create / resize X-Git-Tag: v10.0.3~24^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a2ad7ecb5c8d8560afc7af37edb4061de42e8d46;p=ceph.git tests: update tests to support replaying snap create / resize Signed-off-by: Jason Dillaman --- diff --git a/src/test/librbd/journal/test_mock_Replay.cc b/src/test/librbd/journal/test_mock_Replay.cc index 539a135d5ea5..90475f65cc97 100644 --- a/src/test/librbd/journal/test_mock_Replay.cc +++ b/src/test/librbd/journal/test_mock_Replay.cc @@ -9,6 +9,7 @@ #include "librbd/journal/Replay.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +#include namespace librbd { @@ -61,6 +62,15 @@ MATCHER_P(CStrEq, str, "") { return (strncmp(arg, str, strlen(str)) == 0); } +ACTION_P2(CompleteAioCompletion, r, image_ctx) { + CephContext *cct = image_ctx->cct; + image_ctx->op_work_queue->queue(new FunctionContext([cct, arg0](int r) { + arg0->get(); + arg0->set_request_count(cct, 1); + arg0->complete_request(cct, r); + }), r); +} + namespace librbd { namespace journal { @@ -82,6 +92,12 @@ public: .WillOnce(SaveArg<0>(aio_comp)); } + void expect_aio_flush(MockImageCtx &mock_image_ctx, + MockAioImageRequest &mock_aio_image_request, int r) { + EXPECT_CALL(mock_aio_image_request, aio_flush(_)) + .WillOnce(CompleteAioCompletion(r, mock_image_ctx.image_ctx)); + } + void expect_aio_write(MockAioImageRequest &mock_aio_image_request, AioCompletion **aio_comp, uint64_t off, uint64_t len, const char *data) { @@ -102,14 +118,16 @@ public: } void expect_resize(MockImageCtx &mock_image_ctx, Context **on_finish, - uint64_t size) { - EXPECT_CALL(*mock_image_ctx.operations, resize(size, _, _)) + uint64_t size, uint64_t op_tid) { + EXPECT_CALL(*mock_image_ctx.operations, resize(size, _, _, op_tid)) .WillOnce(SaveArg<2>(on_finish)); } void expect_snap_create(MockImageCtx &mock_image_ctx, - Context **on_finish, const char *snap_name) { - EXPECT_CALL(*mock_image_ctx.operations, snap_create(CStrEq(snap_name), _)) + Context **on_finish, const char *snap_name, + uint64_t op_tid) { + EXPECT_CALL(*mock_image_ctx.operations, snap_create(CStrEq(snap_name), _, + op_tid)) .WillOnce(SaveArg<1>(on_finish)); } @@ -145,12 +163,19 @@ public: } void when_process(MockJournalReplay &mock_journal_replay, - EventEntry &&event_entry, Context *on_safe) { + EventEntry &&event_entry, Context *on_ready, + Context *on_safe) { bufferlist bl; ::encode(event_entry, bl); bufferlist::iterator it = bl.begin(); - mock_journal_replay.process(it, on_safe); + when_process(mock_journal_replay, &it, on_ready, on_safe); + } + + void when_process(MockJournalReplay &mock_journal_replay, + bufferlist::iterator *it, Context *on_ready, + Context *on_safe) { + mock_journal_replay.process(it, on_ready, on_safe); } void when_complete(MockImageCtx &mock_image_ctx, AioCompletion *aio_comp, @@ -160,6 +185,17 @@ public: aio_comp->complete_request(mock_image_ctx.cct, r); } + int when_flush(MockJournalReplay &mock_journal_replay) { + C_SaferCond ctx; + mock_journal_replay.flush(&ctx); + return ctx.wait(); + } + + void when_replay_op_ready(MockJournalReplay &mock_journal_replay, + uint64_t op_tid, Context *on_resume) { + mock_journal_replay.replay_op_ready(op_tid, on_resume); + } + bufferlist to_bl(const std::string &str) { bufferlist bl; bl.append(str); @@ -176,16 +212,22 @@ TEST_F(TestMockJournalReplay, AioDiscard) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); InSequence seq; AioCompletion *aio_comp; + C_SaferCond on_ready; C_SaferCond on_safe; expect_aio_discard(mock_aio_image_request, &aio_comp, 123, 456); when_process(mock_journal_replay, EventEntry{AioDiscardEvent(123, 456)}, - &on_safe); + &on_ready, &on_safe); when_complete(mock_image_ctx, aio_comp, 0); + ASSERT_EQ(0, on_ready.wait()); + + expect_aio_flush(mock_image_ctx, mock_aio_image_request, 0); + ASSERT_EQ(0, when_flush(mock_journal_replay)); ASSERT_EQ(0, on_safe.wait()); } @@ -198,16 +240,22 @@ TEST_F(TestMockJournalReplay, AioWrite) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); InSequence seq; AioCompletion *aio_comp; + C_SaferCond on_ready; C_SaferCond on_safe; expect_aio_write(mock_aio_image_request, &aio_comp, 123, 456, "test"); when_process(mock_journal_replay, EventEntry{AioWriteEvent(123, 456, to_bl("test"))}, - &on_safe); + &on_ready, &on_safe); when_complete(mock_image_ctx, aio_comp, 0); + ASSERT_EQ(0, on_ready.wait()); + + expect_aio_flush(mock_image_ctx, mock_aio_image_request, 0); + ASSERT_EQ(0, when_flush(mock_journal_replay)); ASSERT_EQ(0, on_safe.wait()); } @@ -220,15 +268,21 @@ TEST_F(TestMockJournalReplay, AioFlush) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); InSequence seq; AioCompletion *aio_comp; + C_SaferCond on_ready; C_SaferCond on_safe; expect_aio_flush(mock_aio_image_request, &aio_comp); - when_process(mock_journal_replay, EventEntry{AioFlushEvent()}, &on_safe); + when_process(mock_journal_replay, EventEntry{AioFlushEvent()}, + &on_ready, &on_safe); when_complete(mock_image_ctx, aio_comp, 0); ASSERT_EQ(0, on_safe.wait()); + + ASSERT_EQ(0, when_flush(mock_journal_replay)); + ASSERT_EQ(0, on_ready.wait()); } TEST_F(TestMockJournalReplay, IOError) { @@ -240,24 +294,181 @@ TEST_F(TestMockJournalReplay, IOError) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); InSequence seq; AioCompletion *aio_comp; + C_SaferCond on_ready; C_SaferCond on_safe; expect_aio_discard(mock_aio_image_request, &aio_comp, 123, 456); when_process(mock_journal_replay, EventEntry{AioDiscardEvent(123, 456)}, - &on_safe); + &on_ready, &on_safe); when_complete(mock_image_ctx, aio_comp, -EINVAL); ASSERT_EQ(-EINVAL, on_safe.wait()); + + expect_aio_flush(mock_image_ctx, mock_aio_image_request, 0); + ASSERT_EQ(0, when_flush(mock_journal_replay)); + ASSERT_EQ(0, on_ready.wait()); } -TEST_F(TestMockJournalReplay, SynchronousUntilFlush) { +TEST_F(TestMockJournalReplay, SoftFlushIO) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + const size_t io_count = 32; + C_SaferCond on_safes[io_count]; + for (size_t i = 0; i < io_count; ++i) { + AioCompletion *aio_comp; + AioCompletion *flush_comp = nullptr; + C_SaferCond on_ready; + expect_aio_discard(mock_aio_image_request, &aio_comp, 123, 456); + if (i == io_count - 1) { + expect_aio_flush(mock_aio_image_request, &flush_comp); + } + when_process(mock_journal_replay, + EventEntry{AioDiscardEvent(123, 456)}, + &on_ready, &on_safes[i]); + ASSERT_EQ(0, on_ready.wait()); + + when_complete(mock_image_ctx, aio_comp, 0); + if (flush_comp != nullptr) { + when_complete(mock_image_ctx, flush_comp, 0); + } + } + for (auto &on_safe : on_safes) { + ASSERT_EQ(0, on_safe.wait()); + } + + ASSERT_EQ(0, when_flush(mock_journal_replay)); +} + +TEST_F(TestMockJournalReplay, PauseIO) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + MockAioImageRequest mock_aio_image_request; + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + const size_t io_count = 64; + std::list flush_comps; + C_SaferCond on_safes[io_count]; + for (size_t i = 0; i < io_count; ++i) { + AioCompletion *aio_comp; + C_SaferCond on_ready; + expect_aio_write(mock_aio_image_request, &aio_comp, 123, 456, "test"); + if ((i + 1) % 32 == 0) { + flush_comps.push_back(nullptr); + expect_aio_flush(mock_aio_image_request, &flush_comps.back()); + } + when_process(mock_journal_replay, + EventEntry{AioWriteEvent(123, 456, to_bl("test"))}, + &on_ready, &on_safes[i]); + if (i < io_count - 1) { + ASSERT_EQ(0, on_ready.wait()); + } + + when_complete(mock_image_ctx, aio_comp, 0); + if (i == io_count - 1) { + for (auto flush_comp : flush_comps) { + when_complete(mock_image_ctx, flush_comp, 0); + ASSERT_EQ(0, on_ready.wait()); + } + } + } + for (auto &on_safe : on_safes) { + ASSERT_EQ(0, on_safe.wait()); + } + + ASSERT_EQ(0, when_flush(mock_journal_replay)); +} + +TEST_F(TestMockJournalReplay, MissingOpFinishEvent) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + C_SaferCond on_ready; + C_SaferCond on_safe; + when_process(mock_journal_replay, EventEntry{SnapRemoveEvent(123, "snap")}, + &on_ready, &on_safe); + + ASSERT_EQ(0, on_ready.wait()); + + ASSERT_EQ(0, when_flush(mock_journal_replay)); + ASSERT_EQ(-ERESTART, on_safe.wait()); +} + +TEST_F(TestMockJournalReplay, MissingCompleteOpFinishEvent) { // TODO } +TEST_F(TestMockJournalReplay, UnknownOpFinishEvent) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + C_SaferCond on_ready; + C_SaferCond on_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_ready, &on_safe); + + ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_ready.wait()); +} + +TEST_F(TestMockJournalReplay, OpEventError) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + + InSequence seq; + Context *on_finish; + expect_snap_remove(mock_image_ctx, &on_finish, "snap"); + + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; + when_process(mock_journal_replay, EventEntry{SnapRemoveEvent(123, "snap")}, + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); + + on_finish->complete(-EINVAL); + ASSERT_EQ(-EINVAL, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(-EINVAL, on_finish_safe.wait()); +} + TEST_F(TestMockJournalReplay, SnapCreateEvent) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); @@ -266,17 +477,32 @@ TEST_F(TestMockJournalReplay, SnapCreateEvent) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); InSequence seq; Context *on_finish; - expect_snap_create(mock_image_ctx, &on_finish, "snap"); + expect_snap_create(mock_image_ctx, &on_finish, "snap", 123); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapCreateEvent(123, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + C_SaferCond on_resume; + when_replay_op_ready(mock_journal_replay, 123, &on_resume); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); + + ASSERT_EQ(0, on_resume.wait()); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, SnapRemoveEvent) { @@ -292,12 +518,21 @@ TEST_F(TestMockJournalReplay, SnapRemoveEvent) { Context *on_finish; expect_snap_remove(mock_image_ctx, &on_finish, "snap"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapRemoveEvent(123, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, SnapRenameEvent) { @@ -313,13 +548,22 @@ TEST_F(TestMockJournalReplay, SnapRenameEvent) { Context *on_finish; expect_snap_rename(mock_image_ctx, &on_finish, 234, "snap"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapRenameEvent(123, 234, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, SnapProtectEvent) { @@ -335,12 +579,21 @@ TEST_F(TestMockJournalReplay, SnapProtectEvent) { Context *on_finish; expect_snap_protect(mock_image_ctx, &on_finish, "snap"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapProtectEvent(123, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, SnapUnprotectEvent) { @@ -356,12 +609,21 @@ TEST_F(TestMockJournalReplay, SnapUnprotectEvent) { Context *on_finish; expect_snap_unprotect(mock_image_ctx, &on_finish, "snap"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapUnprotectEvent(123, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, SnapRollbackEvent) { @@ -377,12 +639,21 @@ TEST_F(TestMockJournalReplay, SnapRollbackEvent) { Context *on_finish; expect_snap_rollback(mock_image_ctx, &on_finish, "snap"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{SnapRollbackEvent(123, "snap")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, RenameEvent) { @@ -398,12 +669,21 @@ TEST_F(TestMockJournalReplay, RenameEvent) { Context *on_finish; expect_rename(mock_image_ctx, &on_finish, "image"); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{RenameEvent(123, "image")}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, ResizeEvent) { @@ -414,17 +694,32 @@ TEST_F(TestMockJournalReplay, ResizeEvent) { MockImageCtx mock_image_ctx(*ictx); MockJournalReplay mock_journal_replay(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); InSequence seq; Context *on_finish; - expect_resize(mock_image_ctx, &on_finish, 234); + expect_resize(mock_image_ctx, &on_finish, 234, 123); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{ResizeEvent(123, 234)}, - &on_safe); + &on_start_ready, &on_start_safe); + + C_SaferCond on_resume; + when_replay_op_ready(mock_journal_replay, 123, &on_resume); + ASSERT_EQ(0, on_start_ready.wait()); + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); + + ASSERT_EQ(0, on_resume.wait()); on_finish->complete(0); - ASSERT_EQ(0, on_safe.wait()); + + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); } TEST_F(TestMockJournalReplay, FlattenEvent) { @@ -440,12 +735,46 @@ TEST_F(TestMockJournalReplay, FlattenEvent) { Context *on_finish; expect_flatten(mock_image_ctx, &on_finish); - C_SaferCond on_safe; + C_SaferCond on_start_ready; + C_SaferCond on_start_safe; when_process(mock_journal_replay, EventEntry{FlattenEvent(123)}, - &on_safe); + &on_start_ready, &on_start_safe); + ASSERT_EQ(0, on_start_ready.wait()); + + C_SaferCond on_finish_ready; + C_SaferCond on_finish_safe; + when_process(mock_journal_replay, EventEntry{OpFinishEvent(123, 0)}, + &on_finish_ready, &on_finish_safe); on_finish->complete(0); + ASSERT_EQ(0, on_start_safe.wait()); + ASSERT_EQ(0, on_finish_ready.wait()); + ASSERT_EQ(0, on_finish_safe.wait()); +} + +TEST_F(TestMockJournalReplay, UnknownEvent) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournalReplay mock_journal_replay(mock_image_ctx); + + InSequence seq; + + bufferlist bl; + ENCODE_START(1, 1, bl); + ::encode(static_cast(-1), bl); + ENCODE_FINISH(bl); + + bufferlist::iterator it = bl.begin(); + C_SaferCond on_ready; + C_SaferCond on_safe; + when_process(mock_journal_replay, &it, &on_ready, &on_safe); + ASSERT_EQ(0, on_safe.wait()); + ASSERT_EQ(0, on_ready.wait()); } } // namespace journal diff --git a/src/test/librbd/mock/MockJournal.h b/src/test/librbd/mock/MockJournal.h index 9b3ecfffd6c0..b71505ee6415 100644 --- a/src/test/librbd/mock/MockJournal.h +++ b/src/test/librbd/mock/MockJournal.h @@ -30,6 +30,7 @@ struct MockJournal { } MOCK_METHOD2(commit_op_event, void(uint64_t, int)); + MOCK_METHOD2(replay_op_ready, void(uint64_t, Context *)); }; } // namespace librbd diff --git a/src/test/librbd/mock/MockOperations.h b/src/test/librbd/mock/MockOperations.h index b3dde8cd151a..c3bf9d9ca8c3 100644 --- a/src/test/librbd/mock/MockOperations.h +++ b/src/test/librbd/mock/MockOperations.h @@ -16,9 +16,10 @@ struct MockOperations { MOCK_METHOD2(rebuild_object_map, void(ProgressContext &prog_ctx, Context *on_finish)); MOCK_METHOD2(rename, void(const char *dstname, Context *on_finish)); - MOCK_METHOD3(resize, void(uint64_t size, ProgressContext &prog_ctx, - Context *on_finish)); - MOCK_METHOD2(snap_create, void(const char *snap_name, Context *on_finish)); + MOCK_METHOD4(resize, void(uint64_t size, ProgressContext &prog_ctx, + Context *on_finish, uint64_t journal_op_tid)); + MOCK_METHOD3(snap_create, void(const char *snap_name, Context *on_finish, + uint64_t journal_op_tid)); MOCK_METHOD2(snap_remove, void(const char *snap_name, Context *on_finish)); MOCK_METHOD3(snap_rename, void(uint64_t src_snap_id, const char *snap_name, Context *on_finish)); diff --git a/src/test/librbd/operation/test_mock_ResizeRequest.cc b/src/test/librbd/operation/test_mock_ResizeRequest.cc index 6067b4feddbf..dfe07bf25736 100644 --- a/src/test/librbd/operation/test_mock_ResizeRequest.cc +++ b/src/test/librbd/operation/test_mock_ResizeRequest.cc @@ -70,38 +70,6 @@ public: .Times(1); } - void expect_is_journal_replaying(MockJournal &mock_journal) { - EXPECT_CALL(mock_journal, is_journal_replaying()).WillOnce(Return(false)); - } - - void expect_is_journal_ready(MockJournal &mock_journal) { - EXPECT_CALL(mock_journal, is_journal_ready()).WillOnce(Return(true)); - } - - void expect_allocate_op_tid(MockImageCtx &mock_image_ctx) { - if (mock_image_ctx.journal != nullptr) { - EXPECT_CALL(*mock_image_ctx.journal, allocate_op_tid()) - .WillOnce(Return(1U)); - } - } - - void expect_append_op_event(MockImageCtx &mock_image_ctx, int r) { - if (mock_image_ctx.journal != nullptr) { - expect_is_journal_replaying(*mock_image_ctx.journal); - expect_allocate_op_tid(mock_image_ctx); - EXPECT_CALL(*mock_image_ctx.journal, append_op_event_mock(_, _, _)) - .WillOnce(WithArg<2>(CompleteContext(r, mock_image_ctx.image_ctx->op_work_queue))); - } - } - - void expect_commit_op_event(MockImageCtx &mock_image_ctx, int r) { - if (mock_image_ctx.journal != nullptr) { - expect_is_journal_replaying(*mock_image_ctx.journal); - expect_is_journal_ready(*mock_image_ctx.journal); - EXPECT_CALL(*mock_image_ctx.journal, commit_op_event(1U, r)); - } - } - void expect_is_lock_owner(MockImageCtx &mock_image_ctx) { if (mock_image_ctx.exclusive_lock != nullptr) { EXPECT_CALL(*mock_image_ctx.exclusive_lock, is_lock_owner()) @@ -159,32 +127,19 @@ public: .WillOnce(WithArg<2>(CompleteContext(0, mock_image_ctx.image_ctx->op_work_queue))); } - int when_resize(MockImageCtx &mock_image_ctx, uint64_t new_size) { + int when_resize(MockImageCtx &mock_image_ctx, uint64_t new_size, + uint64_t journal_op_tid, bool disable_journal) { C_SaferCond cond_ctx; librbd::NoOpProgressContext prog_ctx; MockResizeRequest *req = new MockResizeRequest( - mock_image_ctx, &cond_ctx, new_size, prog_ctx); + mock_image_ctx, &cond_ctx, new_size, prog_ctx, journal_op_tid, + disable_journal); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); } return cond_ctx.wait(); } - - void initialize_features(ImageCtx *ictx, MockImageCtx &mock_image_ctx, - MockExclusiveLock &mock_exclusive_lock, - MockJournal &mock_journal, - MockObjectMap &mock_object_map) { - if (ictx->test_features(RBD_FEATURE_EXCLUSIVE_LOCK)) { - mock_image_ctx.exclusive_lock = &mock_exclusive_lock; - } - if (ictx->test_features(RBD_FEATURE_JOURNALING)) { - mock_image_ctx.journal = &mock_journal; - } - if (ictx->test_features(RBD_FEATURE_OBJECT_MAP)) { - mock_image_ctx.object_map = &mock_object_map; - } - } }; TEST_F(TestMockOperationResizeRequest, NoOpSuccess) { @@ -203,7 +158,7 @@ TEST_F(TestMockOperationResizeRequest, NoOpSuccess) { expect_append_op_event(mock_image_ctx, 0); expect_unblock_writes(mock_image_ctx); expect_commit_op_event(mock_image_ctx, 0); - ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size)); + ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size, 0, false)); } TEST_F(TestMockOperationResizeRequest, GrowSuccess) { @@ -226,7 +181,7 @@ TEST_F(TestMockOperationResizeRequest, GrowSuccess) { expect_update_header(mock_image_ctx, 0); expect_commit_op_event(mock_image_ctx, 0); expect_unblock_writes(mock_image_ctx); - ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size * 2)); + ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size * 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, ShrinkSuccess) { @@ -253,7 +208,7 @@ TEST_F(TestMockOperationResizeRequest, ShrinkSuccess) { expect_commit_op_event(mock_image_ctx, 0); expect_shrink_object_map(mock_image_ctx); expect_unblock_writes(mock_image_ctx); - ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size / 2)); + ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size / 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, PreBlockWritesError) { @@ -270,7 +225,7 @@ TEST_F(TestMockOperationResizeRequest, PreBlockWritesError) { InSequence seq; expect_block_writes(mock_image_ctx, -EINVAL); expect_unblock_writes(mock_image_ctx); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size, 0, false)); } TEST_F(TestMockOperationResizeRequest, TrimError) { @@ -292,7 +247,7 @@ TEST_F(TestMockOperationResizeRequest, TrimError) { MockTrimRequest mock_trim_request; expect_trim(mock_image_ctx, mock_trim_request, -EINVAL); expect_commit_op_event(mock_image_ctx, -EINVAL); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, InvalidateCacheError) { @@ -315,7 +270,7 @@ TEST_F(TestMockOperationResizeRequest, InvalidateCacheError) { expect_trim(mock_image_ctx, mock_trim_request, 0); expect_invalidate_cache(mock_image_ctx, -EINVAL); expect_commit_op_event(mock_image_ctx, -EINVAL); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size / 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, PostBlockWritesError) { @@ -337,7 +292,7 @@ TEST_F(TestMockOperationResizeRequest, PostBlockWritesError) { expect_block_writes(mock_image_ctx, -EINVAL); expect_unblock_writes(mock_image_ctx); expect_commit_op_event(mock_image_ctx, -EINVAL); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size * 2)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size * 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, UpdateHeaderError) { @@ -360,7 +315,7 @@ TEST_F(TestMockOperationResizeRequest, UpdateHeaderError) { expect_update_header(mock_image_ctx, -EINVAL); expect_unblock_writes(mock_image_ctx); expect_commit_op_event(mock_image_ctx, -EINVAL); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size * 2)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size * 2, 0, false)); } TEST_F(TestMockOperationResizeRequest, JournalAppendError) { @@ -380,7 +335,24 @@ TEST_F(TestMockOperationResizeRequest, JournalAppendError) { expect_block_writes(mock_image_ctx, 0); expect_append_op_event(mock_image_ctx, -EINVAL); expect_unblock_writes(mock_image_ctx); - ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size)); + ASSERT_EQ(-EINVAL, when_resize(mock_image_ctx, ictx->size, 0, false)); +} + +TEST_F(TestMockOperationResizeRequest, JournalDisabled) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockExclusiveLock mock_exclusive_lock; + MockJournal mock_journal; + MockObjectMap mock_object_map; + initialize_features(ictx, mock_image_ctx, mock_exclusive_lock, mock_journal, + mock_object_map); + + InSequence seq; + expect_block_writes(mock_image_ctx, 0); + expect_unblock_writes(mock_image_ctx); + ASSERT_EQ(0, when_resize(mock_image_ctx, ictx->size, 0, true)); } } // namespace operation diff --git a/src/test/librbd/operation/test_mock_SnapshotCreateRequest.cc b/src/test/librbd/operation/test_mock_SnapshotCreateRequest.cc index 4d7ad4b63a18..d4019431eb30 100644 --- a/src/test/librbd/operation/test_mock_SnapshotCreateRequest.cc +++ b/src/test/librbd/operation/test_mock_SnapshotCreateRequest.cc @@ -133,7 +133,7 @@ TEST_F(TestMockOperationSnapshotCreateRequest, Success) { C_SaferCond cond_ctx; MockSnapshotCreateRequest *req = new MockSnapshotCreateRequest( - mock_image_ctx, &cond_ctx, "snap1"); + mock_image_ctx, &cond_ctx, "snap1", 0); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); @@ -162,7 +162,7 @@ TEST_F(TestMockOperationSnapshotCreateRequest, AllocateSnapIdError) { C_SaferCond cond_ctx; MockSnapshotCreateRequest *req = new MockSnapshotCreateRequest( - mock_image_ctx, &cond_ctx, "snap1"); + mock_image_ctx, &cond_ctx, "snap1", 0); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); @@ -198,7 +198,7 @@ TEST_F(TestMockOperationSnapshotCreateRequest, CreateSnapStale) { C_SaferCond cond_ctx; MockSnapshotCreateRequest *req = new MockSnapshotCreateRequest( - mock_image_ctx, &cond_ctx, "snap1"); + mock_image_ctx, &cond_ctx, "snap1", 0); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); @@ -228,7 +228,7 @@ TEST_F(TestMockOperationSnapshotCreateRequest, CreateSnapError) { C_SaferCond cond_ctx; MockSnapshotCreateRequest *req = new MockSnapshotCreateRequest( - mock_image_ctx, &cond_ctx, "snap1"); + mock_image_ctx, &cond_ctx, "snap1", 0); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); @@ -258,7 +258,7 @@ TEST_F(TestMockOperationSnapshotCreateRequest, ReleaseSnapIdError) { C_SaferCond cond_ctx; MockSnapshotCreateRequest *req = new MockSnapshotCreateRequest( - mock_image_ctx, &cond_ctx, "snap1"); + mock_image_ctx, &cond_ctx, "snap1", 0); { RWLock::RLocker owner_locker(mock_image_ctx.owner_lock); req->send(); diff --git a/src/test/librbd/test_internal.cc b/src/test/librbd/test_internal.cc index a3fb09653072..7c2f759999aa 100644 --- a/src/test/librbd/test_internal.cc +++ b/src/test/librbd/test_internal.cc @@ -321,7 +321,7 @@ TEST_F(TestInternal, CancelAsyncResize) { size -= MIN(size, 1<<18); { RWLock::RLocker l(ictx->owner_lock); - ictx->operations->resize(size, prog_ctx, &ctx); + ictx->operations->resize(size, prog_ctx, &ctx, 0); } // try to interrupt the in-progress resize @@ -369,7 +369,7 @@ TEST_F(TestInternal, MultipleResize) { RWLock::RLocker l(ictx->owner_lock); contexts.push_back(new C_SaferCond()); - ictx->operations->resize(new_size, prog_ctx, contexts.back()); + ictx->operations->resize(new_size, prog_ctx, contexts.back(), 0); } for (uint32_t i = 0; i < contexts.size(); ++i) { @@ -595,7 +595,8 @@ TEST_F(TestInternal, ResizeCopyup) // verify full / partial object removal properly copyup librbd::NoOpProgressContext no_op; - ASSERT_EQ(0, ictx2->operations->resize(m_image_size - (1 << order) - 32, no_op)); + ASSERT_EQ(0, ictx2->operations->resize(m_image_size - (1 << order) - 32, + no_op)); ASSERT_EQ(0, librbd::snap_set(ictx2, "snap1")); { diff --git a/src/test/librbd/test_mock_Journal.cc b/src/test/librbd/test_mock_Journal.cc index 36e7438dfe67..0de97a382d58 100644 --- a/src/test/librbd/test_mock_Journal.cc +++ b/src/test/librbd/test_mock_Journal.cc @@ -4,7 +4,10 @@ #include "test/librbd/test_mock_fixture.h" #include "test/librbd/test_support.h" #include "test/librbd/mock/MockImageCtx.h" +#include "common/Cond.h" +#include "common/Mutex.h" #include "librbd/Journal.h" +#include "librbd/Utils.h" #include "librbd/journal/Entries.h" #include "librbd/journal/Replay.h" #include "gmock/gmock.h" @@ -191,7 +194,8 @@ struct MockReplay { } MOCK_METHOD1(flush, void(Context *)); - MOCK_METHOD2(process, int(bufferlist::iterator, Context *)); + MOCK_METHOD3(process, void(bufferlist::iterator*, Context *, Context *)); + MOCK_METHOD2(replay_op_ready, void(uint64_t, Context *)); }; template <> @@ -205,8 +209,13 @@ public: MockReplay::get_instance().flush(on_finish); } - int process(bufferlist::iterator it, Context *on_commit) { - return MockReplay::get_instance().process(it, on_commit); + void process(bufferlist::iterator *it, Context *on_ready, + Context *on_commit) { + MockReplay::get_instance().process(it, on_ready, on_commit); + } + + void replay_op_ready(uint64_t op_tid, Context *on_resume) { + MockReplay::get_instance().replay_op_ready(op_tid, on_resume); } }; @@ -223,6 +232,7 @@ using ::testing::_; using ::testing::DoAll; using ::testing::InSequence; using ::testing::Invoke; +using ::testing::InvokeWithoutArgs; using ::testing::MatcherCast; using ::testing::Return; using ::testing::SaveArg; @@ -246,7 +256,17 @@ public: typedef std::list ReplayActions; typedef std::list Contexts; + TestMockJournal() : m_lock("lock") { + } + + ~TestMockJournal() { + assert(m_commit_contexts.empty()); + } + + Mutex m_lock; + Cond m_cond; Contexts m_commit_contexts; + struct C_StartReplay : public Context { ReplayActions replay_actions; ::journal::ReplayHandler *replay_handler; @@ -261,10 +281,6 @@ public: } }; - ~TestMockJournal() { - assert(m_commit_contexts.empty()); - } - void expect_construct_journaler(::journal::MockJournaler &mock_journaler) { EXPECT_CALL(mock_journaler, construct()); } @@ -287,9 +303,11 @@ public: EXPECT_CALL(mock_journaler, stop_replay()); } - void expect_flush_replay(MockJournalReplay &mock_journal_replay, int r) { + void expect_flush_replay(MockImageCtx &mock_image_ctx, + MockJournalReplay &mock_journal_replay, int r) { EXPECT_CALL(mock_journal_replay, flush(_)) - .WillOnce(CompleteContext(r, NULL)); + .WillOnce(Invoke([this, &mock_image_ctx, r](Context *on_flush) { + this->commit_replay(mock_image_ctx, on_flush, r);})); } void expect_get_data(::journal::MockReplayEntry &mock_replay_entry) { @@ -308,14 +326,10 @@ public: } } - void expect_replay_process(MockJournalReplay &mock_journal_replay, int r) { - auto &expect = EXPECT_CALL(mock_journal_replay, process(_, _)); - if (r < 0) { - expect.WillOnce(Return(r)); - } else { - expect.WillOnce(DoAll(WithArg<1>(Invoke(this, &TestMockJournal::save_commit_context)), - Return(0))); - } + void expect_replay_process(MockJournalReplay &mock_journal_replay) { + EXPECT_CALL(mock_journal_replay, process(_, _, _)) + .WillOnce(DoAll(WithArg<2>(Invoke(this, &TestMockJournal::save_commit_context)), + WithArg<1>(CompleteContext(0, NULL)))); } void expect_start_append(::journal::MockJournaler &mock_journaler) { @@ -327,9 +341,10 @@ public: .WillOnce(CompleteContext(r, NULL)); } - void expect_committed(::journal::MockJournaler &mock_journaler) { + void expect_committed(::journal::MockJournaler &mock_journaler, + size_t events) { EXPECT_CALL(mock_journaler, committed(MatcherCast(_))) - .Times(m_commit_contexts.size()); + .Times(events); } void expect_append_journaler(::journal::MockJournaler &mock_journaler) { @@ -357,13 +372,6 @@ public: return ctx.wait(); } - void when_commit_replay(int r) { - for (auto ctx : m_commit_contexts) { - ctx->complete(r); - } - m_commit_contexts.clear(); - } - int when_close(MockJournal &mock_journal) { C_SaferCond ctx; mock_journal.close(&ctx); @@ -379,7 +387,19 @@ public: } void save_commit_context(Context *ctx) { + Mutex::Locker locker(m_lock); m_commit_contexts.push_back(ctx); + m_cond.Signal(); + } + + void commit_replay(MockImageCtx &mock_image_ctx, Context *on_flush, int r) { + Contexts commit_contexts; + std::swap(commit_contexts, m_commit_contexts); + + for (auto ctx : commit_contexts) { + mock_image_ctx.image_ctx->op_work_queue->queue(ctx, r); + } + mock_image_ctx.image_ctx->op_work_queue->queue(on_flush, 0); } void open_journal(MockImageCtx &mock_image_ctx, MockJournal &mock_journal, @@ -396,12 +416,10 @@ public: MockJournalReplay mock_journal_replay; expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); + expect_committed(mock_journaler, 0); expect_start_append(mock_journaler); ASSERT_EQ(0, when_open(mock_journal)); - - expect_committed(mock_journaler); - when_commit_replay(0); } void close_journal(MockJournal &mock_journal, @@ -444,24 +462,22 @@ TEST_F(TestMockJournal, StateTransitions) { ::journal::MockReplayEntry mock_replay_entry; MockJournalReplay mock_journal_replay; expect_try_pop_front(mock_journaler, true, mock_replay_entry); - expect_replay_process(mock_journal_replay, 0); + expect_replay_process(mock_journal_replay); expect_try_pop_front(mock_journaler, true, mock_replay_entry); - expect_replay_process(mock_journal_replay, 0); + expect_replay_process(mock_journal_replay); expect_try_pop_front(mock_journaler, false, mock_replay_entry); expect_try_pop_front(mock_journaler, true, mock_replay_entry); - expect_replay_process(mock_journal_replay, 0); + expect_replay_process(mock_journal_replay); expect_try_pop_front(mock_journaler, false, mock_replay_entry); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); + expect_committed(mock_journaler, 3); expect_start_append(mock_journaler); ASSERT_EQ(0, when_open(mock_journal)); - expect_committed(mock_journaler); - when_commit_replay(0); - expect_stop_append(mock_journaler, 0); ASSERT_EQ(0, when_close(mock_journal)); } @@ -484,7 +500,7 @@ TEST_F(TestMockJournal, InitError) { ASSERT_EQ(-EINVAL, when_open(mock_journal)); } -TEST_F(TestMockJournal, ReplayProcessError) { +TEST_F(TestMockJournal, ReplayCompleteError) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); librbd::ImageCtx *ictx; @@ -501,16 +517,12 @@ TEST_F(TestMockJournal, ReplayProcessError) { expect_init_journaler(mock_journaler, 0); expect_start_replay( mock_image_ctx, mock_journaler, { - std::bind(&invoke_replay_ready, _1) + std::bind(&invoke_replay_complete, _1, -EINVAL) }); - ::journal::MockReplayEntry mock_replay_entry; MockJournalReplay mock_journal_replay; - expect_try_pop_front(mock_journaler, true, mock_replay_entry); - expect_replay_process(mock_journal_replay, -EINVAL); - expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); // replay failure should result in replay-restart expect_construct_journaler(mock_journaler); @@ -521,7 +533,7 @@ TEST_F(TestMockJournal, ReplayProcessError) { }); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); expect_start_append(mock_journaler); ASSERT_EQ(0, when_open(mock_journal)); @@ -529,7 +541,7 @@ TEST_F(TestMockJournal, ReplayProcessError) { ASSERT_EQ(0, when_close(mock_journal)); } -TEST_F(TestMockJournal, ReplayCompleteError) { +TEST_F(TestMockJournal, FlushReplayError) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); librbd::ImageCtx *ictx; @@ -546,14 +558,19 @@ TEST_F(TestMockJournal, ReplayCompleteError) { expect_init_journaler(mock_journaler, 0); expect_start_replay( mock_image_ctx, mock_journaler, { - std::bind(&invoke_replay_complete, _1, -EINVAL) + std::bind(&invoke_replay_ready, _1), + std::bind(&invoke_replay_complete, _1, 0) }); + ::journal::MockReplayEntry mock_replay_entry; MockJournalReplay mock_journal_replay; + expect_try_pop_front(mock_journaler, true, mock_replay_entry); + expect_replay_process(mock_journal_replay); + expect_try_pop_front(mock_journaler, false, mock_replay_entry); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, -EINVAL); - // replay failure should result in replay-restart + // replay flush failure should result in replay-restart expect_construct_journaler(mock_journaler); expect_init_journaler(mock_journaler, 0); expect_start_replay( @@ -562,7 +579,7 @@ TEST_F(TestMockJournal, ReplayCompleteError) { }); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); expect_start_append(mock_journaler); ASSERT_EQ(0, when_open(mock_journal)); @@ -570,7 +587,7 @@ TEST_F(TestMockJournal, ReplayCompleteError) { ASSERT_EQ(0, when_close(mock_journal)); } -TEST_F(TestMockJournal, FlushReplayError) { +TEST_F(TestMockJournal, StopError) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); librbd::ImageCtx *ictx; @@ -592,9 +609,50 @@ TEST_F(TestMockJournal, FlushReplayError) { MockJournalReplay mock_journal_replay; expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, -EINVAL); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); + expect_start_append(mock_journaler); + ASSERT_EQ(0, when_open(mock_journal)); - // replay flush failure should result in replay-restart + expect_stop_append(mock_journaler, -EINVAL); + ASSERT_EQ(-EINVAL, when_close(mock_journal)); +} + +TEST_F(TestMockJournal, ReplayOnDiskPreFlushError) { + REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); + + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockImageCtx mock_image_ctx(*ictx); + MockJournal mock_journal(mock_image_ctx); + expect_op_work_queue(mock_image_ctx); + + InSequence seq; + ::journal::MockJournaler mock_journaler; + expect_construct_journaler(mock_journaler); + expect_init_journaler(mock_journaler, 0); + + ::journal::ReplayHandler *replay_handler = nullptr; + expect_start_replay( + mock_image_ctx, mock_journaler, { + std::bind(&invoke_replay_ready, _1), + [&replay_handler] (::journal::ReplayHandler *handler) {replay_handler = handler;}, + }); + + ::journal::MockReplayEntry mock_replay_entry; + MockJournalReplay mock_journal_replay; + expect_try_pop_front(mock_journaler, true, mock_replay_entry); + + Context *on_ready; + EXPECT_CALL(mock_journal_replay, process(_, _, _)) + .WillOnce(DoAll(SaveArg<1>(&on_ready), + WithArg<2>(Invoke(this, &TestMockJournal::save_commit_context)))); + + expect_try_pop_front(mock_journaler, false, mock_replay_entry); + expect_stop_replay(mock_journaler); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); + + // replay write-to-disk failure should result in replay-restart expect_construct_journaler(mock_journaler); expect_init_journaler(mock_journaler, 0); expect_start_replay( @@ -603,15 +661,36 @@ TEST_F(TestMockJournal, FlushReplayError) { }); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); expect_start_append(mock_journaler); - ASSERT_EQ(0, when_open(mock_journal)); + + C_SaferCond ctx; + mock_journal.open(&ctx); + + // wait for the process callback + { + Mutex::Locker locker(m_lock); + while (m_commit_contexts.empty()) { + m_cond.Wait(m_lock); + } + } + on_ready->complete(0); + + // inject RADOS error in the middle of replay + Context *on_safe = m_commit_contexts.front(); + m_commit_contexts.clear(); + on_safe->complete(-EINVAL); + + // flag the replay as complete + replay_handler->handle_complete(0); + + ASSERT_EQ(0, ctx.wait()); expect_stop_append(mock_journaler, 0); ASSERT_EQ(0, when_close(mock_journal)); } -TEST_F(TestMockJournal, StopError) { +TEST_F(TestMockJournal, ReplayOnDiskPostFlushError) { REQUIRE_FEATURE(RBD_FEATURE_JOURNALING); librbd::ImageCtx *ictx; @@ -628,17 +707,52 @@ TEST_F(TestMockJournal, StopError) { expect_init_journaler(mock_journaler, 0); expect_start_replay( mock_image_ctx, mock_journaler, { + std::bind(&invoke_replay_ready, _1), std::bind(&invoke_replay_complete, _1, 0) }); + ::journal::MockReplayEntry mock_replay_entry; MockJournalReplay mock_journal_replay; + expect_try_pop_front(mock_journaler, true, mock_replay_entry); + expect_replay_process(mock_journal_replay); + expect_try_pop_front(mock_journaler, false, mock_replay_entry); expect_stop_replay(mock_journaler); - expect_flush_replay(mock_journal_replay, 0); + + Context *on_flush; + EXPECT_CALL(mock_journal_replay, flush(_)).WillOnce(SaveArg<0>(&on_flush)); + + // replay write-to-disk failure should result in replay-restart + expect_construct_journaler(mock_journaler); + expect_init_journaler(mock_journaler, 0); + expect_start_replay( + mock_image_ctx, mock_journaler, { + std::bind(&invoke_replay_complete, _1, 0) + }); + + expect_stop_replay(mock_journaler); + expect_flush_replay(mock_image_ctx, mock_journal_replay, 0); expect_start_append(mock_journaler); - ASSERT_EQ(0, when_open(mock_journal)); - expect_stop_append(mock_journaler, -EINVAL); - ASSERT_EQ(-EINVAL, when_close(mock_journal)); + C_SaferCond ctx; + mock_journal.open(&ctx); + + { + // wait for the on_safe process callback + Mutex::Locker locker(m_lock); + while (m_commit_contexts.empty()) { + m_cond.Wait(m_lock); + } + } + m_commit_contexts.front()->complete(-EINVAL); + m_commit_contexts.clear(); + + // proceed with the flush + on_flush->complete(0); + + ASSERT_EQ(0, ctx.wait()); + + expect_stop_append(mock_journaler, 0); + ASSERT_EQ(0, when_close(mock_journal)); } TEST_F(TestMockJournal, EventAndIOCommitOrder) {