From 8f485ec8543defb3d7d55f681fa4d637095d8776 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 13 Dec 2017 22:53:30 -0500 Subject: [PATCH] rbd-mirror: async mirroring trash move state machine Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/CMakeLists.txt | 1 + .../test_mock_TrashMoveRequest.cc | 671 ++++++++++++++++++ src/tools/rbd_mirror/CMakeLists.txt | 1 + .../image_deleter/SnapshotPurgeRequest.cc | 20 +- .../image_deleter/TrashMoveRequest.cc | 382 ++++++++++ .../image_deleter/TrashMoveRequest.h | 136 ++++ src/tools/rbd_mirror/image_deleter/Types.h | 15 + 7 files changed, 1208 insertions(+), 18 deletions(-) create mode 100644 src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc create mode 100644 src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc create mode 100644 src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 8f3db31b8d7..e5b4d3d3054 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(unittest_rbd_mirror test_mock_PoolWatcher.cc image_deleter/test_mock_RemoveRequest.cc image_deleter/test_mock_SnapshotPurgeRequest.cc + image_deleter/test_mock_TrashMoveRequest.cc image_deleter/test_mock_TrashWatcher.cc image_replayer/test_mock_BootstrapRequest.cc image_replayer/test_mock_CreateImageRequest.cc diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc new file mode 100644 index 00000000000..43eb524bb79 --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc @@ -0,0 +1,671 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Operations.h" +#include "librbd/TrashWatcher.h" +#include "librbd/journal/ResetRequest.h" +#include "librbd/trash/MoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockExclusiveLock.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/librbd/mock/MockImageState.h" +#include "test/librbd/mock/MockOperations.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + static MockTestImageCtx *s_instance; + static MockTestImageCtx *create(const std::string &image_name, + const std::string &image_id, + const char *snap, librados::IoCtx& p, + bool read_only) { + assert(s_instance != nullptr); + return s_instance; + } + + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + s_instance = this; + } +}; + +MockTestImageCtx *MockTestImageCtx::s_instance = nullptr; + +} // anonymous namespace + +template <> +struct Journal { + static Journal *s_instance; + + static void get_tag_owner(librados::IoCtx &io_ctx, + const std::string &image_id, + std::string *mirror_uuid, + ContextWQ *work_queue, + Context *on_finish) { + assert(s_instance != nullptr); + s_instance->get_tag_owner(image_id, mirror_uuid, on_finish); + } + + MOCK_METHOD3(get_tag_owner, void(const std::string &, std::string *, + Context *)); + + Journal() { + s_instance = this; + } +}; + +Journal* Journal::s_instance = nullptr; + +template<> +struct TrashWatcher { + static TrashWatcher* s_instance; + static void notify_image_added(librados::IoCtx&, const std::string& image_id, + const cls::rbd::TrashImageSpec& spec, + Context *ctx) { + assert(s_instance != nullptr); + s_instance->notify_image_added(image_id, spec, ctx); + } + + MOCK_METHOD3(notify_image_added, void(const std::string&, + const cls::rbd::TrashImageSpec&, + Context*)); + + TrashWatcher() { + s_instance = this; + } +}; + +TrashWatcher* TrashWatcher::s_instance = nullptr; + +namespace journal { + +template <> +struct ResetRequest { + static ResetRequest* s_instance; + Context* on_finish = nullptr; + + static ResetRequest* create(librados::IoCtx &io_ctx, + const std::string &image_id, + const std::string &client_id, + const std::string &mirror_uuid, + ContextWQ *op_work_queue, Context *on_finish) { + assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + ResetRequest() { + s_instance = this; + } +}; + +ResetRequest* ResetRequest::s_instance = nullptr; + +} // namespace journal + +namespace trash { + +template <> +struct MoveRequest { + static MoveRequest* s_instance; + Context* on_finish = nullptr; + + typedef boost::optional DefermentEndTime; + + static MoveRequest* create(librados::IoCtx& io_ctx, + const std::string& image_id, + const cls::rbd::TrashImageSpec& trash_image_spec, + Context* on_finish) { + assert(s_instance != nullptr); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD0(send, void()); + + MoveRequest() { + s_instance = this; + } +}; + +MoveRequest* MoveRequest::s_instance = nullptr; + +} // namespace trash +} // namespace librbd + +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterTrashMoveRequest : public TestMockFixture { +public: + typedef TrashMoveRequest MockTrashMoveRequest; + typedef librbd::Journal MockJournal; + typedef librbd::journal::ResetRequest MockJournalResetRequest; + typedef librbd::trash::MoveRequest MockLibrbdTrashMoveRequest; + typedef librbd::TrashWatcher MockTrashWatcher; + + void SetUp() override { + TestMockFixture::SetUp(); + + librbd::RBD rbd; + ASSERT_EQ(0, create_image(rbd, m_local_io_ctx, m_image_name, m_image_size)); + ASSERT_EQ(0, open_image(m_local_io_ctx, m_image_name, &m_local_image_ctx)); + } + + void expect_mirror_image_get_image_id(const std::string& image_id, int r) { + bufferlist bl; + ::encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_tag_owner(MockJournal &mock_journal, + const std::string &image_id, + const std::string &tag_owner, int r) { + EXPECT_CALL(mock_journal, get_tag_owner(image_id, _, _)) + .WillOnce(WithArgs<1, 2>(Invoke([this, tag_owner, r](std::string *owner, Context *on_finish) { + *owner = tag_owner; + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_set_journal_policy(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, set_journal_policy(_)) + .WillOnce(Invoke([](librbd::journal::Policy* policy) { + delete policy; + })); + } + + void expect_open(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, open(true, _)) + .WillOnce(WithArg<1>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + + void expect_close(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.state, close(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_block_requests(librbd::MockTestImageCtx &mock_image_ctx) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, block_requests(0)).Times(1); + } + + void expect_acquire_lock(librbd::MockTestImageCtx &mock_image_ctx, int r) { + EXPECT_CALL(*mock_image_ctx.exclusive_lock, acquire_lock(_)) + .WillOnce(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_destroy(librbd::MockTestImageCtx& mock_image_ctx) { + EXPECT_CALL(mock_image_ctx, destroy()); + } + + void expect_mirror_image_set(const std::string& image_id, + const std::string& global_image_id, + cls::rbd::MirrorImageState mirror_image_state, + int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = global_image_id; + mirror_image.state = mirror_image_state; + + bufferlist bl; + ::encode(image_id, bl); + ::encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_set"), ContentsEqual(bl), _, _)) + .WillOnce(Return(r)); + } + + void expect_mirror_image_remove(librados::IoCtx &ioctx, int r) { + EXPECT_CALL(get_mock_io_ctx(ioctx), + exec(StrEq("rbd_mirroring"), _, StrEq("rbd"), StrEq("mirror_image_remove"), + _, _, _)) + .WillOnce(Return(r)); + } + + void expect_journal_reset(MockJournalResetRequest& mock_journal_reset_request, + int r) { + EXPECT_CALL(mock_journal_reset_request, send()) + .WillOnce(Invoke([this, &mock_journal_reset_request, r]() { + m_threads->work_queue->queue(mock_journal_reset_request.on_finish, r); + })); + } + + void expect_trash_move(MockLibrbdTrashMoveRequest& mock_trash_move_request, + int r) { + EXPECT_CALL(mock_trash_move_request, send()) + .WillOnce(Invoke([this, &mock_trash_move_request, r]() { + m_threads->work_queue->queue(mock_trash_move_request.on_finish, r); + })); + } + + void expect_notify_image_added(MockTrashWatcher& mock_trash_watcher, + const std::string& image_id) { + EXPECT_CALL(mock_trash_watcher, notify_image_added(image_id, _, _)) + .WillOnce(WithArg<2>(Invoke([this](Context *ctx) { + m_threads->work_queue->queue(ctx, 0); + }))); + } + + librbd::ImageCtx *m_local_image_ctx; +}; + +TEST_F(TestMockImageDeleterTrashMoveRequest, Success) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -ENOENT); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerLocalPrimary) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::LOCAL_MIRROR_UUID, 0); + + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerOrphan) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + false, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerDNE) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -ENOENT); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, GetTagOwnerError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, DisableMirrorImageError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, ResetJournalError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, -EINVAL); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, OpenImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, AcquireLockError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, TrashMoveError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, RemoveMirrorImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, 0); + expect_mirror_image_remove(m_local_io_ctx, -EINVAL); + + expect_close(mock_image_ctx, 0); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) { + librbd::MockTestImageCtx mock_image_ctx(*m_local_image_ctx); + librbd::MockExclusiveLock mock_exclusive_lock; + mock_image_ctx.exclusive_lock = &mock_exclusive_lock; + + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockJournalResetRequest mock_journal_reset_request; + expect_journal_reset(mock_journal_reset_request, 0); + + expect_set_journal_policy(mock_image_ctx); + expect_open(mock_image_ctx, 0); + expect_block_requests(mock_image_ctx); + expect_acquire_lock(mock_image_ctx, 0); + + MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; + expect_trash_move(mock_librbd_trash_move_request, 0); + expect_mirror_image_remove(m_local_io_ctx, 0); + + expect_close(mock_image_ctx, -EINVAL); + expect_destroy(mock_image_ctx); + + MockTrashWatcher mock_trash_watcher; + expect_notify_image_added(mock_trash_watcher, "image id"); + + C_SaferCond ctx; + auto req = MockTrashMoveRequest::create(m_local_io_ctx, "global image id", + true, + m_local_image_ctx->op_work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 8f74007c1c4..9a3b8e18bcc 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -23,6 +23,7 @@ set(rbd_mirror_internal types.cc image_deleter/RemoveRequest.cc image_deleter/SnapshotPurgeRequest.cc + image_deleter/TrashMoveRequest.cc image_deleter/TrashWatcher.cc image_map/Action.cc image_map/LoadRequest.cc diff --git a/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc index 80e46101daa..c417d9f90e5 100644 --- a/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc +++ b/src/tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.cc @@ -9,6 +9,7 @@ #include "librbd/Operations.h" #include "librbd/Utils.h" #include "librbd/journal/Policy.h" +#include "tools/rbd_mirror/image_deleter/Types.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -20,23 +21,6 @@ namespace rbd { namespace mirror { namespace image_deleter { -namespace { - -struct DeleteJournalPolicy : public librbd::journal::Policy { - bool append_disabled() const override { - return true; - } - bool journal_disabled() const override { - return false; - } - - void allocate_tag_on_lock(Context *on_finish) override { - on_finish->complete(0); - } -}; - -} // anonymous namespace - using librbd::util::create_context_callback; template @@ -51,7 +35,7 @@ void SnapshotPurgeRequest::open_image() { { RWLock::WLocker snap_locker(m_image_ctx->snap_lock); - m_image_ctx->set_journal_policy(new DeleteJournalPolicy()); + m_image_ctx->set_journal_policy(new JournalPolicy()); } Context *ctx = create_context_callback< diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc new file mode 100644 index 00000000000..309894f5a07 --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc @@ -0,0 +1,382 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" +#include "include/rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "common/debug.h" +#include "common/errno.h" +#include "librbd/ExclusiveLock.h" +#include "librbd/ImageCtx.h" +#include "librbd/ImageState.h" +#include "librbd/Journal.h" +#include "librbd/TrashWatcher.h" +#include "librbd/Utils.h" +#include "librbd/journal/ResetRequest.h" +#include "librbd/trash/MoveRequest.h" +#include "tools/rbd_mirror/image_deleter/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashMoveRequest: " \ + << this << " " << __func__ << ": " +namespace rbd { +namespace mirror { +namespace image_deleter { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void TrashMoveRequest::send() { + get_mirror_image_id(); +} + +template +void TrashMoveRequest::get_mirror_image_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id); + + auto aio_comp = create_rados_callback< + TrashMoveRequest, + &TrashMoveRequest::handle_get_mirror_image_id>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void TrashMoveRequest::handle_get_mirror_image_id(int r) { + dout(10) << "r=" << r << dendl; + + if (r == 0) { + auto bl_it = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_get_image_id_finish(&bl_it, + &m_image_id); + } + if (r == -ENOENT) { + dout(10) << "image " << m_global_image_id << " is not mirrored" << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "error retrieving local id for image " << m_global_image_id << ": " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_tag_owner(); +} + +template +void TrashMoveRequest::get_tag_owner() { + dout(10) << dendl; + + auto ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_get_tag_owner>(this); + librbd::Journal::get_tag_owner(m_io_ctx, m_image_id, &m_mirror_uuid, + m_op_work_queue, ctx); +} + +template +void TrashMoveRequest::handle_get_tag_owner(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error retrieving image primary info for image " + << m_global_image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (r != -ENOENT) { + if (m_mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(10) << "image " << m_global_image_id << " is local primary" << dendl; + finish(-EPERM); + return; + } else if (m_mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && + !m_resync) { + dout(10) << "image " << m_global_image_id << " is orphaned" << dendl; + finish(-EPERM); + return; + } + } + + disable_mirror_image(); +} + +template +void TrashMoveRequest::disable_mirror_image() { + dout(10) << dendl; + + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = m_global_image_id; + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_set(&op, m_image_id, mirror_image); + + auto aio_comp = create_rados_callback< + TrashMoveRequest, + &TrashMoveRequest::handle_disable_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + assert(r == 0); + aio_comp->release(); +} + +template +void TrashMoveRequest::handle_disable_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "local image is not mirrored, aborting deletion." << dendl; + finish(r); + return; + } else if (r == -EEXIST || r == -EINVAL) { + derr << "cannot disable mirroring for image " << m_global_image_id + << ": global_image_id has changed/reused: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (r < 0) { + derr << "cannot disable mirroring for image " << m_global_image_id + << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + reset_journal(); +} + +template +void TrashMoveRequest::reset_journal() { + dout(10) << dendl; + + // ensure that if the image is recovered any peers will split-brain + auto ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_reset_journal>(this); + auto req = librbd::journal::ResetRequest::create( + m_io_ctx, m_image_id, librbd::Journal<>::IMAGE_CLIENT_ID, + m_mirror_uuid, m_op_work_queue, ctx); + req->send(); +} + +template +void TrashMoveRequest::handle_reset_journal(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "failed to reset journal: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + open_image(); +} + +template +void TrashMoveRequest::open_image() { + dout(10) << dendl; + + m_image_ctx = I::create("", m_image_id, nullptr, m_io_ctx, false); + + { + // don't attempt to open the journal + RWLock::WLocker snap_locker(m_image_ctx->snap_lock); + m_image_ctx->set_journal_policy(new JournalPolicy()); + } + + Context *ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_open_image>(this); + m_image_ctx->state->open(true, ctx); +} + +template +void TrashMoveRequest::handle_open_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to open image: " << cpp_strerror(r) << dendl; + m_image_ctx->destroy(); + m_image_ctx = nullptr; + finish(r); + return; + } + + if (m_image_ctx->old_format) { + derr << "cannot move v1 image to trash" << dendl; + m_ret_val = -EINVAL; + close_image(); + return; + } + + acquire_lock(); +} + +template +void TrashMoveRequest::acquire_lock() { + m_image_ctx->owner_lock.get_read(); + if (m_image_ctx->exclusive_lock == nullptr) { + derr << "exclusive lock feature not enabled" << dendl; + m_image_ctx->owner_lock.put_read(); + m_ret_val = -EINVAL; + close_image(); + return; + } + + dout(10) << dendl; + + Context *ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_acquire_lock>(this); + m_image_ctx->exclusive_lock->block_requests(0); + m_image_ctx->exclusive_lock->acquire_lock(ctx); + m_image_ctx->owner_lock.put_read(); +} + +template +void TrashMoveRequest::handle_acquire_lock(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to acquire exclusive lock: " << cpp_strerror(r) << dendl; + m_ret_val = r; + close_image(); + return; + } + + trash_move(); +} + +template +void TrashMoveRequest::trash_move() { + dout(10) << dendl; + + // TODO support configurable "deletion delay" + utime_t delete_time{ceph_clock_now()}; + utime_t deferment_end_time{delete_time}; + m_trash_image_spec = { + cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, m_image_ctx->name, delete_time, + deferment_end_time}; + + Context *ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_trash_move>(this); + auto req = librbd::trash::MoveRequest::create( + m_io_ctx, m_image_id, m_trash_image_spec, ctx); + req->send(); +} + +template +void TrashMoveRequest::handle_trash_move(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to move image to trash: " << cpp_strerror(r) << dendl; + m_ret_val = r; + close_image(); + return; + } + + m_moved_to_trash = true; + remove_mirror_image(); +} + +template +void TrashMoveRequest::remove_mirror_image() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_remove(&op, m_image_id); + + auto aio_comp = create_rados_callback< + TrashMoveRequest, + &TrashMoveRequest::handle_remove_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + assert(r == 0); + aio_comp->release(); +} + +template +void TrashMoveRequest::handle_remove_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "local image is not mirrored" << dendl; + } else if (r < 0) { + derr << "failed to remove mirror image state for " << m_global_image_id + << ": " << cpp_strerror(r) << dendl; + m_ret_val = r; + } + + close_image(); +} + +template +void TrashMoveRequest::close_image() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_close_image>(this); + m_image_ctx->state->close(ctx); +} + +template +void TrashMoveRequest::handle_close_image(int r) { + dout(10) << "r=" << r << dendl; + + m_image_ctx->destroy(); + m_image_ctx = nullptr; + + if (r < 0) { + derr << "failed to close image: " << cpp_strerror(r) << dendl; + } + + // don't send notification if we failed + if (!m_moved_to_trash) { + finish(0); + return; + } + + notify_trash_add(); +} + +template +void TrashMoveRequest::notify_trash_add() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + TrashMoveRequest, &TrashMoveRequest::handle_notify_trash_add>(this); + librbd::TrashWatcher::notify_image_added(m_io_ctx, m_image_id, + m_trash_image_spec, ctx); +} + +template +void TrashMoveRequest::handle_notify_trash_add(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0) { + derr << "failed to notify trash watchers: " << cpp_strerror(r) << dendl; + } + + finish(0); +} + +template +void TrashMoveRequest::finish(int r) { + if (m_ret_val < 0) { + r = m_ret_val; + } + + dout(10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_deleter::TrashMoveRequest; + diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h new file mode 100644 index 00000000000..07b7432edf1 --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.h @@ -0,0 +1,136 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_MOVE_REQUEST_H +#define CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_MOVE_REQUEST_H + +#include "include/buffer.h" +#include "include/rados/librados.hpp" +#include "cls/rbd/cls_rbd_types.h" +#include +#include + +struct Context; +class ContextWQ; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_deleter { + +template +class TrashMoveRequest { +public: + static TrashMoveRequest* create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + bool resync, ContextWQ* op_work_queue, + Context* on_finish) { + return new TrashMoveRequest(io_ctx, global_image_id, resync, op_work_queue, + on_finish); + } + + TrashMoveRequest(librados::IoCtx& io_ctx, const std::string& global_image_id, + bool resync, ContextWQ* op_work_queue, Context* on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), m_resync(resync), + m_op_work_queue(op_work_queue), m_on_finish(on_finish) { + } + + void send(); + +private: + /* + * @verbatim + * + * + * | + * v + * GET_MIRROR_IMAGE_ID + * | + * v + * GET_TAG_OWNER + * | + * v + * DISABLE_MIRROR_IMAGE + * | + * v + * RESET_JOURNAL + * | + * v + * OPEN_IMAGE + * | + * v + * ACQUIRE_LOCK + * | + * v + * TRASH_MOVE + * | + * v + * REMOVE_MIRROR_IMAGE + * | + * v + * CLOSE_IMAGE + * | + * v + * NOTIFY_TRASH_ADD + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx &m_io_ctx; + std::string m_global_image_id; + bool m_resync; + ContextWQ *m_op_work_queue; + Context *m_on_finish; + + ceph::bufferlist m_out_bl; + std::string m_image_id; + std::string m_mirror_uuid; + cls::rbd::TrashImageSpec m_trash_image_spec; + ImageCtxT *m_image_ctx = nullptr;; + int m_ret_val = 0; + bool m_moved_to_trash = false; + + void get_mirror_image_id(); + void handle_get_mirror_image_id(int r); + + void get_tag_owner(); + void handle_get_tag_owner(int r); + + void disable_mirror_image(); + void handle_disable_mirror_image(int r); + + void reset_journal(); + void handle_reset_journal(int r); + + void open_image(); + void handle_open_image(int r); + + void acquire_lock(); + void handle_acquire_lock(int r); + + void trash_move(); + void handle_trash_move(int r); + + void remove_mirror_image(); + void handle_remove_mirror_image(int r); + + void close_image(); + void handle_close_image(int r); + + void notify_trash_add(); + void handle_notify_trash_add(int r); + + void finish(int r); + +}; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_deleter::TrashMoveRequest; + +#endif // CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H diff --git a/src/tools/rbd_mirror/image_deleter/Types.h b/src/tools/rbd_mirror/image_deleter/Types.h index 12131f15ed5..ac3bc64afb7 100644 --- a/src/tools/rbd_mirror/image_deleter/Types.h +++ b/src/tools/rbd_mirror/image_deleter/Types.h @@ -4,6 +4,8 @@ #ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H #define CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H +#include "include/Context.h" +#include "librbd/journal/Policy.h" #include struct utime_t; @@ -32,6 +34,19 @@ struct TrashListener { }; +struct JournalPolicy : public librbd::journal::Policy { + bool append_disabled() const override { + return true; + } + bool journal_disabled() const override { + return true; + } + + void allocate_tag_on_lock(Context *on_finish) override { + on_finish->complete(0); + } +}; + } // namespace image_deleter } // namespace mirror } // namespace rbd -- 2.39.5