From: Jason Dillaman Date: Thu, 16 Nov 2017 21:59:33 +0000 (-0500) Subject: rbd-mirror: move image deletion to standalone state machine X-Git-Tag: v13.0.1~134^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a2925d317bdb85c5dcbae0df697fc98141b9b9cc;p=ceph.git rbd-mirror: move image deletion to standalone state machine Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 48f2e249f942..43df18ef1395 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -24,6 +24,7 @@ add_executable(unittest_rbd_mirror test_mock_InstanceWatcher.cc test_mock_LeaderWatcher.cc test_mock_PoolWatcher.cc + image_deleter/test_mock_RemoveRequest.cc image_deleter/test_mock_SnapshotPurgeRequest.cc image_replayer/test_mock_BootstrapRequest.cc image_replayer/test_mock_CreateImageRequest.cc diff --git a/src/test/rbd_mirror/image_deleter/test_mock_RemoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_RemoveRequest.cc new file mode 100644 index 000000000000..8eed1f8e8f5f --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_RemoveRequest.cc @@ -0,0 +1,545 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "cls/rbd/cls_rbd_types.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/image/RemoveRequest.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/RemoveRequest.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librbd/mock/MockImageCtx.h" + +namespace librbd { + +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +template <> +struct Journal { + static Journal *s_instance; + + static void get_tag_owner(librados::IoCtx &io_ctx, + const std::string &image_id, + std::string *mirror_uuid, + ContextWQ *work_queue, + Context *on_finish) { + assert(s_instance != nullptr); + s_instance->get_tag_owner(image_id, mirror_uuid, on_finish); + } + + MOCK_METHOD3(get_tag_owner, void(const std::string &, std::string *, + Context *)); + + Journal() { + s_instance = this; + } +}; + +Journal* Journal::s_instance = nullptr; + +namespace image { + +template <> +struct RemoveRequest { + static RemoveRequest *s_instance; + Context *on_finish = nullptr; + + static RemoveRequest *create(librados::IoCtx &io_ctx, + const std::string &image_name, + const std::string &image_id, + bool force, + bool remove_from_trash, + librbd::ProgressContext &progress_ctx, + ContextWQ *work_queue, + Context *on_finish) { + assert(s_instance != nullptr); + EXPECT_TRUE(image_name.empty()); + EXPECT_TRUE(force); + EXPECT_FALSE(remove_from_trash); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + RemoveRequest() { + s_instance = this; + } +}; + +RemoveRequest* RemoveRequest::s_instance = nullptr; + +} // namespace image +} // namespace librbd + +namespace rbd { +namespace mirror { +namespace image_deleter { + +template <> +struct SnapshotPurgeRequest { + static SnapshotPurgeRequest *s_instance; + Context *on_finish = nullptr; + + static SnapshotPurgeRequest *create(librados::IoCtx &io_ctx, + const std::string &image_id, + Context *on_finish) { + assert(s_instance != nullptr); + s_instance->construct(image_id); + s_instance->on_finish = on_finish; + return s_instance; + } + + MOCK_METHOD1(construct, void(const std::string&)); + MOCK_METHOD0(send, void()); + + SnapshotPurgeRequest() { + s_instance = this; + } +}; + +SnapshotPurgeRequest* SnapshotPurgeRequest::s_instance = nullptr; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/RemoveRequest.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::InSequence; +using ::testing::Return; +using ::testing::StrEq; +using ::testing::WithArg; +using ::testing::WithArgs; + +class TestMockImageDeleterRemoveRequest : public TestMockFixture { +public: + typedef RemoveRequest MockRemoveRequest; + typedef SnapshotPurgeRequest MockSnapshotPurgeRequest; + typedef librbd::Journal MockJournal; + typedef librbd::image::RemoveRequest MockImageRemoveRequest; + + void expect_mirror_image_get_image_id(const std::string& image_id, int r) { + bufferlist bl; + ::encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), StrEq("mirror_image_get_image_id"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_get_tag_owner(MockJournal &mock_journal, + const std::string &image_id, + const std::string &tag_owner, int r) { + EXPECT_CALL(mock_journal, get_tag_owner(image_id, _, _)) + .WillOnce(WithArgs<1, 2>(Invoke([this, tag_owner, r](std::string *owner, Context *on_finish) { + *owner = tag_owner; + m_threads->work_queue->queue(on_finish, r); + }))); + } + + void expect_get_snapcontext(const std::string& image_id, + const ::SnapContext &snapc, int r) { + bufferlist bl; + ::encode(snapc, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(librbd::util::header_name(image_id), _, StrEq("rbd"), + StrEq("get_snapcontext"), _, _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([bl](bufferlist *out_bl) { + *out_bl = bl; + })), + Return(r))); + } + + void expect_mirror_image_set(const std::string& image_id, + const std::string& global_image_id, + cls::rbd::MirrorImageState mirror_image_state, + int r) { + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = global_image_id; + mirror_image.state = mirror_image_state; + + bufferlist bl; + ::encode(image_id, bl); + ::encode(mirror_image, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_set"), ContentsEqual(bl), _, _)) + .WillOnce(Return(r)); + } + + void expect_snapshot_purge(MockSnapshotPurgeRequest &snapshot_purge_request, + const std::string &image_id, int r) { + EXPECT_CALL(snapshot_purge_request, construct(image_id)); + EXPECT_CALL(snapshot_purge_request, send()) + .WillOnce(Invoke([this, &snapshot_purge_request, r]() { + m_threads->work_queue->queue(snapshot_purge_request.on_finish, r); + })); + } + + void expect_image_remove(MockImageRemoveRequest &image_remove_request, + const std::string &image_id, int r) { + EXPECT_CALL(image_remove_request, construct(image_id)); + EXPECT_CALL(image_remove_request, send()) + .WillOnce(Invoke([this, &image_remove_request, r]() { + m_threads->work_queue->queue(image_remove_request.on_finish, r); + })); + } + + void expect_mirror_image_remove(const std::string& image_id, + int r) { + bufferlist bl; + ::encode(image_id, bl); + + EXPECT_CALL(get_mock_io_ctx(m_local_io_ctx), + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_remove"), ContentsEqual(bl), _, _)) + .WillOnce(Return(r)); + } +}; + +TEST_F(TestMockImageDeleterRemoveRequest, Success) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + expect_mirror_image_remove("image id", 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetImageIdDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_COMPLETE, error_result); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetImageIdError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetTagOwnerLocalPrimary) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::LOCAL_MIRROR_UUID, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_COMPLETE, error_result); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetTagOwnerOrphan) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", + librbd::Journal<>::ORPHAN_MIRROR_UUID, 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", false, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EPERM, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_COMPLETE, error_result); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetTagOwnerDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -ENOENT); + + expect_get_snapcontext("image id", {1, {1}}, -ENOENT); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + expect_mirror_image_remove("image id", 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetTagOwnerError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetSnapContextDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, -ENOENT); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + expect_mirror_image_remove("image id", 0); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(0, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, GetSnapContextError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, SetMirrorDNE) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, -ENOENT); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_COMPLETE, error_result); +} + +TEST_F(TestMockImageDeleterRemoveRequest, SetMirrorError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, PurgeSnapshotBusy) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EBUSY); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EBUSY, ctx.wait()); + ASSERT_EQ(ERROR_RESULT_RETRY_IMMEDIATELY, error_result); +} + +TEST_F(TestMockImageDeleterRemoveRequest, PurgeSnapshotError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, RemoveError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +TEST_F(TestMockImageDeleterRemoveRequest, RemoveMirrorError) { + InSequence seq; + expect_mirror_image_get_image_id("image id", 0); + + MockJournal mock_journal; + expect_get_tag_owner(mock_journal, "image id", "remote uuid", 0); + + expect_get_snapcontext("image id", {1, {1}}, 0); + expect_mirror_image_set("image id", "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING, 0); + + MockSnapshotPurgeRequest mock_snapshot_purge_request; + expect_snapshot_purge(mock_snapshot_purge_request, "image id", 0); + + MockImageRemoveRequest mock_image_remove_request; + expect_image_remove(mock_image_remove_request, "image id", 0); + + expect_mirror_image_remove("image id", -EINVAL); + + C_SaferCond ctx; + ErrorResult error_result; + auto req = MockRemoveRequest::create(m_local_io_ctx, "global image id", true, + &error_result, m_threads->work_queue, + &ctx); + req->send(); + ASSERT_EQ(-EINVAL, ctx.wait()); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc index c06a4662532a..f52322704925 100644 --- a/src/test/rbd_mirror/test_ImageDeleter.cc +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -264,7 +264,7 @@ TEST_F(TestImageDeleter, Fail_Delete_Primary_Image) { C_SaferCond ctx; m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, &ctx); - EXPECT_EQ(-rbd::mirror::ImageDeleter<>::EISPRM, ctx.wait()); + EXPECT_EQ(-EPERM, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); @@ -280,7 +280,7 @@ TEST_F(TestImageDeleter, Fail_Delete_Orphan_Image) { C_SaferCond ctx; m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, &ctx); - EXPECT_EQ(-rbd::mirror::ImageDeleter<>::EISPRM, ctx.wait()); + EXPECT_EQ(-EPERM, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); ASSERT_EQ(0u, m_deleter->get_failed_queue_items().size()); diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index 238a6085ef01..1571433d73ff 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -34,7 +34,7 @@ #include "cls/rbd/cls_rbd_types.h" #include "librbd/Utils.h" #include "ImageDeleter.h" -#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" +#include "tools/rbd_mirror/image_deleter/RemoveRequest.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rbd_mirror @@ -266,12 +266,10 @@ bool ImageDeleter::process_image_delete() { m_active_delete->to_string(ss); std::string del_info_str = ss.str(); dout(10) << "start processing delete request: " << del_info_str << dendl; - int r; - cls::rbd::MirrorImage mirror_image; // remote image was disabled, now we need to delete local image IoCtx ioctx; - r = m_active_delete->local_rados->ioctx_create2( + int r = m_active_delete->local_rados->ioctx_create2( m_active_delete->local_pool_id, ioctx); if (r < 0) { derr << "error accessing local pool " << m_active_delete->local_pool_id @@ -282,144 +280,33 @@ bool ImageDeleter::process_image_delete() { dout(20) << "connected to local pool: " << ioctx.get_pool_name() << dendl; - auto &global_image_id = m_active_delete->global_image_id; - std::string local_image_id; - r = librbd::cls_client::mirror_image_get_image_id( - &ioctx, global_image_id, &local_image_id); - if (r == -ENOENT) { - dout(10) << "image " << global_image_id << " is not mirrored" << dendl; - complete_active_delete(r); - return true; - } else if (r < 0) { - derr << "error retrieving local id for image " << global_image_id - << ": " << cpp_strerror(r) << dendl; - enqueue_failed_delete(r); - return true; - } - - std::string mirror_uuid; - C_SaferCond tag_owner_ctx; - Journal<>::get_tag_owner(ioctx, local_image_id, &mirror_uuid, m_work_queue, - &tag_owner_ctx); - r = tag_owner_ctx.wait(); - if (r < 0 && r != -ENOENT) { - derr << "error retrieving image primary info for image " << global_image_id - << ": " << cpp_strerror(r) << dendl; - enqueue_failed_delete(r); - return true; - } else if (r != -ENOENT) { - if (mirror_uuid == Journal<>::LOCAL_MIRROR_UUID) { - dout(10) << "image " << global_image_id << " is local primary" << dendl; - complete_active_delete(-EISPRM); - return true; - } else if (mirror_uuid == Journal<>::ORPHAN_MIRROR_UUID && - !m_active_delete->ignore_orphaned) { - dout(10) << "image " << global_image_id << " is orphaned" << dendl; - complete_active_delete(-EISPRM); - return true; - } - } + C_SaferCond remove_ctx; + image_deleter::ErrorResult error_result; + auto req = image_deleter::RemoveRequest::create( + ioctx, m_active_delete->global_image_id, m_active_delete->ignore_orphaned, + &error_result, m_work_queue, &remove_ctx); + req->send(); - dout(20) << "local image is not the primary" << dendl; - bool has_snapshots; - r = image_has_snapshots_and_children(&ioctx, local_image_id, &has_snapshots); + r = remove_ctx.wait(); if (r < 0) { - enqueue_failed_delete(r); - return true; - } - - mirror_image.global_image_id = global_image_id; - mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; - r = cls_client::mirror_image_set(&ioctx, local_image_id, mirror_image); - if (r == -ENOENT) { - dout(10) << "local image is not mirrored, aborting deletion..." << dendl; - complete_active_delete(r); - return true; - } else if (r == -EEXIST || r == -EINVAL) { - derr << "cannot disable mirroring for image " << global_image_id - << ": global_image_id has changed/reused: " - << cpp_strerror(r) << dendl; - complete_active_delete(r); - return true; - } else if (r < 0) { - derr << "cannot disable mirroring for image " << global_image_id - << ": " << cpp_strerror(r) << dendl; - enqueue_failed_delete(r); - return true; - } - - dout(20) << "set local image mirroring to disable" << dendl; - - if (has_snapshots) { - dout(20) << "local image has snapshots" << dendl; - - C_SaferCond purge_ctx; - auto req = image_deleter::SnapshotPurgeRequest::create( - ioctx, local_image_id, &purge_ctx); - req->send(); - - r = purge_ctx.wait(); - if (r == -EBUSY) { + if (error_result == image_deleter::ERROR_RESULT_COMPLETE) { + complete_active_delete(r); + return true; + } else if (error_result == image_deleter::ERROR_RESULT_RETRY_IMMEDIATELY) { Mutex::Locker l(m_delete_lock); m_active_delete->notify(r); m_delete_queue.push_front(std::move(m_active_delete)); return false; - } else if (r < 0) { + } else { enqueue_failed_delete(r); return true; } } - librbd::NoOpProgressContext ctx; - r = librbd::remove(ioctx, "", local_image_id, ctx, true); - if (r < 0 && r != -ENOENT) { - derr << "error removing image " << global_image_id << " " - << "(" << local_image_id << ") from local pool: " - << cpp_strerror(r) << dendl; - enqueue_failed_delete(r); - return true; - } - - // image was already deleted from rbd_directory, now we will make sure - // that will be also removed from rbd_mirroring - if (r == -ENOENT) { - dout(20) << "local image does not exist, removing image from rbd_mirroring" - << dendl; - } - - r = cls_client::mirror_image_remove(&ioctx, local_image_id); - if (r < 0 && r != -ENOENT) { - derr << "error removing image from mirroring directory: " - << cpp_strerror(r) << dendl; - enqueue_failed_delete(r); - return true; - } - - dout(10) << "Successfully deleted image " - << global_image_id << " " << "(" << local_image_id << ")" << dendl; - complete_active_delete(0); return true; } -template -int ImageDeleter::image_has_snapshots_and_children(IoCtx *ioctx, - string& image_id, - bool *has_snapshots) { - string header_oid = librbd::util::header_name(image_id); - ::SnapContext snapc; - int r = cls_client::get_snapcontext(ioctx, header_oid, &snapc); - if (r < 0 && r != -ENOENT) { - derr << "error retrieving snapshot context for image id " << image_id - << ": " << cpp_strerror(r) << dendl; - return r; - } - - *has_snapshots = !snapc.snaps.empty(); - - return 0; -} - template void ImageDeleter::complete_active_delete(int r) { dout(20) << dendl; diff --git a/src/tools/rbd_mirror/ImageDeleter.h b/src/tools/rbd_mirror/ImageDeleter.h index d3df9ed63304..7742a1c8453c 100644 --- a/src/tools/rbd_mirror/ImageDeleter.h +++ b/src/tools/rbd_mirror/ImageDeleter.h @@ -128,9 +128,6 @@ private: void run(); bool process_image_delete(); - int image_has_snapshots_and_children(librados::IoCtx *ioctx, - std::string& image_id, - bool *has_snapshots); void complete_active_delete(int r); void enqueue_failed_delete(int error_code); diff --git a/src/tools/rbd_mirror/image_deleter/RemoveRequest.cc b/src/tools/rbd_mirror/image_deleter/RemoveRequest.cc index e69de29bb2d1..fde01c62336c 100644 --- a/src/tools/rbd_mirror/image_deleter/RemoveRequest.cc +++ b/src/tools/rbd_mirror/image_deleter/RemoveRequest.cc @@ -0,0 +1,293 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_deleter/RemoveRequest.h" +#include "include/assert.h" +#include "common/dout.h" +#include "common/errno.h" +#include "common/WorkQueue.h" +#include "cls/rbd/cls_rbd_client.h" +#include "librbd/ImageCtx.h" +#include "librbd/Journal.h" +#include "librbd/Utils.h" +#include "librbd/image/RemoveRequest.h" +#include "tools/rbd_mirror/image_deleter/SnapshotPurgeRequest.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_deleter::RemoveRequest: " \ + << this << " " << __func__ << ": " + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +template +void RemoveRequest::send() { + *m_error_result = ERROR_RESULT_RETRY; + + get_mirror_image_id(); +} + +template +void RemoveRequest::get_mirror_image_id() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_get_image_id_start(&op, m_global_image_id); + + auto aio_comp = create_rados_callback< + RemoveRequest, &RemoveRequest::handle_get_mirror_image_id>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void RemoveRequest::handle_get_mirror_image_id(int r) { + dout(10) << "r=" << r << dendl; + + if (r == 0) { + auto bl_it = m_out_bl.begin(); + r = librbd::cls_client::mirror_image_get_image_id_finish(&bl_it, + &m_image_id); + } + if (r == -ENOENT) { + dout(10) << "image " << m_global_image_id << " is not mirrored" << dendl; + *m_error_result = ERROR_RESULT_COMPLETE; + finish(r); + return; + } else if (r < 0) { + derr << "error retrieving local id for image " << m_global_image_id + << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + get_tag_owner(); +} + +template +void RemoveRequest::get_tag_owner() { + dout(10) << dendl; + + auto ctx = create_context_callback< + RemoveRequest, &RemoveRequest::handle_get_tag_owner>(this); + librbd::Journal::get_tag_owner(m_io_ctx, m_image_id, &m_mirror_uuid, + m_op_work_queue, ctx); +} + +template +void RemoveRequest::handle_get_tag_owner(int r) { + dout(10) << "r=" << r << dendl; + + if (r < 0 && r != -ENOENT) { + derr << "error retrieving image primary info for image " + << m_global_image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } else if (r != -ENOENT) { + if (m_mirror_uuid == librbd::Journal<>::LOCAL_MIRROR_UUID) { + dout(10) << "image " << m_global_image_id << " is local primary" << dendl; + + *m_error_result = ERROR_RESULT_COMPLETE; + finish(-EPERM); + return; + } else if (m_mirror_uuid == librbd::Journal<>::ORPHAN_MIRROR_UUID && + !m_ignore_orphaned) { + dout(10) << "image " << m_global_image_id << " is orphaned" << dendl; + + *m_error_result = ERROR_RESULT_COMPLETE; + finish(-EPERM); + return; + } + } + + get_snap_context(); +} + +template +void RemoveRequest::get_snap_context() { + dout(10) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::get_snapcontext_start(&op); + + std::string header_oid = librbd::util::header_name(m_image_id); + + auto aio_comp = create_rados_callback< + RemoveRequest, &RemoveRequest::handle_get_snap_context>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(header_oid, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void RemoveRequest::handle_get_snap_context(int r) { + dout(10) << "r=" << r << dendl; + + ::SnapContext snapc; + if (r == 0) { + auto bl_it = m_out_bl.begin(); + r = librbd::cls_client::get_snapcontext_finish(&bl_it, &snapc); + } + if (r < 0 && r != -ENOENT) { + derr << "error retrieving snapshot context for image " + << m_global_image_id << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + m_has_snapshots = (!snapc.empty()); + set_mirror_image_disabling(); +} + +template +void RemoveRequest::set_mirror_image_disabling() { + dout(10) << dendl; + + cls::rbd::MirrorImage mirror_image; + mirror_image.global_image_id = m_global_image_id; + mirror_image.state = cls::rbd::MIRROR_IMAGE_STATE_DISABLING; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_set(&op, m_image_id, mirror_image); + + auto aio_comp = create_rados_callback< + RemoveRequest, + &RemoveRequest::handle_set_mirror_image_disabling>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + assert(r == 0); + aio_comp->release(); +} + +template +void RemoveRequest::handle_set_mirror_image_disabling(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -ENOENT) { + dout(10) << "local image is not mirrored, aborting deletion." << dendl; + *m_error_result = ERROR_RESULT_COMPLETE; + finish(r); + return; + } else if (r == -EEXIST || r == -EINVAL) { + derr << "cannot disable mirroring for image " << m_global_image_id + << ": global_image_id has changed/reused: " + << cpp_strerror(r) << dendl; + *m_error_result = ERROR_RESULT_COMPLETE; + finish(r); + return; + } else if (r < 0) { + derr << "cannot disable mirroring for image " << m_global_image_id + << ": " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + purge_snapshots(); +} + +template +void RemoveRequest::purge_snapshots() { + if (!m_has_snapshots) { + remove_image(); + return; + } + + dout(10) << dendl; + auto ctx = create_context_callback< + RemoveRequest, &RemoveRequest::handle_purge_snapshots>(this); + auto req = SnapshotPurgeRequest::create(m_io_ctx, m_image_id, ctx); + req->send(); +} + +template +void RemoveRequest::handle_purge_snapshots(int r) { + dout(10) << "r=" << r << dendl; + + if (r == -EBUSY) { + dout(10) << "snapshots still in-use" << dendl; + *m_error_result = ERROR_RESULT_RETRY_IMMEDIATELY; + finish(r); + return; + } else if (r < 0) { + derr << "failed to purge image snapshots: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_image(); +} + +template +void RemoveRequest::remove_image() { + dout(10) << dendl; + + auto ctx = create_context_callback< + RemoveRequest, &RemoveRequest::handle_remove_image>(this); + auto req = librbd::image::RemoveRequest::create( + m_io_ctx, "", m_image_id, true, false, m_progress_ctx, m_op_work_queue, + ctx); + req->send(); +} + +template +void RemoveRequest::handle_remove_image(int r) { + dout(10) << "r=" << r << dendl; + if (r < 0 && r != -ENOENT) { + derr << "error removing image " << m_global_image_id << " " + << "(" << m_image_id << ") from local pool: " + << cpp_strerror(r) << dendl; + finish(r); + return; + } + + remove_mirror_image(); +} + +template +void RemoveRequest::remove_mirror_image() { + dout(10) << dendl; + + librados::ObjectWriteOperation op; + librbd::cls_client::mirror_image_remove(&op, m_image_id); + + auto aio_comp = create_rados_callback< + RemoveRequest, &RemoveRequest::handle_remove_mirror_image>(this); + int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); + assert(r == 0); + aio_comp->release(); +} + +template +void RemoveRequest::handle_remove_mirror_image(int r) { + dout(10) << "r=" << r << dendl; + if (r < 0 && r != -ENOENT) { + derr << "error removing image " << m_global_image_id << " from mirroring " + << "directory: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + finish(0); +} + +template +void RemoveRequest::finish(int r) { + dout(10) << "r=" << r << dendl; + + m_on_finish->complete(r); + delete this; +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_deleter::RemoveRequest; diff --git a/src/tools/rbd_mirror/image_deleter/RemoveRequest.h b/src/tools/rbd_mirror/image_deleter/RemoveRequest.h new file mode 100644 index 000000000000..da9107afab96 --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/RemoveRequest.h @@ -0,0 +1,120 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_REMOVE_REQUEST_H +#define CEPH_RBD_MIRROR_IMAGE_DELETER_REMOVE_REQUEST_H + +#include "include/rados/librados.hpp" +#include "include/buffer.h" +#include "librbd/internal.h" +#include "tools/rbd_mirror/image_deleter/Types.h" +#include +#include + +class Context; +class ContextWQ; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { +namespace image_deleter { + +template +class RemoveRequest { +public: + static RemoveRequest* create(librados::IoCtx &io_ctx, + const std::string &global_image_id, + bool ignore_orphaned, ErrorResult *error_result, + ContextWQ *op_work_queue, Context *on_finish) { + return new RemoveRequest(io_ctx, global_image_id, ignore_orphaned, + error_result, op_work_queue, on_finish); + } + + RemoveRequest(librados::IoCtx &io_ctx, const std::string &global_image_id, + bool ignore_orphaned, ErrorResult *error_result, + ContextWQ *op_work_queue, Context *on_finish) + : m_io_ctx(io_ctx), m_global_image_id(global_image_id), + m_ignore_orphaned(ignore_orphaned), m_error_result(error_result), + m_op_work_queue(op_work_queue), m_on_finish(on_finish) { + } + + void send(); + +private: + /* + * @verbatim + * + * + * | + * v + * GET_MIRROR_IMAGE_ID + * | + * v + * GET_TAG_OWNER + * | + * v + * GET_SNAP_CONTEXT + * | + * v + * SET_MIRROR_IMAGE_DISABLING + * | + * v + * PURGE_SNAPSHOTS + * | + * v + * REMOVE_IMAGE + * | + * v + * REMOVE_MIRROR_IMAGE + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx &m_io_ctx; + std::string m_global_image_id; + bool m_ignore_orphaned; + ErrorResult *m_error_result; + ContextWQ *m_op_work_queue; + Context *m_on_finish; + + ceph::bufferlist m_out_bl; + std::string m_image_id; + std::string m_mirror_uuid; + bool m_has_snapshots = false; + librbd::NoOpProgressContext m_progress_ctx; + + void get_mirror_image_id(); + void handle_get_mirror_image_id(int r); + + void get_tag_owner(); + void handle_get_tag_owner(int r); + + void get_snap_context(); + void handle_get_snap_context(int r); + + void set_mirror_image_disabling(); + void handle_set_mirror_image_disabling(int r); + + void purge_snapshots(); + void handle_purge_snapshots(int r); + + void remove_image(); + void handle_remove_image(int r); + + void remove_mirror_image(); + void handle_remove_mirror_image(int r); + + void finish(int r); + +}; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_deleter::RemoveRequest; + +#endif // CEPH_RBD_MIRROR_IMAGE_DELETER_REMOVE_REQUEST_H diff --git a/src/tools/rbd_mirror/image_deleter/Types.h b/src/tools/rbd_mirror/image_deleter/Types.h new file mode 100644 index 000000000000..13e94c5d3f70 --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/Types.h @@ -0,0 +1,21 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H +#define CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H + +namespace rbd { +namespace mirror { +namespace image_deleter { + +enum ErrorResult { + ERROR_RESULT_COMPLETE, + ERROR_RESULT_RETRY, + ERROR_RESULT_RETRY_IMMEDIATELY +}; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +#endif // CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H