From 62998f3de5d14ab8261c54b095997c54654eb598 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 29 Nov 2017 17:26:45 -0500 Subject: [PATCH] rbd-mirror: watch notifications for images added to trash Signed-off-by: Jason Dillaman --- src/test/rbd_mirror/CMakeLists.txt | 1 + .../image_deleter/test_mock_TrashWatcher.cc | 480 ++++++++++++++++++ src/tools/rbd_mirror/CMakeLists.txt | 1 + .../rbd_mirror/image_deleter/TrashWatcher.cc | 370 ++++++++++++++ .../rbd_mirror/image_deleter/TrashWatcher.h | 133 +++++ src/tools/rbd_mirror/image_deleter/Types.h | 18 + 6 files changed, 1003 insertions(+) create mode 100644 src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc create mode 100644 src/tools/rbd_mirror/image_deleter/TrashWatcher.cc create mode 100644 src/tools/rbd_mirror/image_deleter/TrashWatcher.h diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt index 43df18ef139..8f3db31b8d7 100644 --- a/src/test/rbd_mirror/CMakeLists.txt +++ b/src/test/rbd_mirror/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(unittest_rbd_mirror test_mock_PoolWatcher.cc image_deleter/test_mock_RemoveRequest.cc image_deleter/test_mock_SnapshotPurgeRequest.cc + image_deleter/test_mock_TrashWatcher.cc image_replayer/test_mock_BootstrapRequest.cc image_replayer/test_mock_CreateImageRequest.cc image_replayer/test_mock_EventPreprocessor.cc diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc new file mode 100644 index 00000000000..c401e37cb3d --- /dev/null +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashWatcher.cc @@ -0,0 +1,480 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "test/rbd_mirror/test_mock_fixture.h" +#include "test/librados_test_stub/MockTestMemIoCtxImpl.h" +#include "test/librados_test_stub/MockTestMemRadosClient.h" +#include "test/librbd/mock/MockImageCtx.h" +#include "test/rbd_mirror/mock/MockContextWQ.h" +#include "test/rbd_mirror/mock/MockSafeTimer.h" +#include "librbd/TrashWatcher.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/TrashWatcher.h" + +namespace librbd { +namespace { + +struct MockTestImageCtx : public librbd::MockImageCtx { + MockTestImageCtx(librbd::ImageCtx &image_ctx) + : librbd::MockImageCtx(image_ctx) { + } +}; + +} // anonymous namespace + +struct MockTrashWatcher { + static MockTrashWatcher *s_instance; + static MockTrashWatcher &get_instance() { + assert(s_instance != nullptr); + return *s_instance; + } + + MockTrashWatcher() { + s_instance = this; + } + + MOCK_CONST_METHOD0(is_unregistered, bool()); + MOCK_METHOD1(register_watch, void(Context*)); + MOCK_METHOD1(unregister_watch, void(Context*)); +}; + +template <> +struct TrashWatcher { + static TrashWatcher *s_instance; + + TrashWatcher(librados::IoCtx &io_ctx, ::MockContextWQ *work_queue) { + s_instance = this; + } + virtual ~TrashWatcher() { + } + + static TrashWatcher &get_instance() { + assert(s_instance != nullptr); + return *s_instance; + } + + virtual void handle_rewatch_complete(int r) = 0; + + virtual void handle_image_added(const std::string &image_id, + const cls::rbd::TrashImageSpec& spec) = 0; + virtual void handle_image_removed(const std::string &image_id) = 0; + + bool is_unregistered() const { + return MockTrashWatcher::get_instance().is_unregistered(); + } + void register_watch(Context *ctx) { + MockTrashWatcher::get_instance().register_watch(ctx); + } + void unregister_watch(Context *ctx) { + MockTrashWatcher::get_instance().unregister_watch(ctx); + } +}; + +MockTrashWatcher *MockTrashWatcher::s_instance = nullptr; +TrashWatcher *TrashWatcher::s_instance = nullptr; + +} // namespace librbd + +namespace rbd { +namespace mirror { + +template <> +struct Threads { + MockSafeTimer *timer; + Mutex &timer_lock; + + MockContextWQ *work_queue; + + Threads(Threads *threads) + : timer(new MockSafeTimer()), + timer_lock(threads->timer_lock), + work_queue(new MockContextWQ()) { + } + ~Threads() { + delete timer; + delete work_queue; + } +}; + +} // namespace mirror +} // namespace rbd + +#include "tools/rbd_mirror/image_deleter/TrashWatcher.cc" + +namespace rbd { +namespace mirror { +namespace image_deleter { + +using ::testing::_; +using ::testing::DoAll; +using ::testing::InSequence; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::ReturnArg; +using ::testing::StrEq; +using ::testing::WithArg; + +class TestMockImageDeleterTrashWatcher : public TestMockFixture { +public: + typedef TrashWatcher MockTrashWatcher; + typedef Threads MockThreads; + typedef librbd::MockTrashWatcher MockLibrbdTrashWatcher; + typedef librbd::TrashWatcher LibrbdTrashWatcher; + + struct MockListener : TrashListener { + MOCK_METHOD2(handle_trash_image, void(const std::string&, const utime_t&)); + }; + + void expect_work_queue(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.work_queue, queue(_, _)) + .WillRepeatedly(Invoke([this](Context *ctx, int r) { + m_threads->work_queue->queue(ctx, r); + })); + } + + void expect_trash_watcher_is_unregistered(MockLibrbdTrashWatcher &mock_trash_watcher, + bool unregistered) { + EXPECT_CALL(mock_trash_watcher, is_unregistered()) + .WillOnce(Return(unregistered)); + } + + void expect_trash_watcher_register(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, register_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_trash_watcher_unregister(MockLibrbdTrashWatcher &mock_trash_watcher, + int r) { + EXPECT_CALL(mock_trash_watcher, unregister_watch(_)) + .WillOnce(CompleteContext(r)); + } + + void expect_create_trash(librados::IoCtx &io_ctx, int r) { + EXPECT_CALL(get_mock_io_ctx(io_ctx), create(RBD_TRASH, false)) + .WillOnce(Return(r)); + } + + void expect_trash_list(librados::IoCtx &io_ctx, + const std::string& last_image_id, + std::map&& images, + int r) { + bufferlist bl; + ::encode(last_image_id, bl); + ::encode(static_cast(1024), bl); + + bufferlist out_bl; + ::encode(images, out_bl); + + EXPECT_CALL(get_mock_io_ctx(io_ctx), + exec(RBD_TRASH, _, StrEq("rbd"), StrEq("trash_list"), + ContentsEqual(bl), _, _)) + .WillOnce(DoAll(WithArg<5>(Invoke([this, out_bl](bufferlist *bl) { + *bl = out_bl; + })), + Return(r))); + } + + void expect_timer_add_event(MockThreads &mock_threads) { + EXPECT_CALL(*mock_threads.timer, add_event_after(_, _)) + .WillOnce(DoAll(WithArg<1>(Invoke([this](Context *ctx) { + auto wrapped_ctx = + new FunctionContext([this, ctx](int r) { + Mutex::Locker timer_locker(m_threads->timer_lock); + ctx->complete(r); + }); + m_threads->work_queue->queue(wrapped_ctx, 0); + })), + ReturnArg<1>())); + } + + void expect_handle_trash_image(MockListener& mock_listener, + const std::string& global_image_id) { + EXPECT_CALL(mock_listener, handle_trash_image(global_image_id, _)); + } + + int when_shut_down(MockTrashWatcher &mock_trash_watcher) { + C_SaferCond ctx; + mock_trash_watcher.shut_down(&ctx); + return ctx.wait(); + } + +}; + +TEST_F(TestMockImageDeleterTrashWatcher, EmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, NonEmptyPool) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image0"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + std::map images; + images["image0"] = {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}; + for (auto idx = 1; idx < 1024; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "", std::move(images), 0); + + images.clear(); + for (auto idx = 1024; idx < 2000; ++idx) { + images["image" + stringify(idx)] = {}; + } + expect_trash_list(m_local_io_ctx, "image999", std::move(images), 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Notify) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + MockListener mock_listener; + expect_handle_trash_image(mock_listener, "image1"); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_image_added( + "image1", {cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING, "name", {}, {}}); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, CreateError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EBLACKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RegisterWatcherError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, -EINVAL); + expect_timer_add_event(mock_threads); + + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EBLACKLISTED); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(-EBLACKLISTED, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, TrashListError) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, -EINVAL); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, Rewatch) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + expect_timer_add_event(mock_threads); + expect_create_trash(m_local_io_ctx, 0); + + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, false); + expect_trash_list(m_local_io_ctx, "", {}, 0); + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(0); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +TEST_F(TestMockImageDeleterTrashWatcher, RewatchBlacklist) { + MockThreads mock_threads(m_threads); + expect_work_queue(mock_threads); + + InSequence seq; + expect_create_trash(m_local_io_ctx, 0); + + MockLibrbdTrashWatcher mock_librbd_trash_watcher; + expect_trash_watcher_is_unregistered(mock_librbd_trash_watcher, true); + expect_trash_watcher_register(mock_librbd_trash_watcher, 0); + expect_trash_list(m_local_io_ctx, "", {}, 0); + + MockListener mock_listener; + MockTrashWatcher mock_trash_watcher(m_local_io_ctx, &mock_threads, + mock_listener); + C_SaferCond ctx; + mock_trash_watcher.init(&ctx); + ASSERT_EQ(0, ctx.wait()); + + LibrbdTrashWatcher::get_instance().handle_rewatch_complete(-EBLACKLISTED); + m_threads->work_queue->drain(); + + expect_trash_watcher_unregister(mock_librbd_trash_watcher, 0); + ASSERT_EQ(0, when_shut_down(mock_trash_watcher)); +} + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt index 661ac44bff5..8f74007c1c4 100644 --- a/src/tools/rbd_mirror/CMakeLists.txt +++ b/src/tools/rbd_mirror/CMakeLists.txt @@ -23,6 +23,7 @@ set(rbd_mirror_internal types.cc image_deleter/RemoveRequest.cc image_deleter/SnapshotPurgeRequest.cc + image_deleter/TrashWatcher.cc image_map/Action.cc image_map/LoadRequest.cc image_map/Policy.cc diff --git a/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc b/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc new file mode 100644 index 00000000000..05b90ad59fe --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/TrashWatcher.cc @@ -0,0 +1,370 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "tools/rbd_mirror/image_deleter/TrashWatcher.h" +#include "include/rbd_types.h" +#include "cls/rbd/cls_rbd_client.h" +#include "common/debug.h" +#include "common/errno.h" +#include "common/Timer.h" +#include "librbd/ImageCtx.h" +#include "librbd/Utils.h" +#include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/image_deleter/Types.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rbd_mirror +#undef dout_prefix +#define dout_prefix *_dout << "rbd::mirror::image_deleter::TrashWatcher: " \ + << this << " " << __func__ << ": " + +using librbd::util::create_context_callback; +using librbd::util::create_rados_callback; + +namespace rbd { +namespace mirror { +namespace image_deleter { + +namespace { + +const size_t MAX_RETURN = 1024; + +} // anonymous namespace + +template +TrashWatcher::TrashWatcher(librados::IoCtx &io_ctx, Threads *threads, + TrashListener& trash_listener) + : librbd::TrashWatcher(io_ctx, threads->work_queue), + m_io_ctx(io_ctx), m_threads(threads), m_trash_listener(trash_listener), + m_lock(librbd::util::unique_lock_name( + "rbd::mirror::image_deleter::TrashWatcher", this)) { +} + +template +void TrashWatcher::init(Context *on_finish) { + dout(5) << dendl; + + { + Mutex::Locker locker(m_lock); + m_on_init_finish = on_finish; + + assert(!m_trash_list_in_progress); + m_trash_list_in_progress = true; + } + + create_trash(); +} + +template +void TrashWatcher::shut_down(Context *on_finish) { + dout(5) << dendl; + + { + Mutex::Locker timer_locker(m_threads->timer_lock); + Mutex::Locker locker(m_lock); + + assert(!m_shutting_down); + m_shutting_down = true; + if (m_timer_ctx != nullptr) { + m_threads->timer->cancel_event(m_timer_ctx); + m_timer_ctx = nullptr; + } + } + + auto ctx = new FunctionContext([this, on_finish](int r) { + unregister_watcher(on_finish); + }); + m_async_op_tracker.wait_for_ops(ctx); +} + +template +void TrashWatcher::handle_image_added(const std::string &image_id, + const cls::rbd::TrashImageSpec& spec) { + dout(10) << "image_id=" << image_id << dendl; + + Mutex::Locker locker(m_lock); + add_image(image_id, spec); +} + +template +void TrashWatcher::handle_image_removed(const std::string &image_id) { + // ignore removals -- the image deleter will ignore -ENOENTs +} + +template +void TrashWatcher::handle_rewatch_complete(int r) { + dout(5) << "r=" << r << dendl; + + if (r == -EBLACKLISTED) { + dout(0) << "detected client is blacklisted" << dendl; + return; + } else if (r == -ENOENT) { + dout(5) << "trash directory deleted" << dendl; + } else if (r < 0) { + derr << "unexpected error re-registering trash directory watch: " + << cpp_strerror(r) << dendl; + } + schedule_trash_list(30); +} + +template +void TrashWatcher::create_trash() { + dout(20) << dendl; + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + } + + librados::ObjectWriteOperation op; + op.create(false); + + m_async_op_tracker.start_op(); + auto aio_comp = create_rados_callback< + TrashWatcher, &TrashWatcher::handle_create_trash>(this); + int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op); + assert(r == 0); + aio_comp->release(); +} + +template +void TrashWatcher::handle_create_trash(int r) { + dout(20) << "r=" << r << dendl; + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + } + + if (r < 0 && r != -EEXIST) { + derr << "failed to create trash object: " << cpp_strerror(r) << dendl; + { + Mutex::Locker locker(m_lock); + m_trash_list_in_progress = false; + } + + schedule_trash_list(30); + } else { + register_watcher(); + } + + m_async_op_tracker.finish_op(); +} + +template +void TrashWatcher::register_watcher() { + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + } + + // if the watch registration is in-flight, let the watcher + // handle the transition -- only (re-)register if it's not registered + if (!this->is_unregistered()) { + trash_list(true); + return; + } + + // first time registering or the watch failed + dout(5) << dendl; + m_async_op_tracker.start_op(); + + Context *ctx = create_context_callback< + TrashWatcher, &TrashWatcher::handle_register_watcher>(this); + this->register_watch(ctx); +} + +template +void TrashWatcher::handle_register_watcher(int r) { + dout(5) << "r=" << r << dendl; + + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + if (r < 0) { + m_trash_list_in_progress = false; + } + } + + Context *on_init_finish = nullptr; + if (r >= 0) { + trash_list(true); + } else if (r == -EBLACKLISTED) { + dout(0) << "detected client is blacklisted" << dendl; + + Mutex::Locker locker(m_lock); + std::swap(on_init_finish, m_on_init_finish); + } else { + derr << "unexpected error registering trash directory watch: " + << cpp_strerror(r) << dendl; + schedule_trash_list(10); + } + + m_async_op_tracker.finish_op(); + if (on_init_finish != nullptr) { + on_init_finish->complete(r); + } +} + +template +void TrashWatcher::unregister_watcher(Context* on_finish) { + dout(5) << dendl; + + m_async_op_tracker.start_op(); + Context *ctx = new FunctionContext([this, on_finish](int r) { + handle_unregister_watcher(r, on_finish); + }); + this->unregister_watch(ctx); +} + +template +void TrashWatcher::handle_unregister_watcher(int r, Context* on_finish) { + dout(5) << "unregister_watcher: r=" << r << dendl; + if (r < 0) { + derr << "error unregistering watcher for trash directory: " + << cpp_strerror(r) << dendl; + } + m_async_op_tracker.finish_op(); + on_finish->complete(0); +} + +template +void TrashWatcher::trash_list(bool initial_request) { + if (initial_request) { + m_async_op_tracker.start_op(); + m_last_image_id = ""; + } + + dout(5) << "last_image_id=" << m_last_image_id << dendl; + + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + } + + librados::ObjectReadOperation op; + librbd::cls_client::trash_list_start(&op, m_last_image_id, MAX_RETURN); + + librados::AioCompletion *aio_comp = create_rados_callback< + TrashWatcher, &TrashWatcher::handle_trash_list>(this); + m_out_bl.clear(); + int r = m_io_ctx.aio_operate(RBD_TRASH, aio_comp, &op, &m_out_bl); + assert(r == 0); + aio_comp->release(); +} + +template +void TrashWatcher::handle_trash_list(int r) { + dout(5) << "r=" << r << dendl; + + std::map images; + if (r >= 0) { + auto bl_it = m_out_bl.begin(); + r = librbd::cls_client::trash_list_finish(&bl_it, &images); + } + + Context *on_init_finish = nullptr; + { + Mutex::Locker locker(m_lock); + assert(m_trash_list_in_progress); + if (r >= 0) { + for (auto& image : images) { + add_image(image.first, image.second); + } + } else if (r == -ENOENT) { + r = 0; + } + + if (r == -EBLACKLISTED) { + dout(0) << "detected client is blacklisted during trash refresh" << dendl; + m_trash_list_in_progress = false; + std::swap(on_init_finish, m_on_init_finish); + } else if (r >= 0 && images.size() < MAX_RETURN) { + m_trash_list_in_progress = false; + std::swap(on_init_finish, m_on_init_finish); + } else if (r < 0) { + m_trash_list_in_progress = false; + } + } + + if (r >= 0 && images.size() == MAX_RETURN) { + m_last_image_id = images.rbegin()->first; + trash_list(false); + return; + } else if (r < 0 && r != -EBLACKLISTED) { + derr << "failed to retrieve trash directory: " << cpp_strerror(r) << dendl; + schedule_trash_list(10); + } + + m_async_op_tracker.finish_op(); + if (on_init_finish != nullptr) { + on_init_finish->complete(r); + } +} + +template +void TrashWatcher::schedule_trash_list(double interval) { + Mutex::Locker timer_locker(m_threads->timer_lock); + Mutex::Locker locker(m_lock); + if (m_shutting_down || m_trash_list_in_progress || m_timer_ctx != nullptr) { + if (m_trash_list_in_progress && !m_deferred_trash_list) { + dout(5) << "deferring refresh until in-flight refresh completes" << dendl; + m_deferred_trash_list = true; + } + return; + } + + dout(5) << dendl; + m_timer_ctx = m_threads->timer->add_event_after( + interval, + new FunctionContext([this](int r) { + process_trash_list(); + })); +} + +template +void TrashWatcher::process_trash_list() { + dout(5) << dendl; + + assert(m_threads->timer_lock.is_locked()); + assert(m_timer_ctx != nullptr); + m_timer_ctx = nullptr; + + { + Mutex::Locker locker(m_lock); + assert(!m_trash_list_in_progress); + m_trash_list_in_progress = true; + } + + // execute outside of the timer's lock + m_async_op_tracker.start_op(); + Context *ctx = new FunctionContext([this](int r) { + create_trash(); + m_async_op_tracker.finish_op(); + }); + m_threads->work_queue->queue(ctx, 0); +} + +template +void TrashWatcher::add_image(const std::string& image_id, + const cls::rbd::TrashImageSpec& spec) { + if (spec.source != cls::rbd::TRASH_IMAGE_SOURCE_MIRRORING) { + return; + } + + assert(m_lock.is_locked()); + auto& deferment_end_time = spec.deferment_end_time; + dout(10) << "image_id=" << image_id << ", " + << "deferment_end_time=" << deferment_end_time << dendl; + + m_async_op_tracker.start_op(); + auto ctx = new FunctionContext([this, image_id, deferment_end_time](int r) { + m_trash_listener.handle_trash_image(image_id, deferment_end_time); + m_async_op_tracker.finish_op(); + }); + m_threads->work_queue->queue(ctx, 0); +} + +} // namespace image_deleter; +} // namespace mirror +} // namespace rbd + +template class rbd::mirror::image_deleter::TrashWatcher; diff --git a/src/tools/rbd_mirror/image_deleter/TrashWatcher.h b/src/tools/rbd_mirror/image_deleter/TrashWatcher.h new file mode 100644 index 00000000000..8d9ff09e565 --- /dev/null +++ b/src/tools/rbd_mirror/image_deleter/TrashWatcher.h @@ -0,0 +1,133 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H +#define CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H + +#include "include/rados/librados.hpp" +#include "common/AsyncOpTracker.h" +#include "common/Mutex.h" +#include "librbd/TrashWatcher.h" +#include +#include + +struct Context; +namespace librbd { struct ImageCtx; } + +namespace rbd { +namespace mirror { + +template struct Threads; + +namespace image_deleter { + +struct TrashListener; + +template +class TrashWatcher : public librbd::TrashWatcher { +public: + TrashWatcher(librados::IoCtx &io_ctx, Threads *threads, + TrashListener& trash_listener); + TrashWatcher(const TrashWatcher&) = delete; + TrashWatcher& operator=(const TrashWatcher&) = delete; + + void init(Context *on_finish); + void shut_down(Context *on_finish); + +protected: + void handle_image_added(const std::string &image_id, + const cls::rbd::TrashImageSpec& spec) override; + + void handle_image_removed(const std::string &image_id) override; + + void handle_rewatch_complete(int r) override; + +private: + /** + * @verbatim + * + * + * | + * v + * INIT + * | + * v + * CREATE_TRASH + * | + * v + * REGISTER_WATCHER + * | + * |/--------------------------------\ + * | | + * |/---------\ | + * | | | + * v | (more images) | + * TRASH_LIST ---/ | + * | | + * |/----------------------------\ | + * | | | + * v | | + * --\ | | + * | | | | + * | |\---> IMAGE_ADDED -----/ | + * | | | + * | \----> WATCH_ERROR ---------/ + * v + * SHUT_DOWN + * | + * v + * UNREGISTER_WATCHER + * | + * v + * + * + * @endverbatim + */ + + librados::IoCtx m_io_ctx; + Threads *m_threads; + TrashListener& m_trash_listener; + + std::string m_last_image_id; + bufferlist m_out_bl; + + mutable Mutex m_lock; + + Context *m_on_init_finish = nullptr; + Context *m_timer_ctx = nullptr; + + AsyncOpTracker m_async_op_tracker; + bool m_trash_list_in_progress = false; + bool m_deferred_trash_list = false; + bool m_shutting_down = false; + + void register_watcher(); + void handle_register_watcher(int r); + + void create_trash(); + void handle_create_trash(int r); + + void unregister_watcher(Context* on_finish); + void handle_unregister_watcher(int r, Context* on_finish); + + void trash_list(bool initial_request); + void handle_trash_list(int r); + + void schedule_trash_list(double interval); + void process_trash_list(); + + void get_mirror_uuid(); + void handle_get_mirror_uuid(int r); + + void add_image(const std::string& image_id, + const cls::rbd::TrashImageSpec& spec); + +}; + +} // namespace image_deleter +} // namespace mirror +} // namespace rbd + +extern template class rbd::mirror::image_deleter::TrashWatcher; + +#endif // CEPH_RBD_MIRROR_IMAGE_DELETE_TRASH_WATCHER_H diff --git a/src/tools/rbd_mirror/image_deleter/Types.h b/src/tools/rbd_mirror/image_deleter/Types.h index 13e94c5d3f7..12131f15ed5 100644 --- a/src/tools/rbd_mirror/image_deleter/Types.h +++ b/src/tools/rbd_mirror/image_deleter/Types.h @@ -4,6 +4,10 @@ #ifndef CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H #define CEPH_RBD_MIRROR_IMAGE_DELETER_TYPES_H +#include + +struct utime_t; + namespace rbd { namespace mirror { namespace image_deleter { @@ -14,6 +18,20 @@ enum ErrorResult { ERROR_RESULT_RETRY_IMMEDIATELY }; +struct TrashListener { + TrashListener() { + } + TrashListener(const TrashListener&) = delete; + TrashListener& operator=(const TrashListener&) = delete; + + virtual ~TrashListener() { + } + + virtual void handle_trash_image(const std::string& image_id, + const utime_t& deferment_end_time) = 0; + +}; + } // namespace image_deleter } // namespace mirror } // namespace rbd -- 2.47.3