From: Jason Dillaman Date: Wed, 13 Dec 2017 21:26:08 +0000 (-0500) Subject: rbd-mirror: use one image deleter per pool X-Git-Tag: v13.0.2~714^2~8 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=01f2d887db9dea2cb2ee91136a3aec3dbbd6ea89;p=ceph.git rbd-mirror: use one image deleter per pool Signed-off-by: Jason Dillaman --- diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc index 6ee55d7c0836..3325297155b7 100644 --- a/src/test/rbd_mirror/test_ImageDeleter.cc +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -49,32 +49,18 @@ void register_test_rbd_mirror_image_deleter() { class TestImageDeleter : public TestFixture { public: - - static int64_t m_local_pool_id; - const std::string m_local_mirror_uuid = "local mirror uuid"; const std::string m_remote_mirror_uuid = "remote mirror uuid"; - static void SetUpTestCase() { - TestFixture::SetUpTestCase(); - - m_local_pool_id = _rados->pool_lookup(_local_pool_name.c_str()); - } - void SetUp() override { TestFixture::SetUp(); - m_local_io_ctx_ref.reset(new librados::IoCtx{}); - ASSERT_EQ(0, _rados->ioctx_create2(m_local_pool_id, *m_local_io_ctx_ref)); - m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, _rados, m_threads)); librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); - m_deleter = new rbd::mirror::ImageDeleter<>(m_threads->work_queue, - m_threads->timer, - &m_threads->timer_lock, + m_deleter = new rbd::mirror::ImageDeleter<>(m_local_io_ctx, m_threads, m_service_daemon.get()); m_local_image_id = librbd::util::generate_image_id(m_local_io_ctx); @@ -220,21 +206,15 @@ public: librbd::RBD rbd; std::string m_local_image_id; - rbd::mirror::IoCtxRef m_local_io_ctx_ref; std::unique_ptr> m_service_daemon; rbd::mirror::ImageDeleter<> *m_deleter; }; -int64_t TestImageDeleter::m_local_pool_id; - - TEST_F(TestImageDeleter, Delete_NonPrimary_Image) { - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -247,12 +227,10 @@ TEST_F(TestImageDeleter, Delete_Split_Brain_Image) { promote_image(); demote_image(); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, true, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, true, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -265,8 +243,7 @@ TEST_F(TestImageDeleter, Fail_Delete_Primary_Image) { promote_image(); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); EXPECT_EQ(-EPERM, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -278,8 +255,7 @@ TEST_F(TestImageDeleter, Fail_Delete_Orphan_Image) { demote_image(); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); EXPECT_EQ(-EPERM, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -289,12 +265,10 @@ TEST_F(TestImageDeleter, Fail_Delete_Orphan_Image) { TEST_F(TestImageDeleter, Delete_Image_With_Child) { create_snapshot(); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -305,12 +279,10 @@ TEST_F(TestImageDeleter, Delete_Image_With_Children) { create_snapshot("snap1"); create_snapshot("snap2"); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -320,12 +292,10 @@ TEST_F(TestImageDeleter, Delete_Image_With_Children) { TEST_F(TestImageDeleter, Delete_Image_With_ProtectedChild) { create_snapshot("snap1", true); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -336,12 +306,10 @@ TEST_F(TestImageDeleter, Delete_Image_With_ProtectedChildren) { create_snapshot("snap1", true); create_snapshot("snap2", true); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -352,19 +320,16 @@ TEST_F(TestImageDeleter, Delete_Image_With_Clone) { std::string clone_id = create_clone(); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); m_deleter->set_busy_timer_interval(0.1); EXPECT_EQ(-EBUSY, ctx.wait()); C_SaferCond ctx2; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_CLONE_IMAGE_ID, - false, &ctx2); + m_deleter->schedule_image_delete(GLOBAL_CLONE_IMAGE_ID, false, &ctx2); EXPECT_EQ(0, ctx2.wait()); C_SaferCond ctx3; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx3); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx3); EXPECT_EQ(0, ctx3.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -379,12 +344,10 @@ TEST_F(TestImageDeleter, Delete_NonExistent_Image) { EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, m_local_image_id, mirror_image)); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -404,12 +367,10 @@ TEST_F(TestImageDeleter, Delete_NonExistent_Image_With_MirroringState) { EXPECT_EQ(0, cls_client::mirror_image_set(&m_local_io_ctx, m_local_image_id, mirror_image)); - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - nullptr); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, nullptr); C_SaferCond ctx; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx); EXPECT_EQ(0, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -422,8 +383,7 @@ TEST_F(TestImageDeleter, Delete_NonExistent_Image_Without_MirroringState) { remove_image(); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); EXPECT_EQ(-ENOENT, ctx.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); @@ -438,8 +398,7 @@ TEST_F(TestImageDeleter, Fail_Delete_NonPrimary_Image) { EXPECT_EQ(0, ictx->state->open(false)); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); EXPECT_EQ(-EBUSY, ctx.wait()); EXPECT_EQ(0, ictx->state->close()); @@ -452,15 +411,13 @@ TEST_F(TestImageDeleter, Retry_Failed_Deletes) { EXPECT_EQ(0, ictx->state->open(false)); C_SaferCond ctx; - m_deleter->schedule_image_delete(m_local_io_ctx_ref, GLOBAL_IMAGE_ID, false, - &ctx); + m_deleter->schedule_image_delete(GLOBAL_IMAGE_ID, false, &ctx); EXPECT_EQ(-EBUSY, ctx.wait()); EXPECT_EQ(0, ictx->state->close()); C_SaferCond ctx2; - m_deleter->wait_for_scheduled_deletion(m_local_pool_id, GLOBAL_IMAGE_ID, - &ctx2); + m_deleter->wait_for_scheduled_deletion(GLOBAL_IMAGE_ID, &ctx2); EXPECT_EQ(0, ctx2.wait()); ASSERT_EQ(0u, m_deleter->get_delete_queue_items().size()); diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index d8e159f5f3d4..945418bce629 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -124,8 +124,7 @@ public: m_local_cluster, m_threads.get())); m_image_deleter.reset(new rbd::mirror::ImageDeleter<>( - m_threads->work_queue, m_threads->timer, &m_threads->timer_lock, - m_service_daemon.get())); + m_local_ioctx, m_threads.get(), m_service_daemon.get())); m_instance_watcher = rbd::mirror::InstanceWatcher<>::create( m_local_ioctx, m_threads->work_queue, nullptr); m_instance_watcher->handle_acquire_leader(); @@ -140,6 +139,10 @@ public: delete m_replayer; delete m_instance_watcher; + C_SaferCond ctx; + m_image_deleter->shut_down(&ctx); + ctx.wait(); + EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str())); EXPECT_EQ(0, m_local_cluster->pool_delete(m_local_pool_name.c_str())); } @@ -662,7 +665,7 @@ TEST_F(TestImageReplayer, Resync) C_SaferCond delete_ctx; m_image_deleter->wait_for_scheduled_deletion( - m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); + m_replayer->get_global_image_id(), &delete_ctx); EXPECT_EQ(0, delete_ctx.wait()); C_SaferCond cond; @@ -724,7 +727,7 @@ TEST_F(TestImageReplayer, Resync_While_Stop) C_SaferCond delete_ctx; m_image_deleter->wait_for_scheduled_deletion( - m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); + m_replayer->get_global_image_id(), &delete_ctx); EXPECT_EQ(0, delete_ctx.wait()); C_SaferCond cond3; @@ -762,7 +765,7 @@ TEST_F(TestImageReplayer, Resync_StartInterrupted) C_SaferCond delete_ctx; m_image_deleter->wait_for_scheduled_deletion( - m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_ctx); + m_replayer->get_global_image_id(), &delete_ctx); EXPECT_EQ(0, delete_ctx.wait()); C_SaferCond cond2; @@ -949,7 +952,7 @@ TEST_F(TestImageReplayer, Disconnect) ASSERT_EQ(0, cond2.wait()); C_SaferCond delete_cond; m_image_deleter->wait_for_scheduled_deletion( - m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond); + m_replayer->get_global_image_id(), &delete_cond); EXPECT_EQ(0, delete_cond.wait()); start(); @@ -990,7 +993,7 @@ TEST_F(TestImageReplayer, Disconnect) ASSERT_EQ(-ENOTCONN, cond5.wait()); C_SaferCond delete_cond1; m_image_deleter->wait_for_scheduled_deletion( - m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond1); + m_replayer->get_global_image_id(), &delete_cond1); EXPECT_EQ(0, delete_cond1.wait()); C_SaferCond cond6; diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 3df247be94a4..63919514b649 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -85,11 +85,11 @@ struct Threads { template <> struct ImageDeleter { - MOCK_METHOD4(schedule_image_delete, void(IoCtxRef, const std::string&, bool, + MOCK_METHOD3(schedule_image_delete, void(const std::string&, bool, Context*)); - MOCK_METHOD4(wait_for_scheduled_deletion, - void(int64_t, const std::string&, Context*, bool)); - MOCK_METHOD2(cancel_waiter, void(int64_t, const std::string&)); + MOCK_METHOD3(wait_for_scheduled_deletion, + void(const std::string&, Context*, bool)); + MOCK_METHOD1(cancel_waiter, void(const std::string&)); }; template<> @@ -385,22 +385,21 @@ public: const std::string& global_image_id, int r) { EXPECT_CALL(mock_image_deleter, - wait_for_scheduled_deletion(_, global_image_id, _, false)) - .WillOnce(WithArg<2>(Invoke([this, r](Context *ctx) { + wait_for_scheduled_deletion(global_image_id, _, false)) + .WillOnce(WithArg<1>(Invoke([this, r](Context *ctx) { m_threads->work_queue->queue(ctx, r); }))); } void expect_cancel_waiter(MockImageDeleter& mock_image_deleter) { - EXPECT_CALL(mock_image_deleter, cancel_waiter(m_local_io_ctx.get_id(), - "global image id")); + EXPECT_CALL(mock_image_deleter, cancel_waiter("global image id")); } void expect_schedule_image_delete(MockImageDeleter& mock_image_deleter, const std::string& global_image_id, bool ignore_orphan) { EXPECT_CALL(mock_image_deleter, - schedule_image_delete(_, global_image_id, ignore_orphan, nullptr)); + schedule_image_delete(global_image_id, ignore_orphan, nullptr)); } bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) { diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index 19a3c79aa848..94168ad6166d 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -30,6 +30,7 @@ #include "cls/rbd/cls_rbd_types.h" #include "librbd/Utils.h" #include "ImageDeleter.h" +#include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_deleter/RemoveRequest.h" #include #include @@ -79,13 +80,14 @@ private: template class ImageDeleterAdminSocketHook : public AdminSocketHook { public: - ImageDeleterAdminSocketHook(CephContext *cct, ImageDeleter *image_del) : + ImageDeleterAdminSocketHook(CephContext *cct, const std::string& pool_name, + ImageDeleter *image_del) : admin_socket(cct->get_admin_socket()) { std::string command; int r; - command = "rbd mirror deletion status"; + command = "rbd mirror deletion status " + pool_name; r = admin_socket->register_command(command, command, this, "get status for image deleter"); if (r == 0) { @@ -121,19 +123,32 @@ private: }; template -ImageDeleter::ImageDeleter(ContextWQ *work_queue, SafeTimer *timer, - Mutex *timer_lock, +ImageDeleter::ImageDeleter(librados::IoCtx& local_io_ctx, + Threads* threads, ServiceDaemon* service_daemon) - : m_work_queue(work_queue), m_timer(timer), m_timer_lock(timer_lock), + : m_local_io_ctx(local_io_ctx), m_work_queue(threads->work_queue), + m_timer(threads->timer), m_timer_lock(&threads->timer_lock), m_service_daemon(service_daemon), - m_lock("rbd::mirror::ImageDeleter::m_lock"), - m_asok_hook(new ImageDeleterAdminSocketHook(g_ceph_context, this)) -{ + m_lock(librbd::util::unique_lock_name("rbd::mirror::ImageDeleter::m_lock", + this)) { } template -ImageDeleter::~ImageDeleter() { - dout(20) << dendl; +void ImageDeleter::init(Context* on_finish) { + dout(10) << dendl; + + m_asok_hook = new ImageDeleterAdminSocketHook( + g_ceph_context, m_local_io_ctx.get_pool_name(), this); + + on_finish->complete(0); +} + +template +void ImageDeleter::shut_down(Context* on_finish) { + dout(10) << dendl; + + delete m_asok_hook; + m_asok_hook = nullptr; { Mutex::Locker timer_locker(*m_timer_lock); @@ -142,34 +157,40 @@ ImageDeleter::~ImageDeleter() { cancel_retry_timer(); } - C_SaferCond ctx; - m_async_op_tracker.wait_for_ops(&ctx); - ctx.wait(); + wait_for_ops(on_finish); +} - // wake up any external state machines waiting on deletions - assert(m_in_flight_delete_queue.empty()); - for (auto& info : m_delete_queue) { - if (info->on_delete != nullptr) { - info->on_delete->complete(-ECANCELED); - } - } - for (auto& info : m_retry_delete_queue) { - if (info->on_delete != nullptr) { - info->on_delete->complete(-ECANCELED); +template +void ImageDeleter::wait_for_ops(Context* on_finish) { + auto ctx = new FunctionContext([this, on_finish](int) { + cancel_all_deletions(on_finish); + }); + m_async_op_tracker.wait_for_ops(ctx); +} + +template +void ImageDeleter::cancel_all_deletions(Context* on_finish) { + { + Mutex::Locker locker(m_lock); + // wake up any external state machines waiting on deletions + assert(m_in_flight_delete_queue.empty()); + for (auto& queue : {&m_delete_queue, &m_retry_delete_queue}) { + for (auto& info : *queue) { + if (info->on_delete != nullptr) { + info->on_delete->complete(-ECANCELED); + } + } + queue->clear(); } } - - delete m_asok_hook; + on_finish->complete(0); } template -void ImageDeleter::schedule_image_delete(IoCtxRef local_io_ctx, - const std::string& global_image_id, +void ImageDeleter::schedule_image_delete(const std::string& global_image_id, bool ignore_orphaned, Context *on_delete) { - int64_t local_pool_id = local_io_ctx->get_id(); - dout(5) << "local_pool_id=" << local_pool_id << ", " - << "global_image_id=" << global_image_id << dendl; + dout(5) << "global_image_id=" << global_image_id << dendl; if (on_delete != nullptr) { on_delete = new FunctionContext([this, on_delete](int r) { @@ -179,7 +200,7 @@ void ImageDeleter::schedule_image_delete(IoCtxRef local_io_ctx, { Mutex::Locker locker(m_lock); - auto del_info = find_delete_info(local_pool_id, global_image_id); + auto del_info = find_delete_info(global_image_id); if (del_info != nullptr) { dout(20) << "image " << global_image_id << " " << "was already scheduled for deletion" << dendl; @@ -194,27 +215,24 @@ void ImageDeleter::schedule_image_delete(IoCtxRef local_io_ctx, return; } - m_delete_queue.emplace_back(new DeleteInfo(local_pool_id, global_image_id, - local_io_ctx, ignore_orphaned, + m_delete_queue.emplace_back(new DeleteInfo(global_image_id, ignore_orphaned, on_delete)); } remove_images(); } template -void ImageDeleter::wait_for_scheduled_deletion(int64_t local_pool_id, - const std::string &global_image_id, +void ImageDeleter::wait_for_scheduled_deletion(const std::string &global_image_id, Context *ctx, bool notify_on_failed_retry) { - dout(5) << "local_pool_id=" << local_pool_id << ", " - << "global_image_id=" << global_image_id << dendl; + dout(5) << "global_image_id=" << global_image_id << dendl; ctx = new FunctionContext([this, ctx](int r) { m_work_queue->queue(ctx, r); }); Mutex::Locker locker(m_lock); - auto del_info = find_delete_info(local_pool_id, global_image_id); + auto del_info = find_delete_info(global_image_id); if (!del_info) { // image not scheduled for deletion ctx->complete(0); @@ -229,13 +247,11 @@ void ImageDeleter::wait_for_scheduled_deletion(int64_t local_pool_id, } template -void ImageDeleter::cancel_waiter(int64_t local_pool_id, - const std::string &global_image_id) { - dout(5) << "local_pool_id=" << local_pool_id << ", " - << "global_image_id=" << global_image_id << dendl; +void ImageDeleter::cancel_waiter(const std::string &global_image_id) { + dout(5) << "global_image_id=" << global_image_id << dendl; Mutex::Locker locker(m_lock); - auto del_info = find_delete_info(local_pool_id, global_image_id); + auto del_info = find_delete_info(global_image_id); if (!del_info) { return; } @@ -283,14 +299,13 @@ void ImageDeleter::enqueue_failed_delete(DeleteInfoRef* delete_info, template typename ImageDeleter::DeleteInfoRef -ImageDeleter::find_delete_info(int64_t local_pool_id, - const std::string &global_image_id) { +ImageDeleter::find_delete_info(const std::string &global_image_id) { assert(m_lock.is_locked()); DeleteQueue delete_queues[] = {m_in_flight_delete_queue, m_retry_delete_queue, m_delete_queue}; - DeleteInfo delete_info{local_pool_id, global_image_id}; + DeleteInfo delete_info{global_image_id}; for (auto& queue : delete_queues) { auto it = std::find_if(queue.begin(), queue.end(), [&delete_info](const DeleteInfoRef& ref) { @@ -394,7 +409,7 @@ void ImageDeleter::remove_image(DeleteInfoRef delete_info) { }); auto req = image_deleter::RemoveRequest::create( - *delete_info->local_io_ctx, delete_info->global_image_id, + m_local_io_ctx, delete_info->global_image_id, delete_info->ignore_orphaned, &delete_info->error_result, m_work_queue, ctx); req->send(); @@ -512,7 +527,6 @@ void ImageDeleter::DeleteInfo::print_status(Formatter *f, stringstream *ss, bool print_failure_info) { if (f) { f->open_object_section("delete_info"); - f->dump_int("local_pool_id", local_pool_id); f->dump_string("global_image_id", global_image_id); if (print_failure_info) { f->dump_string("error_code", cpp_strerror(error_code)); diff --git a/src/tools/rbd_mirror/ImageDeleter.h b/src/tools/rbd_mirror/ImageDeleter.h index 83a6aba21796..56920ee812d3 100644 --- a/src/tools/rbd_mirror/ImageDeleter.h +++ b/src/tools/rbd_mirror/ImageDeleter.h @@ -36,6 +36,7 @@ namespace rbd { namespace mirror { template class ServiceDaemon; +template class Threads; /** * Manage deletion of non-primary images. @@ -43,22 +44,23 @@ template class ServiceDaemon; template class ImageDeleter { public: - ImageDeleter(ContextWQ *work_queue, SafeTimer *timer, Mutex *timer_lock, + ImageDeleter(librados::IoCtx& local_io_ctx, + Threads* threads, ServiceDaemon* service_daemon); - ~ImageDeleter(); + ImageDeleter(const ImageDeleter&) = delete; ImageDeleter& operator=(const ImageDeleter&) = delete; - void schedule_image_delete(IoCtxRef local_io_ctx, - const std::string& global_image_id, + void init(Context* on_finish); + void shut_down(Context* on_finish); + + void schedule_image_delete(const std::string& global_image_id, bool ignore_orphaned, Context *on_finish); - void wait_for_scheduled_deletion(int64_t local_pool_id, - const std::string &global_image_id, + void wait_for_scheduled_deletion(const std::string &global_image_id, Context *ctx, bool notify_on_failed_retry=true); - void cancel_waiter(int64_t local_pool_id, - const std::string &global_image_id); + void cancel_waiter(const std::string &global_image_id); void print_status(Formatter *f, std::stringstream *ss); @@ -73,9 +75,7 @@ public: private: struct DeleteInfo { - int64_t local_pool_id; std::string global_image_id; - IoCtxRef local_io_ctx; bool ignore_orphaned = false; Context *on_delete = nullptr; @@ -85,26 +85,22 @@ private: int retries = 0; bool notify_on_failed_retry = true; - DeleteInfo(int64_t local_pool_id, const std::string& global_image_id) - : local_pool_id(local_pool_id), global_image_id(global_image_id) { + DeleteInfo(const std::string& global_image_id) + : global_image_id(global_image_id) { } - DeleteInfo(int64_t local_pool_id, const std::string& global_image_id, - IoCtxRef local_io_ctx, bool ignore_orphaned, - Context *on_delete) - : local_pool_id(local_pool_id), global_image_id(global_image_id), - local_io_ctx(local_io_ctx), ignore_orphaned(ignore_orphaned), + DeleteInfo(const std::string& global_image_id, + bool ignore_orphaned, Context *on_delete) + : global_image_id(global_image_id), ignore_orphaned(ignore_orphaned), on_delete(on_delete) { } inline bool operator==(const DeleteInfo& delete_info) const { - return (local_pool_id == delete_info.local_pool_id && - global_image_id == delete_info.global_image_id); + return (global_image_id == delete_info.global_image_id); } friend std::ostream& operator<<(std::ostream& os, DeleteInfo& delete_info) { - os << "[" << "local_pool_id=" << delete_info.local_pool_id << ", " - << "global_image_id=" << delete_info.global_image_id << "]"; + os << "[global_image_id=" << delete_info.global_image_id << "]"; return os; } @@ -115,6 +111,7 @@ private: typedef std::shared_ptr DeleteInfoRef; typedef std::deque DeleteQueue; + librados::IoCtx& m_local_io_ctx; ContextWQ *m_work_queue; SafeTimer *m_timer; Mutex *m_timer_lock; @@ -131,7 +128,7 @@ private: DeleteQueue m_retry_delete_queue; DeleteQueue m_in_flight_delete_queue; - AdminSocketHook *m_asok_hook; + AdminSocketHook *m_asok_hook = nullptr; Context *m_timer_ctx = nullptr; @@ -141,8 +138,7 @@ private: void enqueue_failed_delete(DeleteInfoRef* delete_info, int error_code, double retry_delay); - DeleteInfoRef find_delete_info(int64_t local_pool_id, - const std::string &global_image_id); + DeleteInfoRef find_delete_info(const std::string &global_image_id); void remove_images(); void remove_image(DeleteInfoRef delete_info); @@ -152,6 +148,9 @@ private: void cancel_retry_timer(); void handle_retry_timer(); + void wait_for_ops(Context* on_finish); + void cancel_all_deletions(Context* on_finish); + }; } // namespace mirror diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 64c4ee3f047f..692dfe617acb 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -401,7 +401,7 @@ void ImageReplayer::wait_for_deletion() { Context *ctx = create_context_callback< ImageReplayer, &ImageReplayer::handle_wait_for_deletion>(this); m_image_deleter->wait_for_scheduled_deletion( - m_local_pool_id, m_global_image_id, ctx, false); + m_global_image_id, ctx, false); } template @@ -744,7 +744,7 @@ void ImageReplayer::stop(Context *on_finish, bool manual, int r, dout(20) << "on_finish=" << on_finish << ", manual=" << manual << ", desc=" << desc << dendl; - m_image_deleter->cancel_waiter(m_local_pool_id, m_global_image_id); + m_image_deleter->cancel_waiter(m_global_image_id); image_replayer::BootstrapRequest *bootstrap_request = nullptr; bool shut_down_replay = false; @@ -1657,7 +1657,7 @@ void ImageReplayer::handle_shut_down(int r) { delete_requested = true; } if (delete_requested || m_resync_requested) { - m_image_deleter->schedule_image_delete(m_local_ioctx, m_global_image_id, + m_image_deleter->schedule_image_delete(m_global_image_id, m_resync_requested, nullptr); m_local_image_id = ""; diff --git a/src/tools/rbd_mirror/Mirror.cc b/src/tools/rbd_mirror/Mirror.cc index 61dc9bf2d89c..f3bc77b09507 100644 --- a/src/tools/rbd_mirror/Mirror.cc +++ b/src/tools/rbd_mirror/Mirror.cc @@ -245,11 +245,6 @@ int Mirror::init() m_local_cluster_watcher.reset(new ClusterWatcher(m_local, m_lock, m_service_daemon.get())); - - m_image_deleter.reset(new ImageDeleter<>(m_threads->work_queue, - m_threads->timer, - &m_threads->timer_lock, - m_service_daemon.get())); return r; } @@ -296,10 +291,7 @@ void Mirror::print_status(Formatter *f, stringstream *ss) if (f) { f->close_section(); - f->open_object_section("image_deleter"); } - - m_image_deleter->print_status(f, ss); } void Mirror::start() @@ -419,8 +411,7 @@ void Mirror::update_pool_replayers(const PoolPeers &pool_peers) } else { dout(20) << "starting pool replayer for " << peer << dendl; unique_ptr pool_replayer(new PoolReplayer( - m_threads, m_service_daemon.get(), m_image_deleter.get(), kv.first, - peer, m_args)); + m_threads, m_service_daemon.get(), kv.first, peer, m_args)); // TODO: make async pool_replayer->init(); diff --git a/src/tools/rbd_mirror/Mirror.h b/src/tools/rbd_mirror/Mirror.h index 92f7eb4508a7..ebea3d823f2c 100644 --- a/src/tools/rbd_mirror/Mirror.h +++ b/src/tools/rbd_mirror/Mirror.h @@ -9,7 +9,6 @@ #include "include/rados/librados.hpp" #include "ClusterWatcher.h" #include "PoolReplayer.h" -#include "ImageDeleter.h" #include "types.h" #include @@ -66,7 +65,6 @@ private: // monitor local cluster for config changes in peers std::unique_ptr m_local_cluster_watcher; - std::unique_ptr> m_image_deleter; std::map > m_pool_replayers; std::atomic m_stopping = { false }; bool m_manual_stop = false; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 817d3434c398..a75566c66779 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -215,12 +215,10 @@ private: PoolReplayer::PoolReplayer(Threads *threads, ServiceDaemon* service_daemon, - ImageDeleter<>* image_deleter, int64_t local_pool_id, const peer_t &peer, const std::vector &args) : m_threads(threads), m_service_daemon(service_daemon), - m_image_deleter(image_deleter), m_local_pool_id(local_pool_id), m_peer(peer), m_args(args), @@ -313,8 +311,15 @@ void PoolReplayer::init() dout(20) << "connected to " << m_peer << dendl; + // TODO + C_SaferCond image_deleter_ctx; + m_image_deleter.reset(new ImageDeleter<>(m_local_io_ctx, m_threads, + m_service_daemon)); + m_image_deleter->init(&image_deleter_ctx); + image_deleter_ctx.wait(); + m_instance_replayer.reset(InstanceReplayer<>::create( - m_threads, m_service_daemon, m_image_deleter, m_local_rados, + m_threads, m_service_daemon, m_image_deleter.get(), m_local_rados, local_mirror_uuid, m_local_pool_id)); m_instance_replayer->init(); m_instance_replayer->add_peer(m_peer.uuid, m_remote_io_ctx); @@ -371,6 +376,13 @@ void PoolReplayer::shut_down() { m_instance_replayer.reset(); } + // TODO + if (m_image_deleter) { + C_SaferCond image_deleter_ctx; + m_image_deleter->shut_down(&image_deleter_ctx); + image_deleter_ctx.wait(); + } + assert(!m_local_pool_watcher); assert(!m_remote_pool_watcher); m_local_rados.reset(); @@ -554,6 +566,10 @@ void PoolReplayer::print_status(Formatter *f, stringstream *ss) m_instance_replayer->print_status(f, ss); + f->open_object_section("image_deleter"); + m_image_deleter->print_status(f, ss); + f->close_section(); + f->close_section(); f->flush(*ss); } diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 49aaee3c4ae7..1cfa897ca119 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -42,7 +42,6 @@ class PoolReplayer { public: PoolReplayer(Threads *threads, ServiceDaemon* service_daemon, - ImageDeleter<>* image_deleter, int64_t local_pool_id, const peer_t &peer, const std::vector &args); ~PoolReplayer(); @@ -110,7 +109,6 @@ private: Threads *m_threads; ServiceDaemon* m_service_daemon; - ImageDeleter<>* m_image_deleter; int64_t m_local_pool_id = -1; peer_t m_peer; std::vector m_args; @@ -134,6 +132,7 @@ private: std::unique_ptr > m_remote_pool_watcher; std::unique_ptr> m_instance_replayer; + std::unique_ptr> m_image_deleter; std::string m_asok_hook_name; AdminSocketHook *m_asok_hook = nullptr;