From e57ac6bf204bc057b592aed97a9cd6d25e3534ba Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Wed, 28 Aug 2019 11:12:54 +0100 Subject: [PATCH] rbd-mirror: per pool throttler shared between all namespace image deleters Signed-off-by: Mykola Golub --- src/test/rbd_mirror/test_ImageDeleter.cc | 11 ++++- .../rbd_mirror/test_mock_NamespaceReplayer.cc | 15 ++++--- src/test/rbd_mirror/test_mock_PoolReplayer.cc | 1 + src/tools/rbd_mirror/ImageDeleter.cc | 45 +++++++++++++------ src/tools/rbd_mirror/ImageDeleter.h | 13 ++++-- src/tools/rbd_mirror/NamespaceReplayer.cc | 4 +- src/tools/rbd_mirror/NamespaceReplayer.h | 7 ++- src/tools/rbd_mirror/PoolReplayer.cc | 17 +++++-- src/tools/rbd_mirror/PoolReplayer.h | 1 + 9 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc index 9772094b256..0add0612cd8 100644 --- a/src/test/rbd_mirror/test_ImageDeleter.cc +++ b/src/test/rbd_mirror/test_ImageDeleter.cc @@ -19,6 +19,7 @@ #include "tools/rbd_mirror/ImageDeleter.h" #include "tools/rbd_mirror/ServiceDaemon.h" #include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Throttler.h" #include "tools/rbd_mirror/Types.h" #include "librbd/ImageCtx.h" #include "librbd/ImageState.h" @@ -56,13 +57,18 @@ public: void SetUp() override { TestFixture::SetUp(); + m_image_deletion_throttler.reset( + new rbd::mirror::Throttler<>(g_ceph_context, + "rbd_mirror_concurrent_image_deletions")); + m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context, _rados, m_threads)); librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE); - m_deleter = new rbd::mirror::ImageDeleter<>(m_local_io_ctx, m_threads, - m_service_daemon.get()); + m_deleter = new rbd::mirror::ImageDeleter<>( + m_local_io_ctx, m_threads, m_image_deletion_throttler.get(), + m_service_daemon.get()); m_local_image_id = librbd::util::generate_image_id(m_local_io_ctx); librbd::ImageOptions image_opts; @@ -225,6 +231,7 @@ public: librbd::RBD rbd; std::string m_local_image_id; + std::unique_ptr> m_image_deletion_throttler; std::unique_ptr> m_service_daemon; rbd::mirror::ImageDeleter<> *m_deleter; }; diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc index b52e932ed5c..7da614f9a67 100644 --- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc +++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc @@ -38,6 +38,7 @@ struct ImageDeleter { static ImageDeleter* create( librados::IoCtx &ioctx, Threads *threads, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon) { ceph_assert(s_instance != nullptr); return s_instance; @@ -411,7 +412,7 @@ TEST_F(TestMockNamespaceReplayer, Init_MirrorStatusWatcherError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr); + "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -431,7 +432,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr); + "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -457,7 +458,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr); + "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -484,8 +485,8 @@ TEST_F(TestMockNamespaceReplayer, Init) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon, - nullptr); + "remote mirror uuid", m_mock_threads, nullptr, nullptr, + &mock_service_daemon, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); @@ -523,8 +524,8 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) { MockNamespaceReplayer namespace_replayer( {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid", - "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon, - nullptr); + "remote mirror uuid", m_mock_threads, nullptr, nullptr, + &mock_service_daemon, nullptr); C_SaferCond on_init; namespace_replayer.init(&on_init); diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc index e1f050f8f53..339ed936b0b 100644 --- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc +++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc @@ -133,6 +133,7 @@ struct NamespaceReplayer { const std::string &remote_mirror_uuid, Threads *threads, Throttler *image_sync_throttler, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler) { ceph_assert(s_instances.count(name)); diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc index 6a77955a294..193f2465848 100644 --- a/src/tools/rbd_mirror/ImageDeleter.cc +++ b/src/tools/rbd_mirror/ImageDeleter.cc @@ -31,6 +31,7 @@ #include "librbd/Utils.h" #include "ImageDeleter.h" #include "tools/rbd_mirror/Threads.h" +#include "tools/rbd_mirror/Throttler.h" #include "tools/rbd_mirror/image_deleter/RemoveRequest.h" #include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" #include "tools/rbd_mirror/image_deleter/TrashWatcher.h" @@ -52,6 +53,8 @@ using namespace librbd; namespace rbd { namespace mirror { +using librbd::util::create_async_context_callback; + namespace { class ImageDeleterAdminSocketCommand { @@ -123,10 +126,12 @@ private: }; template -ImageDeleter::ImageDeleter(librados::IoCtx& local_io_ctx, - Threads* threads, - ServiceDaemon* service_daemon) +ImageDeleter::ImageDeleter( + librados::IoCtx& local_io_ctx, Threads* threads, + Throttler* image_deletion_throttler, + ServiceDaemon* service_daemon) : m_local_io_ctx(local_io_ctx), m_threads(threads), + m_image_deletion_throttler(image_deletion_throttler), m_service_daemon(service_daemon), m_trash_listener(this), m_lock(ceph::make_mutex( librbd::util::unique_lock_name("rbd::mirror::ImageDeleter::m_lock", @@ -174,6 +179,9 @@ void ImageDeleter::shut_down(Context* on_finish) { delete m_asok_hook; m_asok_hook = nullptr; + m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(), + -ESTALE); + shut_down_trash_watcher(on_finish); } @@ -206,6 +214,8 @@ void ImageDeleter::wait_for_ops(Context* on_finish) { template void ImageDeleter::cancel_all_deletions(Context* on_finish) { + m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(), + -ECANCELED); { std::lock_guard locker{m_lock}; // wake up any external state machines waiting on deletions @@ -355,29 +365,34 @@ template void ImageDeleter::remove_images() { dout(10) << dendl; - auto cct = reinterpret_cast(m_local_io_ctx.cct()); - uint64_t max_concurrent_deletions = cct->_conf.get_val( - "rbd_mirror_concurrent_image_deletions"); - std::lock_guard locker{m_lock}; - while (true) { - if (!m_running || m_delete_queue.empty() || - m_in_flight_delete_queue.size() >= max_concurrent_deletions) { - return; - } + while (m_running && !m_delete_queue.empty()) { DeleteInfoRef delete_info = m_delete_queue.front(); m_delete_queue.pop_front(); ceph_assert(delete_info); - remove_image(delete_info); + + auto on_start = create_async_context_callback( + m_threads->work_queue, new FunctionContext( + [this, delete_info](int r) { + if (r < 0) { + notify_on_delete(delete_info->image_id, r); + return; + } + remove_image(delete_info); + })); + + m_image_deletion_throttler->start_op(m_local_io_ctx.get_namespace(), + delete_info->image_id, on_start); } } template void ImageDeleter::remove_image(DeleteInfoRef delete_info) { dout(10) << "info=" << *delete_info << dendl; - ceph_assert(ceph_mutex_is_locked(m_lock)); + + std::lock_guard locker{m_lock}; m_in_flight_delete_queue.push_back(delete_info); m_async_op_tracker.start_op(); @@ -398,6 +413,8 @@ void ImageDeleter::handle_remove_image(DeleteInfoRef delete_info, int r) { dout(10) << "info=" << *delete_info << ", r=" << r << dendl; + m_image_deletion_throttler->finish_op(m_local_io_ctx.get_namespace(), + delete_info->image_id); { std::lock_guard locker{m_lock}; ceph_assert(ceph_mutex_is_locked(m_lock)); diff --git a/src/tools/rbd_mirror/ImageDeleter.h b/src/tools/rbd_mirror/ImageDeleter.h index 5b8f46a35bc..e5db2f05c69 100644 --- a/src/tools/rbd_mirror/ImageDeleter.h +++ b/src/tools/rbd_mirror/ImageDeleter.h @@ -38,6 +38,7 @@ namespace mirror { template class ServiceDaemon; template class Threads; +template class Throttler; namespace image_deleter { template struct TrashWatcher; } @@ -47,14 +48,17 @@ namespace image_deleter { template struct TrashWatcher; } template class ImageDeleter { public: - static ImageDeleter* create(librados::IoCtx& local_io_ctx, - Threads* threads, - ServiceDaemon* service_daemon) { - return new ImageDeleter(local_io_ctx, threads, service_daemon); + static ImageDeleter* create( + librados::IoCtx& local_io_ctx, Threads* threads, + Throttler* image_deletion_throttler, + ServiceDaemon* service_daemon) { + return new ImageDeleter(local_io_ctx, threads, image_deletion_throttler, + service_daemon); } ImageDeleter(librados::IoCtx& local_io_ctx, Threads* threads, + Throttler* image_deletion_throttler, ServiceDaemon* service_daemon); ImageDeleter(const ImageDeleter&) = delete; @@ -124,6 +128,7 @@ private: librados::IoCtx& m_local_io_ctx; Threads* m_threads; + Throttler* m_image_deletion_throttler; ServiceDaemon* m_service_daemon; image_deleter::TrashWatcher* m_trash_watcher = nullptr; diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc index 35703ab8bf4..b4c3a2f2580 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.cc +++ b/src/tools/rbd_mirror/NamespaceReplayer.cc @@ -41,11 +41,12 @@ NamespaceReplayer::NamespaceReplayer( librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx, const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid, Threads *threads, Throttler *image_sync_throttler, - ServiceDaemon *service_daemon, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler) : m_local_mirror_uuid(local_mirror_uuid), m_remote_mirror_uuid(remote_mirror_uuid), m_threads(threads), m_image_sync_throttler(image_sync_throttler), + m_image_deletion_throttler(image_deletion_throttler), m_service_daemon(service_daemon), m_cache_manager_handler(cache_manager_handler), m_lock(ceph::make_mutex(librbd::util::unique_lock_name( @@ -611,6 +612,7 @@ void NamespaceReplayer::init_image_deleter(Context *on_finish) { handle_init_image_deleter(r, on_finish); }); m_image_deleter.reset(ImageDeleter::create(m_local_io_ctx, m_threads, + m_image_deletion_throttler, m_service_daemon)); m_image_deleter->init(create_async_context_callback( m_threads->work_queue, on_finish)); diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h index c6f519f5590..7e2b01a368d 100644 --- a/src/tools/rbd_mirror/NamespaceReplayer.h +++ b/src/tools/rbd_mirror/NamespaceReplayer.h @@ -50,12 +50,13 @@ public: const std::string &remote_mirror_uuid, Threads *threads, Throttler *image_sync_throttler, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler) { return new NamespaceReplayer(name, local_ioctx, remote_ioctx, local_mirror_uuid, remote_mirror_uuid, threads, - image_sync_throttler, service_daemon, - cache_manager_handler); + image_sync_throttler, image_deletion_throttler, + service_daemon, cache_manager_handler); } NamespaceReplayer(const std::string &name, @@ -65,6 +66,7 @@ public: const std::string &remote_mirror_uuid, Threads *threads, Throttler *image_sync_throttler, + Throttler *image_deletion_throttler, ServiceDaemon *service_daemon, journal::CacheManagerHandler *cache_manager_handler); NamespaceReplayer(const NamespaceReplayer&) = delete; @@ -252,6 +254,7 @@ private: std::string m_remote_mirror_uuid; Threads *m_threads; Throttler *m_image_sync_throttler; + Throttler *m_image_deletion_throttler; ServiceDaemon *m_service_daemon; journal::CacheManagerHandler *m_cache_manager_handler; diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc index 588ea5e0749..571c22185f0 100644 --- a/src/tools/rbd_mirror/PoolReplayer.cc +++ b/src/tools/rbd_mirror/PoolReplayer.cc @@ -315,10 +315,13 @@ void PoolReplayer::init() { m_image_sync_throttler.reset( Throttler::create(cct, "rbd_mirror_concurrent_image_syncs")); + m_image_deletion_throttler.reset( + Throttler::create(cct, "rbd_mirror_concurrent_image_deletions")); + m_default_namespace_replayer.reset(NamespaceReplayer::create( "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, - m_threads, m_image_sync_throttler.get(), m_service_daemon, - m_cache_manager_handler)); + m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(), + m_service_daemon, m_cache_manager_handler)); C_SaferCond on_init; m_default_namespace_replayer->init(&on_init); @@ -377,6 +380,7 @@ void PoolReplayer::shut_down() { m_default_namespace_replayer.reset(); m_image_sync_throttler.reset(); + m_image_deletion_throttler.reset(); m_local_rados.reset(); m_remote_rados.reset(); @@ -594,7 +598,8 @@ void PoolReplayer::update_namespace_replayers() { for (auto &name : mirroring_namespaces) { auto namespace_replayer = NamespaceReplayer::create( name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid, - m_threads, m_image_sync_throttler.get(), m_service_daemon, + m_threads, m_image_sync_throttler.get(), + m_image_deletion_throttler.get(), m_service_daemon, m_cache_manager_handler); auto on_init = new FunctionContext( [this, namespace_replayer, name, &mirroring_namespaces, @@ -773,6 +778,12 @@ void PoolReplayer::print_status(Formatter *f, stringstream *ss) { f->close_section(); // sync_throttler } + if (m_image_deletion_throttler) { + f->open_object_section("deletion_throttler"); + m_image_deletion_throttler->print_status(f, ss); + f->close_section(); // deletion_throttler + } + m_default_namespace_replayer->print_status(f, ss); f->open_array_section("namespaces"); diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h index 33a8eebd3bf..aacff90a248 100644 --- a/src/tools/rbd_mirror/PoolReplayer.h +++ b/src/tools/rbd_mirror/PoolReplayer.h @@ -260,6 +260,7 @@ private: std::unique_ptr> m_leader_watcher; std::unique_ptr> m_image_sync_throttler; + std::unique_ptr> m_image_deletion_throttler; }; } // namespace mirror -- 2.39.5