rbd-mirror: per pool throttler shared between all namespace image deleters

Signed-off-by: Mykola Golub <mgolub@suse.com>

diff --git a/src/test/rbd_mirror/test_ImageDeleter.cc b/src/test/rbd_mirror/test_ImageDeleter.cc
index 9772094b25604bfc4b17ae2814f0689dbd874b93..0add0612cd8755f32157a0c79aef0f7f22fc1ce9 100644 (file)
--- a/src/test/rbd_mirror/test_ImageDeleter.cc
+++ b/src/test/rbd_mirror/test_ImageDeleter.cc
@@ -19,6 +19,7 @@
 #include "tools/rbd_mirror/ImageDeleter.h"
 #include "tools/rbd_mirror/ServiceDaemon.h"
 #include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/Throttler.h"
 #include "tools/rbd_mirror/Types.h"
 #include "librbd/ImageCtx.h"
 #include "librbd/ImageState.h"
@@ -56,13 +57,18 @@ public:
   void SetUp() override {
     TestFixture::SetUp();
 
+    m_image_deletion_throttler.reset(
+        new rbd::mirror::Throttler<>(g_ceph_context,
+                                     "rbd_mirror_concurrent_image_deletions"));
+
     m_service_daemon.reset(new rbd::mirror::ServiceDaemon<>(g_ceph_context,
                                                             _rados, m_threads));
 
     librbd::api::Mirror<>::mode_set(m_local_io_ctx, RBD_MIRROR_MODE_IMAGE);
 
-    m_deleter = new rbd::mirror::ImageDeleter<>(m_local_io_ctx, m_threads,
-                                                m_service_daemon.get());
+    m_deleter = new rbd::mirror::ImageDeleter<>(
+        m_local_io_ctx, m_threads, m_image_deletion_throttler.get(),
+        m_service_daemon.get());
 
     m_local_image_id = librbd::util::generate_image_id(m_local_io_ctx);
     librbd::ImageOptions image_opts;
@@ -225,6 +231,7 @@ public:
 
   librbd::RBD rbd;
   std::string m_local_image_id;
+  std::unique_ptr<rbd::mirror::Throttler<>> m_image_deletion_throttler;
   std::unique_ptr<rbd::mirror::ServiceDaemon<>> m_service_daemon;
   rbd::mirror::ImageDeleter<> *m_deleter;
 };

diff --git a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
index b52e932ed5c49219447781358a21021765ab2323..7da614f9a6751fc7229ad23810414d1e45d6bd59 100644 (file)
--- a/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_NamespaceReplayer.cc
@@ -38,6 +38,7 @@ struct ImageDeleter<librbd::MockTestImageCtx> {
 
   static ImageDeleter* create(
       librados::IoCtx &ioctx, Threads<librbd::MockTestImageCtx> *threads,
+      Throttler<librbd::MockTestImageCtx> *image_deletion_throttler,
       ServiceDaemon<librbd::MockTestImageCtx> *service_daemon) {
     ceph_assert(s_instance != nullptr);
     return s_instance;
@@ -411,7 +412,7 @@ TEST_F(TestMockNamespaceReplayer, Init_MirrorStatusWatcherError) {
 
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
-      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -431,7 +432,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceReplayerError) {
 
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
-      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -457,7 +458,7 @@ TEST_F(TestMockNamespaceReplayer, Init_InstanceWatcherError) {
 
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
-      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr);
+      "remote mirror uuid", m_mock_threads, nullptr, nullptr, nullptr, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -484,8 +485,8 @@ TEST_F(TestMockNamespaceReplayer, Init) {
 
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
-      "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon,
-      nullptr);
+      "remote mirror uuid", m_mock_threads, nullptr, nullptr,
+      &mock_service_daemon, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);
@@ -523,8 +524,8 @@ TEST_F(TestMockNamespaceReplayer, AcuqireLeader) {
 
   MockNamespaceReplayer namespace_replayer(
       {}, m_local_io_ctx, m_remote_io_ctx, "local mirror uuid",
-      "remote mirror uuid", m_mock_threads, nullptr, &mock_service_daemon,
-      nullptr);
+      "remote mirror uuid", m_mock_threads, nullptr, nullptr,
+      &mock_service_daemon, nullptr);
 
   C_SaferCond on_init;
   namespace_replayer.init(&on_init);

diff --git a/src/test/rbd_mirror/test_mock_PoolReplayer.cc b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
index e1f050f8f53365cd96d5a1f18b966e1418bf5d58..339ed936b0b61c6c622f7875a6a69ff82e8c9edb 100644 (file)
--- a/src/test/rbd_mirror/test_mock_PoolReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_PoolReplayer.cc
@@ -133,6 +133,7 @@ struct NamespaceReplayer<librbd::MockTestImageCtx> {
       const std::string &remote_mirror_uuid,
       Threads<librbd::MockTestImageCtx> *threads,
       Throttler<librbd::MockTestImageCtx> *image_sync_throttler,
+      Throttler<librbd::MockTestImageCtx> *image_deletion_throttler,
       ServiceDaemon<librbd::MockTestImageCtx> *service_daemon,
       journal::CacheManagerHandler *cache_manager_handler) {
     ceph_assert(s_instances.count(name));

diff --git a/src/tools/rbd_mirror/ImageDeleter.cc b/src/tools/rbd_mirror/ImageDeleter.cc
index 6a77955a29486d96e69afbfb4f466207f98ef78d..193f24658484a7d3310113e8a7c6dde72f4886e7 100644 (file)
--- a/src/tools/rbd_mirror/ImageDeleter.cc
+++ b/src/tools/rbd_mirror/ImageDeleter.cc
@@ -31,6 +31,7 @@
 #include "librbd/Utils.h"
 #include "ImageDeleter.h"
 #include "tools/rbd_mirror/Threads.h"
+#include "tools/rbd_mirror/Throttler.h"
 #include "tools/rbd_mirror/image_deleter/RemoveRequest.h"
 #include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h"
 #include "tools/rbd_mirror/image_deleter/TrashWatcher.h"
@@ -52,6 +53,8 @@ using namespace librbd;
 namespace rbd {
 namespace mirror {
 
+using librbd::util::create_async_context_callback;
+
 namespace {
 
 class ImageDeleterAdminSocketCommand {
@@ -123,10 +126,12 @@ private:
 };
 
 template <typename I>
-ImageDeleter<I>::ImageDeleter(librados::IoCtx& local_io_ctx,
-                              Threads<librbd::ImageCtx>* threads,
-                              ServiceDaemon<librbd::ImageCtx>* service_daemon)
+ImageDeleter<I>::ImageDeleter(
+    librados::IoCtx& local_io_ctx, Threads<librbd::ImageCtx>* threads,
+    Throttler<librbd::ImageCtx>* image_deletion_throttler,
+    ServiceDaemon<librbd::ImageCtx>* service_daemon)
   : m_local_io_ctx(local_io_ctx), m_threads(threads),
+    m_image_deletion_throttler(image_deletion_throttler),
     m_service_daemon(service_daemon), m_trash_listener(this),
     m_lock(ceph::make_mutex(
       librbd::util::unique_lock_name("rbd::mirror::ImageDeleter::m_lock",
@@ -174,6 +179,9 @@ void ImageDeleter<I>::shut_down(Context* on_finish) {
   delete m_asok_hook;
   m_asok_hook = nullptr;
 
+  m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
+                                    -ESTALE);
+
   shut_down_trash_watcher(on_finish);
 }
 
@@ -206,6 +214,8 @@ void ImageDeleter<I>::wait_for_ops(Context* on_finish) {
 
 template <typename I>
 void ImageDeleter<I>::cancel_all_deletions(Context* on_finish) {
+  m_image_deletion_throttler->drain(m_local_io_ctx.get_namespace(),
+                                    -ECANCELED);
   {
     std::lock_guard locker{m_lock};
     // wake up any external state machines waiting on deletions
@@ -355,29 +365,34 @@ template <typename I>
 void ImageDeleter<I>::remove_images() {
   dout(10) << dendl;
 
-  auto cct = reinterpret_cast<CephContext *>(m_local_io_ctx.cct());
-  uint64_t max_concurrent_deletions = cct->_conf.get_val<uint64_t>(
-    "rbd_mirror_concurrent_image_deletions");
-
   std::lock_guard locker{m_lock};
-  while (true) {
-    if (!m_running || m_delete_queue.empty() ||
-        m_in_flight_delete_queue.size() >= max_concurrent_deletions) {
-      return;
-    }
+  while (m_running && !m_delete_queue.empty()) {
 
     DeleteInfoRef delete_info = m_delete_queue.front();
     m_delete_queue.pop_front();
 
     ceph_assert(delete_info);
-    remove_image(delete_info);
+
+    auto on_start = create_async_context_callback(
+        m_threads->work_queue, new FunctionContext(
+            [this, delete_info](int r) {
+              if (r < 0) {
+                notify_on_delete(delete_info->image_id, r);
+                return;
+              }
+              remove_image(delete_info);
+            }));
+
+    m_image_deletion_throttler->start_op(m_local_io_ctx.get_namespace(),
+                                         delete_info->image_id, on_start);
   }
 }
 
 template <typename I>
 void ImageDeleter<I>::remove_image(DeleteInfoRef delete_info) {
   dout(10) << "info=" << *delete_info << dendl;
-  ceph_assert(ceph_mutex_is_locked(m_lock));
+
+  std::lock_guard locker{m_lock};
 
   m_in_flight_delete_queue.push_back(delete_info);
   m_async_op_tracker.start_op();
@@ -398,6 +413,8 @@ void ImageDeleter<I>::handle_remove_image(DeleteInfoRef delete_info,
                                           int r) {
   dout(10) << "info=" << *delete_info << ", r=" << r << dendl;
 
+  m_image_deletion_throttler->finish_op(m_local_io_ctx.get_namespace(),
+                                        delete_info->image_id);
   {
     std::lock_guard locker{m_lock};
     ceph_assert(ceph_mutex_is_locked(m_lock));

diff --git a/src/tools/rbd_mirror/ImageDeleter.h b/src/tools/rbd_mirror/ImageDeleter.h
index 5b8f46a35bcf375820e27a308c184028cf81eebe..e5db2f05c693b26dfe25ec5891bcbbddf101ef58 100644 (file)
--- a/src/tools/rbd_mirror/ImageDeleter.h
+++ b/src/tools/rbd_mirror/ImageDeleter.h
@@ -38,6 +38,7 @@ namespace mirror {
 
 template <typename> class ServiceDaemon;
 template <typename> class Threads;
+template <typename> class Throttler;
 
 namespace image_deleter { template <typename> struct TrashWatcher; }
 
@@ -47,14 +48,17 @@ namespace image_deleter { template <typename> struct TrashWatcher; }
 template <typename ImageCtxT = librbd::ImageCtx>
 class ImageDeleter {
 public:
-  static ImageDeleter* create(librados::IoCtx& local_io_ctx,
-                              Threads<librbd::ImageCtx>* threads,
-                              ServiceDaemon<librbd::ImageCtx>* service_daemon) {
-    return new ImageDeleter(local_io_ctx, threads, service_daemon);
+  static ImageDeleter* create(
+      librados::IoCtx& local_io_ctx, Threads<librbd::ImageCtx>* threads,
+      Throttler<librbd::ImageCtx>* image_deletion_throttler,
+      ServiceDaemon<librbd::ImageCtx>* service_daemon) {
+    return new ImageDeleter(local_io_ctx, threads, image_deletion_throttler,
+                            service_daemon);
   }
 
   ImageDeleter(librados::IoCtx& local_io_ctx,
                Threads<librbd::ImageCtx>* threads,
+               Throttler<librbd::ImageCtx>* image_deletion_throttler,
                ServiceDaemon<librbd::ImageCtx>* service_daemon);
 
   ImageDeleter(const ImageDeleter&) = delete;
@@ -124,6 +128,7 @@ private:
 
   librados::IoCtx& m_local_io_ctx;
   Threads<librbd::ImageCtx>* m_threads;
+  Throttler<librbd::ImageCtx>* m_image_deletion_throttler;
   ServiceDaemon<librbd::ImageCtx>* m_service_daemon;
 
   image_deleter::TrashWatcher<ImageCtxT>* m_trash_watcher = nullptr;

diff --git a/src/tools/rbd_mirror/NamespaceReplayer.cc b/src/tools/rbd_mirror/NamespaceReplayer.cc
index 35703ab8bf4c193c1a15d44555bd6030568567e7..b4c3a2f258017fc7243a7d9160ceb871d75913d8 100644 (file)
--- a/src/tools/rbd_mirror/NamespaceReplayer.cc
+++ b/src/tools/rbd_mirror/NamespaceReplayer.cc
@@ -41,11 +41,12 @@ NamespaceReplayer<I>::NamespaceReplayer(
     librados::IoCtx &local_io_ctx, librados::IoCtx &remote_io_ctx,
     const std::string &local_mirror_uuid, const std::string &remote_mirror_uuid,
     Threads<I> *threads, Throttler<I> *image_sync_throttler,
-    ServiceDaemon<I> *service_daemon,
+    Throttler<I> *image_deletion_throttler, ServiceDaemon<I> *service_daemon,
     journal::CacheManagerHandler *cache_manager_handler) :
   m_local_mirror_uuid(local_mirror_uuid),
   m_remote_mirror_uuid(remote_mirror_uuid),
   m_threads(threads), m_image_sync_throttler(image_sync_throttler),
+  m_image_deletion_throttler(image_deletion_throttler),
   m_service_daemon(service_daemon),
   m_cache_manager_handler(cache_manager_handler),
   m_lock(ceph::make_mutex(librbd::util::unique_lock_name(
@@ -611,6 +612,7 @@ void NamespaceReplayer<I>::init_image_deleter(Context *on_finish) {
       handle_init_image_deleter(r, on_finish);
     });
   m_image_deleter.reset(ImageDeleter<I>::create(m_local_io_ctx, m_threads,
+                                                m_image_deletion_throttler,
                                                 m_service_daemon));
   m_image_deleter->init(create_async_context_callback(
     m_threads->work_queue, on_finish));

diff --git a/src/tools/rbd_mirror/NamespaceReplayer.h b/src/tools/rbd_mirror/NamespaceReplayer.h
index c6f519f5590ab9bee931465ffd64726d6350a71d..7e2b01a368d676d54bef6167a47bc808fa6e23ca 100644 (file)
--- a/src/tools/rbd_mirror/NamespaceReplayer.h
+++ b/src/tools/rbd_mirror/NamespaceReplayer.h
@@ -50,12 +50,13 @@ public:
       const std::string &remote_mirror_uuid,
       Threads<ImageCtxT> *threads,
       Throttler<ImageCtxT> *image_sync_throttler,
+      Throttler<ImageCtxT> *image_deletion_throttler,
       ServiceDaemon<ImageCtxT> *service_daemon,
       journal::CacheManagerHandler *cache_manager_handler) {
     return new NamespaceReplayer(name, local_ioctx, remote_ioctx,
                                  local_mirror_uuid, remote_mirror_uuid, threads,
-                                 image_sync_throttler, service_daemon,
-                                 cache_manager_handler);
+                                 image_sync_throttler, image_deletion_throttler,
+                                 service_daemon, cache_manager_handler);
   }
 
   NamespaceReplayer(const std::string &name,
@@ -65,6 +66,7 @@ public:
                     const std::string &remote_mirror_uuid,
                     Threads<ImageCtxT> *threads,
                     Throttler<ImageCtxT> *image_sync_throttler,
+                    Throttler<ImageCtxT> *image_deletion_throttler,
                     ServiceDaemon<ImageCtxT> *service_daemon,
                     journal::CacheManagerHandler *cache_manager_handler);
   NamespaceReplayer(const NamespaceReplayer&) = delete;
@@ -252,6 +254,7 @@ private:
   std::string m_remote_mirror_uuid;
   Threads<ImageCtxT> *m_threads;
   Throttler<ImageCtxT> *m_image_sync_throttler;
+  Throttler<ImageCtxT> *m_image_deletion_throttler;
   ServiceDaemon<ImageCtxT> *m_service_daemon;
   journal::CacheManagerHandler *m_cache_manager_handler;
 

diff --git a/src/tools/rbd_mirror/PoolReplayer.cc b/src/tools/rbd_mirror/PoolReplayer.cc
index 588ea5e0749398d56fca529c3c974a45d083748b..571c22185f0141d3504d1aaea3efcee2ebf9b603 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.cc
+++ b/src/tools/rbd_mirror/PoolReplayer.cc
@@ -315,10 +315,13 @@ void PoolReplayer<I>::init() {
   m_image_sync_throttler.reset(
       Throttler<I>::create(cct, "rbd_mirror_concurrent_image_syncs"));
 
+  m_image_deletion_throttler.reset(
+      Throttler<I>::create(cct, "rbd_mirror_concurrent_image_deletions"));
+
   m_default_namespace_replayer.reset(NamespaceReplayer<I>::create(
       "", m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
-      m_threads, m_image_sync_throttler.get(), m_service_daemon,
-      m_cache_manager_handler));
+      m_threads, m_image_sync_throttler.get(), m_image_deletion_throttler.get(),
+      m_service_daemon, m_cache_manager_handler));
 
   C_SaferCond on_init;
   m_default_namespace_replayer->init(&on_init);
@@ -377,6 +380,7 @@ void PoolReplayer<I>::shut_down() {
   m_default_namespace_replayer.reset();
 
   m_image_sync_throttler.reset();
+  m_image_deletion_throttler.reset();
 
   m_local_rados.reset();
   m_remote_rados.reset();
@@ -594,7 +598,8 @@ void PoolReplayer<I>::update_namespace_replayers() {
   for (auto &name : mirroring_namespaces) {
     auto namespace_replayer = NamespaceReplayer<I>::create(
         name, m_local_io_ctx, m_remote_io_ctx, m_local_mirror_uuid, m_peer.uuid,
-        m_threads, m_image_sync_throttler.get(), m_service_daemon,
+        m_threads, m_image_sync_throttler.get(),
+        m_image_deletion_throttler.get(), m_service_daemon,
         m_cache_manager_handler);
     auto on_init = new FunctionContext(
         [this, namespace_replayer, name, &mirroring_namespaces,
@@ -773,6 +778,12 @@ void PoolReplayer<I>::print_status(Formatter *f, stringstream *ss) {
     f->close_section(); // sync_throttler
   }
 
+  if (m_image_deletion_throttler) {
+    f->open_object_section("deletion_throttler");
+    m_image_deletion_throttler->print_status(f, ss);
+    f->close_section(); // deletion_throttler
+  }
+
   m_default_namespace_replayer->print_status(f, ss);
 
   f->open_array_section("namespaces");

diff --git a/src/tools/rbd_mirror/PoolReplayer.h b/src/tools/rbd_mirror/PoolReplayer.h
index 33a8eebd3bf749ee6d3978aa8f47534187572080..aacff90a248577470ea9284cf3ac02a72bbd79cf 100644 (file)
--- a/src/tools/rbd_mirror/PoolReplayer.h
+++ b/src/tools/rbd_mirror/PoolReplayer.h
@@ -260,6 +260,7 @@ private:
 
   std::unique_ptr<LeaderWatcher<ImageCtxT>> m_leader_watcher;
   std::unique_ptr<Throttler<ImageCtxT>> m_image_sync_throttler;
+  std::unique_ptr<Throttler<ImageCtxT>> m_image_deletion_throttler;
 };
 
 } // namespace mirror