From e8f65389d09a477ab064c3ad3cf5878eebb2a226 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 29 May 2018 01:45:40 -0400
Subject: [PATCH] rbd-mirror: schedule rebalancer to level-load instances

Policy implementation takes care of evenly balancing images
across rbd mirror instances. This is done when images are
added to the map and/or instances are added or removed with
the exception of image removal -- removing images does not
reshuffle other (mapped) images which can result in some of
the instances under loaded (in worst case, if one removes
images which all map to a particular instance, that instance
would remain idle until more images are added or a shuffle is
triggered).

We could possibly trigger map shuffle when images are removed,
but that would change the interface between Policy and ImageMap
class (in the form of changes to Policy::remove_images()). Also,
policy (and its implementations) would have to do more work when
the above class method is invoked.

Therefore, an interval based rebalancer is added to ImageMap for
periodic rebalancing of images only if the following conditions
are met:

    - policy has been idle for a configured time duration
    - no scheduled or in-transit operations

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit d96cee13d7427bca617e473c522ad7cf00274566)
---
 src/common/options.cc            |  4 ++
 src/tools/rbd_mirror/ImageMap.cc | 65 ++++++++++++++++++++++++++++++++
 src/tools/rbd_mirror/ImageMap.h  |  6 +++
 3 files changed, 75 insertions(+)

diff --git a/src/common/options.cc b/src/common/options.cc
index 9b9260b426752..7810a2d964204 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6471,6 +6471,10 @@ static std::vector<Option> get_rbd_mirror_options() {
     .set_default(1)
     .set_min(1)
     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
+
+    Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
   });
 }
 
diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc
index 2ec371892f5d6..3bd9fed469c9f 100644
--- a/src/tools/rbd_mirror/ImageMap.cc
+++ b/src/tools/rbd_mirror/ImageMap.cc
@@ -65,6 +65,7 @@ template <typename I>
 ImageMap<I>::~ImageMap() {
   assert(m_async_op_tracker.empty());
   assert(m_timer_task == nullptr);
+  assert(m_rebalance_task == nullptr);
 }
 
 template <typename I>
@@ -199,6 +200,15 @@ void ImageMap<I>::process_updates() {
 template <typename I>
 void ImageMap<I>::schedule_update_task() {
   Mutex::Locker timer_lock(m_threads->timer_lock);
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_update_task(const Mutex &timer_lock) {
+  assert(m_threads->timer_lock.is_locked());
+
+  schedule_rebalance_task();
+
   if (m_timer_task != nullptr) {
     return;
   }
@@ -225,6 +235,57 @@ void ImageMap<I>::schedule_update_task() {
   m_threads->timer->add_event_after(after, m_timer_task);
 }
 
+template <typename I>
+void ImageMap<I>::rebalance() {
+  assert(m_rebalance_task == nullptr);
+
+  {
+    Mutex::Locker locker(m_lock);
+    if (m_async_op_tracker.empty() && m_global_image_ids.empty()){
+      dout(20) << "starting rebalance" << dendl;
+
+      std::set<std::string> remap_global_image_ids;
+      m_policy->add_instances({}, &remap_global_image_ids);
+
+      for (auto const &global_image_id : remap_global_image_ids) {
+        schedule_action(global_image_id);
+      }
+    }
+  }
+
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_rebalance_task() {
+  assert(m_threads->timer_lock.is_locked());
+
+  CephContext *cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
+
+  // fetch the updated value of idle timeout for (re)scheduling
+  double resched_after = cct->_conf->get_val<double>(
+    "rbd_mirror_image_policy_rebalance_timeout");
+  if (!resched_after) {
+    return;
+  }
+
+  // cancel existing rebalance task if any before scheduling
+  if (m_rebalance_task != nullptr) {
+    m_threads->timer->cancel_event(m_rebalance_task);
+  }
+
+  m_rebalance_task = new FunctionContext([this](int _) {
+      assert(m_threads->timer_lock.is_locked());
+      m_rebalance_task = nullptr;
+
+      rebalance();
+    });
+
+  dout(20) << "scheduling rebalance (" << m_rebalance_task << ")"
+           << " after " << resched_after << " second(s)" << dendl;
+  m_threads->timer->add_event_after(resched_after, m_rebalance_task);
+}
+
 template <typename I>
 void ImageMap<I>::schedule_action(const std::string &global_image_id) {
   dout(20) << "global_image_id=" << global_image_id << dendl;
@@ -499,6 +560,10 @@ void ImageMap<I>::shut_down(Context *on_finish) {
       m_threads->timer->cancel_event(m_timer_task);
       m_timer_task = nullptr;
     }
+    if (m_rebalance_task != nullptr) {
+      m_threads->timer->cancel_event(m_rebalance_task);
+      m_rebalance_task = nullptr;
+    }
   }
 
   wait_for_async_ops(on_finish);
diff --git a/src/tools/rbd_mirror/ImageMap.h b/src/tools/rbd_mirror/ImageMap.h
index f3a9d71e26574..283f55db3627e 100644
--- a/src/tools/rbd_mirror/ImageMap.h
+++ b/src/tools/rbd_mirror/ImageMap.h
@@ -98,6 +98,8 @@ private:
 
   std::set<std::string> m_global_image_ids;
 
+  Context *m_rebalance_task = nullptr;
+
   struct C_LoadMap : Context {
     ImageMap *image_map;
     Context *on_finish;
@@ -145,10 +147,14 @@ private:
   void schedule_action(const std::string &global_image_id);
 
   void schedule_update_task();
+  void schedule_update_task(const Mutex &timer_lock);
   void process_updates();
   void update_image_mapping(Updates&& map_updates,
                             std::set<std::string>&& map_removals);
 
+  void rebalance();
+  void schedule_rebalance_task();
+
   void notify_listener_acquire_release_images(const Updates &acquire, const Updates &release);
   void notify_listener_remove_images(const std::string &peer_uuid, const Updates &remove);
 
-- 
2.39.5