rbd-mirror: schedule rebalancer to level-load instances

Policy implementation takes care of evenly balancing images
across rbd mirror instances. This is done when images are
added to the map and/or instances are added or removed with
the exception of image removal -- removing images does not
reshuffle other (mapped) images which can result in some of
the instances under loaded (in worst case, if one removes
images which all map to a particular instance, that instance
would remain idle until more images are added or a shuffle is
triggered).

We could possibly trigger map shuffle when images are removed,
but that would change the interface between Policy and ImageMap
class (in the form of changes to Policy::remove_images()). Also,
policy (and its implementations) would have to do more work when
the above class method is invoked.

Therefore, an interval based rebalancer is added to ImageMap for
periodic rebalancing of images only if the following conditions
are met:

    - policy has been idle for a configured time duration
    - no scheduled or in-transit operations

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit d96cee13d7427bca617e473c522ad7cf00274566)

diff --git a/src/common/options.cc b/src/common/options.cc
index 9b9260b426752df63496c8a082b86213e7dcb362..7810a2d9642040265bd5f32976654fdefb2b883b 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6471,6 +6471,10 @@ static std::vector<Option> get_rbd_mirror_options() {
     .set_default(1)
     .set_min(1)
     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
+
+    Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
   });
 }
 

diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc
index 2ec371892f5d620fd2988514c6b49d2dfe8ea8dd..3bd9fed469c9fd3a14d53c1a32154a8e03e92a96 100644 (file)
--- a/src/tools/rbd_mirror/ImageMap.cc
+++ b/src/tools/rbd_mirror/ImageMap.cc
@@ -65,6 +65,7 @@ template <typename I>
 ImageMap<I>::~ImageMap() {
   assert(m_async_op_tracker.empty());
   assert(m_timer_task == nullptr);
+  assert(m_rebalance_task == nullptr);
 }
 
 template <typename I>
@@ -199,6 +200,15 @@ void ImageMap<I>::process_updates() {
 template <typename I>
 void ImageMap<I>::schedule_update_task() {
   Mutex::Locker timer_lock(m_threads->timer_lock);
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_update_task(const Mutex &timer_lock) {
+  assert(m_threads->timer_lock.is_locked());
+
+  schedule_rebalance_task();
+
   if (m_timer_task != nullptr) {
     return;
   }
@@ -225,6 +235,57 @@ void ImageMap<I>::schedule_update_task() {
   m_threads->timer->add_event_after(after, m_timer_task);
 }
 
+template <typename I>
+void ImageMap<I>::rebalance() {
+  assert(m_rebalance_task == nullptr);
+
+  {
+    Mutex::Locker locker(m_lock);
+    if (m_async_op_tracker.empty() && m_global_image_ids.empty()){
+      dout(20) << "starting rebalance" << dendl;
+
+      std::set<std::string> remap_global_image_ids;
+      m_policy->add_instances({}, &remap_global_image_ids);
+
+      for (auto const &global_image_id : remap_global_image_ids) {
+        schedule_action(global_image_id);
+      }
+    }
+  }
+
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_rebalance_task() {
+  assert(m_threads->timer_lock.is_locked());
+
+  CephContext *cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
+
+  // fetch the updated value of idle timeout for (re)scheduling
+  double resched_after = cct->_conf->get_val<double>(
+    "rbd_mirror_image_policy_rebalance_timeout");
+  if (!resched_after) {
+    return;
+  }
+
+  // cancel existing rebalance task if any before scheduling
+  if (m_rebalance_task != nullptr) {
+    m_threads->timer->cancel_event(m_rebalance_task);
+  }
+
+  m_rebalance_task = new FunctionContext([this](int _) {
+      assert(m_threads->timer_lock.is_locked());
+      m_rebalance_task = nullptr;
+
+      rebalance();
+    });
+
+  dout(20) << "scheduling rebalance (" << m_rebalance_task << ")"
+           << " after " << resched_after << " second(s)" << dendl;
+  m_threads->timer->add_event_after(resched_after, m_rebalance_task);
+}
+
 template <typename I>
 void ImageMap<I>::schedule_action(const std::string &global_image_id) {
   dout(20) << "global_image_id=" << global_image_id << dendl;
@@ -499,6 +560,10 @@ void ImageMap<I>::shut_down(Context *on_finish) {
       m_threads->timer->cancel_event(m_timer_task);
       m_timer_task = nullptr;
     }
+    if (m_rebalance_task != nullptr) {
+      m_threads->timer->cancel_event(m_rebalance_task);
+      m_rebalance_task = nullptr;
+    }
   }
 
   wait_for_async_ops(on_finish);

diff --git a/src/tools/rbd_mirror/ImageMap.h b/src/tools/rbd_mirror/ImageMap.h
index f3a9d71e26574bf64e25c36a7af74bb65997d61c..283f55db3627e09752ced4652cb10a031b4fa3ef 100644 (file)
--- a/src/tools/rbd_mirror/ImageMap.h
+++ b/src/tools/rbd_mirror/ImageMap.h
@@ -98,6 +98,8 @@ private:
 
   std::set<std::string> m_global_image_ids;
 
+  Context *m_rebalance_task = nullptr;
+
   struct C_LoadMap : Context {
     ImageMap *image_map;
     Context *on_finish;
@@ -145,10 +147,14 @@ private:
   void schedule_action(const std::string &global_image_id);
 
   void schedule_update_task();
+  void schedule_update_task(const Mutex &timer_lock);
   void process_updates();
   void update_image_mapping(Updates&& map_updates,
                             std::set<std::string>&& map_removals);
 
+  void rebalance();
+  void schedule_rebalance_task();
+
   void notify_listener_acquire_release_images(const Updates &acquire, const Updates &release);
   void notify_listener_remove_images(const std::string &peer_uuid, const Updates &remove);