From: Venky Shankar <vshankar@redhat.com>
Date: Tue, 29 May 2018 05:45:40 +0000 (-0400)
Subject: rbd-mirror: schedule rebalancer to level-load instances
X-Git-Tag: v14.0.1~1123^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d96cee13d7427bca617e473c522ad7cf00274566;p=ceph.git

rbd-mirror: schedule rebalancer to level-load instances

Policy implementation takes care of evenly balancing images
across rbd mirror instances. This is done when images are
added to the map and/or instances are added or removed with
the exception of image removal -- removing images does not
reshuffle other (mapped) images which can result in some of
the instances under loaded (in worst case, if one removes
images which all map to a particular instance, that instance
would remain idle until more images are added or a shuffle is
triggered).

We could possibly trigger map shuffle when images are removed,
but that would change the interface between Policy and ImageMap
class (in the form of changes to Policy::remove_images()). Also,
policy (and its implementations) would have to do more work when
the above class method is invoked.

Therefore, an interval based rebalancer is added to ImageMap for
periodic rebalancing of images only if the following conditions
are met:

    - policy has been idle for a configured time duration
    - no scheduled or in-transit operations

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---

diff --git a/src/common/options.cc b/src/common/options.cc
index 2a31285ff228..fc7df3b4df89 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6511,6 +6511,10 @@ static std::vector<Option> get_rbd_mirror_options() {
     .set_default(1)
     .set_min(1)
     .set_description("interval (in seconds) to throttle images for mirror daemon peer updates"),
+
+    Option("rbd_mirror_image_policy_rebalance_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("number of seconds policy should be idle before trigerring reshuffle (rebalance) of images"),
   });
 }
 
diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc
index 2ec371892f5d..3bd9fed469c9 100644
--- a/src/tools/rbd_mirror/ImageMap.cc
+++ b/src/tools/rbd_mirror/ImageMap.cc
@@ -65,6 +65,7 @@ template <typename I>
 ImageMap<I>::~ImageMap() {
   assert(m_async_op_tracker.empty());
   assert(m_timer_task == nullptr);
+  assert(m_rebalance_task == nullptr);
 }
 
 template <typename I>
@@ -199,6 +200,15 @@ void ImageMap<I>::process_updates() {
 template <typename I>
 void ImageMap<I>::schedule_update_task() {
   Mutex::Locker timer_lock(m_threads->timer_lock);
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_update_task(const Mutex &timer_lock) {
+  assert(m_threads->timer_lock.is_locked());
+
+  schedule_rebalance_task();
+
   if (m_timer_task != nullptr) {
     return;
   }
@@ -225,6 +235,57 @@ void ImageMap<I>::schedule_update_task() {
   m_threads->timer->add_event_after(after, m_timer_task);
 }
 
+template <typename I>
+void ImageMap<I>::rebalance() {
+  assert(m_rebalance_task == nullptr);
+
+  {
+    Mutex::Locker locker(m_lock);
+    if (m_async_op_tracker.empty() && m_global_image_ids.empty()){
+      dout(20) << "starting rebalance" << dendl;
+
+      std::set<std::string> remap_global_image_ids;
+      m_policy->add_instances({}, &remap_global_image_ids);
+
+      for (auto const &global_image_id : remap_global_image_ids) {
+        schedule_action(global_image_id);
+      }
+    }
+  }
+
+  schedule_update_task(m_threads->timer_lock);
+}
+
+template <typename I>
+void ImageMap<I>::schedule_rebalance_task() {
+  assert(m_threads->timer_lock.is_locked());
+
+  CephContext *cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
+
+  // fetch the updated value of idle timeout for (re)scheduling
+  double resched_after = cct->_conf->get_val<double>(
+    "rbd_mirror_image_policy_rebalance_timeout");
+  if (!resched_after) {
+    return;
+  }
+
+  // cancel existing rebalance task if any before scheduling
+  if (m_rebalance_task != nullptr) {
+    m_threads->timer->cancel_event(m_rebalance_task);
+  }
+
+  m_rebalance_task = new FunctionContext([this](int _) {
+      assert(m_threads->timer_lock.is_locked());
+      m_rebalance_task = nullptr;
+
+      rebalance();
+    });
+
+  dout(20) << "scheduling rebalance (" << m_rebalance_task << ")"
+           << " after " << resched_after << " second(s)" << dendl;
+  m_threads->timer->add_event_after(resched_after, m_rebalance_task);
+}
+
 template <typename I>
 void ImageMap<I>::schedule_action(const std::string &global_image_id) {
   dout(20) << "global_image_id=" << global_image_id << dendl;
@@ -499,6 +560,10 @@ void ImageMap<I>::shut_down(Context *on_finish) {
       m_threads->timer->cancel_event(m_timer_task);
       m_timer_task = nullptr;
     }
+    if (m_rebalance_task != nullptr) {
+      m_threads->timer->cancel_event(m_rebalance_task);
+      m_rebalance_task = nullptr;
+    }
   }
 
   wait_for_async_ops(on_finish);
diff --git a/src/tools/rbd_mirror/ImageMap.h b/src/tools/rbd_mirror/ImageMap.h
index f3a9d71e2657..283f55db3627 100644
--- a/src/tools/rbd_mirror/ImageMap.h
+++ b/src/tools/rbd_mirror/ImageMap.h
@@ -98,6 +98,8 @@ private:
 
   std::set<std::string> m_global_image_ids;
 
+  Context *m_rebalance_task = nullptr;
+
   struct C_LoadMap : Context {
     ImageMap *image_map;
     Context *on_finish;
@@ -145,10 +147,14 @@ private:
   void schedule_action(const std::string &global_image_id);
 
   void schedule_update_task();
+  void schedule_update_task(const Mutex &timer_lock);
   void process_updates();
   void update_image_mapping(Updates&& map_updates,
                             std::set<std::string>&& map_removals);
 
+  void rebalance();
+  void schedule_rebalance_task();
+
   void notify_listener_acquire_release_images(const Updates &acquire, const Updates &release);
   void notify_listener_remove_images(const std::string &peer_uuid, const Updates &remove);