mds: add 2nd order recall throttle

Purpose of this is to moderate the ramp up of recalling caps on a session.  In
particular, we don't want to exhaust the mds_recall_max_decay_threshold within
a short amount of time. This would happen with cache drop on clients which are
trying to unpin dentries (via remount callbacks). The cache drop recall worker
context is driving new recalls in response to flush ACKs but no caps are being
released yet.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit aa6756d531445a03505f1a15e2f5d10ca10de697)

 Conflicts:
	src/mds/Server.cc
	src/mds/SessionMap.cc
	src/mds/SessionMap.h

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index d30371099e200c1a11367e34b08216250dca7356..afd5b5532ba8ec942e2632df0a2449635b998775 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1286,11 +1286,16 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
       uint64_t recall = std::min<uint64_t>(recall_max_caps, num_caps-newlim);
       newlim = num_caps-recall;
       const uint64_t session_recall_throttle = session->get_recall_caps_throttle();
+      const uint64_t session_recall_throttle2o = session->get_recall_caps_throttle2o();
       const uint64_t global_recall_throttle = recall_throttle.get(ceph_clock_now());
       if (session_recall_throttle+recall > recall_max_decay_threshold) {
         dout(15) << "  session recall threshold (" << recall_max_decay_threshold << ") hit at " << session_recall_throttle << "; skipping!" << dendl;
         throttled = true;
         continue;
+      } else if (session_recall_throttle2o+recall > recall_max_caps*2) {
+        dout(15) << "  session recall 2nd-order threshold (" << 2*recall_max_caps << ") hit at " << session_recall_throttle2o << "; skipping!" << dendl;
+        throttled = true;
+        continue;
       } else if (global_recall_throttle+recall > recall_global_max_decay_threshold) {
         dout(15) << "  global recall threshold (" << recall_global_max_decay_threshold << ") hit at " << global_recall_throttle << "; skipping!" << dendl;
         throttled = true;

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index ae10833c71857b82cddbb23642fa5fe0be016109..ddc106075bca83c505522ddb5e12c658a4c5b2fc 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -887,6 +887,7 @@ uint64_t Session::notify_recall_sent(size_t new_limit)
    * throttle future RECALL messages).
    */
   recall_caps_throttle.hit(ceph_clock_now(), count);
+  recall_caps_throttle2o.hit(ceph_clock_now(), count);
   recall_caps.hit(ceph_clock_now(), count);
   return new_change;
 }

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 23e874831e40acc412b1bab220f51b9b8e8f08d3..8e0f5d13db7f253a6e6d3b5c42254feb54e9beb4 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -120,6 +120,8 @@ private:
   mutable DecayCounter release_caps;
   // throttle on caps recalled
   mutable DecayCounter recall_caps_throttle;
+  // second order throttle that prevents recalling too quickly
+  mutable DecayCounter recall_caps_throttle2o;
   // New limit in SESSION_RECALL
   uint32_t recall_limit = 0;
 
@@ -180,6 +182,9 @@ public:
   double get_recall_caps_throttle() const {
     return recall_caps_throttle.get(ceph_clock_now());
   }
+  double get_recall_caps_throttle2o() const {
+    return recall_caps_throttle2o.get(ceph_clock_now());
+  }
   double get_recall_caps() const {
     return recall_caps.get(ceph_clock_now());
   }
@@ -385,6 +390,7 @@ public:
     recall_caps(ceph_clock_now(), g_conf->get_val<double>("mds_recall_warning_decay_rate")),
     release_caps(ceph_clock_now(), g_conf->get_val<double>("mds_recall_warning_decay_rate")),
     recall_caps_throttle(ceph_clock_now(), g_conf->get_val<double>("mds_recall_max_decay_rate")),
+    recall_caps_throttle2o(ceph_clock_now(), 0.5),
     birth_time(clock::now()),
     auth_caps(g_ceph_context),
     item_session_list(this),