From: Patrick Donnelly <pdonnell@redhat.com>
Date: Tue, 19 Feb 2019 00:07:25 +0000 (-0800)
Subject: mds: add 2nd order recall throttle
X-Git-Tag: v12.2.13~224^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e459c3ef8021937009a9975eaf385d2ff91399ab;p=ceph.git

mds: add 2nd order recall throttle

Purpose of this is to moderate the ramp up of recalling caps on a session.  In
particular, we don't want to exhaust the mds_recall_max_decay_threshold within
a short amount of time. This would happen with cache drop on clients which are
trying to unpin dentries (via remount callbacks). The cache drop recall worker
context is driving new recalls in response to flush ACKs but no caps are being
released yet.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>
(cherry picked from commit aa6756d531445a03505f1a15e2f5d10ca10de697)

 Conflicts:
	src/mds/Server.cc
	src/mds/SessionMap.cc
	src/mds/SessionMap.h
---

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index d30371099e20..afd5b5532ba8 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1286,11 +1286,16 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
       uint64_t recall = std::min<uint64_t>(recall_max_caps, num_caps-newlim);
       newlim = num_caps-recall;
       const uint64_t session_recall_throttle = session->get_recall_caps_throttle();
+      const uint64_t session_recall_throttle2o = session->get_recall_caps_throttle2o();
       const uint64_t global_recall_throttle = recall_throttle.get(ceph_clock_now());
       if (session_recall_throttle+recall > recall_max_decay_threshold) {
         dout(15) << "  session recall threshold (" << recall_max_decay_threshold << ") hit at " << session_recall_throttle << "; skipping!" << dendl;
         throttled = true;
         continue;
+      } else if (session_recall_throttle2o+recall > recall_max_caps*2) {
+        dout(15) << "  session recall 2nd-order threshold (" << 2*recall_max_caps << ") hit at " << session_recall_throttle2o << "; skipping!" << dendl;
+        throttled = true;
+        continue;
       } else if (global_recall_throttle+recall > recall_global_max_decay_threshold) {
         dout(15) << "  global recall threshold (" << recall_global_max_decay_threshold << ") hit at " << global_recall_throttle << "; skipping!" << dendl;
         throttled = true;
diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index ae10833c7185..ddc106075bca 100644
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -887,6 +887,7 @@ uint64_t Session::notify_recall_sent(size_t new_limit)
    * throttle future RECALL messages).
    */
   recall_caps_throttle.hit(ceph_clock_now(), count);
+  recall_caps_throttle2o.hit(ceph_clock_now(), count);
   recall_caps.hit(ceph_clock_now(), count);
   return new_change;
 }
diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 23e874831e40..8e0f5d13db7f 100644
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -120,6 +120,8 @@ private:
   mutable DecayCounter release_caps;
   // throttle on caps recalled
   mutable DecayCounter recall_caps_throttle;
+  // second order throttle that prevents recalling too quickly
+  mutable DecayCounter recall_caps_throttle2o;
   // New limit in SESSION_RECALL
   uint32_t recall_limit = 0;
 
@@ -180,6 +182,9 @@ public:
   double get_recall_caps_throttle() const {
     return recall_caps_throttle.get(ceph_clock_now());
   }
+  double get_recall_caps_throttle2o() const {
+    return recall_caps_throttle2o.get(ceph_clock_now());
+  }
   double get_recall_caps() const {
     return recall_caps.get(ceph_clock_now());
   }
@@ -385,6 +390,7 @@ public:
     recall_caps(ceph_clock_now(), g_conf->get_val<double>("mds_recall_warning_decay_rate")),
     release_caps(ceph_clock_now(), g_conf->get_val<double>("mds_recall_warning_decay_rate")),
     recall_caps_throttle(ceph_clock_now(), g_conf->get_val<double>("mds_recall_max_decay_rate")),
+    recall_caps_throttle2o(ceph_clock_now(), 0.5),
     birth_time(clock::now()),
     auth_caps(g_ceph_context),
     item_session_list(this),