mds: add 2nd order recall throttle

Purpose of this is to moderate the ramp up of recalling caps on a session.  In
particular, we don't want to exhaust the mds_recall_max_decay_threshold within
a short amount of time. This would happen with cache drop on clients which are
trying to unpin dentries (via remount callbacks). The cache drop recall worker
context is driving new recalls in response to flush ACKs but no caps are being
released yet.

Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 582b16e4bd280d73cca709dd0122efb18c8ab176..1fa41c35e6eca7088ddae7332f996d214276934d 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1582,11 +1582,16 @@ std::pair<bool, uint64_t> Server::recall_client_state(MDSGatherBuilder* gather,
       uint64_t recall = std::min<uint64_t>(recall_max_caps, num_caps-newlim);
       newlim = num_caps-recall;
       const uint64_t session_recall_throttle = session->get_recall_caps_throttle();
+      const uint64_t session_recall_throttle2o = session->get_recall_caps_throttle2o();
       const uint64_t global_recall_throttle = recall_throttle.get();
       if (session_recall_throttle+recall > recall_max_decay_threshold) {
         dout(15) << "  session recall threshold (" << recall_max_decay_threshold << ") hit at " << session_recall_throttle << "; skipping!" << dendl;
         throttled = true;
         continue;
+      } else if (session_recall_throttle2o+recall > recall_max_caps*2) {
+        dout(15) << "  session recall 2nd-order threshold (" << 2*recall_max_caps << ") hit at " << session_recall_throttle2o << "; skipping!" << dendl;
+        throttled = true;
+        continue;
       } else if (global_recall_throttle+recall > recall_global_max_decay_threshold) {
         dout(15) << "  global recall threshold (" << recall_global_max_decay_threshold << ") hit at " << global_recall_throttle << "; skipping!" << dendl;
         throttled = true;

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index c3d752af995df113f21a644dcfb06f40fa445d7d..b3a73e8797a274a3361ec6138cd79539995b2e17 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -883,6 +883,7 @@ uint64_t Session::notify_recall_sent(size_t new_limit)
    * throttle future RECALL messages).
    */
   recall_caps_throttle.hit(count);
+  recall_caps_throttle2o.hit(count);
   recall_caps.hit(count);
   return new_change;
 }

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index dfd5f978ec831f5163ae2f3e06d00c3fde4a03a8..d9b2104d27df6a177ff955b9f9286dffd14ba46d 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -120,6 +120,8 @@ private:
   DecayCounter release_caps;
   // throttle on caps recalled
   DecayCounter recall_caps_throttle;
+  // second order throttle that prevents recalling too quickly
+  DecayCounter recall_caps_throttle2o;
   // New limit in SESSION_RECALL
   uint32_t recall_limit = 0;
 
@@ -185,6 +187,9 @@ public:
   auto get_recall_caps_throttle() const {
     return recall_caps_throttle.get();
   }
+  auto get_recall_caps_throttle2o() const {
+    return recall_caps_throttle2o.get();
+  }
   auto get_recall_caps() const {
     return recall_caps.get();
   }
@@ -392,6 +397,7 @@ public:
     recall_caps(g_conf().get_val<double>("mds_recall_warning_decay_rate")),
     release_caps(g_conf().get_val<double>("mds_recall_warning_decay_rate")),
     recall_caps_throttle(g_conf().get_val<double>("mds_recall_max_decay_rate")),
+    recall_caps_throttle2o(0.5),
     birth_time(clock::now()),
     auth_caps(g_ceph_context),
     item_session_list(this),