mds: decay counter for tracking request load average

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit 816d8aaea01aef81467e3df7a4b3ab3eff86d27a)

Conflicts:
	src/common/options.cc
	src/mds/SessionMap.cc
	src/mds/SessionMap.h

diff --git a/src/common/options.cc b/src/common/options.cc
index c6b421dc9a7e10eff05f7a64a3ea0dae261d034a..f0f04a6cccb7bb2c20a15ba048c38e5717eac037 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6496,6 +6496,10 @@ std::vector<Option> get_mds_options() {
     Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
      .set_default(5)
      .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
+
+    Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+    .set_default(60)
+    .set_description("rate of decay in seconds for calculating request load average"),
   });
 }
 

diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index 31dfb3122cd8c81931f0eb78d6b14848f7a438e3..d7ff8d1a49fce4d5d3846d9fb0dfc450f091869a 100644 (file)
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -1387,6 +1387,9 @@ void Server::early_reply(MDRequestRef& mdr, CInode *tracei, CDentry *tracedn)
   mds->logger->inc(l_mds_reply);
   utime_t lat = ceph_clock_now() - req->get_recv_stamp();
   mds->logger->tinc(l_mds_reply_latency, lat);
+  if (client_inst.name.is_client()) {
+    mds->sessionmap.hit_session(mdr->session);
+  }
   perf_gather_op_latency(req, lat);
   dout(20) << "lat " << lat << dendl;
 
@@ -1444,6 +1447,9 @@ void Server::reply_client_request(MDRequestRef& mdr, MClientReply *reply)
     mds->logger->inc(l_mds_reply);
     utime_t lat = ceph_clock_now() - mdr->client_request->get_recv_stamp();
     mds->logger->tinc(l_mds_reply_latency, lat);
+    if (client_inst.name.is_client()) {
+      mds->sessionmap.hit_session(session);
+    }
     perf_gather_op_latency(req, lat);
     dout(20) << "lat " << lat << dendl;
     

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index 56f097c0219da21a00964719a949afe5184fd9cf..849b497260b5cb202e406673eba297b5cde9220b 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -21,6 +21,7 @@
 
 #include "common/config.h"
 #include "common/errno.h"
+#include "common/DecayCounter.h"
 #include "include/assert.h"
 #include "include/stringify.h"
 
@@ -56,6 +57,8 @@ void SessionMap::register_perfcounters()
               "Sessions currently open");
   plb.add_u64(l_mdssm_session_stale, "sessions_stale",
               "Sessions currently stale");
+  plb.add_u64(l_mdssm_total_load, "total_load", "Total Load");
+  plb.add_u64(l_mdssm_avg_load, "average_load", "Average Load");
   logger = plb.create_perf_counters();
   g_ceph_context->get_perfcounters_collection()->add(logger);
 }
@@ -148,9 +151,11 @@ void SessionMapStore::decode_values(std::map<std::string, bufferlist> &session_v
     }
 
     Session *s = get_or_add_session(inst);
-    if (s->is_closed())
+    if (s->is_closed()) {
       s->set_state(Session::STATE_OPEN);
-    bufferlist::iterator q = i->second.begin();
+      s->set_load_avg_decay_rate(decay_rate);
+    }
+    auto q = i->second.begin();
     s->decode(q);
   }
 }
@@ -488,6 +493,10 @@ uint64_t SessionMap::set_state(Session *session, int s) {
       by_state_entry = by_state.emplace(s, new xlist<Session*>).first;
     by_state_entry->second->push_back(&session->item_session_list);
 
+    if (session->is_open() || session->is_stale()) {
+      session->set_load_avg_decay_rate(decay_rate);
+    }
+
     // refresh number of sessions for states which have perf
     // couters associated
     logger->set(l_mdssm_session_open,
@@ -514,8 +523,10 @@ void SessionMapStore::decode_legacy(bufferlist::iterator& p)
       entity_inst_t inst;
       ::decode(inst.name, p);
       Session *s = get_or_add_session(inst);
-      if (s->is_closed())
+      if (s->is_closed()) {
         s->set_state(Session::STATE_OPEN);
+        s->set_load_avg_decay_rate(decay_rate);
+      }
       s->decode(p);
     }
 
@@ -544,6 +555,7 @@ void SessionMapStore::decode_legacy(bufferlist::iterator& p)
        session_map[s->info.inst.name] = s;
       }
       s->set_state(Session::STATE_OPEN);
+      s->set_load_avg_decay_rate(decay_rate);
       s->last_cap_renew = now;
     }
   }
@@ -952,6 +964,21 @@ int Session::check_access(CInode *in, unsigned mask,
   return 0;
 }
 
+// track total and per session load
+void SessionMap::hit_session(Session *session) {
+  uint64_t sessions = get_session_count_in_state(Session::STATE_OPEN) +
+                      get_session_count_in_state(Session::STATE_STALE);
+  assert(sessions != 0);
+
+  double total_load = total_load_avg.hit(ceph_clock_now(), total_load_avg_rate);
+  double avg_load = total_load / sessions;
+
+  logger->set(l_mdssm_total_load, (uint64_t)total_load);
+  logger->set(l_mdssm_avg_load, (uint64_t)avg_load);
+
+  session->hit_session();
+}
+
 int SessionFilter::parse(
     const std::vector<std::string> &args,
     std::stringstream *ss)

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 6aeb83d7f0032af3d86e0a4b6136fc29dedf0f61..8e2ddccf8596a707e8bb8c61f92cd968cc6f18d3 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -30,6 +30,7 @@ using std::set;
 
 class CInode;
 struct MDRequestImpl;
+class DecayCounter;
 
 #include "CInode.h"
 #include "Capability.h"
@@ -42,6 +43,8 @@ enum {
   l_mdssm_session_remove,
   l_mdssm_session_open,
   l_mdssm_session_stale,
+  l_mdssm_total_load,
+  l_mdssm_avg_load,
   l_mdssm_last,
 };
 
@@ -100,7 +103,9 @@ private:
   // that appropriate mark_dirty calls follow.
   std::deque<version_t> projected;
 
-
+  // request load average for this session
+  DecayCounter load_avg;
+  DecayRate    load_avg_rate;
 
 public:
 
@@ -200,6 +205,14 @@ public:
   }
   bool is_importing() const { return importing_count > 0; }
 
+  void set_load_avg_decay_rate(double rate) {
+    assert(is_open() || is_stale());
+    load_avg_rate.set_halflife(rate);
+  }
+  void hit_session() {
+    load_avg.hit(ceph_clock_now(), load_avg_rate);
+  }
+
   // -- caps --
 private:
   version_t cap_push_seq;        // cap push seq #
@@ -395,6 +408,12 @@ protected:
   version_t version;
   ceph::unordered_map<entity_name_t, Session*> session_map;
   PerfCounters *logger;
+
+  // total request load avg
+  double decay_rate;
+  DecayCounter total_load_avg;
+  DecayRate    total_load_avg_rate;
+
 public:
   mds_rank_t rank;
 
@@ -436,7 +455,11 @@ public:
     session_map.clear();
   }
 
-  SessionMapStore() : version(0), logger(nullptr), rank(MDS_RANK_NONE) {}
+  SessionMapStore()
+    : version(0), logger(nullptr),
+      decay_rate(g_conf->get_val<double>("mds_request_load_average_decay_rate")),
+      total_load_avg_rate(decay_rate), rank(MDS_RANK_NONE) {
+  }
   virtual ~SessionMapStore() {};
 };
 
@@ -668,6 +691,9 @@ private:
   uint64_t get_session_count_in_state(int state) {
     return !is_any_state(state) ? 0 : by_state[state]->size();
   }
+
+public:
+  void hit_session(Session *session);
 };
 
 std::ostream& operator<<(std::ostream &out, const Session &s);