mds: fix false "failing to respond to cache pressure" warning

the false warning happens in following sequence of events
- MDS has cache pressure, sends recall state messages to clients
- Client does not trim as many caps as MDS expected. So MDS
  does not reset session->recalled_at
- MDS no longer has cache pressure, it stop sending recall state
  messages to clients.
- Client does not release its caps. So session->recalled_at in
  MDS keeps unchanged

Signed-off-by: Yan, Zheng <zyan@redhat.com>

diff --git a/src/mds/Beacon.cc b/src/mds/Beacon.cc
index c6fd128b8a3da15daa4718f7a42cbc7e631a81e1..713e5764cf12a595e5a4818c261645d6884d44b2 100644 (file)
--- a/src/mds/Beacon.cc
+++ b/src/mds/Beacon.cc
@@ -385,8 +385,10 @@ void Beacon::notify_health(MDSRank const *mds)
   {
     set<Session*> sessions;
     mds->sessionmap.get_client_session_set(sessions);
+
     utime_t cutoff = ceph_clock_now(g_ceph_context);
     cutoff -= g_conf->mds_recall_state_timeout;
+    utime_t last_recall = mds->mdcache->last_recall_state;
 
     std::list<MDSHealthMetric> late_recall_metrics;
     std::list<MDSHealthMetric> large_completed_requests_metrics;
@@ -396,7 +398,10 @@ void Beacon::notify_health(MDSRank const *mds)
         dout(20) << "Session servicing RECALL " << session->info.inst
           << ": " << session->recalled_at << " " << session->recall_release_count
           << "/" << session->recall_count << dendl;
-        if (session->recalled_at < cutoff) {
+       if (last_recall < cutoff || session->last_recall_sent < last_recall) {
+         dout(20) << "  no longer recall" << dendl;
+         session->clear_recalled_at();
+       } else if (session->recalled_at < cutoff) {
           dout(20) << "  exceeded timeout " << session->recalled_at << " vs. " << cutoff << dendl;
           std::ostringstream oss;
          oss << "Client " << session->get_human_name() << " failing to respond to cache pressure";

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index e6c600d84b98599facc9870762e937385311392f..613b15fc93610423be2e8750c80cfc947e187085 100644 (file)
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -7334,10 +7334,11 @@ void MDCache::check_memory_usage()
 
   if (num_inodes_with_caps > g_conf->mds_cache_size) {
     float ratio = (float)g_conf->mds_cache_size * .9 / (float)num_inodes_with_caps;
-    if (ratio < 1.0)
+    if (ratio < 1.0) {
+      last_recall_state = ceph_clock_now(g_ceph_context);
       mds->server->recall_client_state(ratio);
+    }
   }
-
 }
 
 

diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 374b27d7c0ef63f2be303b1c7110c13fbab792ac..222855d13f101360906ccbf3f2c83f9fa8c1ef2a 100644 (file)
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -693,6 +693,8 @@ public:
   void trim_client_leases();
   void check_memory_usage();
 
+  utime_t last_recall_state;
+
   // shutdown
   void shutdown_start();
   void shutdown_check();

diff --git a/src/mds/SessionMap.cc b/src/mds/SessionMap.cc
index efe43f5d11338773aba22cecff8beb40c1e1e212..297c6e1d37be382bbf53e2619684ba833d6abe6d 100644 (file)
--- a/src/mds/SessionMap.cc
+++ b/src/mds/SessionMap.cc
@@ -789,11 +789,8 @@ void Session::notify_cap_release(size_t n_caps)
 {
   if (!recalled_at.is_zero()) {
     recall_release_count += n_caps;
-    if (recall_release_count >= recall_count) {
-      recalled_at = utime_t();
-      recall_count = 0;
-      recall_release_count = 0;
-    }
+    if (recall_release_count >= recall_count)
+      clear_recalled_at();
   }
 }
 
@@ -808,13 +805,22 @@ void Session::notify_recall_sent(int const new_limit)
   if (recalled_at.is_zero()) {
     // Entering recall phase, set up counters so we can later
     // judge whether the client has respected the recall request
-    recalled_at = ceph_clock_now(g_ceph_context);
+    recalled_at = last_recall_sent = ceph_clock_now(g_ceph_context);
     assert (new_limit < caps.size());  // Behaviour of Server::recall_client_state
     recall_count = caps.size() - new_limit;
     recall_release_count = 0;
+  } else {
+    last_recall_sent = ceph_clock_now(g_ceph_context);
   }
 }
 
+void Session::clear_recalled_at()
+{
+  recalled_at = last_recall_sent = utime_t();
+  recall_count = 0;
+  recall_release_count = 0;
+}
+
 void Session::set_client_metadata(map<string, string> const &meta)
 {
   info.client_metadata = meta;

diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h
index 29597904875b4c02a41900e52e2971b7199d14b2..241f10b19066d5284da345aeae292aeaeb1a6386 100644 (file)
--- a/src/mds/SessionMap.h
+++ b/src/mds/SessionMap.h
@@ -129,6 +129,7 @@ public:
 
   // Ephemeral state for tracking progress of capability recalls
   utime_t recalled_at;  // When was I asked to SESSION_RECALL?
+  utime_t last_recall_sent;
   uint32_t recall_count;  // How many caps was I asked to SESSION_RECALL?
   uint32_t recall_release_count;  // How many caps have I actually revoked?
 
@@ -148,6 +149,7 @@ public:
 
   void notify_cap_release(size_t n_caps);
   void notify_recall_sent(int const new_limit);
+  void clear_recalled_at();
 
   inodeno_t next_ino() {
     if (info.prealloc_inos.empty())
@@ -315,7 +317,7 @@ public:
 
   Session() : 
     state(STATE_CLOSED), state_seq(0), importing_count(0),
-    recalled_at(), recall_count(0), recall_release_count(0),
+    recall_count(0), recall_release_count(0),
     auth_caps(g_ceph_context),
     connection(NULL), item_session_list(this),
     requests(0),  // member_offset passed to front() manually