From 4cf7815cdcd8efbbb981ef45b3eabee387b4de21 Mon Sep 17 00:00:00 2001
From: Venky Shankar <vshankar@redhat.com>
Date: Sun, 5 Aug 2018 23:37:18 -0400
Subject: [PATCH] mds: evict clients that do not respond to cap revoke by MDS

By default, preserve old behaviour. When configured with a non
default value, evict clients that have not responded to cap
revoke by MDS for the configured amount of seconds.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
---
 src/common/options.cc |  4 ++++
 src/mds/MDSDaemon.cc  |  1 +
 src/mds/MDSRank.cc    |  2 +-
 src/mds/MDSRank.h     |  3 ++-
 src/mds/Server.cc     | 30 ++++++++++++++++++++++++++++++
 src/mds/Server.h      |  6 ++++++
 6 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 7d52edd0a47fc..d9fdc8315b177 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7177,6 +7177,10 @@ std::vector<Option> get_mds_options() {
     Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(60)
     .set_description("rate of decay in seconds for calculating request load average"),
+
+    Option("mds_cap_revoke_eviction_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
+     .set_default(0)
+     .set_description("number of seconds after which clients which have not responded to cap revoke messages by the MDS are evicted."),
   });
 }
 
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 33144d267a543..164a1fea7f976 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -361,6 +361,7 @@ const char** MDSDaemon::get_tracked_conf_keys() const
     "host",
     "fsid",
     "mds_request_load_average_decay_rate",
+    "mds_cap_revoke_eviction_timeout",
     NULL
   };
   return KEYS;
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index a8b78664764b2..818ae454b31a0 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -28,7 +28,6 @@
 #include "MDBalancer.h"
 #include "Migrator.h"
 #include "Locker.h"
-#include "Server.h"
 #include "InoTable.h"
 #include "mon/MonClient.h"
 #include "common/HeartbeatMap.h"
@@ -321,6 +320,7 @@ void MDSRankDispatcher::tick()
   // ...
   if (is_clientreplay() || is_active() || is_stopping()) {
     server->find_idle_sessions();
+    server->evict_cap_revoke_non_responders();
     locker->tick();
   }
 
diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h
index 86b9c76022d0f..660a40a4842f1 100644
--- a/src/mds/MDSRank.h
+++ b/src/mds/MDSRank.h
@@ -34,6 +34,7 @@
 #include "MDLog.h"
 #include "MDSContext.h"
 #include "PurgeQueue.h"
+#include "Server.h"
 #include "osdc/Journaler.h"
 
 // Full .h import instead of forward declaration for PerfCounter, for the
@@ -104,7 +105,6 @@ namespace ceph {
   struct heartbeat_handle_d;
 }
 
-class Server;
 class Locker;
 class MDCache;
 class MDLog;
@@ -226,6 +226,7 @@ class MDSRank {
                             const std::set <std::string> &changed)
     {
       sessionmap.handle_conf_change(conf, changed);
+      server->handle_conf_change(conf, changed);
       mdcache->handle_conf_change(conf, changed, *mdsmap);
       purge_queue.handle_conf_change(conf, changed, *mdsmap);
     }
diff --git a/src/mds/Server.cc b/src/mds/Server.cc
index e15c46ead1417..21e59f263f998 100644
--- a/src/mds/Server.cc
+++ b/src/mds/Server.cc
@@ -826,6 +826,36 @@ void Server::find_idle_sessions()
   }
 }
 
+void Server::evict_cap_revoke_non_responders() {
+  if (!cap_revoke_eviction_timeout) {
+    return;
+  }
+
+  std::list<client_t> to_evict;
+  mds->locker->get_late_revoking_clients(&to_evict, cap_revoke_eviction_timeout);
+
+  for (auto const &client: to_evict) {
+    mds->clog->warn() << "client id " << client << " has not responded to"
+                      << " cap revoke by MDS for over " << cap_revoke_eviction_timeout
+                      << " seconds, evicting";
+    dout(1) << __func__ << ": evicting cap revoke non-responder client id "
+            << client << dendl;
+
+    std::stringstream ss;
+    mds->evict_client(client.v, false, g_conf()->mds_session_blacklist_on_evict,
+                      ss, nullptr);
+  }
+}
+
+void Server::handle_conf_change(const ConfigProxy& conf,
+                                const std::set <std::string> &changed) {
+  if (changed.count("mds_cap_revoke_eviction_timeout")) {
+    cap_revoke_eviction_timeout = g_conf().get_val<double>("mds_cap_revoke_eviction_timeout");
+    dout(20) << __func__ << " cap revoke eviction timeout changed to "
+            << cap_revoke_eviction_timeout << dendl;
+  }
+}
+
 /*
  * XXX bump in the interface here, not using an MDSInternalContextBase here
  * because all the callers right now happen to use a SaferCond
diff --git a/src/mds/Server.h b/src/mds/Server.h
index d848bd6cab270..36bbbbbe51c10 100644
--- a/src/mds/Server.h
+++ b/src/mds/Server.h
@@ -95,6 +95,8 @@ private:
   feature_bitset_t supported_features;
   feature_bitset_t required_client_features;
 
+  double cap_revoke_eviction_timeout = 0;
+
   friend class MDSContinuation;
   friend class ServerContext;
   friend class ServerLogContext;
@@ -319,6 +321,10 @@ public:
 			       CDentry *destdn, CDentry *staydn, map<client_t,MClientSnap::ref> splits[2],
 			       bool finish_mdr);
 
+  void evict_cap_revoke_non_responders();
+  void handle_conf_change(const ConfigProxy& conf,
+                          const std::set <std::string> &changed);
+
 private:
   void reply_client_request(MDRequestRef& mdr, const MClientReply::ref &reply);
 };
-- 
2.39.5