client: retry remount on dcache invalidation failure

For some (unknown) reason, there have been reports of
ceph-fuse crash due to failure is remounting at the
time of invalidating kernel dentry cache. This issue
is not also reproducible yet.

Therefore, as suggested by Patrick and Zheng, for a
temporary workaround, client would ignore the failure
as the invalidation would be retried again. There is
a max cap on the number of consecutive remount failures
after which client would abort.

Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit d1471f070cd1ad9c0f773e00d2552161d1ad5955)

 Conflicts:
	src/client/Client.cc
	src/client/Client.h
	src/common/options.cc

diff --git a/src/client/Client.cc b/src/client/Client.cc
index 1380eb5646ee98902dfdea976b4de278e2c0f3a3..3ef1c4b2d595701de7a3257be168408101322f12 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -4047,11 +4047,15 @@ void Client::remove_session_caps(MetaSession *s)
   sync_cond.Signal();
 }
 
-int Client::_do_remount(void)
+int Client::_do_remount(bool retry_on_error)
 {
+  uint64_t max_retries = cct->_conf->get_val<uint64_t>("mds_max_retries_on_remount_failure");
+
   errno = 0;
   int r = remount_cb(callback_handle);
-  if (r != 0) {
+  if (r == 0) {
+    retries_on_invalidate = 0;
+  } else {
     int e = errno;
     client_t whoami = get_nodeid();
     if (r == -1) {
@@ -4063,8 +4067,10 @@ int Client::_do_remount(void)
           "failed to remount (to trim kernel dentries): "
           "return code = " << r << dendl;
     }
-    bool should_abort = cct->_conf->get_val<bool>("client_die_on_failed_remount") ||
-        cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate");
+    bool should_abort =
+      (cct->_conf->get_val<bool>("client_die_on_failed_remount") ||
+       cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate")) &&
+      !(retry_on_error && (++retries_on_invalidate < max_retries));
     if (should_abort && !unmounting) {
       lderr(cct) << "failed to remount for kernel dentry trimming; quitting!" << dendl;
       ceph_abort();
@@ -4080,7 +4086,7 @@ public:
   explicit C_Client_Remount(Client *c) : client(c) {}
   void finish(int r) override {
     assert(r == 0);
-    client->_do_remount();
+    client->_do_remount(true);
   }
 };
 
@@ -10116,7 +10122,7 @@ int Client::test_dentry_handling(bool can_invalidate)
     r = 0;
   } else if (remount_cb) {
     ldout(cct, 1) << "using remount_cb" << dendl;
-    r = _do_remount();
+    r = _do_remount(false);
   }
   if (r) {
     bool should_abort = cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate");

diff --git a/src/client/Client.h b/src/client/Client.h
index 6350242fab04b829cad2a07539600794c7049668..9c75a1c4590cec383f13883f09971dc1893e746e 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -762,7 +762,7 @@ private:
   int _release_fh(Fh *fh);
   void _put_fh(Fh *fh);
 
-  int _do_remount(void);
+  int _do_remount(bool retry_on_error);
 
   struct C_Readahead : public Context {
     Client *client;
@@ -1255,6 +1255,9 @@ public:
   uint32_t get_deleg_timeout() { return deleg_timeout; }
   int set_deleg_timeout(uint32_t timeout);
   int ll_delegation(Fh *fh, unsigned cmd, ceph_deleg_cb_t cb, void *priv);
+
+private:
+  uint64_t retries_on_invalidate = 0;
 };
 
 /**

diff --git a/src/common/options.cc b/src/common/options.cc
index d58ecb168b226e831a7c79ebc37f366d2bbaf237..98dad74817e99443c217fcd6e9ab6098cd35ed77 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7031,6 +7031,10 @@ std::vector<Option> get_mds_options() {
     Option("mds_request_load_average_decay_rate", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(60)
     .set_description("rate of decay in seconds for calculating request load average"),
+
+    Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+     .set_default(5)
+     .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
   });
 }