client: retry remount on dcache invalidation failure

For some (unknown) reason, there have been reports of
ceph-fuse crash due to failure is remounting at the
time of invalidating kernel dentry cache. This issue
is not also reproducible yet.

Therefore, as suggested by Patrick and Zheng, for a
temporary workaround, client would ignore the failure
as the invalidation would be retried again. There is
a max cap on the number of consecutive remount failures
after which client would abort.

Fixes: http://tracker.ceph.com/issues/35931
Signed-off-by: Venky Shankar <vshankar@redhat.com>
(cherry picked from commit d1471f070cd1ad9c0f773e00d2552161d1ad5955)

 Conflicts:
	src/client/Client.cc
	src/client/Client.h
	src/common/options.cc

diff --git a/src/client/Client.cc b/src/client/Client.cc
index aacc68f1fd338d0d2f3d9b48a77c11679012206b..20009ea055a1f37595a294d17a20495310079981 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -4049,11 +4049,15 @@ void Client::remove_session_caps(MetaSession *s)
   sync_cond.Signal();
 }
 
-int Client::_do_remount(void)
+int Client::_do_remount(bool retry_on_error)
 {
+  uint64_t max_retries = cct->_conf->get_val<uint64_t>("mds_max_retries_on_remount_failure");
+
   errno = 0;
   int r = remount_cb(callback_handle);
-  if (r != 0) {
+  if (r == 0) {
+    retries_on_invalidate = 0;
+  } else {
     int e = errno;
     client_t whoami = get_nodeid();
     if (r == -1) {
@@ -4065,8 +4069,10 @@ int Client::_do_remount(void)
           "failed to remount (to trim kernel dentries): "
           "return code = " << r << dendl;
     }
-    bool should_abort = cct->_conf->get_val<bool>("client_die_on_failed_remount") ||
-        cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate");
+    bool should_abort =
+      (cct->_conf->get_val<bool>("client_die_on_failed_remount") ||
+       cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate")) &&
+      !(retry_on_error && (++retries_on_invalidate < max_retries));
     if (should_abort && !unmounting) {
       lderr(cct) << "failed to remount for kernel dentry trimming; quitting!" << dendl;
       ceph_abort();
@@ -4082,7 +4088,7 @@ public:
   explicit C_Client_Remount(Client *c) : client(c) {}
   void finish(int r) override {
     assert(r == 0);
-    client->_do_remount();
+    client->_do_remount(true);
   }
 };
 
@@ -10113,7 +10119,7 @@ int Client::test_dentry_handling(bool can_invalidate)
     r = 0;
   } else if (remount_cb) {
     ldout(cct, 1) << "using remount_cb" << dendl;
-    r = _do_remount();
+    r = _do_remount(false);
   }
   if (r) {
     bool should_abort = cct->_conf->get_val<bool>("client_die_on_failed_dentry_invalidate");

diff --git a/src/client/Client.h b/src/client/Client.h
index 2616f6d7169194eccaec4709c58b04bb348f9b8f..99707483821b0b4e7d4ff919051523c5b8856357 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -764,7 +764,7 @@ private:
   int _release_fh(Fh *fh);
   void _put_fh(Fh *fh);
 
-  int _do_remount(void);
+  int _do_remount(bool retry_on_error);
   friend class C_Client_Remount;
 
   struct C_Readahead : public Context {
@@ -1253,6 +1253,9 @@ public:
   uint32_t get_deleg_timeout() { return deleg_timeout; }
   int set_deleg_timeout(uint32_t timeout);
   int ll_delegation(Fh *fh, unsigned cmd, ceph_deleg_cb_t cb, void *priv);
+
+private:
+  uint64_t retries_on_invalidate = 0;
 };
 
 /**

diff --git a/src/common/options.cc b/src/common/options.cc
index c45f900154f787cc55eba207a25eb07734f65375..c6b421dc9a7e10eff05f7a64a3ea0dae261d034a 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -6493,6 +6493,9 @@ std::vector<Option> get_mds_options() {
     .set_default(0)
     .set_description(""),
 
+    Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+     .set_default(5)
+     .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
   });
 }