client: retry remount on dcache invalidation failure

For some (unknown) reason, there have been reports of
ceph-fuse crash due to failure is remounting at the
time of invalidating kernel dentry cache. This issue
is not also reproducible yet.

Therefore, as suggested by Patrick and Zheng, for a
temporary workaround, client would ignore the failure
as the invalidation would be retried again. There is
a max cap on the number of consecutive remount failures
after which client would abort.

Fixes: http://tracker.ceph.com/issues/27657
Signed-off-by: Venky Shankar <vshankar@redhat.com>

diff --git a/src/client/Client.cc b/src/client/Client.cc
index e8b83e10eb729dff4b60654949519b1f1faeb132..006bed108afdf973fa104a14d7fa05847f127dfc 100644 (file)
--- a/src/client/Client.cc
+++ b/src/client/Client.cc
@@ -4047,11 +4047,15 @@ void Client::remove_session_caps(MetaSession *s)
   sync_cond.Signal();
 }
 
-int Client::_do_remount(void)
+int Client::_do_remount(bool retry_on_error)
 {
+  uint64_t max_retries = g_conf().get_val<uint64_t>("mds_max_retries_on_remount_failure");
+
   errno = 0;
   int r = remount_cb(callback_handle);
-  if (r != 0) {
+  if (r == 0) {
+    retries_on_invalidate = 0;
+  } else {
     int e = errno;
     client_t whoami = get_nodeid();
     if (r == -1) {
@@ -4063,8 +4067,10 @@ int Client::_do_remount(void)
           "failed to remount (to trim kernel dentries): "
           "return code = " << r << dendl;
     }
-    bool should_abort = cct->_conf.get_val<bool>("client_die_on_failed_remount") ||
-        cct->_conf.get_val<bool>("client_die_on_failed_dentry_invalidate");
+    bool should_abort =
+      (cct->_conf.get_val<bool>("client_die_on_failed_remount") ||
+       cct->_conf.get_val<bool>("client_die_on_failed_dentry_invalidate")) &&
+      !(retry_on_error && (++retries_on_invalidate < max_retries));
     if (should_abort && !unmounting) {
       lderr(cct) << "failed to remount for kernel dentry trimming; quitting!" << dendl;
       ceph_abort();
@@ -4080,7 +4086,7 @@ public:
   explicit C_Client_Remount(Client *c) : client(c) {}
   void finish(int r) override {
     ceph_assert(r == 0);
-    client->_do_remount();
+    client->_do_remount(true);
   }
 };
 
@@ -10259,7 +10265,7 @@ int Client::test_dentry_handling(bool can_invalidate)
     r = 0;
   } else if (remount_cb) {
     ldout(cct, 1) << "using remount_cb" << dendl;
-    r = _do_remount();
+    r = _do_remount(false);
   }
   if (r) {
     bool should_abort = cct->_conf.get_val<bool>("client_die_on_failed_dentry_invalidate");

diff --git a/src/client/Client.h b/src/client/Client.h
index 482e45df1dd89ea65fd80e35238c829371b4ca04..896bde0e6ba1fa2cb98663fb5faa4947e71992fc 100644 (file)
--- a/src/client/Client.h
+++ b/src/client/Client.h
@@ -1034,7 +1034,7 @@ private:
   int _release_fh(Fh *fh);
   void _put_fh(Fh *fh);
 
-  int _do_remount(void);
+  int _do_remount(bool retry_on_error);
 
   int _read_sync(Fh *f, uint64_t off, uint64_t len, bufferlist *bl, bool *checkeof);
   int _read_async(Fh *f, uint64_t off, uint64_t len, bufferlist *bl);
@@ -1274,6 +1274,8 @@ private:
 
   std::map<std::pair<int64_t,std::string>, int> pool_perms;
   list<Cond*> waiting_for_pool_perm;
+
+  uint64_t retries_on_invalidate = 0;
 };
 
 /**

diff --git a/src/common/options.cc b/src/common/options.cc
index 68f036ce6bccf137fdb4a59dfe6531dbc7f16794..e261827bfc722d4d583e73ccd0fac9909b173c0c 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7199,6 +7199,10 @@ std::vector<Option> get_mds_options() {
     Option("mds_cap_revoke_eviction_timeout", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
      .set_default(0)
      .set_description("number of seconds after which clients which have not responded to cap revoke messages by the MDS are evicted."),
+
+    Option("mds_max_retries_on_remount_failure", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+     .set_default(5)
+     .set_description("number of consecutive failed remount attempts for invalidating kernel dcache after which client would abort."),
   });
 }