From: Simon Gao <simon29rock@gmail.com>
Date: Wed, 26 Aug 2020 13:45:36 +0000 (+0800)
Subject: mds : add timeout to the command, dump cache, to prevent it from running too long... 
X-Git-Tag: v17.1.0~1614^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b45563d814b4b4eddf87bdc3c659267dc26e5cbf;p=ceph.git

mds : add timeout to the command, dump cache, to prevent it from running too long and affecting the service

the default value for timeout is half the heartbeat time, mds_beacon_interval

Signed-off-by: Simon Gao <simon29rock@gmail.com>
---

diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc
index 4af8c901802..f798c21f463 100644
--- a/src/mds/MDCache.cc
+++ b/src/mds/MDCache.cc
@@ -12514,21 +12514,21 @@ void MDCache::dump_tree(CInode *in, const int cur_depth, const int max_depth, Fo
   f->close_section();
 }
 
-int MDCache::dump_cache(std::string_view file_name)
+int MDCache::dump_cache(std::string_view file_name, double timeout)
 {
-  return dump_cache(file_name, NULL);
+  return dump_cache(file_name, NULL, timeout);
 }
 
-int MDCache::dump_cache(Formatter *f)
+int MDCache::dump_cache(Formatter *f, double timeout)
 {
-  return dump_cache(std::string_view(""), f);
+  return dump_cache(std::string_view(""), f, timeout);
 }
 
 /**
  * Dump the metadata cache, either to a Formatter, if
  * provided, else to a plain text file.
  */
-int MDCache::dump_cache(std::string_view fn, Formatter *f)
+int MDCache::dump_cache(std::string_view fn, Formatter *f, double timeout)
 {
   int r = 0;
 
@@ -12614,22 +12614,50 @@ int MDCache::dump_cache(std::string_view fn, Formatter *f)
     return 1;
   };
 
+  auto start = mono_clock::now();
+  int64_t count = 0;
   for (auto &p : inode_map) {
     r = dump_func(p.second);
     if (r < 0)
       goto out;
+    if (!(++count % 1000) &&
+	timeout > 0 &&
+	std::chrono::duration<double>(mono_clock::now() - start).count() > timeout) {
+      r = -ETIMEDOUT;
+      goto out;
+    }
   }
   for (auto &p : snap_inode_map) {
     r = dump_func(p.second);
     if (r < 0)
       goto out;
+    if (!(++count % 1000) &&
+		timeout > 0 &&
+	std::chrono::duration<double>(mono_clock::now() - start).count() > timeout) {
+      r = -ETIMEDOUT;
+      goto out;
+    }
+
   }
   r = 0;
 
  out:
   if (f) {
+    if (r == -ETIMEDOUT)
+    {
+      f->close_section();
+      f->open_object_section("result");
+      f->dump_string("error", "the operation timeout");
+    }
     f->close_section();  // inodes
   } else {
+    if (r == -ETIMEDOUT)
+    {
+      CachedStackStringStream css;
+      *css << "error : the operation timeout" << std::endl;
+      auto sv = css->strv();
+      r = safe_write(fd, sv.data(), sv.size());
+    }
     ::close(fd);
   }
   return r;
diff --git a/src/mds/MDCache.h b/src/mds/MDCache.h
index 1805c54e396..c04f38e1c12 100644
--- a/src/mds/MDCache.h
+++ b/src/mds/MDCache.h
@@ -909,9 +909,9 @@ class MDCache {
   // -- mdsmap --
   void handle_mdsmap(const MDSMap &mdsmap, const MDSMap &oldmap);
 
-  int dump_cache() { return dump_cache({}, nullptr); }
-  int dump_cache(std::string_view filename);
-  int dump_cache(Formatter *f);
+  int dump_cache() { return dump_cache({}, nullptr, 0); }
+  int dump_cache(std::string_view filename, double timeout);
+  int dump_cache(Formatter *f, double timeout);
   void dump_tree(CInode *in, const int cur_depth, const int max_depth, Formatter *f);
 
   void cache_status(Formatter *f);
@@ -1112,7 +1112,7 @@ class MDCache {
   void handle_dentry_link(const cref_t<MDentryLink> &m);
   void handle_dentry_unlink(const cref_t<MDentryUnlink> &m);
 
-  int dump_cache(std::string_view fn, Formatter *f);
+  int dump_cache(std::string_view fn, Formatter *f, double timeout);
 
   void flush_dentry_work(MDRequestRef& mdr);
   /**
diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index 0278eb638b3..edc69c7ac50 100644
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -309,7 +309,9 @@ void MDSDaemon::set_up_admin_socket()
                                      asok_hook,
                                      "migrate a subtree to named MDS");
   ceph_assert(r == 0);
-  r = admin_socket->register_command("dump cache name=path,type=CephString,req=false",
+  r = admin_socket->register_command("dump cache "
+				     "name=path,type=CephString,req=false "
+				     "name=timeout,type=CephInt,range=0,req=false",
                                      asok_hook,
                                      "dump metadata cache (optionally to a file)");
   ceph_assert(r == 0);
diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 14607b9514a..9650d4e4816 100644
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -2728,11 +2728,18 @@ void MDSRankDispatcher::handle_asok_command(
     command_export_dir(f, path, (mds_rank_t)rank);
   } else if (command == "dump cache") {
     std::lock_guard l(mds_lock);
+    int64_t timeout = 0;
+    cmd_getval(cmdmap, "timeout", timeout);
+    auto mds_beacon_interval = g_conf().get_val<double>("mds_beacon_interval");
+    if (timeout <= 0)
+      timeout = mds_beacon_interval / 2;
+    else if (timeout > mds_beacon_interval)
+      timeout = mds_beacon_interval;
     string path;
     if (!cmd_getval(cmdmap, "path", path)) {
-      r = mdcache->dump_cache(f);
+      r = mdcache->dump_cache(f, timeout);
     } else {
-      r = mdcache->dump_cache(path);
+      r = mdcache->dump_cache(path, timeout);
     }
   } else if (command == "cache drop") {
     int64_t timeout = 0;