mds: add asok command that dumps metadata popularity

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>
(cherry picked from commit c078591ad1a33447f4de731c846844f4a0433513)

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 1157e07dfe1e1b5fdd6aff0c112494310d13c469..dd19ac02cec1ef79d8c5c89024e0fcfcbcf6576d 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -3057,6 +3057,28 @@ void CDir::dump(Formatter *f) const
   MDSCacheObject::dump(f);
 }
 
+void CDir::dump_load(Formatter *f, utime_t now, const DecayRate& rate)
+{
+  f->dump_stream("path") << get_path();
+  f->dump_stream("dirfrag") << dirfrag();
+
+  f->open_object_section("pop_me");
+  pop_me.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_nested");
+  pop_nested.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_auth_subtree");
+  pop_auth_subtree.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_auth_subtree_nested");
+  pop_auth_subtree_nested.dump(f, now, rate);
+  f->close_section();
+}
+
 /****** Scrub Stuff *******/
 
 void CDir::scrub_info_create() const

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 8de1867329df9aae00a736fc667b3ee69a241e2f..b8ad1b13311a76562e9d1889a8b3fca4064913c1 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -747,6 +747,7 @@ public:
   ostream& print_db_line_prefix(ostream& out) override;
   void print(ostream& out) override;
   void dump(Formatter *f) const;
+  void dump_load(Formatter *f, utime_t now, const DecayRate& rate);
 };
 
 #endif

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index d8886ce96bd3255ef9a2bbc25968df336597ae50..ba117ca58d4b6be349f8f717560f8615408d7f83 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -615,6 +615,8 @@ void MDBalancer::prep_rebalance(int beat)
              << dendl;
     }
 
+    mds_meta_load.clear();
+
     double total_load = 0.0;
     multimap<double,mds_rank_t> load_map;
     for (mds_rank_t i=mds_rank_t(0); i < mds_rank_t(cluster_size); i++) {
@@ -1241,3 +1243,101 @@ void MDBalancer::handle_mds_failure(mds_rank_t who)
     last_epoch_under = 0;
   }
 }
+
+int MDBalancer::dump_loads(Formatter *f)
+{
+  utime_t now = ceph_clock_now();
+  DecayRate& decayrate = mds->mdcache->decayrate;
+
+  list<CDir*> dfs;
+  if (mds->mdcache->get_root()) {
+    mds->mdcache->get_root()->get_dirfrags(dfs);
+  } else {
+    dout(5) << "dump_load no root" << dendl;
+  }
+
+  f->open_object_section("loads");
+
+  f->open_array_section("dirfrags");
+  while (!dfs.empty()) {
+    CDir *dir = dfs.front();
+    dfs.pop_front();
+
+    if (f) {
+      f->open_object_section("dir");
+      dir->dump_load(f, now, decayrate);
+      f->close_section();
+    }
+
+    for (auto it = dir->begin(); it != dir->end(); ++it) {
+      CInode *in = it->second->get_linkage()->get_inode();
+      if (!in || !in->is_dir())
+       continue;
+
+      list<CDir*> ls;
+      in->get_dirfrags(ls);
+      for (auto subdir : ls) {
+       if (subdir->pop_nested.meta_load() < .001)
+         continue;
+       dfs.push_back(subdir);
+      }
+    }
+  }
+  f->close_section();  // dirfrags array
+
+  f->open_object_section("mds_load");
+  {
+
+    auto dump_mds_load = [f, now](mds_load_t& load) {
+      f->dump_float("request_rate", load.req_rate);
+      f->dump_float("cache_hit_rate", load.cache_hit_rate);
+      f->dump_float("queue_length", load.queue_len);
+      f->dump_float("cpu_load", load.cpu_load_avg);
+      f->dump_float("mds_load", load.mds_load());
+
+      DecayRate rate; // no decay
+      f->open_object_section("auth_dirfrags");
+      load.auth.dump(f, now, rate);
+      f->close_section();
+      f->open_object_section("all_dirfrags");
+      load.all.dump(f, now, rate);
+      f->close_section();
+    };
+
+    for (auto p : mds_load) {
+      stringstream name;
+      name << "mds." << p.first;
+      f->open_object_section(name.str().c_str());
+      dump_mds_load(p.second);
+      f->close_section();
+    }
+  }
+  f->close_section(); // mds_load
+
+  f->open_object_section("mds_meta_load");
+  for (auto p : mds_meta_load) {
+    stringstream name;
+    name << "mds." << p.first;
+    f->dump_float(name.str().c_str(), p.second);
+  }
+  f->close_section(); // mds_meta_load
+
+  f->open_object_section("mds_import_map");
+  for (auto p : mds_import_map) {
+    stringstream name1;
+    name1 << "mds." << p.first;
+    f->open_array_section(name1.str().c_str());
+    for (auto q : p.second) {
+      f->open_object_section("from");
+      stringstream name2;
+      name2 << "mds." << q.first;
+      f->dump_float(name2.str().c_str(), q.second);
+      f->close_section();
+    }
+    f->close_section(); // mds.? array
+  }
+  f->close_section(); // mds_import_map
+
+  f->close_section(); // loads
+  return 0;
+}

diff --git a/src/mds/MDBalancer.h b/src/mds/MDBalancer.h
index d23185b22f898e1ffe278178813bfc62b5683991..976892654b486c157deadf1295149db4321ebb4c 100644 (file)
--- a/src/mds/MDBalancer.h
+++ b/src/mds/MDBalancer.h
@@ -75,6 +75,8 @@ public:
 
   void handle_mds_failure(mds_rank_t who);
 
+  int dump_loads(Formatter *f);
+
 private:
   typedef struct {
     std::map<mds_rank_t, double> targets;

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index d1a22df71ab9ef7d49ab4fe013560d491d9ac043..7841cc6dbed21f6934d98e9e3f03d1ad93c2d991 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -258,6 +258,11 @@ void MDSDaemon::set_up_admin_socket()
                                     asok_hook,
                                     "dump metadata cache for subtree");
   assert(r == 0);
+  r = admin_socket->register_command("dump loads",
+                                     "dump loads",
+                                     asok_hook,
+                                     "dump metadata loads");
+  assert(r == 0);
   r = admin_socket->register_command("session evict",
                                     "session evict name=client_id,type=CephString",
                                     asok_hook,
@@ -327,6 +332,7 @@ void MDSDaemon::clean_up_admin_socket()
   admin_socket->unregister_command("dump cache");
   admin_socket->unregister_command("cache status");
   admin_socket->unregister_command("dump tree");
+  admin_socket->unregister_command("dump loads");
   admin_socket->unregister_command("session evict");
   admin_socket->unregister_command("osdmap barrier");
   admin_socket->unregister_command("session ls");

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index de0a7c3e60af05b90987d7c67cd5ebfce0e46dcb..65dd5efb430df20e845a4fa507841d13a9b1cf1c 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1991,6 +1991,13 @@ bool MDSRankDispatcher::handle_asok_command(
         f->reset();
       }
     }
+  } else if (command == "dump loads") {
+    Mutex::Locker l(mds_lock);
+    int r = balancer->dump_loads(f);
+    if (r != 0) {
+      ss << "Failed to dump loads: " << cpp_strerror(r);
+      f->reset();
+    }
   } else if (command == "force_readonly") {
     Mutex::Locker l(mds_lock);
     mdcache->force_readonly();

diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 84fff861f444cfdd55577bdf5dafc81479d669fb..8ca1ced1bdf5aba2fd0e8c8954fd545285ebb501 100644 (file)
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -639,6 +639,16 @@ void dirfrag_load_vec_t::dump(Formatter *f) const
   f->close_section();
 }
 
+void dirfrag_load_vec_t::dump(Formatter *f, utime_t now, const DecayRate& rate)
+{
+  f->dump_float("meta_load", meta_load(now, rate));
+  f->dump_float("IRD", get(META_POP_IRD).get(now, rate));
+  f->dump_float("IWR", get(META_POP_IWR).get(now, rate));
+  f->dump_float("READDIR", get(META_POP_READDIR).get(now, rate));
+  f->dump_float("FETCH", get(META_POP_FETCH).get(now, rate));
+  f->dump_float("STORE", get(META_POP_STORE).get(now, rate));
+}
+
 void dirfrag_load_vec_t::generate_test_instances(list<dirfrag_load_vec_t*>& ls)
 {
   utime_t sample;

diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index 0ede619d54337b4874363190919cc5bc24215a39..eaaa36caf349dd32d0504c33aa5101154b60456c 100644 (file)
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -1466,6 +1466,7 @@ public:
     decode(sample, p);
   }
   void dump(Formatter *f) const;
+  void dump(Formatter *f, utime_t now, const DecayRate& rate);
   static void generate_test_instances(list<dirfrag_load_vec_t*>& ls);
 
   DecayCounter &get(int t) {