mds: add asok command that dumps metadata popularity

Signed-off-by: "Yan, Zheng" <zyan@redhat.com>

diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc
index 3c4a5428193846935902d3949f1bd943cf916665..e8cbe637b5a0539f1cada3169ee578f653575431 100644 (file)
--- a/src/mds/CDir.cc
+++ b/src/mds/CDir.cc
@@ -3032,6 +3032,28 @@ void CDir::dump(Formatter *f) const
   MDSCacheObject::dump(f);
 }
 
+void CDir::dump_load(Formatter *f, utime_t now, const DecayRate& rate)
+{
+  f->dump_stream("path") << get_path();
+  f->dump_stream("dirfrag") << dirfrag();
+
+  f->open_object_section("pop_me");
+  pop_me.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_nested");
+  pop_nested.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_auth_subtree");
+  pop_auth_subtree.dump(f, now, rate);
+  f->close_section();
+
+  f->open_object_section("pop_auth_subtree_nested");
+  pop_auth_subtree_nested.dump(f, now, rate);
+  f->close_section();
+}
+
 /****** Scrub Stuff *******/
 
 void CDir::scrub_info_create() const

diff --git a/src/mds/CDir.h b/src/mds/CDir.h
index 45b2998b3d24d00ec67268b9d00b51978094f1d6..b73dfeba069767492ad5b9b1dff9ffa290c3cc8b 100644 (file)
--- a/src/mds/CDir.h
+++ b/src/mds/CDir.h
@@ -749,6 +749,7 @@ public:
   ostream& print_db_line_prefix(ostream& out) override;
   void print(ostream& out) override;
   void dump(Formatter *f) const;
+  void dump_load(Formatter *f, utime_t now, const DecayRate& rate);
 };
 
 #endif

diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc
index 186d5f40e91e5278c0d9cb2aef963188e908ea5f..bda41a13ae1bdbe369128f3fec6565a4649a32e8 100644 (file)
--- a/src/mds/MDBalancer.cc
+++ b/src/mds/MDBalancer.cc
@@ -604,6 +604,8 @@ void MDBalancer::prep_rebalance(int beat)
              << dendl;
     }
 
+    mds_meta_load.clear();
+
     double total_load = 0.0;
     multimap<double,mds_rank_t> load_map;
     for (mds_rank_t i=mds_rank_t(0); i < mds_rank_t(cluster_size); i++) {
@@ -1232,3 +1234,101 @@ void MDBalancer::handle_mds_failure(mds_rank_t who)
     last_epoch_under = 0;
   }
 }
+
+int MDBalancer::dump_loads(Formatter *f)
+{
+  utime_t now = ceph_clock_now();
+  DecayRate& decayrate = mds->mdcache->decayrate;
+
+  list<CDir*> dfs;
+  if (mds->mdcache->get_root()) {
+    mds->mdcache->get_root()->get_dirfrags(dfs);
+  } else {
+    dout(5) << "dump_load no root" << dendl;
+  }
+
+  f->open_object_section("loads");
+
+  f->open_array_section("dirfrags");
+  while (!dfs.empty()) {
+    CDir *dir = dfs.front();
+    dfs.pop_front();
+
+    if (f) {
+      f->open_object_section("dir");
+      dir->dump_load(f, now, decayrate);
+      f->close_section();
+    }
+
+    for (CDir::map_t::iterator it = dir->begin(); it != dir->end(); ++it) {
+      CInode *in = it->second->get_linkage()->get_inode();
+      if (!in || !in->is_dir())
+       continue;
+
+      list<CDir*> ls;
+      in->get_dirfrags(ls);
+      for (auto subdir : ls) {
+       if (subdir->pop_nested.meta_load() < .001)
+         continue;
+       dfs.push_back(subdir);
+      }
+    }
+  }
+  f->close_section();  // dirfrags array
+
+  f->open_object_section("mds_load");
+  {
+
+    auto dump_mds_load = [this, f, now](mds_load_t& load) {
+      f->dump_float("request_rate", load.req_rate);
+      f->dump_float("cache_hit_rate", load.cache_hit_rate);
+      f->dump_float("queue_length", load.queue_len);
+      f->dump_float("cpu_load", load.cpu_load_avg);
+      f->dump_float("mds_load", load.mds_load());
+
+      DecayRate rate; // no decay
+      f->open_object_section("auth_dirfrags");
+      load.auth.dump(f, now, rate);
+      f->close_section();
+      f->open_object_section("all_dirfrags");
+      load.all.dump(f, now, rate);
+      f->close_section();
+    };
+
+    for (auto p : mds_load) {
+      stringstream name;
+      name << "mds." << p.first;
+      f->open_object_section(name.str().c_str());
+      dump_mds_load(p.second);
+      f->close_section();
+    }
+  }
+  f->close_section(); // mds_load
+
+  f->open_object_section("mds_meta_load");
+  for (auto p : mds_meta_load) {
+    stringstream name;
+    name << "mds." << p.first;
+    f->dump_float(name.str().c_str(), p.second);
+  }
+  f->close_section(); // mds_meta_load
+
+  f->open_object_section("mds_import_map");
+  for (auto p : mds_import_map) {
+    stringstream name1;
+    name1 << "mds." << p.first;
+    f->open_array_section(name1.str().c_str());
+    for (auto q : p.second) {
+      f->open_object_section("from");
+      stringstream name2;
+      name2 << "mds." << q.first;
+      f->dump_float(name2.str().c_str(), q.second);
+      f->close_section();
+    }
+    f->close_section(); // mds.? array
+  }
+  f->close_section(); // mds_import_map
+
+  f->close_section(); // loads
+  return 0;
+}

diff --git a/src/mds/MDBalancer.h b/src/mds/MDBalancer.h
index d23185b22f898e1ffe278178813bfc62b5683991..976892654b486c157deadf1295149db4321ebb4c 100644 (file)
--- a/src/mds/MDBalancer.h
+++ b/src/mds/MDBalancer.h
@@ -75,6 +75,8 @@ public:
 
   void handle_mds_failure(mds_rank_t who);
 
+  int dump_loads(Formatter *f);
+
 private:
   typedef struct {
     std::map<mds_rank_t, double> targets;

diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc
index c8a83360b7bdbfb4f03e164268b08c9e4abbf916..364bac02220a4131ba4065adb069e27b73dc8560 100644 (file)
--- a/src/mds/MDSDaemon.cc
+++ b/src/mds/MDSDaemon.cc
@@ -254,6 +254,11 @@ void MDSDaemon::set_up_admin_socket()
                                     asok_hook,
                                     "dump metadata cache for subtree");
   assert(r == 0);
+  r = admin_socket->register_command("dump loads",
+                                     "dump loads",
+                                     asok_hook,
+                                     "dump metadata loads");
+  assert(r == 0);
   r = admin_socket->register_command("session evict",
                                     "session evict name=client_id,type=CephString",
                                     asok_hook,
@@ -323,6 +328,7 @@ void MDSDaemon::clean_up_admin_socket()
   admin_socket->unregister_command("dump cache");
   admin_socket->unregister_command("cache status");
   admin_socket->unregister_command("dump tree");
+  admin_socket->unregister_command("dump loads");
   admin_socket->unregister_command("session evict");
   admin_socket->unregister_command("osdmap barrier");
   admin_socket->unregister_command("session ls");

diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc
index 51ae378947f57f30fbb44557670b2ce98e36a775..0e66ef141095ce88cb90e4bd90e268bac5a2aa3f 100644 (file)
--- a/src/mds/MDSRank.cc
+++ b/src/mds/MDSRank.cc
@@ -1976,6 +1976,13 @@ bool MDSRankDispatcher::handle_asok_command(
         f->reset();
       }
     }
+  } else if (command == "dump loads") {
+    Mutex::Locker l(mds_lock);
+    int r = balancer->dump_loads(f);
+    if (r != 0) {
+      ss << "Failed to dump loads: " << cpp_strerror(r);
+      f->reset();
+    }
   } else if (command == "force_readonly") {
     Mutex::Locker l(mds_lock);
     mdcache->force_readonly();

diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc
index 71c5f11d3c2c0bd04cb70377c544e922b6fc5144..481c7f3f82cb2e9557daebb90e92955ad42c1c44 100644 (file)
--- a/src/mds/mdstypes.cc
+++ b/src/mds/mdstypes.cc
@@ -969,6 +969,16 @@ void dirfrag_load_vec_t::dump(Formatter *f) const
   f->close_section();
 }
 
+void dirfrag_load_vec_t::dump(Formatter *f, utime_t now, const DecayRate& rate)
+{
+  f->dump_float("meta_load", meta_load(now, rate));
+  f->dump_float("IRD", get(META_POP_IRD).get(now, rate));
+  f->dump_float("IWR", get(META_POP_IWR).get(now, rate));
+  f->dump_float("READDIR", get(META_POP_READDIR).get(now, rate));
+  f->dump_float("FETCH", get(META_POP_FETCH).get(now, rate));
+  f->dump_float("STORE", get(META_POP_STORE).get(now, rate));
+}
+
 void dirfrag_load_vec_t::generate_test_instances(list<dirfrag_load_vec_t*>& ls)
 {
   utime_t sample;

diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h
index 59f53520e0a657515473a6c0c0e55f2a00761c05..3dbbd18092e55167deb399aef84c7f767f80ef13 100644 (file)
--- a/src/mds/mdstypes.h
+++ b/src/mds/mdstypes.h
@@ -1097,6 +1097,7 @@ public:
     decode(sample, p);
   }
   void dump(Formatter *f) const;
+  void dump(Formatter *f, utime_t now, const DecayRate& rate);
   static void generate_test_instances(list<dirfrag_load_vec_t*>& ls);
 
   DecayCounter &get(int t) {