From: Yan, Zheng Date: Mon, 13 Nov 2017 05:37:55 +0000 (+0800) Subject: mds: add asok command that dumps metadata popularity X-Git-Tag: v12.2.6~41^2~18 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5f868f0c95c291f8a3331faa52eaadf6afb9f3d2;p=ceph.git mds: add asok command that dumps metadata popularity Signed-off-by: "Yan, Zheng" (cherry picked from commit c078591ad1a33447f4de731c846844f4a0433513) --- diff --git a/src/mds/CDir.cc b/src/mds/CDir.cc index 1157e07dfe1e..dd19ac02cec1 100644 --- a/src/mds/CDir.cc +++ b/src/mds/CDir.cc @@ -3057,6 +3057,28 @@ void CDir::dump(Formatter *f) const MDSCacheObject::dump(f); } +void CDir::dump_load(Formatter *f, utime_t now, const DecayRate& rate) +{ + f->dump_stream("path") << get_path(); + f->dump_stream("dirfrag") << dirfrag(); + + f->open_object_section("pop_me"); + pop_me.dump(f, now, rate); + f->close_section(); + + f->open_object_section("pop_nested"); + pop_nested.dump(f, now, rate); + f->close_section(); + + f->open_object_section("pop_auth_subtree"); + pop_auth_subtree.dump(f, now, rate); + f->close_section(); + + f->open_object_section("pop_auth_subtree_nested"); + pop_auth_subtree_nested.dump(f, now, rate); + f->close_section(); +} + /****** Scrub Stuff *******/ void CDir::scrub_info_create() const diff --git a/src/mds/CDir.h b/src/mds/CDir.h index 8de1867329df..b8ad1b13311a 100644 --- a/src/mds/CDir.h +++ b/src/mds/CDir.h @@ -747,6 +747,7 @@ public: ostream& print_db_line_prefix(ostream& out) override; void print(ostream& out) override; void dump(Formatter *f) const; + void dump_load(Formatter *f, utime_t now, const DecayRate& rate); }; #endif diff --git a/src/mds/MDBalancer.cc b/src/mds/MDBalancer.cc index d8886ce96bd3..ba117ca58d4b 100644 --- a/src/mds/MDBalancer.cc +++ b/src/mds/MDBalancer.cc @@ -615,6 +615,8 @@ void MDBalancer::prep_rebalance(int beat) << dendl; } + mds_meta_load.clear(); + double total_load = 0.0; multimap load_map; for (mds_rank_t i=mds_rank_t(0); i < mds_rank_t(cluster_size); i++) { @@ -1241,3 +1243,101 @@ void MDBalancer::handle_mds_failure(mds_rank_t who) last_epoch_under = 0; } } + +int MDBalancer::dump_loads(Formatter *f) +{ + utime_t now = ceph_clock_now(); + DecayRate& decayrate = mds->mdcache->decayrate; + + list dfs; + if (mds->mdcache->get_root()) { + mds->mdcache->get_root()->get_dirfrags(dfs); + } else { + dout(5) << "dump_load no root" << dendl; + } + + f->open_object_section("loads"); + + f->open_array_section("dirfrags"); + while (!dfs.empty()) { + CDir *dir = dfs.front(); + dfs.pop_front(); + + if (f) { + f->open_object_section("dir"); + dir->dump_load(f, now, decayrate); + f->close_section(); + } + + for (auto it = dir->begin(); it != dir->end(); ++it) { + CInode *in = it->second->get_linkage()->get_inode(); + if (!in || !in->is_dir()) + continue; + + list ls; + in->get_dirfrags(ls); + for (auto subdir : ls) { + if (subdir->pop_nested.meta_load() < .001) + continue; + dfs.push_back(subdir); + } + } + } + f->close_section(); // dirfrags array + + f->open_object_section("mds_load"); + { + + auto dump_mds_load = [f, now](mds_load_t& load) { + f->dump_float("request_rate", load.req_rate); + f->dump_float("cache_hit_rate", load.cache_hit_rate); + f->dump_float("queue_length", load.queue_len); + f->dump_float("cpu_load", load.cpu_load_avg); + f->dump_float("mds_load", load.mds_load()); + + DecayRate rate; // no decay + f->open_object_section("auth_dirfrags"); + load.auth.dump(f, now, rate); + f->close_section(); + f->open_object_section("all_dirfrags"); + load.all.dump(f, now, rate); + f->close_section(); + }; + + for (auto p : mds_load) { + stringstream name; + name << "mds." << p.first; + f->open_object_section(name.str().c_str()); + dump_mds_load(p.second); + f->close_section(); + } + } + f->close_section(); // mds_load + + f->open_object_section("mds_meta_load"); + for (auto p : mds_meta_load) { + stringstream name; + name << "mds." << p.first; + f->dump_float(name.str().c_str(), p.second); + } + f->close_section(); // mds_meta_load + + f->open_object_section("mds_import_map"); + for (auto p : mds_import_map) { + stringstream name1; + name1 << "mds." << p.first; + f->open_array_section(name1.str().c_str()); + for (auto q : p.second) { + f->open_object_section("from"); + stringstream name2; + name2 << "mds." << q.first; + f->dump_float(name2.str().c_str(), q.second); + f->close_section(); + } + f->close_section(); // mds.? array + } + f->close_section(); // mds_import_map + + f->close_section(); // loads + return 0; +} diff --git a/src/mds/MDBalancer.h b/src/mds/MDBalancer.h index d23185b22f89..976892654b48 100644 --- a/src/mds/MDBalancer.h +++ b/src/mds/MDBalancer.h @@ -75,6 +75,8 @@ public: void handle_mds_failure(mds_rank_t who); + int dump_loads(Formatter *f); + private: typedef struct { std::map targets; diff --git a/src/mds/MDSDaemon.cc b/src/mds/MDSDaemon.cc index d1a22df71ab9..7841cc6dbed2 100644 --- a/src/mds/MDSDaemon.cc +++ b/src/mds/MDSDaemon.cc @@ -258,6 +258,11 @@ void MDSDaemon::set_up_admin_socket() asok_hook, "dump metadata cache for subtree"); assert(r == 0); + r = admin_socket->register_command("dump loads", + "dump loads", + asok_hook, + "dump metadata loads"); + assert(r == 0); r = admin_socket->register_command("session evict", "session evict name=client_id,type=CephString", asok_hook, @@ -327,6 +332,7 @@ void MDSDaemon::clean_up_admin_socket() admin_socket->unregister_command("dump cache"); admin_socket->unregister_command("cache status"); admin_socket->unregister_command("dump tree"); + admin_socket->unregister_command("dump loads"); admin_socket->unregister_command("session evict"); admin_socket->unregister_command("osdmap barrier"); admin_socket->unregister_command("session ls"); diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index de0a7c3e60af..65dd5efb430d 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -1991,6 +1991,13 @@ bool MDSRankDispatcher::handle_asok_command( f->reset(); } } + } else if (command == "dump loads") { + Mutex::Locker l(mds_lock); + int r = balancer->dump_loads(f); + if (r != 0) { + ss << "Failed to dump loads: " << cpp_strerror(r); + f->reset(); + } } else if (command == "force_readonly") { Mutex::Locker l(mds_lock); mdcache->force_readonly(); diff --git a/src/mds/mdstypes.cc b/src/mds/mdstypes.cc index 84fff861f444..8ca1ced1bdf5 100644 --- a/src/mds/mdstypes.cc +++ b/src/mds/mdstypes.cc @@ -639,6 +639,16 @@ void dirfrag_load_vec_t::dump(Formatter *f) const f->close_section(); } +void dirfrag_load_vec_t::dump(Formatter *f, utime_t now, const DecayRate& rate) +{ + f->dump_float("meta_load", meta_load(now, rate)); + f->dump_float("IRD", get(META_POP_IRD).get(now, rate)); + f->dump_float("IWR", get(META_POP_IWR).get(now, rate)); + f->dump_float("READDIR", get(META_POP_READDIR).get(now, rate)); + f->dump_float("FETCH", get(META_POP_FETCH).get(now, rate)); + f->dump_float("STORE", get(META_POP_STORE).get(now, rate)); +} + void dirfrag_load_vec_t::generate_test_instances(list& ls) { utime_t sample; diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h index 0ede619d5433..eaaa36caf349 100644 --- a/src/mds/mdstypes.h +++ b/src/mds/mdstypes.h @@ -1466,6 +1466,7 @@ public: decode(sample, p); } void dump(Formatter *f) const; + void dump(Formatter *f, utime_t now, const DecayRate& rate); static void generate_test_instances(list& ls); DecayCounter &get(int t) {