From 8b164903bebd2248816ed6fcb83913fd99d9093c Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Wed, 31 May 2017 18:36:47 -0700 Subject: [PATCH] mon: mgr: pgstats: do available-space calculations on mgr instead of mon This means we don't need the osd_stat map on the monitor at all. We're about to remove it entirely! Signed-off-by: Greg Farnum --- src/mgr/DaemonServer.cc | 7 ++++--- src/mon/PGMap.cc | 32 ++++++++++++++++++++++++++------ src/mon/PGMap.h | 16 ++++++++++++++-- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/mgr/DaemonServer.cc b/src/mgr/DaemonServer.cc index d061b20475366..3be8dd87b6fd3 100644 --- a/src/mgr/DaemonServer.cc +++ b/src/mgr/DaemonServer.cc @@ -821,11 +821,12 @@ void DaemonServer::send_report() cluster_state.with_pgmap([&](const PGMap& pg_map) { cluster_state.update_delta_stats(); - // FIXME: no easy way to get mon features here. this will do for - // now, though, as long as we don't make a backward-incompat change. - pg_map.encode_digest(m->get_data(), CEPH_FEATURES_ALL); // FIXME: reporting health detail here might be a bad idea? cluster_state.with_osdmap([&](const OSDMap& osdmap) { + // FIXME: no easy way to get mon features here. this will do for + // now, though, as long as we don't make a backward-incompat change. + pg_map.encode_digest(osdmap, m->get_data(), CEPH_FEATURES_ALL); + pg_map.get_health(g_ceph_context, osdmap, m->health_summary, &m->health_detail); diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index b10da306fb8e5..b9d769e669d70 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -25,7 +25,7 @@ MEMPOOL_DEFINE_OBJECT_FACTORY(PGMap::Incremental, pgmap_inc, pgmap); void PGMapDigest::encode(bufferlist& bl, uint64_t features) const { // NOTE: see PGMap::encode_digest - ENCODE_START(5, 1, bl); + ENCODE_START(6, 1, bl); ::encode(num_pg, bl); ::encode(num_pg_active, bl); ::encode(num_osd, bl); @@ -42,12 +42,13 @@ void PGMapDigest::encode(bufferlist& bl, uint64_t features) const ::encode(pg_sum_delta, bl, features); ::encode(stamp_delta, bl); ::encode(num_pg_unknown, bl); + ::encode(avail_space_by_rule, bl); ENCODE_FINISH(bl); } void PGMapDigest::decode(bufferlist::iterator& p) { - DECODE_START(5, p); + DECODE_START(6, p); ::decode(num_pg, p); ::decode(num_pg_active, p); ::decode(num_osd, p); @@ -64,6 +65,7 @@ void PGMapDigest::decode(bufferlist::iterator& p) ::decode(pg_sum_delta, p); ::decode(stamp_delta, p); ::decode(num_pg_unknown, p); + ::decode(avail_space_by_rule, p); DECODE_FINISH(p); } @@ -655,7 +657,8 @@ void PGMapDigest::dump_pool_stats_full( int64_t avail; float raw_used_rate; if (avail_by_rule.count(ruleno) == 0) { - avail = get_rule_avail(osd_map, ruleno); + // FIXME: we don't guarantee avail_space_by_rule is up-to-date before this function is invoked + avail = get_rule_avail(ruleno); if (avail < 0) avail = 0; avail_by_rule[ruleno] = avail; @@ -813,7 +816,7 @@ void PGMapDigest::dump_object_stat_sum( } } -int64_t PGMapDigest::get_rule_avail(const OSDMap& osdmap, int ruleno) const +int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const { map wm; int r = osdmap.crush->get_rule_weight_osd_map(ruleno, &wm); @@ -859,7 +862,22 @@ int64_t PGMapDigest::get_rule_avail(const OSDMap& osdmap, int ruleno) const return min; } - +void PGMap::get_rules_avail(const OSDMap& osdmap, + std::map *avail_map) const +{ + avail_map->clear(); + for (auto p : osdmap.get_pools()) { + int64_t pool_id = p.first; + if ((pool_id < 0) || (pg_pool_sum.count(pool_id) == 0)) + continue; + const pg_pool_t *pool = osdmap.get_pg_pool(pool_id); + int ruleno = osdmap.crush->find_rule(pool->get_crush_ruleset(), + pool->get_type(), + pool->get_size()); + if (avail_map->count(ruleno) == 0) + (*avail_map)[ruleno] = get_rule_avail(osdmap, ruleno); + } +} // --------------------- // PGMap @@ -1477,8 +1495,10 @@ epoch_t PGMap::calc_min_last_epoch_clean() const return min; } -void PGMap::encode_digest(bufferlist& bl, uint64_t features) const +void PGMap::encode_digest(const OSDMap& osdmap, + bufferlist& bl, uint64_t features) const { + get_rules_avail(osdmap, &avail_space_by_rule); PGMapDigest::encode(bl, features); } diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index 8359785b901f8..30417733b604f 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -41,6 +41,8 @@ public: mempool::pgmap::vector osd_last_seq; mempool::pgmap::unordered_map osd_stat; + mutable std::map avail_space_by_rule; + // aggregate state, populated by PGMap child int64_t num_pg = 0, num_osd = 0; int64_t num_pg_active = 0; @@ -177,7 +179,13 @@ public: return statfs; } - int64_t get_rule_avail(const OSDMap& osdmap, int ruleno) const; + int64_t get_rule_avail(int ruleno) const { + auto i = avail_space_by_rule.find(ruleno); + if (i != avail_space_by_rule.end()) + return avail_space_by_rule[ruleno]; + else + return 0; + } // kill me post-luminous: virtual float get_fallback_full_ratio() const { @@ -415,10 +423,14 @@ public: void decode(bufferlist::iterator &bl); /// encode subset of our data to a PGMapDigest - void encode_digest(bufferlist& bl, uint64_t features) const; + void encode_digest(const OSDMap& osdmap, + bufferlist& bl, uint64_t features) const; void dirty_all(Incremental& inc); + int64_t get_rule_avail(const OSDMap& osdmap, int ruleno) const; + void get_rules_avail(const OSDMap& osdmap, + std::map *avail_map) const; void dump(Formatter *f) const; void dump_basic(Formatter *f) const; void dump_pg_stats(Formatter *f, bool brief) const; -- 2.39.5