From 3f846d7c806b7f62ead08f0e9fb2ba927ffe0592 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Mon, 15 Jul 2019 13:23:53 -0700 Subject: [PATCH] osd mgr: Store last pingtime for possible graphing Signed-off-by: David Zafman --- src/mgr/ClusterState.cc | 8 ++++++-- src/osd/OSD.cc | 27 ++++++++++++++++++++------- src/osd/osd_types.cc | 12 +++++++++--- src/osd/osd_types.h | 2 ++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc index e5279321b38..af87c982084 100644 --- a/src/mgr/ClusterState.cc +++ b/src/mgr/ClusterState.cc @@ -240,6 +240,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& std::array times; std::array min; std::array max; + uint32_t last; bool operator<(const mgr_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -276,14 +277,15 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.max[0] = j.second.back_max[0]; item.max[1] = j.second.back_max[1]; item.max[2] = j.second.back_max[2]; + item.last = j.second.back_last; item.back = true; sorted.emplace(item); } + if (j.second.front_last == 0) + continue; item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]); item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]); - if (item.pingtime == 0) - continue; if (!value || item.pingtime >= value) { item.from = i.first; item.to = j.first; @@ -296,6 +298,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.max[0] = j.second.front_max[0]; item.max[1] = j.second.front_max[1]; item.max[2] = j.second.front_max[2]; + item.last = j.second.front_last; item.back = false; sorted.emplace(item); } @@ -328,6 +331,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& f->dump_unsigned("5min", sitem.max[1]); f->dump_unsigned("15min", sitem.max[2]); f->close_section(); // max + f->dump_unsigned("last", sitem.last); f->close_section(); // entry } f->close_section(); // entries diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 2ecdb71b1f1..18c13bfc4c7 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -2568,6 +2568,7 @@ will start to track new ops received afterwards."; std::array times; std::array min; std::array max; + uint32_t last; bool operator<(const osd_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -2601,10 +2602,11 @@ will start to track new ops received afterwards."; item.max[0] = j.second.back_max[0]; item.max[1] = j.second.back_max[1]; item.max[2] = j.second.back_max[2]; + item.last = j.second.back_last; item.back = true; sorted.emplace(item); } - if (j.second.front_pingtime[0] == 0) + if (j.second.front_last == 0) continue; item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]); item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]); @@ -2619,6 +2621,7 @@ will start to track new ops received afterwards."; item.max[0] = j.second.front_max[0]; item.max[1] = j.second.front_max[1]; item.max[2] = j.second.front_max[2]; + item.last = j.second.front_last; item.back = false; sorted.emplace(item); } @@ -2650,6 +2653,7 @@ will start to track new ops received afterwards."; f->dump_int("5min", sitem.max[1]); f->dump_int("15min", sitem.max[2]); f->close_section(); // max + f->dump_int("last", sitem.last); f->close_section(); // entry } f->close_section(); // entries @@ -4829,10 +4833,10 @@ void OSD::handle_osd_ping(MOSDPing *m) if (i->second.hb_interval_start == utime_t()) i->second.hb_interval_start = now; if (now - i->second.hb_interval_start >= utime_t(hb_avg, 0)) { - uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count; + uint32_t back_avg = i->second.hb_total_back / i->second.hb_average_count; uint32_t back_min = i->second.hb_min_back; uint32_t back_max = i->second.hb_max_back; - uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count; + uint32_t front_avg = i->second.hb_total_front / i->second.hb_average_count; uint32_t front_min = i->second.hb_min_front; uint32_t front_max = i->second.hb_max_front; @@ -4848,18 +4852,18 @@ void OSD::handle_osd_ping(MOSDPing *m) // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval if (i->second.hb_back_pingtime.size() < hb_vector_size) { ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size()); - i->second.hb_back_pingtime.push_back(back_pingtime); + i->second.hb_back_pingtime.push_back(back_avg); i->second.hb_back_min.push_back(back_min); i->second.hb_back_max.push_back(back_max); - i->second.hb_front_pingtime.push_back(front_pingtime); + i->second.hb_front_pingtime.push_back(front_avg); i->second.hb_front_min.push_back(front_min); i->second.hb_front_max.push_back(front_max); } else { int index = i->second.hb_index & (hb_vector_size - 1); - i->second.hb_back_pingtime[index] = back_pingtime; + i->second.hb_back_pingtime[index] = back_avg; i->second.hb_back_min[index] = back_min; i->second.hb_back_max[index] = back_max; - i->second.hb_front_pingtime[index] = front_pingtime; + i->second.hb_front_pingtime[index] = front_avg; i->second.hb_front_min[index] = front_min; i->second.hb_front_max[index] = front_max; } @@ -4867,6 +4871,8 @@ void OSD::handle_osd_ping(MOSDPing *m) { std::lock_guard l(service.stat_lock); + service.osd_stat.hb_pingtime[from].back_last = back_pingtime; + uint32_t total = 0; uint32_t min = UINT_MAX; uint32_t max = 0; @@ -4892,6 +4898,8 @@ void OSD::handle_osd_ping(MOSDPing *m) } if (i->second.con_front != NULL) { + service.osd_stat.hb_pingtime[from].front_last = front_pingtime; + total = 0; min = UINT_MAX; max = 0; @@ -4916,6 +4924,11 @@ void OSD::handle_osd_ping(MOSDPing *m) } } } + } else { + std::lock_guard l(service.stat_lock); + service.osd_stat.hb_pingtime[from].back_last = back_pingtime; + if (i->second.con_front != NULL) + service.osd_stat.hb_pingtime[from].front_last = front_pingtime; } i->second.ping_history.erase(i->second.ping_history.begin(), ++acked); } diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 0cb1f24fc2f..21645fd854e 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -450,6 +450,7 @@ void osd_stat_t::dump(Formatter *f) const f->dump_int("5min", i.second.back_max[1]); f->dump_int("15min", i.second.back_max[2]); f->close_section(); // max + f->dump_int("last", i.second.back_last); f->close_section(); // interface if (i.second.front_pingtime[0] != 0) { @@ -470,6 +471,7 @@ void osd_stat_t::dump(Formatter *f) const f->dump_int("5min", i.second.front_max[1]); f->dump_int("15min", i.second.front_max[2]); f->close_section(); // max + f->dump_int("last", i.second.front_last); f->close_section(); // interface } f->close_section(); // interfaces @@ -529,6 +531,7 @@ void osd_stat_t::encode(ceph::buffer::list &bl, uint64_t features) const encode(i.second.back_max[0], bl); encode(i.second.back_max[1], bl); encode(i.second.back_max[2], bl); + encode(i.second.back_last, bl); encode(i.second.front_pingtime[0], bl); encode(i.second.front_pingtime[1], bl); encode(i.second.front_pingtime[2], bl); @@ -538,6 +541,7 @@ void osd_stat_t::encode(ceph::buffer::list &bl, uint64_t features) const encode(i.second.front_max[0], bl); encode(i.second.front_max[1], bl); encode(i.second.front_max[2], bl); + encode(i.second.front_last, bl); } ENCODE_FINISH(bl); } @@ -635,6 +639,7 @@ void osd_stat_t::decode(ceph::buffer::list::const_iterator &bl) decode(ifs.back_max[0],bl); decode(ifs.back_max[1], bl); decode(ifs.back_max[2], bl); + decode(ifs.back_last, bl); decode(ifs.front_pingtime[0], bl); decode(ifs.front_pingtime[1], bl); decode(ifs.front_pingtime[2], bl); @@ -644,6 +649,7 @@ void osd_stat_t::decode(ceph::buffer::list::const_iterator &bl) decode(ifs.front_max[0], bl); decode(ifs.front_max[1], bl); decode(ifs.front_max[2], bl); + decode(ifs.front_last, bl); hb_pingtime[osd] = ifs; } } @@ -667,11 +673,11 @@ void osd_stat_t::generate_test_instances(std::list& o) o.back()->os_alerts[1].emplace( "some alert2", "some alert2 details"); struct Interfaces gen_interfaces = { - { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, - { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } }; + { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, 1001, + { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 }, 1101 }; o.back()->hb_pingtime[20] = gen_interfaces; gen_interfaces = { - { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } }; + { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 }, 101 }; o.back()->hb_pingtime[30] = gen_interfaces; } diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 42d179de020..56d0dbead58 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2376,9 +2376,11 @@ struct osd_stat_t { uint32_t back_pingtime[3]; uint32_t back_min[3]; uint32_t back_max[3]; + uint32_t back_last; uint32_t front_pingtime[3]; uint32_t front_min[3]; uint32_t front_max[3]; + uint32_t front_last; }; map hb_pingtime; ///< map of osd id to Interfaces -- 2.39.5