From: David Zafman Date: Mon, 15 Jul 2019 20:23:53 +0000 (-0700) Subject: osd mgr: Store last pingtime for possible graphing X-Git-Tag: v13.2.7~26^2~15 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cb02e003bdfbf9eef301900c5f9b643eef5d74d9;p=ceph.git osd mgr: Store last pingtime for possible graphing Signed-off-by: David Zafman (cherry picked from commit 3f846d7c806b7f62ead08f0e9fb2ba927ffe0592) Conflicts: src/osd/osd_types.h (osd_stat_t location in file changed) --- diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc index 01e83a069867..79c71e9fb71a 100644 --- a/src/mgr/ClusterState.cc +++ b/src/mgr/ClusterState.cc @@ -230,6 +230,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& std::array times; std::array min; std::array max; + uint32_t last; bool operator<(const mgr_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -266,14 +267,15 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.max[0] = j.second.back_max[0]; item.max[1] = j.second.back_max[1]; item.max[2] = j.second.back_max[2]; + item.last = j.second.back_last; item.back = true; sorted.emplace(item); } + if (j.second.front_last == 0) + continue; item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]); item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]); - if (item.pingtime == 0) - continue; if (!value || item.pingtime >= value) { item.from = i.first; item.to = j.first; @@ -286,6 +288,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.max[0] = j.second.front_max[0]; item.max[1] = j.second.front_max[1]; item.max[2] = j.second.front_max[2]; + item.last = j.second.front_last; item.back = false; sorted.emplace(item); } @@ -318,6 +321,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& f->dump_unsigned("5min", sitem.max[1]); f->dump_unsigned("15min", sitem.max[2]); f->close_section(); // max + f->dump_unsigned("last", sitem.last); f->close_section(); // entry } f->close_section(); // entries diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 22f5f286f676..6023ec585307 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -2301,6 +2301,7 @@ will start to track new ops received afterwards."; std::array times; std::array min; std::array max; + uint32_t last; bool operator<(const osd_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -2334,10 +2335,11 @@ will start to track new ops received afterwards."; item.max[0] = j.second.back_max[0]; item.max[1] = j.second.back_max[1]; item.max[2] = j.second.back_max[2]; + item.last = j.second.back_last; item.back = true; sorted.emplace(item); } - if (j.second.front_pingtime[0] == 0) + if (j.second.front_last == 0) continue; item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]); item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]); @@ -2352,6 +2354,7 @@ will start to track new ops received afterwards."; item.max[0] = j.second.front_max[0]; item.max[1] = j.second.front_max[1]; item.max[2] = j.second.front_max[2]; + item.last = j.second.front_last; item.back = false; sorted.emplace(item); } @@ -2383,6 +2386,7 @@ will start to track new ops received afterwards."; f->dump_int("5min", sitem.max[1]); f->dump_int("15min", sitem.max[2]); f->close_section(); // max + f->dump_int("last", sitem.last); f->close_section(); // entry } f->close_section(); // entries @@ -4782,10 +4786,10 @@ void OSD::handle_osd_ping(MOSDPing *m) if (i->second.hb_interval_start == utime_t()) i->second.hb_interval_start = now; if (now - i->second.hb_interval_start >= utime_t(hb_avg, 0)) { - uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count; + uint32_t back_avg = i->second.hb_total_back / i->second.hb_average_count; uint32_t back_min = i->second.hb_min_back; uint32_t back_max = i->second.hb_max_back; - uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count; + uint32_t front_avg = i->second.hb_total_front / i->second.hb_average_count; uint32_t front_min = i->second.hb_min_front; uint32_t front_max = i->second.hb_max_front; @@ -4801,18 +4805,18 @@ void OSD::handle_osd_ping(MOSDPing *m) // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval if (i->second.hb_back_pingtime.size() < hb_vector_size) { ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size()); - i->second.hb_back_pingtime.push_back(back_pingtime); + i->second.hb_back_pingtime.push_back(back_avg); i->second.hb_back_min.push_back(back_min); i->second.hb_back_max.push_back(back_max); - i->second.hb_front_pingtime.push_back(front_pingtime); + i->second.hb_front_pingtime.push_back(front_avg); i->second.hb_front_min.push_back(front_min); i->second.hb_front_max.push_back(front_max); } else { int index = i->second.hb_index & (hb_vector_size - 1); - i->second.hb_back_pingtime[index] = back_pingtime; + i->second.hb_back_pingtime[index] = back_avg; i->second.hb_back_min[index] = back_min; i->second.hb_back_max[index] = back_max; - i->second.hb_front_pingtime[index] = front_pingtime; + i->second.hb_front_pingtime[index] = front_avg; i->second.hb_front_min[index] = front_min; i->second.hb_front_max[index] = front_max; } @@ -4820,6 +4824,8 @@ void OSD::handle_osd_ping(MOSDPing *m) { std::lock_guard l(service.stat_lock); + service.osd_stat.hb_pingtime[from].back_last = back_pingtime; + uint32_t total = 0; uint32_t min = UINT_MAX; uint32_t max = 0; @@ -4845,6 +4851,8 @@ void OSD::handle_osd_ping(MOSDPing *m) } if (i->second.con_front != NULL) { + service.osd_stat.hb_pingtime[from].front_last = front_pingtime; + total = 0; min = UINT_MAX; max = 0; @@ -4869,6 +4877,11 @@ void OSD::handle_osd_ping(MOSDPing *m) } } } + } else { + std::lock_guard l(service.stat_lock); + service.osd_stat.hb_pingtime[from].back_last = back_pingtime; + if (i->second.con_front != NULL) + service.osd_stat.hb_pingtime[from].front_last = front_pingtime; } i->second.ping_history.erase(i->second.ping_history.begin(), ++acked); } diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 131374a20186..092b5f759df9 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -381,6 +381,7 @@ void osd_stat_t::dump(Formatter *f) const f->dump_int("5min", i.second.back_max[1]); f->dump_int("15min", i.second.back_max[2]); f->close_section(); // max + f->dump_int("last", i.second.back_last); f->close_section(); // interface if (i.second.front_pingtime[0] != 0) { @@ -401,6 +402,7 @@ void osd_stat_t::dump(Formatter *f) const f->dump_int("5min", i.second.front_max[1]); f->dump_int("15min", i.second.front_max[2]); f->close_section(); // max + f->dump_int("last", i.second.front_last); f->close_section(); // interface } f->close_section(); // interfaces @@ -440,6 +442,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const encode(i.second.back_max[0], bl); encode(i.second.back_max[1], bl); encode(i.second.back_max[2], bl); + encode(i.second.back_last, bl); encode(i.second.front_pingtime[0], bl); encode(i.second.front_pingtime[1], bl); encode(i.second.front_pingtime[2], bl); @@ -449,6 +452,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const encode(i.second.front_max[0], bl); encode(i.second.front_max[1], bl); encode(i.second.front_max[2], bl); + encode(i.second.front_last, bl); } ENCODE_FINISH(bl); } @@ -501,6 +505,7 @@ void osd_stat_t::decode(bufferlist::iterator &bl) decode(ifs.back_max[0],bl); decode(ifs.back_max[1], bl); decode(ifs.back_max[2], bl); + decode(ifs.back_last, bl); decode(ifs.front_pingtime[0], bl); decode(ifs.front_pingtime[1], bl); decode(ifs.front_pingtime[2], bl); @@ -510,6 +515,7 @@ void osd_stat_t::decode(bufferlist::iterator &bl) decode(ifs.front_max[0], bl); decode(ifs.front_max[1], bl); decode(ifs.front_max[2], bl); + decode(ifs.front_last, bl); hb_pingtime[osd] = ifs; } } @@ -531,11 +537,11 @@ void osd_stat_t::generate_test_instances(std::list& o) o.back()->snap_trim_queue_len = 8; o.back()->num_snap_trimming = 99; struct Interfaces gen_interfaces = { - { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, - { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } }; + { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, 1001, + { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 }, 1101 }; o.back()->hb_pingtime[20] = gen_interfaces; gen_interfaces = { - { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } }; + { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 }, 101 }; o.back()->hb_pingtime[30] = gen_interfaces; } diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index df4018b11c93..6ffe6a0dcc65 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -937,9 +937,11 @@ struct osd_stat_t { uint32_t back_pingtime[3]; uint32_t back_min[3]; uint32_t back_max[3]; + uint32_t back_last; uint32_t front_pingtime[3]; uint32_t front_min[3]; uint32_t front_max[3]; + uint32_t front_last; }; map hb_pingtime; ///< map of osd id to Interfaces