From 306fd2500ed4f96674f197405155b2e711be1759 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Fri, 12 Jul 2019 01:06:23 +0000 Subject: [PATCH] osd mgr: Add minimum and maximum tracking to network ping time Signed-off-by: David Zafman (cherry picked from commit 297a0e7b1de410c094fc9a6e42be14813d6dac5e) Conflicts: src/osd/osd_types.cc (trivial) src/osd/osd_types.h (osd_stat_t location in file changed) --- src/mgr/ClusterState.cc | 26 +++++++++++++ src/osd/OSD.cc | 85 +++++++++++++++++++++++++++++++++++++---- src/osd/OSD.h | 8 ++++ src/osd/osd_types.cc | 69 ++++++++++++++++++++++++++++----- src/osd/osd_types.h | 4 ++ 5 files changed, 174 insertions(+), 18 deletions(-) diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc index 6d5fbe83ff80f..01e83a069867b 100644 --- a/src/mgr/ClusterState.cc +++ b/src/mgr/ClusterState.cc @@ -228,6 +228,8 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& int to; bool back; std::array times; + std::array min; + std::array max; bool operator<(const mgr_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -258,6 +260,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.times[0] = j.second.back_pingtime[0]; item.times[1] = j.second.back_pingtime[1]; item.times[2] = j.second.back_pingtime[2]; + item.min[0] = j.second.back_min[0]; + item.min[1] = j.second.back_min[1]; + item.min[2] = j.second.back_min[2]; + item.max[0] = j.second.back_max[0]; + item.max[1] = j.second.back_max[1]; + item.max[2] = j.second.back_max[2]; item.back = true; sorted.emplace(item); } @@ -272,6 +280,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& item.times[0] = j.second.front_pingtime[0]; item.times[1] = j.second.front_pingtime[1]; item.times[2] = j.second.front_pingtime[2]; + item.min[0] = j.second.front_min[0]; + item.min[1] = j.second.front_min[1]; + item.min[2] = j.second.front_min[2]; + item.max[0] = j.second.front_max[0]; + item.max[1] = j.second.front_max[1]; + item.max[2] = j.second.front_max[2]; item.back = false; sorted.emplace(item); } @@ -289,9 +303,21 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t& f->dump_int("from osd", sitem.from); f->dump_int("to osd", sitem.to); f->dump_string("interface", (sitem.back ? "back" : "front")); + f->open_object_section("average"); f->dump_unsigned("1min", sitem.times[0]); f->dump_unsigned("5min", sitem.times[1]); f->dump_unsigned("15min", sitem.times[2]); + f->close_section(); // average + f->open_object_section("min"); + f->dump_unsigned("1min", sitem.min[0]); + f->dump_unsigned("5min", sitem.min[1]); + f->dump_unsigned("15min", sitem.min[2]); + f->close_section(); // min + f->open_object_section("max"); + f->dump_unsigned("1min", sitem.max[0]); + f->dump_unsigned("5min", sitem.max[1]); + f->dump_unsigned("15min", sitem.max[2]); + f->close_section(); // max f->close_section(); // entry } f->close_section(); // entries diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index fbb730b558b6d..22f5f286f6765 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -2299,6 +2299,8 @@ will start to track new ops received afterwards."; int to; bool back; std::array times; + std::array min; + std::array max; bool operator<(const osd_ping_time_t& rhs) const { if (pingtime < rhs.pingtime) @@ -2326,6 +2328,12 @@ will start to track new ops received afterwards."; item.times[0] = j.second.back_pingtime[0]; item.times[1] = j.second.back_pingtime[1]; item.times[2] = j.second.back_pingtime[2]; + item.min[0] = j.second.back_min[0]; + item.min[1] = j.second.back_min[1]; + item.min[2] = j.second.back_min[2]; + item.max[0] = j.second.back_max[0]; + item.max[1] = j.second.back_max[1]; + item.max[2] = j.second.back_max[2]; item.back = true; sorted.emplace(item); } @@ -2338,6 +2346,12 @@ will start to track new ops received afterwards."; item.times[0] = j.second.front_pingtime[0]; item.times[1] = j.second.front_pingtime[1]; item.times[2] = j.second.front_pingtime[2]; + item.min[0] = j.second.front_min[0]; + item.min[1] = j.second.front_min[1]; + item.min[2] = j.second.front_min[2]; + item.max[0] = j.second.front_max[0]; + item.max[1] = j.second.front_max[1]; + item.max[2] = j.second.front_max[2]; item.back = false; sorted.emplace(item); } @@ -2354,9 +2368,21 @@ will start to track new ops received afterwards."; f->dump_int("from osd", whoami); f->dump_int("to osd", sitem.to); f->dump_string("interface", (sitem.back ? "back" : "front")); + f->open_object_section("average"); f->dump_int("1min", sitem.times[0]); f->dump_int("5min", sitem.times[1]); f->dump_int("15min", sitem.times[2]); + f->close_section(); // average + f->open_object_section("min"); + f->dump_int("1min", sitem.min[0]); + f->dump_int("5min", sitem.min[1]); + f->dump_int("15min", sitem.min[2]); + f->close_section(); // min + f->open_object_section("max"); + f->dump_int("1min", sitem.max[0]); + f->dump_int("5min", sitem.max[1]); + f->dump_int("15min", sitem.max[2]); + f->close_section(); // max f->close_section(); // entry } f->close_section(); // entries @@ -4741,45 +4767,78 @@ void OSD::handle_osd_ping(MOSDPing *m) ++i->second.hb_average_count; uint32_t back_pingtime = ROUND_S_TO_USEC(i->second.last_rx_back - m->stamp); i->second.hb_total_back += back_pingtime; + if (back_pingtime < i->second.hb_min_back) + i->second.hb_min_back = back_pingtime; + if (back_pingtime > i->second.hb_max_back) + i->second.hb_max_back = back_pingtime; uint32_t front_pingtime = ROUND_S_TO_USEC(i->second.last_rx_front - m->stamp); i->second.hb_total_front += front_pingtime; + if (front_pingtime < i->second.hb_min_front) + i->second.hb_min_front = front_pingtime; + if (front_pingtime > i->second.hb_max_front) + i->second.hb_max_front = front_pingtime; ceph_assert(i->second.hb_interval_start != utime_t()); if (i->second.hb_interval_start == utime_t()) i->second.hb_interval_start = now; if (now - i->second.hb_interval_start >= utime_t(hb_avg, 0)) { uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count; + uint32_t back_min = i->second.hb_min_back; + uint32_t back_max = i->second.hb_max_back; uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count; + uint32_t front_min = i->second.hb_min_front; + uint32_t front_max = i->second.hb_max_front; // Reset for new interval i->second.hb_average_count = 0; i->second.hb_interval_start = now; - i->second.hb_total_back = 0; - i->second.hb_total_front = 0; + i->second.hb_total_back = i->second.hb_max_back = 0; + i->second.hb_min_back = UINT_MAX; + i->second.hb_total_front = i->second.hb_max_front = 0; + i->second.hb_min_front = UINT_MAX; // Record per osd interace ping times // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval if (i->second.hb_back_pingtime.size() < hb_vector_size) { ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size()); i->second.hb_back_pingtime.push_back(back_pingtime); + i->second.hb_back_min.push_back(back_min); + i->second.hb_back_max.push_back(back_max); i->second.hb_front_pingtime.push_back(front_pingtime); + i->second.hb_front_min.push_back(front_min); + i->second.hb_front_max.push_back(front_max); } else { - i->second.hb_back_pingtime[i->second.hb_index & (hb_vector_size - 1)] = back_pingtime; - i->second.hb_front_pingtime[i->second.hb_index & (hb_vector_size - 1)] = front_pingtime; + int index = i->second.hb_index & (hb_vector_size - 1); + i->second.hb_back_pingtime[index] = back_pingtime; + i->second.hb_back_min[index] = back_min; + i->second.hb_back_max[index] = back_max; + i->second.hb_front_pingtime[index] = front_pingtime; + i->second.hb_front_min[index] = front_min; + i->second.hb_front_max[index] = front_max; } ++i->second.hb_index; { std::lock_guard l(service.stat_lock); uint32_t total = 0; + uint32_t min = UINT_MAX; + uint32_t max = 0; uint32_t count = 0; uint32_t which = 0; uint32_t size = (uint32_t)i->second.hb_back_pingtime.size(); for (int32_t k = size - 1 ; k >= 0; --k) { ++count; - total += i->second.hb_back_pingtime[(i->second.hb_index + k) % size]; + int index = (i->second.hb_index + k) % size; + total += i->second.hb_back_pingtime[index]; + if (i->second.hb_back_min[index] < min) + min = i->second.hb_back_min[index]; + if (i->second.hb_back_max[index] > max) + max = i->second.hb_back_max[index]; if (count == 1 || count == 5 || count == 15) { - service.osd_stat.hb_pingtime[from].back_pingtime[which++] = total / count; + service.osd_stat.hb_pingtime[from].back_pingtime[which] = total / count; + service.osd_stat.hb_pingtime[from].back_min[which] = min; + service.osd_stat.hb_pingtime[from].back_max[which] = max; + which++; if (count == 15) break; } @@ -4787,13 +4846,23 @@ void OSD::handle_osd_ping(MOSDPing *m) if (i->second.con_front != NULL) { total = 0; + min = UINT_MAX; + max = 0; count = 0; which = 0; for (int32_t k = size - 1 ; k >= 0; --k) { ++count; - total += i->second.hb_front_pingtime[(i->second.hb_index + k) % size]; + int index = (i->second.hb_index + k) % size; + total += i->second.hb_front_pingtime[index]; + if (i->second.hb_front_min[index] < min) + min = i->second.hb_front_min[index]; + if (i->second.hb_front_max[index] > max) + max = i->second.hb_front_max[index]; if (count == 1 || count == 5 || count == 15) { - service.osd_stat.hb_pingtime[from].front_pingtime[which++] = total / count; + service.osd_stat.hb_pingtime[from].front_pingtime[which] = total / count; + service.osd_stat.hb_pingtime[from].front_min[which] = min; + service.osd_stat.hb_pingtime[from].front_max[which] = max; + which++; if (count == 15) break; } diff --git a/src/osd/OSD.h b/src/osd/OSD.h index aaf9b2645ec7b..900c9cdd0efed 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1529,10 +1529,18 @@ private: uint32_t hb_index = 0; uint32_t hb_total_back = 0; + uint32_t hb_min_back = UINT_MAX; + uint32_t hb_max_back = 0; vector hb_back_pingtime; + vector hb_back_min; + vector hb_back_max; uint32_t hb_total_front = 0; + uint32_t hb_min_front = UINT_MAX; + uint32_t hb_max_front = 0; vector hb_front_pingtime; + vector hb_front_min; + vector hb_front_max; bool is_unhealthy(utime_t now) { if (ping_history.empty()) { diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 92248a20afa8e..131374a201860 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -359,32 +359,54 @@ void osd_stat_t::dump(Formatter *f) const f->open_object_section("perf_stat"); os_perf_stat.dump(f); f->close_section(); - // Call to update ping times? f->open_array_section("network_ping_times"); for (auto &i : hb_pingtime) { - f->open_object_section("osd"); + f->open_object_section("entry"); f->dump_int("osd", i.first); - f->open_array_section("interfaces"); f->open_object_section("interface"); f->dump_string("interface", "back"); + f->open_object_section("average"); f->dump_int("1min", i.second.back_pingtime[0]); f->dump_int("5min", i.second.back_pingtime[1]); f->dump_int("15min", i.second.back_pingtime[2]); - f->close_section(); + f->close_section(); // average + f->open_object_section("min"); + f->dump_int("1min", i.second.back_min[0]); + f->dump_int("5min", i.second.back_min[1]); + f->dump_int("15min", i.second.back_min[2]); + f->close_section(); // min + f->open_object_section("max"); + f->dump_int("1min", i.second.back_max[0]); + f->dump_int("5min", i.second.back_max[1]); + f->dump_int("15min", i.second.back_max[2]); + f->close_section(); // max + f->close_section(); // interface if (i.second.front_pingtime[0] != 0) { f->open_object_section("interface"); f->dump_string("interface", "front"); + f->open_object_section("average"); f->dump_int("1min", i.second.front_pingtime[0]); f->dump_int("5min", i.second.front_pingtime[1]); f->dump_int("15min", i.second.front_pingtime[2]); - f->close_section(); + f->close_section(); // average + f->open_object_section("min"); + f->dump_int("1min", i.second.front_min[0]); + f->dump_int("5min", i.second.front_min[1]); + f->dump_int("15min", i.second.front_min[2]); + f->close_section(); // min + f->open_object_section("max"); + f->dump_int("1min", i.second.front_max[0]); + f->dump_int("5min", i.second.front_max[1]); + f->dump_int("15min", i.second.front_max[2]); + f->close_section(); // max + f->close_section(); // interface } - f->close_section(); - f->close_section(); + f->close_section(); // interfaces + f->close_section(); // entry } - f->close_section(); + f->close_section(); // network_ping_time } void osd_stat_t::encode(bufferlist &bl, uint64_t features) const @@ -412,9 +434,21 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const encode(i.second.back_pingtime[0], bl); encode(i.second.back_pingtime[1], bl); encode(i.second.back_pingtime[2], bl); + encode(i.second.back_min[0], bl); + encode(i.second.back_min[1], bl); + encode(i.second.back_min[2], bl); + encode(i.second.back_max[0], bl); + encode(i.second.back_max[1], bl); + encode(i.second.back_max[2], bl); encode(i.second.front_pingtime[0], bl); encode(i.second.front_pingtime[1], bl); encode(i.second.front_pingtime[2], bl); + encode(i.second.front_min[0], bl); + encode(i.second.front_min[1], bl); + encode(i.second.front_min[2], bl); + encode(i.second.front_max[0], bl); + encode(i.second.front_max[1], bl); + encode(i.second.front_max[2], bl); } ENCODE_FINISH(bl); } @@ -461,9 +495,21 @@ void osd_stat_t::decode(bufferlist::iterator &bl) decode(ifs.back_pingtime[0],bl); decode(ifs.back_pingtime[1], bl); decode(ifs.back_pingtime[2], bl); + decode(ifs.back_min[0],bl); + decode(ifs.back_min[1], bl); + decode(ifs.back_min[2], bl); + decode(ifs.back_max[0],bl); + decode(ifs.back_max[1], bl); + decode(ifs.back_max[2], bl); decode(ifs.front_pingtime[0], bl); decode(ifs.front_pingtime[1], bl); decode(ifs.front_pingtime[2], bl); + decode(ifs.front_min[0], bl); + decode(ifs.front_min[1], bl); + decode(ifs.front_min[2], bl); + decode(ifs.front_max[0], bl); + decode(ifs.front_max[1], bl); + decode(ifs.front_max[2], bl); hb_pingtime[osd] = ifs; } } @@ -484,9 +530,12 @@ void osd_stat_t::generate_test_instances(std::list& o) o.back()->hb_peers.push_back(7); o.back()->snap_trim_queue_len = 8; o.back()->num_snap_trimming = 99; - struct Interfaces gen_interfaces = { { 1000, 995, 990 }, {980, 985, 990} }; + struct Interfaces gen_interfaces = { + { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, + { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } }; o.back()->hb_pingtime[20] = gen_interfaces; - gen_interfaces = { { 700, 800, 900 } }; + gen_interfaces = { + { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } }; o.back()->hb_pingtime[30] = gen_interfaces; } diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index d5b287c00e58e..df4018b11c93f 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -935,7 +935,11 @@ struct osd_stat_t { struct Interfaces { uint32_t back_pingtime[3]; + uint32_t back_min[3]; + uint32_t back_max[3]; uint32_t front_pingtime[3]; + uint32_t front_min[3]; + uint32_t front_max[3]; }; map hb_pingtime; ///< map of osd id to Interfaces -- 2.39.5