osd mgr: Add minimum and maximum tracking to network ping time

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 297a0e7b1de410c094fc9a6e42be14813d6dac5e)

Conflicts:
	src/osd/osd_types.cc (trivial)
	src/osd/osd_types.h (osd_stat_t location in file changed)

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 6d5fbe83ff80fdfa79cfa5b90550922cacd8467c..01e83a069867b0afd671e9a5c01ba6f22bccba2c 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -228,6 +228,8 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       int to;
       bool back;
       std::array<uint32_t,3> times;
+      std::array<uint32_t,3> min;
+      std::array<uint32_t,3> max;
 
       bool operator<(const mgr_ping_time_t& rhs) const {
         if (pingtime < rhs.pingtime)
@@ -258,6 +260,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.times[0] = j.second.back_pingtime[0];
          item.times[1] = j.second.back_pingtime[1];
          item.times[2] = j.second.back_pingtime[2];
+         item.min[0] = j.second.back_min[0];
+         item.min[1] = j.second.back_min[1];
+         item.min[2] = j.second.back_min[2];
+         item.max[0] = j.second.back_max[0];
+         item.max[1] = j.second.back_max[1];
+         item.max[2] = j.second.back_max[2];
          item.back = true;
          sorted.emplace(item);
        }
@@ -272,6 +280,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.times[0] = j.second.front_pingtime[0];
          item.times[1] = j.second.front_pingtime[1];
          item.times[2] = j.second.front_pingtime[2];
+         item.min[0] = j.second.front_min[0];
+         item.min[1] = j.second.front_min[1];
+         item.min[2] = j.second.front_min[2];
+         item.max[0] = j.second.front_max[0];
+         item.max[1] = j.second.front_max[1];
+         item.max[2] = j.second.front_max[2];
          item.back = false;
          sorted.emplace(item);
        }
@@ -289,9 +303,21 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       f->dump_int("from osd", sitem.from);
       f->dump_int("to osd", sitem.to);
       f->dump_string("interface", (sitem.back ? "back" : "front"));
+      f->open_object_section("average");
       f->dump_unsigned("1min", sitem.times[0]);
       f->dump_unsigned("5min", sitem.times[1]);
       f->dump_unsigned("15min", sitem.times[2]);
+      f->close_section(); // average
+      f->open_object_section("min");
+      f->dump_unsigned("1min", sitem.min[0]);
+      f->dump_unsigned("5min", sitem.min[1]);
+      f->dump_unsigned("15min", sitem.min[2]);
+      f->close_section(); // min
+      f->open_object_section("max");
+      f->dump_unsigned("1min", sitem.max[0]);
+      f->dump_unsigned("5min", sitem.max[1]);
+      f->dump_unsigned("15min", sitem.max[2]);
+      f->close_section(); // max
       f->close_section(); // entry
     }
     f->close_section(); // entries

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index fbb730b558b6d760e89c0b4ab2f08957b09437ed..22f5f286f6765e3ac02140b7fb210756a5dc081f 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2299,6 +2299,8 @@ will start to track new ops received afterwards.";
       int to;
       bool back;
       std::array<uint32_t,3> times;
+      std::array<uint32_t,3> min;
+      std::array<uint32_t,3> max;
 
       bool operator<(const osd_ping_time_t& rhs) const {
        if (pingtime < rhs.pingtime)
@@ -2326,6 +2328,12 @@ will start to track new ops received afterwards.";
        item.times[0] = j.second.back_pingtime[0];
        item.times[1] = j.second.back_pingtime[1];
        item.times[2] = j.second.back_pingtime[2];
+       item.min[0] = j.second.back_min[0];
+       item.min[1] = j.second.back_min[1];
+       item.min[2] = j.second.back_min[2];
+       item.max[0] = j.second.back_max[0];
+       item.max[1] = j.second.back_max[1];
+       item.max[2] = j.second.back_max[2];
        item.back = true;
        sorted.emplace(item);
       }
@@ -2338,6 +2346,12 @@ will start to track new ops received afterwards.";
        item.times[0] = j.second.front_pingtime[0];
        item.times[1] = j.second.front_pingtime[1];
        item.times[2] = j.second.front_pingtime[2];
+       item.min[0] = j.second.front_min[0];
+       item.min[1] = j.second.front_min[1];
+       item.min[2] = j.second.front_min[2];
+       item.max[0] = j.second.front_max[0];
+       item.max[1] = j.second.front_max[1];
+       item.max[2] = j.second.front_max[2];
        item.back = false;
        sorted.emplace(item);
       }
@@ -2354,9 +2368,21 @@ will start to track new ops received afterwards.";
       f->dump_int("from osd", whoami);
       f->dump_int("to osd", sitem.to);
       f->dump_string("interface", (sitem.back ? "back" : "front"));
+      f->open_object_section("average");
       f->dump_int("1min", sitem.times[0]);
       f->dump_int("5min", sitem.times[1]);
       f->dump_int("15min", sitem.times[2]);
+      f->close_section();  // average
+      f->open_object_section("min");
+      f->dump_int("1min", sitem.min[0]);
+      f->dump_int("5min", sitem.min[1]);
+      f->dump_int("15min", sitem.min[2]);
+      f->close_section();  // min
+      f->open_object_section("max");
+      f->dump_int("1min", sitem.max[0]);
+      f->dump_int("5min", sitem.max[1]);
+      f->dump_int("15min", sitem.max[2]);
+      f->close_section();  // max
       f->close_section();  // entry
     }
     f->close_section(); // entries
@@ -4741,45 +4767,78 @@ void OSD::handle_osd_ping(MOSDPing *m)
            ++i->second.hb_average_count;
            uint32_t back_pingtime = ROUND_S_TO_USEC(i->second.last_rx_back - m->stamp);
            i->second.hb_total_back += back_pingtime;
+           if (back_pingtime < i->second.hb_min_back)
+             i->second.hb_min_back = back_pingtime;
+           if (back_pingtime > i->second.hb_max_back)
+             i->second.hb_max_back = back_pingtime;
            uint32_t front_pingtime = ROUND_S_TO_USEC(i->second.last_rx_front - m->stamp);
            i->second.hb_total_front += front_pingtime;
+           if (front_pingtime < i->second.hb_min_front)
+             i->second.hb_min_front = front_pingtime;
+           if (front_pingtime > i->second.hb_max_front)
+             i->second.hb_max_front = front_pingtime;
 
            ceph_assert(i->second.hb_interval_start != utime_t());
            if (i->second.hb_interval_start == utime_t())
              i->second.hb_interval_start = now;
            if (now - i->second.hb_interval_start >=  utime_t(hb_avg, 0)) {
               uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count;
+              uint32_t back_min = i->second.hb_min_back;
+              uint32_t back_max = i->second.hb_max_back;
               uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count;
+              uint32_t front_min = i->second.hb_min_front;
+              uint32_t front_max = i->second.hb_max_front;
 
              // Reset for new interval
              i->second.hb_average_count = 0;
              i->second.hb_interval_start = now;
-             i->second.hb_total_back = 0;
-             i->second.hb_total_front = 0;
+             i->second.hb_total_back = i->second.hb_max_back = 0;
+             i->second.hb_min_back =  UINT_MAX;
+             i->second.hb_total_front = i->second.hb_max_front = 0;
+             i->second.hb_min_front = UINT_MAX;
 
              // Record per osd interace ping times
              // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval
              if (i->second.hb_back_pingtime.size() < hb_vector_size) {
                ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size());
                i->second.hb_back_pingtime.push_back(back_pingtime);
+               i->second.hb_back_min.push_back(back_min);
+               i->second.hb_back_max.push_back(back_max);
                i->second.hb_front_pingtime.push_back(front_pingtime);
+               i->second.hb_front_min.push_back(front_min);
+               i->second.hb_front_max.push_back(front_max);
              } else {
-               i->second.hb_back_pingtime[i->second.hb_index & (hb_vector_size - 1)] = back_pingtime;
-               i->second.hb_front_pingtime[i->second.hb_index & (hb_vector_size - 1)] = front_pingtime;
+               int index = i->second.hb_index & (hb_vector_size - 1);
+               i->second.hb_back_pingtime[index] = back_pingtime;
+               i->second.hb_back_min[index] = back_min;
+               i->second.hb_back_max[index] = back_max;
+               i->second.hb_front_pingtime[index] = front_pingtime;
+               i->second.hb_front_min[index] = front_min;
+               i->second.hb_front_max[index] = front_max;
              }
              ++i->second.hb_index;
 
              {
                std::lock_guard l(service.stat_lock);
                uint32_t total = 0;
+               uint32_t min = UINT_MAX;
+               uint32_t max = 0;
                uint32_t count = 0;
                uint32_t which = 0;
                uint32_t size = (uint32_t)i->second.hb_back_pingtime.size();
                for (int32_t k = size - 1 ; k >= 0; --k) {
                  ++count;
-                 total += i->second.hb_back_pingtime[(i->second.hb_index + k) % size];
+                 int index = (i->second.hb_index + k) % size;
+                 total += i->second.hb_back_pingtime[index];
+                 if (i->second.hb_back_min[index] < min)
+                   min = i->second.hb_back_min[index];
+                 if (i->second.hb_back_max[index] > max)
+                   max = i->second.hb_back_max[index];
                  if (count == 1 || count == 5 || count == 15) {
-                   service.osd_stat.hb_pingtime[from].back_pingtime[which++] = total / count;
+                   service.osd_stat.hb_pingtime[from].back_pingtime[which] = total / count;
+                   service.osd_stat.hb_pingtime[from].back_min[which] = min;
+                   service.osd_stat.hb_pingtime[from].back_max[which] = max;
+                   which++;
                    if (count == 15)
                      break;
                  }
@@ -4787,13 +4846,23 @@ void OSD::handle_osd_ping(MOSDPing *m)
 
                 if (i->second.con_front != NULL) {
                  total = 0;
+                 min = UINT_MAX;
+                 max = 0;
                  count = 0;
                  which = 0;
                  for (int32_t k = size - 1 ; k >= 0; --k) {
                    ++count;
-                   total += i->second.hb_front_pingtime[(i->second.hb_index + k) % size];
+                   int index = (i->second.hb_index + k) % size;
+                   total += i->second.hb_front_pingtime[index];
+                   if (i->second.hb_front_min[index] < min)
+                     min = i->second.hb_front_min[index];
+                   if (i->second.hb_front_max[index] > max)
+                     max = i->second.hb_front_max[index];
                    if (count == 1 || count == 5 || count == 15) {
-                     service.osd_stat.hb_pingtime[from].front_pingtime[which++] = total / count;
+                     service.osd_stat.hb_pingtime[from].front_pingtime[which] = total / count;
+                     service.osd_stat.hb_pingtime[from].front_min[which] = min;
+                     service.osd_stat.hb_pingtime[from].front_max[which] = max;
+                     which++;
                      if (count == 15)
                        break;
                    }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index aaf9b2645ec7be03c066f0ec29ab0c32116518e1..900c9cdd0efed794d331204c926bd9c2a6068a9b 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1529,10 +1529,18 @@ private:
     uint32_t hb_index = 0;
 
     uint32_t hb_total_back = 0;
+    uint32_t hb_min_back = UINT_MAX;
+    uint32_t hb_max_back = 0;
     vector<uint32_t> hb_back_pingtime;
+    vector<uint32_t> hb_back_min;
+    vector<uint32_t> hb_back_max;
 
     uint32_t hb_total_front = 0;
+    uint32_t hb_min_front = UINT_MAX;
+    uint32_t hb_max_front = 0;
     vector<uint32_t> hb_front_pingtime;
+    vector<uint32_t> hb_front_min;
+    vector<uint32_t> hb_front_max;
 
     bool is_unhealthy(utime_t now) {
       if (ping_history.empty()) {

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 92248a20afa8ef180543a0ff82f9907d0d5df7c7..131374a20186077417c9729af01652c58d90083e 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -359,32 +359,54 @@ void osd_stat_t::dump(Formatter *f) const
   f->open_object_section("perf_stat");
   os_perf_stat.dump(f);
   f->close_section();
-  // Call to update ping times?
   f->open_array_section("network_ping_times");
   for (auto &i : hb_pingtime) {
-    f->open_object_section("osd");
+    f->open_object_section("entry");
     f->dump_int("osd", i.first);
-
     f->open_array_section("interfaces");
     f->open_object_section("interface");
     f->dump_string("interface", "back");
+    f->open_object_section("average");
     f->dump_int("1min", i.second.back_pingtime[0]);
     f->dump_int("5min", i.second.back_pingtime[1]);
     f->dump_int("15min", i.second.back_pingtime[2]);
-    f->close_section();
+    f->close_section(); // average
+    f->open_object_section("min");
+    f->dump_int("1min", i.second.back_min[0]);
+    f->dump_int("5min", i.second.back_min[1]);
+    f->dump_int("15min", i.second.back_min[2]);
+    f->close_section(); // min
+    f->open_object_section("max");
+    f->dump_int("1min", i.second.back_max[0]);
+    f->dump_int("5min", i.second.back_max[1]);
+    f->dump_int("15min", i.second.back_max[2]);
+    f->close_section(); // max
+    f->close_section(); // interface
 
     if (i.second.front_pingtime[0] != 0) {
       f->open_object_section("interface");
       f->dump_string("interface", "front");
+      f->open_object_section("average");
       f->dump_int("1min", i.second.front_pingtime[0]);
       f->dump_int("5min", i.second.front_pingtime[1]);
       f->dump_int("15min", i.second.front_pingtime[2]);
-      f->close_section();
+      f->close_section(); // average
+      f->open_object_section("min");
+      f->dump_int("1min", i.second.front_min[0]);
+      f->dump_int("5min", i.second.front_min[1]);
+      f->dump_int("15min", i.second.front_min[2]);
+      f->close_section(); // min
+      f->open_object_section("max");
+      f->dump_int("1min", i.second.front_max[0]);
+      f->dump_int("5min", i.second.front_max[1]);
+      f->dump_int("15min", i.second.front_max[2]);
+      f->close_section(); // max
+      f->close_section(); // interface
     }
-    f->close_section();
-    f->close_section();
+    f->close_section(); // interfaces
+    f->close_section(); // entry
   }
-  f->close_section();
+  f->close_section(); // network_ping_time
 }
 
 void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
@@ -412,9 +434,21 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.back_pingtime[0], bl);
     encode(i.second.back_pingtime[1], bl);
     encode(i.second.back_pingtime[2], bl);
+    encode(i.second.back_min[0], bl);
+    encode(i.second.back_min[1], bl);
+    encode(i.second.back_min[2], bl);
+    encode(i.second.back_max[0], bl);
+    encode(i.second.back_max[1], bl);
+    encode(i.second.back_max[2], bl);
     encode(i.second.front_pingtime[0], bl);
     encode(i.second.front_pingtime[1], bl);
     encode(i.second.front_pingtime[2], bl);
+    encode(i.second.front_min[0], bl);
+    encode(i.second.front_min[1], bl);
+    encode(i.second.front_min[2], bl);
+    encode(i.second.front_max[0], bl);
+    encode(i.second.front_max[1], bl);
+    encode(i.second.front_max[2], bl);
   }
   ENCODE_FINISH(bl);
 }
@@ -461,9 +495,21 @@ void osd_stat_t::decode(bufferlist::iterator &bl)
       decode(ifs.back_pingtime[0],bl);
       decode(ifs.back_pingtime[1], bl);
       decode(ifs.back_pingtime[2], bl);
+      decode(ifs.back_min[0],bl);
+      decode(ifs.back_min[1], bl);
+      decode(ifs.back_min[2], bl);
+      decode(ifs.back_max[0],bl);
+      decode(ifs.back_max[1], bl);
+      decode(ifs.back_max[2], bl);
       decode(ifs.front_pingtime[0], bl);
       decode(ifs.front_pingtime[1], bl);
       decode(ifs.front_pingtime[2], bl);
+      decode(ifs.front_min[0], bl);
+      decode(ifs.front_min[1], bl);
+      decode(ifs.front_min[2], bl);
+      decode(ifs.front_max[0], bl);
+      decode(ifs.front_max[1], bl);
+      decode(ifs.front_max[2], bl);
       hb_pingtime[osd] = ifs;
     }
   }
@@ -484,9 +530,12 @@ void osd_stat_t::generate_test_instances(std::list<osd_stat_t*>& o)
   o.back()->hb_peers.push_back(7);
   o.back()->snap_trim_queue_len = 8;
   o.back()->num_snap_trimming = 99;
-  struct Interfaces gen_interfaces = { { 1000, 995, 990 }, {980, 985, 990} };
+  struct Interfaces gen_interfaces = {
+        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 },
+        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } };
   o.back()->hb_pingtime[20] = gen_interfaces;
-  gen_interfaces = { { 700, 800, 900 } };
+  gen_interfaces = {
+        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } };
   o.back()->hb_pingtime[30] = gen_interfaces;
 }
 

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index d5b287c00e58e3cb428918d9c1e05fdc489fd91b..df4018b11c93f04fb42d14dea6deb97a11c3d07f 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -935,7 +935,11 @@ struct osd_stat_t {
 
   struct Interfaces {
     uint32_t back_pingtime[3];
+    uint32_t back_min[3];
+    uint32_t back_max[3];
     uint32_t front_pingtime[3];
+    uint32_t front_min[3];
+    uint32_t front_max[3];
   };
   map<int, Interfaces> hb_pingtime;  ///< map of osd id to Interfaces