osd mgr: Add minimum and maximum tracking to network ping time

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 297a0e7b1de410c094fc9a6e42be14813d6dac5e)

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 00a723cd5b6ab695a3807c02d08a28aa6c4e71d6..24fbe9a51043cba09b2f1e0976c2fd46eb0a9392 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -239,6 +239,8 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       int to;
       bool back;
       std::array<uint32_t,3> times;
+      std::array<uint32_t,3> min;
+      std::array<uint32_t,3> max;
 
       bool operator<(const mgr_ping_time_t& rhs) const {
         if (pingtime < rhs.pingtime)
@@ -269,6 +271,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.times[0] = j.second.back_pingtime[0];
          item.times[1] = j.second.back_pingtime[1];
          item.times[2] = j.second.back_pingtime[2];
+         item.min[0] = j.second.back_min[0];
+         item.min[1] = j.second.back_min[1];
+         item.min[2] = j.second.back_min[2];
+         item.max[0] = j.second.back_max[0];
+         item.max[1] = j.second.back_max[1];
+         item.max[2] = j.second.back_max[2];
          item.back = true;
          sorted.emplace(item);
        }
@@ -283,6 +291,12 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.times[0] = j.second.front_pingtime[0];
          item.times[1] = j.second.front_pingtime[1];
          item.times[2] = j.second.front_pingtime[2];
+         item.min[0] = j.second.front_min[0];
+         item.min[1] = j.second.front_min[1];
+         item.min[2] = j.second.front_min[2];
+         item.max[0] = j.second.front_max[0];
+         item.max[1] = j.second.front_max[1];
+         item.max[2] = j.second.front_max[2];
          item.back = false;
          sorted.emplace(item);
        }
@@ -300,9 +314,21 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       f->dump_int("from osd", sitem.from);
       f->dump_int("to osd", sitem.to);
       f->dump_string("interface", (sitem.back ? "back" : "front"));
+      f->open_object_section("average");
       f->dump_unsigned("1min", sitem.times[0]);
       f->dump_unsigned("5min", sitem.times[1]);
       f->dump_unsigned("15min", sitem.times[2]);
+      f->close_section(); // average
+      f->open_object_section("min");
+      f->dump_unsigned("1min", sitem.min[0]);
+      f->dump_unsigned("5min", sitem.min[1]);
+      f->dump_unsigned("15min", sitem.min[2]);
+      f->close_section(); // min
+      f->open_object_section("max");
+      f->dump_unsigned("1min", sitem.max[0]);
+      f->dump_unsigned("5min", sitem.max[1]);
+      f->dump_unsigned("15min", sitem.max[2]);
+      f->close_section(); // max
       f->close_section(); // entry
     }
     f->close_section(); // entries

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 1cc3a8e34111294f4ec385d6add560b0ae222dfa..04a3526b7bd945e428f1dbb37ac2cd80184066fc 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2730,6 +2730,8 @@ will start to track new ops received afterwards.";
       int to;
       bool back;
       std::array<uint32_t,3> times;
+      std::array<uint32_t,3> min;
+      std::array<uint32_t,3> max;
 
       bool operator<(const osd_ping_time_t& rhs) const {
        if (pingtime < rhs.pingtime)
@@ -2757,6 +2759,12 @@ will start to track new ops received afterwards.";
        item.times[0] = j.second.back_pingtime[0];
        item.times[1] = j.second.back_pingtime[1];
        item.times[2] = j.second.back_pingtime[2];
+       item.min[0] = j.second.back_min[0];
+       item.min[1] = j.second.back_min[1];
+       item.min[2] = j.second.back_min[2];
+       item.max[0] = j.second.back_max[0];
+       item.max[1] = j.second.back_max[1];
+       item.max[2] = j.second.back_max[2];
        item.back = true;
        sorted.emplace(item);
       }
@@ -2769,6 +2777,12 @@ will start to track new ops received afterwards.";
        item.times[0] = j.second.front_pingtime[0];
        item.times[1] = j.second.front_pingtime[1];
        item.times[2] = j.second.front_pingtime[2];
+       item.min[0] = j.second.front_min[0];
+       item.min[1] = j.second.front_min[1];
+       item.min[2] = j.second.front_min[2];
+       item.max[0] = j.second.front_max[0];
+       item.max[1] = j.second.front_max[1];
+       item.max[2] = j.second.front_max[2];
        item.back = false;
        sorted.emplace(item);
       }
@@ -2785,9 +2799,21 @@ will start to track new ops received afterwards.";
       f->dump_int("from osd", whoami);
       f->dump_int("to osd", sitem.to);
       f->dump_string("interface", (sitem.back ? "back" : "front"));
+      f->open_object_section("average");
       f->dump_int("1min", sitem.times[0]);
       f->dump_int("5min", sitem.times[1]);
       f->dump_int("15min", sitem.times[2]);
+      f->close_section();  // average
+      f->open_object_section("min");
+      f->dump_int("1min", sitem.min[0]);
+      f->dump_int("5min", sitem.min[1]);
+      f->dump_int("15min", sitem.min[2]);
+      f->close_section();  // min
+      f->open_object_section("max");
+      f->dump_int("1min", sitem.max[0]);
+      f->dump_int("5min", sitem.max[1]);
+      f->dump_int("15min", sitem.max[2]);
+      f->close_section();  // max
       f->close_section();  // entry
     }
     f->close_section(); // entries
@@ -5201,45 +5227,78 @@ void OSD::handle_osd_ping(MOSDPing *m)
            ++i->second.hb_average_count;
            uint32_t back_pingtime = ROUND_S_TO_USEC(i->second.last_rx_back - m->stamp);
            i->second.hb_total_back += back_pingtime;
+           if (back_pingtime < i->second.hb_min_back)
+             i->second.hb_min_back = back_pingtime;
+           if (back_pingtime > i->second.hb_max_back)
+             i->second.hb_max_back = back_pingtime;
            uint32_t front_pingtime = ROUND_S_TO_USEC(i->second.last_rx_front - m->stamp);
            i->second.hb_total_front += front_pingtime;
+           if (front_pingtime < i->second.hb_min_front)
+             i->second.hb_min_front = front_pingtime;
+           if (front_pingtime > i->second.hb_max_front)
+             i->second.hb_max_front = front_pingtime;
 
            ceph_assert(i->second.hb_interval_start != utime_t());
            if (i->second.hb_interval_start == utime_t())
              i->second.hb_interval_start = now;
            if (now - i->second.hb_interval_start >=  utime_t(hb_avg, 0)) {
               uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count;
+              uint32_t back_min = i->second.hb_min_back;
+              uint32_t back_max = i->second.hb_max_back;
               uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count;
+              uint32_t front_min = i->second.hb_min_front;
+              uint32_t front_max = i->second.hb_max_front;
 
              // Reset for new interval
              i->second.hb_average_count = 0;
              i->second.hb_interval_start = now;
-             i->second.hb_total_back = 0;
-             i->second.hb_total_front = 0;
+             i->second.hb_total_back = i->second.hb_max_back = 0;
+             i->second.hb_min_back =  UINT_MAX;
+             i->second.hb_total_front = i->second.hb_max_front = 0;
+             i->second.hb_min_front = UINT_MAX;
 
              // Record per osd interace ping times
              // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval
              if (i->second.hb_back_pingtime.size() < hb_vector_size) {
                ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size());
                i->second.hb_back_pingtime.push_back(back_pingtime);
+               i->second.hb_back_min.push_back(back_min);
+               i->second.hb_back_max.push_back(back_max);
                i->second.hb_front_pingtime.push_back(front_pingtime);
+               i->second.hb_front_min.push_back(front_min);
+               i->second.hb_front_max.push_back(front_max);
              } else {
-               i->second.hb_back_pingtime[i->second.hb_index & (hb_vector_size - 1)] = back_pingtime;
-               i->second.hb_front_pingtime[i->second.hb_index & (hb_vector_size - 1)] = front_pingtime;
+               int index = i->second.hb_index & (hb_vector_size - 1);
+               i->second.hb_back_pingtime[index] = back_pingtime;
+               i->second.hb_back_min[index] = back_min;
+               i->second.hb_back_max[index] = back_max;
+               i->second.hb_front_pingtime[index] = front_pingtime;
+               i->second.hb_front_min[index] = front_min;
+               i->second.hb_front_max[index] = front_max;
              }
              ++i->second.hb_index;
 
              {
                std::lock_guard l(service.stat_lock);
                uint32_t total = 0;
+               uint32_t min = UINT_MAX;
+               uint32_t max = 0;
                uint32_t count = 0;
                uint32_t which = 0;
                uint32_t size = (uint32_t)i->second.hb_back_pingtime.size();
                for (int32_t k = size - 1 ; k >= 0; --k) {
                  ++count;
-                 total += i->second.hb_back_pingtime[(i->second.hb_index + k) % size];
+                 int index = (i->second.hb_index + k) % size;
+                 total += i->second.hb_back_pingtime[index];
+                 if (i->second.hb_back_min[index] < min)
+                   min = i->second.hb_back_min[index];
+                 if (i->second.hb_back_max[index] > max)
+                   max = i->second.hb_back_max[index];
                  if (count == 1 || count == 5 || count == 15) {
-                   service.osd_stat.hb_pingtime[from].back_pingtime[which++] = total / count;
+                   service.osd_stat.hb_pingtime[from].back_pingtime[which] = total / count;
+                   service.osd_stat.hb_pingtime[from].back_min[which] = min;
+                   service.osd_stat.hb_pingtime[from].back_max[which] = max;
+                   which++;
                    if (count == 15)
                      break;
                  }
@@ -5247,13 +5306,23 @@ void OSD::handle_osd_ping(MOSDPing *m)
 
                 if (i->second.con_front != NULL) {
                  total = 0;
+                 min = UINT_MAX;
+                 max = 0;
                  count = 0;
                  which = 0;
                  for (int32_t k = size - 1 ; k >= 0; --k) {
                    ++count;
-                   total += i->second.hb_front_pingtime[(i->second.hb_index + k) % size];
+                   int index = (i->second.hb_index + k) % size;
+                   total += i->second.hb_front_pingtime[index];
+                   if (i->second.hb_front_min[index] < min)
+                     min = i->second.hb_front_min[index];
+                   if (i->second.hb_front_max[index] > max)
+                     max = i->second.hb_front_max[index];
                    if (count == 1 || count == 5 || count == 15) {
-                     service.osd_stat.hb_pingtime[from].front_pingtime[which++] = total / count;
+                     service.osd_stat.hb_pingtime[from].front_pingtime[which] = total / count;
+                     service.osd_stat.hb_pingtime[from].front_min[which] = min;
+                     service.osd_stat.hb_pingtime[from].front_max[which] = max;
+                     which++;
                      if (count == 15)
                        break;
                    }

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 00805d0d46f5b0fe01c321a2f47f7882b82110cb..2e9bae1da7c69cb832143c67b1680fd8ccc2f019 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1574,10 +1574,18 @@ private:
     uint32_t hb_index = 0;
 
     uint32_t hb_total_back = 0;
+    uint32_t hb_min_back = UINT_MAX;
+    uint32_t hb_max_back = 0;
     vector<uint32_t> hb_back_pingtime;
+    vector<uint32_t> hb_back_min;
+    vector<uint32_t> hb_back_max;
 
     uint32_t hb_total_front = 0;
+    uint32_t hb_min_front = UINT_MAX;
+    uint32_t hb_max_front = 0;
     vector<uint32_t> hb_front_pingtime;
+    vector<uint32_t> hb_front_min;
+    vector<uint32_t> hb_front_max;
 
     bool is_stale(utime_t stale) {
       if (ping_history.empty()) {

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 3a75c9c0cfb0fd48dc678ec19ccbae0e010554c2..a24ed3e35b5560ffc4afc126cf37f6ca21712816 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -393,32 +393,54 @@ void osd_stat_t::dump(Formatter *f) const
   f->open_array_section("alerts");
   ::dump(f, os_alerts);
   f->close_section();
-  // Call to update ping times?
   f->open_array_section("network_ping_times");
   for (auto &i : hb_pingtime) {
-    f->open_object_section("osd");
+    f->open_object_section("entry");
     f->dump_int("osd", i.first);
-
     f->open_array_section("interfaces");
     f->open_object_section("interface");
     f->dump_string("interface", "back");
+    f->open_object_section("average");
     f->dump_int("1min", i.second.back_pingtime[0]);
     f->dump_int("5min", i.second.back_pingtime[1]);
     f->dump_int("15min", i.second.back_pingtime[2]);
-    f->close_section();
+    f->close_section(); // average
+    f->open_object_section("min");
+    f->dump_int("1min", i.second.back_min[0]);
+    f->dump_int("5min", i.second.back_min[1]);
+    f->dump_int("15min", i.second.back_min[2]);
+    f->close_section(); // min
+    f->open_object_section("max");
+    f->dump_int("1min", i.second.back_max[0]);
+    f->dump_int("5min", i.second.back_max[1]);
+    f->dump_int("15min", i.second.back_max[2]);
+    f->close_section(); // max
+    f->close_section(); // interface
 
     if (i.second.front_pingtime[0] != 0) {
       f->open_object_section("interface");
       f->dump_string("interface", "front");
+      f->open_object_section("average");
       f->dump_int("1min", i.second.front_pingtime[0]);
       f->dump_int("5min", i.second.front_pingtime[1]);
       f->dump_int("15min", i.second.front_pingtime[2]);
-      f->close_section();
+      f->close_section(); // average
+      f->open_object_section("min");
+      f->dump_int("1min", i.second.front_min[0]);
+      f->dump_int("5min", i.second.front_min[1]);
+      f->dump_int("15min", i.second.front_min[2]);
+      f->close_section(); // min
+      f->open_object_section("max");
+      f->dump_int("1min", i.second.front_max[0]);
+      f->dump_int("5min", i.second.front_max[1]);
+      f->dump_int("15min", i.second.front_max[2]);
+      f->close_section(); // max
+      f->close_section(); // interface
     }
-    f->close_section();
-    f->close_section();
+    f->close_section(); // interfaces
+    f->close_section(); // entry
   }
-  f->close_section();
+  f->close_section(); // network_ping_time
 }
 
 void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
@@ -465,9 +487,21 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.back_pingtime[0], bl);
     encode(i.second.back_pingtime[1], bl);
     encode(i.second.back_pingtime[2], bl);
+    encode(i.second.back_min[0], bl);
+    encode(i.second.back_min[1], bl);
+    encode(i.second.back_min[2], bl);
+    encode(i.second.back_max[0], bl);
+    encode(i.second.back_max[1], bl);
+    encode(i.second.back_max[2], bl);
     encode(i.second.front_pingtime[0], bl);
     encode(i.second.front_pingtime[1], bl);
     encode(i.second.front_pingtime[2], bl);
+    encode(i.second.front_min[0], bl);
+    encode(i.second.front_min[1], bl);
+    encode(i.second.front_min[2], bl);
+    encode(i.second.front_max[0], bl);
+    encode(i.second.front_max[1], bl);
+    encode(i.second.front_max[2], bl);
   }
   ENCODE_FINISH(bl);
 }
@@ -554,9 +588,21 @@ void osd_stat_t::decode(bufferlist::const_iterator &bl)
       decode(ifs.back_pingtime[0],bl);
       decode(ifs.back_pingtime[1], bl);
       decode(ifs.back_pingtime[2], bl);
+      decode(ifs.back_min[0],bl);
+      decode(ifs.back_min[1], bl);
+      decode(ifs.back_min[2], bl);
+      decode(ifs.back_max[0],bl);
+      decode(ifs.back_max[1], bl);
+      decode(ifs.back_max[2], bl);
       decode(ifs.front_pingtime[0], bl);
       decode(ifs.front_pingtime[1], bl);
       decode(ifs.front_pingtime[2], bl);
+      decode(ifs.front_min[0], bl);
+      decode(ifs.front_min[1], bl);
+      decode(ifs.front_min[2], bl);
+      decode(ifs.front_max[0], bl);
+      decode(ifs.front_max[1], bl);
+      decode(ifs.front_max[2], bl);
       hb_pingtime[osd] = ifs;
     }
   }
@@ -579,9 +625,12 @@ void osd_stat_t::generate_test_instances(std::list<osd_stat_t*>& o)
     "some alert", "some alert details");
   o.back()->os_alerts[1].emplace(
     "some alert2", "some alert2 details");
-  struct Interfaces gen_interfaces = { { 1000, 995, 990 }, {980, 985, 990} };
+  struct Interfaces gen_interfaces = {
+        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 },
+        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } };
   o.back()->hb_pingtime[20] = gen_interfaces;
-  gen_interfaces = { { 700, 800, 900 } };
+  gen_interfaces = {
+        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } };
   o.back()->hb_pingtime[30] = gen_interfaces;
 }
 

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 9a598c7a9d82c134e7680d59e7db06eb1e2d09d1..83887308c7350b9045dfaaa0fbdc0fefbe635e17 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -2358,7 +2358,11 @@ struct osd_stat_t {
 
   struct Interfaces {
     uint32_t back_pingtime[3];
+    uint32_t back_min[3];
+    uint32_t back_max[3];
     uint32_t front_pingtime[3];
+    uint32_t front_min[3];
+    uint32_t front_max[3];
   };
   map<int, Interfaces> hb_pingtime;  ///< map of osd id to Interfaces