osd mgr: Store last pingtime for possible graphing

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 3f846d7c806b7f62ead08f0e9fb2ba927ffe0592)

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 24fbe9a51043cba09b2f1e0976c2fd46eb0a9392..a84614d0140c130ec599e0fa8ec210188c7e84ac 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -241,6 +241,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const mgr_ping_time_t& rhs) const {
         if (pingtime < rhs.pingtime)
@@ -277,14 +278,15 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.back_max[0];
          item.max[1] = j.second.back_max[1];
          item.max[2] = j.second.back_max[2];
+         item.last = j.second.back_last;
          item.back = true;
          sorted.emplace(item);
        }
 
+       if (j.second.front_last == 0)
+         continue;
        item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
        item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
-       if (item.pingtime == 0)
-         continue;
        if (!value || item.pingtime >= value) {
          item.from = i.first;
          item.to = j.first;
@@ -297,6 +299,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.front_max[0];
          item.max[1] = j.second.front_max[1];
          item.max[2] = j.second.front_max[2];
+         item.last = j.second.front_last;
          item.back = false;
          sorted.emplace(item);
        }
@@ -329,6 +332,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       f->dump_unsigned("5min", sitem.max[1]);
       f->dump_unsigned("15min", sitem.max[2]);
       f->close_section(); // max
+      f->dump_unsigned("last", sitem.last);
       f->close_section(); // entry
     }
     f->close_section(); // entries

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 04a3526b7bd945e428f1dbb37ac2cd80184066fc..3b0ace922067177ec9ae353bb558092b6a5c0106 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2732,6 +2732,7 @@ will start to track new ops received afterwards.";
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const osd_ping_time_t& rhs) const {
        if (pingtime < rhs.pingtime)
@@ -2765,10 +2766,11 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.back_max[0];
        item.max[1] = j.second.back_max[1];
        item.max[2] = j.second.back_max[2];
+       item.last = j.second.back_last;
        item.back = true;
        sorted.emplace(item);
       }
-      if (j.second.front_pingtime[0] == 0)
+      if (j.second.front_last == 0)
        continue;
       item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
@@ -2783,6 +2785,7 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.front_max[0];
        item.max[1] = j.second.front_max[1];
        item.max[2] = j.second.front_max[2];
+       item.last = j.second.front_last;
        item.back = false;
        sorted.emplace(item);
       }
@@ -2814,6 +2817,7 @@ will start to track new ops received afterwards.";
       f->dump_int("5min", sitem.max[1]);
       f->dump_int("15min", sitem.max[2]);
       f->close_section();  // max
+      f->dump_int("last", sitem.last);
       f->close_section();  // entry
     }
     f->close_section(); // entries
@@ -5242,10 +5246,10 @@ void OSD::handle_osd_ping(MOSDPing *m)
            if (i->second.hb_interval_start == utime_t())
              i->second.hb_interval_start = now;
            if (now - i->second.hb_interval_start >=  utime_t(hb_avg, 0)) {
-              uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count;
+              uint32_t back_avg = i->second.hb_total_back / i->second.hb_average_count;
               uint32_t back_min = i->second.hb_min_back;
               uint32_t back_max = i->second.hb_max_back;
-              uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count;
+              uint32_t front_avg = i->second.hb_total_front / i->second.hb_average_count;
               uint32_t front_min = i->second.hb_min_front;
               uint32_t front_max = i->second.hb_max_front;
 
@@ -5261,18 +5265,18 @@ void OSD::handle_osd_ping(MOSDPing *m)
              // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval
              if (i->second.hb_back_pingtime.size() < hb_vector_size) {
                ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size());
-               i->second.hb_back_pingtime.push_back(back_pingtime);
+               i->second.hb_back_pingtime.push_back(back_avg);
                i->second.hb_back_min.push_back(back_min);
                i->second.hb_back_max.push_back(back_max);
-               i->second.hb_front_pingtime.push_back(front_pingtime);
+               i->second.hb_front_pingtime.push_back(front_avg);
                i->second.hb_front_min.push_back(front_min);
                i->second.hb_front_max.push_back(front_max);
              } else {
                int index = i->second.hb_index & (hb_vector_size - 1);
-               i->second.hb_back_pingtime[index] = back_pingtime;
+               i->second.hb_back_pingtime[index] = back_avg;
                i->second.hb_back_min[index] = back_min;
                i->second.hb_back_max[index] = back_max;
-               i->second.hb_front_pingtime[index] = front_pingtime;
+               i->second.hb_front_pingtime[index] = front_avg;
                i->second.hb_front_min[index] = front_min;
                i->second.hb_front_max[index] = front_max;
              }
@@ -5280,6 +5284,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
 
              {
                std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+
                uint32_t total = 0;
                uint32_t min = UINT_MAX;
                uint32_t max = 0;
@@ -5305,6 +5311,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
                }
 
                 if (i->second.con_front != NULL) {
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
+
                  total = 0;
                  min = UINT_MAX;
                  max = 0;
@@ -5329,6 +5337,11 @@ void OSD::handle_osd_ping(MOSDPing *m)
                  }
                }
              }
+           } else {
+               std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+                if (i->second.con_front != NULL)
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
            }
             i->second.ping_history.erase(i->second.ping_history.begin(), ++acked);
           }

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index a24ed3e35b5560ffc4afc126cf37f6ca21712816..2a05689547ff263a0e8b985455c677f5b85ea480 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -415,6 +415,7 @@ void osd_stat_t::dump(Formatter *f) const
     f->dump_int("5min", i.second.back_max[1]);
     f->dump_int("15min", i.second.back_max[2]);
     f->close_section(); // max
+    f->dump_int("last", i.second.back_last);
     f->close_section(); // interface
 
     if (i.second.front_pingtime[0] != 0) {
@@ -435,6 +436,7 @@ void osd_stat_t::dump(Formatter *f) const
       f->dump_int("5min", i.second.front_max[1]);
       f->dump_int("15min", i.second.front_max[2]);
       f->close_section(); // max
+      f->dump_int("last", i.second.front_last);
       f->close_section(); // interface
     }
     f->close_section(); // interfaces
@@ -493,6 +495,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.back_max[0], bl);
     encode(i.second.back_max[1], bl);
     encode(i.second.back_max[2], bl);
+    encode(i.second.back_last, bl);
     encode(i.second.front_pingtime[0], bl);
     encode(i.second.front_pingtime[1], bl);
     encode(i.second.front_pingtime[2], bl);
@@ -502,6 +505,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.front_max[0], bl);
     encode(i.second.front_max[1], bl);
     encode(i.second.front_max[2], bl);
+    encode(i.second.front_last, bl);
   }
   ENCODE_FINISH(bl);
 }
@@ -594,6 +598,7 @@ void osd_stat_t::decode(bufferlist::const_iterator &bl)
       decode(ifs.back_max[0],bl);
       decode(ifs.back_max[1], bl);
       decode(ifs.back_max[2], bl);
+      decode(ifs.back_last, bl);
       decode(ifs.front_pingtime[0], bl);
       decode(ifs.front_pingtime[1], bl);
       decode(ifs.front_pingtime[2], bl);
@@ -603,6 +608,7 @@ void osd_stat_t::decode(bufferlist::const_iterator &bl)
       decode(ifs.front_max[0], bl);
       decode(ifs.front_max[1], bl);
       decode(ifs.front_max[2], bl);
+      decode(ifs.front_last, bl);
       hb_pingtime[osd] = ifs;
     }
   }
@@ -626,11 +632,11 @@ void osd_stat_t::generate_test_instances(std::list<osd_stat_t*>& o)
   o.back()->os_alerts[1].emplace(
     "some alert2", "some alert2 details");
   struct Interfaces gen_interfaces = {
-        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 },
-        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } };
+        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, 1001,
+        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 }, 1101 };
   o.back()->hb_pingtime[20] = gen_interfaces;
   gen_interfaces = {
-        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } };
+        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 }, 101 };
   o.back()->hb_pingtime[30] = gen_interfaces;
 }
 

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 83887308c7350b9045dfaaa0fbdc0fefbe635e17..554577922cb58ed13e8c93d50ff7d340ee6662c7 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -2360,9 +2360,11 @@ struct osd_stat_t {
     uint32_t back_pingtime[3];
     uint32_t back_min[3];
     uint32_t back_max[3];
+    uint32_t back_last;
     uint32_t front_pingtime[3];
     uint32_t front_min[3];
     uint32_t front_max[3];
+    uint32_t front_last;
   };
   map<int, Interfaces> hb_pingtime;  ///< map of osd id to Interfaces