osd mgr: Store last pingtime for possible graphing

Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index e5279321b382ee3846ddff0855184eb0b11337ab..af87c982084930f2b5d60c115fb1ea7cd13c46d8 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -240,6 +240,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const mgr_ping_time_t& rhs) const {
         if (pingtime < rhs.pingtime)
@@ -276,14 +277,15 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.back_max[0];
          item.max[1] = j.second.back_max[1];
          item.max[2] = j.second.back_max[2];
+         item.last = j.second.back_last;
          item.back = true;
          sorted.emplace(item);
        }
 
+       if (j.second.front_last == 0)
+         continue;
        item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
        item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
-       if (item.pingtime == 0)
-         continue;
        if (!value || item.pingtime >= value) {
          item.from = i.first;
          item.to = j.first;
@@ -296,6 +298,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.front_max[0];
          item.max[1] = j.second.front_max[1];
          item.max[2] = j.second.front_max[2];
+         item.last = j.second.front_last;
          item.back = false;
          sorted.emplace(item);
        }
@@ -328,6 +331,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       f->dump_unsigned("5min", sitem.max[1]);
       f->dump_unsigned("15min", sitem.max[2]);
       f->close_section(); // max
+      f->dump_unsigned("last", sitem.last);
       f->close_section(); // entry
     }
     f->close_section(); // entries

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 2ecdb71b1f174780168ec86f70171156820c2bff..18c13bfc4c7d0c8b12de938e08f0ed394b374c70 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2568,6 +2568,7 @@ will start to track new ops received afterwards.";
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const osd_ping_time_t& rhs) const {
        if (pingtime < rhs.pingtime)
@@ -2601,10 +2602,11 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.back_max[0];
        item.max[1] = j.second.back_max[1];
        item.max[2] = j.second.back_max[2];
+       item.last = j.second.back_last;
        item.back = true;
        sorted.emplace(item);
       }
-      if (j.second.front_pingtime[0] == 0)
+      if (j.second.front_last == 0)
        continue;
       item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
@@ -2619,6 +2621,7 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.front_max[0];
        item.max[1] = j.second.front_max[1];
        item.max[2] = j.second.front_max[2];
+       item.last = j.second.front_last;
        item.back = false;
        sorted.emplace(item);
       }
@@ -2650,6 +2653,7 @@ will start to track new ops received afterwards.";
       f->dump_int("5min", sitem.max[1]);
       f->dump_int("15min", sitem.max[2]);
       f->close_section();  // max
+      f->dump_int("last", sitem.last);
       f->close_section();  // entry
     }
     f->close_section(); // entries
@@ -4829,10 +4833,10 @@ void OSD::handle_osd_ping(MOSDPing *m)
            if (i->second.hb_interval_start == utime_t())
              i->second.hb_interval_start = now;
            if (now - i->second.hb_interval_start >=  utime_t(hb_avg, 0)) {
-              uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count;
+              uint32_t back_avg = i->second.hb_total_back / i->second.hb_average_count;
               uint32_t back_min = i->second.hb_min_back;
               uint32_t back_max = i->second.hb_max_back;
-              uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count;
+              uint32_t front_avg = i->second.hb_total_front / i->second.hb_average_count;
               uint32_t front_min = i->second.hb_min_front;
               uint32_t front_max = i->second.hb_max_front;
 
@@ -4848,18 +4852,18 @@ void OSD::handle_osd_ping(MOSDPing *m)
              // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval
              if (i->second.hb_back_pingtime.size() < hb_vector_size) {
                ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size());
-               i->second.hb_back_pingtime.push_back(back_pingtime);
+               i->second.hb_back_pingtime.push_back(back_avg);
                i->second.hb_back_min.push_back(back_min);
                i->second.hb_back_max.push_back(back_max);
-               i->second.hb_front_pingtime.push_back(front_pingtime);
+               i->second.hb_front_pingtime.push_back(front_avg);
                i->second.hb_front_min.push_back(front_min);
                i->second.hb_front_max.push_back(front_max);
              } else {
                int index = i->second.hb_index & (hb_vector_size - 1);
-               i->second.hb_back_pingtime[index] = back_pingtime;
+               i->second.hb_back_pingtime[index] = back_avg;
                i->second.hb_back_min[index] = back_min;
                i->second.hb_back_max[index] = back_max;
-               i->second.hb_front_pingtime[index] = front_pingtime;
+               i->second.hb_front_pingtime[index] = front_avg;
                i->second.hb_front_min[index] = front_min;
                i->second.hb_front_max[index] = front_max;
              }
@@ -4867,6 +4871,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
 
              {
                std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+
                uint32_t total = 0;
                uint32_t min = UINT_MAX;
                uint32_t max = 0;
@@ -4892,6 +4898,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
                }
 
                 if (i->second.con_front != NULL) {
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
+
                  total = 0;
                  min = UINT_MAX;
                  max = 0;
@@ -4916,6 +4924,11 @@ void OSD::handle_osd_ping(MOSDPing *m)
                  }
                }
              }
+           } else {
+               std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+                if (i->second.con_front != NULL)
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
            }
             i->second.ping_history.erase(i->second.ping_history.begin(), ++acked);
           }

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 0cb1f24fc2fd77c15a085024bbf446d5615504f4..21645fd854ec6a736f46b89ef7dceb9cc5d11aba 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -450,6 +450,7 @@ void osd_stat_t::dump(Formatter *f) const
     f->dump_int("5min", i.second.back_max[1]);
     f->dump_int("15min", i.second.back_max[2]);
     f->close_section(); // max
+    f->dump_int("last", i.second.back_last);
     f->close_section(); // interface
 
     if (i.second.front_pingtime[0] != 0) {
@@ -470,6 +471,7 @@ void osd_stat_t::dump(Formatter *f) const
       f->dump_int("5min", i.second.front_max[1]);
       f->dump_int("15min", i.second.front_max[2]);
       f->close_section(); // max
+      f->dump_int("last", i.second.front_last);
       f->close_section(); // interface
     }
     f->close_section(); // interfaces
@@ -529,6 +531,7 @@ void osd_stat_t::encode(ceph::buffer::list &bl, uint64_t features) const
     encode(i.second.back_max[0], bl);
     encode(i.second.back_max[1], bl);
     encode(i.second.back_max[2], bl);
+    encode(i.second.back_last, bl);
     encode(i.second.front_pingtime[0], bl);
     encode(i.second.front_pingtime[1], bl);
     encode(i.second.front_pingtime[2], bl);
@@ -538,6 +541,7 @@ void osd_stat_t::encode(ceph::buffer::list &bl, uint64_t features) const
     encode(i.second.front_max[0], bl);
     encode(i.second.front_max[1], bl);
     encode(i.second.front_max[2], bl);
+    encode(i.second.front_last, bl);
   }
   ENCODE_FINISH(bl);
 }
@@ -635,6 +639,7 @@ void osd_stat_t::decode(ceph::buffer::list::const_iterator &bl)
       decode(ifs.back_max[0],bl);
       decode(ifs.back_max[1], bl);
       decode(ifs.back_max[2], bl);
+      decode(ifs.back_last, bl);
       decode(ifs.front_pingtime[0], bl);
       decode(ifs.front_pingtime[1], bl);
       decode(ifs.front_pingtime[2], bl);
@@ -644,6 +649,7 @@ void osd_stat_t::decode(ceph::buffer::list::const_iterator &bl)
       decode(ifs.front_max[0], bl);
       decode(ifs.front_max[1], bl);
       decode(ifs.front_max[2], bl);
+      decode(ifs.front_last, bl);
       hb_pingtime[osd] = ifs;
     }
   }
@@ -667,11 +673,11 @@ void osd_stat_t::generate_test_instances(std::list<osd_stat_t*>& o)
   o.back()->os_alerts[1].emplace(
     "some alert2", "some alert2 details");
   struct Interfaces gen_interfaces = {
-        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 },
-        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } };
+        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, 1001,
+        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 }, 1101 };
   o.back()->hb_pingtime[20] = gen_interfaces;
   gen_interfaces = {
-        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } };
+        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 }, 101 };
   o.back()->hb_pingtime[30] = gen_interfaces;
 }
 

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index 42d179de0207bfa6b26c85c09e5d7069ebd8a1e1..56d0dbead5837fc329d6146e6be5292c3377acb9 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -2376,9 +2376,11 @@ struct osd_stat_t {
     uint32_t back_pingtime[3];
     uint32_t back_min[3];
     uint32_t back_max[3];
+    uint32_t back_last;
     uint32_t front_pingtime[3];
     uint32_t front_min[3];
     uint32_t front_max[3];
+    uint32_t front_last;
   };
   map<int, Interfaces> hb_pingtime;  ///< map of osd id to Interfaces