osd mgr: Store last pingtime for possible graphing

Signed-off-by: David Zafman <dzafman@redhat.com>
(cherry picked from commit 3f846d7c806b7f62ead08f0e9fb2ba927ffe0592)

Conflicts:
	src/osd/osd_types.h (osd_stat_t location in file changed)

diff --git a/src/mgr/ClusterState.cc b/src/mgr/ClusterState.cc
index 01e83a069867b0afd671e9a5c01ba6f22bccba2c..79c71e9fb71a5cc91181ce11c7553eb4ade38ecf 100644 (file)
--- a/src/mgr/ClusterState.cc
+++ b/src/mgr/ClusterState.cc
@@ -230,6 +230,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const mgr_ping_time_t& rhs) const {
         if (pingtime < rhs.pingtime)
@@ -266,14 +267,15 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.back_max[0];
          item.max[1] = j.second.back_max[1];
          item.max[2] = j.second.back_max[2];
+         item.last = j.second.back_last;
          item.back = true;
          sorted.emplace(item);
        }
 
+       if (j.second.front_last == 0)
+         continue;
        item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
        item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
-       if (item.pingtime == 0)
-         continue;
        if (!value || item.pingtime >= value) {
          item.from = i.first;
          item.to = j.first;
@@ -286,6 +288,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
          item.max[0] = j.second.front_max[0];
          item.max[1] = j.second.front_max[1];
          item.max[2] = j.second.front_max[2];
+         item.last = j.second.front_last;
          item.back = false;
          sorted.emplace(item);
        }
@@ -318,6 +321,7 @@ bool ClusterState::asok_command(std::string_view admin_command, const cmdmap_t&
       f->dump_unsigned("5min", sitem.max[1]);
       f->dump_unsigned("15min", sitem.max[2]);
       f->close_section(); // max
+      f->dump_unsigned("last", sitem.last);
       f->close_section(); // entry
     }
     f->close_section(); // entries

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 22f5f286f6765e3ac02140b7fb210756a5dc081f..6023ec58530782c62d35752a430d02f808c0f325 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -2301,6 +2301,7 @@ will start to track new ops received afterwards.";
       std::array<uint32_t,3> times;
       std::array<uint32_t,3> min;
       std::array<uint32_t,3> max;
+      uint32_t last;
 
       bool operator<(const osd_ping_time_t& rhs) const {
        if (pingtime < rhs.pingtime)
@@ -2334,10 +2335,11 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.back_max[0];
        item.max[1] = j.second.back_max[1];
        item.max[2] = j.second.back_max[2];
+       item.last = j.second.back_last;
        item.back = true;
        sorted.emplace(item);
       }
-      if (j.second.front_pingtime[0] == 0)
+      if (j.second.front_last == 0)
        continue;
       item.pingtime = std::max(j.second.front_pingtime[0], j.second.front_pingtime[1]);
       item.pingtime = std::max(item.pingtime, j.second.front_pingtime[2]);
@@ -2352,6 +2354,7 @@ will start to track new ops received afterwards.";
        item.max[0] = j.second.front_max[0];
        item.max[1] = j.second.front_max[1];
        item.max[2] = j.second.front_max[2];
+       item.last = j.second.front_last;
        item.back = false;
        sorted.emplace(item);
       }
@@ -2383,6 +2386,7 @@ will start to track new ops received afterwards.";
       f->dump_int("5min", sitem.max[1]);
       f->dump_int("15min", sitem.max[2]);
       f->close_section();  // max
+      f->dump_int("last", sitem.last);
       f->close_section();  // entry
     }
     f->close_section(); // entries
@@ -4782,10 +4786,10 @@ void OSD::handle_osd_ping(MOSDPing *m)
            if (i->second.hb_interval_start == utime_t())
              i->second.hb_interval_start = now;
            if (now - i->second.hb_interval_start >=  utime_t(hb_avg, 0)) {
-              uint32_t back_pingtime = i->second.hb_total_back / i->second.hb_average_count;
+              uint32_t back_avg = i->second.hb_total_back / i->second.hb_average_count;
               uint32_t back_min = i->second.hb_min_back;
               uint32_t back_max = i->second.hb_max_back;
-              uint32_t front_pingtime = i->second.hb_total_front / i->second.hb_average_count;
+              uint32_t front_avg = i->second.hb_total_front / i->second.hb_average_count;
               uint32_t front_min = i->second.hb_min_front;
               uint32_t front_max = i->second.hb_max_front;
 
@@ -4801,18 +4805,18 @@ void OSD::handle_osd_ping(MOSDPing *m)
              // Based on osd_heartbeat_interval ignoring that it is randomly short than this interval
              if (i->second.hb_back_pingtime.size() < hb_vector_size) {
                ceph_assert(i->second.hb_front_pingtime.size() == i->second.hb_back_pingtime.size());
-               i->second.hb_back_pingtime.push_back(back_pingtime);
+               i->second.hb_back_pingtime.push_back(back_avg);
                i->second.hb_back_min.push_back(back_min);
                i->second.hb_back_max.push_back(back_max);
-               i->second.hb_front_pingtime.push_back(front_pingtime);
+               i->second.hb_front_pingtime.push_back(front_avg);
                i->second.hb_front_min.push_back(front_min);
                i->second.hb_front_max.push_back(front_max);
              } else {
                int index = i->second.hb_index & (hb_vector_size - 1);
-               i->second.hb_back_pingtime[index] = back_pingtime;
+               i->second.hb_back_pingtime[index] = back_avg;
                i->second.hb_back_min[index] = back_min;
                i->second.hb_back_max[index] = back_max;
-               i->second.hb_front_pingtime[index] = front_pingtime;
+               i->second.hb_front_pingtime[index] = front_avg;
                i->second.hb_front_min[index] = front_min;
                i->second.hb_front_max[index] = front_max;
              }
@@ -4820,6 +4824,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
 
              {
                std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+
                uint32_t total = 0;
                uint32_t min = UINT_MAX;
                uint32_t max = 0;
@@ -4845,6 +4851,8 @@ void OSD::handle_osd_ping(MOSDPing *m)
                }
 
                 if (i->second.con_front != NULL) {
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
+
                  total = 0;
                  min = UINT_MAX;
                  max = 0;
@@ -4869,6 +4877,11 @@ void OSD::handle_osd_ping(MOSDPing *m)
                  }
                }
              }
+           } else {
+               std::lock_guard l(service.stat_lock);
+               service.osd_stat.hb_pingtime[from].back_last =  back_pingtime;
+                if (i->second.con_front != NULL)
+                 service.osd_stat.hb_pingtime[from].front_last = front_pingtime;
            }
             i->second.ping_history.erase(i->second.ping_history.begin(), ++acked);
           }

diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc
index 131374a20186077417c9729af01652c58d90083e..092b5f759df9cfd080ab5c0b20e8ab9a8e15093d 100644 (file)
--- a/src/osd/osd_types.cc
+++ b/src/osd/osd_types.cc
@@ -381,6 +381,7 @@ void osd_stat_t::dump(Formatter *f) const
     f->dump_int("5min", i.second.back_max[1]);
     f->dump_int("15min", i.second.back_max[2]);
     f->close_section(); // max
+    f->dump_int("last", i.second.back_last);
     f->close_section(); // interface
 
     if (i.second.front_pingtime[0] != 0) {
@@ -401,6 +402,7 @@ void osd_stat_t::dump(Formatter *f) const
       f->dump_int("5min", i.second.front_max[1]);
       f->dump_int("15min", i.second.front_max[2]);
       f->close_section(); // max
+      f->dump_int("last", i.second.front_last);
       f->close_section(); // interface
     }
     f->close_section(); // interfaces
@@ -440,6 +442,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.back_max[0], bl);
     encode(i.second.back_max[1], bl);
     encode(i.second.back_max[2], bl);
+    encode(i.second.back_last, bl);
     encode(i.second.front_pingtime[0], bl);
     encode(i.second.front_pingtime[1], bl);
     encode(i.second.front_pingtime[2], bl);
@@ -449,6 +452,7 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const
     encode(i.second.front_max[0], bl);
     encode(i.second.front_max[1], bl);
     encode(i.second.front_max[2], bl);
+    encode(i.second.front_last, bl);
   }
   ENCODE_FINISH(bl);
 }
@@ -501,6 +505,7 @@ void osd_stat_t::decode(bufferlist::iterator &bl)
       decode(ifs.back_max[0],bl);
       decode(ifs.back_max[1], bl);
       decode(ifs.back_max[2], bl);
+      decode(ifs.back_last, bl);
       decode(ifs.front_pingtime[0], bl);
       decode(ifs.front_pingtime[1], bl);
       decode(ifs.front_pingtime[2], bl);
@@ -510,6 +515,7 @@ void osd_stat_t::decode(bufferlist::iterator &bl)
       decode(ifs.front_max[0], bl);
       decode(ifs.front_max[1], bl);
       decode(ifs.front_max[2], bl);
+      decode(ifs.front_last, bl);
       hb_pingtime[osd] = ifs;
     }
   }
@@ -531,11 +537,11 @@ void osd_stat_t::generate_test_instances(std::list<osd_stat_t*>& o)
   o.back()->snap_trim_queue_len = 8;
   o.back()->num_snap_trimming = 99;
   struct Interfaces gen_interfaces = {
-        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 },
-        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 } };
+        { 1000, 900, 800 }, { 990, 890, 790 }, { 1010, 910, 810 }, 1001,
+        { 1100, 1000, 900 }, { 1090, 990, 890 }, { 1110, 1010, 910 }, 1101 };
   o.back()->hb_pingtime[20] = gen_interfaces;
   gen_interfaces = {
-        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 } };
+        { 100, 200, 300 }, { 90, 190, 290 }, { 110, 210, 310 }, 101 };
   o.back()->hb_pingtime[30] = gen_interfaces;
 }
 

diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h
index df4018b11c93f04fb42d14dea6deb97a11c3d07f..6ffe6a0dcc653161d5516ef702d7dfc967ef7847 100644 (file)
--- a/src/osd/osd_types.h
+++ b/src/osd/osd_types.h
@@ -937,9 +937,11 @@ struct osd_stat_t {
     uint32_t back_pingtime[3];
     uint32_t back_min[3];
     uint32_t back_max[3];
+    uint32_t back_last;
     uint32_t front_pingtime[3];
     uint32_t front_min[3];
     uint32_t front_max[3];
+    uint32_t front_last;
   };
   map<int, Interfaces> hb_pingtime;  ///< map of osd id to Interfaces