osd: decouple statfs update from hb peers, pg count

These don't need to be updated in synchrony.  (In fact, the statfs update
could be much more infrequent.)

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 71d8fe7faf4952f1743103b16f590d46a9488684)

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index c776a2bb5ca33ae929ee1b26ea8bfc2a8e26da6b..fcd67367ddcb0301c2605f763dbedebceb81834a 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -927,9 +927,7 @@ void OSDService::set_injectfull(s_names type, int64_t count)
   injectfull = count;
 }
 
-osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf,
-                                    vector<int>& hb_peers,
-                                   int num_pgs)
+void OSDService::set_statfs(const struct store_statfs_t &stbuf)
 {
   uint64_t bytes = stbuf.total;
   uint64_t used = bytes - stbuf.available;
@@ -939,16 +937,23 @@ osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf,
   osd->logger->set(l_osd_stat_bytes_used, used);
   osd->logger->set(l_osd_stat_bytes_avail, avail);
 
-  {
-    Mutex::Locker l(stat_lock);
-    osd_stat.hb_peers.swap(hb_peers);
-    osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
-    osd_stat.kb = bytes >> 10;
-    osd_stat.kb_used = used >> 10;
-    osd_stat.kb_avail = avail >> 10;
-    osd_stat.num_pgs = num_pgs;
-    return osd_stat;
-  }
+  Mutex::Locker l(stat_lock);
+  osd_stat.kb = bytes >> 10;
+  osd_stat.kb_used = used >> 10;
+  osd_stat.kb_avail = avail >> 10;
+  osd_stat.kb_used_data = stbuf.allocated >> 10;
+  osd_stat.kb_used_omap = stbuf.omap_allocated >> 10;
+  osd_stat.kb_used_meta = stbuf.internal_metadata >> 10;
+}
+
+osd_stat_t OSDService::set_osd_stat(vector<int>& hb_peers,
+                                   int num_pgs)
+{
+  Mutex::Locker l(stat_lock);
+  osd_stat.hb_peers.swap(hb_peers);
+  osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
+  osd_stat.num_pgs = num_pgs;
+  return osd_stat;
 }
 
 bool OSDService::check_osdmap_full(const set<pg_shard_t> &missing_on)
@@ -2591,6 +2596,14 @@ int OSD::init()
 
   create_logger();
 
+  // prime osd stats
+  {
+    struct store_statfs_t stbuf;
+    int r = store->statfs(&stbuf);
+    assert(r == 0);
+    service.set_statfs(stbuf);
+  }
+
   // i'm ready!
   client_messenger->add_dispatcher_head(this);
   cluster_messenger->add_dispatcher_head(this);
@@ -5213,20 +5226,13 @@ void OSD::heartbeat()
        ++p)
     hb_peers.push_back(p->first);
 
-  // refresh osd stats
-  struct store_statfs_t stbuf;
-  int r = store->statfs(&stbuf);
-  assert(r == 0);
-
-  auto new_stat = service.set_osd_stat(stbuf, hb_peers, get_num_pgs());
-  dout(20) << __func__ << new_stat << dendl;
+  auto new_stat = service.set_osd_stat(hb_peers, get_num_pgs());
+  dout(5) << __func__ << " " << new_stat << dendl;
   assert(new_stat.kb);
 
   float ratio = ((float)new_stat.kb_used) / ((float)new_stat.kb);
   service.check_full_status(ratio);
 
-  dout(5) << "heartbeat: " << service.get_osd_stat() << dendl;
-
   utime_t now = ceph_clock_now();
 
   // send heartbeats
@@ -5350,6 +5356,12 @@ void OSD::tick_without_osd_lock()
   logger->set(l_osd_missed_crc, buffer::get_missed_crc());
   logger->set(l_osd_pg_removing, remove_wq.get_remove_queue_len());
 
+  // refresh osd stats
+  struct store_statfs_t stbuf;
+  int r = store->statfs(&stbuf);
+  assert(r == 0);
+  service.set_statfs(stbuf);
+
   // osd_lock is not being held, which means the OSD state
   // might change when doing the monitor report
   if (is_active() || is_waiting_for_healthy()) {

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index b5c896ef347602b3403d152cbaa676b6ecba6e7b..24b4200d1ecc8cf47ed30e786d478fda8c4fb8d2 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -1057,9 +1057,8 @@ public:
   osd_stat_t osd_stat;
   uint32_t seq = 0;
 
-  osd_stat_t set_osd_stat(const struct store_statfs_t &stbuf,
-                          vector<int>& hb_peers,
-                         int num_pgs);
+  void set_statfs(const struct store_statfs_t &stbuf);
+  osd_stat_t set_osd_stat(vector<int>& hb_peers, int num_pgs);
   osd_stat_t get_osd_stat() {
     Mutex::Locker l(stat_lock);
     ++seq;