osd: decouple statfs update from hb peers, pg count

These don't need to be updated in synchrony.  (In fact, the statfs update
could be much more infrequent.)

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 47f62e02ea17bdb9dde6c9e778ff8d93f4016559..dd7ec8a32f481e5913f0c8bad15bc31274f99e27 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -786,9 +786,7 @@ void OSDService::set_injectfull(s_names type, int64_t count)
   injectfull = count;
 }
 
-osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf,
-                                    vector<int>& hb_peers,
-                                   int num_pgs)
+void OSDService::set_statfs(const struct store_statfs_t &stbuf)
 {
   uint64_t bytes = stbuf.total;
   uint64_t used = bytes - stbuf.available;
@@ -798,16 +796,23 @@ osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf,
   osd->logger->set(l_osd_stat_bytes_used, used);
   osd->logger->set(l_osd_stat_bytes_avail, avail);
 
-  {
-    Mutex::Locker l(stat_lock);
-    osd_stat.hb_peers.swap(hb_peers);
-    osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
-    osd_stat.kb = bytes >> 10;
-    osd_stat.kb_used = used >> 10;
-    osd_stat.kb_avail = avail >> 10;
-    osd_stat.num_pgs = num_pgs;
-    return osd_stat;
-  }
+  Mutex::Locker l(stat_lock);
+  osd_stat.kb = bytes >> 10;
+  osd_stat.kb_used = used >> 10;
+  osd_stat.kb_avail = avail >> 10;
+  osd_stat.kb_used_data = stbuf.allocated >> 10;
+  osd_stat.kb_used_omap = stbuf.omap_allocated >> 10;
+  osd_stat.kb_used_meta = stbuf.internal_metadata >> 10;
+}
+
+osd_stat_t OSDService::set_osd_stat(vector<int>& hb_peers,
+                                   int num_pgs)
+{
+  Mutex::Locker l(stat_lock);
+  osd_stat.hb_peers.swap(hb_peers);
+  osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist);
+  osd_stat.num_pgs = num_pgs;
+  return osd_stat;
 }
 
 bool OSDService::check_osdmap_full(const set<pg_shard_t> &missing_on)
@@ -2559,6 +2564,14 @@ int OSD::init()
 
   create_logger();
 
+  // prime osd stats
+  {
+    struct store_statfs_t stbuf;
+    int r = store->statfs(&stbuf);
+    assert(r == 0);
+    service.set_statfs(stbuf);
+  }
+
   // i'm ready!
   client_messenger->add_dispatcher_head(this);
   cluster_messenger->add_dispatcher_head(this);
@@ -4696,20 +4709,13 @@ void OSD::heartbeat()
        ++p)
     hb_peers.push_back(p->first);
 
-  // refresh osd stats
-  struct store_statfs_t stbuf;
-  int r = store->statfs(&stbuf);
-  assert(r == 0);
-
-  auto new_stat = service.set_osd_stat(stbuf, hb_peers, get_num_pgs());
-  dout(20) << __func__ << new_stat << dendl;
+  auto new_stat = service.set_osd_stat(hb_peers, get_num_pgs());
+  dout(5) << __func__ << " " << new_stat << dendl;
   assert(new_stat.kb);
 
   float ratio = ((float)new_stat.kb_used) / ((float)new_stat.kb);
   service.check_full_status(ratio);
 
-  dout(5) << "heartbeat: " << service.get_osd_stat() << dendl;
-
   utime_t now = ceph_clock_now();
   utime_t deadline = now;
   deadline += cct->_conf->osd_heartbeat_grace;
@@ -4831,6 +4837,12 @@ void OSD::tick_without_osd_lock()
   logger->set(l_osd_cached_crc_adjusted, buffer::get_cached_crc_adjusted());
   logger->set(l_osd_missed_crc, buffer::get_missed_crc());
 
+  // refresh osd stats
+  struct store_statfs_t stbuf;
+  int r = store->statfs(&stbuf);
+  assert(r == 0);
+  service.set_statfs(stbuf);
+
   // osd_lock is not being held, which means the OSD state
   // might change when doing the monitor report
   if (is_active() || is_waiting_for_healthy()) {

diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index ea58807a632bad4643553466a77422a45c85165e..21bb7d236bab52d91ed0e6bd231a78522c22cebe 100644 (file)
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -881,9 +881,8 @@ public:
   osd_stat_t osd_stat;
   uint32_t seq = 0;
 
-  osd_stat_t set_osd_stat(const struct store_statfs_t &stbuf,
-                          vector<int>& hb_peers,
-                         int num_pgs);
+  void set_statfs(const struct store_statfs_t &stbuf);
+  osd_stat_t set_osd_stat(vector<int>& hb_peers, int num_pgs);
   osd_stat_t get_osd_stat() {
     Mutex::Locker l(stat_lock);
     ++seq;