From 5dec686effb352aa4ff82d3cd2f61879ad98c52f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 2 Oct 2017 21:35:48 -0500 Subject: [PATCH] osd: decouple statfs update from hb peers, pg count These don't need to be updated in synchrony. (In fact, the statfs update could be much more infrequent.) Signed-off-by: Sage Weil (cherry picked from commit 71d8fe7faf4952f1743103b16f590d46a9488684) --- src/osd/OSD.cc | 56 ++++++++++++++++++++++++++++++-------------------- src/osd/OSD.h | 5 ++--- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index c776a2bb5ca..fcd67367ddc 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -927,9 +927,7 @@ void OSDService::set_injectfull(s_names type, int64_t count) injectfull = count; } -osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf, - vector& hb_peers, - int num_pgs) +void OSDService::set_statfs(const struct store_statfs_t &stbuf) { uint64_t bytes = stbuf.total; uint64_t used = bytes - stbuf.available; @@ -939,16 +937,23 @@ osd_stat_t OSDService::set_osd_stat(const struct store_statfs_t &stbuf, osd->logger->set(l_osd_stat_bytes_used, used); osd->logger->set(l_osd_stat_bytes_avail, avail); - { - Mutex::Locker l(stat_lock); - osd_stat.hb_peers.swap(hb_peers); - osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist); - osd_stat.kb = bytes >> 10; - osd_stat.kb_used = used >> 10; - osd_stat.kb_avail = avail >> 10; - osd_stat.num_pgs = num_pgs; - return osd_stat; - } + Mutex::Locker l(stat_lock); + osd_stat.kb = bytes >> 10; + osd_stat.kb_used = used >> 10; + osd_stat.kb_avail = avail >> 10; + osd_stat.kb_used_data = stbuf.allocated >> 10; + osd_stat.kb_used_omap = stbuf.omap_allocated >> 10; + osd_stat.kb_used_meta = stbuf.internal_metadata >> 10; +} + +osd_stat_t OSDService::set_osd_stat(vector& hb_peers, + int num_pgs) +{ + Mutex::Locker l(stat_lock); + osd_stat.hb_peers.swap(hb_peers); + osd->op_tracker.get_age_ms_histogram(&osd_stat.op_queue_age_hist); + osd_stat.num_pgs = num_pgs; + return osd_stat; } bool OSDService::check_osdmap_full(const set &missing_on) @@ -2591,6 +2596,14 @@ int OSD::init() create_logger(); + // prime osd stats + { + struct store_statfs_t stbuf; + int r = store->statfs(&stbuf); + assert(r == 0); + service.set_statfs(stbuf); + } + // i'm ready! client_messenger->add_dispatcher_head(this); cluster_messenger->add_dispatcher_head(this); @@ -5213,20 +5226,13 @@ void OSD::heartbeat() ++p) hb_peers.push_back(p->first); - // refresh osd stats - struct store_statfs_t stbuf; - int r = store->statfs(&stbuf); - assert(r == 0); - - auto new_stat = service.set_osd_stat(stbuf, hb_peers, get_num_pgs()); - dout(20) << __func__ << new_stat << dendl; + auto new_stat = service.set_osd_stat(hb_peers, get_num_pgs()); + dout(5) << __func__ << " " << new_stat << dendl; assert(new_stat.kb); float ratio = ((float)new_stat.kb_used) / ((float)new_stat.kb); service.check_full_status(ratio); - dout(5) << "heartbeat: " << service.get_osd_stat() << dendl; - utime_t now = ceph_clock_now(); // send heartbeats @@ -5350,6 +5356,12 @@ void OSD::tick_without_osd_lock() logger->set(l_osd_missed_crc, buffer::get_missed_crc()); logger->set(l_osd_pg_removing, remove_wq.get_remove_queue_len()); + // refresh osd stats + struct store_statfs_t stbuf; + int r = store->statfs(&stbuf); + assert(r == 0); + service.set_statfs(stbuf); + // osd_lock is not being held, which means the OSD state // might change when doing the monitor report if (is_active() || is_waiting_for_healthy()) { diff --git a/src/osd/OSD.h b/src/osd/OSD.h index b5c896ef347..24b4200d1ec 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -1057,9 +1057,8 @@ public: osd_stat_t osd_stat; uint32_t seq = 0; - osd_stat_t set_osd_stat(const struct store_statfs_t &stbuf, - vector& hb_peers, - int num_pgs); + void set_statfs(const struct store_statfs_t &stbuf); + osd_stat_t set_osd_stat(vector& hb_peers, int num_pgs); osd_stat_t get_osd_stat() { Mutex::Locker l(stat_lock); ++seq; -- 2.47.3