From 7ca25dfd5a4f05d1f942b73f08e82b8694a73c99 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 6 Mar 2018 14:19:54 +0300 Subject: [PATCH] osd,os,mon: extend 'ceph df' report to provide both USED and RAW_USED totals where USED is space allocated for object keeping while RAW_USED also includes space allocated/reserved for internal purposes, e.g. DB/WAL or journal. Signed-off-by: Igor Fedotv --- src/mon/MgrStatMonitor.cc | 6 +- src/mon/PGMap.cc | 105 ++++++++------- src/os/bluestore/BlueStore.cc | 32 +++-- src/os/bluestore/BlueStore.h | 1 + src/os/filestore/FileStore.cc | 1 + src/osd/OSD.cc | 14 +- src/osd/OSDMap.cc | 42 +++--- src/osd/osd_types.cc | 57 +++++++-- src/osd/osd_types.h | 184 ++++++++++++++------------- src/test/mon/test_mon_workloadgen.cc | 6 +- 10 files changed, 252 insertions(+), 196 deletions(-) diff --git a/src/mon/MgrStatMonitor.cc b/src/mon/MgrStatMonitor.cc index caf1dcb04a862..006ef4231e3b5 100644 --- a/src/mon/MgrStatMonitor.cc +++ b/src/mon/MgrStatMonitor.cc @@ -65,11 +65,11 @@ void MgrStatMonitor::update_logger() { dout(20) << __func__ << dendl; - mon->cluster_logger->set(l_cluster_osd_bytes, digest.osd_sum.kb * 1024ull); + mon->cluster_logger->set(l_cluster_osd_bytes, digest.osd_sum.statfs.total); mon->cluster_logger->set(l_cluster_osd_bytes_used, - digest.osd_sum.kb_used * 1024ull); + digest.osd_sum.statfs.get_used_raw()); mon->cluster_logger->set(l_cluster_osd_bytes_avail, - digest.osd_sum.kb_avail * 1024ull); + digest.osd_sum.statfs.available); mon->cluster_logger->set(l_cluster_num_pool, digest.pg_pool_sum.size()); uint64_t num_pg = 0; diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index a4c1b1731dde8..ce01dc7f4d537 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -208,18 +208,18 @@ void PGMapDigest::print_summary(Formatter *f, ostream *out) const f->dump_unsigned("num_pools", pg_pool_sum.size()); f->dump_unsigned("num_objects", pg_sum.stats.sum.num_objects); f->dump_unsigned("data_bytes", pg_sum.stats.sum.num_bytes); - f->dump_unsigned("bytes_used", osd_sum.kb_used * 1024ull); - f->dump_unsigned("bytes_avail", osd_sum.kb_avail * 1024ull); - f->dump_unsigned("bytes_total", osd_sum.kb * 1024ull); + f->dump_unsigned("bytes_used", osd_sum.statfs.get_used_raw()); + f->dump_unsigned("bytes_avail", osd_sum.statfs.available); + f->dump_unsigned("bytes_total", osd_sum.statfs.total); } else { *out << " pools: " << pg_pool_sum.size() << " pools, " << num_pg << " pgs\n"; *out << " objects: " << si_u_t(pg_sum.stats.sum.num_objects) << " objects, " << byte_u_t(pg_sum.stats.sum.num_bytes) << "\n"; *out << " usage: " - << byte_u_t(osd_sum.kb_used << 10) << " used, " - << byte_u_t(osd_sum.kb_avail << 10) << " / " - << byte_u_t(osd_sum.kb << 10) << " avail\n"; + << byte_u_t(osd_sum.statfs.get_used_raw()) << " used, " + << byte_u_t(osd_sum.statfs.available) << " / " + << byte_u_t(osd_sum.statfs.total) << " avail\n"; *out << " pgs: "; } @@ -340,15 +340,15 @@ void PGMapDigest::print_oneline_summary(Formatter *f, ostream *out) const *out << num_pg << " pgs: " << states << "; " << byte_u_t(pg_sum.stats.sum.num_bytes) << " data, " - << byte_u_t(osd_sum.kb_used << 10) << " used, " - << byte_u_t(osd_sum.kb_avail << 10) << " / " - << byte_u_t(osd_sum.kb << 10) << " avail"; + << byte_u_t(osd_sum.statfs.get_used_raw()) << " used, " + << byte_u_t(osd_sum.statfs.available) << " / " + << byte_u_t(osd_sum.statfs.total) << " avail"; if (f) { f->dump_unsigned("num_pgs", num_pg); f->dump_unsigned("num_bytes", pg_sum.stats.sum.num_bytes); - f->dump_unsigned("raw_bytes_used", osd_sum.kb_used << 10); - f->dump_unsigned("raw_bytes_avail", osd_sum.kb_avail << 10); - f->dump_unsigned("raw_bytes", osd_sum.kb << 10); + f->dump_unsigned("raw_bytes_used", osd_sum.statfs.get_used_raw()); + f->dump_unsigned("raw_bytes_avail", osd_sum.statfs.available); + f->dump_unsigned("raw_bytes", osd_sum.statfs.total); } // make non-negative; we can get negative values if osds send @@ -740,9 +740,9 @@ ceph_statfs PGMapDigest::get_statfs(OSDMap &osdmap, statfs.kb = statfs.kb_used + statfs.kb_avail; } else { // these are in KB. - statfs.kb = osd_sum.kb; - statfs.kb_used = osd_sum.kb_used; - statfs.kb_avail = osd_sum.kb_avail; + statfs.kb = osd_sum.statfs.kb(); + statfs.kb_used = osd_sum.statfs.kb_used_raw(); + statfs.kb_avail = osd_sum.statfs.kb_avail(); statfs.num_objects = pg_sum.stats.sum.num_objects; } @@ -848,16 +848,19 @@ void PGMapDigest::dump_pool_stats_full( void PGMapDigest::dump_fs_stats(stringstream *ss, Formatter *f, bool verbose) const { + auto total = osd_sum.statfs.total; + auto used_raw = osd_sum.statfs.get_used_raw(); float used = 0.0; - if (osd_sum.kb > 0) { - used = ((float)osd_sum.kb_used / osd_sum.kb); + if (total > 0) { + used = ((float)used_raw / total); } if (f) { f->open_object_section("stats"); - f->dump_int("total_bytes", osd_sum.kb * 1024ull); - f->dump_int("total_used_bytes", osd_sum.kb_used * 1024ull); - f->dump_int("total_avail_bytes", osd_sum.kb_avail * 1024ull); + f->dump_int("total_bytes", total); + f->dump_int("total_avail_bytes", osd_sum.statfs.available); + f->dump_int("total_used_bytes", osd_sum.statfs.get_used()); + f->dump_int("total_used_raw_bytes", used_raw); f->dump_float("total_percent_used", used * 100); if (verbose) { f->dump_int("total_objects", pg_sum.stats.sum.num_objects); @@ -868,14 +871,16 @@ void PGMapDigest::dump_fs_stats(stringstream *ss, Formatter *f, bool verbose) co TextTable tbl; tbl.define_column("SIZE", TextTable::LEFT, TextTable::RIGHT); tbl.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT); + tbl.define_column("USED", TextTable::LEFT, TextTable::RIGHT); tbl.define_column("RAW USED", TextTable::LEFT, TextTable::RIGHT); tbl.define_column("%RAW USED", TextTable::LEFT, TextTable::RIGHT); if (verbose) { tbl.define_column("OBJECTS", TextTable::LEFT, TextTable::RIGHT); } - tbl << stringify(byte_u_t(osd_sum.kb*1024)) - << stringify(byte_u_t(osd_sum.kb_avail*1024)) - << stringify(byte_u_t(osd_sum.kb_used*1024)); + tbl << stringify(byte_u_t(total)) + << stringify(byte_u_t(osd_sum.statfs.available)) + << stringify(byte_u_t(osd_sum.statfs.get_used())) + << stringify(byte_u_t(used_raw)); tbl << percentify(used*100); if (verbose) { tbl << stringify(si_u_t(pg_sum.stats.sum.num_objects)); @@ -913,8 +918,7 @@ void PGMapDigest::dump_object_stat_sum( } auto avail_res = raw_used_rate ? avail / raw_used_rate : 0; // an approximation for actually stored user data - auto stored_normalized = - raw_used_rate ? statfs.data_stored / raw_used_rate : 0; + auto stored_normalized = pool_stat.get_user_bytes(raw_used_rate); if (f) { f->dump_int("kb_used", shift_round_up(used_bytes, 10)); f->dump_int("bytes_used", used_bytes); @@ -933,7 +937,7 @@ void PGMapDigest::dump_object_stat_sum( f->dump_int("compress_bytes_used", statfs.data_compressed_allocated); f->dump_int("compress_under_bytes", statfs.data_compressed_original); // Stored by user amplified by replication - f->dump_int("stored_raw", statfs.data_stored); + f->dump_int("stored_raw", pool_stat.get_user_bytes(1.0)); } } else { tbl << stringify(byte_u_t(statfs.allocated)); @@ -984,7 +988,7 @@ int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const for (auto p = wm.begin(); p != wm.end(); ++p) { auto osd_info = osd_stat.find(p->first); if (osd_info != osd_stat.end()) { - if (osd_info->second.kb == 0 || p->second == 0) { + if (osd_info->second.statfs.total == 0 || p->second == 0) { // osd must be out, hence its stats have been zeroed // (unless we somehow managed to have a disk with size 0...) // @@ -992,9 +996,9 @@ int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const // calculate proj below. continue; } - double unusable = (double)osd_info->second.kb * + double unusable = (double)osd_info->second.statfs.kb() * (1.0 - fratio); - double avail = std::max(0.0, (double)osd_info->second.kb_avail - unusable); + double avail = std::max(0.0, (double)osd_info->second.statfs.kb_avail() - unusable); avail *= 1024.0; int64_t proj = (int64_t)(avail / (double)p->second); if (min < 0 || proj < min) { @@ -1745,6 +1749,7 @@ void PGMap::dump_osd_stats(ostream& ss) const tab.define_column("OSD_STAT", TextTable::LEFT, TextTable::LEFT); tab.define_column("USED", TextTable::LEFT, TextTable::RIGHT); tab.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("USED_RAW", TextTable::LEFT, TextTable::RIGHT); tab.define_column("TOTAL", TextTable::LEFT, TextTable::RIGHT); tab.define_column("HB_PEERS", TextTable::LEFT, TextTable::RIGHT); tab.define_column("PG_SUM", TextTable::LEFT, TextTable::RIGHT); @@ -1754,9 +1759,10 @@ void PGMap::dump_osd_stats(ostream& ss) const p != osd_stat.end(); ++p) { tab << p->first - << byte_u_t(p->second.kb_used << 10) - << byte_u_t(p->second.kb_avail << 10) - << byte_u_t(p->second.kb << 10) + << byte_u_t(p->second.statfs.get_used()) + << byte_u_t(p->second.statfs.available) + << byte_u_t(p->second.statfs.get_used_raw()) + << byte_u_t(p->second.statfs.total) << p->second.hb_peers << get_num_pg_by_osd(p->first) << get_num_primary_pg_by_osd(p->first) @@ -1764,9 +1770,10 @@ void PGMap::dump_osd_stats(ostream& ss) const } tab << "sum" - << byte_u_t(osd_sum.kb_used << 10) - << byte_u_t(osd_sum.kb_avail << 10) - << byte_u_t(osd_sum.kb << 10) + << byte_u_t(osd_sum.statfs.get_used()) + << byte_u_t(osd_sum.statfs.available) + << byte_u_t(osd_sum.statfs.get_used_raw()) + << byte_u_t(osd_sum.statfs.total) << TextTable::endrow; ss << tab; @@ -1779,12 +1786,14 @@ void PGMap::dump_osd_sum_stats(ostream& ss) const tab.define_column("OSD_STAT", TextTable::LEFT, TextTable::LEFT); tab.define_column("USED", TextTable::LEFT, TextTable::RIGHT); tab.define_column("AVAIL", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("USED_RAW", TextTable::LEFT, TextTable::RIGHT); tab.define_column("TOTAL", TextTable::LEFT, TextTable::RIGHT); tab << "sum" - << byte_u_t(osd_sum.kb_used << 10) - << byte_u_t(osd_sum.kb_avail << 10) - << byte_u_t(osd_sum.kb << 10) + << byte_u_t(osd_sum.statfs.get_used()) + << byte_u_t(osd_sum.statfs.available) + << byte_u_t(osd_sum.statfs.get_used_raw()) + << byte_u_t(osd_sum.statfs.total) << TextTable::endrow; ss << tab; @@ -3367,7 +3376,7 @@ void PGMapUpdater::check_osd_map( pending_inc->rm_stat(p.first); } else if (osdmap.is_out(p.first)) { // zero osd_stat - if (p.second.kb != 0) { + if (p.second.statfs.total != 0) { pending_inc->stat_osd_out(p.first); } } else if (!osdmap.is_up(p.first)) { @@ -3589,20 +3598,22 @@ int reweight::by_utilization( } else { // by osd utilization int num_osd = std::max(1, pgm.osd_stat.size()); - if ((uint64_t)pgm.osd_sum.kb * 1024 / num_osd + if ((uint64_t)pgm.osd_sum.statfs.total / num_osd < g_conf()->mon_reweight_min_bytes_per_osd) { - *ss << "Refusing to reweight: we only have " << pgm.osd_sum.kb + *ss << "Refusing to reweight: we only have " << pgm.osd_sum.statfs.kb() << " kb across all osds!\n"; return -EDOM; } - if ((uint64_t)pgm.osd_sum.kb_used * 1024 / num_osd + if ((uint64_t)pgm.osd_sum.statfs.get_used_raw() / num_osd < g_conf()->mon_reweight_min_bytes_per_osd) { - *ss << "Refusing to reweight: we only have " << pgm.osd_sum.kb_used + *ss << "Refusing to reweight: we only have " + << pgm.osd_sum.statfs.kb_used_raw() << " kb used across all osds!\n"; return -EDOM; } - average_util = (double)pgm.osd_sum.kb_used / (double)pgm.osd_sum.kb; + average_util = (double)pgm.osd_sum.statfs.get_used_raw() / + (double)pgm.osd_sum.statfs.total; } // adjust down only if we are above the threshold @@ -3649,9 +3660,11 @@ int reweight::by_utilization( continue; } - osd_util.second = pgs_by_osd[p.first] / osdmap.crush->get_item_weightf(p.first); + osd_util.second = + pgs_by_osd[p.first] / osdmap.crush->get_item_weightf(p.first); } else { - osd_util.second = (double)p.second.kb_used / (double)p.second.kb; + osd_util.second = + (double)p.second.statfs.get_used_raw() / (double)p.second.statfs.total; } util_by_osd.push_back(osd_util); } diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 4f57253b982d7..189566a5b6f38 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6666,6 +6666,7 @@ int BlueStore::_fsck(bool deep, bool repair) // structs statfs(&actual_statfs); expected_statfs.total = actual_statfs.total; + expected_statfs.internally_reserved = actual_statfs.internally_reserved; expected_statfs.available = actual_statfs.available; expected_statfs.internal_metadata = actual_statfs.internal_metadata; expected_statfs.omap_allocated = actual_statfs.omap_allocated; @@ -7591,7 +7592,7 @@ int BlueStore::get_devices(set *ls) return 0; } -int BlueStore::statfs(struct store_statfs_t *buf) +void BlueStore::_get_statfs_overall(struct store_statfs_t *buf) { buf->reset(); @@ -7600,11 +7601,14 @@ int BlueStore::statfs(struct store_statfs_t *buf) uint64_t bfree = alloc->get_free(); if (bluefs) { + int64_t bluefs_total = bluefs->get_total(bluefs_shared_bdev); + int64_t bluefs_free = bluefs->get_free(bluefs_shared_bdev); // part of our shared device is "free" according to BlueFS, but we // can't touch bluestore_bluefs_min of it. int64_t shared_available = std::min( - bluefs->get_free(bluefs_shared_bdev), - bluefs->get_total(bluefs_shared_bdev) - cct->_conf->bluestore_bluefs_min); + bluefs_free, + int64_t(bluefs_total - cct->_conf->bluestore_bluefs_min)); + buf->internally_reserved = bluefs_total - shared_available; if (shared_available > 0) { bfree += shared_available; } @@ -7618,15 +7622,6 @@ int BlueStore::statfs(struct store_statfs_t *buf) - buf->omap_allocated; } - { - std::lock_guard l(vstatfs_lock); - buf->allocated = vstatfs.allocated(); - buf->data_stored = vstatfs.stored(); - buf->data_compressed = vstatfs.compressed(); - buf->data_compressed_original = vstatfs.compressed_original(); - buf->data_compressed_allocated = vstatfs.compressed_allocated(); - } - uint64_t thin_total, thin_avail; if (bdev->get_thin_utilization(&thin_total, &thin_avail)) { buf->total += thin_total; @@ -7640,6 +7635,19 @@ int BlueStore::statfs(struct store_statfs_t *buf) buf->total += bdev->get_size(); } buf->available = bfree; +} + +int BlueStore::statfs(struct store_statfs_t *buf) +{ + _get_statfs_overall(buf); + { + std::lock_guard l(vstatfs_lock); + buf->allocated = vstatfs.allocated(); + buf->data_stored = vstatfs.stored(); + buf->data_compressed = vstatfs.compressed(); + buf->data_compressed_original = vstatfs.compressed_original(); + buf->data_compressed_allocated = vstatfs.compressed_allocated(); + } dout(20) << __func__ << " " << *buf << dendl; return 0; diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 4f89ceeae763b..f897038d7b2ad 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2176,6 +2176,7 @@ private: int _open_super_meta(); void _open_statfs(); + void _get_statfs_overall(struct store_statfs_t *buf); void _dump_alloc_on_rebalance_failure(); int _reconcile_bluefs_freespace(); diff --git a/src/os/filestore/FileStore.cc b/src/os/filestore/FileStore.cc index 211b26fc09f9c..cd03f6e36dd0f 100644 --- a/src/os/filestore/FileStore.cc +++ b/src/os/filestore/FileStore.cc @@ -769,6 +769,7 @@ int FileStore::statfs(struct store_statfs_t *buf0) // Adjust for writes pending in the journal if (journal) { uint64_t estimate = journal->get_journal_size_estimate(); + buf0->internally_reserved = estimate; if (buf0->available > estimate) buf0->available -= estimate; else diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 4ac1b38905205..dc43e5a4c3972 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -858,20 +858,15 @@ void OSDService::set_injectfull(s_names type, int64_t count) void OSDService::set_statfs(const struct store_statfs_t &stbuf) { uint64_t bytes = stbuf.total; - uint64_t used = bytes - stbuf.available; uint64_t avail = stbuf.available; + uint64_t used = stbuf.get_used_raw(); osd->logger->set(l_osd_stat_bytes, bytes); osd->logger->set(l_osd_stat_bytes_used, used); osd->logger->set(l_osd_stat_bytes_avail, avail); std::lock_guard l(stat_lock); - osd_stat.kb = bytes >> 10; - osd_stat.kb_used = used >> 10; - osd_stat.kb_avail = avail >> 10; - osd_stat.kb_used_data = stbuf.allocated >> 10; - osd_stat.kb_used_omap = stbuf.omap_allocated >> 10; - osd_stat.kb_used_meta = stbuf.internal_metadata >> 10; + osd_stat.statfs = stbuf; } osd_stat_t OSDService::set_osd_stat(vector& hb_peers, @@ -4949,9 +4944,10 @@ void OSD::heartbeat() auto new_stat = service.set_osd_stat(hb_peers, get_num_pgs()); dout(5) << __func__ << " " << new_stat << dendl; - ceph_assert(new_stat.kb); + ceph_assert(new_stat.statfs.total); - float ratio = ((float)new_stat.kb_used) / ((float)new_stat.kb); + float ratio = + ((float)new_stat.statfs.get_used()) / ((float)new_stat.statfs.total); service.check_full_status(ratio); utime_t now = ceph_clock_now(); diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc index 4367ea39c5545..676638fdfbc43 100644 --- a/src/osd/OSDMap.cc +++ b/src/osd/OSDMap.cc @@ -4617,12 +4617,13 @@ protected: int64_t* kb_avail) const { const osd_stat_t *p = pgmap.get_osd_stat(id); if (!p) return false; - *kb = p->kb; - *kb_used = p->kb_used; - *kb_used_data = p->kb_used_data; - *kb_used_omap = p->kb_used_omap; - *kb_used_meta = p->kb_used_meta; - *kb_avail = p->kb_avail; + *kb = p->statfs.kb(); + *kb_used = p->statfs.kb_used_raw(); + *kb_used_data = p->statfs.kb_used_data(); + *kb_used_omap = p->statfs.kb_used_omap(); + *kb_used_meta = p->statfs.kb_used_internal_metadata(); + *kb_avail = p->statfs.kb_avail(); + return *kb > 0; } @@ -4696,7 +4697,7 @@ public: tbl->define_column("WEIGHT", TextTable::LEFT, TextTable::RIGHT); tbl->define_column("REWEIGHT", TextTable::LEFT, TextTable::RIGHT); tbl->define_column("SIZE", TextTable::LEFT, TextTable::RIGHT); - tbl->define_column("USE", TextTable::LEFT, TextTable::RIGHT); + tbl->define_column("RAW USE", TextTable::LEFT, TextTable::RIGHT); tbl->define_column("DATA", TextTable::LEFT, TextTable::RIGHT); tbl->define_column("OMAP", TextTable::LEFT, TextTable::RIGHT); tbl->define_column("META", TextTable::LEFT, TextTable::RIGHT); @@ -4715,12 +4716,12 @@ public: *tbl << "" << "" << "" << "TOTAL" - << byte_u_t(pgmap.get_osd_sum().kb << 10) - << byte_u_t(pgmap.get_osd_sum().kb_used << 10) - << byte_u_t(pgmap.get_osd_sum().kb_used_data << 10) - << byte_u_t(pgmap.get_osd_sum().kb_used_omap << 10) - << byte_u_t(pgmap.get_osd_sum().kb_used_meta << 10) - << byte_u_t(pgmap.get_osd_sum().kb_avail << 10) + << byte_u_t(pgmap.get_osd_sum().statfs.total) + << byte_u_t(pgmap.get_osd_sum().statfs.get_used_raw()) + << byte_u_t(pgmap.get_osd_sum().statfs.allocated) + << byte_u_t(pgmap.get_osd_sum().statfs.omap_allocated) + << byte_u_t(pgmap.get_osd_sum().statfs.internal_metadata) + << byte_u_t(pgmap.get_osd_sum().statfs.available) << lowprecision_t(average_util) << "" << TextTable::endrow; @@ -4876,13 +4877,14 @@ protected: public: void summary(Formatter *f) { f->open_object_section("summary"); - auto& s = pgmap.get_osd_sum(); - f->dump_int("total_kb", s.kb); - f->dump_int("total_kb_used", s.kb_used); - f->dump_int("total_kb_used_data", s.kb_used_data); - f->dump_int("total_kb_used_omap", s.kb_used_omap); - f->dump_int("total_kb_used_meta", s.kb_used_meta); - f->dump_int("total_kb_avail", s.kb_avail); + auto& s = pgmap.get_osd_sum().statfs; + + f->dump_int("total_kb", s.kb()); + f->dump_int("total_kb_used", s.kb_used_raw()); + f->dump_int("total_kb_used_data", s.kb_used_data()); + f->dump_int("total_kb_used_omap", s.kb_used_omap()); + f->dump_int("total_kb_used_meta", s.kb_used_internal_metadata()); + f->dump_int("total_kb_avail", s.kb_avail()); f->dump_float("average_utilization", average_util); f->dump_float("min_var", min_var); f->dump_float("max_var", max_var); diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 2be1b862aca95..539689ad10f27 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -344,12 +344,9 @@ void osd_stat_t::dump(Formatter *f) const f->dump_unsigned("up_from", up_from); f->dump_unsigned("seq", seq); f->dump_unsigned("num_pgs", num_pgs); - f->dump_unsigned("kb", kb); - f->dump_unsigned("kb_used", kb_used); - f->dump_unsigned("kb_used_data", kb_used_data); - f->dump_unsigned("kb_used_omap", kb_used_omap); - f->dump_unsigned("kb_used_meta", kb_used_meta); - f->dump_unsigned("kb_avail", kb_avail); + f->open_object_section("statfs"); + statfs.dump(f); + f->close_section(); f->open_array_section("hb_peers"); for (auto p : hb_peers) f->dump_int("osd", p); @@ -366,10 +363,17 @@ void osd_stat_t::dump(Formatter *f) const void osd_stat_t::encode(bufferlist &bl, uint64_t features) const { - ENCODE_START(8, 2, bl); + ENCODE_START(9, 2, bl); + + //////// for compatibility //////// + int64_t kb = statfs.kb(); + int64_t kb_used = statfs.kb_used_raw(); + int64_t kb_avail = statfs.kb_avail(); encode(kb, bl); encode(kb_used, bl); encode(kb_avail, bl); + /////////////////////////////////// + encode(snap_trim_queue_len, bl); encode(num_snap_trimming, bl); encode(hb_peers, bl); @@ -379,15 +383,25 @@ void osd_stat_t::encode(bufferlist &bl, uint64_t features) const encode(up_from, bl); encode(seq, bl); encode(num_pgs, bl); + + //////// for compatibility //////// + int64_t kb_used_data = statfs.kb_used_data(); + int64_t kb_used_omap = statfs.kb_used_omap(); + int64_t kb_used_meta = statfs.kb_used_internal_metadata(); encode(kb_used_data, bl); encode(kb_used_omap, bl); encode(kb_used_meta, bl); + encode(statfs, bl); + /////////////////////////////////// + ENCODE_FINISH(bl); } void osd_stat_t::decode(bufferlist::const_iterator &bl) { - DECODE_START_LEGACY_COMPAT_LEN(8, 2, 2, bl); + int64_t kb, kb_used,kb_avail; + int64_t kb_used_data, kb_used_omap, kb_used_meta; + DECODE_START_LEGACY_COMPAT_LEN(9, 2, 2, bl); decode(kb, bl); decode(kb_used, bl); decode(kb_avail, bl); @@ -416,6 +430,24 @@ void osd_stat_t::decode(bufferlist::const_iterator &bl) kb_used_omap = 0; kb_used_meta = 0; } + if (struct_v >= 9) { + decode(statfs, bl); + } else { + statfs.reset(); + statfs.total = kb << 10; + statfs.available = kb_avail << 10; + assert(statfs.total >= statfs.available); + statfs.internally_reserved = statfs.total - statfs.available; + kb_used <<= 10; + if ((int64_t)statfs.internally_reserved > kb_used) { + statfs.internally_reserved -= kb_used; + } else { + statfs.internally_reserved = 0; + } + statfs.allocated = kb_used_data << 10; + statfs.omap_allocated = kb_used_omap << 10; + statfs.internal_metadata = kb_used_meta << 10; + } DECODE_FINISH(bl); } @@ -424,12 +456,9 @@ void osd_stat_t::generate_test_instances(std::list& o) o.push_back(new osd_stat_t); o.push_back(new osd_stat_t); - o.back()->kb = 9; - o.back()->kb_used = 6; - o.back()->kb_used_data = 3; - o.back()->kb_used_omap = 2; - o.back()->kb_used_meta = 1; - o.back()->kb_avail = 3; + list ll; + store_statfs_t::generate_test_instances(ll); + o.back()->statfs = *ll.back(); o.back()->hb_peers.push_back(7); o.back()->snap_trim_queue_len = 8; o.back()->num_snap_trimming = 99; diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 17b92812ebd5b..3336db4e926ba 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -921,95 +921,6 @@ struct objectstore_perf_stat_t { }; WRITE_CLASS_ENCODER_FEATURES(objectstore_perf_stat_t) -/** osd_stat - * aggregate stats for an osd - */ -struct osd_stat_t { - int64_t kb = 0; ///< total device size - int64_t kb_used = 0; ///< total used - int64_t kb_used_data = 0; ///< total used by object data - int64_t kb_used_omap = 0; ///< total used by omap data - int64_t kb_used_meta = 0; ///< total used by internal metadata - int64_t kb_avail = 0; ///< total available/free - - vector hb_peers; - int32_t snap_trim_queue_len = 0, num_snap_trimming = 0; - - pow2_hist_t op_queue_age_hist; - - objectstore_perf_stat_t os_perf_stat; - - epoch_t up_from = 0; - uint64_t seq = 0; - - uint32_t num_pgs = 0; - - void add(const osd_stat_t& o) { - kb += o.kb; - kb_used += o.kb_used; - kb_used_data += o.kb_used_data; - kb_used_omap += o.kb_used_omap; - kb_used_meta += o.kb_used_meta; - kb_avail += o.kb_avail; - snap_trim_queue_len += o.snap_trim_queue_len; - num_snap_trimming += o.num_snap_trimming; - op_queue_age_hist.add(o.op_queue_age_hist); - os_perf_stat.add(o.os_perf_stat); - num_pgs += o.num_pgs; - } - void sub(const osd_stat_t& o) { - kb -= o.kb; - kb_used -= o.kb_used; - kb_used_data -= o.kb_used_data; - kb_used_omap -= o.kb_used_omap; - kb_used_meta -= o.kb_used_meta; - kb_avail -= o.kb_avail; - snap_trim_queue_len -= o.snap_trim_queue_len; - num_snap_trimming -= o.num_snap_trimming; - op_queue_age_hist.sub(o.op_queue_age_hist); - os_perf_stat.sub(o.os_perf_stat); - num_pgs -= o.num_pgs; - } - - void dump(Formatter *f) const; - void encode(bufferlist &bl, uint64_t features) const; - void decode(bufferlist::const_iterator &bl); - static void generate_test_instances(std::list& o); -}; -WRITE_CLASS_ENCODER_FEATURES(osd_stat_t) - -inline bool operator==(const osd_stat_t& l, const osd_stat_t& r) { - return l.kb == r.kb && - l.kb_used == r.kb_used && - l.kb_used_data == r.kb_used_data && - l.kb_used_omap == r.kb_used_omap && - l.kb_used_meta == r.kb_used_meta && - l.kb_avail == r.kb_avail && - l.snap_trim_queue_len == r.snap_trim_queue_len && - l.num_snap_trimming == r.num_snap_trimming && - l.hb_peers == r.hb_peers && - l.op_queue_age_hist == r.op_queue_age_hist && - l.os_perf_stat == r.os_perf_stat && - l.num_pgs == r.num_pgs; -} -inline bool operator!=(const osd_stat_t& l, const osd_stat_t& r) { - return !(l == r); -} - -inline ostream& operator<<(ostream& out, const osd_stat_t& s) { - return out << "osd_stat(" - << byte_u_t(s.kb_used << 10) << " used (" - << byte_u_t(s.kb_used_data << 10) << " data, " - << byte_u_t(s.kb_used_omap << 10) << " omap, " - << byte_u_t(s.kb_used_meta << 10) << " meta), " - << byte_u_t(s.kb_avail << 10) << " avail, " - << byte_u_t(s.kb << 10) << " total, " - << "peers " << s.hb_peers - << " op hist " << s.op_queue_age_hist.h - << ")"; -} - - /* * pg states */ @@ -2228,6 +2139,38 @@ struct store_statfs_t bool is_zero() const { return *this == store_statfs_t(); } + + uint64_t get_used() const { + return total - available - internally_reserved; + } + + // this accumulates both actually used and statfs's internally_reserved + uint64_t get_used_raw() const { + return total - available; + } + + // helpers to ease legacy code porting + uint64_t kb_avail() const { + return available >> 10; + } + uint64_t kb() const { + return total >> 10; + } + uint64_t kb_used_raw() const { + return get_used_raw() >> 10; + } + + uint64_t kb_used_data() const { + return allocated >> 10; + } + uint64_t kb_used_omap() const { + return omap_allocated >> 10; + } + + uint64_t kb_used_internal_metadata() const { + return internal_metadata >> 10; + } + void add(const store_statfs_t& o) { total += o.total; available += o.available; @@ -2273,6 +2216,69 @@ WRITE_CLASS_DENC(store_statfs_t) ostream &operator<<(ostream &lhs, const store_statfs_t &rhs); +/** osd_stat + * aggregate stats for an osd + */ +struct osd_stat_t { + store_statfs_t statfs; + vector hb_peers; + int32_t snap_trim_queue_len, num_snap_trimming; + + pow2_hist_t op_queue_age_hist; + + objectstore_perf_stat_t os_perf_stat; + + epoch_t up_from = 0; + uint64_t seq = 0; + + uint32_t num_pgs = 0; + + osd_stat_t() : snap_trim_queue_len(0), num_snap_trimming(0) {} + + void add(const osd_stat_t& o) { + statfs.add(o.statfs); + snap_trim_queue_len += o.snap_trim_queue_len; + num_snap_trimming += o.num_snap_trimming; + op_queue_age_hist.add(o.op_queue_age_hist); + os_perf_stat.add(o.os_perf_stat); + num_pgs += o.num_pgs; + } + void sub(const osd_stat_t& o) { + statfs.sub(o.statfs); + snap_trim_queue_len -= o.snap_trim_queue_len; + num_snap_trimming -= o.num_snap_trimming; + op_queue_age_hist.sub(o.op_queue_age_hist); + os_perf_stat.sub(o.os_perf_stat); + num_pgs -= o.num_pgs; + } + + void dump(Formatter *f) const; + void encode(bufferlist &bl, uint64_t features) const; + void decode(bufferlist::const_iterator &bl); + static void generate_test_instances(std::list& o); +}; +WRITE_CLASS_ENCODER_FEATURES(osd_stat_t) + +inline bool operator==(const osd_stat_t& l, const osd_stat_t& r) { + return l.statfs == r.statfs && + l.snap_trim_queue_len == r.snap_trim_queue_len && + l.num_snap_trimming == r.num_snap_trimming && + l.hb_peers == r.hb_peers && + l.op_queue_age_hist == r.op_queue_age_hist && + l.os_perf_stat == r.os_perf_stat && + l.num_pgs == r.num_pgs; +} +inline bool operator!=(const osd_stat_t& l, const osd_stat_t& r) { + return !(l == r); +} + +inline ostream& operator<<(ostream& out, const osd_stat_t& s) { + return out << "osd_stat(" << s.statfs << ", " + << "peers " << s.hb_peers + << " op hist " << s.op_queue_age_hist.h + << ")"; +} + /* * summation over an entire pool */ diff --git a/src/test/mon/test_mon_workloadgen.cc b/src/test/mon/test_mon_workloadgen.cc index ba82c2b1bc3aa..7cd8432daaf37 100644 --- a/src/test/mon/test_mon_workloadgen.cc +++ b/src/test/mon/test_mon_workloadgen.cc @@ -536,9 +536,9 @@ class OSDStub : public TestStub return; } - osd_stat.kb = stbuf.f_blocks * stbuf.f_bsize / 1024; - osd_stat.kb_used = (stbuf.f_blocks - stbuf.f_bfree) * stbuf.f_bsize / 1024; - osd_stat.kb_avail = stbuf.f_bavail * stbuf.f_bsize / 1024; + osd_stat.statfs.total = stbuf.f_blocks * stbuf.f_bsize; + osd_stat.statfs.available = stbuf.f_bavail * stbuf.f_bsize; + osd_stat.statfs.internally_reserved = 0; } void send_pg_stats() { -- 2.39.5