From 5ebc4744ddb0bb1e07425326dcb7fe014504a637 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Sat, 7 May 2022 19:15:12 +0800 Subject: [PATCH] crimson/os/seastore/segment_cleaner: exclude utilizations from empty/open segments Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/segment_cleaner.cc | 25 ++++++--- src/crimson/os/seastore/segment_cleaner.h | 59 +++++++++++++++++----- 2 files changed, 63 insertions(+), 21 deletions(-) diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index ae3535f2d633d..637d4f4c829af 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -409,15 +409,16 @@ SegmentCleaner::SegmentCleaner( void SegmentCleaner::register_metrics() { namespace sm = seastar::metrics; - stats.segment_util.buckets.resize(11); - for (int i = 0; i < 11; i++) { + stats.segment_util.buckets.resize(UTIL_BUCKETS); + std::size_t i; + for (i = 0; i < UTIL_BUCKETS; ++i) { stats.segment_util.buckets[i].upper_bound = ((double)(i + 1)) / 10; - if (!i) { - stats.segment_util.buckets[i].count = segments.get_num_segments(); - } else { - stats.segment_util.buckets[i].count = 0; - } + stats.segment_util.buckets[i].count = 0; } + // NOTE: by default the segments are empty + i = get_bucket_index(UTIL_STATE_EMPTY); + stats.segment_util.buckets[i].count = segments.get_num_segments(); + metrics.add_group("segment_cleaner", { sm::make_counter("segments_number", [this] { return segments.get_num_segments(); }, @@ -510,7 +511,10 @@ segment_id_t SegmentCleaner::allocate_segment( auto seg_id = it->first; auto& segment_info = it->second; if (segment_info.is_empty()) { + auto old_usage = calc_utilization(seg_id); segments.mark_open(seg_id, seq, type); + auto new_usage = calc_utilization(seg_id); + adjust_segment_util(old_usage, new_usage); INFO("opened, should_block_on_gc {}, projected_avail_ratio {}, " "reclaim_ratio {}", should_block_on_gc(), @@ -591,7 +595,10 @@ void SegmentCleaner::update_journal_tail_committed(journal_seq_t committed) void SegmentCleaner::close_segment(segment_id_t segment) { LOG_PREFIX(SegmentCleaner::close_segment); + auto old_usage = calc_utilization(segment); segments.mark_closed(segment); + auto new_usage = calc_utilization(segment); + adjust_segment_util(old_usage, new_usage); INFO("closed, should_block_on_gc {}, projected_avail_ratio {}, " "reclaim_ratio {}", should_block_on_gc(), @@ -1124,7 +1131,11 @@ SegmentCleaner::maybe_release_segment(Transaction &t) INFOT("releasing segment {}", t, to_release); return sm_group->release_segment(to_release ).safe_then([this, FNAME, &t, to_release] { + auto old_usage = calc_utilization(to_release); + ceph_assert(old_usage == 0); segments.mark_empty(to_release); + auto new_usage = calc_utilization(to_release); + adjust_segment_util(old_usage, new_usage); INFOT("released, should_block_on_gc {}, projected_avail_ratio {}, " "reclaim_ratio {}", t, diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index 6979d9d38d2b3..b3e842ed54890 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -749,9 +749,11 @@ public: release_ertr::future<> maybe_release_segment(Transaction &t); void adjust_segment_util(double old_usage, double new_usage) { - assert(stats.segment_util.buckets[std::floor(old_usage * 10)].count > 0); - stats.segment_util.buckets[std::floor(old_usage * 10)].count--; - stats.segment_util.buckets[std::floor(new_usage * 10)].count++; + auto old_index = get_bucket_index(old_usage); + auto new_index = get_bucket_index(new_usage); + assert(stats.segment_util.buckets[old_index].count > 0); + stats.segment_util.buckets[old_index].count--; + stats.segment_util.buckets[new_index].count++; } void mark_space_used( @@ -766,12 +768,12 @@ public: return; stats.used_bytes += len; - auto old_usage = space_tracker->calc_utilization(seg_addr.get_segment_id()); + auto old_usage = calc_utilization(seg_addr.get_segment_id()); [[maybe_unused]] auto ret = space_tracker->allocate( seg_addr.get_segment_id(), seg_addr.get_segment_off(), len); - auto new_usage = space_tracker->calc_utilization(seg_addr.get_segment_id()); + auto new_usage = calc_utilization(seg_addr.get_segment_id()); adjust_segment_util(old_usage, new_usage); // use the last extent's last modified time for the calculation of the projected @@ -809,12 +811,12 @@ public: seg_addr.get_segment_id(), addr, len); - auto old_usage = space_tracker->calc_utilization(seg_addr.get_segment_id()); + auto old_usage = calc_utilization(seg_addr.get_segment_id()); [[maybe_unused]] auto ret = space_tracker->release( seg_addr.get_segment_id(), seg_addr.get_segment_off(), len); - auto new_usage = space_tracker->calc_utilization(seg_addr.get_segment_id()); + auto new_usage = calc_utilization(seg_addr.get_segment_id()); adjust_segment_util(old_usage, new_usage); maybe_wake_gc_blocked_io(); assert(ret >= 0); @@ -865,11 +867,36 @@ public: using work_iertr = ExtentCallbackInterface::extent_mapping_iertr; private: + /* + * 10 buckets for the number of closed segments by usage + * 2 extra buckets for the number of open and empty segments + */ + static constexpr double UTIL_STATE_OPEN = 1.05; + static constexpr double UTIL_STATE_EMPTY = 1.15; + static constexpr std::size_t UTIL_BUCKETS = 12; + static std::size_t get_bucket_index(double util) { + auto index = std::floor(util * 10); + assert(index < UTIL_BUCKETS); + return index; + } + double calc_utilization(segment_id_t id) const { + auto& info = segments[id]; + if (info.is_open()) { + return UTIL_STATE_OPEN; + } else if (info.is_empty()) { + return UTIL_STATE_EMPTY; + } else { + auto ret = space_tracker->calc_utilization(id); + assert(ret >= 0 && ret < 1); + return ret; + } + } // journal status helpers double calc_gc_benefit_cost(segment_id_t id) const { - double util = space_tracker->calc_utilization(id); + double util = calc_utilization(id); + ceph_assert(util >= 0 && util < 1); auto cur_time = seastar::lowres_system_clock::now(); auto segment = segments[id]; assert(cur_time >= segment.last_modified); @@ -888,13 +915,14 @@ private: ++it) { auto _id = it->first; const auto& segment_info = it->second; - double benefit_cost = calc_gc_benefit_cost(_id); if (segment_info.is_closed() && - !segment_info.is_in_journal(journal_tail_committed) && - benefit_cost > max_benefit_cost) { - id = _id; - seq = segment_info.seq; - max_benefit_cost = benefit_cost; + !segment_info.is_in_journal(journal_tail_committed)) { + double benefit_cost = calc_gc_benefit_cost(_id); + if (benefit_cost > max_benefit_cost) { + id = _id; + seq = segment_info.seq; + max_benefit_cost = benefit_cost; + } } } if (id != NULL_SEG_ID) { @@ -1273,7 +1301,10 @@ private: segment_seq_t seq, segment_type_t s_type) { ceph_assert(!init_complete); + auto old_usage = calc_utilization(segment); segments.init_closed(segment, seq, s_type); + auto new_usage = calc_utilization(segment); + adjust_segment_util(old_usage, new_usage); if (s_type == segment_type_t::OOL) { ool_segment_seq_allocator->set_next_segment_seq(seq); } -- 2.39.5