From 236c8a41a6e426916f84cf30e62ebbd3d1e54af2 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Thu, 5 May 2022 13:48:58 +0800 Subject: [PATCH] crimson/os/seastore/segment_cleaner: improve space calculations Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/segment_cleaner.cc | 31 ++++-- src/crimson/os/seastore/segment_cleaner.h | 117 +++++++-------------- 2 files changed, 57 insertions(+), 91 deletions(-) diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index edd21b1f5b9..ad55494c1ce 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -434,6 +434,7 @@ void SegmentCleaner::register_metrics() sm::make_counter("segments_closed", [this] { return segments.get_num_closed(); }, sm::description("the number of closed segments")), + sm::make_counter("segments_count_open", [this] { return segments.get_count_open(); }, sm::description("the count of open segment operations")), @@ -443,12 +444,26 @@ void SegmentCleaner::register_metrics() sm::make_counter("segments_count_close", [this] { return segments.get_count_close(); }, sm::description("the count of close segment operations")), + sm::make_counter("total_bytes", [this] { return segments.get_total_bytes(); }, sm::description("the size of the space")), sm::make_counter("available_bytes", [this] { return segments.get_available_bytes(); }, sm::description("the size of the space is available")), + sm::make_counter("unavailable_unreclaimable_bytes", + [this] { return segments.get_unavailable_unreclaimable_bytes(); }, + sm::description("the size of the space is unavailable and unreclaimable")), + sm::make_counter("unavailable_reclaimable_bytes", + [this] { return segments.get_unavailable_reclaimable_bytes(); }, + sm::description("the size of the space is unavailable and reclaimable")), + sm::make_counter("used_bytes", stats.used_bytes, + sm::description("the size of the space occupied by live extents")), + sm::make_counter("unavailable_unused_bytes", + [this] { return get_unavailable_unused_bytes(); }, + sm::description("the size of the space is unavailable and not alive")), + sm::make_counter("projected_used_bytes", stats.projected_used_bytes, + sm::description("the size of the space going to be occupied by new extents")), sm::make_counter("accumulated_blocked_ios", stats.accumulated_blocked_ios, sm::description("accumulated total number of ios that were blocked by gc")), @@ -458,10 +473,6 @@ void SegmentCleaner::register_metrics() sm::description("bytes being reclaimed")), sm::make_counter("ios_blocking", stats.ios_blocking, sm::description("IOs that are blocking on space usage")), - sm::make_counter("used_bytes", stats.used_bytes, - sm::description("the size of the space occupied by live extents")), - sm::make_counter("projected_used_bytes", stats.projected_used_bytes, - sm::description("the size of the space going to be occupied by new extents")), sm::make_histogram("segment_utilization_distribution", [this]() -> seastar::metrics::histogram& { return stats.segment_util; @@ -484,10 +495,10 @@ segment_id_t SegmentCleaner::allocate_segment( if (segment_info.is_empty()) { segments.mark_open(seg_id, seq, type); INFO("opened, should_block_on_gc {}, projected_avail_ratio {}, " - "projected_reclaim_ratio {}", + "reclaim_ratio {}", should_block_on_gc(), get_projected_available_ratio(), - get_projected_reclaim_ratio()); + get_reclaim_ratio()); return seg_id; } } @@ -565,10 +576,10 @@ void SegmentCleaner::close_segment(segment_id_t segment) LOG_PREFIX(SegmentCleaner::close_segment); segments.mark_closed(segment); INFO("closed, should_block_on_gc {}, projected_avail_ratio {}, " - "projected_reclaim_ratio {}", + "reclaim_ratio {}", should_block_on_gc(), get_projected_available_ratio(), - get_projected_reclaim_ratio()); + get_reclaim_ratio()); } SegmentCleaner::trim_backrefs_ret SegmentCleaner::trim_backrefs( @@ -1097,11 +1108,11 @@ SegmentCleaner::maybe_release_segment(Transaction &t) ).safe_then([this, FNAME, &t, to_release] { segments.mark_empty(to_release); INFOT("released, should_block_on_gc {}, projected_avail_ratio {}, " - "projected_reclaim_ratio {}", + "reclaim_ratio {}", t, should_block_on_gc(), get_projected_available_ratio(), - get_projected_reclaim_ratio()); + get_reclaim_ratio()); if (space_tracker->get_usage(to_release) != 0) { space_tracker->dump_usage(to_release); ceph_abort(); diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index 2bb1e8520ac..12ca08954c9 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -177,6 +177,9 @@ public: assert(ret + get_unavailable_unreclaimable_bytes() == get_unavailable_bytes()); return ret; } + double get_available_ratio() const { + return (double)get_available_bytes() / (double)total_bytes; + } journal_seq_t get_journal_head() const { if (unlikely(journal_segment_id == NULL_SEG_ID)) { @@ -636,6 +639,11 @@ private: bool init_complete = false; struct { + /** + * used_bytes + * + * Bytes occupied by live extents + */ uint64_t used_bytes = 0; /** * projected_used_bytes @@ -829,10 +837,10 @@ public: store_statfs_t stat() const { store_statfs_t st; - st.total = get_total_bytes(); - st.available = get_total_bytes() - get_used_bytes(); - st.allocated = get_used_bytes(); - st.data_stored = get_used_bytes(); + st.total = segments.get_total_bytes(); + st.available = segments.get_total_bytes() - stats.used_bytes; + st.allocated = stats.used_bytes; + st.data_stored = stats.used_bytes; // TODO add per extent type counters for omap_allocated and // internal metadata @@ -1067,81 +1075,29 @@ private: backref_buf_entry_t::cmp_t> &&backrefs, std::vector &extents); - /// Returns free space available for writes - size_t get_available_bytes() const { - return segments.get_available_bytes(); - } - size_t get_projected_available_bytes() const { - return (get_available_bytes() > stats.projected_used_bytes) ? - get_available_bytes() - stats.projected_used_bytes: - 0; - } - - /// Returns total space available - size_t get_total_bytes() const { - return segments.get_total_bytes(); - } - - /// Returns total space not free - size_t get_unavailable_bytes() const { - return segments.get_total_bytes() - segments.get_available_bytes(); - } - size_t get_projected_unavailable_bytes() const { - assert(get_total_bytes() >= get_projected_available_bytes()); - return get_total_bytes() - get_projected_available_bytes(); - } - - /// Returns bytes currently occupied by live extents (not journal) - size_t get_used_bytes() const { - return stats.used_bytes; - } - - /// Return bytes contained in segments in journal - size_t get_journal_segment_bytes() const { - return segments.get_num_in_journal() * segments.get_segment_size(); - } - - /** - * get_reclaimable_bytes - * - * Returns the number of bytes in unavailable segments that can be - * reclaimed. + /* + * Space calculations */ - size_t get_reclaimable_bytes() const { - auto ret = get_unavailable_bytes() - get_used_bytes(); - if (ret > get_journal_segment_bytes()) - return ret - get_journal_segment_bytes(); - else - return 0; + std::size_t get_unavailable_unused_bytes() const { + assert(segments.get_unavailable_bytes() > stats.used_bytes); + return segments.get_unavailable_bytes() - stats.used_bytes; } - - /** - * get_reclaim_ratio - * - * Returns the ratio of space reclaimable unavailable space to - * total unavailable space. - */ double get_reclaim_ratio() const { - if (get_unavailable_bytes() == 0) return 0; - return (double)get_reclaimable_bytes() / (double)get_unavailable_bytes(); - } - double get_projected_reclaim_ratio() const { - if (get_projected_unavailable_bytes() == 0) return 0; - return (double)get_reclaimable_bytes() / - (double)get_projected_unavailable_bytes(); + if (segments.get_unavailable_bytes() == 0) return 0; + return (double)get_unavailable_unused_bytes() / (double)segments.get_unavailable_bytes(); } - /** - * get_available_ratio - * - * Returns ratio of available space to write to total space + /* + * Space calculations (projected) */ - double get_available_ratio() const { - return (double)get_available_bytes() / (double)get_total_bytes(); + std::size_t get_projected_available_bytes() const { + return (segments.get_available_bytes() > stats.projected_used_bytes) ? + segments.get_available_bytes() - stats.projected_used_bytes: + 0; } double get_projected_available_ratio() const { return (double)get_projected_available_bytes() / - (double)get_total_bytes(); + (double)segments.get_total_bytes(); } /** @@ -1150,9 +1106,8 @@ private: * Encapsulates whether block pending gc. */ bool should_block_on_gc() const { - // TODO: probably worth projecting journal usage as well auto aratio = get_projected_available_ratio(); - auto rratio = get_projected_reclaim_ratio(); + auto rratio = get_reclaim_ratio(); return ( (aratio < config.available_ratio_hard_limit) || ((aratio < config.available_ratio_gc_max) && @@ -1169,8 +1124,8 @@ private: "total {}, " "available {}, " "unavailable {}, " - "used {}, " - "reclaimable {}, " + "unavailable_used {}, " + "unavailable_unused {}, " "reclaim_ratio {}, " "available_ratio {}, " "should_block_on_gc {}, " @@ -1182,13 +1137,13 @@ private: "dirty_tail_limit {}, " "gc_should_trim_journal {}, ", caller, - get_total_bytes(), - get_available_bytes(), - get_unavailable_bytes(), - get_used_bytes(), - get_reclaimable_bytes(), + segments.get_total_bytes(), + segments.get_available_bytes(), + segments.get_unavailable_bytes(), + stats.used_bytes, + get_unavailable_unused_bytes(), get_reclaim_ratio(), - get_available_ratio(), + segments.get_available_ratio(), should_block_on_gc(), gc_should_reclaim_space(), segments.get_journal_head(), @@ -1258,7 +1213,7 @@ private: * Encapsulates logic for whether gc should be reclaiming segment space. */ bool gc_should_reclaim_space() const { - auto aratio = get_available_ratio(); + auto aratio = segments.get_available_ratio(); auto rratio = get_reclaim_ratio(); return ( (aratio < config.available_ratio_hard_limit) || -- 2.39.5