From 98caf058cfd211184db9db83c5e93c81972e464d Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 23 Sep 2021 14:27:02 +0800 Subject: [PATCH] crimson/os/seastore: enable SegmentCleaner to hold multiple segmented devices For now, all segmented devices are treated the same as SEGMENTED. In the future, there may be different kinds of segmeted devices, like SEGMENTED_NVME, SEGMENTED_SSD, and even SEGMENTED_SATA. We plan to use a dedicated segment cleaner for each kind of those devices, and if there are multiple devices of the same kind, they share the same segment cleaner. Signed-off-by: Xuehan Xu --- .../os/seastore/extent_placement_manager.cc | 4 +- src/crimson/os/seastore/journal.cc | 2 +- src/crimson/os/seastore/segment_cleaner.cc | 90 ++-- src/crimson/os/seastore/segment_cleaner.h | 411 +++++++++++++++--- .../os/seastore/transaction_manager.cc | 4 +- .../seastore/test_btree_lba_manager.cc | 4 +- .../crimson/seastore/test_seastore_journal.cc | 4 +- .../seastore/transaction_manager_test_state.h | 1 + 8 files changed, 431 insertions(+), 89 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 43e6235ddab..63c0919640b 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -273,7 +273,9 @@ SegmentedAllocator::Writer::roll_segment(bool set_rolling) { }); } - return segment_provider.get_segment().safe_then([this](auto segment) { + return segment_provider.get_segment( + segment_manager.get_device_id() + ).safe_then([this](auto segment) { return segment_manager.open(segment); }).safe_then([this](auto segref) { LOG_PREFIX(SegmentedAllocator::Writer::roll_segment); diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index db59b7d1373..3725aeee867 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -166,7 +166,7 @@ Journal::roll_journal_segment() return (current_journal_segment ? current_journal_segment->close() : Segment::close_ertr::now()).safe_then([this] { - return segment_provider->get_segment(); + return segment_provider->get_segment(segment_manager.get_device_id()); }).safe_then([this](auto segment) { return segment_manager.open(segment); }).safe_then([this](auto sref) { diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index d23c5407c76..89d98875f21 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -15,6 +15,17 @@ namespace { namespace crimson::os::seastore { +template +void segment_manager_info_t::init_segment_infos( + SegmentInfoT&& segment_info) +{ + segment_infos.resize(num_segments, std::move(segment_info)); + for (device_segment_id_t j = 0; j < num_segments; j++) { + segment_infos[j].segment = segment_id_t(device_id, j); + crimson::get_logger(ceph_subsys_seastore).debug("added segment: {}", segment_id_t(device_id, j)); + } +} + bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const { const auto &other = static_cast(_other); @@ -26,22 +37,24 @@ bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const } bool all_match = true; - for (segment_id_t i = 0; i < live_bytes_by_segment.size(); ++i) { - if (other.live_bytes_by_segment[i] != live_bytes_by_segment[i]) { + for (auto i = live_bytes_by_segment.begin(), j = other.live_bytes_by_segment.begin(); + i != live_bytes_by_segment.end(); ++i, ++j) { + assert(i->segment == j->segment); + if (i->live_bytes != j->live_bytes) { all_match = false; logger().debug( "{}: segment_id {} live bytes mismatch *this: {}, other: {}", __func__, - i, - live_bytes_by_segment[i], - other.live_bytes_by_segment[i]); + i->segment, + i->live_bytes, + j->live_bytes); } } return all_match; } int64_t SpaceTrackerDetailed::SegmentMap::allocate( - segment_id_t segment, + device_segment_id_t segment, segment_off_t offset, extent_len_t len, const extent_len_t block_size) @@ -73,7 +86,7 @@ int64_t SpaceTrackerDetailed::SegmentMap::allocate( } int64_t SpaceTrackerDetailed::SegmentMap::release( - segment_id_t segment, + device_segment_id_t segment, segment_off_t offset, extent_len_t len, const extent_len_t block_size) @@ -115,15 +128,17 @@ bool SpaceTrackerDetailed::equals(const SpaceTrackerI &_other) const } bool all_match = true; - for (segment_id_t i = 0; i < segment_usage.size(); ++i) { - if (other.segment_usage[i].get_usage() != segment_usage[i].get_usage()) { + for (auto i = segment_usage.begin(), j = other.segment_usage.begin(); + i != segment_usage.end(); ++i, ++j) { + assert(i->segment == j->segment); + if (i->segment_map.get_usage() != j->segment_map.get_usage()) { all_match = false; logger().error( "{}: segment_id {} live bytes mismatch *this: {}, other: {}", __func__, - i, - segment_usage[i].get_usage(), - other.segment_usage[i].get_usage()); + i->segment, + i->segment_map.get_usage(), + j->segment_map.get_usage()); } } return all_match; @@ -141,7 +156,8 @@ void SpaceTrackerDetailed::SegmentMap::dump_usage(extent_len_t block_size) const void SpaceTrackerDetailed::dump_usage(segment_id_t id) const { logger().debug("SpaceTrackerDetailed::dump_usage {}", id); - segment_usage[id].dump_usage(block_size); + segment_usage[id].segment_map.dump_usage( + segment_usage[id.device_id()]->block_size); } SegmentCleaner::SegmentCleaner( @@ -167,13 +183,33 @@ void SegmentCleaner::register_metrics() SegmentCleaner::get_segment_ret SegmentCleaner::get_segment() { - for (size_t i = 0; i < segments.size(); ++i) { - if (segments[i].is_empty()) { - mark_open(i); - logger().debug("{}: returning segment {}", __func__, i); + for (auto& segment_info : segments) { + if (segment_info.is_empty()) { + mark_open(segment_info.segment); + logger().debug("{}: returning segment {}", __func__, segment_info.segment); + return get_segment_ret( + get_segment_ertr::ready_future_marker{}, + segment_info.segment); + } + } + assert(0 == "out of space handling todo"); + return get_segment_ret( + get_segment_ertr::ready_future_marker{}, + 0); +} + +SegmentCleaner::get_segment_ret SegmentCleaner::get_segment(device_id_t id) +{ + for (auto it = segments.find_begin(id); + it != segments.find_end(id); + it++) { + auto& segment_info = *it; + if (segment_info.is_empty()) { + mark_open(segment_info.segment); + logger().debug("{}: returning segment {}", __func__, segment_info.segment); return get_segment_ret( get_segment_ertr::ready_future_marker{}, - i); + segment_info.segment); } } assert(0 == "out of space handling todo"); @@ -378,15 +414,17 @@ SegmentCleaner::gc_reclaim_space_ret SegmentCleaner::gc_reclaim_space() } SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() { + logger().debug("SegmentCleaner::init_segments: {} segments", segments.size()); return seastar::do_with( std::vector>(), - [this](auto& segments) { + [this](auto& segment_set) { return crimson::do_for_each( - boost::make_counting_iterator(segment_id_t{0}), - boost::make_counting_iterator(segment_id_t{num_segments}), - [this, &segments](auto segment_id) { + segments.begin(), + segments.end(), + [this, &segment_set](auto& segment_info) { + auto segment_id = segment_info.segment; return scanner->read_segment_header(segment_id) - .safe_then([&segments, segment_id, this](auto header) { + .safe_then([&segment_set, segment_id, this](auto header) { if (header.out_of_line) { logger().debug("ExtentReader::init_segments: out-of-line segment {}", segment_id); init_mark_segment_closed( @@ -395,7 +433,7 @@ SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() { true); } else { logger().debug("ExtentReader::init_segments: journal segment {}", segment_id); - segments.emplace_back(std::make_pair(segment_id, std::move(header))); + segment_set.emplace_back(std::make_pair(segment_id, std::move(header))); } return seastar::now(); }).handle_error( @@ -407,10 +445,10 @@ SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() { }), crimson::ct_error::input_output_error::pass_further{} ); - }).safe_then([&segments] { + }).safe_then([&segment_set] { return seastar::make_ready_future< std::vector>>( - std::move(segments)); + std::move(segment_set)); }); }); } diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index 470504ac44c..7060a55e0cd 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -18,12 +18,52 @@ namespace crimson::os::seastore { +struct segment_info_t; +using unavail_segment_info_set_t = + std::list; +using unavail_segment_info_iterator = + unavail_segment_info_set_t::iterator; + +// position within segment_info_set_t +using segment_pos_t = uint64_t; + +template +struct segment_manager_info_t { + segment_manager_info_t() = default; + segment_manager_info_t( + device_id_t device_id, + device_segment_id_t num_segments, + segment_off_t segment_size, + segment_off_t block_size, + size_t empty_segments, + size_t size) + : device_id(device_id), + num_segments(num_segments), + segment_size(segment_size), + block_size(block_size), + empty_segments(empty_segments), + size(size) + {} + + void init_segment_infos(SegmentInfoT&&); + + unavail_segment_info_set_t usis; + device_id_t device_id = 0; + device_segment_id_t num_segments = 0; + segment_off_t segment_size = 0; + segment_off_t block_size = 0; + size_t empty_segments = 0; + size_t size = 0; + std::vector segment_infos; +}; + struct segment_info_t { Segment::segment_state_t state = Segment::segment_state_t::EMPTY; // Will be non-null for any segments in the current journal segment_seq_t journal_segment_seq = NULL_SEG_SEQ; + segment_id_t segment; bool out_of_line = false; @@ -46,6 +86,173 @@ struct segment_info_t { } }; +// for keeping track of segment managers' various information, +// like empty segments, opened segments and so on. +template +class segment_info_set_t { +public: + template + class iterator { + public: + iterator( + device_id_t sm_id, + std::conditional_t< + is_const, + const segment_info_set_t*, + segment_info_set_t*> segment_info_set, + bool end = false) + : sm_id(sm_id), segment_info_set(segment_info_set) { + auto& sm_info = segment_info_set->sm_info_vec[sm_id]; + assert(sm_info); + if (end) { + iter = sm_info->segment_infos.end(); + } else { + iter = sm_info->segment_infos.begin(); + } + } + iterator& operator++() { + iter++; + auto& sm_info = segment_info_set->sm_info_vec[sm_id]; + if (iter == sm_info->segment_infos.end()) { + device_id_t t_sm_id = sm_id; + while (++t_sm_id < max_devices && !segment_info_set->sm_info_vec[t_sm_id]); + if (t_sm_id < max_devices) { + auto& sm = segment_info_set->sm_info_vec[t_sm_id]; + assert(sm); + iter = sm->segment_infos.begin(); + sm_id = t_sm_id; + } + } + return *this; + } + bool operator==(iterator rit) { + return sm_id == rit.sm_id && iter == rit.iter; + } + bool operator!=(iterator rit) { + return !(*this == rit); + } + template = 0> + const SegmentInfoT& operator*() { + return *iter; + } + template = 0> + SegmentInfoT& operator*() { + return *iter; + } + template = 0> + const SegmentInfoT* operator->() { + return &(*iter); + } + template = 0> + SegmentInfoT* operator->() { + return &(*iter); + } + private: + device_id_t sm_id = 0; + std::conditional_t< + is_const, + const segment_info_set_t*, + segment_info_set_t*> segment_info_set = nullptr; + std::conditional_t< + is_const, + typename std::vector::const_iterator, + typename std::vector::iterator> iter; + }; + SegmentInfoT& operator[](segment_id_t id) { + auto d_id = id.device_id(); + auto& sm_info = sm_info_vec[d_id]; + assert(sm_info); + assert(sm_info->device_id == d_id); + auto& segment_info = sm_info->segment_infos[id.device_segment_id()]; + assert(segment_info.segment == id); + return segment_info; + } + const SegmentInfoT& operator[](segment_id_t id) const { + auto d_id = id.device_id(); + auto& sm_info = sm_info_vec.at(d_id); + assert(sm_info); + assert(sm_info->device_id == d_id); + auto& segment_info = sm_info->segment_infos[id.device_segment_id()]; + assert(segment_info.segment == id); + return segment_info; + } + std::optional>& + operator[](device_id_t id) { + auto& sm_info = sm_info_vec[id]; + assert(sm_info && sm_info->device_id == id); + return sm_info; + } + const std::optional>& + operator[](device_id_t id) const { + auto& sm_info = sm_info_vec[id]; + assert(sm_info && sm_info->device_id == id); + return sm_info; + } + void clear() { + sm_info_vec.clear(); + total_segments = 0; + } + void add_segment_manager( + SegmentManager& segment_manager, + SegmentInfoT&& segment_info) + { + if (!sm_info_vec.size()) { + sm_info_vec.resize(max_devices); + } + device_id_t d_id = segment_manager.get_device_id(); + sm_info_vec[segment_manager.get_device_id()] = std::make_optional< + segment_manager_info_t>( + d_id, + segment_manager.get_num_segments(), + segment_manager.get_segment_size(), + segment_manager.get_block_size(), + segment_manager.get_num_segments(), + segment_manager.get_size()); + sm_info_vec[segment_manager.get_device_id()]->init_segment_infos(std::move(segment_info)); + total_segments += segment_manager.get_num_segments(); + } + + device_segment_id_t size() const { + return total_segments; + } + auto begin() { + device_id_t sm_id = 0; + for (;sm_id < max_devices && !sm_info_vec[sm_id]; + sm_id ++); + return iterator(sm_id, this); + } + auto begin() const { + device_id_t sm_id = 0; + for (;sm_id < max_devices && !sm_info_vec[sm_id]; + sm_id ++); + return iterator(sm_id, this); + } + auto end() { + device_id_t sm_id = max_devices - 1; + for (;!sm_info_vec[sm_id]; + sm_id --); + return iterator(sm_id, this, true); + } + auto end() const { + device_id_t sm_id = max_devices - 1; + for (;!sm_info_vec[sm_id]; + sm_id --); + return iterator(sm_id, this, true); + } + auto find_begin(device_id_t id) { + auto& sm_info = sm_info_vec[id]; + return sm_info->segment_infos.begin(); + } + auto find_end(device_id_t id) { + auto& sm_info = sm_info_vec[id]; + return sm_info->segment_infos.end(); + } +private: + device_segment_id_t total_segments; + std::vector>> sm_info_vec; + + friend class SegmentCleaner; +}; /** * Callback interface for managing available segments */ @@ -55,6 +262,7 @@ public: crimson::ct_error::input_output_error>; using get_segment_ret = get_segment_ertr::future; virtual get_segment_ret get_segment() = 0; + virtual get_segment_ret get_segment(device_id_t id) = 0; virtual void close_segment(segment_id_t) {} @@ -102,19 +310,37 @@ public: }; using SpaceTrackerIRef = std::unique_ptr; +struct segment_space_tracker_t { + segment_id_t segment = NULL_SEG_ID; + int64_t live_bytes = 0; +}; + class SpaceTrackerSimple : public SpaceTrackerI { // Tracks live space for each segment - std::vector live_bytes_by_segment; + segment_info_set_t live_bytes_by_segment; int64_t update_usage(segment_id_t segment, int64_t delta) { - assert(segment < live_bytes_by_segment.size()); - live_bytes_by_segment[segment] += delta; - assert(live_bytes_by_segment[segment] >= 0); - return live_bytes_by_segment[segment]; + live_bytes_by_segment[segment].live_bytes += delta; + assert(live_bytes_by_segment[segment].live_bytes >= 0); + return live_bytes_by_segment[segment].live_bytes; } public: - SpaceTrackerSimple(segment_id_t num_segments) - : live_bytes_by_segment(num_segments, 0) {} + SpaceTrackerSimple(std::vector sms) { + for (auto sm : sms) { + live_bytes_by_segment.add_segment_manager( + *sm, + segment_space_tracker_t{}); + } + } + SpaceTrackerSimple( + const segment_info_set_t& live_bytes_by_segment) + : live_bytes_by_segment(live_bytes_by_segment) { + for (auto it = this->live_bytes_by_segment.begin(); + it != this->live_bytes_by_segment.end(); + ++it) { + it->live_bytes = 0; + } + } int64_t allocate( segment_id_t segment, @@ -131,20 +357,21 @@ public: } int64_t get_usage(segment_id_t segment) const final { - assert(segment < live_bytes_by_segment.size()); - return live_bytes_by_segment[segment]; + return live_bytes_by_segment[segment].live_bytes; } void dump_usage(segment_id_t) const final {} void reset() final { - for (auto &i: live_bytes_by_segment) - i = 0; + for (auto it = live_bytes_by_segment.begin(); + it != live_bytes_by_segment.end(); + ++it) + it->live_bytes = 0; } SpaceTrackerIRef make_empty() const final { return SpaceTrackerIRef( - new SpaceTrackerSimple(live_bytes_by_segment.size())); + new SpaceTrackerSimple(live_bytes_by_segment)); } bool equals(const SpaceTrackerI &other) const; @@ -164,13 +391,13 @@ class SpaceTrackerDetailed : public SpaceTrackerI { } int64_t allocate( - segment_id_t segment, + device_segment_id_t segment, segment_off_t offset, extent_len_t len, const extent_len_t block_size); int64_t release( - segment_id_t segment, + device_segment_id_t segment, segment_off_t offset, extent_len_t len, const extent_len_t block_size); @@ -188,52 +415,81 @@ class SpaceTrackerDetailed : public SpaceTrackerI { } } }; - const size_t block_size; - const size_t segment_size; + struct segment_tracker_t { + segment_tracker_t(size_t blocks) + : segment_map(blocks) {} + segment_id_t segment; + SegmentMap segment_map; + }; // Tracks live space for each segment - std::vector segment_usage; + segment_info_set_t segment_usage; public: - SpaceTrackerDetailed(segment_id_t num_segments, size_t segment_size, size_t block_size) - : block_size(block_size), - segment_size(segment_size), - segment_usage(num_segments, segment_size / block_size) {} + SpaceTrackerDetailed( + std::vector sms) + { + for (auto sm : sms) { + // sms is a vector that is indexed by device id and + // always has "max_device" elements, some of which + // may be null. + if (!sm) { + continue; + } + segment_usage.add_segment_manager( + *sm, + segment_tracker_t( + sm->get_segment_size() / sm->get_block_size())); + } + } + SpaceTrackerDetailed( + const segment_info_set_t& segment_usage) + : segment_usage(segment_usage) + { + for (auto& tracker : this->segment_usage) { + tracker.segment_map.reset(); + } + } int64_t allocate( segment_id_t segment, segment_off_t offset, extent_len_t len) final { assert(segment < segment_usage.size()); - return segment_usage[segment].allocate(segment, offset, len, block_size); + return segment_usage[segment].segment_map.allocate( + segment.device_segment_id(), + offset, + len, + segment_usage[segment.device_id()]->block_size); } int64_t release( segment_id_t segment, segment_off_t offset, extent_len_t len) final { - assert(segment < segment_usage.size()); - return segment_usage[segment].release(segment, offset, len, block_size); + return segment_usage[segment].segment_map.release( + segment.device_segment_id(), + offset, + len, + segment_usage[segment.device_id()]->block_size); } int64_t get_usage(segment_id_t segment) const final { assert(segment < segment_usage.size()); - return segment_usage[segment].get_usage(); + return segment_usage[segment].segment_map.get_usage(); } void dump_usage(segment_id_t seg) const final; void reset() final { for (auto &i: segment_usage) - i.reset(); + i.segment_map.reset(); } SpaceTrackerIRef make_empty() const final { return SpaceTrackerIRef( new SpaceTrackerDetailed( - segment_usage.size(), - segment_size, - block_size)); + segment_usage)); } bool equals(const SpaceTrackerI &other) const; @@ -379,14 +635,12 @@ private: const bool detailed; const config_t config; - segment_id_t num_segments = 0; - size_t segment_size = 0; - size_t block_size = 0; + internal_segment_id_t num_segments = 0; ExtentReaderRef scanner; SpaceTrackerIRef space_tracker; - std::vector segments; + segment_info_set_t segments; size_t empty_segments; uint64_t used_bytes = 0; bool init_complete = false; @@ -414,40 +668,53 @@ private: /// head of journal journal_seq_t journal_head; + device_id_t journal_device_id; + ExtentCallbackInterface *ecb = nullptr; /// populated if there is an IO blocked on hard limits std::optional> blocked_io_wake; + std::vector effective_devices; + public: SegmentCleaner( config_t config, ExtentReaderRef&& scanner, bool detailed = false); - void mount(SegmentManager &sm) { + void mount(SegmentManager &psm, std::vector& sms) { + crimson::get_logger(ceph_subsys_seastore).debug( + "SegmentCleaner::mount: {} segment managers", sms.size()); init_complete = false; used_bytes = 0; journal_tail_target = journal_seq_t{}; journal_tail_committed = journal_seq_t{}; journal_head = journal_seq_t{}; + journal_device_id = psm.get_device_id(); - num_segments = sm.get_num_segments(); - segment_size = static_cast(sm.get_segment_size()); - block_size = static_cast(sm.get_block_size()); + for (auto& sm : sms) { + if (sm) + effective_devices.push_back(sm->get_device_id()); + } space_tracker.reset( detailed ? (SpaceTrackerI*)new SpaceTrackerDetailed( - num_segments, - segment_size, - block_size) : + sms) : (SpaceTrackerI*)new SpaceTrackerSimple( - num_segments)); + sms)); segments.clear(); - segments.resize(num_segments); - empty_segments = num_segments; + for (auto sm : sms) { + // sms is a vector that is indexed by device id and + // always has "max_device" elements, some of which + // may be null. + if (!sm) { + continue; + } + segments.add_segment_manager(*sm, segment_info_t{}); + } } using init_segments_ertr = crimson::errorator< @@ -458,12 +725,16 @@ public: init_segments_ret init_segments(); get_segment_ret get_segment() final; + get_segment_ret get_segment(device_id_t id) final; void close_segment(segment_id_t segment) final; void set_journal_segment( segment_id_t segment, segment_seq_t seq) final { - assert(segment < segments.size()); + assert(segment.device_id() == + segments[segment.device_id()]->device_id); + assert(segment.device_segment_id() < + segments[segment.device_id()]->num_segments); segments[segment].journal_segment_seq = seq; assert(segments[segment].is_open()); } @@ -523,7 +794,10 @@ public: paddr_t addr, extent_len_t len, bool init_scan = false) { - assert(addr.segment < segments.size()); + assert(addr.segment.device_id() == + segments[addr.segment.device_id()]->device_id); + assert(addr.segment.device_segment_id() < + segments[addr.segment.device_id()]->num_segments); if (!init_scan && !init_complete) return; @@ -545,7 +819,10 @@ public: ceph_assert(used_bytes >= len); used_bytes -= len; - assert(addr.segment < segments.size()); + assert(addr.segment.device_id() == + segments[addr.segment.device_id()]->device_id); + assert(addr.segment.device_segment_id() < + segments[addr.segment.device_id()]->num_segments); [[maybe_unused]] auto ret = space_tracker->release( addr.segment, @@ -557,20 +834,25 @@ public: segment_id_t get_next_gc_target() const { segment_id_t ret = NULL_SEG_ID; + segment_seq_t seq = NULL_SEG_SEQ; int64_t least_live_bytes = std::numeric_limits::max(); - for (segment_id_t i = 0; i < segments.size(); ++i) { - if (segments[i].is_closed() && - !segments[i].is_in_journal(journal_tail_committed) && - space_tracker->get_usage(i) < least_live_bytes) { - ret = i; - least_live_bytes = space_tracker->get_usage(i); + for (auto it = segments.begin(); + it != segments.end(); + ++it) { + const auto& segment_info = *it; + if (segment_info.is_closed() && + !segment_info.is_in_journal(journal_tail_committed) && + space_tracker->get_usage(segment_info.segment) < least_live_bytes) { + ret = segment_info.segment; + seq = segment_info.journal_segment_seq; + least_live_bytes = space_tracker->get_usage(segment_info.segment); } } if (ret != NULL_SEG_ID) { crimson::get_logger(ceph_subsys_seastore).debug( "SegmentCleaner::get_next_gc_target: segment {} seq {}", ret, - segments[ret].journal_segment_seq); + seq); } return ret; } @@ -979,7 +1261,10 @@ private: } void mark_closed(segment_id_t segment) { - assert(segments.size() > segment); + assert(segment.device_id() == + segments[segment.device_id()]->device_id); + assert(segment.device_segment_id() < + segments[segment.device_id()]->num_segments); if (init_complete) { assert(segments[segment].is_open()); } else { @@ -994,9 +1279,11 @@ private: } void mark_empty(segment_id_t segment) { - assert(segments.size() > segment); + assert(segment.device_id() == + segments[segment.device_id()]->device_id); + assert(segment.device_segment_id() < + segments[segment.device_id()]->num_segments); assert(segments[segment].is_closed()); - assert(segments.size() > empty_segments); ++empty_segments; if (space_tracker->get_usage(segment) != 0) { space_tracker->dump_usage(segment); @@ -1007,7 +1294,11 @@ private: } void mark_open(segment_id_t segment) { - assert(segments.size() > segment); + crimson::get_logger(ceph_subsys_seastore).debug("mark open: {}", segment); + assert(segment.device_id() == + segments[segment.device_id()]->device_id); + assert(segment.device_segment_id() < + segments[segment.device_id()]->num_segments); assert(segments[segment].is_empty()); assert(empty_segments > 0); --empty_segments; @@ -1016,4 +1307,10 @@ private: }; using SegmentCleanerRef = std::unique_ptr; +template struct segment_manager_info_t; +template class segment_info_set_t; +template struct segment_manager_info_t; +template class segment_info_set_t; +template struct segment_manager_info_t; +template class segment_info_set_t; } diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 170742df09e..2bb4686bdaa 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -33,7 +33,7 @@ TransactionManager::TransactionManager( TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs() { LOG_PREFIX(TransactionManager::mkfs); - segment_cleaner->mount(segment_manager); + segment_cleaner->mount(segment_manager, segment_managers); return journal->open_for_write().safe_then([this, FNAME](auto addr) { DEBUG("about to do_with"); segment_cleaner->init_mkfs(addr); @@ -64,7 +64,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() { LOG_PREFIX(TransactionManager::mount); cache->init(); - segment_cleaner->mount(segment_manager); + segment_cleaner->mount(segment_manager, segment_managers); return segment_cleaner->init_segments().safe_then( [this](auto&& segments) { return journal->replay( diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 7399bd26eac..330c3c0102a 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -50,7 +50,7 @@ struct btree_lba_manager_test : } segment_id_t next = 0; - get_segment_ret get_segment() final { + get_segment_ret get_segment(device_id_t id) final { auto ret = next; next = segment_id_t{ next.device_id(), @@ -60,6 +60,8 @@ struct btree_lba_manager_test : ret); } + void set_segment_written_to(paddr_t addr) final {} + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } void update_journal_tail_committed(journal_seq_t committed) final {} diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index 3d937d5b67a..b18a842e793 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -85,7 +85,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { } segment_id_t next = 0; - get_segment_ret get_segment() final { + get_segment_ret get_segment(device_id_t id) final { auto ret = next; next = segment_id_t{ next.device_id(), @@ -95,6 +95,8 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { ret); } + void set_segment_written_to(paddr_t addr) final {} + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } void update_journal_tail_committed(journal_seq_t paddr) final {} diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index a5428556eb7..e12b79eddbb 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -76,6 +76,7 @@ auto get_transaction_manager( auto segment_cleaner = std::make_unique( SegmentCleaner::config_t::get_default(), std::move(scanner), + segment_manager, true); auto journal = std::make_unique(segment_manager, scanner_ref); auto cache = std::make_unique(scanner_ref, segment_manager.get_block_size()); -- 2.39.5