From feaf582481f7ec83bc298be82cc41ba706b04b00 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 31 Mar 2022 13:48:42 +0800 Subject: [PATCH] crimson/os/seastore: use different segment seq allocator from journal and ool segments Fixes: https://tracker.ceph.com/issues/55143 Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/cache.cc | 4 ++++ src/crimson/os/seastore/journal.h | 2 -- .../os/seastore/journal/segmented_journal.cc | 12 ++++++---- .../os/seastore/journal/segmented_journal.h | 3 --- src/crimson/os/seastore/seastore_types.cc | 1 + src/crimson/os/seastore/seastore_types.h | 4 +++- src/crimson/os/seastore/segment_cleaner.cc | 2 ++ src/crimson/os/seastore/segment_cleaner.h | 24 ++++++++++++++++--- .../os/seastore/segment_seq_allocator.h | 21 ++++++++++++++-- src/crimson/os/seastore/transaction_manager.h | 2 +- .../seastore/test_btree_lba_manager.cc | 9 +++++-- .../crimson/seastore/test_seastore_journal.cc | 9 ++++++- 12 files changed, 74 insertions(+), 19 deletions(-) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index d29c2549a5e..c3d0c3fc9c3 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -985,6 +985,7 @@ record_t Cache::prepare_record( 0, t.root->get_version() - 1, MAX_SEG_SEQ, + segment_type_t::NULL_SEG, std::move(delta_bl) }); } else { @@ -1002,6 +1003,9 @@ record_t Cache::prepare_record( cleaner ? cleaner->get_seq(i->get_paddr().as_seg_paddr().get_segment_id()) : MAX_SEG_SEQ, + cleaner + ? cleaner->get_type(i->get_paddr().as_seg_paddr().get_segment_id()) + : segment_type_t::NULL_SEG, std::move(delta_bl) }); i->last_committed_crc = final_crc; diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 7c0a1a8cabd..77ecc34df93 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -85,8 +85,6 @@ public: virtual replay_ret replay( delta_handler_t &&delta_handler) = 0; - virtual SegmentSeqAllocator& get_segment_seq_allocator() = 0; - virtual ~Journal() {} }; using JournalRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/journal/segmented_journal.cc b/src/crimson/os/seastore/journal/segmented_journal.cc index ddd9c3d2513..e4a29d795ef 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.cc +++ b/src/crimson/os/seastore/journal/segmented_journal.cc @@ -31,7 +31,8 @@ SegmentedJournal::SegmentedJournal( ExtentReader &scanner, SegmentProvider &segment_provider) : segment_provider(segment_provider), - segment_seq_allocator(new SegmentSeqAllocator), + segment_seq_allocator( + new SegmentSeqAllocator(segment_type_t::JOURNAL)), journal_segment_allocator("JOURNAL", segment_type_t::JOURNAL, segment_provider, @@ -209,13 +210,16 @@ SegmentedJournal::replay_segment( if (delta.paddr != P_ADDR_NULL) { auto& seg_addr = delta.paddr.as_seg_paddr(); auto delta_paddr_segment_seq = segment_provider.get_seq(seg_addr.get_segment_id()); + auto delta_paddr_segment_type = segment_provider.get_type(seg_addr.get_segment_id()); if (s_type == segment_type_t::NULL_SEG || - (delta_paddr_segment_seq != delta.ext_seq)) { + (delta_paddr_segment_seq != delta.ext_seq || + delta_paddr_segment_type != delta.seg_type)) { SUBDEBUG(seastore_cache, - "delta is obsolete, delta_paddr_segment_seq={}, -- {}", + "delta is obsolete, delta_paddr_segment_seq={}," + " delta_paddr_segment_type={} -- {}", segment_seq_printer_t{delta_paddr_segment_seq}, + delta_paddr_segment_type, delta); - assert(delta_paddr_segment_seq > delta.ext_seq); return replay_ertr::now(); } } diff --git a/src/crimson/os/seastore/journal/segmented_journal.h b/src/crimson/os/seastore/journal/segmented_journal.h index 665a33ec938..e571583bcb9 100644 --- a/src/crimson/os/seastore/journal/segmented_journal.h +++ b/src/crimson/os/seastore/journal/segmented_journal.h @@ -46,9 +46,6 @@ public: write_pipeline = _write_pipeline; } - SegmentSeqAllocator& get_segment_seq_allocator() final { - return *segment_seq_allocator; - } private: submit_record_ret do_submit_record( record_t &&record, diff --git a/src/crimson/os/seastore/seastore_types.cc b/src/crimson/os/seastore/seastore_types.cc index a52ccfc5909..5e9365b38c0 100644 --- a/src/crimson/os/seastore/seastore_types.cc +++ b/src/crimson/os/seastore/seastore_types.cc @@ -199,6 +199,7 @@ std::ostream &operator<<(std::ostream &out, const delta_info_t &delta) << ", length: " << delta.length << ", pversion: " << delta.pversion << ", ext_seq: " << delta.ext_seq + << ", seg_type: " << delta.seg_type << ")"; } diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index 3afbf6571e4..d54add7c7c9 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -210,7 +210,7 @@ static constexpr segment_seq_t MAX_SEG_SEQ = static constexpr segment_seq_t NULL_SEG_SEQ = MAX_SEG_SEQ; static constexpr segment_seq_t MAX_VALID_SEG_SEQ = MAX_SEG_SEQ - 2; -enum class segment_type_t { +enum class segment_type_t : uint8_t { JOURNAL = 0, OOL, NULL_SEG, @@ -912,6 +912,7 @@ struct delta_info_t { seastore_off_t length = NULL_SEG_OFF; ///< extent length extent_version_t pversion; ///< prior version segment_seq_t ext_seq; ///< seq of the extent's segment + segment_type_t seg_type; ceph::bufferlist bl; ///< payload DENC(delta_info_t, v, p) { @@ -924,6 +925,7 @@ struct delta_info_t { denc(v.length, p); denc(v.pversion, p); denc(v.ext_seq, p); + denc(v.seg_type, p); denc(v.bl, p); DENC_FINISH(p); } diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc index 48f7b464fc8..006a196c197 100644 --- a/src/crimson/os/seastore/segment_cleaner.cc +++ b/src/crimson/os/seastore/segment_cleaner.cc @@ -183,6 +183,8 @@ SegmentCleaner::SegmentCleaner( : detailed(detailed), config(config), scanner(std::move(scr)), + ool_segment_seq_allocator( + new SegmentSeqAllocator(segment_type_t::OOL)), gc_process(*this) {} diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index 5f7a6a5e7c8..2cdc636c2a7 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -16,6 +16,7 @@ #include "crimson/os/seastore/seastore_types.h" #include "crimson/os/seastore/segment_manager.h" #include "crimson/os/seastore/transaction.h" +#include "crimson/os/seastore/segment_seq_allocator.h" namespace crimson::os::seastore { @@ -299,6 +300,8 @@ public: virtual segment_seq_t get_seq(segment_id_t id) { return 0; } + virtual segment_type_t get_type(segment_id_t id) = 0; + virtual seastar::lowres_system_clock::time_point get_last_modified( segment_id_t id) const = 0; @@ -718,12 +721,18 @@ private: /// populated if there is an IO blocked on hard limits std::optional> blocked_io_wake; + SegmentSeqAllocatorRef ool_segment_seq_allocator; + public: SegmentCleaner( config_t config, ExtentReaderRef&& scanner, bool detailed = false); + SegmentSeqAllocator& get_ool_segment_seq_allocator() { + return *ool_segment_seq_allocator; + } + using mount_ertr = crimson::errorator< crimson::ct_error::input_output_error>; using mount_ret = mount_ertr::future<>; @@ -763,6 +772,10 @@ public: return segments[id].journal_segment_seq; } + segment_type_t get_type(segment_id_t id) final { + return segments[id].get_type(); + } + void mark_segment_released(segment_id_t segment) { stats.segments_released++; return mark_empty(segment); @@ -1321,9 +1334,10 @@ private: segment_seq_t seq, segment_type_t s_type) { crimson::get_logger(ceph_subsys_seastore_cleaner).debug( - "SegmentCleaner::init_mark_segment_closed: segment {}, seq {}", + "SegmentCleaner::init_mark_segment_closed: segment {}, seq {}, s_type {}", segment, - segment_seq_printer_t{seq}); + segment_seq_printer_t{seq}, + s_type); mark_closed(segment); segments[segment].journal_segment_seq = seq; assert(s_type != segment_type_t::NULL_SEG); @@ -1331,6 +1345,9 @@ private: if (s_type == segment_type_t::JOURNAL) { assert(journal_device_id == segment.device_id()); segments.new_journal_segment(); + } else { + assert(s_type == segment_type_t::OOL); + ool_segment_seq_allocator->set_next_segment_seq(seq); } } @@ -1411,11 +1428,12 @@ private: assert(stats.empty_segments > 0); stats.empty_segments--; crimson::get_logger(ceph_subsys_seastore_cleaner - ).info("mark open: {} {}, empty_segments {}" + ).info("mark open: {} {} {}, empty_segments {}" ", opened_segments {}, should_block_on_gc {}" ", projected_avail_ratio {}, projected_reclaim_ratio {}", segment, segment_seq_printer_t{seq}, + segment_info.type, stats.empty_segments, segments.get_opened_segments(), should_block_on_gc(), diff --git a/src/crimson/os/seastore/segment_seq_allocator.h b/src/crimson/os/seastore/segment_seq_allocator.h index 9003b3a5a26..7bbca15572a 100644 --- a/src/crimson/os/seastore/segment_seq_allocator.h +++ b/src/crimson/os/seastore/segment_seq_allocator.h @@ -6,6 +6,10 @@ #include "crimson/os/seastore/logging.h" #include "crimson/os/seastore/seastore_types.h" +namespace crimson::os::seastore { +class SegmentCleaner; +} + namespace crimson::os::seastore::journal { class SegmentedJournal; } @@ -14,17 +18,30 @@ namespace crimson::os::seastore { class SegmentSeqAllocator { public: + SegmentSeqAllocator(segment_type_t type) + : type(type) {} segment_seq_t get_and_inc_next_segment_seq() { return next_segment_seq++; } private: void set_next_segment_seq(segment_seq_t seq) { LOG_PREFIX(SegmentSeqAllocator::set_next_segment_seq); - SUBINFO(seastore_journal, "next_segment_seq={}", segment_seq_printer_t{seq}); - next_segment_seq = seq; + SUBINFO( + seastore_journal, + "type {}, next_segment_seq={}, cur_segment_seq={}", + type, + segment_seq_printer_t{seq}, + segment_seq_printer_t{next_segment_seq}); + assert(type == segment_type_t::JOURNAL + ? seq >= next_segment_seq + : true); + if (seq > next_segment_seq) + next_segment_seq = seq; } segment_seq_t next_segment_seq = 0; + segment_type_t type = segment_type_t::NULL_SEG; friend class journal::SegmentedJournal; + friend class SegmentCleaner; }; using SegmentSeqAllocatorRef = diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index 4c7ff591ca6..b54e4475122 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -550,7 +550,7 @@ public: std::make_unique( *segment_cleaner, *sm, - journal->get_segment_seq_allocator())); + segment_cleaner->get_ool_segment_seq_allocator())); } ~TransactionManager(); diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 3e24f5accca..c36936c5024 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -42,7 +42,7 @@ struct btree_test_base : btree_test_base() = default; std::map segment_seqs; - + std::map segment_types; seastar::lowres_system_clock::time_point get_last_modified( @@ -59,13 +59,14 @@ struct btree_test_base : segment_id_t get_segment( device_id_t id, segment_seq_t seq, - segment_type_t) final + segment_type_t type) final { auto ret = next; next = segment_id_t{ next.device_id(), next.device_segment_id() + 1}; segment_seqs[ret] = seq; + segment_types[ret] = type; return ret; } @@ -73,6 +74,10 @@ struct btree_test_base : return segment_seqs[id]; } + segment_type_t get_type(segment_id_t id) { + return segment_types[id]; + } + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } void update_journal_tail_committed(journal_seq_t committed) final {} diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index f9d5dc4f77d..3599dc16033 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -81,6 +81,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { segment_id_t next; std::map segment_seqs; + std::map segment_types; journal_test_t() = default; @@ -99,13 +100,14 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { segment_id_t get_segment( device_id_t id, segment_seq_t seq, - segment_type_t) final + segment_type_t type) final { auto ret = next; next = segment_id_t{ next.device_id(), next.device_segment_id() + 1}; segment_seqs[ret] = seq; + segment_types[ret] = type; return ret; } @@ -113,6 +115,10 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { return segment_seqs[id]; } + segment_type_t get_type(segment_id_t id) final { + return segment_types[id]; + } + journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; } void update_journal_tail_committed(journal_seq_t paddr) final {} @@ -236,6 +242,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { block_size, 1, MAX_SEG_SEQ, + segment_type_t::NULL_SEG, bl }; } -- 2.39.5