From: Zhang Song Date: Tue, 10 Jan 2023 08:47:43 +0000 (+0800) Subject: crimson/os/seastore/EPM: refactor reserve process X-Git-Tag: v19.0.0~1546^2~17 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=86e2f4b24bff98b775a76289aa3fd382333d42ad;p=ceph.git crimson/os/seastore/EPM: refactor reserve process Signed-off-by: Zhang Song --- diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 78ca3796a655..b8fd2b18072a 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -264,15 +264,16 @@ ExtentPlacementManager::dispatch_delayed_extents(Transaction &t) for (auto &extent : t.get_inline_block_list()) { if (extent->is_valid()) { res.usage.inline_usage += extent->get_length(); + res.usage.cleaner_usage.main_usage += extent->get_length(); } } for (auto &extent : res.delayed_extents) { + res.usage.cleaner_usage.main_usage += extent->get_length(); if (dispatch_delayed_extent(extent)) { res.usage.inline_usage += extent->get_length(); t.mark_delayed_extent_inline(extent); } else { - res.usage.ool_usage += extent->get_length(); t.mark_delayed_extent_ool(extent); auto writer_ptr = get_writer( extent->get_user_hint(), @@ -417,29 +418,9 @@ ExtentPlacementManager::BackgroundProcess::run_until_halt() ); } -ExtentPlacementManager::BackgroundProcess::reserve_result_t -ExtentPlacementManager::BackgroundProcess::try_reserve( - const projected_usage_t &usage) -{ - reserve_result_t res { - trimmer->try_reserve_inline_usage(usage.inline_usage), - main_cleaner->try_reserve_projected_usage(usage.inline_usage + usage.ool_usage) - }; - - if (!res.is_successful()) { - if (res.reserve_inline_success) { - trimmer->release_inline_usage(usage.inline_usage); - } - if (res.reserve_ool_success) { - main_cleaner->release_projected_usage(usage.inline_usage + usage.ool_usage); - } - } - return res; -} - seastar::future<> ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( - projected_usage_t usage) + io_usage_t usage) { if (!is_ready()) { return seastar::now(); @@ -449,14 +430,15 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( // prepare until the prior one exits and clears this. ++stats.io_count; - auto res = try_reserve(usage); + auto res = try_reserve_io(usage); if (res.is_successful()) { return seastar::now(); } else { + abort_io_usage(usage, res); if (!res.reserve_inline_success) { ++stats.io_blocked_count_trim; } - if (!res.reserve_ool_success) { + if (!res.cleaner_result.is_successful()) { ++stats.io_blocked_count_clean; } ++stats.io_blocking_num; @@ -468,13 +450,14 @@ ExtentPlacementManager::BackgroundProcess::reserve_projected_usage( return blocking_io->get_future( ).then([this, usage] { ceph_assert(!blocking_io); - auto res = try_reserve(usage); + auto res = try_reserve_io(usage); if (res.is_successful()) { assert(stats.io_blocking_num == 1); --stats.io_blocking_num; return seastar::make_ready_future( seastar::stop_iteration::yes); } else { + abort_io_usage(usage, res); return seastar::make_ready_future( seastar::stop_iteration::no); } @@ -509,28 +492,72 @@ ExtentPlacementManager::BackgroundProcess::run() }); } +reserve_cleaner_result_t +ExtentPlacementManager::BackgroundProcess::try_reserve_cleaner( + const cleaner_usage_t &usage) +{ + return { + main_cleaner->try_reserve_projected_usage(usage.main_usage) + }; +} + +void ExtentPlacementManager::BackgroundProcess::abort_cleaner_usage( + const cleaner_usage_t &usage, + const reserve_cleaner_result_t &result) +{ + if (result.reserve_main_success) { + main_cleaner->release_projected_usage(usage.main_usage); + } +} + +reserve_io_result_t +ExtentPlacementManager::BackgroundProcess::try_reserve_io( + const io_usage_t &usage) +{ + return { + trimmer->try_reserve_inline_usage(usage.inline_usage), + try_reserve_cleaner(usage.cleaner_usage) + }; +} + +void ExtentPlacementManager::BackgroundProcess::abort_io_usage( + const io_usage_t &usage, + const reserve_io_result_t &result) +{ + if (result.reserve_inline_success) { + trimmer->release_inline_usage(usage.inline_usage); + } + abort_cleaner_usage(usage.cleaner_usage, result.cleaner_result); +} + seastar::future<> ExtentPlacementManager::BackgroundProcess::do_background_cycle() { assert(is_ready()); - bool trimmer_reserve_success = true; - if (trimmer->should_trim()) { - trimmer_reserve_success = - main_cleaner->try_reserve_projected_usage( - trimmer->get_trim_size_per_cycle()); + bool should_trim = trimmer->should_trim(); + bool proceed_trim = false; + auto trim_size = trimmer->get_trim_size_per_cycle(); + cleaner_usage_t trim_usage{trim_size}; + + if (should_trim) { + auto res = try_reserve_cleaner(trim_usage); + if (res.is_successful()) { + proceed_trim = true; + } else { + abort_cleaner_usage(trim_usage, trim_reserve_res); + } } - if (trimmer->should_trim() && trimmer_reserve_success) { + if (proceed_trim) { return trimmer->trim( - ).finally([this] { - main_cleaner->release_projected_usage( - trimmer->get_trim_size_per_cycle()); + ).finally([this, trim_usage] { + abort_cleaner_usage(trim_usage, {true}); }); } else if (main_cleaner->should_clean_space() || // make sure cleaner will start // when the trimmer should run but // failed to reserve space. - !trimmer_reserve_success) { + (should_trim && !proceed_trim)) { return main_cleaner->clean_space( ).handle_error( crimson::ct_error::assert_all{ @@ -609,12 +636,4 @@ RandomBlockOolWriter::do_write( }); } -std::ostream &operator<<(std::ostream &out, const ExtentPlacementManager::projected_usage_t &usage) -{ - return out << "projected_usage_t(" - << "inline_usage=" << usage.inline_usage - << ", ool_usage=" << usage.ool_usage << ")"; -} - } - diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index e5f8e53dc692..d700a0b6c8b4 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -128,6 +128,49 @@ private: seastar::gate write_guard; }; +struct cleaner_usage_t { + // The size of all extents write to the main devices, including inline extents + // and out-of-line extents. + std::size_t main_usage = 0; + // TODO: add cold_ool_usage +}; + +struct reserve_cleaner_result_t { + bool reserve_main_success = true; + + bool is_successful() const { + return reserve_main_success; + } +}; + +/** + * io_usage_t + * + * io_usage_t describes the space usage consumed by client IO. + */ +struct io_usage_t { + // The total size of all inlined extents, not including deltas and other metadata + // produced by Cache::prepare_record. + std::size_t inline_usage = 0; + cleaner_usage_t cleaner_usage; + friend std::ostream &operator<<(std::ostream &out, const io_usage_t &usage) { + return out << "io_usage_t(" + << "inline_usage=" << usage.inline_usage + << ", main_cleaner_usage=" << usage.cleaner_usage.main_usage + << ")"; + } +}; + +struct reserve_io_result_t { + bool reserve_inline_success = true; + reserve_cleaner_result_t cleaner_result; + + bool is_successful() const { + return reserve_inline_success && + cleaner_result.is_successful(); + } +}; + class ExtentPlacementManager { public: ExtentPlacementManager() { @@ -269,16 +312,12 @@ public: * delayed_extents is used to update lba mapping. * usage is used to reserve projected space */ - struct projected_usage_t { - std::size_t inline_usage = 0; - std::size_t ool_usage = 0; - }; using extents_by_writer_t = std::map>; struct dispatch_result_t { extents_by_writer_t alloc_map; std::list delayed_extents; - projected_usage_t usage; + io_usage_t usage; }; /** @@ -337,11 +376,11 @@ public: return background_process.commit_space_used(addr, len); } - seastar::future<> reserve_projected_usage(projected_usage_t usage) { + seastar::future<> reserve_projected_usage(io_usage_t usage) { return background_process.reserve_projected_usage(usage); } - void release_projected_usage(projected_usage_t usage) { + void release_projected_usage(const io_usage_t &usage) { background_process.release_projected_usage(usage); } @@ -478,12 +517,12 @@ private: return main_cleaner->commit_space_used(addr, len); } - seastar::future<> reserve_projected_usage(projected_usage_t usage); + seastar::future<> reserve_projected_usage(io_usage_t usage); - void release_projected_usage(projected_usage_t usage) { + void release_projected_usage(const io_usage_t &usage) { if (is_ready()) { trimmer->release_inline_usage(usage.inline_usage); - main_cleaner->release_projected_usage(usage.inline_usage + usage.ool_usage); + main_cleaner->release_projected_usage(usage.cleaner_usage.main_usage); } } @@ -529,6 +568,15 @@ private: } private: + // reserve helpers + reserve_cleaner_result_t try_reserve_cleaner(const cleaner_usage_t &usage); + void abort_cleaner_usage(const cleaner_usage_t &usage, + const reserve_cleaner_result_t &result); + + reserve_io_result_t try_reserve_io(const io_usage_t &usage); + void abort_io_usage(const io_usage_t &usage, + const reserve_io_result_t &result); + bool is_running() const { if (state == state_t::RUNNING) { assert(process_join); @@ -563,17 +611,6 @@ private: main_cleaner->should_block_io_on_clean(); } - struct reserve_result_t { - bool reserve_inline_success = true; - bool reserve_ool_success = true; - - bool is_successful() const { - return reserve_inline_success && reserve_ool_success; - } - }; - - reserve_result_t try_reserve(const projected_usage_t &usage); - seastar::future<> do_background_cycle(); void register_metrics(); @@ -612,12 +649,8 @@ private: using ExtentPlacementManagerRef = std::unique_ptr; -std::ostream &operator<<(std::ostream &, const ExtentPlacementManager::projected_usage_t &); - } #if FMT_VERSION >= 90000 -template <> -struct fmt::formatter - : fmt::ostream_formatter {}; +template <> struct fmt::formatter : fmt::ostream_formatter {}; #endif