From 9521fc2ab8cf0a31d15a14c8d3fe2c2699f36f81 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Mon, 21 Mar 2022 15:47:16 +0800 Subject: [PATCH] crimson/os/seastore: introduce the generic Device class Implement the read interface of Device and integrate it with EPM, so that Cache doesn't need to depend on the segment specific ExtentReader class. Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/cache.cc | 4 +- src/crimson/os/seastore/cache.h | 5 +- src/crimson/os/seastore/device.h | 46 +++++++++++++++++++ .../os/seastore/extent_placement_manager.h | 23 +++++++++- src/crimson/os/seastore/extent_reader.h | 8 ---- src/crimson/os/seastore/segment_manager.h | 30 ++---------- .../os/seastore/transaction_manager.cc | 2 +- src/crimson/os/seastore/transaction_manager.h | 1 + .../seastore/test_btree_lba_manager.cc | 3 +- .../crimson/seastore/test_seastore_cache.cc | 7 +-- 10 files changed, 82 insertions(+), 47 deletions(-) create mode 100644 src/crimson/os/seastore/device.h diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index c3d0c3fc9c36c..50c2a9b72ecce 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -27,10 +27,8 @@ SET_SUBSYS(seastore_cache); namespace crimson::os::seastore { Cache::Cache( - ExtentReader &reader, ExtentPlacementManager &epm) - : reader(reader), - epm(epm), + : epm(epm), lru(crimson::common::get_conf( "seastore_cache_lru_size")) { diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index b2f5b213aceab..789e5fa7a8577 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -100,7 +100,7 @@ public: crimson::ct_error::input_output_error>; using base_iertr = trans_iertr; - Cache(ExtentReader &reader, ExtentPlacementManager &epm); + Cache(ExtentPlacementManager &epm); ~Cache(); /// Creates empty transaction by source @@ -739,7 +739,6 @@ public: void dump_contents(); private: - ExtentReader &reader; ///< ref to extent reader ExtentPlacementManager& epm; RootBlockRef root; ///< ref to current root ExtentIndex extents; ///< set of live extents @@ -1020,7 +1019,7 @@ private: ) { assert(extent->state == CachedExtent::extent_state_t::CLEAN_PENDING); extent->set_io_wait(); - return reader.read( + return epm.read( extent->get_paddr(), extent->get_length(), extent->get_bptr() diff --git a/src/crimson/os/seastore/device.h b/src/crimson/os/seastore/device.h new file mode 100644 index 0000000000000..bc050a0225fc8 --- /dev/null +++ b/src/crimson/os/seastore/device.h @@ -0,0 +1,46 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*- +// vim: ts=8 sw=2 smarttab + +#pragma once + +#include + +#include "include/buffer_fwd.h" + +#include "crimson/common/errorator.h" + +namespace crimson::os::seastore { + +/** + * Device + * + * Represents a general device regardless of the underlying medium. + */ +class Device { +public: + virtual device_id_t get_device_id() const = 0; + + using read_ertr = crimson::errorator< + crimson::ct_error::input_output_error, + crimson::ct_error::invarg, + crimson::ct_error::enoent, + crimson::ct_error::erange>; + virtual read_ertr::future<> read( + paddr_t addr, + size_t len, + ceph::bufferptr &out) = 0; + + read_ertr::future read( + paddr_t addr, + size_t len + ) { + auto ptrref = std::make_unique( + buffer::create_page_aligned(len)); + return read(addr, len, *ptrref + ).safe_then([ptrref=std::move(ptrref)]() mutable { + return read_ertr::make_ready_future(std::move(*ptrref)); + }); + } +}; + +} diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index fcf1375e3bd84..e7dddf3534152 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -171,7 +171,15 @@ private: class ExtentPlacementManager { public: - ExtentPlacementManager() = default; + ExtentPlacementManager() { + devices_by_id.resize(DEVICE_ID_MAX, nullptr); + } + + void add_device(Device* device) { + auto device_id = device->get_device_id(); + ceph_assert(devices_by_id[device_id] == nullptr); + devices_by_id[device_id] = device; + } void add_allocator(device_type_t type, ExtentAllocatorRef&& allocator) { allocators[type].emplace_back(std::move(allocator)); @@ -270,9 +278,21 @@ public: }); }).safe_then([this] { allocators.clear(); + devices_by_id.clear(); + devices_by_id.resize(DEVICE_ID_MAX, nullptr); }); } + using read_ertr = Device::read_ertr; + read_ertr::future<> read( + paddr_t addr, + size_t len, + ceph::bufferptr &out + ) { + assert(devices_by_id[addr.get_device_id()] != nullptr); + return devices_by_id[addr.get_device_id()]->read(addr, len, out); + } + private: device_type_t get_allocator_type(placement_hint_t hint) { return device_type_t::SEGMENTED; @@ -286,6 +306,7 @@ private: } std::map> allocators; + std::vector devices_by_id; }; using ExtentPlacementManagerRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/extent_reader.h b/src/crimson/os/seastore/extent_reader.h index 1016f50f23846..4d77dbe6ef813 100644 --- a/src/crimson/os/seastore/extent_reader.h +++ b/src/crimson/os/seastore/extent_reader.h @@ -85,14 +85,6 @@ public: segment_managers[segment_manager->get_device_id()] = segment_manager; } - read_ertr::future<> read( - paddr_t addr, - size_t len, - ceph::bufferptr &out) { - assert(segment_managers[addr.get_device_id()]); - return segment_managers[addr.get_device_id()]->read(addr, len, out); - } - void reset() { segment_managers.clear(); segment_managers.resize(DEVICE_ID_MAX, nullptr); diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 235a9d1cad505..644aaa71444a8 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -10,11 +10,13 @@ #include #include -#include "include/ceph_assert.h" -#include "crimson/os/seastore/seastore_types.h" #include "include/buffer_fwd.h" +#include "include/ceph_assert.h" + #include "crimson/common/config_proxy.h" +#include "crimson/os/seastore/seastore_types.h" #include "crimson/osd/exceptions.h" +#include "device.h" namespace crimson::os::seastore { @@ -178,7 +180,7 @@ class SegmentManager; using SegmentManagerRef = std::unique_ptr; -class SegmentManager { +class SegmentManager : public Device { public: using access_ertr = crimson::errorator< crimson::ct_error::input_output_error, @@ -210,26 +212,6 @@ public: crimson::ct_error::enoent>; virtual release_ertr::future<> release(segment_id_t id) = 0; - using read_ertr = crimson::errorator< - crimson::ct_error::input_output_error, - crimson::ct_error::invarg, - crimson::ct_error::enoent, - crimson::ct_error::erange>; - virtual read_ertr::future<> read( - paddr_t addr, - size_t len, - ceph::bufferptr &out) = 0; - read_ertr::future read( - paddr_t addr, - size_t len) { - auto ptrref = std::make_unique( - buffer::create_page_aligned(len)); - return read(addr, len, *ptrref).safe_then( - [ptrref=std::move(ptrref)]() mutable { - return read_ertr::make_ready_future(std::move(*ptrref)); - }); - } - /* Methods for discovering device geometry, segmentid set, etc */ virtual size_t get_size() const = 0; virtual seastore_off_t get_block_size() const = 0; @@ -245,8 +227,6 @@ public: } virtual const seastore_meta_t &get_meta() const = 0; - virtual device_id_t get_device_id() const = 0; - virtual secondary_device_set_t& get_secondary_devices() = 0; virtual device_spec_t get_device_spec() const = 0; diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index a01652601d0c1..237e90966d24a 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -567,7 +567,7 @@ TransactionManagerRef make_transaction_manager( detailed); auto journal = journal::make_segmented(sm, scanner_ref, *segment_cleaner); auto epm = std::make_unique(); - auto cache = std::make_unique(scanner_ref, *epm); + auto cache = std::make_unique(*epm); auto lba_manager = lba_manager::create_lba_manager(sm, *cache); return std::make_unique( diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h index b54e44751225d..34171b63f98a7 100644 --- a/src/crimson/os/seastore/transaction_manager.h +++ b/src/crimson/os/seastore/transaction_manager.h @@ -545,6 +545,7 @@ public: LOG_PREFIX(TransactionManager::add_segment_manager); SUBDEBUG(seastore_tm, "adding segment manager {}", sm->get_device_id()); scanner.add_segment_manager(sm); + epm->add_device(sm); epm->add_allocator( device_type_t::SEGMENTED, std::make_unique( diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index c36936c5024ed..79bbce63c6aa9 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -103,11 +103,12 @@ struct btree_test_base : journal = journal::make_segmented( *segment_manager, scanner_ref, *this); epm.reset(new ExtentPlacementManager()); - cache.reset(new Cache(scanner_ref, *epm)); + cache.reset(new Cache(*epm)); block_size = segment_manager->get_block_size(); next = segment_id_t{segment_manager->get_device_id(), 0}; scanner_ref.add_segment_manager(segment_manager.get()); + epm->add_device(segment_manager.get()); journal->set_write_pipeline(&pipeline); return segment_manager->init( diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 979883be44f35..94efc4fd48db4 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -21,7 +21,6 @@ namespace { struct cache_test_t : public seastar_test_suite_t { segment_manager::EphemeralSegmentManagerRef segment_manager; - ExtentReaderRef reader; ExtentPlacementManagerRef epm; CacheRef cache; paddr_t current; @@ -84,11 +83,10 @@ struct cache_test_t : public seastar_test_suite_t { seastar::future<> set_up_fut() final { segment_manager = segment_manager::create_test_ephemeral(); - reader.reset(new ExtentReader()); epm.reset(new ExtentPlacementManager()); - cache.reset(new Cache(*reader, *epm)); + cache.reset(new Cache(*epm)); current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0); - reader->add_segment_manager(segment_manager.get()); + epm->add_device(segment_manager.get()); return segment_manager->init( ).safe_then([this] { return seastar::do_with( @@ -113,7 +111,6 @@ struct cache_test_t : public seastar_test_suite_t { return cache->close( ).safe_then([this] { segment_manager.reset(); - reader.reset(); epm.reset(); cache.reset(); }).handle_error( -- 2.39.5