From 7de7f9524631dfac1f3a00579372a949e623e575 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 23 Sep 2021 14:30:06 +0800 Subject: [PATCH] crimson/os/seastore: make segment managers device-id aware Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/segment_manager.h | 7 ++-- .../os/seastore/segment_manager/block.cc | 35 ++++++++++++------- .../os/seastore/segment_manager/block.h | 18 +++++++--- .../os/seastore/segment_manager/ephemeral.cc | 21 ++++++----- .../os/seastore/segment_manager/ephemeral.h | 13 +++++-- .../seastore/transaction_manager_test_state.h | 10 +++++- 6 files changed, 73 insertions(+), 31 deletions(-) diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h index 192564a4b18..53a5ff7a514 100644 --- a/src/crimson/os/seastore/segment_manager.h +++ b/src/crimson/os/seastore/segment_manager.h @@ -7,6 +7,7 @@ #include #include +#include #include #include "include/ceph_assert.h" @@ -128,12 +129,14 @@ public: virtual size_t get_size() const = 0; virtual segment_off_t get_block_size() const = 0; virtual segment_off_t get_segment_size() const = 0; - virtual segment_id_t get_num_segments() const { + virtual device_segment_id_t get_num_segments() const { ceph_assert(get_size() % get_segment_size() == 0); - return ((segment_id_t)(get_size() / get_segment_size())); + return ((device_segment_id_t)(get_size() / get_segment_size())); } virtual const seastore_meta_t &get_meta() const = 0; + virtual device_id_t get_device_id() const = 0; + virtual ~SegmentManager() {} }; using SegmentManagerRef = std::unique_ptr; diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index b1970838644..12e45d88a6b 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -343,8 +343,10 @@ Segment::write_ertr::future<> BlockSegment::write( Segment::close_ertr::future<> BlockSegmentManager::segment_close( segment_id_t id, segment_off_t write_pointer) { + assert(id.device_id() == superblock.device_id); + auto s_id = id.device_segment_id(); assert(tracker); - tracker->set(id, segment_state_t::CLOSED); + tracker->set(s_id, segment_state_t::CLOSED); ++stats.closed_segments; int unused_bytes = get_segment_size() - write_pointer; assert(unused_bytes >= 0); @@ -358,6 +360,7 @@ Segment::write_ertr::future<> BlockSegmentManager::segment_write( ceph::bufferlist bl, bool ignore_check) { + assert(addr.segment.device_id() == get_device_id()); assert((bl.length() % superblock.block_size) == 0); logger().debug( "BlockSegmentManager::segment_write: " @@ -394,7 +397,7 @@ BlockSegmentManager::mount_ret BlockSegmentManager::mount() device, superblock.tracker_offset ).safe_then([this] { - for (segment_id_t i = 0; i < tracker->get_capacity(); ++i) { + for (device_segment_id_t i = 0; i < tracker->get_capacity(); ++i) { if (tracker->get(i) == segment_state_t::OPEN) { tracker->set(i, segment_state_t::CLOSED); } @@ -446,26 +449,29 @@ BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs(seastore_meta_t meta) BlockSegmentManager::close_ertr::future<> BlockSegmentManager::close() { + metrics.clear(); return device.close(); } SegmentManager::open_ertr::future BlockSegmentManager::open( segment_id_t id) { - if (id >= get_num_segments()) { + assert(id.device_id() == superblock.device_id); + auto s_id = id.device_segment_id(); + if (s_id >= get_num_segments()) { logger().error("BlockSegmentManager::open: invalid segment {}", id); return crimson::ct_error::invarg::make(); } - if (tracker->get(id) != segment_state_t::EMPTY) { + if (tracker->get(s_id) != segment_state_t::EMPTY) { logger().error( "BlockSegmentManager::open: invalid segment {} state {}", id, - tracker->get(id)); + tracker->get(s_id)); return crimson::ct_error::invarg::make(); } - tracker->set(id, segment_state_t::OPEN); + tracker->set(s_id, segment_state_t::OPEN); stats.metadata_write.increment(tracker->get_size()); return tracker->write_out(device, superblock.tracker_offset ).safe_then([this, id] { @@ -479,24 +485,26 @@ SegmentManager::open_ertr::future BlockSegmentManager::open( SegmentManager::release_ertr::future<> BlockSegmentManager::release( segment_id_t id) { + assert(id.device_id() == superblock.device_id); + auto s_id = id.device_segment_id(); logger().debug("BlockSegmentManager::release: {}", id); - if (id >= get_num_segments()) { + if (s_id >= get_num_segments()) { logger().error( "BlockSegmentManager::release: invalid segment {}", id); return crimson::ct_error::invarg::make(); } - if (tracker->get(id) != segment_state_t::CLOSED) { + if (tracker->get(s_id) != segment_state_t::CLOSED) { logger().error( "BlockSegmentManager::release: invalid segment {} state {}", id, - tracker->get(id)); + tracker->get(s_id)); return crimson::ct_error::invarg::make(); } - tracker->set(id, segment_state_t::EMPTY); + tracker->set(s_id, segment_state_t::EMPTY); ++stats.released_segments; stats.metadata_write.increment(tracker->get_size()); return tracker->write_out(device, superblock.tracker_offset); @@ -507,7 +515,8 @@ SegmentManager::read_ertr::future<> BlockSegmentManager::read( size_t len, ceph::bufferptr &out) { - if (addr.segment >= get_num_segments()) { + assert(addr.segment.device_id() == get_device_id()); + if (addr.segment.device_segment_id() >= get_num_segments()) { logger().error( "BlockSegmentManager::read: invalid segment {}", addr); @@ -522,11 +531,11 @@ SegmentManager::read_ertr::future<> BlockSegmentManager::read( return crimson::ct_error::invarg::make(); } - if (tracker->get(addr.segment) == segment_state_t::EMPTY) { + if (tracker->get(addr.segment.device_segment_id()) == segment_state_t::EMPTY) { logger().error( "BlockSegmentManager::read: read on invalid segment {} state {}", addr.segment, - tracker->get(addr.segment)); + tracker->get(addr.segment.device_segment_id())); return crimson::ct_error::enoent::make(); } diff --git a/src/crimson/os/seastore/segment_manager/block.h b/src/crimson/os/seastore/segment_manager/block.h index 860401baeeb..2712243e3b4 100644 --- a/src/crimson/os/seastore/segment_manager/block.h +++ b/src/crimson/os/seastore/segment_manager/block.h @@ -83,14 +83,14 @@ public: return bptr.length(); } - segment_state_t get(segment_id_t offset) const { + segment_state_t get(device_segment_id_t offset) const { assert(offset < get_capacity()); return static_cast( layout.template Pointer<0>( bptr.c_str())[offset]); } - void set(segment_id_t offset, segment_state_t state) { + void set(device_segment_id_t offset, segment_state_t state) { assert(offset < get_capacity()); layout.template Pointer<0>(bptr.c_str())[offset] = static_cast(state); @@ -141,7 +141,11 @@ public: >; close_ertr::future<> close(); - BlockSegmentManager(const std::string &path) : device_path(path) { + BlockSegmentManager( + const std::string &path, + device_id_t device_id = 0) + : device_path(path), + device_id(device_id) { register_metrics(); } ~BlockSegmentManager(); @@ -165,6 +169,10 @@ public: return superblock.segment_size; } + device_id_t get_device_id() const final { + return device_id; + } + // public so tests can bypass segment interface when simpler Segment::write_ertr::future<> segment_write( paddr_t addr, @@ -213,9 +221,11 @@ private: block_sm_superblock_t superblock; seastar::file device; + device_id_t device_id = 0; + size_t get_offset(paddr_t addr) { return superblock.first_segment_offset + - (addr.segment * superblock.segment_size) + + (addr.segment.device_segment_id() * superblock.segment_size) + addr.offset; } diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.cc b/src/crimson/os/seastore/segment_manager/ephemeral.cc index 77013ec838d..244a0806c6c 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.cc +++ b/src/crimson/os/seastore/segment_manager/ephemeral.cc @@ -59,10 +59,11 @@ Segment::write_ertr::future<> EphemeralSegment::write( Segment::close_ertr::future<> EphemeralSegmentManager::segment_close(segment_id_t id) { - if (segment_state[id] != segment_state_t::OPEN) + auto s_id = id.device_segment_id(); + if (segment_state[s_id] != segment_state_t::OPEN) return crimson::ct_error::invarg::make(); - segment_state[id] = segment_state_t::CLOSED; + segment_state[s_id] = segment_state_t::CLOSED; return Segment::close_ertr::now().safe_then([] { return seastar::sleep(std::chrono::milliseconds(1)); }); @@ -80,7 +81,7 @@ Segment::write_ertr::future<> EphemeralSegmentManager::segment_write( get_offset(addr), bl.length(), bl.crc32c(1)); - if (!ignore_check && segment_state[addr.segment] != segment_state_t::OPEN) + if (!ignore_check && segment_state[addr.segment.device_segment_id()] != segment_state_t::OPEN) return crimson::ct_error::invarg::make(); bl.begin().copy(bl.length(), buffer + get_offset(addr)); @@ -145,33 +146,35 @@ void EphemeralSegmentManager::remount() SegmentManager::open_ertr::future EphemeralSegmentManager::open( segment_id_t id) { - if (id >= get_num_segments()) { + auto s_id = id.device_segment_id(); + if (s_id >= get_num_segments()) { logger().error("EphemeralSegmentManager::open: invalid segment {}", id); return crimson::ct_error::invarg::make(); } - if (segment_state[id] != segment_state_t::EMPTY) { + if (segment_state[s_id] != segment_state_t::EMPTY) { logger().error("EphemeralSegmentManager::open: segment {} not empty", id); return crimson::ct_error::invarg::make(); } - segment_state[id] = segment_state_t::OPEN; + segment_state[s_id] = segment_state_t::OPEN; return open_ertr::make_ready_future(new EphemeralSegment(*this, id)); } SegmentManager::release_ertr::future<> EphemeralSegmentManager::release( segment_id_t id) { + auto s_id = id.device_segment_id(); logger().debug("EphemeralSegmentManager::release: {}", id); - if (id >= get_num_segments()) { + if (s_id >= get_num_segments()) { logger().error( "EphemeralSegmentManager::release: invalid segment {}", id); return crimson::ct_error::invarg::make(); } - if (segment_state[id] != segment_state_t::CLOSED) { + if (segment_state[s_id] != segment_state_t::CLOSED) { logger().error( "EphemeralSegmentManager::release: segment id {} not closed", id); @@ -179,7 +182,7 @@ SegmentManager::release_ertr::future<> EphemeralSegmentManager::release( } ::memset(buffer + get_offset({id, 0}), 0, config.segment_size); - segment_state[id] = segment_state_t::EMPTY; + segment_state[s_id] = segment_state_t::EMPTY; return release_ertr::now().safe_then([] { return seastar::sleep(std::chrono::milliseconds(1)); }); diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.h b/src/crimson/os/seastore/segment_manager/ephemeral.h index 2096fab3910..fd8b28c43e2 100644 --- a/src/crimson/os/seastore/segment_manager/ephemeral.h +++ b/src/crimson/os/seastore/segment_manager/ephemeral.h @@ -56,7 +56,7 @@ class EphemeralSegmentManager final : public SegmentManager { std::optional meta; size_t get_offset(paddr_t addr) { - return (addr.segment * config.segment_size) + addr.offset; + return (addr.segment.device_segment_id() * config.segment_size) + addr.offset; } std::vector segment_state; @@ -65,10 +65,19 @@ class EphemeralSegmentManager final : public SegmentManager { Segment::close_ertr::future<> segment_close(segment_id_t id); + device_id_t device_id = 0; + public: - EphemeralSegmentManager(ephemeral_config_t config) : config(config) {} + EphemeralSegmentManager( + ephemeral_config_t config, + device_id_t device_id = 0) + : config(config), device_id(device_id) {} ~EphemeralSegmentManager(); + device_id_t get_device_id() const { + return device_id; + } + using init_ertr = crimson::errorator< crimson::ct_error::enospc, crimson::ct_error::invarg, diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 9b2af2e779b..bec8cb1dcf3 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -192,8 +192,16 @@ protected: class TestSegmentManagerWrapper final : public SegmentManager { SegmentManager &sm; + device_id_t device_id = 0; public: - TestSegmentManagerWrapper(SegmentManager &sm) : sm(sm) {} + TestSegmentManagerWrapper( + SegmentManager &sm, + device_id_t device_id = 0) + : sm(sm), device_id(device_id) {} + + device_id_t get_device_id() const { + return device_id; + } mount_ret mount() final { return mount_ertr::now(); // we handle this above -- 2.39.5