crimson/os/seastore: make segment managers device-id aware

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>

diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h
index 192564a4b18c7518a823a41e85c8593261bd3105..53a5ff7a514598a8a2f92bec4fcb4d36a3851573 100644 (file)
--- a/src/crimson/os/seastore/segment_manager.h
+++ b/src/crimson/os/seastore/segment_manager.h
@@ -7,6 +7,7 @@
 
 #include <boost/intrusive_ptr.hpp>
 #include <boost/smart_ptr/intrusive_ref_counter.hpp>
+#include <boost/iterator/counting_iterator.hpp>
 #include <seastar/core/future.hh>
 
 #include "include/ceph_assert.h"
@@ -128,12 +129,14 @@ public:
   virtual size_t get_size() const = 0;
   virtual segment_off_t get_block_size() const = 0;
   virtual segment_off_t get_segment_size() const = 0;
-  virtual segment_id_t get_num_segments() const {
+  virtual device_segment_id_t get_num_segments() const {
     ceph_assert(get_size() % get_segment_size() == 0);
-    return ((segment_id_t)(get_size() / get_segment_size()));
+    return ((device_segment_id_t)(get_size() / get_segment_size()));
   }
   virtual const seastore_meta_t &get_meta() const = 0;
 
+  virtual device_id_t get_device_id() const = 0;
+
   virtual ~SegmentManager() {}
 };
 using SegmentManagerRef = std::unique_ptr<SegmentManager>;

diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc
index b197083864463ed3549097fb6d6eefdb6ae4afc6..12e45d88a6bd842d24c96f2b473f1a1faf1dbb6f 100644 (file)
--- a/src/crimson/os/seastore/segment_manager/block.cc
+++ b/src/crimson/os/seastore/segment_manager/block.cc
@@ -343,8 +343,10 @@ Segment::write_ertr::future<> BlockSegment::write(
 Segment::close_ertr::future<> BlockSegmentManager::segment_close(
     segment_id_t id, segment_off_t write_pointer)
 {
+  assert(id.device_id() == superblock.device_id);
+  auto s_id = id.device_segment_id();
   assert(tracker);
-  tracker->set(id, segment_state_t::CLOSED);
+  tracker->set(s_id, segment_state_t::CLOSED);
   ++stats.closed_segments;
   int unused_bytes = get_segment_size() - write_pointer;
   assert(unused_bytes >= 0);
@@ -358,6 +360,7 @@ Segment::write_ertr::future<> BlockSegmentManager::segment_write(
   ceph::bufferlist bl,
   bool ignore_check)
 {
+  assert(addr.segment.device_id() == get_device_id());
   assert((bl.length() % superblock.block_size) == 0);
   logger().debug(
     "BlockSegmentManager::segment_write: "
@@ -394,7 +397,7 @@ BlockSegmentManager::mount_ret BlockSegmentManager::mount()
       device,
       superblock.tracker_offset
     ).safe_then([this] {
-      for (segment_id_t i = 0; i < tracker->get_capacity(); ++i) {
+      for (device_segment_id_t i = 0; i < tracker->get_capacity(); ++i) {
        if (tracker->get(i) == segment_state_t::OPEN) {
          tracker->set(i, segment_state_t::CLOSED);
        }
@@ -446,26 +449,29 @@ BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs(seastore_meta_t meta)
 
 BlockSegmentManager::close_ertr::future<> BlockSegmentManager::close()
 {
+  metrics.clear();
   return device.close();
 }
 
 SegmentManager::open_ertr::future<SegmentRef> BlockSegmentManager::open(
   segment_id_t id)
 {
-  if (id >= get_num_segments()) {
+  assert(id.device_id() == superblock.device_id);
+  auto s_id = id.device_segment_id();
+  if (s_id >= get_num_segments()) {
     logger().error("BlockSegmentManager::open: invalid segment {}", id);
     return crimson::ct_error::invarg::make();
   }
 
-  if (tracker->get(id) != segment_state_t::EMPTY) {
+  if (tracker->get(s_id) != segment_state_t::EMPTY) {
     logger().error(
       "BlockSegmentManager::open: invalid segment {} state {}",
       id,
-      tracker->get(id));
+      tracker->get(s_id));
     return crimson::ct_error::invarg::make();
   }
 
-  tracker->set(id, segment_state_t::OPEN);
+  tracker->set(s_id, segment_state_t::OPEN);
   stats.metadata_write.increment(tracker->get_size());
   return tracker->write_out(device, superblock.tracker_offset
   ).safe_then([this, id] {
@@ -479,24 +485,26 @@ SegmentManager::open_ertr::future<SegmentRef> BlockSegmentManager::open(
 SegmentManager::release_ertr::future<> BlockSegmentManager::release(
   segment_id_t id)
 {
+  assert(id.device_id() == superblock.device_id);
+  auto s_id = id.device_segment_id();
   logger().debug("BlockSegmentManager::release: {}", id);
 
-  if (id >= get_num_segments()) {
+  if (s_id >= get_num_segments()) {
     logger().error(
       "BlockSegmentManager::release: invalid segment {}",
       id);
     return crimson::ct_error::invarg::make();
   }
 
-  if (tracker->get(id) != segment_state_t::CLOSED) {
+  if (tracker->get(s_id) != segment_state_t::CLOSED) {
     logger().error(
       "BlockSegmentManager::release: invalid segment {} state {}",
       id,
-      tracker->get(id));
+      tracker->get(s_id));
     return crimson::ct_error::invarg::make();
   }
 
-  tracker->set(id, segment_state_t::EMPTY);
+  tracker->set(s_id, segment_state_t::EMPTY);
   ++stats.released_segments;
   stats.metadata_write.increment(tracker->get_size());
   return tracker->write_out(device, superblock.tracker_offset);
@@ -507,7 +515,8 @@ SegmentManager::read_ertr::future<> BlockSegmentManager::read(
   size_t len,
   ceph::bufferptr &out)
 {
-  if (addr.segment >= get_num_segments()) {
+  assert(addr.segment.device_id() == get_device_id());
+  if (addr.segment.device_segment_id() >= get_num_segments()) {
     logger().error(
       "BlockSegmentManager::read: invalid segment {}",
       addr);
@@ -522,11 +531,11 @@ SegmentManager::read_ertr::future<> BlockSegmentManager::read(
     return crimson::ct_error::invarg::make();
   }
 
-  if (tracker->get(addr.segment) == segment_state_t::EMPTY) {
+  if (tracker->get(addr.segment.device_segment_id()) == segment_state_t::EMPTY) {
     logger().error(
       "BlockSegmentManager::read: read on invalid segment {} state {}",
       addr.segment,
-      tracker->get(addr.segment));
+      tracker->get(addr.segment.device_segment_id()));
     return crimson::ct_error::enoent::make();
   }
 

diff --git a/src/crimson/os/seastore/segment_manager/block.h b/src/crimson/os/seastore/segment_manager/block.h
index 860401baeeb9fd81541404ffa1fde9e8d389907d..2712243e3b40fdb2b43ec7440fbc7adf4181fa8a 100644 (file)
--- a/src/crimson/os/seastore/segment_manager/block.h
+++ b/src/crimson/os/seastore/segment_manager/block.h
@@ -83,14 +83,14 @@ public:
     return bptr.length();
   }
 
-  segment_state_t get(segment_id_t offset) const {
+  segment_state_t get(device_segment_id_t offset) const {
     assert(offset < get_capacity());
     return static_cast<segment_state_t>(
       layout.template Pointer<0>(
        bptr.c_str())[offset]);
   }
 
-  void set(segment_id_t offset, segment_state_t state) {
+  void set(device_segment_id_t offset, segment_state_t state) {
     assert(offset < get_capacity());
     layout.template Pointer<0>(bptr.c_str())[offset] =
       static_cast<uint8_t>(state);
@@ -141,7 +141,11 @@ public:
     >;
   close_ertr::future<> close();
 
-  BlockSegmentManager(const std::string &path) : device_path(path) {
+  BlockSegmentManager(
+    const std::string &path,
+    device_id_t device_id = 0)
+  : device_path(path),
+    device_id(device_id) {
     register_metrics();
   }
   ~BlockSegmentManager();
@@ -165,6 +169,10 @@ public:
     return superblock.segment_size;
   }
 
+  device_id_t get_device_id() const final {
+    return device_id;
+  }
+
   // public so tests can bypass segment interface when simpler
   Segment::write_ertr::future<> segment_write(
     paddr_t addr,
@@ -213,9 +221,11 @@ private:
   block_sm_superblock_t superblock;
   seastar::file device;
 
+  device_id_t device_id = 0;
+
   size_t get_offset(paddr_t addr) {
     return superblock.first_segment_offset +
-      (addr.segment * superblock.segment_size) +
+      (addr.segment.device_segment_id() * superblock.segment_size) +
       addr.offset;
   }
 

diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.cc b/src/crimson/os/seastore/segment_manager/ephemeral.cc
index 77013ec838d79abbc5a33a18081c4cabc526bc13..244a0806c6c0717efa5311ae4f2d9a71146e9b88 100644 (file)
--- a/src/crimson/os/seastore/segment_manager/ephemeral.cc
+++ b/src/crimson/os/seastore/segment_manager/ephemeral.cc
@@ -59,10 +59,11 @@ Segment::write_ertr::future<> EphemeralSegment::write(
 
 Segment::close_ertr::future<> EphemeralSegmentManager::segment_close(segment_id_t id)
 {
-  if (segment_state[id] != segment_state_t::OPEN)
+  auto s_id = id.device_segment_id();
+  if (segment_state[s_id] != segment_state_t::OPEN)
     return crimson::ct_error::invarg::make();
 
-  segment_state[id] = segment_state_t::CLOSED;
+  segment_state[s_id] = segment_state_t::CLOSED;
   return Segment::close_ertr::now().safe_then([] {
     return seastar::sleep(std::chrono::milliseconds(1));
   });
@@ -80,7 +81,7 @@ Segment::write_ertr::future<> EphemeralSegmentManager::segment_write(
     get_offset(addr),
     bl.length(),
     bl.crc32c(1));
-  if (!ignore_check && segment_state[addr.segment] != segment_state_t::OPEN)
+  if (!ignore_check && segment_state[addr.segment.device_segment_id()] != segment_state_t::OPEN)
     return crimson::ct_error::invarg::make();
 
   bl.begin().copy(bl.length(), buffer + get_offset(addr));
@@ -145,33 +146,35 @@ void EphemeralSegmentManager::remount()
 SegmentManager::open_ertr::future<SegmentRef> EphemeralSegmentManager::open(
   segment_id_t id)
 {
-  if (id >= get_num_segments()) {
+  auto s_id = id.device_segment_id();
+  if (s_id >= get_num_segments()) {
     logger().error("EphemeralSegmentManager::open: invalid segment {}", id);
     return crimson::ct_error::invarg::make();
   }
 
-  if (segment_state[id] != segment_state_t::EMPTY) {
+  if (segment_state[s_id] != segment_state_t::EMPTY) {
     logger().error("EphemeralSegmentManager::open: segment {} not empty", id);
     return crimson::ct_error::invarg::make();
   }
 
-  segment_state[id] = segment_state_t::OPEN;
+  segment_state[s_id] = segment_state_t::OPEN;
   return open_ertr::make_ready_future<SegmentRef>(new EphemeralSegment(*this, id));
 }
 
 SegmentManager::release_ertr::future<> EphemeralSegmentManager::release(
   segment_id_t id)
 {
+  auto s_id = id.device_segment_id();
   logger().debug("EphemeralSegmentManager::release: {}", id);
 
-  if (id >= get_num_segments()) {
+  if (s_id >= get_num_segments()) {
     logger().error(
       "EphemeralSegmentManager::release: invalid segment {}",
       id);
     return crimson::ct_error::invarg::make();
   }
 
-  if (segment_state[id] != segment_state_t::CLOSED) {
+  if (segment_state[s_id] != segment_state_t::CLOSED) {
     logger().error(
       "EphemeralSegmentManager::release: segment id {} not closed",
       id);
@@ -179,7 +182,7 @@ SegmentManager::release_ertr::future<> EphemeralSegmentManager::release(
   }
 
   ::memset(buffer + get_offset({id, 0}), 0, config.segment_size);
-  segment_state[id] = segment_state_t::EMPTY;
+  segment_state[s_id] = segment_state_t::EMPTY;
   return release_ertr::now().safe_then([] {
     return seastar::sleep(std::chrono::milliseconds(1));
   });

diff --git a/src/crimson/os/seastore/segment_manager/ephemeral.h b/src/crimson/os/seastore/segment_manager/ephemeral.h
index 2096fab3910fc694cb21681164d1dc12dbe05568..fd8b28c43e20a54176593796ef728861c00a6dc5 100644 (file)
--- a/src/crimson/os/seastore/segment_manager/ephemeral.h
+++ b/src/crimson/os/seastore/segment_manager/ephemeral.h
@@ -56,7 +56,7 @@ class EphemeralSegmentManager final : public SegmentManager {
   std::optional<seastore_meta_t> meta;
 
   size_t get_offset(paddr_t addr) {
-    return (addr.segment * config.segment_size) + addr.offset;
+    return (addr.segment.device_segment_id() * config.segment_size) + addr.offset;
   }
 
   std::vector<segment_state_t> segment_state;
@@ -65,10 +65,19 @@ class EphemeralSegmentManager final : public SegmentManager {
 
   Segment::close_ertr::future<> segment_close(segment_id_t id);
 
+  device_id_t device_id = 0;
+
 public:
-  EphemeralSegmentManager(ephemeral_config_t config) : config(config) {}
+  EphemeralSegmentManager(
+    ephemeral_config_t config,
+    device_id_t device_id = 0)
+    : config(config), device_id(device_id) {}
   ~EphemeralSegmentManager();
 
+  device_id_t get_device_id() const {
+    return device_id;
+  }
+
   using init_ertr = crimson::errorator<
     crimson::ct_error::enospc,
     crimson::ct_error::invarg,

diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h
index 9b2af2e779b1513d0a1c8346ef2fddbc0e71bc76..bec8cb1dcf3923b68b6e6950634c433334697cf0 100644 (file)
--- a/src/test/crimson/seastore/transaction_manager_test_state.h
+++ b/src/test/crimson/seastore/transaction_manager_test_state.h
@@ -192,8 +192,16 @@ protected:
 
 class TestSegmentManagerWrapper final : public SegmentManager {
   SegmentManager &sm;
+  device_id_t device_id = 0;
 public:
-  TestSegmentManagerWrapper(SegmentManager &sm) : sm(sm) {}
+  TestSegmentManagerWrapper(
+    SegmentManager &sm,
+    device_id_t device_id = 0)
+    : sm(sm), device_id(device_id) {}
+
+  device_id_t get_device_id() const {
+    return device_id;
+  }
 
   mount_ret mount() final {
     return mount_ertr::now(); // we handle this above