]> git-server-git.apps.pok.os.sepia.ceph.com Git - ceph.git/commitdiff
crimson/os/seastore: enable SegmentCleaner to hold multiple segmented devices
authorXuehan Xu <xxhdx1985126@gmail.com>
Thu, 23 Sep 2021 06:27:02 +0000 (14:27 +0800)
committerXuehan Xu <xxhdx1985126@gmail.com>
Sun, 10 Oct 2021 06:21:07 +0000 (14:21 +0800)
For now, all segmented devices are treated the same as SEGMENTED. In the future,
there may be different kinds of segmeted devices, like SEGMENTED_NVME, SEGMENTED_SSD,
and even SEGMENTED_SATA. We plan to use a dedicated segment cleaner for each kind of
those devices, and if there are multiple devices of the same kind, they share the same
segment cleaner.

Signed-off-by: Xuehan Xu <xxhdx1985126@gmail.com>
src/crimson/os/seastore/extent_placement_manager.cc
src/crimson/os/seastore/journal.cc
src/crimson/os/seastore/segment_cleaner.cc
src/crimson/os/seastore/segment_cleaner.h
src/crimson/os/seastore/transaction_manager.cc
src/test/crimson/seastore/test_btree_lba_manager.cc
src/test/crimson/seastore/test_seastore_journal.cc
src/test/crimson/seastore/transaction_manager_test_state.h

index 43e6235ddabe974cbc22c09a98e664735bb0d141..63c0919640b01ba201693260dcc46c418e0bee48 100644 (file)
@@ -273,7 +273,9 @@ SegmentedAllocator::Writer::roll_segment(bool set_rolling) {
     });
   }
 
-  return segment_provider.get_segment().safe_then([this](auto segment) {
+  return segment_provider.get_segment(
+    segment_manager.get_device_id()
+  ).safe_then([this](auto segment) {
     return segment_manager.open(segment);
   }).safe_then([this](auto segref) {
     LOG_PREFIX(SegmentedAllocator::Writer::roll_segment);
index db59b7d1373a14906b046b6593ea5c715922e951..3725aeee86741727f472e522efdbe0d6e33c1db1 100644 (file)
@@ -166,7 +166,7 @@ Journal::roll_journal_segment()
   return (current_journal_segment ?
          current_journal_segment->close() :
          Segment::close_ertr::now()).safe_then([this] {
-      return segment_provider->get_segment();
+      return segment_provider->get_segment(segment_manager.get_device_id());
     }).safe_then([this](auto segment) {
       return segment_manager.open(segment);
     }).safe_then([this](auto sref) {
index d23c5407c76975d78486e276c9800bd918943ffd..89d98875f21f49e6c716952ac4f192c7c4e2454f 100644 (file)
@@ -15,6 +15,17 @@ namespace {
 
 namespace crimson::os::seastore {
 
+template <typename SegmentInfoT>
+void segment_manager_info_t<SegmentInfoT>::init_segment_infos(
+  SegmentInfoT&& segment_info)
+{
+  segment_infos.resize(num_segments, std::move(segment_info));
+  for (device_segment_id_t j = 0; j < num_segments; j++) {
+    segment_infos[j].segment = segment_id_t(device_id, j);
+    crimson::get_logger(ceph_subsys_seastore).debug("added segment: {}", segment_id_t(device_id, j));
+  }
+}
+
 bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const
 {
   const auto &other = static_cast<const SpaceTrackerSimple&>(_other);
@@ -26,22 +37,24 @@ bool SpaceTrackerSimple::equals(const SpaceTrackerI &_other) const
   }
 
   bool all_match = true;
-  for (segment_id_t i = 0; i < live_bytes_by_segment.size(); ++i) {
-    if (other.live_bytes_by_segment[i] != live_bytes_by_segment[i]) {
+  for (auto i = live_bytes_by_segment.begin(), j = other.live_bytes_by_segment.begin();
+       i != live_bytes_by_segment.end(); ++i, ++j) {
+    assert(i->segment == j->segment);
+    if (i->live_bytes != j->live_bytes) {
       all_match = false;
       logger().debug(
        "{}: segment_id {} live bytes mismatch *this: {}, other: {}",
        __func__,
-       i,
-       live_bytes_by_segment[i],
-       other.live_bytes_by_segment[i]);
+       i->segment,
+       i->live_bytes,
+       j->live_bytes);
     }
   }
   return all_match;
 }
 
 int64_t SpaceTrackerDetailed::SegmentMap::allocate(
-  segment_id_t segment,
+  device_segment_id_t segment,
   segment_off_t offset,
   extent_len_t len,
   const extent_len_t block_size)
@@ -73,7 +86,7 @@ int64_t SpaceTrackerDetailed::SegmentMap::allocate(
 }
 
 int64_t SpaceTrackerDetailed::SegmentMap::release(
-  segment_id_t segment,
+  device_segment_id_t segment,
   segment_off_t offset,
   extent_len_t len,
   const extent_len_t block_size)
@@ -115,15 +128,17 @@ bool SpaceTrackerDetailed::equals(const SpaceTrackerI &_other) const
   }
 
   bool all_match = true;
-  for (segment_id_t i = 0; i < segment_usage.size(); ++i) {
-    if (other.segment_usage[i].get_usage() != segment_usage[i].get_usage()) {
+  for (auto i = segment_usage.begin(), j = other.segment_usage.begin();
+       i != segment_usage.end(); ++i, ++j) {
+    assert(i->segment == j->segment);
+    if (i->segment_map.get_usage() != j->segment_map.get_usage()) {
       all_match = false;
       logger().error(
        "{}: segment_id {} live bytes mismatch *this: {}, other: {}",
        __func__,
-       i,
-       segment_usage[i].get_usage(),
-       other.segment_usage[i].get_usage());
+       i->segment,
+       i->segment_map.get_usage(),
+       j->segment_map.get_usage());
     }
   }
   return all_match;
@@ -141,7 +156,8 @@ void SpaceTrackerDetailed::SegmentMap::dump_usage(extent_len_t block_size) const
 void SpaceTrackerDetailed::dump_usage(segment_id_t id) const
 {
   logger().debug("SpaceTrackerDetailed::dump_usage {}", id);
-  segment_usage[id].dump_usage(block_size);
+  segment_usage[id].segment_map.dump_usage(
+    segment_usage[id.device_id()]->block_size);
 }
 
 SegmentCleaner::SegmentCleaner(
@@ -167,13 +183,33 @@ void SegmentCleaner::register_metrics()
 
 SegmentCleaner::get_segment_ret SegmentCleaner::get_segment()
 {
-  for (size_t i = 0; i < segments.size(); ++i) {
-    if (segments[i].is_empty()) {
-      mark_open(i);
-      logger().debug("{}: returning segment {}", __func__, i);
+  for (auto& segment_info : segments) {
+    if (segment_info.is_empty()) {
+      mark_open(segment_info.segment);
+      logger().debug("{}: returning segment {}", __func__, segment_info.segment);
+      return get_segment_ret(
+       get_segment_ertr::ready_future_marker{},
+       segment_info.segment);
+    }
+  }
+  assert(0 == "out of space handling todo");
+  return get_segment_ret(
+    get_segment_ertr::ready_future_marker{},
+    0);
+}
+
+SegmentCleaner::get_segment_ret SegmentCleaner::get_segment(device_id_t id)
+{
+  for (auto it = segments.find_begin(id);
+       it != segments.find_end(id);
+       it++) {
+    auto& segment_info = *it;
+     if (segment_info.is_empty()) {
+      mark_open(segment_info.segment);
+      logger().debug("{}: returning segment {}", __func__, segment_info.segment);
       return get_segment_ret(
        get_segment_ertr::ready_future_marker{},
-       i);
+       segment_info.segment);
     }
   }
   assert(0 == "out of space handling todo");
@@ -378,15 +414,17 @@ SegmentCleaner::gc_reclaim_space_ret SegmentCleaner::gc_reclaim_space()
 }
 
 SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() {
+  logger().debug("SegmentCleaner::init_segments: {} segments", segments.size());
   return seastar::do_with(
     std::vector<std::pair<segment_id_t, segment_header_t>>(),
-    [this](auto& segments) {
+    [this](auto& segment_set) {
     return crimson::do_for_each(
-      boost::make_counting_iterator(segment_id_t{0}),
-      boost::make_counting_iterator(segment_id_t{num_segments}),
-      [this, &segments](auto segment_id) {
+      segments.begin(),
+      segments.end(),
+      [this, &segment_set](auto& segment_info) {
+      auto segment_id = segment_info.segment;
       return scanner->read_segment_header(segment_id)
-      .safe_then([&segments, segment_id, this](auto header) {
+      .safe_then([&segment_set, segment_id, this](auto header) {
        if (header.out_of_line) {
          logger().debug("ExtentReader::init_segments: out-of-line segment {}", segment_id);
          init_mark_segment_closed(
@@ -395,7 +433,7 @@ SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() {
            true);
        } else {
          logger().debug("ExtentReader::init_segments: journal segment {}", segment_id);
-         segments.emplace_back(std::make_pair(segment_id, std::move(header)));
+         segment_set.emplace_back(std::make_pair(segment_id, std::move(header)));
        }
        return seastar::now();
       }).handle_error(
@@ -407,10 +445,10 @@ SegmentCleaner::init_segments_ret SegmentCleaner::init_segments() {
        }),
        crimson::ct_error::input_output_error::pass_further{}
       );
-    }).safe_then([&segments] {
+    }).safe_then([&segment_set] {
       return seastar::make_ready_future<
        std::vector<std::pair<segment_id_t, segment_header_t>>>(
-         std::move(segments));
+         std::move(segment_set));
     });
   });
 }
index 470504ac44c66401c9c074d38f775826ffb30e2a..7060a55e0cd23f47ad829f9dd9f79a8ca61425e6 100644 (file)
 
 namespace crimson::os::seastore {
 
+struct segment_info_t;
+using unavail_segment_info_set_t =
+  std::list<segment_info_t*>;
+using unavail_segment_info_iterator =
+  unavail_segment_info_set_t::iterator;
+
+// position within segment_info_set_t
+using segment_pos_t = uint64_t;
+
+template <typename SegmentInfoT>
+struct segment_manager_info_t {
+  segment_manager_info_t() = default;
+  segment_manager_info_t(
+    device_id_t device_id,
+    device_segment_id_t num_segments,
+    segment_off_t segment_size,
+    segment_off_t block_size,
+    size_t empty_segments,
+    size_t size)
+    : device_id(device_id),
+      num_segments(num_segments),
+      segment_size(segment_size),
+      block_size(block_size),
+      empty_segments(empty_segments),
+      size(size)
+  {}
+
+  void init_segment_infos(SegmentInfoT&&);
+
+  unavail_segment_info_set_t usis;
+  device_id_t device_id = 0;
+  device_segment_id_t num_segments = 0;
+  segment_off_t segment_size = 0;
+  segment_off_t block_size = 0;
+  size_t empty_segments = 0;
+  size_t size = 0;
+  std::vector<SegmentInfoT> segment_infos;
+};
+
 struct segment_info_t {
   Segment::segment_state_t state = Segment::segment_state_t::EMPTY;
 
   // Will be non-null for any segments in the current journal
   segment_seq_t journal_segment_seq = NULL_SEG_SEQ;
 
+  segment_id_t segment;
 
   bool out_of_line = false;
 
@@ -46,6 +86,173 @@ struct segment_info_t {
   }
 };
 
+// for keeping track of segment managers' various information,
+// like empty segments, opened segments and so on.
+template <typename SegmentInfoT>
+class segment_info_set_t {
+public:
+  template <bool is_const = false>
+  class iterator {
+  public:
+    iterator(
+      device_id_t sm_id,
+      std::conditional_t<
+       is_const,
+       const segment_info_set_t*,
+       segment_info_set_t*> segment_info_set,
+      bool end = false)
+      : sm_id(sm_id), segment_info_set(segment_info_set) {
+      auto& sm_info = segment_info_set->sm_info_vec[sm_id];
+      assert(sm_info);
+      if (end) {
+       iter = sm_info->segment_infos.end();
+      } else {
+       iter = sm_info->segment_infos.begin();
+      }
+    }
+    iterator<is_const>& operator++() {
+      iter++;
+      auto& sm_info = segment_info_set->sm_info_vec[sm_id];
+      if (iter == sm_info->segment_infos.end()) {
+       device_id_t t_sm_id = sm_id;
+       while (++t_sm_id < max_devices && !segment_info_set->sm_info_vec[t_sm_id]);
+       if (t_sm_id < max_devices) {
+         auto& sm = segment_info_set->sm_info_vec[t_sm_id];
+         assert(sm);
+         iter = sm->segment_infos.begin();
+         sm_id = t_sm_id;
+       }
+      }
+      return *this;
+    }
+    bool operator==(iterator<is_const> rit) {
+      return sm_id == rit.sm_id && iter == rit.iter;
+    }
+    bool operator!=(iterator<is_const> rit) {
+      return !(*this == rit);
+    }
+    template <bool c = is_const, std::enable_if_t<c, int> = 0>
+    const SegmentInfoT& operator*() {
+      return *iter;
+    }
+    template <bool c = is_const, std::enable_if_t<!c, int> = 0>
+    SegmentInfoT& operator*() {
+      return *iter;
+    }
+    template <bool c = is_const, std::enable_if_t<c, int> = 0>
+    const SegmentInfoT* operator->() {
+      return &(*iter);
+    }
+    template <bool c = is_const, std::enable_if_t<!c, int> = 0>
+    SegmentInfoT* operator->() {
+      return &(*iter);
+    }
+  private:
+    device_id_t sm_id = 0;
+    std::conditional_t<
+      is_const,
+      const segment_info_set_t*,
+      segment_info_set_t*> segment_info_set = nullptr;
+    std::conditional_t<
+      is_const,
+      typename std::vector<SegmentInfoT>::const_iterator,
+      typename std::vector<SegmentInfoT>::iterator> iter;
+  };
+  SegmentInfoT& operator[](segment_id_t id) {
+    auto d_id = id.device_id();
+    auto& sm_info = sm_info_vec[d_id];
+    assert(sm_info);
+    assert(sm_info->device_id == d_id);
+    auto& segment_info = sm_info->segment_infos[id.device_segment_id()];
+    assert(segment_info.segment == id);
+    return segment_info;
+  }
+  const SegmentInfoT& operator[](segment_id_t id) const {
+    auto d_id = id.device_id();
+    auto& sm_info = sm_info_vec.at(d_id);
+    assert(sm_info);
+    assert(sm_info->device_id == d_id);
+    auto& segment_info = sm_info->segment_infos[id.device_segment_id()];
+    assert(segment_info.segment == id);
+    return segment_info;
+  }
+  std::optional<segment_manager_info_t<SegmentInfoT>>&
+  operator[](device_id_t id) {
+    auto& sm_info = sm_info_vec[id];
+    assert(sm_info && sm_info->device_id == id);
+    return sm_info;
+  }
+  const std::optional<segment_manager_info_t<SegmentInfoT>>&
+  operator[](device_id_t id) const {
+    auto& sm_info = sm_info_vec[id];
+    assert(sm_info && sm_info->device_id == id);
+    return sm_info;
+  }
+  void clear() {
+    sm_info_vec.clear();
+    total_segments = 0;
+  }
+  void add_segment_manager(
+    SegmentManager& segment_manager,
+    SegmentInfoT&& segment_info)
+  {
+    if (!sm_info_vec.size()) {
+      sm_info_vec.resize(max_devices);
+    }
+    device_id_t d_id = segment_manager.get_device_id();
+    sm_info_vec[segment_manager.get_device_id()] = std::make_optional<
+      segment_manager_info_t<SegmentInfoT>>(
+       d_id,
+       segment_manager.get_num_segments(),
+       segment_manager.get_segment_size(),
+       segment_manager.get_block_size(),
+       segment_manager.get_num_segments(),
+       segment_manager.get_size());
+    sm_info_vec[segment_manager.get_device_id()]->init_segment_infos(std::move(segment_info));
+    total_segments += segment_manager.get_num_segments();
+  }
+
+  device_segment_id_t size() const {
+    return total_segments;
+  }
+  auto begin() {
+    device_id_t sm_id = 0;
+    for (;sm_id < max_devices && !sm_info_vec[sm_id];
+       sm_id ++);
+    return iterator<false>(sm_id, this);
+  }
+  auto begin() const {
+    device_id_t sm_id = 0;
+    for (;sm_id < max_devices && !sm_info_vec[sm_id];
+       sm_id ++);
+    return iterator<true>(sm_id, this);
+  }
+  auto end() {
+    device_id_t sm_id = max_devices - 1;
+    for (;!sm_info_vec[sm_id];
+       sm_id --);
+    return iterator<false>(sm_id, this, true);
+  }
+  auto end() const {
+    device_id_t sm_id = max_devices - 1;
+    for (;!sm_info_vec[sm_id];
+       sm_id --);
+    return iterator<true>(sm_id, this, true);
+  }
+  auto find_begin(device_id_t id) {
+    auto& sm_info = sm_info_vec[id];
+    return sm_info->segment_infos.begin();
+  }
+  auto find_end(device_id_t id) {
+    auto& sm_info = sm_info_vec[id];
+    return sm_info->segment_infos.end();
+  }
+private:
+  device_segment_id_t total_segments;
+  std::vector<std::optional<segment_manager_info_t<SegmentInfoT>>> sm_info_vec;
+
+  friend class SegmentCleaner;
+};
 /**
  * Callback interface for managing available segments
  */
@@ -55,6 +262,7 @@ public:
     crimson::ct_error::input_output_error>;
   using get_segment_ret = get_segment_ertr::future<segment_id_t>;
   virtual get_segment_ret get_segment() = 0;
+  virtual get_segment_ret get_segment(device_id_t id) = 0;
 
   virtual void close_segment(segment_id_t) {}
 
@@ -102,19 +310,37 @@ public:
 };
 using SpaceTrackerIRef = std::unique_ptr<SpaceTrackerI>;
 
+struct segment_space_tracker_t {
+  segment_id_t segment = NULL_SEG_ID;
+  int64_t live_bytes = 0;
+};
+
 class SpaceTrackerSimple : public SpaceTrackerI {
   // Tracks live space for each segment
-  std::vector<int64_t> live_bytes_by_segment;
+  segment_info_set_t<segment_space_tracker_t> live_bytes_by_segment;
 
   int64_t update_usage(segment_id_t segment, int64_t delta) {
-    assert(segment < live_bytes_by_segment.size());
-    live_bytes_by_segment[segment] += delta;
-    assert(live_bytes_by_segment[segment] >= 0);
-    return live_bytes_by_segment[segment];
+    live_bytes_by_segment[segment].live_bytes += delta;
+    assert(live_bytes_by_segment[segment].live_bytes >= 0);
+    return live_bytes_by_segment[segment].live_bytes;
   }
 public:
-  SpaceTrackerSimple(segment_id_t num_segments)
-    : live_bytes_by_segment(num_segments, 0) {}
+  SpaceTrackerSimple(std::vector<SegmentManager*> sms) {
+    for (auto sm : sms) {
+      live_bytes_by_segment.add_segment_manager(
+       *sm,
+       segment_space_tracker_t{});
+    }
+  }
+  SpaceTrackerSimple(
+    const segment_info_set_t<segment_space_tracker_t>& live_bytes_by_segment)
+    : live_bytes_by_segment(live_bytes_by_segment) {
+    for (auto it = this->live_bytes_by_segment.begin();
+       it != this->live_bytes_by_segment.end();
+       ++it) {
+      it->live_bytes = 0;
+    }
+  }
 
   int64_t allocate(
     segment_id_t segment,
@@ -131,20 +357,21 @@ public:
   }
 
   int64_t get_usage(segment_id_t segment) const final {
-    assert(segment < live_bytes_by_segment.size());
-    return live_bytes_by_segment[segment];
+    return live_bytes_by_segment[segment].live_bytes;
   }
 
   void dump_usage(segment_id_t) const final {}
 
   void reset() final {
-    for (auto &i: live_bytes_by_segment)
-      i = 0;
+    for (auto it = live_bytes_by_segment.begin();
+       it != live_bytes_by_segment.end();
+       ++it)
+      it->live_bytes = 0;
   }
 
   SpaceTrackerIRef make_empty() const final {
     return SpaceTrackerIRef(
-      new SpaceTrackerSimple(live_bytes_by_segment.size()));
+      new SpaceTrackerSimple(live_bytes_by_segment));
   }
 
   bool equals(const SpaceTrackerI &other) const;
@@ -164,13 +391,13 @@ class SpaceTrackerDetailed : public SpaceTrackerI {
     }
 
     int64_t allocate(
-      segment_id_t segment,
+      device_segment_id_t segment,
       segment_off_t offset,
       extent_len_t len,
       const extent_len_t block_size);
 
     int64_t release(
-      segment_id_t segment,
+      device_segment_id_t segment,
       segment_off_t offset,
       extent_len_t len,
       const extent_len_t block_size);
@@ -188,52 +415,81 @@ class SpaceTrackerDetailed : public SpaceTrackerI {
       }
     }
   };
-  const size_t block_size;
-  const size_t segment_size;
+  struct segment_tracker_t {
+    segment_tracker_t(size_t blocks)
+      : segment_map(blocks) {}
+    segment_id_t segment;
+    SegmentMap segment_map;
+  };
 
   // Tracks live space for each segment
-  std::vector<SegmentMap> segment_usage;
+  segment_info_set_t<segment_tracker_t> segment_usage;
 
 public:
-  SpaceTrackerDetailed(segment_id_t num_segments, size_t segment_size, size_t block_size)
-    : block_size(block_size),
-      segment_size(segment_size),
-      segment_usage(num_segments, segment_size / block_size) {}
+  SpaceTrackerDetailed(
+    std::vector<SegmentManager*> sms)
+  {
+    for (auto sm : sms) {
+      // sms is a vector that is indexed by device id and
+      // always has "max_device" elements, some of which
+      // may be null.
+      if (!sm) {
+       continue;
+      }
+      segment_usage.add_segment_manager(
+       *sm,
+       segment_tracker_t(
+         sm->get_segment_size() / sm->get_block_size()));
+    }
+  }
+  SpaceTrackerDetailed(
+    const segment_info_set_t<segment_tracker_t>& segment_usage)
+    : segment_usage(segment_usage)
+  {
+    for (auto& tracker : this->segment_usage) {
+      tracker.segment_map.reset();
+    }
+  }
 
   int64_t allocate(
     segment_id_t segment,
     segment_off_t offset,
     extent_len_t len) final {
     assert(segment < segment_usage.size());
-    return segment_usage[segment].allocate(segment, offset, len, block_size);
+    return segment_usage[segment].segment_map.allocate(
+      segment.device_segment_id(),
+      offset,
+      len,
+      segment_usage[segment.device_id()]->block_size);
   }
 
   int64_t release(
     segment_id_t segment,
     segment_off_t offset,
     extent_len_t len) final {
-    assert(segment < segment_usage.size());
-    return segment_usage[segment].release(segment, offset, len, block_size);
+    return segment_usage[segment].segment_map.release(
+      segment.device_segment_id(),
+      offset,
+      len,
+      segment_usage[segment.device_id()]->block_size);
   }
 
   int64_t get_usage(segment_id_t segment) const final {
     assert(segment < segment_usage.size());
-    return segment_usage[segment].get_usage();
+    return segment_usage[segment].segment_map.get_usage();
   }
 
   void dump_usage(segment_id_t seg) const final;
 
   void reset() final {
     for (auto &i: segment_usage)
-      i.reset();
+      i.segment_map.reset();
   }
 
   SpaceTrackerIRef make_empty() const final {
     return SpaceTrackerIRef(
       new SpaceTrackerDetailed(
-       segment_usage.size(),
-       segment_size,
-       block_size));
+       segment_usage));
   }
 
   bool equals(const SpaceTrackerI &other) const;
@@ -379,14 +635,12 @@ private:
   const bool detailed;
   const config_t config;
 
-  segment_id_t num_segments = 0;
-  size_t segment_size = 0;
-  size_t block_size = 0;
+  internal_segment_id_t num_segments = 0;
 
   ExtentReaderRef scanner;
 
   SpaceTrackerIRef space_tracker;
-  std::vector<segment_info_t> segments;
+  segment_info_set_t<segment_info_t> segments;
   size_t empty_segments;
   uint64_t used_bytes = 0;
   bool init_complete = false;
@@ -414,40 +668,53 @@ private:
   /// head of journal
   journal_seq_t journal_head;
 
+  device_id_t journal_device_id;
+
   ExtentCallbackInterface *ecb = nullptr;
 
   /// populated if there is an IO blocked on hard limits
   std::optional<seastar::promise<>> blocked_io_wake;
 
+  std::vector<device_id_t> effective_devices;
+
 public:
   SegmentCleaner(
     config_t config,
     ExtentReaderRef&& scanner,
     bool detailed = false);
 
-  void mount(SegmentManager &sm) {
+  void mount(SegmentManager &psm, std::vector<SegmentManager*>& sms) {
+    crimson::get_logger(ceph_subsys_seastore).debug(
+      "SegmentCleaner::mount: {} segment managers", sms.size());
     init_complete = false;
     used_bytes = 0;
     journal_tail_target = journal_seq_t{};
     journal_tail_committed = journal_seq_t{};
     journal_head = journal_seq_t{};
+    journal_device_id = psm.get_device_id();
 
-    num_segments = sm.get_num_segments();
-    segment_size = static_cast<size_t>(sm.get_segment_size());
-    block_size = static_cast<size_t>(sm.get_block_size());
+    for (auto& sm : sms) {
+      if (sm)
+       effective_devices.push_back(sm->get_device_id());
+    }
 
     space_tracker.reset(
       detailed ?
       (SpaceTrackerI*)new SpaceTrackerDetailed(
-       num_segments,
-       segment_size,
-       block_size) :
+       sms) :
       (SpaceTrackerI*)new SpaceTrackerSimple(
-       num_segments));
+       sms));
 
     segments.clear();
-    segments.resize(num_segments);
-    empty_segments = num_segments;
+    for (auto sm : sms) {
+      // sms is a vector that is indexed by device id and
+      // always has "max_device" elements, some of which
+      // may be null.
+      if (!sm) {
+       continue;
+      }
+      segments.add_segment_manager(*sm, segment_info_t{});
+    }
   }
 
   using init_segments_ertr = crimson::errorator<
@@ -458,12 +725,16 @@ public:
   init_segments_ret init_segments();
 
   get_segment_ret get_segment() final;
+  get_segment_ret get_segment(device_id_t id) final;
 
   void close_segment(segment_id_t segment) final;
 
   void set_journal_segment(
     segment_id_t segment, segment_seq_t seq) final {
-    assert(segment < segments.size());
+    assert(segment.device_id() ==
+      segments[segment.device_id()]->device_id);
+    assert(segment.device_segment_id() <
+      segments[segment.device_id()]->num_segments);
     segments[segment].journal_segment_seq = seq;
     assert(segments[segment].is_open());
   }
@@ -523,7 +794,10 @@ public:
     paddr_t addr,
     extent_len_t len,
     bool init_scan = false) {
-    assert(addr.segment < segments.size());
+    assert(addr.segment.device_id() ==
+      segments[addr.segment.device_id()]->device_id);
+    assert(addr.segment.device_segment_id() <
+      segments[addr.segment.device_id()]->num_segments);
 
     if (!init_scan && !init_complete)
       return;
@@ -545,7 +819,10 @@ public:
 
     ceph_assert(used_bytes >= len);
     used_bytes -= len;
-    assert(addr.segment < segments.size());
+    assert(addr.segment.device_id() ==
+      segments[addr.segment.device_id()]->device_id);
+    assert(addr.segment.device_segment_id() <
+      segments[addr.segment.device_id()]->num_segments);
 
     [[maybe_unused]] auto ret = space_tracker->release(
       addr.segment,
@@ -557,20 +834,25 @@ public:
 
   segment_id_t get_next_gc_target() const {
     segment_id_t ret = NULL_SEG_ID;
+    segment_seq_t seq = NULL_SEG_SEQ;
     int64_t least_live_bytes = std::numeric_limits<int64_t>::max();
-    for (segment_id_t i = 0; i < segments.size(); ++i) {
-      if (segments[i].is_closed() &&
-         !segments[i].is_in_journal(journal_tail_committed) &&
-         space_tracker->get_usage(i) < least_live_bytes) {
-       ret = i;
-       least_live_bytes = space_tracker->get_usage(i);
+    for (auto it = segments.begin();
+        it != segments.end();
+        ++it) {
+      const auto& segment_info = *it;
+      if (segment_info.is_closed() &&
+         !segment_info.is_in_journal(journal_tail_committed) &&
+         space_tracker->get_usage(segment_info.segment) < least_live_bytes) {
+       ret = segment_info.segment;
+       seq = segment_info.journal_segment_seq;
+       least_live_bytes = space_tracker->get_usage(segment_info.segment);
       }
     }
     if (ret != NULL_SEG_ID) {
       crimson::get_logger(ceph_subsys_seastore).debug(
        "SegmentCleaner::get_next_gc_target: segment {} seq {}",
        ret,
-       segments[ret].journal_segment_seq);
+       seq);
     }
     return ret;
   }
@@ -979,7 +1261,10 @@ private:
   }
 
   void mark_closed(segment_id_t segment) {
-    assert(segments.size() > segment);
+    assert(segment.device_id() ==
+      segments[segment.device_id()]->device_id);
+    assert(segment.device_segment_id() <
+      segments[segment.device_id()]->num_segments);
     if (init_complete) {
       assert(segments[segment].is_open());
     } else {
@@ -994,9 +1279,11 @@ private:
   }
 
   void mark_empty(segment_id_t segment) {
-    assert(segments.size() > segment);
+    assert(segment.device_id() ==
+      segments[segment.device_id()]->device_id);
+    assert(segment.device_segment_id() <
+      segments[segment.device_id()]->num_segments);
     assert(segments[segment].is_closed());
-    assert(segments.size() > empty_segments);
     ++empty_segments;
     if (space_tracker->get_usage(segment) != 0) {
       space_tracker->dump_usage(segment);
@@ -1007,7 +1294,11 @@ private:
   }
 
   void mark_open(segment_id_t segment) {
-    assert(segments.size() > segment);
+    crimson::get_logger(ceph_subsys_seastore).debug("mark open: {}", segment);
+    assert(segment.device_id() ==
+      segments[segment.device_id()]->device_id);
+    assert(segment.device_segment_id() <
+      segments[segment.device_id()]->num_segments);
     assert(segments[segment].is_empty());
     assert(empty_segments > 0);
     --empty_segments;
@@ -1016,4 +1307,10 @@ private:
 };
 using SegmentCleanerRef = std::unique_ptr<SegmentCleaner>;
 
+template struct segment_manager_info_t<segment_info_t>;
+template class segment_info_set_t<segment_info_t>;
+template struct segment_manager_info_t<SpaceTrackerDetailed::segment_tracker_t>;
+template class segment_info_set_t<SpaceTrackerDetailed::segment_tracker_t>;
+template struct segment_manager_info_t<segment_space_tracker_t>;
+template class segment_info_set_t<segment_space_tracker_t>;
 }
index 170742df09ed334eae4eb20f49e338166d5e20a4..2bb4686bdaaaa954c5cc3c3b122c2035c05a353b 100644 (file)
@@ -33,7 +33,7 @@ TransactionManager::TransactionManager(
 TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs()
 {
   LOG_PREFIX(TransactionManager::mkfs);
-  segment_cleaner->mount(segment_manager);
+  segment_cleaner->mount(segment_manager, segment_managers);
   return journal->open_for_write().safe_then([this, FNAME](auto addr) {
     DEBUG("about to do_with");
     segment_cleaner->init_mkfs(addr);
@@ -64,7 +64,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount()
 {
   LOG_PREFIX(TransactionManager::mount);
   cache->init();
-  segment_cleaner->mount(segment_manager);
+  segment_cleaner->mount(segment_manager, segment_managers);
   return segment_cleaner->init_segments().safe_then(
     [this](auto&& segments) {
     return journal->replay(
index 7399bd26eac8723c4c69742714c4b0d9e934d255..330c3c0102a3d3992a6c49aa141b6e4b7219bf0e 100644 (file)
@@ -50,7 +50,7 @@ struct btree_lba_manager_test :
   }
 
   segment_id_t next = 0;
-  get_segment_ret get_segment() final {
+  get_segment_ret get_segment(device_id_t id) final {
     auto ret = next;
     next = segment_id_t{
       next.device_id(),
@@ -60,6 +60,8 @@ struct btree_lba_manager_test :
       ret);
   }
 
+  void set_segment_written_to(paddr_t addr) final {}
+
   journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; }
   void update_journal_tail_committed(journal_seq_t committed) final {}
 
index 3d937d5b67a5e6a5ea9e50457adf3f1c18131f05..b18a842e793b3ecf5d2acb6acbc54995a1e66f61 100644 (file)
@@ -85,7 +85,7 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider {
   }
 
   segment_id_t next = 0;
-  get_segment_ret get_segment() final {
+  get_segment_ret get_segment(device_id_t id) final {
     auto ret = next;
     next = segment_id_t{
       next.device_id(),
@@ -95,6 +95,8 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider {
       ret);
   }
 
+  void set_segment_written_to(paddr_t addr) final {}
+
   journal_seq_t get_journal_tail_target() const final { return journal_seq_t{}; }
   void update_journal_tail_committed(journal_seq_t paddr) final {}
 
index a5428556eb77dc3850a21a04abed69f9e9cf6a2d..e12b79eddbb1846754fc0579315d213b3de741a8 100644 (file)
@@ -76,6 +76,7 @@ auto get_transaction_manager(
   auto segment_cleaner = std::make_unique<SegmentCleaner>(
     SegmentCleaner::config_t::get_default(),
     std::move(scanner),
+    segment_manager,
     true);
   auto journal = std::make_unique<Journal>(segment_manager, scanner_ref);
   auto cache = std::make_unique<Cache>(scanner_ref, segment_manager.get_block_size());