]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
crimson/os/seastore: cleanup dependency between epm and lba_manager
authorYingxin Cheng <yingxin.cheng@intel.com>
Thu, 27 Jan 2022 15:35:45 +0000 (23:35 +0800)
committerYingxin Cheng <yingxin.cheng@intel.com>
Fri, 11 Feb 2022 01:26:22 +0000 (09:26 +0800)
Move lba_manamger->update_mapping() from epm to transaction manager.

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
14 files changed:
src/crimson/os/seastore/cache.cc
src/crimson/os/seastore/cache.h
src/crimson/os/seastore/extent_placement_manager.cc
src/crimson/os/seastore/extent_placement_manager.h
src/crimson/os/seastore/lba_manager.cc
src/crimson/os/seastore/lba_manager.h
src/crimson/os/seastore/seastore.cc
src/crimson/os/seastore/transaction.h
src/crimson/os/seastore/transaction_manager.cc
src/crimson/os/seastore/transaction_manager.h
src/crimson/tools/store_nbd/tm_driver.cc
src/test/crimson/seastore/test_btree_lba_manager.cc
src/test/crimson/seastore/test_seastore_cache.cc
src/test/crimson/seastore/transaction_manager_test_state.h

index 1270881241530c2329eef5b1e79d2fe7f3dc823a..5a711a1b45c8f407d1b552267a71ec438387ba9d 100644 (file)
@@ -25,8 +25,10 @@ SET_SUBSYS(seastore_cache);
 namespace crimson::os::seastore {
 
 Cache::Cache(
-  ExtentReader &reader)
+  ExtentReader &reader,
+  ExtentPlacementManager &epm)
   : reader(reader),
+    epm(epm),
     lru(crimson::common::get_conf<Option::size_t>(
          "seastore_cache_lru_size"))
 {
index 524b72707ec46a3372f6da49fc31e91b10ca48b2..2d7042bd643a38f994e9729c26f234c57b00f1be 100644 (file)
@@ -100,13 +100,9 @@ public:
     crimson::ct_error::input_output_error>;
   using base_iertr = trans_iertr<base_ertr>;
 
-  Cache(ExtentReader &reader);
+  Cache(ExtentReader &reader, ExtentPlacementManager &epm);
   ~Cache();
 
-  void set_epm(ExtentPlacementManager& epm) {
-    p_epm = &epm;
-  }
-
   /// Creates empty transaction by source
   TransactionRef create_transaction(
       Transaction::src_t src,
@@ -501,7 +497,7 @@ public:
     LOG_PREFIX(Cache::alloc_new_extent);
     SUBTRACET(seastore_cache, "allocate {} {}B, hint={}",
               t, T::TYPE, length, hint);
-    auto result = p_epm->alloc_new_extent(t, T::TYPE, length, hint);
+    auto result = epm.alloc_new_extent(t, T::TYPE, length, hint);
     auto ret = CachedExtent::make_cached_extent_ref<T>(std::move(result.bp));
     ret->set_paddr(result.paddr);
     t.add_fresh_extent(ret);
@@ -740,7 +736,7 @@ public:
 
 private:
   ExtentReader &reader;                   ///< ref to extent reader
-  ExtentPlacementManager* p_epm = nullptr;
+  ExtentPlacementManager& epm;
   RootBlockRef root;               ///< ref to current root
   ExtentIndex extents;             ///< set of live extents
 
index 71d86e00c8bb293091440b8c85d41548f2299c13..6db0b9b9c7631ae9bb3c889fb0b2a8e27f27a333 100644 (file)
@@ -3,7 +3,6 @@
 
 #include "crimson/os/seastore/extent_placement_manager.h"
 
-#include "crimson/os/seastore/lba_manager.h"
 #include "crimson/os/seastore/segment_cleaner.h"
 
 namespace {
@@ -19,11 +18,9 @@ namespace crimson::os::seastore {
 SegmentedAllocator::SegmentedAllocator(
   SegmentProvider& sp,
   SegmentManager& sm,
-  LBAManager& lba_manager,
   Journal& journal)
   : segment_provider(sp),
     segment_manager(sm),
-    lba_manager(lba_manager),
     journal(journal)
 {
   std::generate_n(
@@ -34,39 +31,10 @@ SegmentedAllocator::SegmentedAllocator(
       return Writer{
        segment_provider,
        segment_manager,
-       lba_manager,
        journal};
       });
 }
 
-SegmentedAllocator::Writer::finish_record_ret
-SegmentedAllocator::Writer::finish_write(
-  Transaction& t,
-  ool_record_t& record) {
-  return trans_intr::do_for_each(record.get_extents(),
-    [this, &t](auto& ool_extent) {
-    LOG_PREFIX(SegmentedAllocator::Writer::finish_write);
-    auto& lextent = ool_extent.get_lextent();
-    DEBUGT("extent: {}, ool_paddr: {}",
-      t,
-      *lextent,
-      ool_extent.get_ool_paddr());
-    return lba_manager.update_mapping(
-      t,
-      lextent->get_laddr(),
-      lextent->get_paddr(),
-      ool_extent.get_ool_paddr()
-    ).si_then([&ool_extent, &t, &lextent, this, FNAME] {
-      lextent->hint = {};
-      TRACET("mark extent as ool at {} -- {}", t, ool_extent.get_ool_paddr(), *lextent);
-      t.mark_delayed_extent_ool(lextent, ool_extent.get_ool_paddr());
-      return finish_record_iertr::now();
-    });
-  }).si_then([&record] {
-    record.clear();
-  });
-}
-
 SegmentedAllocator::Writer::write_iertr::future<>
 SegmentedAllocator::Writer::_write(
   Transaction& t,
@@ -104,10 +72,9 @@ SegmentedAllocator::Writer::_write(
 
   return trans_intr::make_interruptible(
     current_segment->segment->write(record.get_base(), bl).safe_then(
-      [this, pr=std::move(pr), &t,
-      it=(--current_segment->inflight_writes.end()),
-      cs=current_segment]() mutable {
-        LOG_PREFIX(SegmentedAllocator::Writer::_write);
+      [this, FNAME, pr=std::move(pr), &t,
+       it=(--current_segment->inflight_writes.end()),
+       cs=current_segment]() mutable {
         if (cs->outdated) {
           DEBUGT("segment rolled", t);
           pr.set_value();
@@ -117,8 +84,15 @@ SegmentedAllocator::Writer::_write(
         }
         return seastar::now();
     })
-  ).si_then([this, &record, &t]() mutable {
-    return finish_write(t, record);
+  ).si_then([FNAME, &record, &t] {
+    for (auto& ool_extent : record.get_extents()) {
+      auto& lextent = ool_extent.get_lextent();
+      auto paddr = ool_extent.get_ool_paddr();
+      TRACET("ool extent written at {} -- {}", t, *lextent, paddr);
+      lextent->hint = {};
+      t.mark_delayed_extent_ool(lextent, paddr);
+    }
+    record.clear();
   });
 }
 
index 9d56e3e0263536471048f00a8ae81196f1c85797..b78e5b28b2eeadf5d48158c17da0de09017bc4fa 100644 (file)
@@ -163,7 +163,6 @@ struct open_segment_wrapper_t : public boost::intrusive_ref_counter<
 using open_segment_wrapper_ref =
   boost::intrusive_ptr<open_segment_wrapper_t>;
 
-class LBAManager;
 class SegmentProvider;
 
 /**
@@ -185,11 +184,9 @@ class SegmentedAllocator : public ExtentAllocator {
     Writer(
       SegmentProvider& sp,
       SegmentManager& sm,
-      LBAManager& lba_manager,
       Journal& journal)
       : segment_provider(sp),
         segment_manager(sm),
-        lba_manager(lba_manager),
         journal(journal)
     {}
     Writer(Writer &&) = default;
@@ -205,13 +202,6 @@ class SegmentedAllocator : public ExtentAllocator {
       });
     }
   private:
-    using finish_record_ertr = crimson::errorator<
-      crimson::ct_error::input_output_error>;
-    using finish_record_iertr = trans_iertr<finish_record_ertr>;
-    using finish_record_ret = finish_record_iertr::future<>;
-    finish_record_ret finish_write(
-      Transaction& t,
-      ool_record_t& record);
     bool _needs_roll(seastore_off_t length) const;
 
     write_iertr::future<> _write(
@@ -235,7 +225,6 @@ class SegmentedAllocator : public ExtentAllocator {
     open_segment_wrapper_ref current_segment;
     std::list<open_segment_wrapper_ref> open_segments;
     seastore_off_t allocated_to = 0;
-    LBAManager& lba_manager;
     Journal& journal;
     crimson::condition_variable segment_rotation_guard;
     seastar::gate writer_guard;
@@ -245,7 +234,6 @@ public:
   SegmentedAllocator(
     SegmentProvider& sp,
     SegmentManager& sm,
-    LBAManager& lba_manager,
     Journal& journal);
 
   Writer &get_writer(placement_hint_t hint) {
@@ -281,15 +269,12 @@ private:
   SegmentProvider& segment_provider;
   SegmentManager& segment_manager;
   std::vector<Writer> writers;
-  LBAManager& lba_manager;
   Journal& journal;
 };
 
 class ExtentPlacementManager {
 public:
-  ExtentPlacementManager(
-    LBAManager& lba_manager
-  ) : lba_manager(lba_manager) {}
+  ExtentPlacementManager() = default;
 
   struct alloc_result_t {
     paddr_t paddr;
@@ -332,34 +317,25 @@ public:
   /**
    * delayed_alloc_or_ool_write
    *
-   * Performs any outstanding ool writes and updates pending lba updates
-   * accordingly
+   * Performs delayed allocation and do writes for out-of-line extents.
    */
   using alloc_paddr_iertr = ExtentOolWriter::write_iertr;
   alloc_paddr_iertr::future<> delayed_alloc_or_ool_write(
-    Transaction& t) {
+    Transaction& t,
+    const std::list<LogicalCachedExtentRef>& delayed_extents) {
     LOG_PREFIX(ExtentPlacementManager::delayed_alloc_or_ool_write);
-    SUBDEBUGT(seastore_tm, "start", t);
+    SUBDEBUGT(seastore_tm, "start with {} delayed extents",
+              t, delayed_extents.size());
     return seastar::do_with(
         std::map<ExtentAllocator*, std::list<LogicalCachedExtentRef>>(),
-        [this, &t](auto& alloc_map) {
-      LOG_PREFIX(ExtentPlacementManager::delayed_alloc_or_ool_write);
-      auto& alloc_list = t.get_delayed_alloc_list();
-      uint64_t num_ool_extents = 0;
-      for (auto& extent : alloc_list) {
-        // extents may be invalidated
-        if (!extent->is_valid()) {
-          t.increment_delayed_invalid_extents();
-          continue;
-        }
+        [this, &t, &delayed_extents](auto& alloc_map) {
+      for (auto& extent : delayed_extents) {
         // For now, just do ool allocation for any delayed extent
         auto& allocator_ptr = get_allocator(
           get_allocator_type(extent->hint), extent->hint
         );
         alloc_map[allocator_ptr.get()].emplace_back(extent);
-        num_ool_extents++;
       }
-      SUBDEBUGT(seastore_tm, "{} ool extents", t, num_ool_extents);
       return trans_intr::do_for_each(alloc_map, [&t](auto& p) {
         auto allocator = p.first;
         auto& extents = p.second;
@@ -388,7 +364,6 @@ private:
     return devices[std::rand() % devices.size()];
   }
 
-  LBAManager& lba_manager;
   std::map<device_type_t, std::vector<ExtentAllocatorRef>> allocators;
 };
 using ExtentPlacementManagerRef = std::unique_ptr<ExtentPlacementManager>;
index 73411dcf7e3acaa6aae9223320e0a74ca3b79da6..c6026ad246224693746e7b3a02d47aefe81e3ac8 100644 (file)
@@ -6,9 +6,43 @@
 #include "crimson/os/seastore/lba_manager.h"
 #include "crimson/os/seastore/lba_manager/btree/btree_lba_manager.h"
 
-namespace crimson::os::seastore::lba_manager {
+namespace crimson::os::seastore {
 
-LBAManagerRef create_lba_manager(
+LBAManager::update_mappings_ret
+LBAManager::update_mappings(
+  Transaction& t,
+  const std::list<LogicalCachedExtentRef>& extents,
+  const std::vector<paddr_t>& original_paddrs)
+{
+  assert(extents.size() == original_paddrs.size());
+  auto extents_end = extents.end();
+  return seastar::do_with(
+      extents.begin(),
+      original_paddrs.begin(),
+      [this, extents_end, &t](auto& iter_extents,
+                              auto& iter_original_paddrs) {
+    return trans_intr::repeat(
+      [this, extents_end, &t, &iter_extents, &iter_original_paddrs]
+    {
+      if (extents_end == iter_extents) {
+        return update_mappings_iertr::make_ready_future<
+          seastar::stop_iteration>(seastar::stop_iteration::yes);
+      }
+      return update_mapping(
+          t,
+          (*iter_extents)->get_laddr(),
+          *iter_original_paddrs,
+          (*iter_extents)->get_paddr()
+      ).si_then([&iter_extents, &iter_original_paddrs] {
+        ++iter_extents;
+        ++iter_original_paddrs;
+        return seastar::stop_iteration::no;
+      });
+    });
+  });
+}
+
+LBAManagerRef lba_manager::create_lba_manager(
   SegmentManager &segment_manager,
   Cache &cache) {
   return LBAManagerRef(new btree::BtreeLBAManager(segment_manager, cache));
index fe0a5fa331bcb36e5558fadba4e2dc5c3ffe8bfe..172052bac55bacecd2a405dbaf5a370d81a4c6b3 100644 (file)
@@ -175,6 +175,19 @@ public:
     laddr_t laddr,
     paddr_t prev_addr,
     paddr_t paddr) = 0;
+
+  /**
+   * update_mappings
+   *
+   * update lba mappings for delayed allocated extents
+   */
+  using update_mappings_iertr = update_mapping_iertr;
+  using update_mappings_ret = update_mapping_ret;
+  update_mappings_ret update_mappings(
+    Transaction& t,
+    const std::list<LogicalCachedExtentRef>& extents,
+    const std::vector<paddr_t>& original_paddrs);
+
   /**
    * get_physical_extent_if_live
    *
index bc3e5ca7674fd254bb85a4d7b742f9525555895e..1ff54fca9fd77010645c9b1f6b86f2bba0718a1e 100644 (file)
@@ -1415,12 +1415,10 @@ seastar::future<std::unique_ptr<SeaStore>> make_seastore(
       false /* detailed */);
 
     auto journal = std::make_unique<Journal>(*sm, scanner_ref);
-    auto cache = std::make_unique<Cache>(scanner_ref);
+    auto epm = std::make_unique<ExtentPlacementManager>();
+    auto cache = std::make_unique<Cache>(scanner_ref, *epm);
     auto lba_manager = lba_manager::create_lba_manager(*sm, *cache);
 
-    auto epm = std::make_unique<ExtentPlacementManager>(*lba_manager);
-    cache->set_epm(*epm);
-
     journal->set_segment_provider(&*segment_cleaner);
 
     auto tm = std::make_unique<TransactionManager>(
index 4090744412841e3c6bc723205be6af68b6e0ce05..d613261fa366a26990d99d22abe3ea7cc464f41f 100644 (file)
@@ -190,8 +190,18 @@ public:
     return to_release;
   }
 
-  auto& get_delayed_alloc_list() {
-    return delayed_alloc_list;
+  auto get_delayed_alloc_list() {
+    std::list<LogicalCachedExtentRef> ret;
+    for (auto& extent : delayed_alloc_list) {
+      // delayed extents may be invalidated
+      if (extent->is_valid()) {
+        ret.push_back(std::move(extent));
+      } else {
+        ++num_delayed_invalid_extents;
+      }
+    }
+    delayed_alloc_list.clear();
+    return ret;
   }
 
   const auto &get_mutated_block_list() {
@@ -348,10 +358,6 @@ public:
     return ool_write_stats;
   }
 
-  void increment_delayed_invalid_extents() {
-    ++num_delayed_invalid_extents;
-  }
-
 private:
   friend class Cache;
   friend Ref make_test_transaction();
index db37abdd096349cc17b424df0efc6fd2ab6d5f24..ec80eb92341773d58a83526cd20f46da3c6ebf61 100644 (file)
@@ -285,12 +285,29 @@ TransactionManager::submit_transaction_direct(
   return trans_intr::make_interruptible(
     tref.get_handle().enter(write_pipeline.ool_writes)
   ).then_interruptible([this, FNAME, &tref] {
-    SUBTRACET(seastore_t, "process delayed and out-of-line extents", tref);
-    return epm->delayed_alloc_or_ool_write(tref
-    ).handle_error_interruptible(
-      crimson::ct_error::input_output_error::pass_further(),
-      crimson::ct_error::assert_all("invalid error")
-    );
+    auto delayed_extents = tref.get_delayed_alloc_list();
+    auto num_extents = delayed_extents.size();
+    SUBTRACET(seastore_t, "process {} delayed extents", tref, num_extents);
+    std::vector<paddr_t> delayed_paddrs;
+    delayed_paddrs.reserve(num_extents);
+    for (auto& ext : delayed_extents) {
+      assert(ext->get_paddr().is_delayed());
+      delayed_paddrs.push_back(ext->get_paddr());
+    }
+    return seastar::do_with(
+      std::move(delayed_extents),
+      std::move(delayed_paddrs),
+      [this, FNAME, &tref](auto& delayed_extents, auto& delayed_paddrs)
+    {
+      return epm->delayed_alloc_or_ool_write(tref, delayed_extents
+      ).si_then([this, FNAME, &tref, &delayed_extents, &delayed_paddrs] {
+        SUBTRACET(seastore_t, "update delayed extent mappings", tref);
+        return lba_manager->update_mappings(tref, delayed_extents, delayed_paddrs);
+      }).handle_error_interruptible(
+        crimson::ct_error::input_output_error::pass_further(),
+        crimson::ct_error::assert_all("invalid error")
+      );
+    });
   }).si_then([this, FNAME, &tref] {
     SUBTRACET(seastore_t, "about to prepare", tref);
     return tref.get_handle().enter(write_pipeline.prepare);
index a5bb254badb8a3a9aecb5d22d683b2b19e09958a..a7371ae4a3ce6d318c5b4c5589ef947a0129ccba 100644 (file)
@@ -550,7 +550,6 @@ public:
       std::make_unique<SegmentedAllocator>(
        *segment_cleaner,
        *sm,
-       *lba_manager,
        *journal));
   }
 
index 19cbd56e5e62962d3566e4eab3d7c5beeaac4868..3ac4cccef1616cd920e5e107200a11ad1df4e106 100644 (file)
@@ -140,20 +140,17 @@ void TMDriver::init()
     false /* detailed */);
   std::vector<SegmentManager*> sms;
   segment_cleaner->mount(segment_manager->get_device_id(), sms);
-  auto journal = std::make_unique<Journal>(*segment_manager, scanner_ref);
-  auto cache = std::make_unique<Cache>(scanner_ref);
+  auto journal = std::make_unique<Journal>(*segment_manager, *scanner);
+  auto epm = std::make_unique<ExtentPlacementManager>();
+  auto cache = std::make_unique<Cache>(scanner_ref, *epm);
   auto lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache);
 
-  auto epm = std::make_unique<ExtentPlacementManager>(*cache, *lba_manager);
-
   epm->add_allocator(
     device_type_t::SEGMENTED,
     std::make_unique<SegmentedAllocator>(
       *segment_cleaner,
       *segment_manager,
-      *lba_manager,
-      *journal,
-      *cache));
+      *journal));
 
   journal->set_segment_provider(&*segment_cleaner);
 
index f9d5d195057b11887eb9b1511b40ff56729b866c..7bf2e01a4ad385e9493efd63c7e5f96c099b7f60 100644 (file)
@@ -30,6 +30,7 @@ struct btree_test_base :
   segment_manager::EphemeralSegmentManagerRef segment_manager;
   ExtentReaderRef scanner;
   JournalRef journal;
+  ExtentPlacementManagerRef epm;
   CacheRef cache;
 
   size_t block_size;
@@ -74,7 +75,8 @@ struct btree_test_base :
     segment_manager = segment_manager::create_test_ephemeral();
     scanner.reset(new ExtentReader());
     journal.reset(new Journal(*segment_manager, *scanner));
-    cache.reset(new Cache(*scanner));
+    epm.reset(new ExtentPlacementManager());
+    cache.reset(new Cache(*scanner, *epm));
 
     block_size = segment_manager->get_block_size();
     next = segment_id_t{segment_manager->get_device_id(), 0};
@@ -117,6 +119,7 @@ struct btree_test_base :
       segment_manager.reset();
       scanner.reset();
       journal.reset();
+      epm.reset();
       cache.reset();
     }).handle_error(
       crimson::ct_error::all_same_way([] {
index 4b49f020acf5ae5da1cf69be6a8507a636ec9840..90bf0f4d485b6aa4110531f378b16ef57fdf1bb8 100644 (file)
@@ -22,6 +22,7 @@ namespace {
 struct cache_test_t : public seastar_test_suite_t {
   segment_manager::EphemeralSegmentManagerRef segment_manager;
   ExtentReaderRef reader;
+  ExtentPlacementManagerRef epm;
   CacheRef cache;
   paddr_t current;
   journal_seq_t seq;
@@ -84,7 +85,8 @@ struct cache_test_t : public seastar_test_suite_t {
   seastar::future<> set_up_fut() final {
     segment_manager = segment_manager::create_test_ephemeral();
     reader.reset(new ExtentReader());
-    cache.reset(new Cache(*reader));
+    epm.reset(new ExtentPlacementManager());
+    cache.reset(new Cache(*reader, *epm));
     current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0);
     reader->add_segment_manager(segment_manager.get());
     return segment_manager->init(
@@ -112,6 +114,7 @@ struct cache_test_t : public seastar_test_suite_t {
     ).safe_then([this] {
       segment_manager.reset();
       reader.reset();
+      epm.reset();
       cache.reset();
     }).handle_error(
       Cache::close_ertr::assert_all{}
index 9213d2b8243a022ea8dac2b3eef4c3f8260ced1c..c177eafbf10115e5ad9b825175b18313bb523528 100644 (file)
@@ -80,19 +80,16 @@ auto get_transaction_manager(
     std::move(scanner),
     true);
   auto journal = std::make_unique<Journal>(segment_manager, scanner_ref);
-  auto cache = std::make_unique<Cache>(scanner_ref);
+  auto epm = std::make_unique<ExtentPlacementManager>();
+  auto cache = std::make_unique<Cache>(scanner_ref, *epm);
   auto lba_manager = lba_manager::create_lba_manager(segment_manager, *cache);
 
-  auto epm = std::make_unique<ExtentPlacementManager>(*cache, *lba_manager);
-
   epm->add_allocator(
     device_type_t::SEGMENTED,
     std::make_unique<SegmentedAllocator>(
       *segment_cleaner,
       segment_manager,
-      *lba_manager,
-      *journal,
-      *cache));
+      *journal));
 
   journal->set_segment_provider(&*segment_cleaner);