crimson/os/seastore: decouple SegmentManager and TransactionManager

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

diff --git a/src/crimson/os/seastore/device.h b/src/crimson/os/seastore/device.h
index bc050a0225fc885cce405e088f715db19af416d2..33a97a6d70c787f2bdaa357bff3e546cf93793d9 100644 (file)
--- a/src/crimson/os/seastore/device.h
+++ b/src/crimson/os/seastore/device.h
@@ -20,6 +20,8 @@ class Device {
 public:
   virtual device_id_t get_device_id() const = 0;
 
+  virtual seastore_off_t get_block_size() const = 0;
+
   using read_ertr = crimson::errorator<
     crimson::ct_error::input_output_error,
     crimson::ct_error::invarg,

diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h
index e7dddf3534152d8b2b12620798964ce6c80042e9..8865aa9201e5edf31a06d06d6c7f017853087fa2 100644 (file)
--- a/src/crimson/os/seastore/extent_placement_manager.h
+++ b/src/crimson/os/seastore/extent_placement_manager.h
@@ -175,10 +175,14 @@ public:
     devices_by_id.resize(DEVICE_ID_MAX, nullptr);
   }
 
-  void add_device(Device* device) {
+  void add_device(Device* device, bool is_primary) {
     auto device_id = device->get_device_id();
     ceph_assert(devices_by_id[device_id] == nullptr);
     devices_by_id[device_id] = device;
+    if (is_primary) {
+      ceph_assert(primary_device == nullptr);
+      primary_device = device;
+    }
   }
 
   void add_allocator(device_type_t type, ExtentAllocatorRef&& allocator) {
@@ -189,6 +193,17 @@ public:
       allocators[type].size());
   }
 
+  seastore_off_t get_block_size() const {
+    assert(primary_device != nullptr);
+    // assume all the devices have the same block size
+    return primary_device->get_block_size();
+  }
+
+  Device& get_primary_device() {
+    assert(primary_device != nullptr);
+    return *primary_device;
+  }
+
   using open_ertr = ExtentOolWriter::open_ertr;
   open_ertr::future<> open() {
     LOG_PREFIX(ExtentPlacementManager::open);
@@ -280,6 +295,7 @@ public:
       allocators.clear();
       devices_by_id.clear();
       devices_by_id.resize(DEVICE_ID_MAX, nullptr);
+      primary_device = nullptr;
     });
   }
 
@@ -307,6 +323,7 @@ private:
 
   std::map<device_type_t, std::vector<ExtentAllocatorRef>> allocators;
   std::vector<Device*> devices_by_id;
+  Device* primary_device = nullptr;
 };
 using ExtentPlacementManagerRef = std::unique_ptr<ExtentPlacementManager>;
 

diff --git a/src/crimson/os/seastore/extent_reader.h b/src/crimson/os/seastore/extent_reader.h
index 68202f6a29ab27c19b1d15906aa64d58459798e2..b32ae81e64b4ce329fd482a65fdef2e80a52c8dd 100644 (file)
--- a/src/crimson/os/seastore/extent_reader.h
+++ b/src/crimson/os/seastore/extent_reader.h
@@ -15,12 +15,6 @@ class TransactionManager;
 
 class ExtentReader {
 public:
-  seastore_off_t get_block_size() const {
-    assert(segment_managers.size());
-    // assume all segment managers have the same block size
-    return segment_managers[0]->get_block_size();
-  }
-
   std::vector<SegmentManager*>& get_segment_managers() {
     return segment_managers;
   }
@@ -84,6 +78,12 @@ public:
     found_record_handler_t &handler    ///< [in] handler for records
   ); ///< @return used budget
 
+  using release_ertr = SegmentManager::release_ertr;
+  release_ertr::future<> release_segment(segment_id_t id) {
+    assert(segment_managers[id.device_id()] != nullptr);
+    return segment_managers[id.device_id()]->release(id);
+  }
+
   void add_segment_manager(SegmentManager* segment_manager) {
     ceph_assert(!segment_managers[segment_manager->get_device_id()]);
     segment_managers[segment_manager->get_device_id()] = segment_manager;

diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc
index 424c0ea8ee9f01ac0c07c219904513d7a2279ed2..9be8bcd5f151f8f2072e8bed485d1c2b3d249da4 100644 (file)
--- a/src/crimson/os/seastore/seastore.cc
+++ b/src/crimson/os/seastore/seastore.cc
@@ -189,7 +189,7 @@ SeaStore::mount_ertr::future<> SeaStore::mount()
 {
   return segment_manager->mount(
   ).safe_then([this] {
-    transaction_manager->add_segment_manager(segment_manager.get());
+    transaction_manager->add_segment_manager(segment_manager.get(), true);
     auto sec_devices = segment_manager->get_secondary_devices();
     return crimson::do_for_each(sec_devices, [this](auto& device_entry) {
       device_id_t id = device_entry.first;
@@ -203,7 +203,7 @@ SeaStore::mount_ertr::future<> SeaStore::mount()
        [this, sm=std::move(sm), magic]() mutable {
        boost::ignore_unused(magic);  // avoid clang warning;
        assert(sm->get_magic() == magic);
-       transaction_manager->add_segment_manager(sm.get());
+       transaction_manager->add_segment_manager(sm.get(), false);
        secondaries.emplace_back(std::move(sm));
        return seastar::now();
       });
@@ -334,7 +334,7 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid)
         }).safe_then([this] {
           return crimson::do_for_each(secondaries, [this](auto& sec_sm) {
             return sec_sm->mount().safe_then([this, &sec_sm] {
-              transaction_manager->add_segment_manager(sec_sm.get());
+              transaction_manager->add_segment_manager(sec_sm.get(), false);
               return seastar::now();
             });
           });
@@ -342,13 +342,13 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid)
       }).safe_then([this] {
         return segment_manager->mount();
       }).safe_then([this] {
-        transaction_manager->add_segment_manager(segment_manager.get());
+        transaction_manager->add_segment_manager(segment_manager.get(), true);
         return transaction_manager->mkfs();
       }).safe_then([this] {
         for (auto& sec_sm : secondaries) {
-          transaction_manager->add_segment_manager(sec_sm.get());
+          transaction_manager->add_segment_manager(sec_sm.get(), false);
         }
-        transaction_manager->add_segment_manager(segment_manager.get());
+        transaction_manager->add_segment_manager(segment_manager.get(), true);
         return transaction_manager->mount();
       }).safe_then([this] {
         return repeat_eagain([this] {
@@ -643,7 +643,7 @@ seastar::future<struct stat> SeaStore::stat(
       struct stat st;
       auto &olayout = onode.get_layout();
       st.st_size = olayout.size;
-      st.st_blksize = transaction_manager->get_block_size();
+      st.st_blksize = segment_manager->get_block_size();
       st.st_blocks = (st.st_size + st.st_blksize - 1) / st.st_blksize;
       st.st_nlink = 1;
       DEBUGT("cid {}, oid {}, return size {}", t, c->get_cid(), oid, st.st_size);

diff --git a/src/crimson/os/seastore/segment_cleaner.cc b/src/crimson/os/seastore/segment_cleaner.cc
index ac48a00a769f28d8e7cbe075202a271612457088..f629f7e818108bfaf7bcd83bb4043f7afcf28862 100644 (file)
--- a/src/crimson/os/seastore/segment_cleaner.cc
+++ b/src/crimson/os/seastore/segment_cleaner.cc
@@ -683,4 +683,21 @@ SegmentCleaner::scan_extents_ret SegmentCleaner::scan_nonfull_segment(
   return seastar::now();
 }
 
+SegmentCleaner::release_ertr::future<>
+SegmentCleaner::maybe_release_segment(Transaction &t)
+{
+  auto to_release = t.get_segment_to_release();
+  if (to_release != NULL_SEG_ID) {
+    LOG_PREFIX(SegmentCleaner::maybe_release_segment);
+    INFOT("releasing segment {}", t, to_release);
+    return scanner->release_segment(to_release
+    ).safe_then([this, to_release] {
+      stats.segments_released++;
+      mark_empty(to_release);
+    });
+  } else {
+    return SegmentManager::release_ertr::now();
+  }
+}
+
 }

diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h
index afb400c84cd0a3ec4a13dccae657745cdb527446..e703b19793d68dacb7e4fe369698259be3070e53 100644 (file)
--- a/src/crimson/os/seastore/segment_cleaner.h
+++ b/src/crimson/os/seastore/segment_cleaner.h
@@ -648,16 +648,6 @@ public:
       laddr_t laddr,
       seastore_off_t len) = 0;
 
-    /**
-     * release_segment
-     *
-     * Release segment.
-     */
-    using release_segment_ertr = SegmentManager::release_ertr;
-    using release_segment_ret = release_segment_ertr::future<>;
-    virtual release_segment_ret release_segment(
-      segment_id_t id) = 0;
-
     /**
      * submit_transaction_direct
      *
@@ -776,10 +766,8 @@ public:
     return segments[id].get_type();
   }
 
-  void mark_segment_released(segment_id_t segment) {
-    stats.segments_released++;
-    return mark_empty(segment);
-  }
+  using release_ertr = ExtentReader::release_ertr;
+  release_ertr::future<> maybe_release_segment(Transaction &t);
 
   void adjust_segment_util(double old_usage, double new_usage) {
     assert(stats.segment_util.buckets[std::floor(old_usage * 10)].count > 0);

diff --git a/src/crimson/os/seastore/segment_manager.h b/src/crimson/os/seastore/segment_manager.h
index 644aaa71444a86608704c8d96f054897116c4309..22a5aeae05686084cb7510b15f089b1f938b09fd 100644 (file)
--- a/src/crimson/os/seastore/segment_manager.h
+++ b/src/crimson/os/seastore/segment_manager.h
@@ -214,7 +214,6 @@ public:
 
   /* Methods for discovering device geometry, segmentid set, etc */
   virtual size_t get_size() const = 0;
-  virtual seastore_off_t get_block_size() const = 0;
   virtual seastore_off_t get_segment_size() const = 0;
   virtual device_segment_id_t get_num_segments() const {
     ceph_assert(get_size() % get_segment_size() == 0);

diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc
index a40f6632b6b244df5ffdb11d93c3f5472108f46b..31d54a77af60a72050a80d4c9edfe7802414fa86 100644 (file)
--- a/src/crimson/os/seastore/transaction_manager.cc
+++ b/src/crimson/os/seastore/transaction_manager.cc
@@ -6,7 +6,6 @@
 
 #include "crimson/os/seastore/logging.h"
 #include "crimson/os/seastore/transaction_manager.h"
-#include "crimson/os/seastore/segment_manager.h"
 #include "crimson/os/seastore/journal.h"
 
 /*
@@ -23,15 +22,13 @@ SET_SUBSYS(seastore_tm);
 namespace crimson::os::seastore {
 
 TransactionManager::TransactionManager(
-  SegmentManager &_segment_manager,
   SegmentCleanerRef _segment_cleaner,
   JournalRef _journal,
   CacheRef _cache,
   LBAManagerRef _lba_manager,
   ExtentPlacementManagerRef&& epm,
   ExtentReader& scanner)
-  : segment_manager(_segment_manager),
-    segment_cleaner(std::move(_segment_cleaner)),
+  : segment_cleaner(std::move(_segment_cleaner)),
     cache(std::move(_cache)),
     lba_manager(std::move(_lba_manager)),
     journal(std::move(_journal)),
@@ -47,7 +44,7 @@ TransactionManager::mkfs_ertr::future<> TransactionManager::mkfs()
   LOG_PREFIX(TransactionManager::mkfs);
   INFO("enter");
   return segment_cleaner->mount(
-    segment_manager.get_device_id()
+    epm->get_primary_device().get_device_id()
   ).safe_then([this] {
     return journal->open_for_write();
   }).safe_then([this](auto addr) {
@@ -87,7 +84,7 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount()
   INFO("enter");
   cache->init();
   return segment_cleaner->mount(
-    segment_manager.get_device_id()
+    epm->get_primary_device().get_device_id()
   ).safe_then([this] {
     return journal->replay(
       [this](const auto &offsets, const auto &e, auto last_modified) {
@@ -340,16 +337,7 @@ TransactionManager::submit_transaction_direct(
       lba_manager->complete_transaction(tref);
       segment_cleaner->update_journal_tail_target(
        cache->get_oldest_dirty_from().value_or(start_seq));
-      auto to_release = tref.get_segment_to_release();
-      if (to_release != NULL_SEG_ID) {
-        SUBDEBUGT(seastore_t, "releasing segment {}", tref, to_release);
-       return segment_manager.release(to_release
-       ).safe_then([this, to_release] {
-         segment_cleaner->mark_segment_released(to_release);
-       });
-      } else {
-       return SegmentManager::release_ertr::now();
-      }
+      return segment_cleaner->maybe_release_segment(tref);
     }).safe_then([FNAME, &tref] {
       SUBTRACET(seastore_t, "completed", tref);
       return tref.get_handle().complete();
@@ -569,7 +557,6 @@ TransactionManagerRef make_transaction_manager(
   auto lba_manager = lba_manager::create_lba_manager(sm, *cache);
 
   return std::make_unique<TransactionManager>(
-    sm,
     std::move(segment_cleaner),
     std::move(journal),
     std::move(cache),

diff --git a/src/crimson/os/seastore/transaction_manager.h b/src/crimson/os/seastore/transaction_manager.h
index 34171b63f98a7481f9ccb00f13ec03b7dab229f4..46f78ded4909da930cd08f671b813e4afb8a9888 100644 (file)
--- a/src/crimson/os/seastore/transaction_manager.h
+++ b/src/crimson/os/seastore/transaction_manager.h
@@ -24,7 +24,6 @@
 #include "crimson/os/seastore/segment_cleaner.h"
 #include "crimson/os/seastore/seastore_types.h"
 #include "crimson/os/seastore/cache.h"
-#include "crimson/os/seastore/segment_manager.h"
 #include "crimson/os/seastore/lba_manager.h"
 #include "crimson/os/seastore/journal.h"
 #include "crimson/os/seastore/extent_placement_manager.h"
@@ -64,7 +63,6 @@ public:
   using base_iertr = Cache::base_iertr;
 
   TransactionManager(
-    SegmentManager &segment_manager,
     SegmentCleanerRef segment_cleaner,
     JournalRef journal,
     CacheRef cache,
@@ -409,15 +407,6 @@ public:
     laddr_t laddr,
     seastore_off_t len) final;
 
-  using release_segment_ret =
-    SegmentCleaner::ExtentCallbackInterface::release_segment_ret;
-  release_segment_ret release_segment(
-    segment_id_t id) final {
-    LOG_PREFIX(TransactionManager::release_segment);
-    SUBDEBUG(seastore_tm, "{}", id);
-    return segment_manager.release(id);
-  }
-
   /**
    * read_root_meta
    *
@@ -534,18 +523,19 @@ public:
   }
 
   extent_len_t get_block_size() const {
-    return segment_manager.get_block_size();
+    return epm->get_block_size();
   }
 
   store_statfs_t store_stat() const {
     return segment_cleaner->stat();
   }
 
-  void add_segment_manager(SegmentManager* sm) {
+  void add_segment_manager(SegmentManager* sm, bool is_primary) {
     LOG_PREFIX(TransactionManager::add_segment_manager);
-    SUBDEBUG(seastore_tm, "adding segment manager {}", sm->get_device_id());
+    SUBDEBUG(seastore_tm, "adding segment manager {}, is_primary={}",
+             sm->get_device_id(), is_primary);
     scanner.add_segment_manager(sm);
-    epm->add_device(sm);
+    epm->add_device(sm, is_primary);
     epm->add_allocator(
       device_type_t::SEGMENTED,
       std::make_unique<SegmentedAllocator>(
@@ -559,10 +549,6 @@ public:
 private:
   friend class Transaction;
 
-  // although there might be multiple devices backing seastore,
-  // only one of them are supposed to hold the journal. This
-  // segment manager is that device
-  SegmentManager &segment_manager;
   SegmentCleanerRef segment_cleaner;
   CacheRef cache;
   LBAManagerRef lba_manager;

diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc
index 851228e2ad42939d1e9684e8e80d738856c16ec3..54f6912349aff14b10618fbd35e5e53fc48172ee 100644 (file)
--- a/src/crimson/tools/store_nbd/tm_driver.cc
+++ b/src/crimson/tools/store_nbd/tm_driver.cc
@@ -135,7 +135,7 @@ seastar::future<bufferlist> TMDriver::read(
 void TMDriver::init()
 {
   tm = make_transaction_manager(*segment_manager, false /* detailed */);
-  tm->add_segment_manager(segment_manager.get());
+  tm->add_segment_manager(segment_manager.get(), true);
 }
 
 void TMDriver::clear()

diff --git a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
index 77578f3625f88c088aa2d720ef7c3c3875ef84aa..3ff93fb4f85d2ac79aa99bcbc2e277ad5b7ff9d6 100644 (file)
--- a/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
+++ b/src/test/crimson/seastore/onode_tree/test_fltree_onode_manager.cc
@@ -84,7 +84,7 @@ struct fltree_onode_manager_test_t
   virtual FuturizedStore::mkfs_ertr::future<> _mkfs() final {
     return TMTestState::_mkfs(
     ).safe_then([this] {
-      tm->add_segment_manager(segment_manager.get());
+      tm->add_segment_manager(segment_manager.get(), true);
       return tm->mount(
       ).safe_then([this] {
        return repeat_eagain([this] {

diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc
index 79bbce63c6aa92671e2a8582b3b9bccd9f3852a7..f0cf4e03dd854d8dd0f9dbce6d780a8be524c1ec 100644 (file)
--- a/src/test/crimson/seastore/test_btree_lba_manager.cc
+++ b/src/test/crimson/seastore/test_btree_lba_manager.cc
@@ -108,7 +108,7 @@ struct btree_test_base :
     block_size = segment_manager->get_block_size();
     next = segment_id_t{segment_manager->get_device_id(), 0};
     scanner_ref.add_segment_manager(segment_manager.get());
-    epm->add_device(segment_manager.get());
+    epm->add_device(segment_manager.get(), true);
     journal->set_write_pipeline(&pipeline);
 
     return segment_manager->init(

diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc
index 94efc4fd48db4f0b3637e95bacf99fc98c9eedbe..5840739448bef66103b69cc02b1addc28182d53c 100644 (file)
--- a/src/test/crimson/seastore/test_seastore_cache.cc
+++ b/src/test/crimson/seastore/test_seastore_cache.cc
@@ -86,7 +86,7 @@ struct cache_test_t : public seastar_test_suite_t {
     epm.reset(new ExtentPlacementManager());
     cache.reset(new Cache(*epm));
     current = paddr_t::make_seg_paddr(segment_id_t(segment_manager->get_device_id(), 0), 0);
-    epm->add_device(segment_manager.get());
+    epm->add_device(segment_manager.get(), true);
     return segment_manager->init(
     ).safe_then([this] {
       return seastar::do_with(

diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h
index 7c0d3df0878d4eaf0f0feb6072dd220ff453e02a..1dd72325f6727050bc101f6cd1bd74e8e7112a73 100644 (file)
--- a/src/test/crimson/seastore/transaction_manager_test_state.h
+++ b/src/test/crimson/seastore/transaction_manager_test_state.h
@@ -93,7 +93,7 @@ protected:
 
   virtual void _init() override {
     tm = make_transaction_manager(*segment_manager, true);
-    tm->add_segment_manager(segment_manager.get());
+    tm->add_segment_manager(segment_manager.get(), true);
     segment_cleaner = tm->get_segment_cleaner();
     lba_manager = tm->get_lba_manager();
   }