crimson/onode-staged-tree: random insert/split test

Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>

diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
index a6dedb4650989bfccd44f2264838b215611239f3..d88d4fe0bea3cc5c62fe6dce0c882f7c9b84bc62 100644 (file)
--- a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
+++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc
@@ -10,16 +10,27 @@
 #include <vector>
 
 #include "crimson/common/log.h"
+#include "crimson/os/seastore/cache.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/node_layout.h"
 #include "crimson/os/seastore/onode_manager/staged-fltree/tree.h"
+#include "crimson/os/seastore/segment_manager.h"
+#include "crimson/os/seastore/transaction_manager.h"
 
 #include "test/crimson/gtest_seastar.h"
 
 // TODO: use assert instead of logging
 
 using namespace crimson::os::seastore::onode;
+using crimson::os::seastore::Cache;
+using crimson::os::seastore::Journal;
+using crimson::os::seastore::LBAManagerRef;
+using crimson::os::seastore::lba_manager::create_lba_manager;
+using crimson::os::seastore::SegmentCleaner;
+using crimson::os::seastore::SegmentManager;
+using crimson::os::seastore::segment_manager::create_ephemeral;
+using crimson::os::seastore::segment_manager::DEFAULT_TEST_EPHEMERAL;
 
 namespace {
   [[maybe_unused]] seastar::logger& logger() {
@@ -79,6 +90,13 @@ namespace {
       }
     }
 
+    Onodes(std::vector<size_t> sizes) {
+      for (auto& size : sizes) {
+        auto p_onode = &create(size);
+        onodes.push_back(p_onode);
+      }
+    }
+
     ~Onodes() {
       std::for_each(tracked_onodes.begin(), tracked_onodes.end(),
                     [] (onode_t* onode) {
@@ -1081,3 +1099,211 @@ TEST_F(c_dummy_test_t, 5_split_internal_node)
     // Impossible to split at [END, END, END]
   });
 }
+
+class KVPool {
+  struct kv_conf_t {
+    unsigned index_2;
+    unsigned index_1;
+    unsigned index_0;
+    size_t ns_size;
+    size_t oid_size;
+    const onode_t* p_value;
+
+    ghobject_t get_ghobj() const {
+      assert(index_1 < 10);
+      std::ostringstream os_ns;
+      os_ns << "ns" << index_1;
+      unsigned current_size = (unsigned)os_ns.tellp();
+      assert(ns_size >= current_size);
+      os_ns << std::string(ns_size - current_size, '_');
+
+      std::ostringstream os_oid;
+      os_oid << "oid" << index_1;
+      current_size = (unsigned)os_oid.tellp();
+      assert(oid_size >= current_size);
+      os_oid << std::string(oid_size - current_size, '_');
+
+      return make_ghobj(index_2, index_2, index_2,
+                        os_ns.str(), os_oid.str(), index_0, index_0);
+    }
+  };
+  using kv_vector_t = std::vector<kv_conf_t>;
+
+ public:
+  using kv_t = std::pair<ghobject_t, const onode_t*>;
+
+  KVPool(std::vector<size_t> str_sizes,
+         std::vector<size_t> onode_sizes,
+         std::pair<unsigned, unsigned> range_2,
+         std::pair<unsigned, unsigned> range_1,
+         std::pair<unsigned, unsigned> range_0)
+      : str_sizes{str_sizes}, onodes{onode_sizes} {
+    std::random_device rd;
+    for (unsigned i = range_2.first; i < range_2.second; ++i) {
+      for (unsigned j = range_1.first; j < range_1.second; ++j) {
+        auto ns_size = (unsigned)str_sizes[rd() % str_sizes.size()];
+        auto oid_size = (unsigned)str_sizes[rd() % str_sizes.size()];
+        for (unsigned k = range_0.first; k < range_0.second; ++k) {
+          kvs.emplace_back(kv_conf_t{i, j, k, ns_size, oid_size, &onodes.pick()});
+        }
+      }
+    }
+    random_kvs = kvs;
+    std::random_shuffle(random_kvs.begin(), random_kvs.end());
+  }
+
+  class iterator_t {
+   public:
+    kv_t get_kv() const {
+      assert(!is_end());
+      auto& conf = (*p_kvs)[i];
+      return std::make_pair(conf.get_ghobj(), conf.p_value);
+    }
+    bool is_end() const { return i >= p_kvs->size(); }
+    size_t index() const { return i; }
+
+    iterator_t& operator++() {
+      assert(!is_end());
+      ++i;
+      return *this;
+    }
+
+    iterator_t operator++(int) {
+      iterator_t tmp = *this;
+      ++*this;
+      return tmp;
+    }
+
+   private:
+    iterator_t(const kv_vector_t& kvs) : p_kvs{&kvs} {}
+
+    const kv_vector_t* p_kvs;
+    size_t i = 0;
+    friend class KVPool;
+  };
+
+  iterator_t begin() const {
+    return iterator_t(kvs);
+  }
+
+  iterator_t random_begin() const {
+    return iterator_t(random_kvs);
+  }
+
+  size_t size() const {
+    return kvs.size();
+  }
+
+  kv_t make_kv(unsigned index_2, unsigned index_1, unsigned index_0,
+               size_t ns_size, size_t oid_size, size_t onode_size) {
+    auto& onode = onodes.create(onode_size);
+    kv_conf_t conf{index_2, index_1, index_0, ns_size, oid_size, &onode};
+    return std::make_pair(conf.get_ghobj(), &onode);
+  }
+
+ private:
+  std::vector<size_t> str_sizes;
+  Onodes onodes;
+  kv_vector_t kvs;
+  kv_vector_t random_kvs;
+};
+
+struct d_seastore_tree_test_t : public seastar_test_suite_t {
+  std::unique_ptr<SegmentManager> segment_manager;
+  SegmentCleaner segment_cleaner;
+  Journal journal;
+  Cache cache;
+  LBAManagerRef lba_manager;
+  TransactionManager tm;
+  NodeExtentManagerURef moved_nm;
+  TransactionRef ref_t;
+  Transaction& t;
+  Btree tree;
+
+  d_seastore_tree_test_t()
+    : segment_manager(create_ephemeral(DEFAULT_TEST_EPHEMERAL)),
+      segment_cleaner(SegmentCleaner::config_t::default_from_segment_manager(
+                        *segment_manager)),
+      journal(*segment_manager),
+      cache(*segment_manager),
+      lba_manager(create_lba_manager(*segment_manager, cache)),
+      tm(*segment_manager, segment_cleaner, journal, cache, *lba_manager),
+      moved_nm{
+#if 0
+        NodeExtentManager::create_dummy()},
+#else
+        NodeExtentManager::create_seastore(tm)},
+#endif
+      ref_t{make_transaction()},
+      t{*ref_t},
+      tree{std::move(moved_nm)} {
+    journal.set_segment_provider(&segment_cleaner);
+    segment_cleaner.set_extent_callback(&tm);
+  }
+
+  seastar::future<> set_up_fut() override final {
+    return segment_manager->init().safe_then([this] {
+      return tm.mkfs();
+    }).safe_then([this] {
+      return tm.mount();
+    }).safe_then([this] {
+      return tree.mkfs(t);
+    }).handle_error(
+      crimson::ct_error::all_same_way([] {
+        ASSERT_FALSE("Unable to mkfs");
+      })
+    );
+  }
+
+  seastar::future<> tear_down_fut() final {
+    return tm.close().handle_error(
+      crimson::ct_error::all_same_way([] {
+        ASSERT_FALSE("Unable to close");
+      })
+    );
+  }
+};
+
+TEST_F(d_seastore_tree_test_t, 6_random_insert_leaf_node)
+{
+  run_async([this] {
+    KVPool kvs({8, 11, 64, 256, 301, 320},
+               {8, 16, 128, 512, 576, 640},
+               {0, 32}, {0, 10}, {0, 4});
+    auto iter = kvs.random_begin();
+    std::vector<Btree::Cursor> cursors;
+    while (!iter.is_end()) {
+      auto [key, p_value] = iter.get_kv();
+      logger().info("[{}] {} -> {}", iter.index(), key_hobj_t{key}, *p_value);
+      auto [cursor, success] = tree.insert(t, key, *p_value).unsafe_get0();
+      assert(success == true);
+#if 1 // validate cursor tracking
+      cursors.emplace_back(cursor);
+#endif
+      Onodes::validate_cursor(cursor, key, *p_value);
+      auto cursor_ = tree.lower_bound(t, key).unsafe_get0();
+      assert(cursor_.get_ghobj() == key);
+      assert(cursor_.value() == cursor.value());
+      ++iter;
+    }
+
+    logger().info("Insert done! Tree height: {}", tree.height(t).unsafe_get0());
+
+    if (!cursors.empty()) {
+      auto kv_iter = kvs.random_begin();
+      auto c_iter = cursors.begin();
+      while (!kv_iter.is_end()) {
+        assert(c_iter != cursors.end());
+        auto [k, v] = kv_iter.get_kv();
+        // validate values in tree keep intact
+        auto cursor = tree.lower_bound(t, k).unsafe_get0();
+        Onodes::validate_cursor(cursor, k, *v);
+        // validate values in cursors keep intact
+        Onodes::validate_cursor(*c_iter, k, *v);
+
+        ++kv_iter;
+        ++c_iter;
+      }
+    }
+  });
+}