From: Yingxin Cheng Date: Wed, 30 Sep 2020 01:52:59 +0000 (+0800) Subject: crimson/onode-staged-tree: random insert/split test X-Git-Tag: v16.1.0~359^2~22 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=761e67f57db01cf656dd6bb6845a350838279454;p=ceph.git crimson/onode-staged-tree: random insert/split test Signed-off-by: Yingxin Cheng --- diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc index a6dedb465098..d88d4fe0bea3 100644 --- a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc +++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc @@ -10,16 +10,27 @@ #include #include "crimson/common/log.h" +#include "crimson/os/seastore/cache.h" #include "crimson/os/seastore/onode_manager/staged-fltree/node.h" #include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager.h" #include "crimson/os/seastore/onode_manager/staged-fltree/node_layout.h" #include "crimson/os/seastore/onode_manager/staged-fltree/tree.h" +#include "crimson/os/seastore/segment_manager.h" +#include "crimson/os/seastore/transaction_manager.h" #include "test/crimson/gtest_seastar.h" // TODO: use assert instead of logging using namespace crimson::os::seastore::onode; +using crimson::os::seastore::Cache; +using crimson::os::seastore::Journal; +using crimson::os::seastore::LBAManagerRef; +using crimson::os::seastore::lba_manager::create_lba_manager; +using crimson::os::seastore::SegmentCleaner; +using crimson::os::seastore::SegmentManager; +using crimson::os::seastore::segment_manager::create_ephemeral; +using crimson::os::seastore::segment_manager::DEFAULT_TEST_EPHEMERAL; namespace { [[maybe_unused]] seastar::logger& logger() { @@ -79,6 +90,13 @@ namespace { } } + Onodes(std::vector sizes) { + for (auto& size : sizes) { + auto p_onode = &create(size); + onodes.push_back(p_onode); + } + } + ~Onodes() { std::for_each(tracked_onodes.begin(), tracked_onodes.end(), [] (onode_t* onode) { @@ -1081,3 +1099,211 @@ TEST_F(c_dummy_test_t, 5_split_internal_node) // Impossible to split at [END, END, END] }); } + +class KVPool { + struct kv_conf_t { + unsigned index_2; + unsigned index_1; + unsigned index_0; + size_t ns_size; + size_t oid_size; + const onode_t* p_value; + + ghobject_t get_ghobj() const { + assert(index_1 < 10); + std::ostringstream os_ns; + os_ns << "ns" << index_1; + unsigned current_size = (unsigned)os_ns.tellp(); + assert(ns_size >= current_size); + os_ns << std::string(ns_size - current_size, '_'); + + std::ostringstream os_oid; + os_oid << "oid" << index_1; + current_size = (unsigned)os_oid.tellp(); + assert(oid_size >= current_size); + os_oid << std::string(oid_size - current_size, '_'); + + return make_ghobj(index_2, index_2, index_2, + os_ns.str(), os_oid.str(), index_0, index_0); + } + }; + using kv_vector_t = std::vector; + + public: + using kv_t = std::pair; + + KVPool(std::vector str_sizes, + std::vector onode_sizes, + std::pair range_2, + std::pair range_1, + std::pair range_0) + : str_sizes{str_sizes}, onodes{onode_sizes} { + std::random_device rd; + for (unsigned i = range_2.first; i < range_2.second; ++i) { + for (unsigned j = range_1.first; j < range_1.second; ++j) { + auto ns_size = (unsigned)str_sizes[rd() % str_sizes.size()]; + auto oid_size = (unsigned)str_sizes[rd() % str_sizes.size()]; + for (unsigned k = range_0.first; k < range_0.second; ++k) { + kvs.emplace_back(kv_conf_t{i, j, k, ns_size, oid_size, &onodes.pick()}); + } + } + } + random_kvs = kvs; + std::random_shuffle(random_kvs.begin(), random_kvs.end()); + } + + class iterator_t { + public: + kv_t get_kv() const { + assert(!is_end()); + auto& conf = (*p_kvs)[i]; + return std::make_pair(conf.get_ghobj(), conf.p_value); + } + bool is_end() const { return i >= p_kvs->size(); } + size_t index() const { return i; } + + iterator_t& operator++() { + assert(!is_end()); + ++i; + return *this; + } + + iterator_t operator++(int) { + iterator_t tmp = *this; + ++*this; + return tmp; + } + + private: + iterator_t(const kv_vector_t& kvs) : p_kvs{&kvs} {} + + const kv_vector_t* p_kvs; + size_t i = 0; + friend class KVPool; + }; + + iterator_t begin() const { + return iterator_t(kvs); + } + + iterator_t random_begin() const { + return iterator_t(random_kvs); + } + + size_t size() const { + return kvs.size(); + } + + kv_t make_kv(unsigned index_2, unsigned index_1, unsigned index_0, + size_t ns_size, size_t oid_size, size_t onode_size) { + auto& onode = onodes.create(onode_size); + kv_conf_t conf{index_2, index_1, index_0, ns_size, oid_size, &onode}; + return std::make_pair(conf.get_ghobj(), &onode); + } + + private: + std::vector str_sizes; + Onodes onodes; + kv_vector_t kvs; + kv_vector_t random_kvs; +}; + +struct d_seastore_tree_test_t : public seastar_test_suite_t { + std::unique_ptr segment_manager; + SegmentCleaner segment_cleaner; + Journal journal; + Cache cache; + LBAManagerRef lba_manager; + TransactionManager tm; + NodeExtentManagerURef moved_nm; + TransactionRef ref_t; + Transaction& t; + Btree tree; + + d_seastore_tree_test_t() + : segment_manager(create_ephemeral(DEFAULT_TEST_EPHEMERAL)), + segment_cleaner(SegmentCleaner::config_t::default_from_segment_manager( + *segment_manager)), + journal(*segment_manager), + cache(*segment_manager), + lba_manager(create_lba_manager(*segment_manager, cache)), + tm(*segment_manager, segment_cleaner, journal, cache, *lba_manager), + moved_nm{ +#if 0 + NodeExtentManager::create_dummy()}, +#else + NodeExtentManager::create_seastore(tm)}, +#endif + ref_t{make_transaction()}, + t{*ref_t}, + tree{std::move(moved_nm)} { + journal.set_segment_provider(&segment_cleaner); + segment_cleaner.set_extent_callback(&tm); + } + + seastar::future<> set_up_fut() override final { + return segment_manager->init().safe_then([this] { + return tm.mkfs(); + }).safe_then([this] { + return tm.mount(); + }).safe_then([this] { + return tree.mkfs(t); + }).handle_error( + crimson::ct_error::all_same_way([] { + ASSERT_FALSE("Unable to mkfs"); + }) + ); + } + + seastar::future<> tear_down_fut() final { + return tm.close().handle_error( + crimson::ct_error::all_same_way([] { + ASSERT_FALSE("Unable to close"); + }) + ); + } +}; + +TEST_F(d_seastore_tree_test_t, 6_random_insert_leaf_node) +{ + run_async([this] { + KVPool kvs({8, 11, 64, 256, 301, 320}, + {8, 16, 128, 512, 576, 640}, + {0, 32}, {0, 10}, {0, 4}); + auto iter = kvs.random_begin(); + std::vector cursors; + while (!iter.is_end()) { + auto [key, p_value] = iter.get_kv(); + logger().info("[{}] {} -> {}", iter.index(), key_hobj_t{key}, *p_value); + auto [cursor, success] = tree.insert(t, key, *p_value).unsafe_get0(); + assert(success == true); +#if 1 // validate cursor tracking + cursors.emplace_back(cursor); +#endif + Onodes::validate_cursor(cursor, key, *p_value); + auto cursor_ = tree.lower_bound(t, key).unsafe_get0(); + assert(cursor_.get_ghobj() == key); + assert(cursor_.value() == cursor.value()); + ++iter; + } + + logger().info("Insert done! Tree height: {}", tree.height(t).unsafe_get0()); + + if (!cursors.empty()) { + auto kv_iter = kvs.random_begin(); + auto c_iter = cursors.begin(); + while (!kv_iter.is_end()) { + assert(c_iter != cursors.end()); + auto [k, v] = kv_iter.get_kv(); + // validate values in tree keep intact + auto cursor = tree.lower_bound(t, k).unsafe_get0(); + Onodes::validate_cursor(cursor, k, *v); + // validate values in cursors keep intact + Onodes::validate_cursor(*c_iter, k, *v); + + ++kv_iter; + ++c_iter; + } + } + }); +}