So TreeBuilder can be reused by different value implementations.
Signed-off-by: Yingxin Cheng <yingxin.cheng@intel.com>
#include "crimson/common/log.h"
#include "stages/key_layout.h"
#include "tree.h"
-#include "test/crimson/seastore/onode_tree/test_value.h"
/**
* tree_utils.h
namespace crimson::os::seastore::onode {
-using TestBtree = Btree<TestValue>;
+/**
+ * ValueItem template to work with tree utility classes:
+ *
+ * struct ValueItem {
+ * using ValueType = ConcreteValueType;
+ * <public members>
+ *
+ * value_size_t get_payload_size() const;
+ * void initialize(Transaction& t, ValueType& value) const;
+ * void validate(ValueType& value) const;
+ * static ValueItem create(std::size_t expected_size, std::size_t id);
+ * };
+ * std::ostream& operator<<(std::ostream& os, const ValueItem& item);
+ */
-struct value_item_t {
- value_size_t size;
- TestValue::id_t id;
- TestValue::magic_t magic;
+template <typename ValueItem>
+void initialize_cursor_from_item(
+ Transaction& t,
+ const ghobject_t& key,
+ const ValueItem& item,
+ typename Btree<typename ValueItem::ValueType>::Cursor& cursor,
+ bool insert_success) {
+ ceph_assert(insert_success);
+ ceph_assert(!cursor.is_end());
+ ceph_assert(cursor.get_ghobj() == key);
+ auto tree_value = cursor.value();
+ item.initialize(t, tree_value);
+}
- TestBtree::tree_value_config_t get_config() const {
- assert(size > sizeof(value_header_t));
- return {static_cast<value_size_t>(size - sizeof(value_header_t))};
- }
-};
-inline std::ostream& operator<<(std::ostream& os, const value_item_t& item) {
- return os << "ValueItem(#" << item.id << ", " << item.size << "B)";
+
+template <typename ValueItem>
+void validate_cursor_from_item(
+ const ghobject_t& key,
+ const ValueItem& item,
+ typename Btree<typename ValueItem::ValueType>::Cursor& cursor) {
+ ceph_assert(!cursor.is_end());
+ ceph_assert(cursor.get_ghobj() == key);
+ auto value = cursor.value();
+ item.validate(value);
}
+template <typename ValueItem>
class Values {
public:
Values(size_t n) {
~Values() = default;
- value_item_t create(size_t _size) {
- ceph_assert(_size <= std::numeric_limits<value_size_t>::max());
- ceph_assert(_size > sizeof(value_header_t));
- value_size_t size = _size;
- auto current_id = id++;
- return value_item_t{size, current_id, (TestValue::magic_t)current_id * 137};
+ ValueItem create(size_t size) {
+ return ValueItem::create(size, id++);
}
- value_item_t pick() const {
+ ValueItem pick() const {
auto index = rd() % values.size();
return values[index];
}
- static void initialize_cursor(
- Transaction& t,
- TestBtree::Cursor& cursor,
- const value_item_t& item) {
- ceph_assert(!cursor.is_end());
- auto value = cursor.value();
- ceph_assert(value.get_payload_size() + sizeof(value_header_t) == item.size);
- value.set_id_replayable(t, item.id);
- value.set_tail_magic_replayable(t, item.magic);
- }
-
- static void validate_cursor(
- TestBtree::Cursor& cursor,
- const ghobject_t& key,
- const value_item_t& item) {
- ceph_assert(!cursor.is_end());
- ceph_assert(cursor.get_ghobj() == key);
- auto value = cursor.value();
- ceph_assert(value.get_payload_size() + sizeof(value_header_t) == item.size);
- ceph_assert(value.get_id() == item.id);
- ceph_assert(value.get_tail_magic() == item.magic);
- }
-
private:
- TestValue::id_t id = 0;
+ std::size_t id = 0;
mutable std::random_device rd;
- std::vector<value_item_t> values;
+ std::vector<ValueItem> values;
};
+template <typename ValueItem>
class KVPool {
struct kv_conf_t {
index_t index2;
index_t index0;
size_t ns_size;
size_t oid_size;
- value_item_t value;
+ ValueItem value;
ghobject_t get_ghobj() const {
assert(index1 < 10);
using kv_vector_t = std::vector<kv_conf_t>;
public:
- using kv_t = std::pair<ghobject_t, value_item_t>;
+ using kv_t = std::pair<ghobject_t, ValueItem>;
KVPool(const std::vector<size_t>& str_sizes,
const std::vector<size_t>& value_sizes,
private:
std::vector<size_t> str_sizes;
- Values values;
+ Values<ValueItem> values;
kv_vector_t kvs;
kv_vector_t random_kvs;
};
-template <bool TRACK>
+template <bool TRACK, typename ValueItem>
class TreeBuilder {
public:
- using ertr = TestBtree::btree_ertr;
+ using BtreeImpl = Btree<typename ValueItem::ValueType>;
+ using BtreeCursor = typename BtreeImpl::Cursor;
+ using ertr = typename BtreeImpl::btree_ertr;
template <class ValueT=void>
- using future = ertr::future<ValueT>;
+ using future = typename ertr::template future<ValueT>;
- TreeBuilder(KVPool& kvs, NodeExtentManagerURef&& nm)
+ TreeBuilder(KVPool<ValueItem>& kvs, NodeExtentManagerURef&& nm)
: kvs{kvs} {
tree.emplace(std::move(nm));
}
future<> insert(Transaction& t) {
kv_iter = kvs.random_begin();
- auto cursors = seastar::make_lw_shared<std::vector<TestBtree::Cursor>>();
+ auto cursors = seastar::make_lw_shared<std::vector<BtreeCursor>>();
logger().warn("start inserting {} kvs ...", kvs.size());
auto start_time = mono_clock::now();
return crimson::do_until([&t, this, cursors]() -> future<bool> {
if (kv_iter.is_end()) {
- return ertr::make_ready_future<bool>(true);
+ return ertr::template make_ready_future<bool>(true);
}
auto [key, value] = kv_iter.get_kv();
logger().debug("[{}] {} -> {}", kv_iter.index(), key_hobj_t{key}, value);
- return tree->insert(t, key, value.get_config()
- ).safe_then([&t, this, cursors, value](auto ret) {
+ return tree->insert(
+ t, key, {value.get_payload_size()}
+ ).safe_then([&t, this, cursors, key, value](auto ret) {
auto& [cursor, success] = ret;
- assert(success == true);
- Values::initialize_cursor(t, cursor, value);
+ initialize_cursor_from_item(t, key, value, cursor, success);
if constexpr (TRACK) {
cursors->emplace_back(cursor);
}
#ifndef NDEBUG
auto [key, value] = kv_iter.get_kv();
- Values::validate_cursor(cursor, key, value);
+ validate_cursor_from_item(key, value, cursor);
return tree->find(t, key
).safe_then([this, cursor](auto cursor_) mutable {
assert(!cursor_.is_end());
auto [key, value] = kv_iter.get_kv();
ceph_assert(cursor_.get_ghobj() == key);
ceph_assert(cursor_.value() == cursor.value());
- Values::validate_cursor(cursor_, key, value);
+ validate_cursor_from_item(key, value, cursor_);
++kv_iter;
- return ertr::make_ready_future<bool>(false);
+ return ertr::template make_ready_future<bool>(false);
});
#else
++kv_iter;
- return ertr::make_ready_future<bool>(false);
+ return ertr::template make_ready_future<bool>(false);
#endif
});
}).safe_then([&t, this, start_time, cursors] {
return crimson::do_until([&t, this, &c_iter, cursors]() -> future<bool> {
if (kv_iter.is_end()) {
logger().info("Verify done!");
- return ertr::make_ready_future<bool>(true);
+ return ertr::template make_ready_future<bool>(true);
}
assert(c_iter != cursors->end());
auto [k, v] = kv_iter.get_kv();
// validate values in tree keep intact
return tree->find(t, k).safe_then([this, &c_iter](auto cursor) {
auto [k, v] = kv_iter.get_kv();
- Values::validate_cursor(cursor, k, v);
+ validate_cursor_from_item(k, v, cursor);
// validate values in cursors keep intact
- Values::validate_cursor(*c_iter, k, v);
+ validate_cursor_from_item(k, v, *c_iter);
++kv_iter;
++c_iter;
- return ertr::make_ready_future<bool>(false);
+ return ertr::template make_ready_future<bool>(false);
});
});
});
while (!iter.is_end()) {
auto [k, v] = iter.get_kv();
auto cursor = tree->find(t, k).unsafe_get0();
- Values::validate_cursor(cursor, k, v);
+ validate_cursor_from_item(k, v, cursor);
++iter;
}
while (!iter.is_end()) {
assert(!cursor.is_end());
auto [k, v] = iter.get_kv();
- Values::validate_cursor(cursor, k, v);
+ validate_cursor_from_item(k, v, cursor);
cursor = cursor.get_next(t).unsafe_get0();
++iter;
}
return crimson::get_logger(ceph_subsys_filestore);
}
- KVPool& kvs;
- std::optional<TestBtree> tree;
- KVPool::iterator_t kv_iter;
+ KVPool<ValueItem>& kvs;
+ std::optional<BtreeImpl> tree;
+ typename KVPool<ValueItem>::iterator_t kv_iter;
};
}
});
}
+using TestBtree = Btree<TestValue>;
+
struct b_dummy_tree_test_t : public seastar_test_suite_t {
NodeExtentManagerURef moved_nm;
TransactionRef ref_t;
ASSERT_TRUE(tree.last(t).unsafe_get0().is_end());
std::vector<std::tuple<ghobject_t,
- value_item_t,
+ test_item_t,
TestBtree::Cursor>> insert_history;
auto f_validate_insert_new = [this, &insert_history] (
- const ghobject_t& key, const value_item_t& value) {
+ const ghobject_t& key, const test_item_t& value) {
auto [cursor, success] = tree.insert(
- t, key, value.get_config()).unsafe_get0();
- ceph_assert(success);
- ceph_assert(cursor.get_ghobj() == key);
- Values::initialize_cursor(t, cursor, value);
+ t, key, {value.get_payload_size()}).unsafe_get0();
+ initialize_cursor_from_item(t, key, value, cursor, success);
insert_history.emplace_back(key, value, cursor);
auto cursor_ = tree.find(t, key).unsafe_get0();
ceph_assert(cursor_ != tree.end());
ceph_assert(cursor_.value() == cursor.value());
- Values::validate_cursor(cursor_, key, value);
+ validate_cursor_from_item(key, value, cursor_);
return cursor.value();
};
- auto values = Values(15);
+ auto values = Values<test_item_t>(15);
// insert key1, value1 at STAGE_LEFT
auto key1 = make_ghobj(3, 3, 3, "ns3", "oid3", 3, 3);
{
auto value1_dup = values.pick();
auto [cursor1_dup, ret1_dup] = tree.insert(
- t, key1, value1_dup.get_config()).unsafe_get0();
+ t, key1, {value1_dup.get_payload_size()}).unsafe_get0();
ASSERT_FALSE(ret1_dup);
- Values::validate_cursor(cursor1_dup, key1, value1);
+ validate_cursor_from_item(key1, value1, cursor1_dup);
}
// insert key2, value2 to key1's left at STAGE_LEFT
f_validate_insert_new(key11, value11);
// insert key, value randomly until a perfect 3-ary tree is formed
- std::vector<std::pair<ghobject_t, value_item_t>> kvs{
+ std::vector<std::pair<ghobject_t, test_item_t>> kvs{
{make_ghobj(2, 2, 2, "ns2", "oid2", 2, 2), values.pick()},
{make_ghobj(2, 2, 2, "ns2", "oid2", 4, 4), values.pick()},
{make_ghobj(2, 2, 2, "ns3", "oid3", 4, 4), values.pick()},
// validate values in tree keep intact
auto cursor = tree.find(t, k).unsafe_get0();
EXPECT_NE(cursor, tree.end());
- Values::validate_cursor(cursor, k, v);
+ validate_cursor_from_item(k, v, cursor);
// validate values in cursors keep intact
- Values::validate_cursor(c, k, v);
+ validate_cursor_from_item(k, v, c);
}
{
auto cursor = tree.lower_bound(t, key_s).unsafe_get0();
- Values::validate_cursor(cursor, smallest_key, smallest_value);
+ validate_cursor_from_item(smallest_key, smallest_value, cursor);
}
{
auto cursor = tree.begin(t).unsafe_get0();
- Values::validate_cursor(cursor, smallest_key, smallest_value);
+ validate_cursor_from_item(smallest_key, smallest_value, cursor);
}
{
auto cursor = tree.last(t).unsafe_get0();
- Values::validate_cursor(cursor, largest_key, largest_value);
+ validate_cursor_from_item(largest_key, largest_value, cursor);
}
// validate range query
auto cursor = tree.begin(t).unsafe_get0();
for (auto& [k, v] : kvs) {
ASSERT_FALSE(cursor.is_end());
- Values::validate_cursor(cursor, k, v);
+ validate_cursor_from_item(k, v, cursor);
cursor = cursor.get_next(t).unsafe_get0();
}
ASSERT_TRUE(cursor.is_end());
}
seastar::future<> build_tree(
- const std::vector<ghobject_t>& keys, const std::vector<value_item_t>& values) {
+ const std::vector<ghobject_t>& keys, const std::vector<test_item_t>& values) {
return seastar::async([this, keys, values] {
tree.mkfs(t).unsafe_get0();
//logger().info("\n---------------------------------------------"
});
}
- seastar::future<> split(const ghobject_t& key, const value_item_t& value,
+ seastar::future<> split(const ghobject_t& key, const test_item_t& value,
const split_expectation_t& expected) {
return seastar::async([this, key, value, expected] {
TestBtree tree_clone(NodeExtentManager::create_dummy(IS_DUMMY_SYNC));
logger().info("insert {}:", key_hobj_t(key));
auto [cursor, success] = tree_clone.insert(
- t_clone, key, value.get_config()).unsafe_get0();
- ASSERT_TRUE(success);
- ASSERT_EQ(cursor.get_ghobj(), key);
- Values::initialize_cursor(t_clone, cursor, value);
+ t_clone, key, {value.get_payload_size()}).unsafe_get0();
+ initialize_cursor_from_item(t, key, value, cursor, success);
std::ostringstream oss;
tree_clone.dump(t_clone, oss);
for (auto& [k, v, c] : insert_history) {
auto result = tree_clone.find(t_clone, k).unsafe_get0();
EXPECT_NE(result, tree_clone.end());
- Values::validate_cursor(result, k, v);
+ validate_cursor_from_item(k, v, result);
}
auto result = tree_clone.find(t_clone, key).unsafe_get0();
EXPECT_NE(result, tree_clone.end());
- Values::validate_cursor(result, key, value);
+ validate_cursor_from_item(key, value, result);
EXPECT_TRUE(last_split.match(expected));
});
}
- value_item_t create_value(size_t size) {
+ test_item_t create_value(size_t size) {
return values.create(size);
}
private:
- seastar::future<> insert_tree(const ghobject_t& key, const value_item_t& value) {
+ seastar::future<> insert_tree(const ghobject_t& key, const test_item_t& value) {
return seastar::async([this, &key, &value] {
auto [cursor, success] = tree.insert(
- t, key, value.get_config()).unsafe_get0();
- ASSERT_TRUE(success);
- ASSERT_EQ(cursor.get_ghobj(), key);
- Values::initialize_cursor(t, cursor, value);
+ t, key, {value.get_payload_size()}).unsafe_get0();
+ initialize_cursor_from_item(t, key, value, cursor, success);
insert_history.emplace_back(key, value, cursor);
});
}
ValueBuilderImpl<TestValue> vb;
context_t c;
TestBtree tree;
- Values values;
+ Values<test_item_t> values;
std::vector<std::tuple<
- ghobject_t, value_item_t, TestBtree::Cursor>> insert_history;
+ ghobject_t, test_item_t, TestBtree::Cursor>> insert_history;
};
struct c_dummy_test_t : public seastar_test_suite_t {};
std::vector<ghobject_t> keys = {
make_ghobj(2, 2, 2, "ns3", "oid3", 3, 3),
make_ghobj(3, 3, 3, "ns3", "oid3", 3, 3)};
- std::vector<value_item_t> values = {
+ std::vector<test_item_t> values = {
test.create_value(1360),
test.create_value(1632)};
test.build_tree(keys, values).get0();
run_async([this] {
constexpr bool TEST_SEASTORE = true;
constexpr bool TRACK_CURSORS = true;
- KVPool kvs{{8, 11, 64, 256, 301, 320},
- {8, 16, 128, 512, 576, 640},
- {0, 32}, {0, 10}, {0, 4}};
- auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS>>(kvs,
+ KVPool<test_item_t> kvs{{8, 11, 64, 256, 301, 320},
+ {8, 16, 128, 512, 576, 640},
+ {0, 32}, {0, 10}, {0, 4}};
+ auto tree = std::make_unique<TreeBuilder<TRACK_CURSORS, test_item_t>>(kvs,
(TEST_SEASTORE ? NodeExtentManager::create_seastore(*tm)
: NodeExtentManager::create_dummy(IS_DUMMY_SYNC)));
{
}
};
+struct test_item_t {
+ using ValueType = TestValue;
+
+ value_size_t size;
+ TestValue::id_t id;
+ TestValue::magic_t magic;
+
+ value_size_t get_payload_size() const {
+ assert(size > sizeof(value_header_t));
+ return {static_cast<value_size_t>(size - sizeof(value_header_t))};
+ }
+
+ void initialize(Transaction& t, TestValue& value) const {
+ ceph_assert(value.get_payload_size() + sizeof(value_header_t) == size);
+ value.set_id_replayable(t, id);
+ value.set_tail_magic_replayable(t, magic);
+ }
+
+ void validate(TestValue& value) const {
+ ceph_assert(value.get_payload_size() + sizeof(value_header_t) == size);
+ ceph_assert(value.get_id() == id);
+ ceph_assert(value.get_tail_magic() == magic);
+ }
+
+ static test_item_t create(std::size_t _size, std::size_t _id) {
+ ceph_assert(_size <= std::numeric_limits<value_size_t>::max());
+ ceph_assert(_size > sizeof(value_header_t));
+ value_size_t size = _size;
+
+ ceph_assert(_id <= std::numeric_limits<TestValue::id_t>::max());
+ TestValue::id_t id = _id;
+
+ return {size, id, (TestValue::magic_t)id * 137};
+ }
+};
+inline std::ostream& operator<<(std::ostream& os, const test_item_t& item) {
+ return os << "TestItem(#" << item.id << ", " << item.size << "B)";
+}
+
}
#include "crimson/common/log.h"
#include "crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h"
#include "crimson/os/seastore/onode_manager/staged-fltree/node_extent_manager.h"
+
+#include "test/crimson/seastore/onode_tree/test_value.h"
#include "test/crimson/seastore/transaction_manager_test_state.h"
using namespace crimson::os::seastore::onode;
public:
PerfTree(bool is_dummy) : is_dummy{is_dummy} {}
- seastar::future<> run(KVPool& kvs) {
+ seastar::future<> run(KVPool<test_item_t>& kvs) {
return tm_setup().then([this, &kvs] {
return seastar::async([this, &kvs] {
- auto tree = std::make_unique<TreeBuilder<TRACK>>(kvs,
+ auto tree = std::make_unique<TreeBuilder<TRACK, test_item_t>>(kvs,
(is_dummy ? NodeExtentManager::create_dummy(true)
: NodeExtentManager::create_seastore(*tm)));
{
auto range0 = config["range0"].as<std::vector<unsigned>>();
ceph_assert(range0.size() == 2);
- KVPool kvs{str_sizes, onode_sizes,
- {range2[0], range2[1]},
- {range1[0], range1[1]},
- {range0[0], range0[1]}};
+ KVPool<test_item_t> kvs{str_sizes, onode_sizes,
+ {range2[0], range2[1]},
+ {range1[0], range1[1]},
+ {range0[0], range0[1]}};
PerfTree<TRACK> perf{is_dummy};
perf.run(kvs).get0();
});
projects / ceph.git / commitdiff