};
using kv_vector_t = std::vector<kv_t>;
using kvptr_vector_t = std::vector<kv_t*>;
+ using iterator_t = typename kvptr_vector_t::iterator;
- class iterator_t {
- public:
- iterator_t() = default;
- iterator_t(const iterator_t&) = default;
- iterator_t(iterator_t&&) = default;
- iterator_t& operator=(const iterator_t&) = default;
- iterator_t& operator=(iterator_t&&) = default;
-
- kv_t* get_p_kv() const {
- assert(!is_end());
- return (*p_kvptrs)[i];
- }
- bool is_end() const { return i >= p_kvptrs->size(); }
- index_t index() const { return i; }
-
- iterator_t& operator++() {
- assert(!is_end());
- ++i;
- return *this;
- }
-
- iterator_t operator++(int) {
- iterator_t tmp = *this;
- ++*this;
- return tmp;
- }
-
- private:
- iterator_t(const kvptr_vector_t& kvs) : p_kvptrs{&kvs} {}
-
- const kvptr_vector_t* p_kvptrs;
- index_t i = 0;
- friend class KVPool;
- };
-
- iterator_t begin() const {
- return iterator_t(serial_p_kvs);
+ size_t size() const {
+ return kvs.size();
}
- iterator_t random_begin() const {
- return iterator_t(random_p_kvs);
+ iterator_t begin() {
+ return serial_p_kvs.begin();
+ }
+ iterator_t end() {
+ return serial_p_kvs.end();
+ }
+ iterator_t random_begin() {
+ return random_p_kvs.begin();
+ }
+ iterator_t random_end() {
+ return random_p_kvs.end();
}
- size_t size() const {
- return kvs.size();
+ void shuffle() {
+ std::random_shuffle(random_p_kvs.begin(), random_p_kvs.end());
}
static KVPool create_raw_range(
return KVPool(std::move(kvs));
}
+ static KVPool create_range(
+ const std::pair<index_t, index_t>& range_i,
+ const std::vector<size_t>& value_sizes) {
+ kv_vector_t kvs;
+ std::random_device rd;
+ for (index_t i = range_i.first; i < range_i.second; ++i) {
+ auto value_size = value_sizes[rd() % value_sizes.size()];
+ kvs.emplace_back(
+ kv_t{make_oid(i), ValueItem::create(value_size, i)}
+ );
+ }
+ return KVPool(std::move(kvs));
+ }
+
private:
KVPool(kv_vector_t&& _kvs)
: kvs(std::move(_kvs)), serial_p_kvs(kvs.size()), random_p_kvs(kvs.size()) {
[] (kv_t& item) { return &item; });
std::transform(kvs.begin(), kvs.end(), random_p_kvs.begin(),
[] (kv_t& item) { return &item; });
- std::random_shuffle(random_p_kvs.begin(), random_p_kvs.end());
- std::cout << "size: " << kvs.size()
- << ", " << serial_p_kvs.size()
- << ", " << random_p_kvs.size() << std::endl;
+ shuffle();
}
static ghobject_t make_raw_oid(
os_ns.str(), os_oid.str(), index0, index0);
}
+ static ghobject_t make_oid(index_t i) {
+ std::stringstream ss;
+ ss << "object_" << i;
+ auto ret = ghobject_t(
+ hobject_t(
+ sobject_t(ss.str(), CEPH_NOSNAP)));
+ ret.hobj.nspace = "asdf";
+ return ret;
+ }
+
kv_vector_t kvs;
kvptr_vector_t serial_p_kvs;
kvptr_vector_t random_p_kvs;
logger().warn("start inserting {} kvs ...", kvs.size());
auto start_time = mono_clock::now();
return crimson::do_until([&t, this, cursors]() -> future<bool> {
- if (kv_iter.is_end()) {
+ if (kv_iter == kvs.random_end()) {
return ertr::template make_ready_future<bool>(true);
}
- auto p_kv = kv_iter.get_p_kv();
+ auto p_kv = *kv_iter;
logger().debug("[{}] {} -> {}",
- kv_iter.index(), key_hobj_t{p_kv->key}, p_kv->value);
+ kv_iter - kvs.random_begin(),
+ key_hobj_t{p_kv->key},
+ p_kv->value);
return tree->insert(
t, p_kv->key, {p_kv->value.get_payload_size()}
).safe_then([&t, this, cursors](auto ret) {
- auto p_kv = kv_iter.get_p_kv();
+ auto p_kv = *kv_iter;
auto& [cursor, success] = ret;
initialize_cursor_from_item(t, p_kv->key, p_kv->value, cursor, success);
if constexpr (TRACK) {
return tree->find(t, p_kv->key
).safe_then([this, cursor](auto cursor_) mutable {
assert(!cursor_.is_end());
- auto p_kv = kv_iter.get_p_kv();
+ auto p_kv = *kv_iter;
ceph_assert(cursor_.get_ghobj() == p_kv->key);
ceph_assert(cursor_.value() == cursor.value());
validate_cursor_from_item(p_kv->key, p_kv->value, cursor_);
return seastar::do_with(
cursors->begin(), [&t, this, cursors](auto& c_iter) {
return crimson::do_until([&t, this, &c_iter, cursors]() -> future<bool> {
- if (kv_iter.is_end()) {
+ if (kv_iter == kvs.random_end()) {
logger().info("Verify done!");
return ertr::template make_ready_future<bool>(true);
}
assert(c_iter != cursors->end());
- auto p_kv = kv_iter.get_p_kv();
+ auto p_kv = *kv_iter;
// validate values in tree keep intact
return tree->find(t, p_kv->key).safe_then([this, &c_iter](auto cursor) {
- auto p_kv = kv_iter.get_p_kv();
+ auto p_kv = *kv_iter;
validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
// validate values in cursors keep intact
validate_cursor_from_item(p_kv->key, p_kv->value, *c_iter);
future<> validate(Transaction& t) {
return seastar::async([this, &t] {
logger().info("Verifing insertion ...");
- auto iter = kvs.begin();
- while (!iter.is_end()) {
- auto p_kv = iter.get_p_kv();
+ for (auto& p_kv : kvs) {
auto cursor = tree->find(t, p_kv->key).unsafe_get0();
validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
- ++iter;
}
logger().info("Verifing range query ...");
- iter = kvs.begin();
auto cursor = tree->begin(t).unsafe_get0();
- while (!iter.is_end()) {
+ for (auto& p_kv : kvs) {
assert(!cursor.is_end());
- auto p_kv = iter.get_p_kv();
validate_cursor_from_item(p_kv->key, p_kv->value, cursor);
cursor = cursor.get_next(t).unsafe_get0();
- ++iter;
}
assert(cursor.is_end());
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:nil -*-
// vim: ts=8 sw=2 smarttab
-#include <string>
-#include <iostream>
-#include <sstream>
+#include <boost/range/combine.hpp>
#include "test/crimson/gtest_seastar.h"
#include "test/crimson/seastore/transaction_manager_test_state.h"
-#include "crimson/os/futurized_collection.h"
#include "crimson/os/seastore/onode_manager/staged-fltree/fltree_onode_manager.h"
+#include "crimson/os/seastore/onode_manager/staged-fltree/tree_utils.h"
using namespace crimson;
using namespace crimson::os;
}
}
-ghobject_t make_oid(unsigned i) {
- stringstream ss;
- ss << "object_" << i;
- auto ret = ghobject_t(
- hobject_t(
- sobject_t(ss.str(), CEPH_NOSNAP)));
- ret.hobj.nspace = "asdf";
- return ret;
-}
+struct onode_item_t {
+ uint32_t size;
+ uint64_t id;
+ uint32_t cnt_modify = 0;
+
+ void initialize(Transaction& t, Onode& value) const {
+ auto& layout = value.get_mutable_layout(t);
+ layout.size = size;
+ layout.omap_root.update(omap_root_t(id, cnt_modify));
+ validate(value);
+ }
+
+ void validate(Onode& value) const {
+ auto& layout = value.get_layout();
+ ceph_assert(laddr_t(layout.size) == laddr_t{size});
+ ceph_assert(layout.omap_root.get().addr == id);
+ ceph_assert(layout.omap_root.get().depth == cnt_modify);
+ }
+
+ void modify(Transaction& t, Onode& value) {
+ validate(value);
+ ++cnt_modify;
+ initialize(t, value);
+ }
+
+ static onode_item_t create(std::size_t size, std::size_t id) {
+ ceph_assert(size <= std::numeric_limits<uint32_t>::max());
+ return {(uint32_t)size, id};
+ }
+};
+
+struct fltree_onode_manager_test_t
+ : public seastar_test_suite_t, TMTestState {
+ using iterator_t = typename KVPool<onode_item_t>::iterator_t;
-struct fltree_onode_manager_test_t :
- public seastar_test_suite_t,
- TMTestState {
FLTreeOnodeManagerRef manager;
seastar::future<> set_up_fut() final {
return TMTestState::_mkfs(
).then([this] {
return seastar::do_with(
- tm->create_transaction(),
- [this](auto &t) {
- return manager->mkfs(*t
- ).safe_then([this, &t] {
- return tm->submit_transaction(std::move(t));
- }).handle_error(
- crimson::ct_error::assert_all{
- "Invalid error in _mkfs"
- }
- );
- });
+ tm->create_transaction(),
+ [this](auto &t) {
+ return manager->mkfs(*t
+ ).safe_then([this, &t] {
+ return tm->submit_transaction(std::move(t));
+ }).handle_error(
+ crimson::ct_error::assert_all{
+ "Invalid error in _mkfs"
+ }
+ );
+ });
});
}
template <typename F>
- void with_onodes_write(
- std::vector<ghobject_t> oids,
- F &&f) {
+ void with_transaction(F&& f) {
auto t = tm->create_transaction();
- auto onodes = manager->get_or_create_onodes(
- *t,
- oids).unsafe_get0();
- std::invoke(f, *t, onodes);
- manager->write_dirty(
- *t,
- onodes
- ).unsafe_get0();
+ std::invoke(f, *t);
tm->submit_transaction(std::move(t)).unsafe_get0();
}
template <typename F>
- void with_onodes_write_range(
- unsigned start,
- unsigned end,
- unsigned stride,
- F &&f) {
+ void with_onode_write(iterator_t& it, F&& f) {
+ with_transaction([this, &it, f=std::move(f)] (auto& t) {
+ auto p_kv = *it;
+ auto onode = manager->get_or_create_onode(
+ t, p_kv->key).unsafe_get0();
+ std::invoke(f, t, *onode, p_kv->value);
+ manager->write_dirty(t, {onode}).unsafe_get0();
+ });
+ }
+
+ void validate_onode(iterator_t& it) {
+ with_transaction([this, &it] (auto& t) {
+ auto p_kv = *it;
+ auto onode = manager->get_onode(
+ t, p_kv->key).unsafe_get0();
+ p_kv->value.validate(*onode);
+ });
+ }
+
+ template <typename F>
+ void with_onodes_process(
+ iterator_t& start, iterator_t& end, F&& f) {
std::vector<ghobject_t> oids;
- for (unsigned i = start; i < end; i += stride) {
- oids.emplace_back(make_oid(i));
+ std::vector<onode_item_t*> items;
+ auto it = start;
+ while(it != end) {
+ auto p_kv = *it;
+ oids.emplace_back(p_kv->key);
+ items.emplace_back(&p_kv->value);
+ ++it;
}
- with_onodes_write(std::move(oids), std::forward<F>(f));
+ with_transaction([this, &oids, &items, f=std::move(f)] (auto& t) mutable {
+ std::invoke(f, t, oids, items);
+ });
}
template <typename F>
- void with_onode_write(unsigned i, F &&f) {
- with_onodes_write_range(
- i,
- i + 1,
- 1, [this, f=std::forward<F>(f)](auto &t, auto &onodes) {
- assert(onodes.size() == 1);
- std::invoke(f, t, *onodes[0]);
- });
+ void with_onodes_write(
+ iterator_t& start, iterator_t& end, F&& f) {
+ with_onodes_process(start, end,
+ [this, f=std::move(f)] (auto& t, auto& oids, auto& items) {
+ auto onodes = manager->get_or_create_onodes(
+ t, oids).unsafe_get0();
+ for (auto tup : boost::combine(onodes, items)) {
+ OnodeRef onode;
+ onode_item_t* p_item;
+ boost::tie(onode, p_item) = tup;
+ std::invoke(f, t, *onode, *p_item);
+ }
+ manager->write_dirty(t, onodes).unsafe_get0();
+ });
+ }
+
+ void validate_onodes(
+ iterator_t& start, iterator_t& end) {
+ with_onodes_process(start, end,
+ [this] (auto& t, auto& oids, auto& items) {
+ for (auto tup : boost::combine(oids, items)) {
+ ghobject_t oid;
+ onode_item_t* p_item;
+ boost::tie(oid, p_item) = tup;
+ auto onode = manager->get_onode(t, oid).unsafe_get0();
+ p_item->validate(*onode);
+ }
+ });
}
fltree_onode_manager_test_t() {}
};
-TEST_F(fltree_onode_manager_test_t, touch)
+TEST_F(fltree_onode_manager_test_t, 1_single)
{
run_async([this] {
- with_onode_write(0, [](auto &t, auto &onode) {
- onode.get_mutable_layout(t);
+ auto pool = KVPool<onode_item_t>::create_range({0, 1}, {128, 256});
+ auto iter = pool.begin();
+ with_onode_write(iter, [](auto& t, auto& onode, auto& item) {
+ item.initialize(t, onode);
+ });
+ validate_onode(iter);
+
+ with_onode_write(iter, [](auto& t, auto& onode, auto& item) {
+ item.modify(t, onode);
+ });
+ validate_onode(iter);
+ });
+}
+
+TEST_F(fltree_onode_manager_test_t, 2_synthetic)
+{
+ run_async([this] {
+ auto pool = KVPool<onode_item_t>::create_range(
+ {0, 100}, {32, 64, 128, 256, 512});
+ auto start = pool.begin();
+ auto end = pool.end();
+ with_onodes_write(start, end,
+ [](auto& t, auto& onode, auto& item) {
+ item.initialize(t, onode);
+ });
+ validate_onodes(start, end);
+
+ auto rd_start = pool.random_begin();
+ auto rd_end = rd_start + 50;
+ with_onodes_write(rd_start, rd_end,
+ [](auto& t, auto& onode, auto& item) {
+ item.modify(t, onode);
});
- /* Disabled pending fix
- with_onode_write(0, [](auto &t, auto &onode) {
- onode.get_mutable_layout(t);
+ validate_onodes(start, end);
+
+ pool.shuffle();
+ rd_start = pool.random_begin();
+ rd_end = rd_start + 50;
+ with_onodes_write(rd_start, rd_end,
+ [](auto& t, auto& onode, auto& item) {
+ item.modify(t, onode);
});
- */
+ validate_onodes(start, end);
});
}
projects / ceph.git / commitdiff