);
}
+node_future<tree_stats_t> Node::get_tree_stats(context_t c) {
+ return seastar::do_with(
+ tree_stats_t(), [this, c](auto& stats) {
+ return do_get_tree_stats(c, stats).safe_then([&stats] {
+ return stats;
+ });
+ }
+ );
+}
+
std::ostream& Node::dump(std::ostream& os) const {
return impl->dump(os);
}
});
}
+node_future<> InternalNode::do_get_tree_stats(
+ context_t c, tree_stats_t& stats) {
+ auto nstats = impl->get_stats();
+ stats.size_persistent_internal += nstats.size_persistent;
+ stats.size_filled_internal += nstats.size_filled;
+ stats.size_logical_internal += nstats.size_logical;
+ stats.size_overhead_internal += nstats.size_overhead;
+ stats.size_value_internal += nstats.size_value;
+ stats.num_kvs_internal += nstats.num_kvs;
+ stats.num_nodes_internal += 1;
+
+ Ref<const InternalNode> this_ref = this;
+ return seastar::do_with(
+ search_position_t(), [this, this_ref, c, &stats](auto& pos) {
+ pos = search_position_t::begin();
+ return crimson::do_until(
+ [this, this_ref, c, &stats, &pos]() -> node_future<bool> {
+ auto child_addr = impl->get_p_value(pos)->value;
+ return get_or_track_child(c, pos, child_addr
+ ).safe_then([c, &stats](auto child) {
+ return child->do_get_tree_stats(c, stats);
+ }).safe_then([this, this_ref, &pos] {
+ if (pos.is_end()) {
+ return node_ertr::make_ready_future<bool>(true);
+ } else {
+ impl->next_position(pos);
+ if (pos.is_end()) {
+ if (impl->is_level_tail()) {
+ return node_ertr::make_ready_future<bool>(false);
+ } else {
+ return node_ertr::make_ready_future<bool>(true);
+ }
+ } else {
+ return node_ertr::make_ready_future<bool>(false);
+ }
+ }
+ });
+ });
+ }
+ );
+}
+
node_future<> InternalNode::test_clone_root(
context_t c_other, RootNodeTracker& tracker_other) const {
assert(is_root());
search_result_t{cursor, result.mstat});
}
+node_future<> LeafNode::do_get_tree_stats(context_t, tree_stats_t& stats) {
+ auto nstats = impl->get_stats();
+ stats.size_persistent_leaf += nstats.size_persistent;
+ stats.size_filled_leaf += nstats.size_filled;
+ stats.size_logical_leaf += nstats.size_logical;
+ stats.size_overhead_leaf += nstats.size_overhead;
+ stats.size_value_leaf += nstats.size_value;
+ stats.num_kvs_leaf += nstats.num_kvs;
+ stats.num_nodes_leaf += 1;
+ return node_ertr::now();
+}
+
node_future<> LeafNode::test_clone_root(
context_t c_other, RootNodeTracker& tracker_other) const {
assert(is_root());
node_future<search_result_t> lower_bound(context_t c, const key_hobj_t& key);
node_future<std::pair<Ref<tree_cursor_t>, bool>> insert(
context_t, const key_hobj_t&, const onode_t&);
+ node_future<tree_stats_t> get_tree_stats(context_t);
std::ostream& dump(std::ostream&) const;
std::ostream& dump_brief(std::ostream&) const;
}
virtual node_future<search_result_t> lower_bound_tracked(
context_t, const key_hobj_t&, MatchHistory&) = 0;
+ virtual node_future<> do_get_tree_stats(context_t, tree_stats_t&) = 0;
protected:
Node(NodeImplURef&&);
node_future<Ref<tree_cursor_t>> lookup_largest(context_t) override;
node_future<search_result_t> lower_bound_tracked(
context_t, const key_hobj_t&, MatchHistory&) override;
+ node_future<> do_get_tree_stats(context_t, tree_stats_t&) override;
node_future<> test_clone_root(context_t, RootNodeTracker&) const override;
node_future<Ref<tree_cursor_t>> lookup_largest(context_t) override;
node_future<search_result_t> lower_bound_tracked(
context_t, const key_hobj_t&, MatchHistory&) override;
+ node_future<> do_get_tree_stats(context_t, tree_stats_t&) override;
node_future<> test_clone_root(context_t, RootNodeTracker&) const override;
virtual node_offset_t free_size() const = 0;
virtual key_view_t get_key_view(const search_position_t&) const = 0;
virtual key_view_t get_largest_key_view() const = 0;
+ virtual void next_position(search_position_t&) const = 0;
+ virtual node_stats_t get_stats() const = 0;
virtual std::ostream& dump(std::ostream&) const = 0;
virtual std::ostream& dump_brief(std::ostream&) const = 0;
virtual void validate_layout() const = 0;
return index_key;
}
+ void next_position(search_position_t& pos) const override {
+ assert(!pos.is_end());
+ bool find_next = STAGE_T::next_position(extent.read(), cast_down<STAGE>(pos));
+ if (find_next) {
+ pos = search_position_t::end();
+ }
+ }
+
+ node_stats_t get_stats() const override {
+ node_stats_t stats;
+ auto& node_stage = extent.read();
+ key_view_t index_key;
+ if (node_stage.keys()) {
+ STAGE_T::get_stats(node_stage, stats, index_key);
+ }
+ stats.size_persistent = node_stage_t::EXTENT_SIZE;
+ stats.size_filled = filled_size();
+ if constexpr (NODE_TYPE == node_type_t::INTERNAL) {
+ if (is_level_tail()) {
+ stats.size_logical += sizeof(value_t);
+ stats.size_value += sizeof(value_t);
+ stats.num_kvs += 1;
+ }
+ }
+ return stats;
+ }
+
std::ostream& dump(std::ostream& os) const override {
auto& node_stage = extent.read();
auto p_start = node_stage.p_start();
dump_brief(os);
+ auto stats = get_stats();
+ os << " num_kvs=" << stats.num_kvs
+ << ", logical=" << stats.size_logical
+ << "B, overhead=" << stats.size_overhead
+ << "B, value=" << stats.size_value << "B";
os << ":\n header: " << node_stage_t::header_size() << "B";
size_t size = 0u;
if (node_stage.keys()) {
assert(ret < NODE_BLOCK_SIZE);
return ret;
}
+ node_offset_t size_overhead() const {
+ return sizeof(node_offset_t) + get_key().size_overhead();
+ }
memory_range_t get_nxt_container() const {
return {item_range.p_start, get_key().p_start()};
}
assert(ret < NODE_BLOCK_SIZE);
return ret;
}
+ node_offset_t size_logical() const {
+ assert(type() == Type::STR);
+ return length;
+ }
+ node_offset_t size_overhead() const {
+ assert(type() == Type::STR);
+ return sizeof(string_size_t);
+ }
+
std::string_view to_string_view() const {
assert(type() == Type::STR);
return {p_key, length};
return sizeof(string_size_t);
}
}
+ node_offset_t size_logical() const {
+ assert(type() == Type::STR);
+ return nspace.size_logical() + oid.size_logical();
+ }
+ node_offset_t size_overhead() const {
+ assert(type() == Type::STR);
+ return nspace.size_overhead() + oid.size_overhead();
+ }
bool operator==(const ns_oid_view_t& x) const {
return (nspace == x.nspace && oid == x.oid);
}
return *p_snap_gen;
}
+ size_t size_logical() const {
+ return sizeof(shard_t) + sizeof(pool_t) + sizeof(crush_hash_t) +
+ sizeof(snap_t) + sizeof(gen_t) + ns_oid_view().size_logical();
+ }
+
ghobject_t to_ghobj() const {
ghobject_t ghobj;
ghobj.shard_id.id = shard();
return ghobj;
}
+ void replace(const crush_t& key) { p_crush = &key; }
void set(const crush_t& key) {
assert(!has_crush());
- p_crush = &key;
+ replace(key);
}
+ void replace(const shard_pool_crush_t& key) { p_shard_pool = &key.shard_pool; }
void set(const shard_pool_crush_t& key) {
set(key.crush);
assert(!has_shard_pool());
- p_shard_pool = &key.shard_pool;
+ replace(key);
}
+ void replace(const ns_oid_view_t& key) { p_ns_oid = key; }
void set(const ns_oid_view_t& key) {
assert(!has_ns_oid());
- p_ns_oid = key;
+ replace(key);
}
+ void replace(const snap_gen_t& key) { p_snap_gen = &key; }
void set(const snap_gen_t& key) {
assert(!has_snap_gen());
- p_snap_gen = &key;
+ replace(key);
}
std::ostream& dump(std::ostream& os) const {
return total_size() - free_size;
}
node_offset_t size_to_nxt_at(size_t index) const;
+ node_offset_t size_overhead_at(size_t index) const {
+ return FieldType::ITEM_OVERHEAD; }
memory_range_t get_nxt_container(size_t index) const;
template <typename T = FieldType>
struct _slot_t {
using key_t = FixedKeyType;
static constexpr field_type_t FIELD_TYPE = _FIELD_TYPE;
+ static constexpr node_offset_t OVERHEAD = sizeof(_slot_t) - sizeof(key_t);
key_t key;
node_offset_t right_offset;
static constexpr node_offset_t SIZE = NODE_BLOCK_SIZE;
static constexpr node_offset_t HEADER_SIZE =
sizeof(node_header_t) + sizeof(num_keys_t);
+ static constexpr node_offset_t ITEM_OVERHEAD = SlotType::OVERHEAD;
bool is_level_tail() const { return header.get_is_level_tail(); }
node_offset_t total_size() const { return SIZE; }
static constexpr node_offset_t SIZE = NODE_BLOCK_SIZE;
static constexpr node_offset_t HEADER_SIZE =
sizeof(node_header_t) + sizeof(num_keys_t);
+ static constexpr node_offset_t ITEM_OVERHEAD = sizeof(node_offset_t);
bool is_level_tail() const { return header.get_is_level_tail(); }
node_offset_t total_size() const { return SIZE; }
static constexpr node_offset_t SIZE = sizeof(me_t);
static constexpr node_offset_t HEADER_SIZE =
sizeof(node_header_t) + sizeof(num_keys_t);
+ static constexpr node_offset_t ITEM_OVERHEAD = 0u;
bool is_level_tail() const { return header.get_is_level_tail(); }
node_offset_t total_size() const {
* CONTAINER_TYPE = ContainerType::INDEXABLE
* keys() const -> size_t
* operator[](size_t) const -> key_get_type
- * size_before(size_t) const -> size_t
+ * size_before(size_t) const -> node_offset_t
+ * size_overhead_at(size_t) const -> node_offset_t
* (IS_BOTTOM) get_p_value(size_t) const -> const value_t*
* (!IS_BOTTOM) size_to_nxt_at(size_t) const -> node_offset_t
* (!IS_BOTTOM) get_nxt_container(size_t) const
return container.size_before(_index + 1) -
container.size_before(_index);
}
+ node_offset_t size_overhead() const {
+ assert(!is_end());
+ return container.size_overhead_at(_index);
+ }
me_t& operator++() {
assert(!is_end());
* get_key() const -> key_get_type
* size() const -> node_offset_t
* size_to_nxt() const -> node_offset_t
+ * size_overhead() const -> node_offset_t
* get_nxt_container() const
* has_next() const -> bool
* operator++()
assert(!is_end());
return container.size();
}
+ node_offset_t size_overhead() const {
+ assert(!is_end());
+ return container.size_overhead();
+ }
me_t& operator++() {
assert(!is_end());
* is_last() -> bool
* is_end() -> bool
* size() -> node_offset_t
+ * size_overhead() -> node_offset_t
* (IS_BOTTOM) get_p_value() -> const value_t*
* (!IS_BOTTOM) get_nxt_container() -> nxt_stage::container_t
* (!IS_BOTTOM) size_to_nxt() -> node_offset_t
} while (true);
}
+ static void get_stats(const container_t& container, node_stats_t& stats,
+ full_key_t<KeyT::VIEW>& index_key) {
+ auto iter = iterator_t(container);
+ assert(!iter.is_end());
+ stats.size_overhead += iterator_t::header_size();
+ do {
+ index_key.replace(iter.get_key());
+ stats.size_overhead += iter.size_overhead();
+ if constexpr (!IS_BOTTOM) {
+ auto nxt_container = iter.get_nxt_container();
+ NXT_STAGE_T::get_stats(nxt_container, stats, index_key);
+ } else {
+ ++stats.num_kvs;
+ size_t kv_logical_size = index_key.size_logical();
+ size_t value_size;
+ if constexpr (NODE_TYPE == node_type_t::LEAF) {
+ value_size = iter.get_p_value()->size;
+ } else {
+ value_size = sizeof(value_t);
+ }
+ stats.size_value += value_size;
+ kv_logical_size += value_size;
+ stats.size_logical += kv_logical_size;
+ }
+ if (iter.is_last()) {
+ break;
+ } else {
+ ++iter;
+ }
+ } while (true);
+ }
+
+ static bool next_position(const container_t& container, position_t& pos) {
+ auto iter = iterator_t(container);
+ assert(!iter.is_end());
+ iter.seek_at(pos.index);
+ bool find_next;
+ if constexpr (!IS_BOTTOM) {
+ auto nxt_container = iter.get_nxt_container();
+ find_next = NXT_STAGE_T::next_position(nxt_container, pos.nxt);
+ } else {
+ find_next = true;
+ }
+ if (find_next) {
+ if (iter.is_last()) {
+ return true;
+ } else {
+ pos.index = iter.index() + 1;
+ if constexpr (!IS_BOTTOM) {
+ pos.nxt = NXT_STAGE_T::position_t::begin();
+ }
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
struct _BaseEmpty {};
class _BaseWithNxtIterator {
protected:
return {normalize(std::move(result.position)), result.p_value, result.mstat};
}
+struct node_stats_t {
+ size_t size_persistent = 0;
+ size_t size_filled = 0;
+ // filled by staged::get_stats()
+ size_t size_logical = 0;
+ size_t size_overhead = 0;
+ size_t size_value = 0;
+ unsigned num_kvs = 0;
+};
+
}
assert(index < num_items);
return (p_first_item - index)->get_p_value();
}
+ node_offset_t size_overhead_at(size_t index) const { return 0u; }
static node_offset_t header_size() { return 0u; }
assert(ret < NODE_BLOCK_SIZE);
return ret;
}
+ node_offset_t size_overhead_at(size_t index) const { return sizeof(node_offset_t); }
const onode_t* get_p_value(size_t index) const {
assert(index < keys());
auto pointer = get_item_start(index);
});
}
+btree_future<tree_stats_t> Btree::get_stats_slow(Transaction& t) {
+ return get_root(t).safe_then([this, &t](auto root) {
+ unsigned height = root->level() + 1;
+ return root->get_tree_stats(get_context(t)
+ ).safe_then([height](auto stats) {
+ stats.height = height;
+ return btree_ertr::make_ready_future<tree_stats_t>(stats);
+ });
+ });
+}
+
std::ostream& Btree::dump(Transaction& t, std::ostream& os) {
auto root = root_tracker->get_root(t);
if (root) {
// stats
btree_future<size_t> height(Transaction&);
+ btree_future<tree_stats_t> get_stats_slow(Transaction&);
std::ostream& dump(Transaction&, std::ostream&);
// test_only
return os << "onode(" << node.id << ", " << node.size << "B)";
}
+struct tree_stats_t {
+ size_t size_persistent_leaf = 0;
+ size_t size_persistent_internal = 0;
+ size_t size_filled_leaf = 0;
+ size_t size_filled_internal = 0;
+ size_t size_logical_leaf = 0;
+ size_t size_logical_internal = 0;
+ size_t size_overhead_leaf = 0;
+ size_t size_overhead_internal = 0;
+ size_t size_value_leaf = 0;
+ size_t size_value_internal = 0;
+ unsigned num_kvs_leaf = 0;
+ unsigned num_kvs_internal = 0;
+ unsigned num_nodes_leaf = 0;
+ unsigned num_nodes_internal = 0;
+ unsigned height = 0;
+
+ size_t size_persistent() const {
+ return size_persistent_leaf + size_persistent_internal; }
+ size_t size_filled() const {
+ return size_filled_leaf + size_filled_internal; }
+ size_t size_logical() const {
+ return size_logical_leaf + size_logical_internal; }
+ size_t size_overhead() const {
+ return size_overhead_leaf + size_overhead_internal; }
+ size_t size_value() const {
+ return size_value_leaf + size_value_internal; }
+ unsigned num_kvs() const {
+ return num_kvs_leaf + num_kvs_internal; }
+ unsigned num_nodes() const {
+ return num_nodes_leaf + num_nodes_internal; }
+
+ double ratio_fullness() const {
+ return (double)size_filled() / size_persistent(); }
+ double ratio_key_compression() const {
+ return (double)(size_filled() - size_value()) / (size_logical() - size_value()); }
+ double ratio_overhead() const {
+ return (double)size_overhead() / size_filled(); }
+ double ratio_keys_leaf() const {
+ return (double)num_kvs_leaf / num_kvs(); }
+ double ratio_nodes_leaf() const {
+ return (double)num_nodes_leaf / num_nodes(); }
+ double ratio_filled_leaf() const {
+ return (double)size_filled_leaf / size_filled(); }
+};
+inline std::ostream& operator<<(std::ostream& os, const tree_stats_t& stats) {
+ os << "Tree stats:"
+ << "\n height = " << stats.height
+ << "\n num values = " << stats.num_kvs_leaf
+ << "\n num nodes = " << stats.num_nodes()
+ << " (leaf=" << stats.num_nodes_leaf
+ << ", internal=" << stats.num_nodes_internal << ")"
+ << "\n size persistent = " << stats.size_persistent() << "B"
+ << "\n size filled = " << stats.size_filled() << "B"
+ << " (value=" << stats.size_value_leaf << "B"
+ << ", rest=" << stats.size_filled() - stats.size_value_leaf << "B)"
+ << "\n size logical = " << stats.size_logical() << "B"
+ << "\n size overhead = " << stats.size_overhead() << "B"
+ << "\n ratio fullness = " << stats.ratio_fullness()
+ << "\n ratio keys leaf = " << stats.ratio_keys_leaf()
+ << "\n ratio nodes leaf = " << stats.ratio_nodes_leaf()
+ << "\n ratio filled leaf = " << stats.ratio_filled_leaf()
+ << "\n ratio key compression = " << stats.ratio_key_compression();
+ assert(stats.num_kvs_internal + 1 == stats.num_nodes());
+ return os;
+}
+
}
assert(false && "impossible path"); }
key_view_t get_key_view(const search_position_t&) const override {
assert(false && "impossible path"); }
+ void next_position(search_position_t&) const override {
+ assert(false && "impossible path"); }
+ node_stats_t get_stats() const override {
+ assert(false && "impossible path"); }
std::ostream& dump(std::ostream&) const override {
assert(false && "impossible path"); }
std::ostream& dump_brief(std::ostream&) const override {
node_future<search_result_t> lower_bound_tracked(
context_t, const key_hobj_t&, MatchHistory&) override {
assert(false && "impossible path"); }
+ node_future<> do_get_tree_stats(context_t, tree_stats_t&) override {
+ assert(false && "impossible path"); }
private:
DummyChild(DummyChildImpl* impl, DummyChildImpl::URef&& ref, DummyChildPool& pool)
++iter;
}
- logger().info("Insert done! Tree height: {}", tree.height(t).unsafe_get0());
+ logger().info("Insert done!");
+ logger().info("{}", tree.get_stats_slow(t).unsafe_get0());
if (!cursors.empty()) {
auto kv_iter = kvs.random_begin();
projects / ceph.git / commitdiff