From: Igor Fedotov Date: Wed, 10 Feb 2021 11:46:36 +0000 (+0300) Subject: kv/KeyValueHistogram: make Bluestore's DBHistogram class reusable. X-Git-Tag: v17.1.0~2895^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c50b14120090e71a314bbe9efbe23392e8040e5b;p=ceph.git kv/KeyValueHistogram: make Bluestore's DBHistogram class reusable. Signed-off-by: Igor Fedotov --- diff --git a/src/kv/CMakeLists.txt b/src/kv/CMakeLists.txt index 057eb82601a..dce4cd2d181 100644 --- a/src/kv/CMakeLists.txt +++ b/src/kv/CMakeLists.txt @@ -2,6 +2,7 @@ set(kv_srcs KeyValueDB.cc MemDB.cc RocksDBStore.cc + KeyValueHistogram.cc rocksdb_cache/ShardedCache.cc rocksdb_cache/BinnedLRUCache.cc) diff --git a/src/kv/KeyValueHistogram.cc b/src/kv/KeyValueHistogram.cc new file mode 100644 index 00000000000..5015225d5b8 --- /dev/null +++ b/src/kv/KeyValueHistogram.cc @@ -0,0 +1,78 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "include/stringify.h" +#include "KeyValueHistogram.h" +using std::map; +using std::string; +using ceph::Formatter; + +#define KEY_SLAB 32 +#define VALUE_SLAB 64 + +int KeyValueHistogram::get_key_slab(size_t sz) +{ + return (sz / KEY_SLAB); +} + +string KeyValueHistogram::get_key_slab_to_range(int slab) +{ + int lower_bound = slab * KEY_SLAB; + int upper_bound = (slab + 1) * KEY_SLAB; + string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")"; + return ret; +} + +int KeyValueHistogram::get_value_slab(size_t sz) +{ + return (sz / VALUE_SLAB); +} + +string KeyValueHistogram::get_value_slab_to_range(int slab) +{ + int lower_bound = slab * VALUE_SLAB; + int upper_bound = (slab + 1) * VALUE_SLAB; + string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")"; + return ret; +} + +void KeyValueHistogram::update_hist_entry(map >& key_hist, + const string& prefix, size_t key_size, size_t value_size) +{ + uint32_t key_slab = get_key_slab(key_size); + uint32_t value_slab = get_value_slab(value_size); + key_hist[prefix][key_slab].count++; + key_hist[prefix][key_slab].max_len = + std::max(key_size, key_hist[prefix][key_slab].max_len); + key_hist[prefix][key_slab].val_map[value_slab].count++; + key_hist[prefix][key_slab].val_map[value_slab].max_len = + std::max(value_size, + key_hist[prefix][key_slab].val_map[value_slab].max_len); +} + +void KeyValueHistogram::dump(Formatter* f) +{ + f->open_object_section("rocksdb_value_distribution"); + for (auto i : value_hist) { + f->dump_unsigned(get_value_slab_to_range(i.first).data(), i.second); + } + f->close_section(); + + f->open_object_section("rocksdb_key_value_histogram"); + for (auto i : key_hist) { + f->dump_string("prefix", i.first); + f->open_object_section("key_hist"); + for (auto k : i.second) { + f->dump_unsigned(get_key_slab_to_range(k.first).data(), k.second.count); + f->dump_unsigned("max_len", k.second.max_len); + f->open_object_section("value_hist"); + for (auto j : k.second.val_map) { + f->dump_unsigned(get_value_slab_to_range(j.first).data(), j.second.count); + f->dump_unsigned("max_len", j.second.max_len); + } + f->close_section(); + } + f->close_section(); + } + f->close_section(); +} diff --git a/src/kv/KeyValueHistogram.h b/src/kv/KeyValueHistogram.h new file mode 100644 index 00000000000..2da44bd4ec2 --- /dev/null +++ b/src/kv/KeyValueHistogram.h @@ -0,0 +1,38 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#ifndef KeyValueHistogram_H +#define KeyValueHistogram_H + +#include +#include +#include "common/Formatter.h" + +/** + * + * Key Value DB Histogram generator + * + */ +struct KeyValueHistogram { + struct value_dist { + uint64_t count; + uint32_t max_len; + }; + + struct key_dist { + uint64_t count; + uint32_t max_len; + std::map val_map; ///< slab id to count, max length of value and key + }; + + std::map > key_hist; + std::map value_hist; + int get_key_slab(size_t sz); + std::string get_key_slab_to_range(int slab); + int get_value_slab(size_t sz); + std::string get_value_slab_to_range(int slab); + void update_hist_entry(std::map >& key_hist, + const std::string& prefix, size_t key_size, size_t value_size); + void dump(ceph::Formatter* f); +}; + +#endif diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index ddf98cedfe0..6f2d16f93de 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -45,6 +45,7 @@ #include "common/blkdev.h" #include "common/numa.h" #include "common/pretty_binary.h" +#include "kv/KeyValueHistogram.h" #if defined(WITH_LTTNG) #define TRACEPOINT_DEFINE @@ -15656,81 +15657,9 @@ void BlueStore::BlueStoreThrottle::complete(TransContext &txc) } #endif -// DB key value Histogram -#define KEY_SLAB 32 -#define VALUE_SLAB 64 - const string prefix_onode = "o"; const string prefix_onode_shard = "x"; const string prefix_other = "Z"; - -int BlueStore::DBHistogram::get_key_slab(size_t sz) -{ - return (sz/KEY_SLAB); -} - -string BlueStore::DBHistogram::get_key_slab_to_range(int slab) -{ - int lower_bound = slab * KEY_SLAB; - int upper_bound = (slab + 1) * KEY_SLAB; - string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")"; - return ret; -} - -int BlueStore::DBHistogram::get_value_slab(size_t sz) -{ - return (sz/VALUE_SLAB); -} - -string BlueStore::DBHistogram::get_value_slab_to_range(int slab) -{ - int lower_bound = slab * VALUE_SLAB; - int upper_bound = (slab + 1) * VALUE_SLAB; - string ret = "[" + stringify(lower_bound) + "," + stringify(upper_bound) + ")"; - return ret; -} - -void BlueStore::DBHistogram::update_hist_entry(map > &key_hist, - const string &prefix, size_t key_size, size_t value_size) -{ - uint32_t key_slab = get_key_slab(key_size); - uint32_t value_slab = get_value_slab(value_size); - key_hist[prefix][key_slab].count++; - key_hist[prefix][key_slab].max_len = - std::max(key_size, key_hist[prefix][key_slab].max_len); - key_hist[prefix][key_slab].val_map[value_slab].count++; - key_hist[prefix][key_slab].val_map[value_slab].max_len = - std::max(value_size, - key_hist[prefix][key_slab].val_map[value_slab].max_len); -} - -void BlueStore::DBHistogram::dump(Formatter *f) -{ - f->open_object_section("rocksdb_value_distribution"); - for (auto i : value_hist) { - f->dump_unsigned(get_value_slab_to_range(i.first).data(), i.second); - } - f->close_section(); - - f->open_object_section("rocksdb_key_value_histogram"); - for (auto i : key_hist) { - f->dump_string("prefix", i.first); - f->open_object_section("key_hist"); - for ( auto k : i.second) { - f->dump_unsigned(get_key_slab_to_range(k.first).data(), k.second.count); - f->dump_unsigned("max_len", k.second.max_len); - f->open_object_section("value_hist"); - for ( auto j : k.second.val_map) { - f->dump_unsigned(get_value_slab_to_range(j.first).data(), j.second.count); - f->dump_unsigned("max_len", j.second.max_len); - } - f->close_section(); - } - f->close_section(); - } - f->close_section(); -} - //Itrerates through the db and collects the stats void BlueStore::generate_db_histogram(Formatter *f) { @@ -15749,7 +15678,7 @@ void BlueStore::generate_db_histogram(Formatter *f) size_t max_key_size =0, max_value_size = 0; uint64_t total_key_size = 0, total_value_size = 0; size_t key_size = 0, value_size = 0; - DBHistogram hist; + KeyValueHistogram hist; auto start = coarse_mono_clock::now(); diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 14c06a5c26e..640c4c15e9c 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1997,29 +1997,6 @@ public: } }; - struct DBHistogram { - struct value_dist { - uint64_t count; - uint32_t max_len; - }; - - struct key_dist { - uint64_t count; - uint32_t max_len; - std::map val_map; ///< slab id to count, max length of value and key - }; - - std::map > key_hist; - std::map value_hist; - int get_key_slab(size_t sz); - std::string get_key_slab_to_range(int slab); - int get_value_slab(size_t sz); - std::string get_value_slab_to_range(int slab); - void update_hist_entry(std::map > &key_hist, - const std::string &prefix, size_t key_size, size_t value_size); - void dump(ceph::Formatter *f); - }; - struct BigDeferredWriteContext { uint64_t off = 0; // original logical offset uint32_t b_off = 0; // blob relative offset