From: Igor Fedotov Date: Fri, 5 Feb 2021 10:15:16 +0000 (+0300) Subject: tools/kvstore-tool: implement 'dissect' command X-Git-Tag: v17.1.0~2895^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=b437f7141cb3ee319235afdd3fa19eefad351035;p=ceph.git tools/kvstore-tool: implement 'dissect' command This command shows information on key/value counts and lengths for KV DB. Signed-off-by: Igor Fedotov --- diff --git a/doc/man/8/ceph-kvstore-tool.rst b/doc/man/8/ceph-kvstore-tool.rst index 1eb99c03067..e7c859b13d9 100644 --- a/doc/man/8/ceph-kvstore-tool.rst +++ b/doc/man/8/ceph-kvstore-tool.rst @@ -85,6 +85,9 @@ which are as follows: Format and information content may vary between releases. For RocksDB information includes compactions stats, performance counters, memory usage and internal RocksDB stats. +:command:`histogram` + Presents key-value sizes distribution statistics from the underlying KV database. + Availability ============ diff --git a/src/kv/KeyValueDB.h b/src/kv/KeyValueDB.h index a9a7965117c..afbfb6ecfd7 100644 --- a/src/kv/KeyValueDB.h +++ b/src/kv/KeyValueDB.h @@ -264,6 +264,8 @@ public: private: // This class filters a WholeSpaceIterator by a prefix. + // Performs as a dummy wrapper over WholeSpaceIterator + // if prefix is empty class PrefixIteratorImpl : public IteratorImpl { const std::string prefix; WholeSpaceIterator generic_iter; @@ -273,10 +275,14 @@ private: ~PrefixIteratorImpl() override { } int seek_to_first() override { - return generic_iter->seek_to_first(prefix); + return prefix.empty() ? + generic_iter->seek_to_first() : + generic_iter->seek_to_first(prefix); } int seek_to_last() override { - return generic_iter->seek_to_last(prefix); + return prefix.empty() ? + generic_iter->seek_to_last() : + generic_iter->seek_to_last(prefix); } int upper_bound(const std::string &after) override { return generic_iter->upper_bound(prefix, after); @@ -287,7 +293,11 @@ private: bool valid() override { if (!generic_iter->valid()) return false; - return generic_iter->raw_key_is_prefixed(prefix); + if (prefix.empty()) + return true; + return prefix.empty() ? + true : + generic_iter->raw_key_is_prefixed(prefix); } int next() override { return generic_iter->next(); diff --git a/src/test/cli/ceph-kvstore-tool/help.t b/src/test/cli/ceph-kvstore-tool/help.t index 1f984b11e5f..e86c7cd23be 100644 --- a/src/test/cli/ceph-kvstore-tool/help.t +++ b/src/test/cli/ceph-kvstore-tool/help.t @@ -19,4 +19,5 @@ compact-range destructive-repair (use only as last resort! may corrupt healthy data) stats + histogram [prefix] diff --git a/src/tools/ceph_kvstore_tool.cc b/src/tools/ceph_kvstore_tool.cc index a50666850de..fdcf54fb1dd 100644 --- a/src/tools/ceph_kvstore_tool.cc +++ b/src/tools/ceph_kvstore_tool.cc @@ -48,6 +48,7 @@ void usage(const char *pname) << " compact-range \n" << " destructive-repair (use only as last resort! may corrupt healthy data)\n" << " stats\n" + << " histogram [prefix]\n" << std::endl; } @@ -347,6 +348,11 @@ int main(int argc, const char *argv[]) st.compact_range(prefix, start, end); } else if (cmd == "stats") { st.print_stats(); + } else if (cmd == "histogram") { + string prefix; + if (argc > 4) + prefix = url_unescape(argv[4]); + st.build_size_histogram(prefix); } else { std::cerr << "Unrecognized command: " << cmd << std::endl; return 1; diff --git a/src/tools/kvstore_tool.cc b/src/tools/kvstore_tool.cc index d26a195880e..88d2b50722c 100644 --- a/src/tools/kvstore_tool.cc +++ b/src/tools/kvstore_tool.cc @@ -7,8 +7,10 @@ #include "common/errno.h" #include "common/url_escape.h" +#include "common/pretty_binary.h" #include "include/buffer.h" #include "kv/KeyValueDB.h" +#include "kv/KeyValueHistogram.h" StoreTool::StoreTool(const string& type, const string& path, @@ -228,6 +230,66 @@ int StoreTool::print_stats() const return ret; } +//Itrerates through the db and collects the stats +int StoreTool::build_size_histogram(const string& prefix0) const +{ + ostringstream ostr; + Formatter* f = Formatter::create("json-pretty", "json-pretty", "json-pretty"); + + const size_t MAX_PREFIX = 256; + uint64_t num[MAX_PREFIX] = {0}; + + size_t max_key_size = 0, max_value_size = 0; + uint64_t total_key_size = 0, total_value_size = 0; + size_t key_size = 0, value_size = 0; + KeyValueHistogram hist; + + auto start = coarse_mono_clock::now(); + + auto iter = db->get_iterator(prefix0, KeyValueDB::ITERATOR_NOCACHE); + iter->seek_to_first(); + while (iter->valid()) { + pair key(iter->raw_key()); + key_size = key.first.size() + key.second.size(); + value_size = iter->value().length(); + hist.value_hist[hist.get_value_slab(value_size)]++; + max_key_size = std::max(max_key_size, key_size); + max_value_size = std::max(max_value_size, value_size); + total_key_size += key_size; + total_value_size += value_size; + + + unsigned prefix = key.first[0]; + ceph_assert(prefix < MAX_PREFIX); + num[prefix]++; + hist.update_hist_entry(hist.key_hist, key.first, key_size, value_size); + iter->next(); + } + + ceph::timespan duration = coarse_mono_clock::now() - start; + f->open_object_section("rocksdb_key_value_stats"); + for (size_t i = 0; i < MAX_PREFIX; ++i) { + if (num[i]) { + string key = "Records for prefix: "; + key += pretty_binary_string(string(1, char(i))); + f->dump_unsigned(key, num[i]); + } + } + f->dump_unsigned("max_key_size", max_key_size); + f->dump_unsigned("max_value_size", max_value_size); + f->dump_unsigned("total_key_size", total_key_size); + f->dump_unsigned("total_value_size", total_value_size); + hist.dump(f); + f->close_section(); + + f->flush(ostr); + delete f; + + std::cout << ostr.str() << std::endl; + std::cout << __func__ << " finished in " << duration << " seconds" << std::endl; + return 0; +} + int StoreTool::copy_store_to(const string& type, const string& other_path, const int num_keys_per_tx, const string& other_type) diff --git a/src/tools/kvstore_tool.h b/src/tools/kvstore_tool.h index d8c8966130a..e2522124c77 100644 --- a/src/tools/kvstore_tool.h +++ b/src/tools/kvstore_tool.h @@ -77,4 +77,5 @@ public: int destructive_repair(); int print_stats() const; + int build_size_histogram(const string& prefix) const; };