tools/kvstore-tool: implement 'dissect' command

This command shows information on key/value counts and lengths for KV
DB.

Signed-off-by: Igor Fedotov <ifedotov@suse.com>

diff --git a/doc/man/8/ceph-kvstore-tool.rst b/doc/man/8/ceph-kvstore-tool.rst
index 1eb99c03067d6abd3e8d886d7940c3ac5da7cf43..e7c859b13d97903c5d2550d49696de35e80c991d 100644 (file)
--- a/doc/man/8/ceph-kvstore-tool.rst
+++ b/doc/man/8/ceph-kvstore-tool.rst
@@ -85,6 +85,9 @@ which are as follows:
     Format and information content may vary between releases. For RocksDB information includes
     compactions stats, performance counters, memory usage and internal RocksDB stats. 
 
+:command:`histogram`
+    Presents key-value sizes distribution statistics from the underlying KV database.
+
 Availability
 ============
 

diff --git a/src/kv/KeyValueDB.h b/src/kv/KeyValueDB.h
index a9a7965117c0dc0dc62d5e8af55df32aa008a185..afbfb6ecfd7f3629a8eed450aac1292503b60144 100644 (file)
--- a/src/kv/KeyValueDB.h
+++ b/src/kv/KeyValueDB.h
@@ -264,6 +264,8 @@ public:
 
 private:
   // This class filters a WholeSpaceIterator by a prefix.
+  // Performs as a dummy wrapper over WholeSpaceIterator
+  // if prefix is empty
   class PrefixIteratorImpl : public IteratorImpl {
     const std::string prefix;
     WholeSpaceIterator generic_iter;
@@ -273,10 +275,14 @@ private:
     ~PrefixIteratorImpl() override { }
 
     int seek_to_first() override {
-      return generic_iter->seek_to_first(prefix);
+      return prefix.empty() ?
+       generic_iter->seek_to_first() :
+       generic_iter->seek_to_first(prefix);
     }
     int seek_to_last() override {
-      return generic_iter->seek_to_last(prefix);
+      return prefix.empty() ?
+       generic_iter->seek_to_last() :
+       generic_iter->seek_to_last(prefix);
     }
     int upper_bound(const std::string &after) override {
       return generic_iter->upper_bound(prefix, after);
@@ -287,7 +293,11 @@ private:
     bool valid() override {
       if (!generic_iter->valid())
        return false;
-      return generic_iter->raw_key_is_prefixed(prefix);
+      if (prefix.empty())
+        return true;
+      return prefix.empty() ?
+             true :
+             generic_iter->raw_key_is_prefixed(prefix);
     }
     int next() override {
       return generic_iter->next();

diff --git a/src/test/cli/ceph-kvstore-tool/help.t b/src/test/cli/ceph-kvstore-tool/help.t
index 1f984b11e5fe086a77144e749104029bd9ffe0ed..e86c7cd23bedf4b8a8a5ffe124d71f63ba069e60 100644 (file)
--- a/src/test/cli/ceph-kvstore-tool/help.t
+++ b/src/test/cli/ceph-kvstore-tool/help.t
@@ -19,4 +19,5 @@
     compact-range <prefix> <start> <end>
     destructive-repair  (use only as last resort! may corrupt healthy data)
     stats
+    histogram [prefix]
   

diff --git a/src/tools/ceph_kvstore_tool.cc b/src/tools/ceph_kvstore_tool.cc
index a50666850de03a67f62489a766a5a714cdc40f54..fdcf54fb1dddbd4349fd901e500ed1ba6857d23e 100644 (file)
--- a/src/tools/ceph_kvstore_tool.cc
+++ b/src/tools/ceph_kvstore_tool.cc
@@ -48,6 +48,7 @@ void usage(const char *pname)
     << "  compact-range <prefix> <start> <end>\n"
     << "  destructive-repair  (use only as last resort! may corrupt healthy data)\n"
     << "  stats\n"
+    << "  histogram [prefix]\n"
     << std::endl;
 }
 
@@ -347,6 +348,11 @@ int main(int argc, const char *argv[])
     st.compact_range(prefix, start, end);
   } else if (cmd == "stats") {
     st.print_stats();
+  } else if (cmd == "histogram") {
+    string prefix;
+    if (argc > 4)
+      prefix = url_unescape(argv[4]);
+    st.build_size_histogram(prefix);
   } else {
     std::cerr << "Unrecognized command: " << cmd << std::endl;
     return 1;

diff --git a/src/tools/kvstore_tool.cc b/src/tools/kvstore_tool.cc
index d26a195880ef3a93620f2a9f71591846e63ada3d..88d2b50722c39cf6bdca8c09a2510abca71e47aa 100644 (file)
--- a/src/tools/kvstore_tool.cc
+++ b/src/tools/kvstore_tool.cc
@@ -7,8 +7,10 @@
 
 #include "common/errno.h"
 #include "common/url_escape.h"
+#include "common/pretty_binary.h"
 #include "include/buffer.h"
 #include "kv/KeyValueDB.h"
+#include "kv/KeyValueHistogram.h"
 
 StoreTool::StoreTool(const string& type,
                     const string& path,
@@ -228,6 +230,66 @@ int StoreTool::print_stats() const
   return ret;
 }
 
+//Itrerates through the db and collects the stats
+int StoreTool::build_size_histogram(const string& prefix0) const
+{
+  ostringstream ostr;
+  Formatter* f = Formatter::create("json-pretty", "json-pretty", "json-pretty");
+
+  const size_t MAX_PREFIX = 256;
+  uint64_t num[MAX_PREFIX] = {0};
+
+  size_t max_key_size = 0, max_value_size = 0;
+  uint64_t total_key_size = 0, total_value_size = 0;
+  size_t key_size = 0, value_size = 0;
+  KeyValueHistogram hist;
+
+  auto start = coarse_mono_clock::now();
+
+  auto iter = db->get_iterator(prefix0, KeyValueDB::ITERATOR_NOCACHE);
+  iter->seek_to_first();
+  while (iter->valid()) {
+    pair<string, string> key(iter->raw_key());
+    key_size = key.first.size() + key.second.size();
+    value_size = iter->value().length();
+    hist.value_hist[hist.get_value_slab(value_size)]++;
+    max_key_size = std::max(max_key_size, key_size);
+    max_value_size = std::max(max_value_size, value_size);
+    total_key_size += key_size;
+    total_value_size += value_size;
+
+
+    unsigned prefix = key.first[0];
+    ceph_assert(prefix < MAX_PREFIX);
+    num[prefix]++;
+    hist.update_hist_entry(hist.key_hist, key.first, key_size, value_size);
+    iter->next();
+  }
+
+  ceph::timespan duration = coarse_mono_clock::now() - start;
+  f->open_object_section("rocksdb_key_value_stats");
+  for (size_t i = 0; i < MAX_PREFIX; ++i) {
+    if (num[i]) {
+      string key = "Records for prefix: ";
+      key += pretty_binary_string(string(1, char(i)));
+      f->dump_unsigned(key, num[i]);
+    }
+  }
+  f->dump_unsigned("max_key_size", max_key_size);
+  f->dump_unsigned("max_value_size", max_value_size);
+  f->dump_unsigned("total_key_size", total_key_size);
+  f->dump_unsigned("total_value_size", total_value_size);
+  hist.dump(f);
+  f->close_section();
+
+  f->flush(ostr);
+  delete f;
+
+  std::cout << ostr.str() << std::endl;
+  std::cout << __func__ << " finished in " << duration << " seconds" << std::endl;
+  return 0;
+}
+
 int StoreTool::copy_store_to(const string& type, const string& other_path,
                              const int num_keys_per_tx,
                              const string& other_type)

diff --git a/src/tools/kvstore_tool.h b/src/tools/kvstore_tool.h
index d8c8966130aac69703e386ffd4eca76b022af4da..e2522124c77750a33ea25b2ee244cf3bef396304 100644 (file)
--- a/src/tools/kvstore_tool.h
+++ b/src/tools/kvstore_tool.h
@@ -77,4 +77,5 @@ public:
   int destructive_repair();
 
   int print_stats() const;
+  int build_size_histogram(const string& prefix) const;
 };