PlainTableReader to expose index size to users

Summary:
This is a temp solution to expose index sizes to users from PlainTableReader before we persistent them to files.
In this patch, the memory consumption of indexes used by PlainTableReader will be reported as two user defined properties, so that users can monitor them.

Test Plan:
Add a unit test.
make all check`

Reviewers: haobo, ljin

Reviewed By: haobo

CC: nkg-, yhchiang, igor, ljin, dhruba, leveldb
Differential Revision: https://reviews.facebook.net/D18195

diff --git a/db/plain_table_db_test.cc b/db/plain_table_db_test.cc
index 6a95a2585f4ed71d3534a326e27a03b7f2bda424..8a2d85f58e2b03d6458d8741401b5efbc78f1056 100644 (file)
--- a/db/plain_table_db_test.cc
+++ b/db/plain_table_db_test.cc
@@ -190,7 +190,7 @@ class TestPlainTableReader : public PlainTableReader {
                          file_size, bloom_bits_per_key, hash_table_ratio,
                          index_sparseness, table_properties),
         expect_bloom_not_match_(expect_bloom_not_match) {
-    Status s = PopulateIndex();
+    Status s = PopulateIndex(const_cast<TableProperties*>(table_properties));
     ASSERT_TRUE(s.ok());
   }
 
@@ -265,6 +265,19 @@ TEST(PlainTableDBTest, Flush) {
       ASSERT_OK(Put("0000000000000bar", "v2"));
       ASSERT_OK(Put("1000000000000foo", "v3"));
       dbfull()->TEST_FlushMemTable();
+
+      TablePropertiesCollection ptc;
+      reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
+      ASSERT_EQ(1, ptc.size());
+      auto row = ptc.begin();
+      auto tp = row->second;
+      ASSERT_EQ(
+          total_order ? "4" : "12",
+          (tp->user_collected_properties).at("plain_table_hash_table_size"));
+      ASSERT_EQ(
+          total_order ? "9" : "0",
+          (tp->user_collected_properties).at("plain_table_sub_index_size"));
+
       ASSERT_EQ("v3", Get("1000000000000foo"));
       ASSERT_EQ("v2", Get("0000000000000bar"));
     }

diff --git a/include/rocksdb/table_properties.h b/include/rocksdb/table_properties.h
index 55b83f4411611737dc7052fd73244b41f8b05721..aa8b8a0b8a99cc60d3015b893385f58b85a6ec49 100644 (file)
--- a/include/rocksdb/table_properties.h
+++ b/include/rocksdb/table_properties.h
@@ -23,7 +23,7 @@ namespace rocksdb {
 //      ++pos) {
 //   ...
 // }
-typedef std::map<std::string, std::string> UserCollectedProperties;
+typedef std::map<const std::string, std::string> UserCollectedProperties;
 
 // TableProperties contains a bunch of read-only properties of its associated
 // table.

diff --git a/table/plain_table_reader.cc b/table/plain_table_reader.cc
index 436d13bf3825e6f08e4c11b209d98eb754b693db..02e8f12d41185770a6521226131f9554f76b6359 100644 (file)
--- a/table/plain_table_reader.cc
+++ b/table/plain_table_reader.cc
@@ -103,8 +103,8 @@ PlainTableReader::PlainTableReader(
       kHashTableRatio(hash_table_ratio),
       kBloomBitsPerKey(bloom_bits_per_key),
       kIndexIntervalForSamePrefixKeys(index_sparseness),
-      table_properties_(table_properties),
-      data_end_offset_(table_properties_->data_size),
+      table_properties_(nullptr),
+      data_end_offset_(table_properties->data_size),
       user_key_len_(table_properties->fixed_key_len) {
   assert(kHashTableRatio >= 0.0);
 }
@@ -136,7 +136,7 @@ Status PlainTableReader::Open(
       bloom_bits_per_key, hash_table_ratio, index_sparseness, props));
 
   // -- Populate Index
-  s = new_reader->PopulateIndex();
+  s = new_reader->PopulateIndex(props);
   if (!s.ok()) {
     return s;
   }
@@ -363,7 +363,10 @@ void PlainTableReader::FillIndexes(
       index_size_, kSubIndexSize);
 }
 
-Status PlainTableReader::PopulateIndex() {
+Status PlainTableReader::PopulateIndex(TableProperties* props) {
+  assert(props != nullptr);
+  table_properties_.reset(props);
+
   // options.prefix_extractor is requried for a hash-based look-up.
   if (options_.prefix_extractor.get() == nullptr && kHashTableRatio != 0) {
     return Status::NotSupported(
@@ -408,6 +411,14 @@ Status PlainTableReader::PopulateIndex() {
   // From the temp data structure, populate indexes.
   FillIndexes(sub_index_size_needed, hash_to_offsets, entries_per_bucket);
 
+  // Fill two table properties.
+  // TODO(sdong): after we have the feature of storing index in file, this
+  // properties need to be populated to index_size instead.
+  props->user_collected_properties["plain_table_hash_table_size"] =
+      std::to_string(index_size_ * 4U);
+  props->user_collected_properties["plain_table_sub_index_size"] =
+      std::to_string(sub_index_size_needed);
+
   return Status::OK();
 }
 

diff --git a/table/plain_table_reader.h b/table/plain_table_reader.h
index ac2cb874459cdb54527f72b26eb398152fed0b5b..7f0c3b537fea2366440f45003a8758ce32b05f69 100644 (file)
--- a/table/plain_table_reader.h
+++ b/table/plain_table_reader.h
@@ -86,6 +86,9 @@ class PlainTableReader: public TableReader {
   // PopulateIndex() builds index of keys. It must be called before any query
   // to the table.
   //
+  // props: the table properties object that need to be stored. Ownership of
+  //        the object will be passed.
+  //
   // index_ contains buckets size of index_size_, each is a
   // 32-bit integer. The lower 31 bits contain an offset value (explained below)
   // and the first bit of the integer indicates type of the offset.
@@ -121,7 +124,7 @@ class PlainTableReader: public TableReader {
   //    ....
   //   record N file offset:  fixedint32
   // <end>
-  Status PopulateIndex();
+  Status PopulateIndex(TableProperties* props);
 
  private:
   struct IndexRecord;