Hotcold: Added ability to select which levels record metrics

Summary: Added an option to `Options` to specify which levels have metrics recorded and used that option in `Version::Get()` to stop metrics being recorded for wrong levels and in `DBImpl` to stop `DoCompactionWork()` trying to split on hotcold for levels where no metrics are available.

Test Plan: make check

Reviewers: vamsi, dhruba

Reviewed By: vamsi

CC: leveldb
Differential Revision: https://reviews.facebook.net/D8511

diff --git a/db/db_bench.cc b/db/db_bench.cc
index 5258c07e27157f7be2e780e49578b6b05ca3a02f..e7a8e56dda5f83e73423136e23382d4e320dc2dd 100644 (file)
--- a/db/db_bench.cc
+++ b/db/db_bench.cc
@@ -227,6 +227,9 @@ static bool FLAGS_read_only = false;
 // Run benchmarks with hot-cold separation.
 static bool FLAGS_hot_cold = false;
 
+// For hot-cold separation only levels numbered FLAGS_min_hotcold_level and above
+static uint32_t FLAGS_min_hotcold_level = 0;
+
 // Do not auto trigger compactions
 static bool FLAGS_disable_auto_compactions = false;
 
@@ -997,6 +1000,7 @@ class Benchmark {
       FLAGS_max_grandparent_overlap_factor;
     options.disable_auto_compactions = FLAGS_disable_auto_compactions;
     options.source_compaction_factor = FLAGS_source_compaction_factor;
+    options.min_hotcold_level = FLAGS_min_hotcold_level;
     Status s;
     if(FLAGS_read_only) {
       s = DB::OpenForReadOnly(options, FLAGS_db, &db_);
@@ -1359,6 +1363,7 @@ int main(int argc, char** argv) {
   for (int i = 1; i < argc; i++) {
     double d;
     int n;
+    unsigned int un;
     long l;
     char junk;
     char hdfsname[2048];
@@ -1524,6 +1529,8 @@ int main(int argc, char** argv) {
     } else if (sscanf(argv[i], "--source_compaction_factor=%d%c",
                &n, &junk) == 1 && n > 0) {
       FLAGS_source_compaction_factor = n;
+    } else if (sscanf(argv[i], "--min_hotcold_level=%u%c", &un, &junk) == 1) {
+      FLAGS_min_hotcold_level = un;
     } else if (sscanf(argv[i], "--wal_ttl=%d%c", &n, &junk) == 1) {
       FLAGS_WAL_ttl_seconds = static_cast<uint64_t>(n);
     } else {

diff --git a/db/db_impl.cc b/db/db_impl.cc
index 8d8a2440e306ac6a4f0b463f57b19972bca85bd5..86b2c30d2749568aa96ae948b2fd6703101ccae6 100644 (file)
--- a/db/db_impl.cc
+++ b/db/db_impl.cc
@@ -1397,7 +1397,12 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
   assert(compact != NULL);
   assert(compact->builders.empty());
 
-  compact->num_outfiles = is_hotcold_?2:1;
+  compact->num_outfiles = 1;
+  // Create 2 files if we are going to do hot-cold separation.
+  if (is_hotcold_ &&
+      compact->compaction->level()+1 >= options_.min_hotcold_level) {
+    compact->num_outfiles = 2;
+  }
   compact->builders.resize(compact->num_outfiles);
   compact->outfiles.resize(compact->num_outfiles);
 
@@ -1968,6 +1973,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
       if (is_hotcold_ &&
           !encountered_error_key &&
           can_be_hot &&
+          compact->compaction->level()+1 >= options_.min_hotcold_level &&
           IsRecordHot(input.get(), metrics_db_, ReadOptions(),
                       &block_metrics_store)) {
         outfile_idx = 1; // Use 1 as the hot file as it has a larger file

diff --git a/db/version_set.cc b/db/version_set.cc
index 7a1c6854d1bb768d83608d1178c09cdb12ce1a5f..d57adb399528ccf3868064d5e51caaf12dec43fc 100644 (file)
--- a/db/version_set.cc
+++ b/db/version_set.cc
@@ -545,6 +545,11 @@ Status Version::Get(const ReadOptions& options,
     SequenceNumber saved_seq = 0;
     SaverState saved_state = kNotFound;
 
+    ReadOptions read_options = options;
+    if (level < vset_->options_->min_hotcold_level) {
+      read_options.metrics_handler = nullptr;
+    }
+
     for (uint32_t i = 0; i < files.size(); ++i) {
 
       FileMetaData* f = files[i];
@@ -554,7 +559,7 @@ Status Version::Get(const ReadOptions& options,
       saver.value = value;
       saver.didIO = false;
       bool tableIO = false;
-      s = vset_->table_cache_->Get(options, f->number, f->file_size,
+      s = vset_->table_cache_->Get(read_options, f->number, f->file_size,
                                    ikey, &saver, SaveValue, &tableIO);
       if (!s.ok()) {
         return s;

diff --git a/include/leveldb/options.h b/include/leveldb/options.h
index fb29c297a72e54c983fbae08bda15f538066267c..c119a9761888dce7c256ec744d992e87877388ae 100644 (file)
--- a/include/leveldb/options.h
+++ b/include/leveldb/options.h
@@ -327,6 +327,14 @@ struct Options {
   // Number of shards used for table cache.
   int table_cache_numshardbits;
 
+  // In a database with hot cold separation. Levels with a level number greater
+  // or equal to this have metrics recorded, so it can be determined whether
+  // records in that level are hot. Only levles with a level number greater
+  // than or equal to this parameter have records split into hot and cold
+  // files.
+  // Default: 0
+  uint32_t min_hotcold_level;
+
   // Create an Options object with default values for all fields.
   Options();
 

diff --git a/util/options.cc b/util/options.cc
index ea58893ab0b08cb67585273e74574e55b0d97c6b..504640ee612abd2b4fdea1ff87c2e26366e55c3b 100644 (file)
--- a/util/options.cc
+++ b/util/options.cc
@@ -52,6 +52,7 @@ Options::Options()
       max_manifest_file_size(std::numeric_limits<uint64_t>::max()),
       no_block_cache(false),
       table_cache_numshardbits(4),
+      min_hotcold_level(0),
       compaction_filter_args(NULL),
       CompactionFilter(NULL),
       disable_auto_compactions(false),
@@ -149,6 +150,8 @@ Options::Dump(Logger* log) const
         WAL_ttl_seconds);
     Log(log,"            Options.manifest_preallocation_size: %ld",
         manifest_preallocation_size);
+    Log(log,"                      Options.min_hotcold_level: %u",
+        min_hotcold_level);
 }   // Options::Dump
 
 }  // namespace leveldb