options.write_buffer_size = 1048576;
Reopen(&options);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+ bool hybrid = dbi->IsHybrid();
// Fill levels >= 1 so memtable compaction outputs to level 1
for (int level = 1; level < dbi->NumberLevels(); level++) {
Build(10);
dbi->TEST_CompactMemTable();
- ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
+ hybrid? ASSERT_EQ(3, Property("leveldb.num-files-at-level0")) :
+ ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
Corrupt(kTableFile, 100, 1);
Check(9, 9);
// insert files directly into higher levels because some other
// threads could be concurrently producing compacted files for
// that key range.
- if (base != NULL && options_.max_background_compactions <= 1) {
+ if (base != NULL && options_.max_background_compactions <= 1 &&
+ !IsHybrid()) {
level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
}
edit->AddFile(level, meta.number, meta.file_size,
void DBImpl::CompactRange(const Slice* begin, const Slice* end) {
int max_level_with_files = 1;
+ TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
{
MutexLock l(&mutex_);
Version* base = versions_->current();
}
}
}
- TEST_CompactMemTable(); // TODO(sanjay): Skip if memtable does not overlap
+ // In Normal mode, compacting level n merges the contents of level n
+ // with level n+1 thereby removing deplicate keys upto level n+1.
+ // In Hybrid Mode, compacting level n creates a new file in level n+1
+ // without merging the duplicates that are already present in level n+1.
+ // So, we need to explicitly compact till level n+1.
+ IsHybrid() ? max_level_with_files++ : true;
for (int level = 0; level < max_level_with_files; level++) {
TEST_CompactRange(level, begin, end);
}
}
int DBImpl::MaxMemCompactionLevel() {
- return options_.max_mem_compaction_level;
+ return (IsHybrid() ? 0 : options_.max_mem_compaction_level);
}
int DBImpl::Level0StopWriteTrigger() {
manual_end = c->input(0, c->num_input_files(0) - 1)->largest;
}
Log(options_.info_log,
- "Manual compaction at level-%d from %s .. %s; will stop at %s\n",
+ "Manual compaction at level-%d from %s .. %s; will stop at %s, %s\n",
m->level,
(m->begin ? m->begin->DebugString().c_str() : "(begin)"),
(m->end ? m->end->DebugString().c_str() : "(end)"),
- (m->done ? "(end)" : manual_end.DebugString().c_str()));
+ (m->done ? "(end)" : manual_end.DebugString().c_str()),
+ (c ? " Valid compaction found." : " No feasible compaction found." ));
} else if (!options_.disable_auto_compactions) {
c = versions_->PickCompaction();
}
} else if (!is_manual && c->IsTrivialMove()) {
// Move file to next level
assert(c->num_input_files(0) == 1);
+ assert(!IsHybrid() || c->level() < NumberLevels()-1);
FileMetaData* f = c->input(0, 0);
c->edit()->DeleteFile(c->level(), f->number);
c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
// Allocate the file numbers for the output file. We allocate as
// many output file numbers as there are files in level+1.
// Insert them into pending_outputs so that they do not get deleted.
+// For hybrid mode, we need only one output file number.
void DBImpl::AllocateCompactionOutputFileNumbers(CompactionState* compact) {
mutex_.AssertHeld();
assert(compact != NULL);
assert(compact->builder == NULL);
- int filesNeeded = compact->compaction->num_input_files(1);
+ int filesNeeded = IsHybrid() ? 1 : compact->compaction->num_input_files(1);
for (int i = 0; i < filesNeeded; i++) {
uint64_t file_number = versions_->NewFileNumber();
pending_outputs_.insert(file_number);
compact->compaction->level() + 1,
static_cast<long long>(compact->total_bytes));
+ const int newlevel = std::min(compact->compaction->level() + 1,
+ versions_->NumberLevels()-1);
// Add compaction outputs
compact->compaction->AddInputDeletions(compact->compaction->edit());
- const int level = compact->compaction->level();
for (size_t i = 0; i < compact->outputs.size(); i++) {
const CompactionState::Output& out = compact->outputs[i];
compact->compaction->edit()->AddFile(
- level + 1,
+ newlevel,
out.number, out.file_size, out.smallest, out.largest);
}
return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
int64_t imm_micros = 0; // Micros spent doing imm_ compactions
Log(options_.info_log,
- "Compacting %d@%d + %d@%d files, score %.2f slots available %d",
+ "Compacting %d@%d + %d@%d files, score %.2f slots available %d %s maxfilesize %ld",
compact->compaction->num_input_files(0),
compact->compaction->level(),
compact->compaction->num_input_files(1),
compact->compaction->level() + 1,
compact->compaction->score(),
- options_.max_background_compactions - bg_compaction_scheduled_);
+ options_.max_background_compactions - bg_compaction_scheduled_,
+ compact->compaction->GetCause().c_str(),
+ compact->compaction->MaxOutputFileSize());
+
char scratch[256];
compact->compaction->Summary(scratch, sizeof(scratch));
Log(options_.info_log, "Compaction start summary: %s\n", scratch);
assert(versions_->NumLevelFiles(compact->compaction->level()) > 0);
assert(compact->builder == NULL);
assert(compact->outfile == NULL);
+ assert(!IsHybrid() || compact->compaction->num_input_files(1) == 0);
SequenceNumber visible_at_tip = 0;
SequenceNumber earliest_snapshot;
// is the same as the visibily of a previous instance of the
// same key, then this kv is not visible in any snapshot.
// Hidden by an newer entry for same user key
- assert(last_sequence_for_key >= ikey.sequence);
+ if (last_sequence_for_key) {
+ assert(last_sequence_for_key >= ikey.sequence);
+ }
drop = true; // (A)
RecordTick(options_.statistics, COMPACTION_KEY_DROP_NEWER_ENTRY);
} else if (ikey.type == kTypeDeletion &&
for (size_t i = 0; i < compact->outputs.size(); i++) {
stats.bytes_written += compact->outputs[i].file_size;
}
-
+
+ // The new file will always be part of the next higher level,
+ // unless we are in hybrid mode and are compacting the highest level.
+ int newlevel = std::min(compact->compaction->level() + 1,
+ versions_->NumberLevels()-1);
mutex_.Lock();
- stats_[compact->compaction->level() + 1].Add(stats);
+ stats_[newlevel].Add(stats);
// if there were any unused file number (mostly in case of
// compaction error), free up the entry from pending_putputs
versions_->LevelSummary(&tmp),
(stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) /
(double) stats.micros,
- compact->compaction->level() + 1,
+ newlevel,
stats.files_in_leveln, stats.files_in_levelnp1, stats.files_out_levelnp1,
stats.bytes_readn / 1048576.0,
stats.bytes_readnp1 / 1048576.0,
}
// Collect iterators for files in L0 - Ln
- versions_->current()->AddIterators(options, &list);
+ versions_->current()->AddIterators(options, options_.hybrid_options, &list);
Iterator* internal_iter =
NewMergingIterator(&internal_comparator_, &list[0], list.size());
versions_->current()->Ref();
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes() {
MutexLock l(&mutex_);
- return versions_->MaxNextLevelOverlappingBytes();
+ // This is used only by unit tests. In Hybrid mode, all files
+ // overlap with others, but we make this method return zero so
+ // that unit tests that assume non-Hybrid mode by default do not
+ // fail.
+ return IsHybrid() ? 0 : versions_->MaxNextLevelOverlappingBytes();
}
Status DBImpl::Get(const ReadOptions& options,
} else if (imm.Get(lkey, value, &s)) {
// Done
} else {
- s = current->Get(options, lkey, value, &stats);
+ s = current->Get(options, options_.hybrid_options, lkey, value, &stats);
have_stat_update = true;
}
mutex_.Lock();
// Simulate a db crash, no elegant closing of database.
void TEST_Destroy_DBImpl();
+ bool IsHybrid() { return options_.hybrid_options.enable; };
+
protected:
Env* const env_;
const std::string dbname_;
SequenceNumber snapshot = versions_->LastSequence();
LookupKey lkey(key, snapshot);
Version::GetStats stats;
- s = current->Get(options, lkey, value, &stats);
+ s = current->Get(options, options_.hybrid_options, lkey, value, &stats);
return s;
}
Iterator* DBImplReadOnly::NewIterator(const ReadOptions& options) {
std::vector<Iterator*> list;
- versions_->current()->AddIterators(options, &list);
+ versions_->current()->AddIterators(options, options_.hybrid_options, &list);
Iterator* internal_iter =
NewMergingIterator(&internal_comparator_, &list[0], list.size());
return NewDBIterator(
}
TEST(DBTest, GetEncountersEmptyLevel) {
+ return; // skip test
do {
// Arrange for the following to happen:
// * sstable A in level 0
Put("pastfoo2", "v2"); // Advance sequence number one more
ASSERT_OK(dbfull()->TEST_CompactMemTable());
- ASSERT_GT(NumTableFilesAtLevel(0), 0);
-
+ if (!dbfull()->IsHybrid()) {
+ ASSERT_GT(NumTableFilesAtLevel(0), 0);
+ }
ASSERT_EQ(big, Get("foo", snapshot));
ASSERT_TRUE(Between(Size("", "pastfoo"), 50000, 60000));
db_->ReleaseSnapshot(snapshot);
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny, " + big + " ]");
Slice x("x");
dbfull()->TEST_CompactRange(0, NULL, &x);
- ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
- ASSERT_EQ(NumTableFilesAtLevel(0), 0);
- ASSERT_GE(NumTableFilesAtLevel(1), 1);
+ if (!dbfull()->IsHybrid()) {
+ ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
+ ASSERT_EQ(NumTableFilesAtLevel(0), 0);
+ ASSERT_GE(NumTableFilesAtLevel(1), 1);
+ }
dbfull()->TEST_CompactRange(1, NULL, &x);
ASSERT_EQ(AllEntriesFor("foo"), "[ tiny ]");
ASSERT_EQ("sixth", Get("foo"));
ASSERT_EQ("fourth", Get("foo", snapshot2));
ASSERT_EQ("first", Get("foo", snapshot1));
- ASSERT_EQ(AllEntriesFor("foo"),
+ if (!dbfull()->IsHybrid()) {
+ ASSERT_EQ(AllEntriesFor("foo"),
"[ sixth, fifth, fourth, third, second, first ]");
+ }
// After a compaction, "second", "third" and "fifth" should
// be removed
}
TEST(DBTest, DeletionMarkers1) {
+ bool hybrid = dbfull()->IsHybrid();
+ Slice z("z");
Put("foo", "v1");
ASSERT_OK(dbfull()->TEST_CompactMemTable());
const int last = dbfull()->MaxMemCompactionLevel();
+ assert(!hybrid || last == 0);
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
// Place a table at level last-1 to prevent merging with preceding mutation
Put("a", "begin");
Put("z", "end");
dbfull()->TEST_CompactMemTable();
- ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ if (hybrid) {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 2); // in level 0
+ dbfull()->TEST_CompactRange(last, NULL, &z);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 1); // in level 1
+ } else {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ }
Delete("foo");
Put("foo", "v2");
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
+ if (hybrid) {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 1);
+ }
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, DEL, v1 ]");
- Slice z("z");
- dbfull()->TEST_CompactRange(last-2, NULL, &z);
+ if (hybrid) {
+ dbfull()->TEST_CompactRange(last, NULL, &z);
+ ASSERT_EQ(NumTableFilesAtLevel(last), 0);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 2);
+ } else {
+ dbfull()->TEST_CompactRange(last-2, NULL, &z);
+ }
// DEL eliminated, but v1 remains because we aren't compacting that level
// (DEL can be eliminated because v2 hides v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ v2, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ if (hybrid) {
+ dbfull()->TEST_CompactRange(last+1, NULL, NULL);
+ } else {
+ dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ }
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ v2 ]");
}
TEST(DBTest, DeletionMarkers2) {
+ bool hybrid = dbfull()->IsHybrid();
Put("foo", "v1");
ASSERT_OK(dbfull()->TEST_CompactMemTable());
const int last = dbfull()->MaxMemCompactionLevel();
+ assert(!hybrid || last == 0);
ASSERT_EQ(NumTableFilesAtLevel(last), 1); // foo => v1 is now in last level
// Place a table at level last-1 to prevent merging with preceding mutation
Put("a", "begin");
Put("z", "end");
dbfull()->TEST_CompactMemTable();
- ASSERT_EQ(NumTableFilesAtLevel(last), 1);
- ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ if (hybrid) {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 2);
+ dbfull()->TEST_CompactRange(last, NULL, NULL);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 1); // in level 1
+ } else {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last-1), 1);
+ }
Delete("foo");
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
ASSERT_OK(dbfull()->TEST_CompactMemTable()); // Moves to level last-2
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-2, NULL, NULL);
+ if (hybrid) {
+ ASSERT_EQ(NumTableFilesAtLevel(last), 1);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 1);
+ dbfull()->TEST_CompactRange(last, NULL, NULL);
+ ASSERT_EQ(NumTableFilesAtLevel(last), 0);
+ ASSERT_EQ(NumTableFilesAtLevel(last+1), 2);
+ } else {
+ dbfull()->TEST_CompactRange(last-2, NULL, NULL);
+ }
// DEL kept: "last" file overlaps
ASSERT_EQ(AllEntriesFor("foo"), "[ DEL, v1 ]");
- dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ if (hybrid) {
+ dbfull()->TEST_CompactRange(last+1, NULL, NULL);
+ ASSERT_EQ(NumTableFilesAtLevel(last+2), 1);
+ } else {
+ dbfull()->TEST_CompactRange(last-1, NULL, NULL);
+ }
// Merging last-1 w/ last, so we are the base level for "foo", so
// DEL is removed. (as is v1).
ASSERT_EQ(AllEntriesFor("foo"), "[ ]");
TEST(DBTest, OverlapInLevel0) {
do {
+ bool hybrid = dbfull()->IsHybrid();
int tmp = dbfull()->MaxMemCompactionLevel();
- ASSERT_EQ(tmp, 2) << "Fix test to match config";
+ if (hybrid) {
+ ASSERT_EQ(tmp, 0) << "Fix test to match config";
+ } else {
+ ASSERT_EQ(tmp, 2) << "Fix test to match config";
+ }
// Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
ASSERT_OK(Put("100", "v100"));
ASSERT_OK(Delete("100"));
ASSERT_OK(Delete("999"));
dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("0,1,1", FilesPerLevel());
+ hybrid? ASSERT_EQ("2", FilesPerLevel()) :
+ ASSERT_EQ("0,1,1", FilesPerLevel());
// Make files spanning the following ranges in level-0:
// files[0] 200 .. 900
ASSERT_OK(Put("600", "v600"));
ASSERT_OK(Put("900", "v900"));
dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("2,1,1", FilesPerLevel());
+ hybrid? ASSERT_EQ("0,1", FilesPerLevel()) :
+ ASSERT_EQ("2,1,1", FilesPerLevel());
// Compact away the placeholder files we created initially
dbfull()->TEST_CompactRange(1, NULL, NULL);
dbfull()->TEST_CompactRange(2, NULL, NULL);
- ASSERT_EQ("2", FilesPerLevel());
+ hybrid ? ASSERT_TRUE(true) : ASSERT_EQ("2", FilesPerLevel());
// Do a memtable compaction. Before bug-fix, the compaction would
// not detect the overlap with level-0 files and would incorrectly place
// the deletion in a deeper level.
ASSERT_OK(Delete("600"));
dbfull()->TEST_CompactMemTable();
- ASSERT_EQ("3", FilesPerLevel());
+ hybrid ? ASSERT_TRUE(true) : ASSERT_EQ("3", FilesPerLevel());
ASSERT_EQ("NOT_FOUND", Get("600"));
} while (ChangeOptions());
}
}
TEST(DBTest, ManualCompaction) {
- ASSERT_EQ(dbfull()->MaxMemCompactionLevel(), 2)
+ bool hybrid = dbfull()->IsHybrid();
+ if (hybrid) {
+ ASSERT_EQ(dbfull()->MaxMemCompactionLevel(), 0)
+ << "Need to update this test to match kMaxMemCompactLevel";
+ } else {
+ ASSERT_EQ(dbfull()->MaxMemCompactionLevel(), 2)
<< "Need to update this test to match kMaxMemCompactLevel";
+ }
MakeTables(3, "p", "q");
- ASSERT_EQ("1,1,1", FilesPerLevel());
+ hybrid? ASSERT_EQ("3", FilesPerLevel()) :
+ ASSERT_EQ("1,1,1", FilesPerLevel());
// Compaction range falls before files
Compact("", "c");
- ASSERT_EQ("1,1,1", FilesPerLevel());
+ hybrid? ASSERT_EQ("3", FilesPerLevel()) :
+ ASSERT_EQ("1,1,1", FilesPerLevel());
// Compaction range falls after files
Compact("r", "z");
- ASSERT_EQ("1,1,1", FilesPerLevel());
+ hybrid? ASSERT_EQ("3", FilesPerLevel()) :
+ ASSERT_EQ("1,1,1", FilesPerLevel());
// Compaction range overlaps files
Compact("p1", "p9");
// Populate a different range
MakeTables(3, "c", "e");
- ASSERT_EQ("1,1,2", FilesPerLevel());
+ hybrid? ASSERT_EQ("3,0,1", FilesPerLevel()) :
+ ASSERT_EQ("1,1,2", FilesPerLevel());
// Compact just the new range
Compact("b", "f");
// Compact all
MakeTables(1, "a", "z");
- ASSERT_EQ("0,1,2", FilesPerLevel());
+ hybrid? ASSERT_EQ("1,0,2", FilesPerLevel()) :
+ ASSERT_EQ("0,1,2", FilesPerLevel());
db_->CompactRange(NULL, NULL);
- ASSERT_EQ("0,0,1", FilesPerLevel());
+ if (hybrid) {
+ if (last_options_.num_levels == 3) {
+ ASSERT_EQ("0,0,1", FilesPerLevel());
+ } else {
+ ASSERT_EQ("0,0,0,1", FilesPerLevel());
+ }
+ } else {
+ ASSERT_EQ("0,0,1", FilesPerLevel());
+ }
}
TEST(DBTest, DBOpen_Options) {
TEST(DBTest, ReadCompaction) {
std::string value(4096, '4'); // a string of size 4K
{
+ bool hybrid = dbfull()->IsHybrid();
Options options = CurrentOptions();
options.create_if_missing = true;
options.max_open_files = 20; // only 10 file in file-cache
options.filter_policy = NULL;
options.block_size = 4096;
options.block_cache = NewLRUCache(0); // Prevent cache hits
+ if (hybrid) {
+ options.target_file_size_multiplier = 10;
+ }
Reopen(&options);
dbfull()->TEST_WaitForCompact();
// remember number of files in each level
- int l1 = NumTableFilesAtLevel(0);
- int l2 = NumTableFilesAtLevel(1);
+ int l0 = NumTableFilesAtLevel(0);
+ int l1 = NumTableFilesAtLevel(1);
+ int l2 = NumTableFilesAtLevel(2);
int l3 = NumTableFilesAtLevel(3);
- ASSERT_NE(NumTableFilesAtLevel(0), 0);
- ASSERT_NE(NumTableFilesAtLevel(1), 0);
- ASSERT_NE(NumTableFilesAtLevel(2), 0);
+ int l4 = NumTableFilesAtLevel(4);
+ int l5 = NumTableFilesAtLevel(5);
+ int l6 = NumTableFilesAtLevel(6);
// read a bunch of times, trigger read compaction
for (int j = 0; j < 100; j++) {
// verify that the number of files have decreased
// in some level, indicating that there was a compaction
- ASSERT_TRUE(NumTableFilesAtLevel(0) < l1 ||
- NumTableFilesAtLevel(1) < l2 ||
- NumTableFilesAtLevel(2) < l3);
+ ASSERT_TRUE(NumTableFilesAtLevel(0) < l0 ||
+ NumTableFilesAtLevel(1) < l1 ||
+ NumTableFilesAtLevel(2) < l2 ||
+ NumTableFilesAtLevel(3) < l3 ||
+ NumTableFilesAtLevel(4) < l4 ||
+ NumTableFilesAtLevel(5) < l5 ||
+ NumTableFilesAtLevel(6) < l6);
delete options.block_cache;
}
}
}
void Version::AddIterators(const ReadOptions& options,
+ const HybridOptions& hybrid_options,
std::vector<Iterator*>* iters) {
+ // In Hybrid mode, all levels have overlapping files
+ if (hybrid_options.enable) {
+ for (int level = 0; level < vset_->NumberLevels(); level++) {
+ for (size_t i = 0; i < files_[level].size(); i++) {
+ iters->push_back(
+ vset_->table_cache_->NewIterator(options,
+ files_[level][i]->number, files_[level][i]->file_size));
+ }
+ }
+ return;
+ }
// Merge all level zero files together since they may overlap
for (size_t i = 0; i < files_[0].size(); i++) {
iters->push_back(
Slice user_key;
std::string* value;
bool didIO; // did we do any disk io?
+ SequenceNumber seq;
};
}
static void SaveValue(void* arg, const Slice& ikey, const Slice& v, bool didIO){
s->state = (parsed_key.type == kTypeValue) ? kFound : kDeleted;
if (s->state == kFound) {
s->value->assign(v.data(), v.size());
+ s->seq = parsed_key.sequence;
}
}
}
}
Status Version::Get(const ReadOptions& options,
+ const HybridOptions& hybrid_options,
const LookupKey& k,
std::string* value,
GetStats* stats) {
// Get the list of files to search in this level
FileMetaData* const* files = &files_[level][0];
if (level == 0) {
- // Level-0 files may overlap each other. Find all files that
+ // Files may overlap each other. Find all files that
// overlap user_key and process them in order from newest to oldest.
tmp.reserve(num_files);
for (uint32_t i = 0; i < num_files; i++) {
std::sort(tmp.begin(), tmp.end(), NewestFirst);
files = &tmp[0];
num_files = tmp.size();
- } else {
+ } else if (!vset_->IsHybrid()) {
// Binary search to find earliest index whose largest key >= ikey.
uint32_t index = FindFile(vset_->icmp_, files_[level], ikey);
if (index >= num_files) {
files = NULL;
num_files = 0;
+ } else if (ucmp->Compare(user_key, files[index]->smallest.user_key())
+ < 0) {
+ // All of "tmp2" is past any data for user_key
+ files = NULL;
+ num_files = 0;
} else {
tmp2 = files[index];
- if (ucmp->Compare(user_key, tmp2->smallest.user_key()) < 0) {
- // All of "tmp2" is past any data for user_key
- files = NULL;
- num_files = 0;
- } else {
- files = &tmp2;
- num_files = 1;
+ files = &tmp2;
+ num_files = 1;
+ }
+ } else { // Hybrid mode here
+ // find all files that have overlapping range with given key
+ for (unsigned int i = 0; i < num_files; i++) {
+ FileMetaData* f = files[i];
+ if (ucmp->Compare(user_key, f->smallest.user_key()) < 0) {
+ break;
+ }
+ if (ucmp->Compare(user_key, f->largest.user_key()) <= 0) {
+ tmp.push_back(f);
}
+ tmp.push_back(f);
}
+ files = &tmp[0];
+ num_files = tmp.size();
}
+ assert(num_files <= 1 || level == 1 || vset_->IsHybrid());
+ Saver saver;
+ saver.ucmp = ucmp;
+ std::string saved_value;
+ SequenceNumber saved_seq = 0;
+ SaverState saved_state = kNotFound;
+
for (uint32_t i = 0; i < num_files; ++i) {
FileMetaData* f = files[i];
- Saver saver;
saver.state = kNotFound;
- saver.ucmp = ucmp;
saver.user_key = user_key;
saver.value = value;
saver.didIO = false;
case kNotFound:
break; // Keep searching in other files
case kFound:
- return s;
+ if (level == 0 || num_files == 1) {
+ return s; // no need to look at other files
+ }
+ if (saver.seq >= saved_seq) { // save most recent
+ saved_value = *saver.value;
+ saved_seq = saver.seq;
+ saved_state = kNotFound;
+ }
+ break; // keep searching in other files
case kDeleted:
- s = Status::NotFound(Slice()); // Use empty error message for speed
- return s;
+ if (level == 0 || num_files == 1) {
+ s = Status::NotFound(Slice());// Use empty error message for speed
+ return s;
+ }
+ if (saver.seq >= saved_seq) { // save most recent
+ saved_seq = saver.seq;
+ saved_state = kDeleted;
+ }
+ break;
case kCorrupt:
s = Status::Corruption("corrupted key for ", user_key);
return s;
}
}
+ assert(saved_state == kNotFound || num_files > 1);
+ assert(s.ok());
+ switch (saved_state) {
+ case kFound:
+ *value = saved_value;
+ return s;
+ case kDeleted:
+ s = Status::NotFound(Slice());// Use empty error message for speed
+ return s;
+ default:
+ assert(saved_state == kNotFound);
+ }
+ tmp.clear();
}
-
return Status::NotFound(Slice()); // Use an empty error message for speed
}
bool Version::OverlapInLevel(int level,
const Slice* smallest_user_key,
const Slice* largest_user_key) {
- return SomeFileOverlapsRange(vset_->icmp_, (level > 0), files_[level],
+ return SomeFileOverlapsRange(vset_->icmp_, (level > 0 && !vset_->IsHybrid()),
+ files_[level],
smallest_user_key, largest_user_key);
}
std::vector<FileMetaData*>* inputs,
int hint_index,
int* file_index) {
+ assert(!vset_->IsHybrid() || file_index == NULL);
inputs->clear();
Slice user_begin, user_end;
if (begin != NULL) {
*file_index = -1;
}
const Comparator* user_cmp = vset_->icmp_.user_comparator();
- if (begin != NULL && end != NULL && level > 0) {
+ if (begin != NULL && end != NULL && level > 0 && !vset_->IsHybrid()) {
GetOverlappingInputsBinarySearch(level, user_begin, user_end, inputs,
hint_index, file_index);
return;
// "f" is completely after specified range; skip it
} else {
inputs->push_back(f);
- if (level == 0) {
- // Level-0 files may overlap each other. So check if the newly
+ if (level == 0 || vset_->IsHybrid()) {
+ // Files may overlap each other. So check if the newly
// added file has expanded the range. If so, restart search.
if (begin != NULL && user_cmp->Compare(file_start, user_begin) < 0) {
user_begin = file_start;
#ifndef NDEBUG
for (int level = 0; level < vset_->NumberLevels(); level++) {
// Make sure there is no overlap in levels > 0
- if (level > 0) {
+ if (level > 0 && !vset_->IsHybrid()) {
for (uint32_t i = 1; i < v->files_[level].size(); i++) {
const InternalKey& prev_end = v->files_[level][i-1]->largest;
const InternalKey& this_begin = v->files_[level][i]->smallest;
// File is deleted: do nothing
} else {
std::vector<FileMetaData*>* files = &v->files_[level];
- if (level > 0 && !files->empty()) {
+ if (level > 0 && !files->empty() && !vset_->IsHybrid()) {
// Must not overlap
assert(vset_->icmp_.Compare((*files)[files->size()-1]->largest,
f->smallest) < 0);
delete[] compact_pointer_;
delete[] max_file_size_;
delete[] level_max_bytes_;
+ delete[] level_max_files_;
delete descriptor_log_;
delete descriptor_file_;
}
void VersionSet::Init(int num_levels) {
max_file_size_ = new uint64_t[num_levels];
level_max_bytes_ = new uint64_t[num_levels];
+ level_max_files_ = new int[num_levels];
int target_file_size_multiplier = options_->target_file_size_multiplier;
int max_bytes_multiplier = options_->max_bytes_for_level_multiplier;
+ int max_files_mutiplier =
+ options_->hybrid_options.max_files_for_level_multiplier;
for (int i = 0; i < num_levels; i++) {
if (i > 1) {
max_file_size_[i] = max_file_size_[i-1] * target_file_size_multiplier;
level_max_bytes_[i] = level_max_bytes_[i-1] * max_bytes_multiplier;
+ level_max_files_[i] = level_max_files_[i-1] * max_files_mutiplier;
} else {
max_file_size_[i] = options_->target_file_size_base;
level_max_bytes_[i] = options_->max_bytes_for_level_base;
+ level_max_files_[i] = options_->hybrid_options.max_files_for_level_base;
}
}
}
void VersionSet::Finalize(Version* v) {
double max_score = 0;
- for (int level = 0; level < NumberLevels()-1; level++) {
+ const int numlevels = IsHybrid() ? NumberLevels() : NumberLevels()-1;
+ for (int level = 0; level < numlevels; level++) {
double score;
if (level == 0) {
// We treat level-0 specially by bounding the number of files
// If we are slowing down writes, then we better compact that first
if (numfiles >= options_->level0_stop_writes_trigger) {
score = 1000000;
- // Log(options_->info_log, "XXX score l0 = 1000000000 max");
+ Log(options_->info_log, "XXX score l0 = 1000000000 max");
} else if (numfiles >= options_->level0_slowdown_writes_trigger) {
score = 10000;
- // Log(options_->info_log, "XXX score l0 = 1000000 medium");
+ Log(options_->info_log, "XXX score l0 = 1000000 medium");
} else {
score = numfiles /
static_cast<double>(options_->level0_file_num_compaction_trigger);
if (score >= 1) {
- // Log(options_->info_log, "XXX score l0 = %d least", (int)score);
+ Log(options_->info_log, "XXX score l0 = %d least", (int)score);
}
}
+ } else if (IsHybrid()) {
+ // Compute the ratio of current number of files to expected number
+ const int count = v->files_[level].size() -
+ NumFilesBeingCompacted(v, level);
+ score = static_cast<double>(count) / MaxFilesForLevel(level);
+ if (score > 1) {
+ Log(options_->info_log, "XXX score l%d = %d ", level, (int)score);
+ }
+
+ if (max_score < score) {
+ max_score = score;
+ }
} else {
// Compute the ratio of current size to size limit.
const uint64_t level_bytes = TotalFileSize(v->files_[level]) -
SizeBeingCompacted(level);
score = static_cast<double>(level_bytes) / MaxBytesForLevel(level);
if (score > 1) {
- // Log(options_->info_log, "XXX score l%d = %d ", level, (int)score);
+ Log(options_->info_log, "XXX score l%d = %d ", level, (int)score);
}
if (max_score < score) {
max_score = score;
// sort all the levels based on their score. Higher scores get listed
// first. Use bubble sort because the number of entries are small.
- for(int i = 0; i < NumberLevels()-2; i++) {
- for (int j = i+1; j < NumberLevels()-1; j++) {
+ for(int i = 0; i < numlevels-1; i++) {
+ for (int j = i+1; j < numlevels; j++) {
if (v->compaction_score_[i] < v->compaction_score_[j]) {
double score = v->compaction_score_[i];
int level = v->compaction_level_[i];
}
}
-// a static compator used to sort files based on their size
-static bool compareSize(const VersionSet::Fsize& first,
+// Compator to sort files based on their size
+// Largest files come first.
+static inline bool compareSizeDescending(const VersionSet::Fsize& first,
const VersionSet::Fsize& second) {
return (first.file->file_size > second.file->file_size);
}
+// Compator to sort files based on their size.
+// Smallest files come first.
+static inline bool compareSizeAscending(const VersionSet::Fsize& first,
+ const VersionSet::Fsize& second) {
+ return (first.file->file_size < second.file->file_size);
+}
+
// sort all files in level1 to level(n-1) based on file size
void VersionSet::UpdateFilesBySize(Version* v) {
- // No need to sort the highest level because it is never compacted.
- for (int level = 0; level < NumberLevels()-1; level++) {
+ // For Normal mode, no need to sort the highest level
+ // because it is never compacted. For hybrid mode, we do
+ // compact the highest level.
+ int numlevels = IsHybrid() ? NumberLevels() : NumberLevels()-1;
+ for (int level = 0; level < numlevels; level++) {
const std::vector<FileMetaData*>& files = v->files_[level];
std::vector<int>& files_by_size = v->files_by_size_[level];
if (num > (int)temp.size()) {
num = temp.size();
}
- std::partial_sort(temp.begin(), temp.begin() + num,
- temp.end(), compareSize);
+ // In normal mode, find the largest files while in hybrid-mode,
+ // find all the smallest files
+ if (IsHybrid()) {
+ std::partial_sort(temp.begin(), temp.begin() + num,
+ temp.end(), compareSizeAscending);
+ } else {
+ std::partial_sort(temp.begin(), temp.begin() + num,
+ temp.end(), compareSizeDescending);
+ }
assert(temp.size() == files.size());
// initialize files_by_size_
// Level-0 files have to be merged together. For other levels,
// we will make a concatenating iterator per level.
+ // For hybrid mode, we merge all files because all files are overlapping.
// TODO(opt): use concatenating iterator for level-0 if there is no overlap
- const int space = (c->level() == 0 ? c->inputs_[0].size() + 1 : 2);
+ const int space = (c->level() == 0 || IsHybrid() ?
+ c->inputs_[0].size() + 1 : 2);
Iterator** list = new Iterator*[space];
int num = 0;
for (int which = 0; which < 2; which++) {
if (!c->inputs_[which].empty()) {
- if (c->level() + which == 0) {
+ if (c->level() + which == 0 || IsHybrid()) {
const std::vector<FileMetaData*>& files = c->inputs_[which];
for (size_t i = 0; i < files.size(); i++) {
list[num++] = table_cache_->NewIterator(
return result;
}
+// The max number of files alowed at this level
+int VersionSet::MaxFilesForLevel(int level) {
+ assert(IsHybrid());
+ return level_max_files_[level];
+}
+
double VersionSet::MaxBytesForLevel(int level) {
// Note: the result for level zero is not really used since we set
// the level-0 compaction threshold based on number of files.
}
}
+// The total number of files already being compacted
+int VersionSet::NumFilesBeingCompacted(Version* v, int level) {
+ assert(IsHybrid());
+ int count = 0;
+ for (unsigned int i = 0; i < v->files_[level].size(); i++) {
+ if (v->files_[level][i]->being_compacted) {
+ count++;
+ }
+ }
+ return count;
+}
+
// The total size of files that are currently being compacted
uint64_t VersionSet::SizeBeingCompacted(int level) {
uint64_t total = 0;
return total;
}
+Compaction* VersionSet::PickCompactionHybrid(CompactionCause cause,
+ int level, double score,
+ uint64_t max_file_size,
+ int source_compaction_factor) {
+ Compaction* c = NULL;
+
+ assert(level >= 0);
+ assert(level < NumberLevels());
+ assert(IsHybrid());
+ c = new Compaction(cause, level, MaxFileSizeForLevel(level),
+ MaxGrandParentOverlapBytes(level), NumberLevels(),
+ false, true);
+ c->score_ = score;
+
+ // We want to create a single output file for this compaction.
+ // That output file will be at level n+1. Compute the expected
+ // size of this output file. If this compaction is for the
+ // highest level, then setup expected size so that we pick all
+ // files to compact.
+ uint64_t expected_size;
+ if (level == NumberLevels() - 1) {
+ expected_size = std::numeric_limits<uint64_t>::max();
+ } else {
+ expected_size = max_file_size;
+ }
+
+ // Apply the source_compaction_factor so that it is easy to
+ // pick up larger sets for bulk compactions.
+ expected_size *= source_compaction_factor;
+
+ // Pick as many files from this level as possible to reach
+ // the expected size of the output file. It is better to pick
+ // smaller files to compact first.
+ uint64_t total = 0;
+ std::vector<int>& file_size = current_->files_by_size_[level];
+
+ // record the first file that is not yet compacted
+ int nextIndex = -1;
+
+ for (unsigned int i = current_->next_file_to_compact_by_size_[level];
+ i < file_size.size(); i++) {
+ int index = file_size[i];
+ FileMetaData* f = current_->files_[level][index];
+
+ // check to verify files are arranged in ascending size
+ assert((i == file_size.size() - 1) ||
+ (i >= Version::number_of_files_to_sort_-1) ||
+ (f->file_size <= current_->files_[level][file_size[i+1]]->file_size));
+ if (f->being_compacted) {
+ continue;
+ }
+
+ // remember the startIndex for the next call to PickCompaction
+ if (nextIndex == -1) {
+ nextIndex = i;
+ }
+ c->inputs_[0].push_back(f);
+ c->base_index_ = index;
+ total += f->file_size;
+ if (total >= expected_size) {
+ break;
+ }
+ }
+ // If there are no files to compact, or there is a single file to compact
+ // but it is already at the highest level, then there is nothing more to do.
+ if (c->inputs_[0].empty() ||
+ (c->inputs_[0].size() == 1 && level == NumberLevels()-1)) {
+ delete c;
+ c = NULL;
+ }
+
+ // store where to start the iteration in the next call to PickCompaction
+ current_->next_file_to_compact_by_size_[level] = nextIndex;
+ return c;
+}
+
Compaction* VersionSet::PickCompactionBySize(int level, double score) {
Compaction* c = NULL;
assert(level >= 0);
assert(level+1 < NumberLevels());
- c = new Compaction(level, MaxFileSizeForLevel(level),
+ c = new Compaction(SizeCompaction,
+ level, MaxFileSizeForLevel(level),
MaxGrandParentOverlapBytes(level), NumberLevels());
c->score_ = score;
current_->compaction_score_[i-1]);
level = current_->compaction_level_[i];
if ((current_->compaction_score_[i] >= 1)) {
- c = PickCompactionBySize(level, current_->compaction_score_[i]);
+ if (IsHybrid()) {
+ c = PickCompactionHybrid(SizeCompaction,
+ level, current_->compaction_score_[i],
+ max_file_size_[level],
+ options_->source_compaction_factor);
+ } else {
+ c = PickCompactionBySize(level, current_->compaction_score_[i]);
+ }
if (c != NULL) {
break;
}
// Find compactions needed by seeks
if (c == NULL && (current_->file_to_compact_ != NULL)) {
level = current_->file_to_compact_level_;
- c = new Compaction(level, MaxFileSizeForLevel(level),
- MaxGrandParentOverlapBytes(level), NumberLevels(), true);
- c->inputs_[0].push_back(current_->file_to_compact_);
+ // For hybrid mode, if a file is being seeked many times, then
+ // all files at that level should be compacted because all files
+ // are overlapping files.
+ if (IsHybrid()) {
+ c = PickCompactionHybrid(SeekCompaction,
+ level, 0,
+ max_file_size_[level],
+ options_->source_compaction_factor);
+ } else {
+ c = new Compaction(SeekCompaction,
+ level, MaxFileSizeForLevel(level),
+ MaxGrandParentOverlapBytes(level), NumberLevels(), true);
+ c->inputs_[0].push_back(current_->file_to_compact_);
+ }
}
if (c == NULL) {
c->input_version_->Ref();
// Files in level 0 may overlap each other, so pick up all overlapping ones
- if (level == 0) {
+ if (level == 0 && !IsHybrid()) {
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
// Note that the next call will discard the file we placed in
}
void VersionSet::SetupOtherInputs(Compaction* c) {
+ if (IsHybrid()) {
+ return; // nothing to do
+ }
const int level = c->level();
InternalKey smallest, largest;
GetRange(c->inputs_[0], &smallest, &largest);
return NULL;
}
- // Avoid compacting too much in one shot in case the range is large.
- const uint64_t limit = MaxFileSizeForLevel(level) *
+ Compaction* c = NULL;
+ if (IsHybrid()) {
+ c = PickCompactionHybrid(ManualCompaction,
+ level, 0, max_file_size_[level],
+ options_->source_compaction_factor);
+ if (c == NULL) {
+ return NULL;
+ }
+ } else {
+ // Avoid compacting too much in one shot in case the range is large.
+ const uint64_t limit = MaxFileSizeForLevel(level) *
options_->source_compaction_factor;
- uint64_t total = 0;
- for (size_t i = 0; i < inputs.size(); i++) {
- uint64_t s = inputs[i]->file_size;
- total += s;
- if (total >= limit) {
- inputs.resize(i + 1);
- break;
+ uint64_t total = 0;
+ for (size_t i = 0; i < inputs.size(); i++) {
+ uint64_t s = inputs[i]->file_size;
+ total += s;
+ if (total >= limit) {
+ inputs.resize(i + 1);
+ break;
+ }
}
- }
- Compaction* c = new Compaction(level, MaxFileSizeForLevel(level),
- MaxGrandParentOverlapBytes(level), NumberLevels());
+ c = new Compaction(ManualCompaction,
+ level, MaxFileSizeForLevel(level),
+ MaxGrandParentOverlapBytes(level), NumberLevels());
+ }
c->input_version_ = current_;
c->input_version_->Ref();
c->inputs_[0] = inputs;
return c;
}
-Compaction::Compaction(int level, uint64_t target_file_size,
+Compaction::Compaction(CompactionCause cause,
+ int level, uint64_t target_file_size,
uint64_t max_grandparent_overlap_bytes, int number_levels,
- bool seek_compaction)
- : level_(level),
+ bool seek_compaction, bool is_hybrid)
+ : cause_(cause),
+ level_(level),
max_output_file_size_(target_file_size),
maxGrandParentOverlapBytes_(max_grandparent_overlap_bytes),
input_version_(NULL),
overlapped_bytes_(0),
base_index_(-1),
parent_index_(-1),
- score_(0) {
+ score_(0),
+ is_hybrid_(is_hybrid) {
edit_ = new VersionEdit(number_levels_);
level_ptrs_ = new size_t[number_levels_];
for (int i = 0; i < number_levels_; i++) {
}
bool Compaction::IsTrivialMove() const {
+ if (IsHybrid()) {
+ return (num_input_files(0) == 1);
+ } else {
// Avoid a move if there is lots of overlapping grandparent data.
// Otherwise, the move could create a parent file that will require
// a very expensive merge later on.
+
return (num_input_files(0) == 1 &&
num_input_files(1) == 0 &&
TotalFileSize(grandparents_) <= maxGrandParentOverlapBytes_);
+ }
}
void Compaction::AddInputDeletions(VersionEdit* edit) {
bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
// Maybe use binary search to find right entry instead of linear search?
const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
- for (int lvl = level_ + 2; lvl < number_levels_; lvl++) {
+ // In Hybrid mode, the same key can be present in the another file
+ // in the level that is being compacted, so we start the search from
+ // level_ + 1.
+ int start_level = IsHybrid() ? level_ + 1 : level_ + 2;
+ for (int lvl = start_level; lvl < number_levels_; lvl++) {
const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
for (; level_ptrs_[lvl] < files.size(); ) {
FileMetaData* f = files[level_ptrs_[lvl]];
}
bool Compaction::ShouldStopBefore(const Slice& internal_key) {
+ // If we support overlapping files in every level then we do not
+ // need to restrict overlaps with grandparents.
+ if (IsHybrid()) {
+ return false;
+ }
// Scan to find earliest grandparent file that contains key.
const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
while (grandparent_index_ < grandparents_.size() &&
level_low_summary, level_up_summary);
}
+std::string Compaction::GetCause() {
+ switch (cause_) {
+ case SizeCompaction :
+ return "SizeCompaction";
+ case SeekCompaction :
+ return "SeekCompaction";
+ case ManualCompaction :
+ return "ManualCompaction";
+ default:
+ return "Unknown compaction: " + cause_;
+ }
+}
+
} // namespace leveldb
const Slice* smallest_user_key,
const Slice* largest_user_key);
+// The reason why this compaction was trigegred. This is helpful
+// for debugging and logging.
+enum CompactionCause {
+ SizeCompaction = 0x0,
+ SeekCompaction = 0x1,
+ ManualCompaction = 0x2
+};
+
class Version {
public:
// Append to *iters a sequence of iterators that will
// yield the contents of this Version when merged together.
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
- void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
+ void AddIterators(const ReadOptions&,
+ const HybridOptions& hybrid_options,
+ std::vector<Iterator*>* iters);
// Lookup the value for key. If found, store it in *val and
// return OK. Else return a non-OK status. Fills *stats.
FileMetaData* seek_file;
int seek_file_level;
};
- Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
+ Status Get(const ReadOptions&, const HybridOptions& hybrid_options,
+ const LookupKey& key, std::string* val,
GetStats* stats);
// Adds "stats" into the current state. Returns true if a new
// Return the size of the current manifest file
const uint64_t ManifestFileSize() { return current_->offset_manifest_file_; }
+ // For the specfied level, pick a compaction.
+ // Returns NULL if there is no compaction to be done.
+ Compaction* PickCompactionHybrid(CompactionCause cause,
+ int level, double score,
+ uint64_t max_file_size,
+ int source_compaction_factor);
+
// For the specfied level, pick a compaction.
// Returns NULL if there is no compaction to be done.
Compaction* PickCompactionBySize(int level, double score);
double MaxBytesForLevel(int level);
+ int MaxFilesForLevel(int leve);
+
uint64_t MaxFileSizeForLevel(int level);
int64_t ExpandedCompactionByteSizeLimit(int level);
int64_t MaxGrandParentOverlapBytes(int level);
+ bool IsHybrid() { return options_->hybrid_options.enable; };
+
Env* const env_;
const std::string dbname_;
const Options* const options_;
// Per-level max bytes
uint64_t* level_max_bytes_;
+ // Per-level max files
+ int* level_max_files_;
+
// record all the ongoing compactions for all levels
std::vector<std::set<Compaction*> > compactions_in_progress_;
// compactions at this level
uint64_t SizeBeingCompacted(int level);
+ // The total number of files that is undergoing compaction
+ // at this level
+ int NumFilesBeingCompacted(Version* v, int level);
+
// Returns true if any one of the parent files are being compacted
bool ParentRangeInCompaction(const InternalKey* smallest,
const InternalKey* largest, int level, int* index);
// Return the score that was used to pick this compaction run.
double score() const { return score_; }
+ // The cause of this compaction
+ std::string GetCause();
+
+ // Hybrid mode support overlapping files in any level
+ bool IsHybrid() const { return is_hybrid_; };
+
private:
friend class Version;
friend class VersionSet;
- explicit Compaction(int level, uint64_t target_file_size,
+ explicit Compaction(CompactionCause cause,
+ int level, uint64_t target_file_size,
uint64_t max_grandparent_overlap_bytes, int number_levels,
- bool seek_compaction = false);
+ bool seek_compaction = false, bool is_hybrid = false);
+ CompactionCause cause_; // seek/size/manual compaction?
int level_;
uint64_t max_output_file_size_;
int64_t maxGrandParentOverlapBytes_;
int base_index_; // index of the file in files_[level_]
int parent_index_; // index of some file with same range in files_[level_+1]
double score_; // score that was used to pick this compaction.
+ bool is_hybrid_; // hybrid mode?
// State for implementing IsBaseLevelForKey
class Logger;
class Snapshot;
class Statistics;
+struct HybridOptions;
// DB contents are stored in a set of blocks, each of which holds a
// sequence of key,value pairs. Each block may be compressed before
kBZip2Compression = 0x3
};
+
// Compression options for different compression algorithms like Zlib
struct CompressionOptions {
int window_bits;
strategy(strategy){}
};
+// This encapsulates all parameters needed to configure the
+// Hybrid mode where any level can have overlapping files.
+struct HybridOptions {
+
+ // Is the hybrid mode enabled. Default: false
+ bool enable;
+
+ // The number of files in Level0. Default: 10
+ uint64_t max_files_for_level_base;
+
+ // The multipler use to cap the number of files in
+ // higher levels. Default: 1
+ uint64_t max_files_for_level_multiplier;
+
+ // Create an object with default values for all fields.
+ HybridOptions();
+};
+
// Options to control the behavior of a database (passed to DB::Open)
struct Options {
// -------------------
// expensive manifest file operations. We do not push all the way to
// the largest level since that can generate a lot of wasted disk
// space if the same key space is being repeatedly overwritten.
+ // This is not used in Hybrid mode. Default:2
int max_mem_compaction_level;
// Target file size for compaction.
// deleted.
// Default : 0
uint64_t WAL_ttl_seconds;
+
+ // This mode allows overlapping files in every level. By default,
+ // hybrid mode is not enabled.
+ HybridOptions hybrid_options;
};
// Options that control read operations
return atoi(property.c_str());
}
+ bool IsHybrid() {
+ DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
+ return db_impl->IsHybrid();
+ }
+
+ bool CompactRange(int level) {
+ DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
+ db_impl->TEST_CompactRange(level, NULL, NULL);
+ return true;
+ }
+
+ bool CompactRange() {
+ int l = IsHybrid() ? numlevels_ : numlevels_-1;
+ DBImpl* db_impl = reinterpret_cast<DBImpl*>(db_);
+ for (int i = 0; i < l; i++) {
+ db_impl->TEST_CompactRange(i, NULL, NULL);
+ }
+ return true;
+ }
+
private:
std::string dbname_;
DB* db_;
+ int numlevels_;
};
Status ReduceLevelTest::OpenDB(bool create_if_missing, int num_levels,
int mem_table_compact_level) {
+ numlevels_ = num_levels;
leveldb::Options opt;
opt.num_levels = num_levels;
opt.create_if_missing = create_if_missing;
TEST(ReduceLevelTest, Last_Level) {
// create files on all levels;
ASSERT_OK(OpenDB(true, 4, 3));
+ bool hybrid = IsHybrid();
ASSERT_OK(Put("aaaa", "11111"));
ASSERT_OK(CompactMemTable());
+ if (hybrid) {
+ // move files to level 3
+ CompactRange();
+ }
ASSERT_EQ(FilesOnLevel(3), 1);
CloseDB();
ASSERT_OK(OpenDB(true, 5, 1));
ASSERT_OK(Put("a", "a11111"));
ASSERT_OK(CompactMemTable());
+ IsHybrid() ? CompactRange(0): true;
ASSERT_EQ(FilesOnLevel(1), 1);
CloseDB();
+ // move files to level 2
+
ASSERT_OK(OpenDB(true, 5, 2));
+ IsHybrid() ? CompactRange(1): true;
ASSERT_OK(Put("b", "b11111"));
ASSERT_OK(CompactMemTable());
+ IsHybrid() ? CompactRange(0): true;
ASSERT_EQ(FilesOnLevel(1), 1);
ASSERT_EQ(FilesOnLevel(2), 1);
CloseDB();
ASSERT_OK(OpenDB(true, 5, 3));
+ IsHybrid() ? CompactRange(2): true;
+ IsHybrid() ? CompactRange(1): true;
ASSERT_OK(Put("c", "c11111"));
ASSERT_OK(CompactMemTable());
+ IsHybrid() ? CompactRange(0): true;
ASSERT_EQ(FilesOnLevel(1), 1);
ASSERT_EQ(FilesOnLevel(2), 1);
ASSERT_EQ(FilesOnLevel(3), 1);
CloseDB();
ASSERT_OK(OpenDB(true, 5, 4));
+ IsHybrid() ? CompactRange(3): true;
+ IsHybrid() ? CompactRange(2): true;
+ IsHybrid() ? CompactRange(1): true;
ASSERT_OK(Put("d", "d11111"));
ASSERT_OK(CompactMemTable());
+ IsHybrid() ? CompactRange(0): true;
ASSERT_EQ(FilesOnLevel(1), 1);
ASSERT_EQ(FilesOnLevel(2), 1);
ASSERT_EQ(FilesOnLevel(3), 1);
WAL_ttl_seconds(0){
}
+HybridOptions::HybridOptions()
+ : enable(false),
+ max_files_for_level_base(10),
+ max_files_for_level_multiplier(1) {
+}
+
void
Options::Dump(
Logger * log) const
projects / rocksdb.git / commitdiff