#include "db/version_set.h"
#include "db/write_controller.h"
#include "file/sst_file_manager_impl.h"
-#include "memtable/hash_skiplist_rep.h"
#include "monitoring/thread_status_util.h"
#include "options/options_helper.h"
#include "port/port.h"
SpecialEnv env(Env::Default());
db_options_.env = &env;
db_options_.max_background_flushes = 1;
- column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
+ column_family_options_.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
SpecialEnv env(Env::Default());
db_options_.env = &env;
db_options_.max_background_flushes = 1;
- column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
+ column_family_options_.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
env.SetBackgroundThreads(2, Env::HIGH);
db_options_.env = &env;
db_options_.max_background_flushes = 1;
- column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(2));
+ column_family_options_.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(2));
Open();
CreateColumnFamilies({"one"});
ASSERT_OK(Put(1, "fodor", "mirko"));
env.SetBackgroundThreads(2, Env::HIGH);
db_options_.env = &env;
db_options_.max_background_flushes = 1;
- column_family_options_.memtable_factory.reset(new SpecialSkipListFactory(3));
+ column_family_options_.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(3));
column_family_options_.level0_file_num_compaction_trigger = 2;
Open();
CreateColumnFamilies({"one"});
#include "db/write_batch_internal.h"
#include "env/mock_env.h"
#include "file/filename.h"
-#include "memtable/hash_linklist_rep.h"
#include "monitoring/statistics.h"
#include "monitoring/thread_status_util.h"
#include "port/stack_trace.h"
#include "port/stack_trace.h"
#include "rocksdb/perf_context.h"
#include "table/block_based/filter_policy_internal.h"
+#include "test_util/testutil.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
Options options = CurrentOptions();
constexpr size_t kNumKeys = 10000;
static_assert(kNumKeys <= 10000, "kNumKeys have to be <= 10000");
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeys + 10));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeys + 10));
options.create_if_missing = true;
constexpr size_t kPrefixLength = 4;
options.prefix_extractor.reset(NewFixedPrefixTransform(kPrefixLength));
#include "rocksdb/sst_file_writer.h"
#include "rocksdb/utilities/convenience.h"
#include "test_util/sync_point.h"
+#include "test_util/testutil.h"
#include "util/concurrent_task_limiter_impl.h"
#include "util/random.h"
#include "utilities/fault_injection_env.h"
options.num_levels = 3;
options.level0_file_num_compaction_trigger = 3;
options.max_subcompactions = max_subcompactions_;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
CreateAndReopenWithCF({"pikachu"}, options);
Random rnd(301);
options.level0_slowdown_writes_trigger = 20;
options.soft_pending_compaction_bytes_limit = 1 << 30; // Infinitely large
options.max_background_compactions = 3;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
// Block all threads in thread pool.
const size_t kTotalTasks = 4;
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
}
Options options = CurrentOptions();
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 110 << 10; // 110KB
options.arena_block_size = 4 << 10;
options.level0_slowdown_writes_trigger = 64;
options.level0_stop_writes_trigger = 64;
options.max_background_jobs = kMaxBackgroundThreads; // Enough threads
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.max_write_buffer_number = 10; // Enough memtables
DestroyAndReopen(options);
options.level0_file_num_compaction_trigger =
options.level0_stop_writes_trigger;
options.max_subcompactions = max_subcompactions_;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
Random rnd(301);
// `Status::Incomplete`.
Options options = CurrentOptions();
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 3;
Reopen(options);
// succeeds
Options options = CurrentOptions();
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 3;
Reopen(options);
#include "port/stack_trace.h"
#include "rocksdb/utilities/transaction_db.h"
#include "test_util/sync_point.h"
+#include "test_util/testutil.h"
#include "util/cast_util.h"
#include "util/mutexlock.h"
#include "utilities/fault_injection_env.h"
// scheduled in the low-pri (compaction) thread pool.
Options options = CurrentOptions();
options.level0_file_num_compaction_trigger = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(1));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(options);
env_->SetBackgroundThreads(0, Env::HIGH);
options.create_if_missing = true;
options.atomic_flush = atomic_flush;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kNumKeysTriggerFlush));
+ test::NewSpecialSkipListFactory(kNumKeysTriggerFlush));
CreateAndReopenWithCF({"pikachu"}, options);
for (int i = 0; i != kNumKeysTriggerFlush; ++i) {
#include "logging/auto_roll_logger.h"
#include "logging/log_buffer.h"
#include "logging/logging.h"
-#include "memtable/hash_linklist_rep.h"
-#include "memtable/hash_skiplist_rep.h"
#include "monitoring/in_memory_stats_history.h"
#include "monitoring/instrumented_mutex.h"
#include "monitoring/iostats_context_imp.h"
#include "db/db_test_util.h"
#include "port/stack_trace.h"
+#include "test_util/testutil.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
options.writable_file_max_buffer_size = 128 * 1024;
options.bytes_per_sync = 128 * 1024;
options.level0_file_num_compaction_trigger = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(10));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(10));
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
options.error_if_exists = false;
options.paranoid_checks = true;
options.level0_file_num_compaction_trigger = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(2));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
options.paranoid_checks = true;
options.use_fsync = false;
options.level0_file_num_compaction_trigger = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(2));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
DestroyAndReopen(options);
CreateAndReopenWithCF({"pikachu"}, options);
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.level0_file_num_compaction_trigger = kNumFiles;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = 2;
options.target_file_size_base = kFileBytes;
BlockBasedTableOptions table_options;
opts.disable_auto_compactions = true;
opts.level0_file_num_compaction_trigger = kNumFiles;
opts.max_compaction_bytes = kNumPerFile * kBytesPerVal;
- opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
// Want max_compaction_bytes to trigger the end of compaction output file, not
// target_file_size_base, so make the latter much bigger
opts.target_file_size_base = 100 * opts.max_compaction_bytes;
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
opts.disable_auto_compactions = true;
- opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
opts.num_levels = 2;
opts.statistics = CreateDBStatistics();
DestroyAndReopen(opts);
options.level0_file_num_compaction_trigger = kNumFiles;
options.max_bytes_for_level_base = 2 * kFileBytes;
options.max_subcompactions = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = 3;
options.target_file_size_base = kFileBytes;
options.target_file_size_multiplier = 1;
options.compaction_style = kCompactionStyleUniversal;
options.level0_file_num_compaction_trigger = kFilesPerLevel;
options.max_subcompactions = 4;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
options.num_levels = kNumLevels;
options.target_file_size_base = kNumPerFile << 10;
options.target_file_size_multiplier = 1;
const int kNumPerFile = 3, kNumFiles = 3;
Options opts = CurrentOptions();
opts.disable_auto_compactions = true;
- opts.memtable_factory.reset(new SpecialSkipListFactory(2 * kNumPerFile));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(2 * kNumPerFile));
opts.merge_operator = MergeOperators::CreateUInt64AddOperator();
opts.num_levels = 2;
Reopen(opts);
opts.comparator = test::Uint64Comparator();
opts.level0_file_num_compaction_trigger = 4;
opts.level0_stop_writes_trigger = 4;
- opts.memtable_factory.reset(new SpecialSkipListFactory(1));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
opts.num_levels = 2;
BlockBasedTableOptions bbto;
bbto.cache_index_and_filter_blocks = true;
// memtable can hold. It switches the active memtable to immutable (flush is
// prevented by the above options) upon inserting an element that would
// overflow the memtable.
- opts.memtable_factory.reset(new SpecialSkipListFactory(1));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
DestroyAndReopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "key", "val"));
Options opts = CurrentOptions();
opts.max_write_buffer_number = 4;
opts.min_write_buffer_number_to_merge = 3;
- opts.memtable_factory.reset(new SpecialSkipListFactory(1));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val"));
const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25;
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
- opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
DestroyAndReopen(opts);
// Write half of the keys before the tombstone and half after the tombstone.
const int kNum = 200, kRangeBegin = 50, kRangeEnd = 150, kNumPerFile = 25;
Options opts = CurrentOptions();
opts.comparator = test::Uint64Comparator();
- opts.memtable_factory.reset(new SpecialSkipListFactory(kNumPerFile));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(kNumPerFile));
DestroyAndReopen(opts);
const Snapshot* snapshot = nullptr;
Options opts = CurrentOptions();
opts.max_write_buffer_number = 4;
opts.min_write_buffer_number_to_merge = 3;
- opts.memtable_factory.reset(new SpecialSkipListFactory(1));
+ opts.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
Reopen(opts);
ASSERT_OK(db_->Put(WriteOptions(), "sst_key", "val"));
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
options.memtable_factory.reset(
- new SpecialSkipListFactory(2 /* num_entries_flush */));
+ test::NewSpecialSkipListFactory(2 /* num_entries_flush */));
options.target_file_size_base = kValueBytes;
// i == 0: CompactFiles
// i == 1: CompactRange
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumFilesPerLevel;
options.memtable_factory.reset(
- new SpecialSkipListFactory(2 /* num_entries_flush */));
+ test::NewSpecialSkipListFactory(2 /* num_entries_flush */));
options.target_file_size_base = kValueBytes;
options.disable_auto_compactions = true;
#include "port/stack_trace.h"
#include "rocksdb/utilities/transaction_db.h"
#include "test_util/sync_point.h"
+#include "test_util/testutil.h"
#include "utilities/fault_injection_env.h"
namespace ROCKSDB_NAMESPACE {
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kNumKeysPerMemtable));
+ test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
Options options1;
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kNumKeysPerMemtable));
+ test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
CreateAndReopenWithCF({kCFName1}, options);
Options options1;
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 2;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kNumKeysPerMemtable));
+ test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
Options options1;
#include "db/write_batch_internal.h"
#include "env/mock_env.h"
#include "file/filename.h"
-#include "memtable/hash_linklist_rep.h"
#include "monitoring/thread_status_util.h"
#include "port/port.h"
#include "port/stack_trace.h"
};
Options options = CurrentOptions();
- options.memtable_factory.reset(
- new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+ DBTestBase::kNumKeysByGenerateNewRandomFile));
options.compaction_style = kCompactionStyleLevel;
options.compaction_filter_factory.reset(
new CompactionFilterFactoryGetContext());
options.level0_stop_writes_trigger = 999999;
options.delayed_write_rate = 20000000; // Start with 200MB/s
options.memtable_factory.reset(
- new SpecialSkipListFactory(kEntriesPerMemTable));
+ test::NewSpecialSkipListFactory(kEntriesPerMemTable));
SetTimeElapseOnlySleepOnReopen(&options);
CreateAndReopenWithCF({"pikachu"}, options);
options.max_bytes_for_level_base = 10000000000u;
options.max_background_compactions = 1;
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
env_->SetBackgroundThreads(1, Env::LOW);
test::SleepingBackgroundTask sleeping_task_low;
options.disable_auto_compactions = true;
int kNumKeysPerMemtable = 3;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kNumKeysPerMemtable));
+ test::NewSpecialSkipListFactory(kNumKeysPerMemtable));
Reopen(options);
test::SleepingBackgroundTask sleeping_task;
#include "rocksdb/trace_record_result.h"
#include "rocksdb/utilities/replayer.h"
#include "rocksdb/wal_filter.h"
+#include "test_util/testutil.h"
#include "util/random.h"
#include "utilities/fault_injection_env.h"
options.disable_auto_compactions = true;
options.level0_file_num_compaction_trigger = kNumL0Files;
options.memtable_factory.reset(
- new SpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes));
+ test::NewSpecialSkipListFactory(kL0FileBytes / kBlockSizeBytes));
options.num_levels = 2;
options.target_file_size_base = kL0FileBytes;
options.target_file_size_multiplier = 2;
options.compression_opts.max_dict_bytes = kDictLen;
options.compression_opts.max_dict_buffer_bytes = kBlockLen;
}
- options.memtable_factory.reset(new SpecialSkipListFactory(kKeysPerFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(kKeysPerFile));
options.statistics = CreateDBStatistics();
BlockBasedTableOptions bbto;
bbto.block_size = kBlockLen;
#ifndef ROCKSDB_LITE
TEST_F(DBTest2, MaxCompactionBytesTest) {
Options options = CurrentOptions();
- options.memtable_factory.reset(
- new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+ DBTestBase::kNumKeysByGenerateNewRandomFile));
options.compaction_style = kCompactionStyleLevel;
options.write_buffer_size = 200 << 10;
options.arena_block_size = 4 << 10;
Options options = CurrentOptions();
options.compression = kNoCompression;
options.level0_file_num_compaction_trigger = kNumL0Files;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.new_table_reader_for_compaction_inputs = true;
// takes roughly one second, split into 100 x 10ms intervals. Each interval
// permits 5.12KB, which is smaller than the block size, so this test
#include "db/db_impl/db_impl.h"
#include "db/dbformat.h"
#include "file/filename.h"
-#include "memtable/hash_linklist_rep.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/convenience.h"
enum SkipPolicy { kSkipNone = 0, kSkipNoSnapshot = 1, kSkipNoPrefix = 2 };
-// A hacky skip list mem table that triggers flush after number of entries.
-class SpecialMemTableRep : public MemTableRep {
- public:
- explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
- int num_entries_flush)
- : MemTableRep(allocator),
- memtable_(memtable),
- num_entries_flush_(num_entries_flush),
- num_entries_(0) {}
-
- virtual KeyHandle Allocate(const size_t len, char** buf) override {
- return memtable_->Allocate(len, buf);
- }
-
- // Insert key into the list.
- // REQUIRES: nothing that compares equal to key is currently in the list.
- virtual void Insert(KeyHandle handle) override {
- num_entries_++;
- memtable_->Insert(handle);
- }
-
- void InsertConcurrently(KeyHandle handle) override {
- num_entries_++;
- memtable_->Insert(handle);
- }
-
- // Returns true iff an entry that compares equal to key is in the list.
- virtual bool Contains(const char* key) const override {
- return memtable_->Contains(key);
- }
-
- virtual size_t ApproximateMemoryUsage() override {
- // Return a high memory usage when number of entries exceeds the threshold
- // to trigger a flush.
- return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
- }
-
- virtual void Get(const LookupKey& k, void* callback_args,
- bool (*callback_func)(void* arg,
- const char* entry)) override {
- memtable_->Get(k, callback_args, callback_func);
- }
-
- uint64_t ApproximateNumEntries(const Slice& start_ikey,
- const Slice& end_ikey) override {
- return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
- }
-
- virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
- return memtable_->GetIterator(arena);
- }
-
- virtual ~SpecialMemTableRep() override {}
-
- private:
- std::unique_ptr<MemTableRep> memtable_;
- int num_entries_flush_;
- int num_entries_;
-};
-
-// The factory for the hacky skip list mem table that triggers flush after
-// number of entries exceeds a threshold.
-class SpecialSkipListFactory : public MemTableRepFactory {
- public:
- // After number of inserts exceeds `num_entries_flush` in a mem table, trigger
- // flush.
- explicit SpecialSkipListFactory(int num_entries_flush)
- : num_entries_flush_(num_entries_flush) {}
-
- using MemTableRepFactory::CreateMemTableRep;
- virtual MemTableRep* CreateMemTableRep(
- const MemTableRep::KeyComparator& compare, Allocator* allocator,
- const SliceTransform* transform, Logger* /*logger*/) override {
- return new SpecialMemTableRep(
- allocator, factory_.CreateMemTableRep(compare, allocator, transform, 0),
- num_entries_flush_);
- }
- virtual const char* Name() const override { return "SkipListFactory"; }
-
- bool IsInsertConcurrentlySupported() const override {
- return factory_.IsInsertConcurrentlySupported();
- }
-
- private:
- SkipListFactory factory_;
- int num_entries_flush_;
-};
-
// Special Env used to delay background operations
class SpecialEnv : public EnvWrapper {
public:
#if !defined(ROCKSDB_LITE)
#include "rocksdb/utilities/table_properties_collectors.h"
#include "test_util/sync_point.h"
+#include "test_util/testutil.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
options.optimize_filters_for_hits = true;
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
- options.memtable_factory.reset(new SpecialSkipListFactory(3));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(3));
DestroyAndReopen(options);
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 5;
options.write_buffer_size = 111 << 10; // 114KB
options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_style = kCompactionStyleUniversal;
options.compaction_options_universal.size_ratio = 10;
options.write_buffer_size = 111 << 10; // 114KB
options.num_levels = 1;
options.write_buffer_size = 200 << 10; // 200KB
options.level0_file_num_compaction_trigger = 3;
- options.memtable_factory.reset(new SpecialSkipListFactory(KNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(KNumKeysPerFile));
options = CurrentOptions(options);
CreateAndReopenWithCF({"pikachu"}, options);
options.level0_file_num_compaction_trigger = 2;
options.num_levels = 1;
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
std::vector<std::string> filenames;
if (env_->GetChildren(options.db_paths[1].path, &filenames).ok()) {
const int kNumFilesTrigger = 8;
Options options = CurrentOptions();
options.memtable_factory.reset(
- new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
+ test::NewSpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
options.compaction_options_universal.max_size_amplification_percent =
static_cast<unsigned int>(-1);
#if !defined(ROCKSDB_LITE)
#include "test_util/sync_point.h"
#endif
+#include "test_util/testutil.h"
#include "utilities/fault_injection_env.h"
namespace ROCKSDB_NAMESPACE {
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
const std::vector<uint64_t> start_keys = {1, 0};
const std::vector<std::string> write_timestamps = {Timestamp(1, 0),
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
const std::vector<std::string> write_timestamps = {Timestamp(1, 0),
Timestamp(3, 0)};
const size_t kTimestampSize = Timestamp(0, 0).size();
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
std::vector<SequenceNumber> start_seqs;
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
size_t ts_sz = Timestamp(0, 0).size();
TestComparator test_cmp(ts_sz);
options.comparator = &test_cmp;
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
const int kNumKeysPerFile = 1024;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<0>(GetParam());
bbto.whole_key_filtering = true;
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
FlushedFileCollector* collector = new FlushedFileCollector();
options.listeners.emplace_back(collector);
Options options = CurrentOptions();
options.create_if_missing = true;
options.env = env_;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
options.memtable_prefix_bloom_size_ratio = 0.1;
options.memtable_whole_key_filtering = true;
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.prefix_extractor = std::get<0>(GetParam());
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<1>(GetParam());
bbto.index_type = std::get<3>(GetParam());
TestComparator test_cmp(kTimestampSize);
options.comparator = &test_cmp;
options.prefix_extractor = std::get<0>(GetParam());
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
BlockBasedTableOptions bbto;
bbto.filter_policy = std::get<1>(GetParam());
bbto.index_type = std::get<3>(GetParam());
#include "db/compaction/compaction.h"
#include "db/db_test_util.h"
#include "port/stack_trace.h"
+#include "test_util/testutil.h"
namespace ROCKSDB_NAMESPACE {
options.comparator = test::ComparatorWithU64Ts();
options.level0_file_num_compaction_trigger = 3;
constexpr size_t kNumKeysPerFile = 101;
- options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+ options.memtable_factory.reset(
+ test::NewSpecialSkipListFactory(kNumKeysPerFile));
DestroyAndReopen(options);
SyncPoint::GetInstance()->DisableProcessing();
SyncPoint::GetInstance()->ClearAllCallBacks();
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "file/filename.h"
-#include "memtable/hash_linklist_rep.h"
#include "monitoring/statistics.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
- options.memtable_factory.reset(
- new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+ DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
- options.memtable_factory.reset(
- new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+ DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
Options options;
options.env = CurrentOptions().env;
options.create_if_missing = true;
- options.memtable_factory.reset(
- new SpecialSkipListFactory(DBTestBase::kNumKeysByGenerateNewRandomFile));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(
+ DBTestBase::kNumKeysByGenerateNewRandomFile));
TestCompactionReasonListener* listener = new TestCompactionReasonListener();
options.listeners.emplace_back(listener);
options.create_if_missing = true;
options.env = env_;
options.listeners.push_back(listener);
- options.memtable_factory.reset(new SpecialSkipListFactory(1));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(1));
options.paranoid_checks = true;
DestroyAndReopen(options);
options.env = env_;
options.level0_file_num_compaction_trigger = 2;
options.listeners.push_back(listener);
- options.memtable_factory.reset(new SpecialSkipListFactory(2));
+ options.memtable_factory.reset(test::NewSpecialSkipListFactory(2));
options.paranoid_checks = true;
DestroyAndReopen(options);
#include <stdexcept>
#include <unordered_set>
+#include "rocksdb/customizable.h"
#include "rocksdb/slice.h"
namespace ROCKSDB_NAMESPACE {
class LookupKey;
class SliceTransform;
class Logger;
+struct DBOptions;
using KeyHandle = void*;
// This is the base class for all factories that are used by RocksDB to create
// new MemTableRep objects
-class MemTableRepFactory {
+class MemTableRepFactory : public Customizable {
public:
virtual ~MemTableRepFactory() {}
+ static const char* Type() { return "MemTableRepFactory"; }
+ static Status CreateFromString(const ConfigOptions& config_options,
+ const std::string& id,
+ std::unique_ptr<MemTableRepFactory>* factory);
+
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) = 0;
// seeks with consecutive keys.
class SkipListFactory : public MemTableRepFactory {
public:
- explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {}
+ explicit SkipListFactory(size_t lookahead = 0);
+
+ // Methods for Configurable/Customizable class overrides
+ static const char* kClassName() { return "SkipListFactory"; }
+ static const char* kNickName() { return "skip_list"; }
+ virtual const char* Name() const override { return kClassName(); }
+ virtual const char* NickName() const override { return kNickName(); }
+ std::string GetId() const override;
+ // Methods for MemTableRepFactory class overrides
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) override;
- virtual const char* Name() const override { return "SkipListFactory"; }
bool IsInsertConcurrentlySupported() const override { return true; }
bool CanHandleDuplicatedKey() const override { return true; }
private:
- const size_t lookahead_;
+ size_t lookahead_;
};
#ifndef ROCKSDB_LITE
// VectorRep. On initialization, the underlying array will be at least count
// bytes reserved for usage.
class VectorRepFactory : public MemTableRepFactory {
- const size_t count_;
+ size_t count_;
public:
- explicit VectorRepFactory(size_t count = 0) : count_(count) {}
+ explicit VectorRepFactory(size_t count = 0);
+ // Methods for Configurable/Customizable class overrides
+ static const char* kClassName() { return "VectorRepFactory"; }
+ static const char* kNickName() { return "vector"; }
+ const char* Name() const override { return kClassName(); }
+ const char* NickName() const override { return kNickName(); }
+
+ // Methods for MemTableRepFactory class overrides
using MemTableRepFactory::CreateMemTableRep;
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
Allocator*, const SliceTransform*,
Logger* logger) override;
-
- virtual const char* Name() const override { return "VectorRepFactory"; }
};
// This class contains a fixed array of buckets, each
kSliceTransform,
kCompressionType,
kCompactionStopStyle,
- kMemTableRepFactory,
kFilterPolicy,
kChecksumType,
kEncodingType,
//
#ifndef ROCKSDB_LITE
-#include "memtable/hash_linklist_rep.h"
#include <algorithm>
#include <atomic>
+
#include "db/memtable.h"
#include "memory/arena.h"
#include "memtable/skiplist.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
+#include "rocksdb/utilities/options_type.h"
#include "util/hash.h"
namespace ROCKSDB_NAMESPACE {
return x;
}
-} // anon namespace
+struct HashLinkListRepOptions {
+ static const char* kName() { return "HashLinkListRepFactoryOptions"; }
+ size_t bucket_count;
+ uint32_t threshold_use_skiplist;
+ size_t huge_page_tlb_size;
+ int bucket_entries_logging_threshold;
+ bool if_log_bucket_dist_when_flash;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> hash_linklist_info = {
+ {"bucket_count",
+ {offsetof(struct HashLinkListRepOptions, bucket_count), OptionType::kSizeT,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"threshold",
+ {offsetof(struct HashLinkListRepOptions, threshold_use_skiplist),
+ OptionType::kUInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"huge_page_size",
+ {offsetof(struct HashLinkListRepOptions, huge_page_tlb_size),
+ OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"logging_threshold",
+ {offsetof(struct HashLinkListRepOptions, bucket_entries_logging_threshold),
+ OptionType::kInt, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"log_when_flash",
+ {offsetof(struct HashLinkListRepOptions, if_log_bucket_dist_when_flash),
+ OptionType::kBoolean, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+};
+
+class HashLinkListRepFactory : public MemTableRepFactory {
+ public:
+ explicit HashLinkListRepFactory(size_t bucket_count,
+ uint32_t threshold_use_skiplist,
+ size_t huge_page_tlb_size,
+ int bucket_entries_logging_threshold,
+ bool if_log_bucket_dist_when_flash) {
+ options_.bucket_count = bucket_count;
+ options_.threshold_use_skiplist = threshold_use_skiplist;
+ options_.huge_page_tlb_size = huge_page_tlb_size;
+ options_.bucket_entries_logging_threshold =
+ bucket_entries_logging_threshold;
+ options_.if_log_bucket_dist_when_flash = if_log_bucket_dist_when_flash;
+ RegisterOptions(&options_, &hash_linklist_info);
+ }
+
+ using MemTableRepFactory::CreateMemTableRep;
+ virtual MemTableRep* CreateMemTableRep(
+ const MemTableRep::KeyComparator& compare, Allocator* allocator,
+ const SliceTransform* transform, Logger* logger) override;
+
+ static const char* kClassName() { return "HashLinkListRepFactory"; }
+ static const char* kNickName() { return "hash_linkedlist"; }
+ virtual const char* Name() const override { return kClassName(); }
+ virtual const char* NickName() const override { return kNickName(); }
+
+ private:
+ HashLinkListRepOptions options_;
+};
+
+} // namespace
MemTableRep* HashLinkListRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* logger) {
- return new HashLinkListRep(compare, allocator, transform, bucket_count_,
- threshold_use_skiplist_, huge_page_tlb_size_,
- logger, bucket_entries_logging_threshold_,
- if_log_bucket_dist_when_flash_);
+ return new HashLinkListRep(
+ compare, allocator, transform, options_.bucket_count,
+ options_.threshold_use_skiplist, options_.huge_page_tlb_size, logger,
+ options_.bucket_entries_logging_threshold,
+ options_.if_log_bucket_dist_when_flash);
}
MemTableRepFactory* NewHashLinkListRepFactory(
+++ /dev/null
-// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under both the GPLv2 (found in the
-// COPYING file in the root directory) and Apache 2.0 License
-// (found in the LICENSE.Apache file in the root directory).
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#ifndef ROCKSDB_LITE
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/memtablerep.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class HashLinkListRepFactory : public MemTableRepFactory {
- public:
- explicit HashLinkListRepFactory(size_t bucket_count,
- uint32_t threshold_use_skiplist,
- size_t huge_page_tlb_size,
- int bucket_entries_logging_threshold,
- bool if_log_bucket_dist_when_flash)
- : bucket_count_(bucket_count),
- threshold_use_skiplist_(threshold_use_skiplist),
- huge_page_tlb_size_(huge_page_tlb_size),
- bucket_entries_logging_threshold_(bucket_entries_logging_threshold),
- if_log_bucket_dist_when_flash_(if_log_bucket_dist_when_flash) {}
-
- virtual ~HashLinkListRepFactory() {}
-
- using MemTableRepFactory::CreateMemTableRep;
- virtual MemTableRep* CreateMemTableRep(
- const MemTableRep::KeyComparator& compare, Allocator* allocator,
- const SliceTransform* transform, Logger* logger) override;
-
- virtual const char* Name() const override {
- return "HashLinkListRepFactory";
- }
-
- private:
- const size_t bucket_count_;
- const uint32_t threshold_use_skiplist_;
- const size_t huge_page_tlb_size_;
- int bucket_entries_logging_threshold_;
- bool if_log_bucket_dist_when_flash_;
-};
-
-} // namespace ROCKSDB_NAMESPACE
-#endif // ROCKSDB_LITE
//
#ifndef ROCKSDB_LITE
-#include "memtable/hash_skiplist_rep.h"
-
#include <atomic>
#include "db/memtable.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice.h"
#include "rocksdb/slice_transform.h"
+#include "rocksdb/utilities/options_type.h"
#include "util/murmurhash.h"
namespace ROCKSDB_NAMESPACE {
}
}
-} // anon namespace
+struct HashSkipListRepOptions {
+ static const char* kName() { return "HashSkipListRepFactoryOptions"; }
+ size_t bucket_count;
+ int32_t skiplist_height;
+ int32_t skiplist_branching_factor;
+};
+
+static std::unordered_map<std::string, OptionTypeInfo> hash_skiplist_info = {
+ {"bucket_count",
+ {offsetof(struct HashSkipListRepOptions, bucket_count), OptionType::kSizeT,
+ OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
+ {"skiplist_height",
+ {offsetof(struct HashSkipListRepOptions, skiplist_height),
+ OptionType::kInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+ {"branching_factor",
+ {offsetof(struct HashSkipListRepOptions, skiplist_branching_factor),
+ OptionType::kInt32T, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+};
+
+class HashSkipListRepFactory : public MemTableRepFactory {
+ public:
+ explicit HashSkipListRepFactory(size_t bucket_count, int32_t skiplist_height,
+ int32_t skiplist_branching_factor) {
+ options_.bucket_count = bucket_count;
+ options_.skiplist_height = skiplist_height;
+ options_.skiplist_branching_factor = skiplist_branching_factor;
+ RegisterOptions(&options_, &hash_skiplist_info);
+ }
+
+ using MemTableRepFactory::CreateMemTableRep;
+ virtual MemTableRep* CreateMemTableRep(
+ const MemTableRep::KeyComparator& compare, Allocator* allocator,
+ const SliceTransform* transform, Logger* logger) override;
+
+ static const char* kClassName() { return "HashSkipListRepFactory"; }
+ static const char* kNickName() { return "prefix_hash"; }
+
+ virtual const char* Name() const override { return kClassName(); }
+ virtual const char* NickName() const override { return kNickName(); }
+
+ private:
+ HashSkipListRepOptions options_;
+};
+
+} // namespace
MemTableRep* HashSkipListRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) {
- return new HashSkipListRep(compare, allocator, transform, bucket_count_,
- skiplist_height_, skiplist_branching_factor_);
+ return new HashSkipListRep(compare, allocator, transform,
+ options_.bucket_count, options_.skiplist_height,
+ options_.skiplist_branching_factor);
}
MemTableRepFactory* NewHashSkipListRepFactory(
+++ /dev/null
-// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
-// This source code is licensed under both the GPLv2 (found in the
-// COPYING file in the root directory) and Apache 2.0 License
-// (found in the LICENSE.Apache file in the root directory).
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#ifndef ROCKSDB_LITE
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/memtablerep.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class HashSkipListRepFactory : public MemTableRepFactory {
- public:
- explicit HashSkipListRepFactory(
- size_t bucket_count,
- int32_t skiplist_height,
- int32_t skiplist_branching_factor)
- : bucket_count_(bucket_count),
- skiplist_height_(skiplist_height),
- skiplist_branching_factor_(skiplist_branching_factor) { }
-
- virtual ~HashSkipListRepFactory() {}
-
- using MemTableRepFactory::CreateMemTableRep;
- virtual MemTableRep* CreateMemTableRep(
- const MemTableRep::KeyComparator& compare, Allocator* allocator,
- const SliceTransform* transform, Logger* logger) override;
-
- virtual const char* Name() const override {
- return "HashSkipListRepFactory";
- }
-
- private:
- const size_t bucket_count_;
- const int32_t skiplist_height_;
- const int32_t skiplist_branching_factor_;
-};
-
-} // namespace ROCKSDB_NAMESPACE
-#endif // ROCKSDB_LITE
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/comparator.h"
+#include "rocksdb/convenience.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/options.h"
#include "rocksdb/slice_transform.h"
#ifndef ROCKSDB_LITE
} else if (FLAGS_memtablerep == "vector") {
factory.reset(new ROCKSDB_NAMESPACE::VectorRepFactory);
- } else if (FLAGS_memtablerep == "hashskiplist") {
+ } else if (FLAGS_memtablerep == "hashskiplist" ||
+ FLAGS_memtablerep == "prefix_hash") {
factory.reset(ROCKSDB_NAMESPACE::NewHashSkipListRepFactory(
FLAGS_bucket_count, FLAGS_hashskiplist_height,
FLAGS_hashskiplist_branching_factor));
options.prefix_extractor.reset(
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length));
- } else if (FLAGS_memtablerep == "hashlinklist") {
+ } else if (FLAGS_memtablerep == "hashlinklist" ||
+ FLAGS_memtablerep == "hash_linkedlist") {
factory.reset(ROCKSDB_NAMESPACE::NewHashLinkListRepFactory(
FLAGS_bucket_count, FLAGS_huge_page_tlb_size,
FLAGS_bucket_entries_logging_threshold,
ROCKSDB_NAMESPACE::NewFixedPrefixTransform(FLAGS_prefix_length));
#endif // ROCKSDB_LITE
} else {
- fprintf(stdout, "Unknown memtablerep: %s\n", FLAGS_memtablerep.c_str());
- exit(1);
+ ROCKSDB_NAMESPACE::ConfigOptions config_options;
+ config_options.ignore_unsupported_options = false;
+
+ ROCKSDB_NAMESPACE::Status s =
+ ROCKSDB_NAMESPACE::MemTableRepFactory::CreateFromString(
+ config_options, FLAGS_memtablerep, &factory);
+ if (!s.ok()) {
+ fprintf(stdout, "Unknown memtablerep: %s\n", s.ToString().c_str());
+ exit(1);
+ }
}
ROCKSDB_NAMESPACE::InternalKeyComparator internal_key_comp(
#include "memory/arena.h"
#include "memtable/inlineskiplist.h"
#include "rocksdb/memtablerep.h"
+#include "rocksdb/utilities/options_type.h"
+#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
namespace {
};
}
+static std::unordered_map<std::string, OptionTypeInfo> skiplist_factory_info = {
+#ifndef ROCKSDB_LITE
+ {"lookahead",
+ {0, OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kDontSerialize /*Since it is part of the ID*/}},
+#endif
+};
+
+SkipListFactory::SkipListFactory(size_t lookahead) : lookahead_(lookahead) {
+ RegisterOptions("SkipListFactoryOptions", &lookahead_,
+ &skiplist_factory_info);
+}
+
+std::string SkipListFactory::GetId() const {
+ std::string id = Name();
+ if (lookahead_ > 0) {
+ id.append(":").append(ROCKSDB_NAMESPACE::ToString(lookahead_));
+ }
+ return id;
+}
+
MemTableRep* SkipListFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform* transform, Logger* /*logger*/) {
// (found in the LICENSE.Apache file in the root directory).
//
#ifndef ROCKSDB_LITE
-#include "rocksdb/memtablerep.h"
-
-#include <unordered_set>
-#include <set>
-#include <memory>
#include <algorithm>
+#include <memory>
+#include <set>
#include <type_traits>
+#include <unordered_set>
#include "db/memtable.h"
#include "memory/arena.h"
#include "memtable/stl_wrappers.h"
#include "port/port.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/utilities/options_type.h"
#include "util/mutexlock.h"
namespace ROCKSDB_NAMESPACE {
}
} // anon namespace
+static std::unordered_map<std::string, OptionTypeInfo> vector_rep_table_info = {
+ {"count",
+ {0, OptionType::kSizeT, OptionVerificationType::kNormal,
+ OptionTypeFlags::kNone}},
+};
+
+VectorRepFactory::VectorRepFactory(size_t count) : count_(count) {
+ RegisterOptions("VectorRepFactoryOptions", &count_, &vector_rep_table_info);
+}
+
MemTableRep* VectorRepFactory::CreateMemTableRep(
const MemTableRep::KeyComparator& compare, Allocator* allocator,
const SliceTransform*, Logger* /*logger*/) {
OptionTypeFlags::kNone}},
{"memtable_factory",
{offset_of(&ImmutableCFOptions::memtable_factory),
- OptionType::kMemTableRepFactory, OptionVerificationType::kByName,
- OptionTypeFlags::kNone}},
+ OptionType::kCustomizable, OptionVerificationType::kByName,
+ OptionTypeFlags::kShared,
+ [](const ConfigOptions& opts, const std::string&,
+ const std::string& value, void* addr) {
+ std::unique_ptr<MemTableRepFactory> factory;
+ auto* shared =
+ static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
+ Status s =
+ MemTableRepFactory::CreateFromString(opts, value, &factory);
+ if (s.ok()) {
+ shared->reset(factory.release());
+ }
+ return s;
+ }}},
{"memtable",
{offset_of(&ImmutableCFOptions::memtable_factory),
- OptionType::kMemTableRepFactory, OptionVerificationType::kAlias,
- OptionTypeFlags::kNone,
- // Parses the value string and updates the memtable_factory
- [](const ConfigOptions& /*opts*/, const std::string& /*name*/,
+ OptionType::kCustomizable, OptionVerificationType::kAlias,
+ OptionTypeFlags::kShared,
+ [](const ConfigOptions& opts, const std::string&,
const std::string& value, void* addr) {
- std::unique_ptr<MemTableRepFactory> new_mem_factory;
- Status s = GetMemTableRepFactoryFromString(value, &new_mem_factory);
+ std::unique_ptr<MemTableRepFactory> factory;
+ auto* shared =
+ static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
+ Status s =
+ MemTableRepFactory::CreateFromString(opts, value, &factory);
if (s.ok()) {
- auto memtable_factory =
- static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
- memtable_factory->reset(new_mem_factory.release());
+ shared->reset(factory.release());
}
return s;
}}},
}
}
+TEST_F(LoadCustomizableTest, LoadMemTableRepFactoryTest) {
+ std::unique_ptr<MemTableRepFactory> result;
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options_, "SpecialSkipListFactory", &result));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options_, SkipListFactory::kClassName(), &result));
+ ASSERT_NE(result.get(), nullptr);
+ ASSERT_TRUE(result->IsInstanceOf(SkipListFactory::kClassName()));
+
+ if (RegisterTests("Test")) {
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options_, "SpecialSkipListFactory", &result));
+ ASSERT_NE(result, nullptr);
+ ASSERT_STREQ(result->Name(), "SpecialSkipListFactory");
+ }
+}
+
TEST_F(LoadCustomizableTest, LoadMergeOperatorTest) {
std::shared_ptr<MergeOperator> result;
: kNullptrString;
break;
}
- case OptionType::kMemTableRepFactory: {
- const auto* ptr =
- static_cast<const std::shared_ptr<MemTableRepFactory>*>(opt_address);
- *value = ptr->get() ? ptr->get()->Name() : kNullptrString;
- break;
- }
case OptionType::kFilterPolicy: {
const auto* ptr =
static_cast<const std::shared_ptr<FilterPolicy>*>(opt_address);
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
+ ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
}
TEST_F(OptionsTest, CompressionOptionsFromString) {
ASSERT_OK(GetMemTableRepFactoryFromString("skip_list", &new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("skip_list:16", &new_mem_factory));
- ASSERT_EQ(std::string(new_mem_factory->Name()), "SkipListFactory");
+ ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
ASSERT_NOK(GetMemTableRepFactoryFromString("skip_list:16:invalid_opt",
&new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash", &new_mem_factory));
ASSERT_OK(GetMemTableRepFactoryFromString("prefix_hash:1000",
&new_mem_factory));
- ASSERT_EQ(std::string(new_mem_factory->Name()), "HashSkipListRepFactory");
+ ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
ASSERT_NOK(GetMemTableRepFactoryFromString("prefix_hash:1000:invalid_opt",
&new_mem_factory));
}
#endif // !ROCKSDB_LITE
+TEST_F(OptionsTest, MemTableRepFactoryCreateFromString) {
+ std::unique_ptr<MemTableRepFactory> new_mem_factory = nullptr;
+ ConfigOptions config_options;
+ config_options.ignore_unsupported_options = false;
+ config_options.ignore_unknown_options = false;
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "skip_list:16",
+ &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "SkipListFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("skip_list"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("SkipListFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "skip_list:16:invalid_opt", &new_mem_factory));
+
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "invalid_opt=10", &new_mem_factory));
+
+ // Test a reset
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "",
+ &new_mem_factory));
+ ASSERT_EQ(new_mem_factory, nullptr);
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "invalid_opt=10", &new_mem_factory));
+
+#ifndef ROCKSDB_LITE
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "id=skip_list; lookahead=32", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "prefix_hash",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "prefix_hash:1000", &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "HashSkipListRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("prefix_hash"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashSkipListRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "prefix_hash:1000:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=prefix_hash; bucket_count=32; skiplist_height=64; "
+ "branching_factor=16",
+ &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=prefix_hash; bucket_count=32; skiplist_height=64; "
+ "branching_factor=16; invalid=unknown",
+ &new_mem_factory));
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist:1000", &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "HashLinkListRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("hash_linkedlist"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("HashLinkListRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "hash_linkedlist:1000:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
+ "logging_threshold=12; log_when_flash=true",
+ &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options,
+ "id=hash_linkedlist; bucket_count=32; threshold=64; huge_page_size=16; "
+ "logging_threshold=12; log_when_flash=true; invalid=unknown",
+ &new_mem_factory));
+
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector",
+ &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(config_options, "vector:1024",
+ &new_mem_factory));
+ ASSERT_STREQ(new_mem_factory->Name(), "VectorRepFactory");
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("vector"));
+ ASSERT_TRUE(new_mem_factory->IsInstanceOf("VectorRepFactory"));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "vector:1024:invalid_opt", &new_mem_factory));
+ ASSERT_OK(MemTableRepFactory::CreateFromString(
+ config_options, "id=vector; count=42", &new_mem_factory));
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(
+ config_options, "id=vector; invalid=unknown", &new_mem_factory));
+#endif // ROCKSDB_LITE
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo",
+ &new_mem_factory));
+ // CuckooHash memtable is already removed.
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "cuckoo:1024",
+ &new_mem_factory));
+
+ ASSERT_NOK(MemTableRepFactory::CreateFromString(config_options, "bad_factory",
+ &new_mem_factory));
+}
+
#ifndef ROCKSDB_LITE // GetOptionsFromString is not supported in RocksDB Lite
TEST_F(OptionsTest, GetOptionsFromStringTest) {
Options base_options, new_options;
"memtable=skip_list:10;arena_block_size=1024",
&new_cf_opt));
ASSERT_TRUE(new_cf_opt.memtable_factory != nullptr);
- ASSERT_EQ(std::string(new_cf_opt.memtable_factory->Name()), "SkipListFactory");
+ ASSERT_TRUE(new_cf_opt.memtable_factory->IsInstanceOf("SkipListFactory"));
}
TEST_F(OptionsOldApiTest, GetBlockBasedTableOptionsFromString) {
ASSERT_TRUE(new_opt.full_scan_mode);
ASSERT_TRUE(new_opt.store_index_in_file);
+ std::unordered_map<std::string, std::string> opt_map;
+ ASSERT_OK(StringToMap(
+ "user_key_len=55;bloom_bits_per_key=10;huge_page_tlb_size=8;", &opt_map));
+ ASSERT_OK(GetPlainTableOptionsFromMap(table_opt, opt_map, &new_opt));
+ ASSERT_EQ(new_opt.user_key_len, 55u);
+ ASSERT_EQ(new_opt.bloom_bits_per_key, 10);
+ ASSERT_EQ(new_opt.huge_page_tlb_size, 8);
+
// unknown option
ASSERT_NOK(GetPlainTableOptionsFromString(table_opt,
"user_key_len=66;bloom_bits_per_key=20;hash_table_ratio=0.5;"
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#ifndef ROCKSDB_LITE
#include "table/plain/plain_table_factory.h"
#include <stdint.h>
#include <memory>
#include "db/dbformat.h"
-#include "options/configurable_helper.h"
#include "port/port.h"
#include "rocksdb/convenience.h"
+#include "rocksdb/utilities/customizable_util.h"
+#include "rocksdb/utilities/object_registry.h"
#include "rocksdb/utilities/options_type.h"
#include "table/plain/plain_table_builder.h"
#include "table/plain/plain_table_reader.h"
#include "util/string_util.h"
namespace ROCKSDB_NAMESPACE {
+#ifndef ROCKSDB_LITE
static std::unordered_map<std::string, OptionTypeInfo> plain_table_type_info = {
{"user_key_len",
{offsetof(struct PlainTableOptions, user_key_len), OptionType::kUInt32T,
return Status::InvalidArgument(s.getState());
}
}
+#endif // ROCKSDB_LITE
-Status GetMemTableRepFactoryFromString(
- const std::string& opts_str,
- std::unique_ptr<MemTableRepFactory>* new_mem_factory) {
- std::vector<std::string> opts_list = StringSplit(opts_str, ':');
- size_t len = opts_list.size();
+#ifndef ROCKSDB_LITE
+static int RegisterBuiltinMemTableRepFactory(ObjectLibrary& library,
+ const std::string& /*arg*/) {
+ // The MemTableRepFactory built-in classes will be either a class
+ // (VectorRepFactory) or a nickname (vector), followed optionally by ":#",
+ // where # is the "size" of the factory.
+ auto AsRegex = [](const std::string& name, const std::string& alt) {
+ std::string regex;
+ regex.append("(").append(name);
+ regex.append("|").append(alt).append(")(:[0-9]*)?");
+ return regex;
+ };
- if (opts_list.empty() || opts_list.size() > 2) {
- return Status::InvalidArgument("Can't parse memtable_factory option ",
- opts_str);
- }
+ library.Register<MemTableRepFactory>(
+ AsRegex(VectorRepFactory::kClassName(), VectorRepFactory::kNickName()),
+ [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
+ std::string* /*errmsg*/) {
+ auto colon = uri.find(":");
+ if (colon != std::string::npos) {
+ size_t count = ParseSizeT(uri.substr(colon + 1));
+ guard->reset(new VectorRepFactory(count));
+ } else {
+ guard->reset(new VectorRepFactory());
+ }
+ return guard->get();
+ });
+ library.Register<MemTableRepFactory>(
+ AsRegex(SkipListFactory::kClassName(), SkipListFactory::kNickName()),
+ [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
+ std::string* /*errmsg*/) {
+ auto colon = uri.find(":");
+ if (colon != std::string::npos) {
+ size_t lookahead = ParseSizeT(uri.substr(colon + 1));
+ guard->reset(new SkipListFactory(lookahead));
+ } else {
+ guard->reset(new SkipListFactory());
+ }
+ return guard->get();
+ });
+ library.Register<MemTableRepFactory>(
+ AsRegex("HashLinkListRepFactory", "hash_linkedlist"),
+ [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
+ std::string* /*errmsg*/) {
+ // Expecting format: hash_linkedlist:<hash_bucket_count>
+ auto colon = uri.find(":");
+ if (colon != std::string::npos) {
+ size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1));
+ guard->reset(NewHashLinkListRepFactory(hash_bucket_count));
+ } else {
+ guard->reset(NewHashLinkListRepFactory());
+ }
+ return guard->get();
+ });
+ library.Register<MemTableRepFactory>(
+ AsRegex("HashSkipListRepFactory", "prefix_hash"),
+ [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
+ std::string* /*errmsg*/) {
+ // Expecting format: prefix_hash:<hash_bucket_count>
+ auto colon = uri.find(":");
+ if (colon != std::string::npos) {
+ size_t hash_bucket_count = ParseSizeT(uri.substr(colon + 1));
+ guard->reset(NewHashSkipListRepFactory(hash_bucket_count));
+ } else {
+ guard->reset(NewHashSkipListRepFactory());
+ }
+ return guard->get();
+ });
+ library.Register<MemTableRepFactory>(
+ "cuckoo",
+ [](const std::string& /*uri*/,
+ std::unique_ptr<MemTableRepFactory>* /*guard*/, std::string* errmsg) {
+ *errmsg = "cuckoo hash memtable is not supported anymore.";
+ return nullptr;
+ });
- MemTableRepFactory* mem_factory = nullptr;
+ return 5;
+}
+#endif // ROCKSDB_LITE
+Status GetMemTableRepFactoryFromString(
+ const std::string& opts_str, std::unique_ptr<MemTableRepFactory>* result) {
+ ConfigOptions config_options;
+ config_options.ignore_unsupported_options = false;
+ config_options.ignore_unknown_options = false;
+ return MemTableRepFactory::CreateFromString(config_options, opts_str, result);
+}
- if (opts_list[0] == "skip_list" || opts_list[0] == "SkipListFactory") {
- // Expecting format
- // skip_list:<lookahead>
- if (2 == len) {
- size_t lookahead = ParseSizeT(opts_list[1]);
- mem_factory = new SkipListFactory(lookahead);
- } else if (1 == len) {
- mem_factory = new SkipListFactory();
- }
- } else if (opts_list[0] == "prefix_hash" ||
- opts_list[0] == "HashSkipListRepFactory") {
- // Expecting format
- // prfix_hash:<hash_bucket_count>
- if (2 == len) {
- size_t hash_bucket_count = ParseSizeT(opts_list[1]);
- mem_factory = NewHashSkipListRepFactory(hash_bucket_count);
- } else if (1 == len) {
- mem_factory = NewHashSkipListRepFactory();
- }
- } else if (opts_list[0] == "hash_linkedlist" ||
- opts_list[0] == "HashLinkListRepFactory") {
- // Expecting format
- // hash_linkedlist:<hash_bucket_count>
- if (2 == len) {
- size_t hash_bucket_count = ParseSizeT(opts_list[1]);
- mem_factory = NewHashLinkListRepFactory(hash_bucket_count);
- } else if (1 == len) {
- mem_factory = NewHashLinkListRepFactory();
- }
- } else if (opts_list[0] == "vector" || opts_list[0] == "VectorRepFactory") {
- // Expecting format
- // vector:<count>
- if (2 == len) {
- size_t count = ParseSizeT(opts_list[1]);
- mem_factory = new VectorRepFactory(count);
- } else if (1 == len) {
- mem_factory = new VectorRepFactory();
- }
- } else if (opts_list[0] == "cuckoo") {
- return Status::NotSupported(
- "cuckoo hash memtable is not supported anymore.");
+Status MemTableRepFactory::CreateFromString(
+ const ConfigOptions& config_options, const std::string& value,
+ std::unique_ptr<MemTableRepFactory>* result) {
+#ifndef ROCKSDB_LITE
+ static std::once_flag once;
+ std::call_once(once, [&]() {
+ RegisterBuiltinMemTableRepFactory(*(ObjectLibrary::Default().get()), "");
+ });
+#endif // ROCKSDB_LITE
+ std::string id;
+ std::unordered_map<std::string, std::string> opt_map;
+ Status status = Customizable::GetOptionsMap(config_options, result->get(),
+ value, &id, &opt_map);
+ if (!status.ok()) { // GetOptionsMap failed
+ return status;
+ } else if (value.empty()) {
+ // No Id and no options. Clear the object
+ result->reset();
+ return Status::OK();
+ } else if (id.empty()) { // We have no Id but have options. Not good
+ return Status::NotSupported("Cannot reset object ", id);
} else {
- return Status::InvalidArgument("Unrecognized memtable_factory option ",
- opts_str);
- }
-
- if (mem_factory != nullptr) {
- new_mem_factory->reset(mem_factory);
+#ifndef ROCKSDB_LITE
+ status = NewUniqueObject<MemTableRepFactory>(config_options, id, opt_map,
+ result);
+#else
+ // To make it possible to configure the memtables in LITE mode, the ID
+ // is of the form <name>:<size>, where name is the name of the class and
+ // <size> is the length of the object (e.g. skip_list:10).
+ std::vector<std::string> opts_list = StringSplit(id, ':');
+ if (opts_list.empty() || opts_list.size() > 2 || !opt_map.empty()) {
+ status = Status::InvalidArgument("Can't parse memtable_factory option ",
+ value);
+ } else if (opts_list[0] == "skip_list" ||
+ opts_list[0] == SkipListFactory::kClassName()) {
+ // Expecting format
+ // skip_list:<lookahead>
+ if (opts_list.size() == 2) {
+ size_t lookahead = ParseSizeT(opts_list[1]);
+ result->reset(new SkipListFactory(lookahead));
+ } else {
+ result->reset(new SkipListFactory());
+ }
+ } else if (!config_options.ignore_unsupported_options) {
+ status = Status::NotSupported("Cannot load object in LITE mode ", id);
+ }
+#endif // ROCKSDB_LITE
}
-
- return Status::OK();
+ return status;
}
+#ifndef ROCKSDB_LITE
Status GetPlainTableOptionsFromMap(
const PlainTableOptions& table_options,
const std::unordered_map<std::string, std::string>& opts_map,
const std::string PlainTablePropertyNames::kNumBloomBlocks =
"rocksdb.plain.table.bloom.numblocks";
-} // namespace ROCKSDB_NAMESPACE
#endif // ROCKSDB_LITE
+} // namespace ROCKSDB_NAMESPACE
return Status::OK();
}
}
+namespace {
+// A hacky skip list mem table that triggers flush after number of entries.
+class SpecialMemTableRep : public MemTableRep {
+ public:
+ explicit SpecialMemTableRep(Allocator* allocator, MemTableRep* memtable,
+ int num_entries_flush)
+ : MemTableRep(allocator),
+ memtable_(memtable),
+ num_entries_flush_(num_entries_flush),
+ num_entries_(0) {}
+
+ virtual KeyHandle Allocate(const size_t len, char** buf) override {
+ return memtable_->Allocate(len, buf);
+ }
+
+ // Insert key into the list.
+ // REQUIRES: nothing that compares equal to key is currently in the list.
+ virtual void Insert(KeyHandle handle) override {
+ num_entries_++;
+ memtable_->Insert(handle);
+ }
+
+ void InsertConcurrently(KeyHandle handle) override {
+ num_entries_++;
+ memtable_->Insert(handle);
+ }
+
+ // Returns true iff an entry that compares equal to key is in the list.
+ virtual bool Contains(const char* key) const override {
+ return memtable_->Contains(key);
+ }
+
+ virtual size_t ApproximateMemoryUsage() override {
+ // Return a high memory usage when number of entries exceeds the threshold
+ // to trigger a flush.
+ return (num_entries_ < num_entries_flush_) ? 0 : 1024 * 1024 * 1024;
+ }
+
+ virtual void Get(const LookupKey& k, void* callback_args,
+ bool (*callback_func)(void* arg,
+ const char* entry)) override {
+ memtable_->Get(k, callback_args, callback_func);
+ }
+
+ uint64_t ApproximateNumEntries(const Slice& start_ikey,
+ const Slice& end_ikey) override {
+ return memtable_->ApproximateNumEntries(start_ikey, end_ikey);
+ }
+
+ virtual MemTableRep::Iterator* GetIterator(Arena* arena = nullptr) override {
+ return memtable_->GetIterator(arena);
+ }
+
+ virtual ~SpecialMemTableRep() override {}
+
+ private:
+ std::unique_ptr<MemTableRep> memtable_;
+ int num_entries_flush_;
+ int num_entries_;
+};
+class SpecialSkipListFactory : public MemTableRepFactory {
+ public:
+ // After number of inserts exceeds `num_entries_flush` in a mem table, trigger
+ // flush.
+ explicit SpecialSkipListFactory(int num_entries_flush)
+ : num_entries_flush_(num_entries_flush) {}
+
+ using MemTableRepFactory::CreateMemTableRep;
+ virtual MemTableRep* CreateMemTableRep(
+ const MemTableRep::KeyComparator& compare, Allocator* allocator,
+ const SliceTransform* transform, Logger* /*logger*/) override {
+ return new SpecialMemTableRep(
+ allocator,
+ factory_.CreateMemTableRep(compare, allocator, transform, nullptr),
+ num_entries_flush_);
+ }
+ static const char* kClassName() { return "SpecialSkipListFactory"; }
+ virtual const char* Name() const override { return kClassName(); }
+ std::string GetId() const override {
+ std::string id = Name();
+ if (num_entries_flush_ > 0) {
+ id.append(":").append(ROCKSDB_NAMESPACE::ToString(num_entries_flush_));
+ }
+ return id;
+ }
+
+ bool IsInsertConcurrentlySupported() const override {
+ return factory_.IsInsertConcurrentlySupported();
+ }
+
+ private:
+ SkipListFactory factory_;
+ int num_entries_flush_;
+};
+} // namespace
+
+MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush) {
+ RegisterTestLibrary();
+ return new SpecialSkipListFactory(num_entries_per_flush);
+}
#ifndef ROCKSDB_LITE
// This method loads existing test classes into the ObjectRegistry
static test::SimpleSuffixReverseComparator ssrc;
return &ssrc;
});
+ library.Register<MemTableRepFactory>(
+ std::string(SpecialSkipListFactory::kClassName()) + "(:[0-9]*)?",
+ [](const std::string& uri, std::unique_ptr<MemTableRepFactory>* guard,
+ std::string* /* errmsg */) {
+ auto colon = uri.find(":");
+ if (colon != std::string::npos) {
+ auto count = ParseInt(uri.substr(colon + 1));
+ guard->reset(new SpecialSkipListFactory(count));
+ } else {
+ guard->reset(new SpecialSkipListFactory(2));
+ }
+ return guard->get();
+ });
library.Register<MergeOperator>(
"Changling",
[](const std::string& uri, std::unique_ptr<MergeOperator>* guard,
return static_cast<int>(library.GetFactoryCount(&num_types));
}
+
+#endif // ROCKSDB_LITE
+
+void RegisterTestLibrary(const std::string& arg) {
+ static bool registered = false;
+ if (!registered) {
+ registered = true;
+#ifndef ROCKSDB_LITE
+ ObjectRegistry::Default()->AddLibrary("test", RegisterTestObjects, arg);
+#else
+ (void)arg;
#endif // ROCKSDB_LITE
+ }
+}
} // namespace test
} // namespace ROCKSDB_NAMESPACE
namespace ROCKSDB_NAMESPACE {
class FileSystem;
+class MemTableRepFactory;
class ObjectLibrary;
class Random;
class SequentialFile;
std::string name_;
};
+// The factory for the hacky skip list mem table that triggers flush after
+// number of entries exceeds a threshold.
+extern MemTableRepFactory* NewSpecialSkipListFactory(int num_entries_per_flush);
+
extern const Comparator* ComparatorWithU64Ts();
CompressionType RandomCompressionType(Random* rnd);
// Registers the testutil classes with the ObjectLibrary
int RegisterTestObjects(ObjectLibrary& library, const std::string& /*arg*/);
#endif // ROCKSDB_LITE
+
+// Register the testutil classes with the default ObjectRegistry/Library
+void RegisterTestLibrary(const std::string& arg = "");
} // namespace test
} // namespace ROCKSDB_NAMESPACE
"Stride length for the keys in a MultiGet batch");
DEFINE_bool(multiread_batched, false, "Use the new MultiGet API");
-enum RepFactory {
- kSkipList,
- kPrefixHash,
- kVectorRep,
- kHashLinkedList,
-};
-
-static enum RepFactory StringToRepFactory(const char* ctype) {
- assert(ctype);
-
- if (!strcasecmp(ctype, "skip_list"))
- return kSkipList;
- else if (!strcasecmp(ctype, "prefix_hash"))
- return kPrefixHash;
- else if (!strcasecmp(ctype, "vector"))
- return kVectorRep;
- else if (!strcasecmp(ctype, "hash_linkedlist"))
- return kHashLinkedList;
-
- fprintf(stdout, "Cannot parse memreptable %s\n", ctype);
- return kSkipList;
-}
-
-static enum RepFactory FLAGS_rep_factory;
DEFINE_string(memtablerep, "skip_list", "");
DEFINE_int64(hash_bucket_count, 1024 * 1024, "hash bucket count");
DEFINE_bool(use_plain_table, false, "if use plain table "
&ValidateTableCacheNumshardbits);
namespace ROCKSDB_NAMESPACE {
-
namespace {
+static Status CreateMemTableRepFactory(
+ const ConfigOptions& config_options,
+ std::shared_ptr<MemTableRepFactory>* factory) {
+ Status s;
+ if (!strcasecmp(FLAGS_memtablerep.c_str(), SkipListFactory::kNickName())) {
+ factory->reset(new SkipListFactory(FLAGS_skip_list_lookahead));
+#ifndef ROCKSDB_LITE
+ } else if (!strcasecmp(FLAGS_memtablerep.c_str(), "prefix_hash")) {
+ factory->reset(NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
+ } else if (!strcasecmp(FLAGS_memtablerep.c_str(),
+ VectorRepFactory::kNickName())) {
+ factory->reset(new VectorRepFactory());
+ } else if (!strcasecmp(FLAGS_memtablerep.c_str(), "hash_linkedlist")) {
+ factory->reset(NewHashLinkListRepFactory(FLAGS_hash_bucket_count));
+#endif // ROCKSDB_LITE
+ } else {
+ std::unique_ptr<MemTableRepFactory> unique;
+ s = MemTableRepFactory::CreateFromString(config_options, FLAGS_memtablerep,
+ &unique);
+ if (s.ok()) {
+ factory->reset(unique.release());
+ }
+ }
+ return s;
+}
+
struct ReportFileOpCounters {
std::atomic<int> open_counter_;
std::atomic<int> delete_counter_;
compressed);
}
- void PrintHeader() {
+ void PrintHeader(const Options& options) {
PrintEnvironment();
fprintf(stdout,
"Keys: %d bytes each (+ %d bytes user-defined timestamp)\n",
fprintf(stdout, "Compression: %s\n", compression.c_str());
fprintf(stdout, "Compression sampling rate: %" PRId64 "\n",
FLAGS_sample_for_compression);
-
- switch (FLAGS_rep_factory) {
- case kPrefixHash:
- fprintf(stdout, "Memtablerep: prefix_hash\n");
- break;
- case kSkipList:
- fprintf(stdout, "Memtablerep: skip_list\n");
- break;
- case kVectorRep:
- fprintf(stdout, "Memtablerep: vector\n");
- break;
- case kHashLinkedList:
- fprintf(stdout, "Memtablerep: hash_linkedlist\n");
- break;
+ if (options.memtable_factory != nullptr) {
+ fprintf(stdout, "Memtablerep: %s\n",
+ options.memtable_factory->GetId().c_str());
}
fprintf(stdout, "Perf Level: %d\n", FLAGS_perf_level);
ErrorExit();
}
Open(&open_options_);
- PrintHeader();
+ PrintHeader(open_options_);
std::stringstream benchmark_stream(FLAGS_benchmarks);
std::string name;
std::unique_ptr<ExpiredTimeFilter> filter;
printf("Initializing RocksDB Options from command-line flags\n");
Options& options = *opts;
ConfigOptions config_options(options);
+ config_options.ignore_unsupported_options = false;
assert(db_.db == nullptr);
FLAGS_level_compaction_dynamic_level_bytes;
options.max_bytes_for_level_multiplier =
FLAGS_max_bytes_for_level_multiplier;
- if ((FLAGS_prefix_size == 0) && (FLAGS_rep_factory == kPrefixHash ||
- FLAGS_rep_factory == kHashLinkedList)) {
+ Status s =
+ CreateMemTableRepFactory(config_options, &options.memtable_factory);
+ if (!s.ok()) {
+ fprintf(stderr, "Could not create memtable factory: %s\n",
+ s.ToString().c_str());
+ exit(1);
+ } else if ((FLAGS_prefix_size == 0) &&
+ (options.memtable_factory->IsInstanceOf("prefix_hash") ||
+ options.memtable_factory->IsInstanceOf("hash_linkedlist"))) {
fprintf(stderr, "prefix_size should be non-zero if PrefixHash or "
"HashLinkedList memtablerep is used\n");
exit(1);
}
- switch (FLAGS_rep_factory) {
- case kSkipList:
- options.memtable_factory.reset(new SkipListFactory(
- FLAGS_skip_list_lookahead));
- break;
-#ifndef ROCKSDB_LITE
- case kPrefixHash:
- options.memtable_factory.reset(
- NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
- break;
- case kHashLinkedList:
- options.memtable_factory.reset(NewHashLinkListRepFactory(
- FLAGS_hash_bucket_count));
- break;
- case kVectorRep:
- options.memtable_factory.reset(
- new VectorRepFactory
- );
- break;
-#else
- default:
- fprintf(stderr, "Only skip list is supported in lite mode\n");
- exit(1);
-#endif // ROCKSDB_LITE
- }
if (FLAGS_use_plain_table) {
#ifndef ROCKSDB_LITE
- if (FLAGS_rep_factory != kPrefixHash &&
- FLAGS_rep_factory != kHashLinkedList) {
- fprintf(stderr, "Waring: plain table is used with skipList\n");
+ if (!options.memtable_factory->IsInstanceOf("prefix_hash") &&
+ !options.memtable_factory->IsInstanceOf("hash_linkedlist")) {
+ fprintf(stderr, "Warning: plain table is used with %s\n",
+ options.memtable_factory->Name());
}
int bloom_bits_per_key = FLAGS_bloom_bits;
// merge operator options
if (!FLAGS_merge_operator.empty()) {
- Status s = MergeOperator::CreateFromString(
- config_options, FLAGS_merge_operator, &options.merge_operator);
+ s = MergeOperator::CreateFromString(config_options, FLAGS_merge_operator,
+ &options.merge_operator);
if (!s.ok()) {
fprintf(stderr, "invalid merge operator[%s]: %s\n",
FLAGS_merge_operator.c_str(), s.ToString().c_str());
FLAGS_value_size_distribution_type_e =
StringToDistributionType(FLAGS_value_size_distribution_type.c_str());
- FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
-
// Note options sanitization may increase thread pool sizes according to
// max_background_flushes/max_background_compactions/max_background_jobs
FLAGS_env->SetBackgroundThreads(FLAGS_num_high_pri_threads,
projects / rocksdb.git / commitdiff