From b362334c0d9dd474ab249739aa1dbb915cb864bf Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Mon, 28 Sep 2020 15:30:59 +0200 Subject: [PATCH] kv/RocksDBStore: Added ability to set compound options Imported rocksdb StringToMap that is able to properly parse compound options. Now we are properly respecting '{}' and no longer splitting '{}' if '=' is inside them. Signed-off-by: Adam Kupczyk --- src/kv/RocksDBStore.cc | 104 ++++++++++++++++-- .../objectstore/TestRocksdbOptionParse.cc | 6 +- 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/src/kv/RocksDBStore.cc b/src/kv/RocksDBStore.cc index b2f8ef01d4356..90d2f4063a1c8 100644 --- a/src/kv/RocksDBStore.cc +++ b/src/kv/RocksDBStore.cc @@ -231,7 +231,93 @@ static int string2bool(const string &val, bool &b_val) return 0; } } - + +namespace rocksdb { +extern std::string trim(const std::string& str); +} + +// this function is a modification of rocksdb's StringToMap: +// 1) accepts ' \n ; as separators +// 2) leaves compound options with enclosing { and } +rocksdb::Status StringToMap(const std::string& opts_str, + std::unordered_map* opts_map) +{ + using rocksdb::Status; + using rocksdb::trim; + assert(opts_map); + // Example: + // opts_str = "write_buffer_size=1024;max_write_buffer_number=2;" + // "nested_opt={opt1=1;opt2=2};max_bytes_for_level_base=100" + size_t pos = 0; + std::string opts = trim(opts_str); + while (pos < opts.size()) { + size_t eq_pos = opts.find('=', pos); + if (eq_pos == std::string::npos) { + return Status::InvalidArgument("Mismatched key value pair, '=' expected"); + } + std::string key = trim(opts.substr(pos, eq_pos - pos)); + if (key.empty()) { + return Status::InvalidArgument("Empty key found"); + } + + // skip space after '=' and look for '{' for possible nested options + pos = eq_pos + 1; + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + // Empty value at the end + if (pos >= opts.size()) { + (*opts_map)[key] = ""; + break; + } + if (opts[pos] == '{') { + int count = 1; + size_t brace_pos = pos + 1; + while (brace_pos < opts.size()) { + if (opts[brace_pos] == '{') { + ++count; + } else if (opts[brace_pos] == '}') { + --count; + if (count == 0) { + break; + } + } + ++brace_pos; + } + // found the matching closing brace + if (count == 0) { + //include both '{' and '}' + (*opts_map)[key] = trim(opts.substr(pos, brace_pos - pos + 1)); + // skip all whitespace and move to the next ';,' + // brace_pos points to the matching '}' + pos = brace_pos + 1; + while (pos < opts.size() && isspace(opts[pos])) { + ++pos; + } + if (pos < opts.size() && opts[pos] != ';' && opts[pos] != ',') { + return Status::InvalidArgument( + "Unexpected chars after nested options"); + } + ++pos; + } else { + return Status::InvalidArgument( + "Mismatched curly braces for nested options"); + } + } else { + size_t sc_pos = opts.find_first_of(",;", pos); + if (sc_pos == std::string::npos) { + (*opts_map)[key] = trim(opts.substr(pos)); + // It either ends with a trailing , ; or the last key-value pair + break; + } else { + (*opts_map)[key] = trim(opts.substr(pos, sc_pos - pos)); + } + pos = sc_pos + 1; + } + } + return Status::OK(); +} + int RocksDBStore::tryInterpret(const string &key, const string &val, rocksdb::Options &opt) { if (key == "compaction_threads") { @@ -280,13 +366,17 @@ int RocksDBStore::ParseOptionsFromStringStatic( { // keep aligned with func tryInterpret const set need_interp_keys = {"compaction_threads", "flusher_threads", "compact_on_mount", "disableWAL"}; + int r; + rocksdb::Status status; + std::unordered_map str_map; + status = StringToMap(opt_str, &str_map); + if (!status.ok()) { + dout(5) << __func__ << " error '" << status.getState() << + "' while parsing options '" << opt_str << "'" << dendl; + return -EINVAL; + } - map str_map; - int r = get_str_map(opt_str, &str_map, ",\n;"); - if (r < 0) - return r; - map::iterator it; - for (it = str_map.begin(); it != str_map.end(); ++it) { + for (auto it = str_map.begin(); it != str_map.end(); ++it) { string this_opt = it->first + "=" + it->second; rocksdb::Status status = rocksdb::GetOptionsFromString(opt, this_opt, &opt); diff --git a/src/test/objectstore/TestRocksdbOptionParse.cc b/src/test/objectstore/TestRocksdbOptionParse.cc index c2b08735383ee..c34ea6bc2361a 100644 --- a/src/test/objectstore/TestRocksdbOptionParse.cc +++ b/src/test/objectstore/TestRocksdbOptionParse.cc @@ -26,7 +26,8 @@ TEST(RocksDBOption, simple) { "max_bytes_for_level_base = 104857600;" "target_file_size_base = 10485760;" "num_levels = 3;" - "compression = kNoCompression;"; + "compression = kNoCompression;" + "compaction_options_universal = {min_merge_width=4;size_ratio=2;max_size_amplification_percent=500}"; int r = db->ParseOptionsFromString(options_string, options); ASSERT_EQ(0, r); ASSERT_EQ(536870912u, options.write_buffer_size); @@ -39,6 +40,9 @@ TEST(RocksDBOption, simple) { ASSERT_EQ(10485760u, options.target_file_size_base); ASSERT_EQ(3, options.num_levels); ASSERT_EQ(rocksdb::kNoCompression, options.compression); + ASSERT_EQ(2, options.compaction_options_universal.size_ratio); + ASSERT_EQ(4, options.compaction_options_universal.min_merge_width); + ASSERT_EQ(500, options.compaction_options_universal.max_size_amplification_percent); } TEST(RocksDBOption, interpret) { rocksdb::Options options; -- 2.47.3