config: provide settings for the LevelDB stores we use

Now that we can set up the LevelDB options internally, provide
config options on the OSD and the Monitor. We leave the OSD values
at the defaults for now as they're performance-sensitive, but we
set new values on the Monitor so that it can scale to large PGMaps.
(Previously there were issues with large PGMaps taking forever to write;
these changes to the use of compression and the default block and
write buffers counteract them.)

Since we pass these variables through, users who are interested in
doing so now can test and tune them more appropriately.

Reported-by: Jim Schutt <jaschut@sandia.gov>
Signed-off-by: Greg Farnum <greg@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 19f3082ec4b82669bea1821ff2fa9815acff6734..3a9cfd708272241ea9327cca81cbc673e56c91e8 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -180,6 +180,12 @@ OPTION(mon_sync_debug_provider_fallback, OPT_STR, "") // monitor to be used as f
 OPTION(mon_sync_leader_kill_at, OPT_INT, 0) // kill the sync leader at a specifc point in the work flow
 OPTION(mon_sync_provider_kill_at, OPT_INT, 0) // kill the sync provider at a specific point in the work flow
 OPTION(mon_sync_requester_kill_at, OPT_INT, 0) // kill the sync requester at a specific point in the work flow
+OPTION(mon_leveldb_write_buffer_size, OPT_U64, 32*1024*1024) // monitor's leveldb write buffer size
+OPTION(mon_leveldb_cache_size, OPT_U64, 0) // monitor's leveldb cache size
+OPTION(mon_leveldb_block_size, OPT_U64, 4*1024*1024) // monitor's leveldb block size
+OPTION(mon_leveldb_bloom_size, OPT_INT, 0) // monitor's leveldb bloom bits per entry
+OPTION(mon_leveldb_max_open_files, OPT_INT, 0) // monitor's leveldb max open files
+OPTION(mon_leveldb_compression, OPT_BOOL, false) // monitor's leveldb uses compression
 OPTION(paxos_max_join_drift, OPT_INT, 10) // max paxos iterations before we must first sync the monitor stores
 OPTION(paxos_propose_interval, OPT_DOUBLE, 1.0)  // gather updates for this long before proposing a map update
 OPTION(paxos_min_wait, OPT_DOUBLE, 0.05)  // min time to gather updates for after period of inactivity
@@ -429,6 +435,12 @@ OPTION(osd_op_history_duration, OPT_U32, 600) // Oldest completed op to track
 OPTION(osd_target_transaction_size, OPT_INT, 30)     // to adjust various transactions that batch smaller items
 OPTION(osd_failsafe_full_ratio, OPT_FLOAT, .97) // what % full makes an OSD "full" (failsafe)
 OPTION(osd_failsafe_nearfull_ratio, OPT_FLOAT, .90) // what % full makes an OSD near full (failsafe)
+OPTION(osd_leveldb_write_buffer_size, OPT_U64, 0) // OSD's leveldb write buffer size
+OPTION(osd_leveldb_cache_size, OPT_U64, 0) // OSD's leveldb cache size
+OPTION(osd_leveldb_block_size, OPT_U64, 0) // OSD's leveldb block size
+OPTION(osd_leveldb_bloom_size, OPT_INT, 0) // OSD's leveldb bloom bits per entry
+OPTION(osd_leveldb_max_open_files, OPT_INT, 0) // OSD's leveldb max open files
+OPTION(osd_leveldb_compression, OPT_BOOL, true) // OSD's leveldb uses compression
 
 /**
  * osd_client_op_priority and osd_recovery_op_priority adjust the relative

diff --git a/src/mon/MonitorDBStore.h b/src/mon/MonitorDBStore.h
index 99c9b4fe9f51e15a09589f4ed3ffa77fef143e15..ac2703ec5e65965c72c38038a2d7ca905a265d6d 100644 (file)
--- a/src/mon/MonitorDBStore.h
+++ b/src/mon/MonitorDBStore.h
@@ -474,6 +474,12 @@ class MonitorDBStore
       assert(0 != "MonitorDBStore: error initializing level db back storage");
     }
     db.reset(db_ptr);
+    db->options.write_buffer_size = g_conf->mon_leveldb_write_buffer_size;
+    db->options.cache_size = g_conf->mon_leveldb_cache_size;
+    db->options.block_size = g_conf->mon_leveldb_block_size;
+    db->options.bloom_size = g_conf->mon_leveldb_bloom_size;
+    db->options.compression_enabled = g_conf->mon_leveldb_compression;
+    db->options.max_open_files = g_conf->mon_leveldb_max_open_files;
   }
   MonitorDBStore(LevelDBStore *db_ptr) {
     db.reset(db_ptr);

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 9d393022f1475a4e2bd6407d169a4ddc650a458c..5170412183e93328557096134636f2222e23680f 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -1647,6 +1647,14 @@ int FileStore::mount()
 
   {
     LevelDBStore *omap_store = new LevelDBStore(omap_dir);
+
+    omap_store->options.write_buffer_size = g_conf->osd_leveldb_write_buffer_size;
+    omap_store->options.cache_size = g_conf->osd_leveldb_cache_size;
+    omap_store->options.block_size = g_conf->osd_leveldb_block_size;
+    omap_store->options.bloom_size = g_conf->osd_leveldb_bloom_size;
+    omap_store->options.compression_enabled = g_conf->osd_leveldb_compression;
+    omap_store->options.max_open_files = g_conf->osd_leveldb_max_open_files;
+
     stringstream err;
     if (omap_store->create_and_open(err)) {
       delete omap_store;