mon: add a cache layer over MonitorDBStore

the cache of of leveldb does not perform well under some condition,
so we need a cache in our own stack.

* add an option "mon_osd_cache_size" to control the size of cache size
  of MonitorDBStore.

Fixes: #12638
Signed-off-by: Kefu Chai <kchai@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index d30e15e3428a944daae1d99ff0e7deaa29284c02..f7ef5bc6a44a46018cb740944212ccab11ec5ffa 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -186,6 +186,8 @@ OPTION(mon_sync_fs_threshold, OPT_INT, 5)   // sync() when writing this many obj
 OPTION(mon_compact_on_start, OPT_BOOL, false)  // compact leveldb on ceph-mon start
 OPTION(mon_compact_on_bootstrap, OPT_BOOL, false)  // trigger leveldb compaction on bootstrap
 OPTION(mon_compact_on_trim, OPT_BOOL, true)       // compact (a prefix) when we trim old states
+OPTION(mon_osd_cache_size, OPT_INT, 10)  // the size of osdmaps cache, not to rely on underlying store's cache
+
 OPTION(mon_tick_interval, OPT_INT, 5)
 OPTION(mon_subscribe_interval, OPT_DOUBLE, 300)
 OPTION(mon_delta_reset_interval, OPT_DOUBLE, 10)   // seconds of inactivity before we reset the pg delta to 0

diff --git a/src/mon/OSDMonitor.cc b/src/mon/OSDMonitor.cc
index 6c2893d0f503eb225ed00cabc14ba57bb67c12ed..7730670b84e55109e9627cc58ac5856fefabfaab 100644 (file)
--- a/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@ -71,6 +71,14 @@ static ostream& _prefix(std::ostream *_dout, Monitor *mon, OSDMap& osdmap) {
                << ").osd e" << osdmap.get_epoch() << " ";
 }
 
+OSDMonitor::OSDMonitor(CephContext *cct, Monitor *mn, Paxos *p, const string& service_name)
+ : PaxosService(mn, p, service_name),
+   inc_osd_cache(g_conf->mon_osd_cache_size),
+   full_osd_cache(g_conf->mon_osd_cache_size),
+   thrash_map(0), thrash_last_up_osd(-1),
+   op_tracker(cct, true, 1)
+{}
+
 bool OSDMonitor::_have_pending_crush()
 {
   return pending_inc.crush.length();
@@ -2409,6 +2417,29 @@ void OSDMonitor::send_incremental(epoch_t first,
   }
 }
 
+int OSDMonitor::get_version(version_t ver, bufferlist& bl)
+{
+    if (inc_osd_cache.lookup(ver, &bl)) {
+      return 0;
+    }
+    int ret = PaxosService::get_version(ver, bl);
+    if (!ret) {
+      inc_osd_cache.add(ver, bl);
+    }
+    return ret;
+}
+
+int OSDMonitor::get_version_full(version_t ver, bufferlist& bl)
+{
+    if (full_osd_cache.lookup(ver, &bl)) {
+      return 0;
+    }
+    int ret = PaxosService::get_version_full(ver, bl);
+    if (!ret) {
+      full_osd_cache.add(ver, bl);
+    }
+    return ret;
+}
 
 epoch_t OSDMonitor::blacklist(const entity_addr_t& a, utime_t until)
 {

diff --git a/src/mon/OSDMonitor.h b/src/mon/OSDMonitor.h
index ef9f15779c021710588a02c0a580872442413bb1..a185954e11880ccbb10a2a92fd58893185512084 100644 (file)
--- a/src/mon/OSDMonitor.h
+++ b/src/mon/OSDMonitor.h
@@ -26,6 +26,7 @@
 using namespace std;
 
 #include "include/types.h"
+#include "common/simple_cache.hpp"
 #include "msg/Messenger.h"
 
 #include "osd/OSDMap.h"
@@ -136,6 +137,9 @@ private:
 
   map<int,double> osd_weight;
 
+  SimpleLRU<version_t, bufferlist> inc_osd_cache;
+  SimpleLRU<version_t, bufferlist> full_osd_cache;
+
   void check_failures(utime_t now);
   bool check_failure(utime_t now, int target_osd, failure_info_t& fi);
 
@@ -157,7 +161,6 @@ private:
   void encode_pending(MonitorDBStore::TransactionRef t);
   void on_active();
   void on_shutdown();
-
   /**
    * we haven't delegated full version stashing to paxosservice for some time
    * now, making this function useless in current context.
@@ -387,11 +390,7 @@ private:
   int load_metadata(int osd, map<string, string>& m, ostream *err);
 
  public:
-  OSDMonitor(CephContext *cct, Monitor *mn, Paxos *p, string service_name)
-  : PaxosService(mn, p, service_name),
-    thrash_map(0), thrash_last_up_osd(-1),
-    op_tracker(cct, true, 1)
-  { }
+  OSDMonitor(CephContext *cct, Monitor *mn, Paxos *p, const string& service_name);
 
   void tick();  // check state, take actions
 
@@ -417,6 +416,9 @@ private:
     send_incremental(op, start);
   }
 
+  int get_version(version_t ver, bufferlist& bl) override;
+  int get_version_full(version_t ver, bufferlist& bl) override;
+
   epoch_t blacklist(const entity_addr_t& a, utime_t until);
 
   void dump_info(Formatter *f);

diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h
index c7f6cf919f93209db9fd66931c8730ca4392f5ac..54d26a69501dfce5afccf6d92b95c590c0c60896 100644 (file)
--- a/src/mon/PaxosService.h
+++ b/src/mon/PaxosService.h
@@ -892,7 +892,7 @@ public:
    * @param bl The bufferlist to be populated
    * @return 0 on success; <0 otherwise
    */
-  int get_version(version_t ver, bufferlist& bl) {
+  virtual int get_version(version_t ver, bufferlist& bl) {
     return mon->store->get(get_service_name(), ver, bl);
   }
   /**
@@ -902,7 +902,7 @@ public:
    * @param bl The bufferlist to be populated
    * @returns 0 on success; <0 otherwise
    */
-  int get_version_full(version_t ver, bufferlist& bl) {
+  virtual int get_version_full(version_t ver, bufferlist& bl) {
     string key = mon->store->combine_strings(full_prefix_name, ver);
     return mon->store->get(get_service_name(), key, bl);
   }