From 15350de7ae5d5a9c1ce615ffd148882154d09b95 Mon Sep 17 00:00:00 2001
From: Haomai Wang <haomaiwang@gmail.com>
Date: Sat, 17 May 2014 13:41:58 +0800
Subject: [PATCH] Add header cache to DBObjectMap

OMap is used by PG to store PGLog, so it's called by each write operations.
Because leveldb don't play well in random read and "header_lock" limit the
concurrence. Add a LRU cache to avoid too much overload on seeking header.

Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
---
 src/common/config_opts.h    |  1 +
 src/common/simple_cache.hpp | 10 ++++++++++
 src/os/DBObjectMap.cc       | 31 ++++++++++++++++++++++++++-----
 src/os/DBObjectMap.h        |  8 ++++++--
 4 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index ee5d39598aa11..360f2ff511c9c 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -631,6 +631,7 @@ OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0)
 OPTION(filestore_debug_inject_read_err, OPT_BOOL, false)
 
 OPTION(filestore_debug_omap_check, OPT_BOOL, 0) // Expensive debugging check on sync
+OPTION(filestore_omap_header_cache_size, OPT_INT, 1024) 
 
 // Use omap for xattrs for attrs over
 // filestore_max_inline_xattr_size or
diff --git a/src/common/simple_cache.hpp b/src/common/simple_cache.hpp
index 60919fd97319e..30abd0a3ad8d2 100644
--- a/src/common/simple_cache.hpp
+++ b/src/common/simple_cache.hpp
@@ -62,6 +62,16 @@ public:
     }
   }
 
+  void clear(K key) {
+    Mutex::Locker l(lock);
+    typename map<K, typename list<pair<K, V> >::iterator>::iterator i =
+      contents.find(key);
+    if (i == contents.end())
+      return;
+    lru.erase(i->second);
+    contents.erase(i);
+  }
+
   void set_size(size_t new_size) {
     Mutex::Locker l(lock);
     max_size = new_size;
diff --git a/src/os/DBObjectMap.cc b/src/os/DBObjectMap.cc
index 5d2a2e616242c..176e1dc34675d 100644
--- a/src/os/DBObjectMap.cc
+++ b/src/os/DBObjectMap.cc
@@ -1114,18 +1114,31 @@ DBObjectMap::Header DBObjectMap::_lookup_map_header(const ghobject_t &oid)
   while (map_header_in_use.count(oid))
     header_cond.Wait(header_lock);
 
+  _Header *header = new _Header();
+  {
+    Mutex::Locker l(cache_lock);
+    if (caches.lookup(oid, header)) {
+      return Header(header, RemoveMapHeaderOnDelete(this, oid));
+    }
+  }
+
   map<string, bufferlist> out;
   set<string> to_get;
   to_get.insert(map_header_key(oid));
   int r = db->get(HOBJECT_TO_SEQ, to_get, &out);
-  if (r < 0)
-    return Header();
-  if (out.empty())
+  if (r < 0 || out.empty()) {
+    delete header;
     return Header();
-  
-  Header ret(new _Header(), RemoveMapHeaderOnDelete(this, oid));
+  }
+
+  Header ret(header, RemoveMapHeaderOnDelete(this, oid));
   bufferlist::iterator iter = out.begin()->second.begin();
   ret->decode(iter);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.add(oid, *ret);
+  }
+
   return ret;
 }
 
@@ -1220,6 +1233,10 @@ void DBObjectMap::remove_map_header(const ghobject_t &oid,
   set<string> to_remove;
   to_remove.insert(map_header_key(oid));
   t->rmkeys(HOBJECT_TO_SEQ, to_remove);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.clear(oid);
+  }
 }
 
 void DBObjectMap::set_map_header(const ghobject_t &oid, _Header header,
@@ -1231,6 +1248,10 @@ void DBObjectMap::set_map_header(const ghobject_t &oid, _Header header,
   map<string, bufferlist> to_set;
   header.encode(to_set[map_header_key(oid)]);
   t->set(HOBJECT_TO_SEQ, to_set);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.add(oid, header);
+  }
 }
 
 bool DBObjectMap::check_spos(const ghobject_t &oid,
diff --git a/src/os/DBObjectMap.h b/src/os/DBObjectMap.h
index a71c369fc028e..410a3439fec97 100644
--- a/src/os/DBObjectMap.h
+++ b/src/os/DBObjectMap.h
@@ -16,6 +16,7 @@
 #include "osd/osd_types.h"
 #include "common/Mutex.h"
 #include "common/Cond.h"
+#include "common/simple_cache.hpp"
 
 /**
  * DBObjectMap: Implements ObjectMap in terms of KeyValueDB
@@ -68,8 +69,9 @@ public:
   set<uint64_t> in_use;
   set<ghobject_t> map_header_in_use;
 
-  DBObjectMap(KeyValueDB *db) : db(db),
-				header_lock("DBOBjectMap")
+  DBObjectMap(KeyValueDB *db) : db(db), header_lock("DBOBjectMap"),
+                                cache_lock("DBObjectMap::CacheLock"),
+                                caches(g_conf->filestore_omap_header_cache_size)
     {}
 
   int set_keys(
@@ -281,6 +283,8 @@ public:
 private:
   /// Implicit lock on Header->seq
   typedef ceph::shared_ptr<_Header> Header;
+  Mutex cache_lock;
+  SimpleLRU<ghobject_t, _Header> caches;
 
   string map_header_key(const ghobject_t &oid);
   string header_key(uint64_t seq);
-- 
2.39.5