From: Haomai Wang <haomaiwang@gmail.com>
Date: Sat, 17 May 2014 05:41:58 +0000 (+0800)
Subject: Add header cache to DBObjectMap
X-Git-Tag: v0.83~135^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=15350de7ae5d5a9c1ce615ffd148882154d09b95;p=ceph.git

Add header cache to DBObjectMap

OMap is used by PG to store PGLog, so it's called by each write operations.
Because leveldb don't play well in random read and "header_lock" limit the
concurrence. Add a LRU cache to avoid too much overload on seeking header.

Signed-off-by: Haomai Wang <haomaiwang@gmail.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index ee5d39598aa1..360f2ff511c9 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -631,6 +631,7 @@ OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0)
 OPTION(filestore_debug_inject_read_err, OPT_BOOL, false)
 
 OPTION(filestore_debug_omap_check, OPT_BOOL, 0) // Expensive debugging check on sync
+OPTION(filestore_omap_header_cache_size, OPT_INT, 1024) 
 
 // Use omap for xattrs for attrs over
 // filestore_max_inline_xattr_size or
diff --git a/src/common/simple_cache.hpp b/src/common/simple_cache.hpp
index 60919fd97319..30abd0a3ad8d 100644
--- a/src/common/simple_cache.hpp
+++ b/src/common/simple_cache.hpp
@@ -62,6 +62,16 @@ public:
     }
   }
 
+  void clear(K key) {
+    Mutex::Locker l(lock);
+    typename map<K, typename list<pair<K, V> >::iterator>::iterator i =
+      contents.find(key);
+    if (i == contents.end())
+      return;
+    lru.erase(i->second);
+    contents.erase(i);
+  }
+
   void set_size(size_t new_size) {
     Mutex::Locker l(lock);
     max_size = new_size;
diff --git a/src/os/DBObjectMap.cc b/src/os/DBObjectMap.cc
index 5d2a2e616242..176e1dc34675 100644
--- a/src/os/DBObjectMap.cc
+++ b/src/os/DBObjectMap.cc
@@ -1114,18 +1114,31 @@ DBObjectMap::Header DBObjectMap::_lookup_map_header(const ghobject_t &oid)
   while (map_header_in_use.count(oid))
     header_cond.Wait(header_lock);
 
+  _Header *header = new _Header();
+  {
+    Mutex::Locker l(cache_lock);
+    if (caches.lookup(oid, header)) {
+      return Header(header, RemoveMapHeaderOnDelete(this, oid));
+    }
+  }
+
   map<string, bufferlist> out;
   set<string> to_get;
   to_get.insert(map_header_key(oid));
   int r = db->get(HOBJECT_TO_SEQ, to_get, &out);
-  if (r < 0)
-    return Header();
-  if (out.empty())
+  if (r < 0 || out.empty()) {
+    delete header;
     return Header();
-  
-  Header ret(new _Header(), RemoveMapHeaderOnDelete(this, oid));
+  }
+
+  Header ret(header, RemoveMapHeaderOnDelete(this, oid));
   bufferlist::iterator iter = out.begin()->second.begin();
   ret->decode(iter);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.add(oid, *ret);
+  }
+
   return ret;
 }
 
@@ -1220,6 +1233,10 @@ void DBObjectMap::remove_map_header(const ghobject_t &oid,
   set<string> to_remove;
   to_remove.insert(map_header_key(oid));
   t->rmkeys(HOBJECT_TO_SEQ, to_remove);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.clear(oid);
+  }
 }
 
 void DBObjectMap::set_map_header(const ghobject_t &oid, _Header header,
@@ -1231,6 +1248,10 @@ void DBObjectMap::set_map_header(const ghobject_t &oid, _Header header,
   map<string, bufferlist> to_set;
   header.encode(to_set[map_header_key(oid)]);
   t->set(HOBJECT_TO_SEQ, to_set);
+  {
+    Mutex::Locker l(cache_lock);
+    caches.add(oid, header);
+  }
 }
 
 bool DBObjectMap::check_spos(const ghobject_t &oid,
diff --git a/src/os/DBObjectMap.h b/src/os/DBObjectMap.h
index a71c369fc028..410a3439fec9 100644
--- a/src/os/DBObjectMap.h
+++ b/src/os/DBObjectMap.h
@@ -16,6 +16,7 @@
 #include "osd/osd_types.h"
 #include "common/Mutex.h"
 #include "common/Cond.h"
+#include "common/simple_cache.hpp"
 
 /**
  * DBObjectMap: Implements ObjectMap in terms of KeyValueDB
@@ -68,8 +69,9 @@ public:
   set<uint64_t> in_use;
   set<ghobject_t> map_header_in_use;
 
-  DBObjectMap(KeyValueDB *db) : db(db),
-				header_lock("DBOBjectMap")
+  DBObjectMap(KeyValueDB *db) : db(db), header_lock("DBOBjectMap"),
+                                cache_lock("DBObjectMap::CacheLock"),
+                                caches(g_conf->filestore_omap_header_cache_size)
     {}
 
   int set_keys(
@@ -281,6 +283,8 @@ public:
 private:
   /// Implicit lock on Header->seq
   typedef ceph::shared_ptr<_Header> Header;
+  Mutex cache_lock;
+  SimpleLRU<ghobject_t, _Header> caches;
 
   string map_header_key(const ghobject_t &oid);
   string header_key(uint64_t seq);