rgw: use rwlock for cache

Use a window for cache lru updates, so that we don't need to get the
writer lock every read.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/rgw/rgw_cache.cc b/src/rgw/rgw_cache.cc
index d0afdcd389cd3e9b61243e02a4924b9773dc3701..94b3d0404edb623f1484308383fafe31bb495253 100644 (file)
--- a/src/rgw/rgw_cache.cc
+++ b/src/rgw/rgw_cache.cc
@@ -8,7 +8,7 @@ using namespace std;
 
 int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
 {
-  Mutex::Locker l(lock);
+  RWLock::RLocker l(lock);
 
   map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
   if (iter == cache_map.end()) {
@@ -17,7 +17,18 @@ int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
     return -ENOENT;
   }
 
-  touch_lru(name, iter->second.lru_iter);
+  ObjectCacheEntry& entry = iter->second;
+
+  if (lru_counter - entry.lru_promotion_ts > lru_window) {
+    ldout(cct, 20) << "cache get: touching lru, lru_counter=" << lru_counter << " promotion_ts=" << entry.lru_promotion_ts << dendl;
+    lock.unlock();
+    lock.get_write(); /* promote lock to writer */
+
+    /* check again, we might have lost a race here */
+    if (lru_counter - entry.lru_promotion_ts > lru_window) {
+      touch_lru(name, entry, iter->second.lru_iter);
+    }
+  }
 
   ObjectCacheInfo& src = iter->second.info;
   if ((src.flags & mask) != mask) {
@@ -35,7 +46,7 @@ int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask)
 
 void ObjectCache::put(string& name, ObjectCacheInfo& info)
 {
-  Mutex::Locker l(lock);
+  RWLock::WLocker l(lock);
 
   ldout(cct, 10) << "cache put: name=" << name << dendl;
   map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
@@ -48,7 +59,7 @@ void ObjectCache::put(string& name, ObjectCacheInfo& info)
   ObjectCacheEntry& entry = iter->second;
   ObjectCacheInfo& target = entry.info;
 
-  touch_lru(name, entry.lru_iter);
+  touch_lru(name, entry, entry.lru_iter);
 
   target.status = info.status;
 
@@ -93,7 +104,7 @@ void ObjectCache::put(string& name, ObjectCacheInfo& info)
 
 void ObjectCache::remove(string& name)
 {
-  Mutex::Locker l(lock);
+  RWLock::WLocker l(lock);
 
   map<string, ObjectCacheEntry>::iterator iter = cache_map.find(name);
   if (iter == cache_map.end())
@@ -105,7 +116,7 @@ void ObjectCache::remove(string& name)
   cache_map.erase(iter);
 }
 
-void ObjectCache::touch_lru(string& name, std::list<string>::iterator& lru_iter)
+void ObjectCache::touch_lru(string& name, ObjectCacheEntry& entry, std::list<string>::iterator& lru_iter)
 {
   while (lru_size > (size_t)cct->_conf->rgw_cache_lru_size) {
     list<string>::iterator iter = lru.begin();
@@ -136,6 +147,9 @@ void ObjectCache::touch_lru(string& name, std::list<string>::iterator& lru_iter)
     lru_iter = lru.end();
     --lru_iter;
   }
+
+  lru_counter++;
+  entry.lru_promotion_ts = lru_counter;
 }
 
 void ObjectCache::remove_lru(string& name, std::list<string>::iterator& lru_iter)

diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h
index 68720d0e6ac35ef8175d06892046f00fe9a5df79..68cffe0210209fe4ded1defb467f89f4e3d72d5c 100644 (file)
--- a/src/rgw/rgw_cache.h
+++ b/src/rgw/rgw_cache.h
@@ -7,6 +7,7 @@
 #include "include/types.h"
 #include "include/utime.h"
 #include "include/assert.h"
+#include "common/RWLock.h"
 
 enum {
   UPDATE_OBJ,
@@ -126,23 +127,31 @@ WRITE_CLASS_ENCODER(RGWCacheNotifyInfo)
 struct ObjectCacheEntry {
   ObjectCacheInfo info;
   std::list<string>::iterator lru_iter;
+  uint64_t lru_promotion_ts;
+
+  ObjectCacheEntry() : lru_promotion_ts(0) {}
 };
 
 class ObjectCache {
   std::map<string, ObjectCacheEntry> cache_map;
   std::list<string> lru;
   unsigned long lru_size;
-  Mutex lock;
+  unsigned long lru_counter;
+  unsigned long lru_window;
+  RWLock lock;
   CephContext *cct;
 
-  void touch_lru(string& name, std::list<string>::iterator& lru_iter);
+  void touch_lru(string& name, ObjectCacheEntry& entry, std::list<string>::iterator& lru_iter);
   void remove_lru(string& name, std::list<string>::iterator& lru_iter);
 public:
-  ObjectCache() : lru_size(0), lock("ObjectCache"), cct(NULL) { }
+  ObjectCache() : lru_size(0), lru_counter(0), lru_window(0), lock("ObjectCache"), cct(NULL) { }
   int get(std::string& name, ObjectCacheInfo& bl, uint32_t mask);
   void put(std::string& name, ObjectCacheInfo& bl);
   void remove(std::string& name);
-  void set_ctx(CephContext *_cct) { cct = _cct; }
+  void set_ctx(CephContext *_cct) {
+    cct = _cct;
+    lru_window = cct->_conf->rgw_cache_lru_size / 2;
+  }
 };
 
 template <class T>