rgw: Expire entries in bucket info cache

To bound the degree to which an RGW instance can go out to lunch if
the watch/notify breaks down, force refresh of any cache entry over a
certain age.

Fifteen minutes by default, and expiration can be turned off entirely.

This is separate from the LRU. The LRU removes entries based on the
last time of access. This expiration patch forces refresh based on the
last time they were updated.

Fixes: http://tracker.ceph.com/issues/22517
Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
(cherry picked from commit 4489cb58a15647a31ac0546d70400af5668404cb)
Signed-off-by: Matt Benjamin <mbenjamin@redhat.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 062675d08ffc7242d5ed822209e5ae973d1e4e4f..5ac9442114f4a152539b1cbd9e7d6bdb07bb4c6a 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1268,6 +1268,12 @@ OPTION(rgw_override_bucket_index_max_shards, OPT_U32, 0)
  */
 OPTION(rgw_bucket_index_max_aio, OPT_U32, 8)
 
+/**
+ * Number of seconds before entries in the bucket info cache are
+ * assumed stale and re-fetched. Zero is never.
+ */
+OPTION(rgw_cache_expiry_interval, OPT_U64, 900 /* 15 min */)
+
 /**
  * whether or not the quota/gc threads should be started
  */

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 604b8c6a7aba3c44a70ad0ca3c30d181435d2b4d..aa05f0ab2f974731859a2c55e0ad32cdc87a210d 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3147,25 +3147,31 @@ public:
 
 template <class T>
 class RGWChainedCacheImpl : public RGWChainedCache {
+  ceph::timespan expiry;
   RWLock lock;
 
-  map<string, T> entries;
+  map<string, std::pair<T, ceph::coarse_mono_time>> entries;
 
 public:
   RGWChainedCacheImpl() : lock("RGWChainedCacheImpl::lock") {}
 
   void init(RGWRados *store) {
     store->register_chained_cache(this);
+    expiry = std::chrono::seconds(store->ctx()->_conf->rgw_cache_expiry_interval);
   }
 
   bool find(const string& key, T *entry) {
     RWLock::RLocker rl(lock);
-    typename map<string, T>::iterator iter = entries.find(key);
+    auto iter = entries.find(key);
     if (iter == entries.end()) {
       return false;
     }
+    if (expiry.count() &&
+       (ceph::coarse_mono_clock::now() - iter->second.second) > expiry) {
+      return false;
+    }
 
-    *entry = iter->second;
+    *entry = iter->second.first;
     return true;
   }
 
@@ -3179,7 +3185,10 @@ public:
   void chain_cb(const string& key, void *data) {
     T *entry = static_cast<T *>(data);
     RWLock::WLocker wl(lock);
-    entries[key] = *entry;
+    entries[key].first = *entry;
+    if (expiry.count() > 0) {
+      entries[key].second = ceph::coarse_mono_clock::now();
+    }
   }
 
   void invalidate(const string& key) {