common: move config vars to global, add cache eviction

Signed-off-by: Ali Maredia <amaredia@redhat.com>

diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in
index ea3c1f4fdb7e551ee0ecc05d84df1a641ea170c0..a1cbdb5ee843130e4091ff5794e5ca5bca45ee73 100644 (file)
--- a/src/common/options/global.yaml.in
+++ b/src/common/options/global.yaml.in
@@ -6325,3 +6325,19 @@ options:
   default: 0
   services:
   - mgr
+- name: labeled_perfcounters_cache_size
+  type: uint
+  level: advanced
+  desc: Number of labeled perfcounters the perfcounters cache can store
+  default: 1000
+  services:
+  - common
+- name: labeled_perfcounters_cache_eviction
+  type: bool
+  level: advanced
+  desc: Whether or not the labeled perfcounters cache evicts the least recently updated counters once labeled_perfcounters_cache_size is reached
+  default: false
+  see_also:
+  - labeled_perfcounters_cache_size
+  services:
+  - common

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index 976dfa53dac29154eb6181f7e33ad5873502caa6..33fe0a607948ac2fcd3717c12edf96d74898f692 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3692,10 +3692,3 @@ options:
   default: tank
   services:
   - rgw
-- name: rgw_labeled_perfcounters_size
-  type: uint
-  level: advanced
-  desc: number of labeled perfcounters the rgw can store
-  default: -1
-  services:
-  - rgw

diff --git a/src/common/perf_counters_cache.h b/src/common/perf_counters_cache.h
index d165ccc82bae677a5ffd52f9eb251a824a55dcd8..9ef68b2bcce66d3a4a7a78064b7f9b2c77f4ad46 100644 (file)
--- a/src/common/perf_counters_cache.h
+++ b/src/common/perf_counters_cache.h
@@ -4,27 +4,67 @@
 #include "common/perf_counters.h"
 #include "common/ceph_context.h"
 
+typedef std::list<std::string> labels_list;
+
+struct CacheEntry {
+  PerfCounters *counters;
+  labels_list::iterator pos;
+
+  CacheEntry(PerfCounters* _counters, labels_list::iterator _pos) {
+    counters = _counters;
+    pos = _pos;
+  }
+
+  ~CacheEntry() {}
+};
+
 class PerfCountersCache {
 
 private:
   CephContext *cct;
+  bool eviction;
   size_t curr_size = 0; 
   size_t target_size = 0; 
   int lower_bound = 0;
   int upper_bound = 0;
   std::function<void(PerfCountersBuilder*)> lpcb_init;
 
-  std::unordered_map<std::string, PerfCounters*> cache;
+  std::unordered_map<std::string, CacheEntry*> cache;
 
   ceph::common::PerfCounters* base_counters;
   std::string base_counters_name;
+  labels_list labels;
+
+  // move recently updated items in the list to the front
+  void update_labels_list(std::string label) {
+    labels.erase(cache[label]->pos);
+    cache[label]->pos = labels.insert(labels.begin(), label);
+  }
+
+  // removes least recently updated label from labels list
+  // removes oldest label's CacheEntry from cache
+  void remove_oldest_counters() {
+    std::string removed_label = labels.back();
+    labels.pop_back();
+
+    ceph_assert(cache[removed_label]->counters);
+    cct->get_perfcounters_collection()->remove(cache[removed_label]->counters);
+    delete cache[removed_label]->counters;
+    cache[removed_label]->counters = NULL;
+
+    delete cache[removed_label];
+    cache[removed_label] = NULL;
+
+    cache.erase(removed_label);
+    curr_size--;
+  }
 
 public:
 
   PerfCounters* get(std::string key) {
     auto got = cache.find(key);
     if(got != cache.end()) {
-      return got->second;
+      return got->second->counters;
     }
     return NULL;
   }
@@ -32,20 +72,27 @@ public:
   ceph::common::PerfCounters* add(std::string key, bool is_labeled = true) {
     auto counters = get(key);
     if (!counters) {
-      // perf counters instance creation code
-      if(curr_size < target_size) {
-        auto lpcb = new PerfCountersBuilder(cct, key, lower_bound, upper_bound, is_labeled);
-        lpcb_init(lpcb);
-
-        // add counters to builder
-        counters = lpcb->create_perf_counters();
-        delete lpcb;
-
-        // add new counters to collection, cache
-        cct->get_perfcounters_collection()->add(counters);
-        cache[key] = counters;
-        curr_size++;
+      // check to make sure cache isn't full
+      if(eviction) {
+        if(curr_size >= target_size) {
+          remove_oldest_counters();
+        }
       }
+
+      // perf counters instance creation code
+      auto lpcb = new PerfCountersBuilder(cct, key, lower_bound, upper_bound, is_labeled);
+      lpcb_init(lpcb);
+
+      // add counters to builder
+      counters = lpcb->create_perf_counters();
+      delete lpcb;
+
+      // add new counters to collection, cache
+      cct->get_perfcounters_collection()->add(counters);
+      labels_list::iterator pos = labels.insert(labels.begin(), key);
+      CacheEntry *m = new CacheEntry(counters, pos);
+      cache[key] = m;
+      curr_size++;
     }
     return counters;
   }
@@ -143,18 +190,18 @@ public:
   // for use right before destructor would get called
   void clear_cache() {
     for(auto it = cache.begin(); it != cache.end(); ++it ) {
-      ceph_assert(it->second);
-      cct->get_perfcounters_collection()->remove(it->second);
-      delete it->second;
+      ceph_assert(it->second->counters);
+      cct->get_perfcounters_collection()->remove(it->second->counters);
+      delete it->second->counters;
       curr_size--;
     }
   }
 
-  PerfCountersCache(CephContext *_cct, size_t _target_size, int _lower_bound, int _upper_bound, 
+  PerfCountersCache(CephContext *_cct, bool _eviction, size_t _target_size, int _lower_bound, int _upper_bound, 
       std::function<void(PerfCountersBuilder*)> _lpcb_init, std::string _base_counters_name) : cct(_cct), 
-      target_size(_target_size), lower_bound(_lower_bound), upper_bound(_upper_bound), 
-      lpcb_init(_lpcb_init), base_counters_name(_base_counters_name) {
-      base_counters = add(base_counters_name, false);
+      eviction(_eviction), target_size(_target_size), lower_bound(_lower_bound), upper_bound(_upper_bound), 
+      lpcb_init(_lpcb_init) {
+      base_counters = add(_base_counters_name, false);
   }
 
   ~PerfCountersCache() {}

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 5138589cb1e4035d5f883e8edecb0a0640f2d494..bfae8af1fd05a48bbc3ddaba258120859af17510 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -3911,8 +3911,9 @@ void RGWPutObj::execute(optional_yield y)
   std::string labels = ceph::perf_counters::cache_key("z_rgw", {{"Bucket", s->bucket_name}, {"User", s->user->get_display_name()}});
   // TODO delete below logging
   ldpp_dout(this, 20) << "labels for perf counters cache for l_rgw_metrics_put_b: " << labels << dendl;
-  uint64_t rgw_labeled_perfcounters_size = s->cct->_conf.get_val<uint64_t>("rgw_labeled_perfcounters_size");
-  ldpp_dout(this, 20) << "rgw_labeled_perfcounters_size is: " << rgw_labeled_perfcounters_size << dendl;
+  uint64_t target_size = s->cct->_conf.get_val<uint64_t>("labeled_perfcounters_cache_size");
+  bool eviction = s->cct->_conf.get_val<bool>("labeled_perfcounters_cache_eviction");
+  ldpp_dout(this, 20) << "target size for perf counters cache is: " << target_size << " eviction is: " << eviction << dendl;
 
   perf_counters_cache->add(labels);
 

diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc
index b3ae7aff7169306c227a7f34c1464f19fdf3bcff..2bff9346bdf3b1991b8695aa9da74b1e9d679494 100644 (file)
--- a/src/rgw/rgw_perf_counters.cc
+++ b/src/rgw/rgw_perf_counters.cc
@@ -69,8 +69,9 @@ int rgw_perf_start(CephContext *cct)
   cct->get_perfcounters_collection()->add(perfcounter);
   std::function<void(PerfCountersBuilder*)> lpcb_init = add_rgw_counters;
 
-  uint64_t target_size = cct->_conf.get_val<uint64_t>("rgw_labeled_perfcounters_size");
-  perf_counters_cache = new PerfCountersCache(cct, target_size, l_rgw_first, l_rgw_last, lpcb_init, "rgw_base");
+  uint64_t target_size = cct->_conf.get_val<uint64_t>("labeled_perfcounters_cache_size");
+  bool eviction = cct->_conf.get_val<bool>("labeled_perfcounters_cache_eviction");
+  perf_counters_cache = new PerfCountersCache(cct, eviction, target_size, l_rgw_first, l_rgw_last, lpcb_init, "rgw_base");
   return 0;
 }