From 6652213383688c9b6a2ac193ed024ef3ff1c12b3 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Wed, 30 Nov 2022 19:37:34 -0500 Subject: [PATCH] common: move config vars to global, add cache eviction Signed-off-by: Ali Maredia --- src/common/options/global.yaml.in | 16 ++++++ src/common/options/rgw.yaml.in | 7 --- src/common/perf_counters_cache.h | 91 +++++++++++++++++++++++-------- src/rgw/rgw_op.cc | 5 +- src/rgw/rgw_perf_counters.cc | 5 +- 5 files changed, 91 insertions(+), 33 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index ea3c1f4fdb7e5..a1cbdb5ee8431 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -6325,3 +6325,19 @@ options: default: 0 services: - mgr +- name: labeled_perfcounters_cache_size + type: uint + level: advanced + desc: Number of labeled perfcounters the perfcounters cache can store + default: 1000 + services: + - common +- name: labeled_perfcounters_cache_eviction + type: bool + level: advanced + desc: Whether or not the labeled perfcounters cache evicts the least recently updated counters once labeled_perfcounters_cache_size is reached + default: false + see_also: + - labeled_perfcounters_cache_size + services: + - common diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 976dfa53dac29..33fe0a607948a 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3692,10 +3692,3 @@ options: default: tank services: - rgw -- name: rgw_labeled_perfcounters_size - type: uint - level: advanced - desc: number of labeled perfcounters the rgw can store - default: -1 - services: - - rgw diff --git a/src/common/perf_counters_cache.h b/src/common/perf_counters_cache.h index d165ccc82bae6..9ef68b2bcce66 100644 --- a/src/common/perf_counters_cache.h +++ b/src/common/perf_counters_cache.h @@ -4,27 +4,67 @@ #include "common/perf_counters.h" #include "common/ceph_context.h" +typedef std::list labels_list; + +struct CacheEntry { + PerfCounters *counters; + labels_list::iterator pos; + + CacheEntry(PerfCounters* _counters, labels_list::iterator _pos) { + counters = _counters; + pos = _pos; + } + + ~CacheEntry() {} +}; + class PerfCountersCache { private: CephContext *cct; + bool eviction; size_t curr_size = 0; size_t target_size = 0; int lower_bound = 0; int upper_bound = 0; std::function lpcb_init; - std::unordered_map cache; + std::unordered_map cache; ceph::common::PerfCounters* base_counters; std::string base_counters_name; + labels_list labels; + + // move recently updated items in the list to the front + void update_labels_list(std::string label) { + labels.erase(cache[label]->pos); + cache[label]->pos = labels.insert(labels.begin(), label); + } + + // removes least recently updated label from labels list + // removes oldest label's CacheEntry from cache + void remove_oldest_counters() { + std::string removed_label = labels.back(); + labels.pop_back(); + + ceph_assert(cache[removed_label]->counters); + cct->get_perfcounters_collection()->remove(cache[removed_label]->counters); + delete cache[removed_label]->counters; + cache[removed_label]->counters = NULL; + + delete cache[removed_label]; + cache[removed_label] = NULL; + + cache.erase(removed_label); + curr_size--; + } public: PerfCounters* get(std::string key) { auto got = cache.find(key); if(got != cache.end()) { - return got->second; + return got->second->counters; } return NULL; } @@ -32,20 +72,27 @@ public: ceph::common::PerfCounters* add(std::string key, bool is_labeled = true) { auto counters = get(key); if (!counters) { - // perf counters instance creation code - if(curr_size < target_size) { - auto lpcb = new PerfCountersBuilder(cct, key, lower_bound, upper_bound, is_labeled); - lpcb_init(lpcb); - - // add counters to builder - counters = lpcb->create_perf_counters(); - delete lpcb; - - // add new counters to collection, cache - cct->get_perfcounters_collection()->add(counters); - cache[key] = counters; - curr_size++; + // check to make sure cache isn't full + if(eviction) { + if(curr_size >= target_size) { + remove_oldest_counters(); + } } + + // perf counters instance creation code + auto lpcb = new PerfCountersBuilder(cct, key, lower_bound, upper_bound, is_labeled); + lpcb_init(lpcb); + + // add counters to builder + counters = lpcb->create_perf_counters(); + delete lpcb; + + // add new counters to collection, cache + cct->get_perfcounters_collection()->add(counters); + labels_list::iterator pos = labels.insert(labels.begin(), key); + CacheEntry *m = new CacheEntry(counters, pos); + cache[key] = m; + curr_size++; } return counters; } @@ -143,18 +190,18 @@ public: // for use right before destructor would get called void clear_cache() { for(auto it = cache.begin(); it != cache.end(); ++it ) { - ceph_assert(it->second); - cct->get_perfcounters_collection()->remove(it->second); - delete it->second; + ceph_assert(it->second->counters); + cct->get_perfcounters_collection()->remove(it->second->counters); + delete it->second->counters; curr_size--; } } - PerfCountersCache(CephContext *_cct, size_t _target_size, int _lower_bound, int _upper_bound, + PerfCountersCache(CephContext *_cct, bool _eviction, size_t _target_size, int _lower_bound, int _upper_bound, std::function _lpcb_init, std::string _base_counters_name) : cct(_cct), - target_size(_target_size), lower_bound(_lower_bound), upper_bound(_upper_bound), - lpcb_init(_lpcb_init), base_counters_name(_base_counters_name) { - base_counters = add(base_counters_name, false); + eviction(_eviction), target_size(_target_size), lower_bound(_lower_bound), upper_bound(_upper_bound), + lpcb_init(_lpcb_init) { + base_counters = add(_base_counters_name, false); } ~PerfCountersCache() {} diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 5138589cb1e40..bfae8af1fd05a 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -3911,8 +3911,9 @@ void RGWPutObj::execute(optional_yield y) std::string labels = ceph::perf_counters::cache_key("z_rgw", {{"Bucket", s->bucket_name}, {"User", s->user->get_display_name()}}); // TODO delete below logging ldpp_dout(this, 20) << "labels for perf counters cache for l_rgw_metrics_put_b: " << labels << dendl; - uint64_t rgw_labeled_perfcounters_size = s->cct->_conf.get_val("rgw_labeled_perfcounters_size"); - ldpp_dout(this, 20) << "rgw_labeled_perfcounters_size is: " << rgw_labeled_perfcounters_size << dendl; + uint64_t target_size = s->cct->_conf.get_val("labeled_perfcounters_cache_size"); + bool eviction = s->cct->_conf.get_val("labeled_perfcounters_cache_eviction"); + ldpp_dout(this, 20) << "target size for perf counters cache is: " << target_size << " eviction is: " << eviction << dendl; perf_counters_cache->add(labels); diff --git a/src/rgw/rgw_perf_counters.cc b/src/rgw/rgw_perf_counters.cc index b3ae7aff71693..2bff9346bdf3b 100644 --- a/src/rgw/rgw_perf_counters.cc +++ b/src/rgw/rgw_perf_counters.cc @@ -69,8 +69,9 @@ int rgw_perf_start(CephContext *cct) cct->get_perfcounters_collection()->add(perfcounter); std::function lpcb_init = add_rgw_counters; - uint64_t target_size = cct->_conf.get_val("rgw_labeled_perfcounters_size"); - perf_counters_cache = new PerfCountersCache(cct, target_size, l_rgw_first, l_rgw_last, lpcb_init, "rgw_base"); + uint64_t target_size = cct->_conf.get_val("labeled_perfcounters_cache_size"); + bool eviction = cct->_conf.get_val("labeled_perfcounters_cache_eviction"); + perf_counters_cache = new PerfCountersCache(cct, eviction, target_size, l_rgw_first, l_rgw_last, lpcb_init, "rgw_base"); return 0; } -- 2.39.5