From 495d0314e0169d76179484b89b47c8ed1bbd635b Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Sat, 27 Jan 2018 16:18:53 -0500 Subject: [PATCH] rgw: Add cache introspection and manipulation Provide the ability to examine and delete elements from the cache. Fixes: http://tracker.ceph.com/issues/22603 Fixes: http://tracker.ceph.com/issues/22604 Signed-off-by: Adam C. Emerson (cherry picked from commit 69fafcaaed6c5ee32f12ac857dbf4af7e13483f8) Conflicts: Resolved by removing a few C++17isms. Make list helper public to deal with CentOS 7 GCC Bug --- src/rgw/rgw_cache.cc | 17 ++++--- src/rgw/rgw_cache.h | 72 ++++++++++++++++++++++++++--- src/rgw/rgw_rados.cc | 105 ++++++++++++++++++++++++++++++++++++++++++- src/rgw/rgw_rados.h | 42 ++++++++++++++++- 4 files changed, 222 insertions(+), 14 deletions(-) diff --git a/src/rgw/rgw_cache.cc b/src/rgw/rgw_cache.cc index fded2ead97e08..c60d72953665b 100644 --- a/src/rgw/rgw_cache.cc +++ b/src/rgw/rgw_cache.cc @@ -9,7 +9,7 @@ using namespace std; -int ObjectCache::get(string& name, ObjectCacheInfo& info, uint32_t mask, rgw_cache_entry_info *cache_info) +int ObjectCache::get(const string& name, ObjectCacheInfo& info, uint32_t mask, rgw_cache_entry_info *cache_info) { RWLock::RLocker l(lock); @@ -119,7 +119,7 @@ bool ObjectCache::chain_cache_entry(list& cache_info_ent return true; } -void ObjectCache::put(string& name, ObjectCacheInfo& info, rgw_cache_entry_info *cache_info) +void ObjectCache::put(const string& name, ObjectCacheInfo& info, rgw_cache_entry_info *cache_info) { RWLock::WLocker l(lock); @@ -192,17 +192,17 @@ void ObjectCache::put(string& name, ObjectCacheInfo& info, rgw_cache_entry_info target.version = info.version; } -void ObjectCache::remove(string& name) +bool ObjectCache::remove(const string& name) { RWLock::WLocker l(lock); if (!enabled) { - return; + return false; } map::iterator iter = cache_map.find(name); if (iter == cache_map.end()) - return; + return false; ldout(cct, 10) << "removing " << name << " from cache" << dendl; ObjectCacheEntry& entry = iter->second; @@ -215,9 +215,11 @@ void ObjectCache::remove(string& name) remove_lru(name, iter->second.lru_iter); cache_map.erase(iter); + return true; } -void ObjectCache::touch_lru(string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter) +void ObjectCache::touch_lru(const string& name, ObjectCacheEntry& entry, + std::list::iterator& lru_iter) { while (lru_size > (size_t)cct->_conf->rgw_cache_lru_size) { list::iterator iter = lru.begin(); @@ -256,7 +258,8 @@ void ObjectCache::touch_lru(string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter) +void ObjectCache::remove_lru(const string& name, + std::list::iterator& lru_iter) { if (lru_iter == lru.end()) return; diff --git a/src/rgw/rgw_cache.h b/src/rgw/rgw_cache.h index 022dc3624ff17..023db75ff76c2 100644 --- a/src/rgw/rgw_cache.h +++ b/src/rgw/rgw_cache.h @@ -149,16 +149,35 @@ class ObjectCache { bool enabled; ceph::timespan expiry; - void touch_lru(string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter); - void remove_lru(string& name, std::list::iterator& lru_iter); + void touch_lru(const string& name, ObjectCacheEntry& entry, std::list::iterator& lru_iter); + void remove_lru(const string& name, std::list::iterator& lru_iter); void invalidate_lru(ObjectCacheEntry& entry); void do_invalidate_all(); public: ObjectCache() : lru_size(0), lru_counter(0), lru_window(0), lock("ObjectCache"), cct(NULL), enabled(false) { } - int get(std::string& name, ObjectCacheInfo& bl, uint32_t mask, rgw_cache_entry_info *cache_info); - void put(std::string& name, ObjectCacheInfo& bl, rgw_cache_entry_info *cache_info); - void remove(std::string& name); + int get(const std::string& name, ObjectCacheInfo& bl, uint32_t mask, rgw_cache_entry_info *cache_info); + boost::optional get(const std::string& name) { + boost::optional info{boost::in_place_init}; + auto r = get(name, *info, 0, nullptr); + return r < 0 ? boost::none : info; + } + + template + void for_each(const F& f) { + RWLock::RLocker l(lock); + if (enabled) { + auto now = ceph::coarse_mono_clock::now(); + for (const auto& kv : cache_map) { + if (expiry.count() && ((now - kv.second.info.time_added) < expiry)) { + f(kv.first, kv.second); + } + } + } + } + + void put(const std::string& name, ObjectCacheInfo& bl, rgw_cache_entry_info *cache_info); + bool remove(const std::string& name); void set_ctx(CephContext *_cct) { cct = _cct; lru_window = cct->_conf->rgw_cache_lru_size / 2; @@ -254,6 +273,11 @@ public: bool chain_cache_entry(list& cache_info_entries, RGWChainedCache::Entry *chained_entry) override { return cache.chain_cache_entry(cache_info_entries, chained_entry); } + void call_list(const boost::optional& filter, + Formatter* format) override; + bool call_inspect(const std::string& target, Formatter* format) override; + bool call_erase(const std::string& target) override; + void call_zap() override; }; template @@ -589,4 +613,42 @@ int RGWCache::watch_cb(uint64_t notify_id, return 0; } +template +void RGWCache::call_list(const boost::optional& filter, + Formatter* f) +{ + cache.for_each( + [this, &filter, f] (const string& name, const ObjectCacheEntry& entry) { + if (!filter || name.find(*filter) != name.npos) { + T::cache_list_dump_helper(f, name, entry.info.meta.mtime, + entry.info.meta.size); + } + }); +} + +template +bool RGWCache::call_inspect(const std::string& target, Formatter* f) +{ + if (const auto entry = cache.get(target)) { + f->open_object_section("cache_entry"); + f->dump_string("name", target.c_str()); + entry->dump(f); + f->close_section(); + return true; + } else { + return false; + } +} + +template +bool RGWCache::call_erase(const std::string& target) +{ + return cache.remove(target); +} + +template +void RGWCache::call_zap() +{ + cache.invalidate_all(); +} #endif diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 460029be92541..9f10e9e487439 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1,4 +1,3 @@ - // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab @@ -2792,6 +2791,22 @@ int RGWPutObjProcessor_Atomic::do_complete(size_t accounted_size, const string& return 0; } +const char* RGWRados::admin_commands[4][3] = { + { "cache list", + "cache list name=filter,type=CephString,req=false", + "cache list [filter_str]: list object cache, possibly matching substrings" }, + { "cache inspect", + "cache inspect name=target,type=CephString,req=true", + "cache inspect target: print cache element" }, + { "cache erase", + "cache erase name=target,type=CephString,req=true", + "cache erase target: erase element from cache" }, + { "cache zap", + "cache zap", + "cache zap: erase all elements from cache" } +}; + + int RGWRados::watch(const string& oid, uint64_t *watch_handle, librados::WatchCtx2 *ctx) { int r = control_pool_ctx.watch2(oid, watch_handle, ctx); if (r < 0) @@ -3669,6 +3684,15 @@ bool RGWIndexCompletionManager::handle_completion(completion_t cb, complete_op_d void RGWRados::finalize() { + auto admin_socket = cct->get_admin_socket(); + for (auto cmd : admin_commands) { + int r = admin_socket->unregister_command(cmd[0]); + if (r < 0) { + lderr(cct) << "ERROR: fail to unregister admin socket command (r=" << r + << ")" << dendl; + } + } + if (run_sync_thread) { Mutex::Locker l(meta_sync_thread_lock); meta_sync_processor_thread->stop(); @@ -3773,6 +3797,17 @@ void RGWRados::finalize() int RGWRados::init_rados() { int ret = 0; + auto admin_socket = cct->get_admin_socket(); + for (auto cmd : admin_commands) { + int r = admin_socket->register_command(cmd[0], cmd[1], this, + cmd[2]); + if (r < 0) { + lderr(cct) << "ERROR: fail to register admin socket command (r=" << r + << ")" << dendl; + return r; + } + } + auto handles = std::vector{cct->_conf->rgw_num_rados_handles}; for (auto& r : handles) { @@ -14023,3 +14058,71 @@ int rgw_compression_info_from_attrset(map& attrs, bool& need } } +bool RGWRados::call(std::string command, cmdmap_t& cmdmap, std::string format, + bufferlist& out) +{ + if (command == "cache list") { + boost::optional filter; + auto i = cmdmap.find("filter"); + if (i != cmdmap.cend()) { + filter = boost::get(i->second); + } + std::unique_ptr f(ceph::Formatter::create(format, "table")); + if (f) { + f->open_array_section("cache_entries"); + call_list(filter, f.get()); + f->close_section(); + f->flush(out); + return true; + } else { + out.append("Unable to create Formatter.\n"); + return false; + } + } else if (command == "cache inspect") { + std::unique_ptr f(ceph::Formatter::create(format, "json-pretty")); + if (f) { + const auto& target = boost::get(cmdmap["target"]); + if (call_inspect(target, f.get())) { + f->flush(out); + return true; + } else { + out.append(string("Unable to find entry ") + target + string(".\n")); + return false; + } + } else { + out.append("Unable to create Formatter.\n"); + return false; + } + } else if (command == "cache erase") { + const auto& target = boost::get(cmdmap["target"]); + if (call_erase(target)) { + return true; + } else { + out.append(string("Unable to find entry ") + target + string(".\n")); + return false; + } + } else if (command == "cache zap") { + call_zap(); + return true; + } + return false; +} + +void RGWRados::call_list(const boost::optional&, + ceph::Formatter*) +{ + return; +} + +bool RGWRados::call_inspect(const std::string&, Formatter*) +{ + return false; +} + +bool RGWRados::call_erase(const std::string&) { + return false; +} + +void RGWRados::call_zap() { + return; +} diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 58848b0419d90..13de667c06045 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -8,6 +8,7 @@ #include "include/rados/librados.hpp" #include "include/Context.h" +#include "common/admin_socket.h" #include "common/RefCountedObj.h" #include "common/RWLock.h" #include "common/ceph_time.h" @@ -2204,7 +2205,7 @@ struct tombstone_entry { class RGWIndexCompletionManager; -class RGWRados +class RGWRados : public AdminSocketHook { friend class RGWGC; friend class RGWMetaNotifier; @@ -2220,6 +2221,8 @@ class RGWRados friend class BucketIndexLockGuard; friend class RGWCompleteMultipart; + static const char* admin_commands[4][3]; + /** Open the pool used as root for this gateway */ int open_root_pool_ctx(); int open_gc_pool_ctx(); @@ -3411,6 +3414,43 @@ private: boost::optional refresh_version); public: + bool call(std::string command, cmdmap_t& cmdmap, std::string format, + bufferlist& out) override final; + + // Should really be protected, but some older GCCs don't handle + // access control properly with lambdas defined in member functions + // of child classes. + + void cache_list_dump_helper(Formatter* f, + const std::string& name, + const ceph::real_time mtime, + const std::uint64_t size) { + f->dump_string("name", name); + f->dump_string("mtime", ceph::to_iso_8601(mtime)); + f->dump_unsigned("size", size); + } + +protected: + + // `call_list` must iterate over all cache entries and call + // `cache_list_dump_helper` with the supplied Formatter on any that + // include `filter` as a substring. + // + virtual void call_list(const boost::optional& filter, + Formatter* format); + // `call_inspect` must look up the requested target and, if found, + // dump it to the supplied Formatter and return true. If not found, + // it must return false. + // + virtual bool call_inspect(const std::string& target, Formatter* format); + + // `call_erase` must erase the requested target and return true. If + // the requested target does not exist, it should return false. + virtual bool call_erase(const std::string& target); + + // `call_zap` must erase the cache. + virtual void call_zap(); +public: int get_bucket_info(RGWObjectCtx& obj_ctx, const string& tenant_name, const string& bucket_name, -- 2.39.5