From 76f811b7e95944a733a09b72e3667fca28c77da0 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Wed, 7 Oct 2015 12:16:22 -0400 Subject: [PATCH] rgw: refactor reconfigure out of RGWRealmWatcher add a RGWRealmReloader class to handle reconfiguration, and use a Watcher interface to get notifications from RGWRealmWatcher Signed-off-by: Casey Bodley --- src/CMakeLists.txt | 3 +- src/rgw/Makefile.am | 4 +- src/rgw/rgw_main.cc | 11 ++- src/rgw/rgw_realm_reloader.cc | 142 ++++++++++++++++++++++++++++++++++ src/rgw/rgw_realm_reloader.h | 62 +++++++++++++++ src/rgw/rgw_realm_watcher.cc | 142 +++------------------------------- src/rgw/rgw_realm_watcher.h | 52 ++++--------- 7 files changed, 244 insertions(+), 172 deletions(-) create mode 100644 src/rgw/rgw_realm_reloader.cc create mode 100644 src/rgw/rgw_realm_reloader.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e71f43a204f2..472d59ac63d86 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1170,6 +1170,8 @@ if(${WITH_RADOSGW}) rgw/rgw_sync.cc rgw/rgw_data_sync.cc rgw/rgw_dencoder.cc + rgw/rgw_realm_reloader.cc + rgw/rgw_realm_watcher.cc rgw/rgw_coroutine.cc rgw/rgw_cr_rados.cc rgw/rgw_object_expirer_core.cc @@ -1209,7 +1211,6 @@ if(${WITH_RADOSGW}) rgw/rgw_civetweb_log.cc civetweb/src/civetweb.c rgw/rgw_main.cc - rgw/rgw_realm_watcher.cc rgw/rgw_sync.cc rgw/rgw_data_sync.cc) diff --git a/src/rgw/Makefile.am b/src/rgw/Makefile.am index 8d7816b491555..5ac4bda8e74cb 100644 --- a/src/rgw/Makefile.am +++ b/src/rgw/Makefile.am @@ -59,6 +59,8 @@ librgw_la_SOURCES = \ rgw/rgw_keystone.cc \ rgw/rgw_quota.cc \ rgw/rgw_dencoder.cc \ + rgw/rgw_realm_reloader.cc \ + rgw/rgw_realm_watcher.cc \ rgw/rgw_object_expirer_core.cc \ rgw/rgw_website.cc \ rgw/rgw_sync.cc \ @@ -115,7 +117,6 @@ radosgw_SOURCES = \ rgw/rgw_swift.cc \ rgw/rgw_swift_auth.cc \ rgw/rgw_loadgen.cc \ - rgw/rgw_realm_watcher.cc \ rgw/rgw_main.cc radosgw_CFLAGS = -I$(srcdir)/civetweb/include radosgw_LDADD = $(LIBRGW) $(LIBCIVETWEB) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL) @@ -201,6 +202,7 @@ noinst_HEADERS += \ rgw/rgw_user.h \ rgw/rgw_bucket.h \ rgw/rgw_keystone.h \ + rgw/rgw_realm_reloader.h \ rgw/rgw_realm_watcher.h \ rgw/rgw_civetweb.h \ rgw/rgw_boost_asio_coroutine.h \ diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 2ecdfe3b5fddb..cfe60dc2496a3 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -39,7 +39,7 @@ #include "rgw_acl.h" #include "rgw_user.h" #include "rgw_op.h" -#include "rgw_realm_watcher.h" +#include "rgw_realm_reloader.h" #include "rgw_rest.h" #include "rgw_rest_s3.h" #include "rgw_rest_swift.h" @@ -1100,8 +1100,8 @@ public: } }; -// FrontendPauser implementation for RGWRealmWatcher -class RGWFrontendPauser : public RGWRealmWatcher::FrontendPauser { +// FrontendPauser implementation for RGWRealmReloader +class RGWFrontendPauser : public RGWRealmReloader::FrontendPauser { std::list &frontends; public: RGWFrontendPauser(std::list &frontends) @@ -1341,7 +1341,10 @@ int main(int argc, const char **argv) // add a watcher to respond to realm configuration changes RGWFrontendPauser pauser(fes); - RGWRealmWatcher realm_watcher(g_ceph_context, store, &pauser); + RGWRealmReloader reloader(store, &pauser); + + RGWRealmWatcher realm_watcher(g_ceph_context, store->realm); + realm_watcher.set_watcher(&reloader); wait_shutdown(); diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc new file mode 100644 index 0000000000000..f55b1a36ecd33 --- /dev/null +++ b/src/rgw/rgw_realm_reloader.cc @@ -0,0 +1,142 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#include "rgw_realm_reloader.h" +#include "rgw_rados.h" + +#include "rgw_bucket.h" +#include "rgw_log.h" +#include "rgw_rest.h" +#include "rgw_user.h" + +#define dout_subsys ceph_subsys_rgw + +#undef dout_prefix +#define dout_prefix (*_dout << "rgw realm reloader: ") + + +// safe callbacks from SafeTimer are unneccessary. reload() can take a long +// time, so we don't want to hold the mutex and block handle_notify() for the +// duration +static constexpr bool USE_SAFE_TIMER_CALLBACKS = false; + + +RGWRealmReloader::RGWRealmReloader(RGWRados*& store, FrontendPauser* frontends) + : store(store), + frontends(frontends), + timer(store->ctx(), mutex, USE_SAFE_TIMER_CALLBACKS), + mutex("RGWRealmReloader"), + reload_scheduled(nullptr) +{ + timer.init(); +} + +RGWRealmReloader::~RGWRealmReloader() +{ + Mutex::Locker lock(mutex); + timer.shutdown(); +} + +class RGWRealmReloader::C_Reload : public Context { + RGWRealmReloader* reloader; + public: + C_Reload(RGWRealmReloader* reloader) : reloader(reloader) {} + void finish(int r) { reloader->reload(); } +}; + +void RGWRealmReloader::handle_notify(bufferlist::iterator& p) +{ + CephContext *const cct = store->ctx(); + + Mutex::Locker lock(mutex); + if (reload_scheduled) { + ldout(cct, 4) << "Notification on realm, reconfiguration " + "already scheduled" << dendl; + return; + } + + reload_scheduled = new C_Reload(this); + cond.SignalOne(); // wake reload() if it blocked on a bad configuration + + // schedule reload() with a delay so we can batch up changes + auto delay = cct->_conf->rgw_realm_reconfigure_delay; + timer.add_event_after(delay, reload_scheduled); + + ldout(cct, 4) << "Notification on realm, reconfiguration scheduled in " + << delay << 's' << dendl; +} + +void RGWRealmReloader::reload() +{ + CephContext *const cct = store->ctx(); + ldout(cct, 1) << "Pausing frontends for realm update..." << dendl; + + frontends->pause(); + + // destroy the existing store + RGWStoreManager::close_storage(store); + store = nullptr; + + { + // allow a new notify to reschedule us. it's important that we do this + // before we start loading the new realm, or we could miss some updates + Mutex::Locker lock(mutex); + reload_scheduled = nullptr; + } + + while (!store) { + // recreate and initialize a new store + store = RGWStoreManager::get_storage(cct, + cct->_conf->rgw_enable_gc_threads, + cct->_conf->rgw_enable_quota_threads, + cct->_conf->rgw_run_sync_thread); + + RGWRados* store_cleanup = nullptr; + { + Mutex::Locker lock(mutex); + + // failure to recreate RGWRados is not a recoverable error, but we + // don't want to assert or abort the entire cluster. instead, just + // sleep until we get another notification, and retry until we get + // a working configuration + if (store == nullptr) { + lderr(cct) << "Failed to reinitialize RGWRados after a realm " + "configuration update. Waiting for a new update." << dendl; + + // sleep until another event is scheduled + while (!reload_scheduled) + cond.Wait(mutex); + + ldout(cct, 1) << "Woke up with a new configuration, retrying " + "RGWRados initialization." << dendl; + } + + if (reload_scheduled) { + // cancel the event; we'll handle it now + timer.cancel_event(reload_scheduled); + reload_scheduled = nullptr; + + // if we successfully created a store, clean it up outside of the lock, + // then continue to loop and recreate another + std::swap(store, store_cleanup); + } + } + + if (store_cleanup) { + ldout(cct, 4) << "Got another notification, restarting RGWRados " + "initialization." << dendl; + + RGWStoreManager::close_storage(store_cleanup); + } + } + + // finish initializing the new store + rgw_rest_init(cct, store->get_zonegroup()); + rgw_user_init(store); + rgw_bucket_init(store->meta_mgr); + rgw_log_usage_init(cct, store); + + ldout(cct, 1) << "Resuming frontends with new realm configuration." << dendl; + + frontends->resume(store); +} diff --git a/src/rgw/rgw_realm_reloader.h b/src/rgw/rgw_realm_reloader.h new file mode 100644 index 0000000000000..96e1ca68a7a6f --- /dev/null +++ b/src/rgw/rgw_realm_reloader.h @@ -0,0 +1,62 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +#ifndef RGW_REALM_RELOADER_H +#define RGW_REALM_RELOADER_H + +#include "rgw_realm_watcher.h" +#include "common/Cond.h" + +class RGWRados; + +/** + * RGWRealmReloader responds to notifications by recreating RGWRados with the + * updated realm configuration. + */ +class RGWRealmReloader : public RGWRealmWatcher::Watcher { + public: + /** + * FrontendPauser is an interface to pause/resume frontends. Frontend + * cooperation is required to ensure that they stop issuing requests on the + * old RGWRados instance, and restart with the updated configuration. + * + * This abstraction avoids a depency on class RGWFrontend, which is only + * defined in rgw_main.cc + */ + class FrontendPauser { + public: + virtual ~FrontendPauser() = default; + + /// pause all frontends while realm reconfiguration is in progress + virtual void pause() = 0; + /// resume all frontends with the given RGWRados instance + virtual void resume(RGWRados* store) = 0; + }; + + RGWRealmReloader(RGWRados*& store, FrontendPauser* frontends); + ~RGWRealmReloader(); + + /// respond to realm notifications by scheduling a reload() + void handle_notify(bufferlist::iterator& p) override; + + private: + /// pause frontends and replace the RGWRados instance + void reload(); + + class C_Reload; //< Context that calls reload() + + /// main()'s RGWRados pointer as a reference, modified by reload() + RGWRados*& store; + FrontendPauser *const frontends; + + /// reload() takes a significant amount of time, so we don't want to run + /// it in the handle_notify() thread. we choose a timer thread because we + /// also want to add a delay (see rgw_realm_reconfigure_delay) so that we + /// can batch up notifications within that window + SafeTimer timer; + Mutex mutex; //< protects access to timer and reload_scheduled + Cond cond; //< to signal reload() after an invalid realm config + C_Reload* reload_scheduled; //< reload() context if scheduled +}; + +#endif // RGW_REALM_RELOADER_H diff --git a/src/rgw/rgw_realm_watcher.cc b/src/rgw/rgw_realm_watcher.cc index 3878a5c4401fa..bbbe70d5d59ad 100644 --- a/src/rgw/rgw_realm_watcher.cc +++ b/src/rgw/rgw_realm_watcher.cc @@ -1,173 +1,57 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include - #include "common/errno.h" #include "rgw_realm_watcher.h" #include "rgw_rados.h" -#include "rgw_bucket.h" -#include "rgw_log.h" -#include "rgw_rest.h" -#include "rgw_user.h" - #define dout_subsys ceph_subsys_rgw #undef dout_prefix #define dout_prefix (*_dout << "rgw realm watcher: ") -// safe callbacks from SafeTimer are unneccessary. reconfigure() can take a long -// time, so we don't want to hold the mutex and block handle_notify() for the -// duration -static constexpr bool USE_SAFE_TIMER_CALLBACKS = false; - - -RGWRealmWatcher::RGWRealmWatcher(CephContext *cct, RGWRados *&store, - FrontendPauser *frontends) - : cct(cct), - store(store), - frontends(frontends), - timer(cct, mutex, USE_SAFE_TIMER_CALLBACKS), - mutex("RGWRealmWatcher"), - reconfigure_scheduled(nullptr) +RGWRealmWatcher::RGWRealmWatcher(CephContext* cct, RGWRealm& realm) + : cct(cct), watcher(nullptr) { // no default realm, nothing to watch - if (store->realm.get_id().empty()) { + if (realm.get_id().empty()) { ldout(cct, 4) << "No realm, disabling dynamic reconfiguration." << dendl; return; } // establish the watch on RGWRealm - int r = watch_start(); + int r = watch_start(realm); if (r < 0) { lderr(cct) << "Failed to establish a watch on RGWRealm, " "disabling dynamic reconfiguration." << dendl; return; } - - timer.init(); } RGWRealmWatcher::~RGWRealmWatcher() { watch_stop(); - - Mutex::Locker lock(mutex); - timer.shutdown(); } - -void RGWRealmWatcher::reconfigure() +void RGWRealmWatcher::set_watcher(Watcher* watcher) { - ldout(cct, 1) << "Pausing frontends for realm update..." << dendl; - - frontends->pause(); - - // destroy the existing store - RGWStoreManager::close_storage(store); - store = nullptr; - - { - // allow a new notify to reschedule us. it's important that we do this - // before we start loading the new realm, or we could miss some updates - Mutex::Locker lock(mutex); - reconfigure_scheduled = nullptr; - } - - while (!store) { - // recreate and initialize a new store - store = RGWStoreManager::get_storage(cct, - cct->_conf->rgw_enable_gc_threads, - cct->_conf->rgw_enable_quota_threads, - cct->_conf->rgw_run_sync_thread); - - RGWRados *store_cleanup = nullptr; - { - Mutex::Locker lock(mutex); - - // failure to recreate RGWRados is not a recoverable error, but we - // don't want to assert or abort the entire cluster. instead, just - // sleep until we get another notification, and retry until we get - // a working configuration - if (store == nullptr) { - lderr(cct) << "Failed to reinitialize RGWRados after a realm " - "configuration update. Waiting for a new update." << dendl; - - // sleep until another event is scheduled - while (!reconfigure_scheduled) - cond.Wait(mutex); - - ldout(cct, 1) << "Woke up with a new configuration, retrying " - "RGWRados initialization." << dendl; - } - - if (reconfigure_scheduled) { - // cancel the event; we'll handle it now - timer.cancel_event(reconfigure_scheduled); - reconfigure_scheduled = nullptr; - - // if we successfully created a store, clean it up outside of the lock, - // then continue to loop and recreate another - std::swap(store, store_cleanup); - } - } - - if (store_cleanup) { - ldout(cct, 4) << "Got another notification, restarting RGWRados " - "initialization." << dendl; - - RGWStoreManager::close_storage(store_cleanup); - } - } - - // finish initializing the new store - rgw_rest_init(cct, store->get_zonegroup()); - rgw_user_init(store); - rgw_bucket_init(store->meta_mgr); - rgw_log_usage_init(cct, store); - - ldout(cct, 1) << "Resuming frontends with new realm configuration." << dendl; - - frontends->resume(store); + this->watcher = watcher; } - -class RGWRealmWatcher::C_Reconfigure : public Context { - RGWRealmWatcher *watcher; - public: - C_Reconfigure(RGWRealmWatcher *watcher) : watcher(watcher) {} - void finish(int r) { watcher->reconfigure(); } -}; - void RGWRealmWatcher::handle_notify(uint64_t notify_id, uint64_t cookie, uint64_t notifier_id, bufferlist& bl) { if (cookie != watch_handle) return; - // send an empty notify ack - bufferlist reply_bl; - pool_ctx.notify_ack(watch_oid, notify_id, cookie, reply_bl); + auto p = bl.begin(); + watcher->handle_notify(p); - Mutex::Locker lock(mutex); - if (reconfigure_scheduled) { - ldout(cct, 4) << "Notification on " << watch_oid << ", reconfiguration " - "already scheduled" << dendl; - return; - } - - reconfigure_scheduled = new C_Reconfigure(this); - cond.SignalOne(); // wake reconfigure() if it blocked on a bad configuration - - // schedule reconfigure() with a delay so we can batch up changes - auto delay = cct->_conf->rgw_realm_reconfigure_delay; - timer.add_event_after(delay, reconfigure_scheduled); - - ldout(cct, 4) << "Notification on " << watch_oid << ", reconfiguration " - "scheduled in " << delay << 's' << dendl; + // send an empty notify ack + bufferlist reply; + pool_ctx.notify_ack(watch_oid, notify_id, cookie, reply); } void RGWRealmWatcher::handle_error(uint64_t cookie, int err) @@ -181,8 +65,7 @@ void RGWRealmWatcher::handle_error(uint64_t cookie, int err) } } - -int RGWRealmWatcher::watch_start() +int RGWRealmWatcher::watch_start(RGWRealm& realm) { // initialize a Rados client int r = rados.init_with_context(cct); @@ -199,7 +82,6 @@ int RGWRealmWatcher::watch_start() } // open an IoCtx for the realm's pool - auto& realm = store->realm; auto pool = realm.get_pool_name(cct); r = rados.ioctx_create(pool.c_str(), pool_ctx); if (r < 0) { diff --git a/src/rgw/rgw_realm_watcher.h b/src/rgw/rgw_realm_watcher.h index 61b508a2a0e68..d780891fd7884 100644 --- a/src/rgw/rgw_realm_watcher.h +++ b/src/rgw/rgw_realm_watcher.h @@ -9,6 +9,7 @@ #include "common/Cond.h" class RGWRados; +class RGWRealm; /** * RGWRealmWatcher establishes a watch on the current RGWRealm's control object, @@ -18,28 +19,23 @@ class RGWRados; class RGWRealmWatcher : public librados::WatchCtx2 { public: /** - * FrontendPauser is an interface to pause/resume frontends. Frontend - * cooperation is required to ensure that they stop issuing requests on the - * old RGWRados instance, and restart with the updated configuration. - * - * This abstraction avoids a depency on class RGWFrontend, which is only - * defined in rgw_main.cc + * Watcher is an interface that allows the RGWRealmWatcher to pass + * notifications on to other interested objects. */ - class FrontendPauser { + class Watcher { public: - virtual ~FrontendPauser() = default; + virtual ~Watcher() = default; - /// pause all frontends while realm reconfiguration is in progress - virtual void pause() = 0; - /// resume all frontends with the given RGWRados instance - virtual void resume(RGWRados *store) = 0; + virtual void handle_notify(bufferlist::iterator& p) = 0; }; - RGWRealmWatcher(CephContext *cct, RGWRados *&store, - FrontendPauser *frontends); + RGWRealmWatcher(CephContext* cct, RGWRealm& realm); ~RGWRealmWatcher(); - /// respond to realm notifications by scheduling a reconfigure() + /// register a watcher to be notified + void set_watcher(Watcher* watcher); + + /// respond to realm notifications by calling the appropriate watcher void handle_notify(uint64_t notify_id, uint64_t cookie, uint64_t notifier_id, bufferlist& bl) override; @@ -47,36 +43,20 @@ class RGWRealmWatcher : public librados::WatchCtx2 { void handle_error(uint64_t cookie, int err) override; private: - CephContext *cct; - /// main()'s RGWRados pointer as a reference, modified by reconfigure() - RGWRados *&store; - FrontendPauser *frontends; + CephContext *const cct; - /// to prevent a race between reconfigure() and another realm notify, we need - /// to keep the watch open during reconfiguration. this means we need a - /// separate Rados client whose lifetime is independent of RGWRados + /// keep a separate Rados client whose lifetime is independent of RGWRados + /// so that we don't miss notifications during realm reconfiguration librados::Rados rados; librados::IoCtx pool_ctx; uint64_t watch_handle; std::string watch_oid; - int watch_start(); + int watch_start(RGWRealm& realm); int watch_restart(); void watch_stop(); - /// reconfigure() takes a significant amount of time, so we don't want to run - /// it in the handle_notify() thread. we choose a timer thread, because we - /// also want to add a delay (see rgw_realm_reconfigure_delay) so that we can - /// batch up notifications within that window - SafeTimer timer; - Mutex mutex; //< protects access to timer and reconfigure_scheduled - Cond cond; //< to signal reconfigure() after an invalid realm config - Context *reconfigure_scheduled; //< reconfigure() context if scheduled - - /// pause frontends and replace the RGWRados - void reconfigure(); - - class C_Reconfigure; //< Context to call reconfigure() + Watcher* watcher; }; #endif // RGW_REALM_WATCHER_H -- 2.39.5