From 9ecd08fd497298f533512f0982da15430c165d95 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 6 Oct 2015 17:12:14 -0400 Subject: [PATCH] rgw: RGWRealmWatcher supports multiple watch types Signed-off-by: Casey Bodley --- src/rgw/rgw_admin.cc | 1 + src/rgw/rgw_main.cc | 2 +- src/rgw/rgw_rados.cc | 10 +++++++--- src/rgw/rgw_rados.h | 2 +- src/rgw/rgw_realm_reloader.cc | 3 ++- src/rgw/rgw_realm_reloader.h | 2 +- src/rgw/rgw_realm_watcher.cc | 26 ++++++++++++++++++++------ src/rgw/rgw_realm_watcher.h | 15 +++++++++++---- 8 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 85d1532eb5bbc..a2a81f4a70758 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -34,6 +34,7 @@ #include "rgw_sync.h" #include "rgw_data_sync.h" #include "rgw_rest_conn.h" +#include "rgw_realm_watcher.h" using namespace std; diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 8dc5680ffbd89..8718ed48da0a3 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -1344,7 +1344,7 @@ int main(int argc, const char **argv) RGWRealmReloader reloader(store, &pauser); RGWRealmWatcher realm_watcher(g_ceph_context, store->realm); - realm_watcher.set_watcher(&reloader); + realm_watcher.add_watcher(RGWRealmNotify::Reload, reloader); wait_shutdown(); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index c997e892a4ae3..bc379da2f20d9 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -57,6 +57,7 @@ using namespace librados; #include "rgw_object_expirer_core.h" #include "rgw_sync.h" #include "rgw_data_sync.h" +#include "rgw_realm_watcher.h" #define dout_subsys ceph_subsys_rgw @@ -772,7 +773,7 @@ string RGWRealm::get_control_oid() return get_info_oid_prefix() + id + ".control"; } -int RGWRealm::notify_zone() +int RGWRealm::notify_zone(bufferlist& bl) { // open a context on the realm's pool auto pool = get_pool_name(cct); @@ -783,7 +784,6 @@ int RGWRealm::notify_zone() return r; } // send a notify on the realm object - bufferlist bl; r = ctx.notify2(get_control_oid(), bl, 0, nullptr); if (r < 0) { lderr(cct) << "Realm notify failed with " << r << dendl; @@ -794,7 +794,11 @@ int RGWRealm::notify_zone() int RGWRealm::notify_new_period(const RGWPeriod& period) { - return notify_zone(); + bufferlist bl; + // reload the gateway with the new period + ::encode(RGWRealmNotify::Reload, bl); + + return notify_zone(bl); } int RGWPeriod::init(CephContext *_cct, RGWRados *_store, const string& period_realm_id, diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index b2f914722d6b2..7d28ea3e4080a 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1325,7 +1325,7 @@ public: string get_control_oid(); /// send a notify on the realm control object - int notify_zone(); + int notify_zone(bufferlist& bl); /// notify the zone of a new period int notify_new_period(const RGWPeriod& period); }; diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc index 7b6da87e59dcd..295e374c5d2eb 100644 --- a/src/rgw/rgw_realm_reloader.cc +++ b/src/rgw/rgw_realm_reloader.cc @@ -44,7 +44,8 @@ class RGWRealmReloader::C_Reload : public Context { void finish(int r) { reloader->reload(); } }; -void RGWRealmReloader::handle_notify(bufferlist::iterator& p) +void RGWRealmReloader::handle_notify(RGWRealmNotify type, + bufferlist::iterator& p) { CephContext *const cct = store->ctx(); diff --git a/src/rgw/rgw_realm_reloader.h b/src/rgw/rgw_realm_reloader.h index 280262dc0f3c8..3e60cd49d1f4e 100644 --- a/src/rgw/rgw_realm_reloader.h +++ b/src/rgw/rgw_realm_reloader.h @@ -37,7 +37,7 @@ class RGWRealmReloader : public RGWRealmWatcher::Watcher { ~RGWRealmReloader(); /// respond to realm notifications by scheduling a reload() - void handle_notify(bufferlist::iterator& p) override; + void handle_notify(RGWRealmNotify type, bufferlist::iterator& p) override; private: /// pause frontends and replace the RGWRados instance diff --git a/src/rgw/rgw_realm_watcher.cc b/src/rgw/rgw_realm_watcher.cc index bbbe70d5d59ad..6e47c48460aff 100644 --- a/src/rgw/rgw_realm_watcher.cc +++ b/src/rgw/rgw_realm_watcher.cc @@ -13,7 +13,7 @@ RGWRealmWatcher::RGWRealmWatcher(CephContext* cct, RGWRealm& realm) - : cct(cct), watcher(nullptr) + : cct(cct) { // no default realm, nothing to watch if (realm.get_id().empty()) { @@ -35,9 +35,9 @@ RGWRealmWatcher::~RGWRealmWatcher() watch_stop(); } -void RGWRealmWatcher::set_watcher(Watcher* watcher) +void RGWRealmWatcher::add_watcher(RGWRealmNotify type, Watcher& watcher) { - this->watcher = watcher; + watchers.emplace(type, watcher); } void RGWRealmWatcher::handle_notify(uint64_t notify_id, uint64_t cookie, @@ -46,12 +46,26 @@ void RGWRealmWatcher::handle_notify(uint64_t notify_id, uint64_t cookie, if (cookie != watch_handle) return; - auto p = bl.begin(); - watcher->handle_notify(p); - // send an empty notify ack bufferlist reply; pool_ctx.notify_ack(watch_oid, notify_id, cookie, reply); + + try { + auto p = bl.begin(); + while (!p.end()) { + RGWRealmNotify notify; + ::decode(notify, p); + auto watcher = watchers.find(notify); + if (watcher == watchers.end()) { + lderr(cct) << "Failed to find a watcher for notify type " + << static_cast(notify) << dendl; + break; + } + watcher->second.handle_notify(notify, p); + } + } catch (const buffer::error &e) { + lderr(cct) << "Failed to decode realm notifications." << dendl; + } } void RGWRealmWatcher::handle_error(uint64_t cookie, int err) diff --git a/src/rgw/rgw_realm_watcher.h b/src/rgw/rgw_realm_watcher.h index d780891fd7884..d0e87e6b0ea5e 100644 --- a/src/rgw/rgw_realm_watcher.h +++ b/src/rgw/rgw_realm_watcher.h @@ -5,12 +5,18 @@ #define RGW_REALM_WATCHER_H #include "include/rados/librados.hpp" +#include "include/assert.h" #include "common/Timer.h" #include "common/Cond.h" class RGWRados; class RGWRealm; +enum class RGWRealmNotify { + Reload, +}; +WRITE_RAW_ENCODER(RGWRealmNotify); + /** * RGWRealmWatcher establishes a watch on the current RGWRealm's control object, * and responds to notifications by recreating RGWRados with the updated realm @@ -26,14 +32,15 @@ class RGWRealmWatcher : public librados::WatchCtx2 { public: virtual ~Watcher() = default; - virtual void handle_notify(bufferlist::iterator& p) = 0; + virtual void handle_notify(RGWRealmNotify type, + bufferlist::iterator& p) = 0; }; RGWRealmWatcher(CephContext* cct, RGWRealm& realm); ~RGWRealmWatcher(); - /// register a watcher to be notified - void set_watcher(Watcher* watcher); + /// register a watcher for the given notification type + void add_watcher(RGWRealmNotify type, Watcher& watcher); /// respond to realm notifications by calling the appropriate watcher void handle_notify(uint64_t notify_id, uint64_t cookie, @@ -56,7 +63,7 @@ class RGWRealmWatcher : public librados::WatchCtx2 { int watch_restart(); void watch_stop(); - Watcher* watcher; + std::map watchers; }; #endif // RGW_REALM_WATCHER_H -- 2.39.5