rgw/rgw_sync.cc
rgw/rgw_data_sync.cc
rgw/rgw_dencoder.cc
+ rgw/rgw_realm_reloader.cc
+ rgw/rgw_realm_watcher.cc
rgw/rgw_coroutine.cc
rgw/rgw_cr_rados.cc
rgw/rgw_object_expirer_core.cc
rgw/rgw_civetweb_log.cc
civetweb/src/civetweb.c
rgw/rgw_main.cc
- rgw/rgw_realm_watcher.cc
rgw/rgw_sync.cc
rgw/rgw_data_sync.cc)
rgw/rgw_keystone.cc \
rgw/rgw_quota.cc \
rgw/rgw_dencoder.cc \
+ rgw/rgw_realm_reloader.cc \
+ rgw/rgw_realm_watcher.cc \
rgw/rgw_object_expirer_core.cc \
rgw/rgw_website.cc \
rgw/rgw_sync.cc \
rgw/rgw_swift.cc \
rgw/rgw_swift_auth.cc \
rgw/rgw_loadgen.cc \
- rgw/rgw_realm_watcher.cc \
rgw/rgw_main.cc
radosgw_CFLAGS = -I$(srcdir)/civetweb/include
radosgw_LDADD = $(LIBRGW) $(LIBCIVETWEB) $(LIBRGW_DEPS) $(RESOLV_LIBS) $(CEPH_GLOBAL)
rgw/rgw_user.h \
rgw/rgw_bucket.h \
rgw/rgw_keystone.h \
+ rgw/rgw_realm_reloader.h \
rgw/rgw_realm_watcher.h \
rgw/rgw_civetweb.h \
rgw/rgw_boost_asio_coroutine.h \
#include "rgw_acl.h"
#include "rgw_user.h"
#include "rgw_op.h"
-#include "rgw_realm_watcher.h"
+#include "rgw_realm_reloader.h"
#include "rgw_rest.h"
#include "rgw_rest_s3.h"
#include "rgw_rest_swift.h"
}
};
-// FrontendPauser implementation for RGWRealmWatcher
-class RGWFrontendPauser : public RGWRealmWatcher::FrontendPauser {
+// FrontendPauser implementation for RGWRealmReloader
+class RGWFrontendPauser : public RGWRealmReloader::FrontendPauser {
std::list<RGWFrontend*> &frontends;
public:
RGWFrontendPauser(std::list<RGWFrontend*> &frontends)
// add a watcher to respond to realm configuration changes
RGWFrontendPauser pauser(fes);
- RGWRealmWatcher realm_watcher(g_ceph_context, store, &pauser);
+ RGWRealmReloader reloader(store, &pauser);
+
+ RGWRealmWatcher realm_watcher(g_ceph_context, store->realm);
+ realm_watcher.set_watcher(&reloader);
wait_shutdown();
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "rgw_realm_reloader.h"
+#include "rgw_rados.h"
+
+#include "rgw_bucket.h"
+#include "rgw_log.h"
+#include "rgw_rest.h"
+#include "rgw_user.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+#undef dout_prefix
+#define dout_prefix (*_dout << "rgw realm reloader: ")
+
+
+// safe callbacks from SafeTimer are unneccessary. reload() can take a long
+// time, so we don't want to hold the mutex and block handle_notify() for the
+// duration
+static constexpr bool USE_SAFE_TIMER_CALLBACKS = false;
+
+
+RGWRealmReloader::RGWRealmReloader(RGWRados*& store, FrontendPauser* frontends)
+ : store(store),
+ frontends(frontends),
+ timer(store->ctx(), mutex, USE_SAFE_TIMER_CALLBACKS),
+ mutex("RGWRealmReloader"),
+ reload_scheduled(nullptr)
+{
+ timer.init();
+}
+
+RGWRealmReloader::~RGWRealmReloader()
+{
+ Mutex::Locker lock(mutex);
+ timer.shutdown();
+}
+
+class RGWRealmReloader::C_Reload : public Context {
+ RGWRealmReloader* reloader;
+ public:
+ C_Reload(RGWRealmReloader* reloader) : reloader(reloader) {}
+ void finish(int r) { reloader->reload(); }
+};
+
+void RGWRealmReloader::handle_notify(bufferlist::iterator& p)
+{
+ CephContext *const cct = store->ctx();
+
+ Mutex::Locker lock(mutex);
+ if (reload_scheduled) {
+ ldout(cct, 4) << "Notification on realm, reconfiguration "
+ "already scheduled" << dendl;
+ return;
+ }
+
+ reload_scheduled = new C_Reload(this);
+ cond.SignalOne(); // wake reload() if it blocked on a bad configuration
+
+ // schedule reload() with a delay so we can batch up changes
+ auto delay = cct->_conf->rgw_realm_reconfigure_delay;
+ timer.add_event_after(delay, reload_scheduled);
+
+ ldout(cct, 4) << "Notification on realm, reconfiguration scheduled in "
+ << delay << 's' << dendl;
+}
+
+void RGWRealmReloader::reload()
+{
+ CephContext *const cct = store->ctx();
+ ldout(cct, 1) << "Pausing frontends for realm update..." << dendl;
+
+ frontends->pause();
+
+ // destroy the existing store
+ RGWStoreManager::close_storage(store);
+ store = nullptr;
+
+ {
+ // allow a new notify to reschedule us. it's important that we do this
+ // before we start loading the new realm, or we could miss some updates
+ Mutex::Locker lock(mutex);
+ reload_scheduled = nullptr;
+ }
+
+ while (!store) {
+ // recreate and initialize a new store
+ store = RGWStoreManager::get_storage(cct,
+ cct->_conf->rgw_enable_gc_threads,
+ cct->_conf->rgw_enable_quota_threads,
+ cct->_conf->rgw_run_sync_thread);
+
+ RGWRados* store_cleanup = nullptr;
+ {
+ Mutex::Locker lock(mutex);
+
+ // failure to recreate RGWRados is not a recoverable error, but we
+ // don't want to assert or abort the entire cluster. instead, just
+ // sleep until we get another notification, and retry until we get
+ // a working configuration
+ if (store == nullptr) {
+ lderr(cct) << "Failed to reinitialize RGWRados after a realm "
+ "configuration update. Waiting for a new update." << dendl;
+
+ // sleep until another event is scheduled
+ while (!reload_scheduled)
+ cond.Wait(mutex);
+
+ ldout(cct, 1) << "Woke up with a new configuration, retrying "
+ "RGWRados initialization." << dendl;
+ }
+
+ if (reload_scheduled) {
+ // cancel the event; we'll handle it now
+ timer.cancel_event(reload_scheduled);
+ reload_scheduled = nullptr;
+
+ // if we successfully created a store, clean it up outside of the lock,
+ // then continue to loop and recreate another
+ std::swap(store, store_cleanup);
+ }
+ }
+
+ if (store_cleanup) {
+ ldout(cct, 4) << "Got another notification, restarting RGWRados "
+ "initialization." << dendl;
+
+ RGWStoreManager::close_storage(store_cleanup);
+ }
+ }
+
+ // finish initializing the new store
+ rgw_rest_init(cct, store->get_zonegroup());
+ rgw_user_init(store);
+ rgw_bucket_init(store->meta_mgr);
+ rgw_log_usage_init(cct, store);
+
+ ldout(cct, 1) << "Resuming frontends with new realm configuration." << dendl;
+
+ frontends->resume(store);
+}
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef RGW_REALM_RELOADER_H
+#define RGW_REALM_RELOADER_H
+
+#include "rgw_realm_watcher.h"
+#include "common/Cond.h"
+
+class RGWRados;
+
+/**
+ * RGWRealmReloader responds to notifications by recreating RGWRados with the
+ * updated realm configuration.
+ */
+class RGWRealmReloader : public RGWRealmWatcher::Watcher {
+ public:
+ /**
+ * FrontendPauser is an interface to pause/resume frontends. Frontend
+ * cooperation is required to ensure that they stop issuing requests on the
+ * old RGWRados instance, and restart with the updated configuration.
+ *
+ * This abstraction avoids a depency on class RGWFrontend, which is only
+ * defined in rgw_main.cc
+ */
+ class FrontendPauser {
+ public:
+ virtual ~FrontendPauser() = default;
+
+ /// pause all frontends while realm reconfiguration is in progress
+ virtual void pause() = 0;
+ /// resume all frontends with the given RGWRados instance
+ virtual void resume(RGWRados* store) = 0;
+ };
+
+ RGWRealmReloader(RGWRados*& store, FrontendPauser* frontends);
+ ~RGWRealmReloader();
+
+ /// respond to realm notifications by scheduling a reload()
+ void handle_notify(bufferlist::iterator& p) override;
+
+ private:
+ /// pause frontends and replace the RGWRados instance
+ void reload();
+
+ class C_Reload; //< Context that calls reload()
+
+ /// main()'s RGWRados pointer as a reference, modified by reload()
+ RGWRados*& store;
+ FrontendPauser *const frontends;
+
+ /// reload() takes a significant amount of time, so we don't want to run
+ /// it in the handle_notify() thread. we choose a timer thread because we
+ /// also want to add a delay (see rgw_realm_reconfigure_delay) so that we
+ /// can batch up notifications within that window
+ SafeTimer timer;
+ Mutex mutex; //< protects access to timer and reload_scheduled
+ Cond cond; //< to signal reload() after an invalid realm config
+ C_Reload* reload_scheduled; //< reload() context if scheduled
+};
+
+#endif // RGW_REALM_RELOADER_H
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
-#include <system_error>
-
#include "common/errno.h"
#include "rgw_realm_watcher.h"
#include "rgw_rados.h"
-#include "rgw_bucket.h"
-#include "rgw_log.h"
-#include "rgw_rest.h"
-#include "rgw_user.h"
-
#define dout_subsys ceph_subsys_rgw
#undef dout_prefix
#define dout_prefix (*_dout << "rgw realm watcher: ")
-// safe callbacks from SafeTimer are unneccessary. reconfigure() can take a long
-// time, so we don't want to hold the mutex and block handle_notify() for the
-// duration
-static constexpr bool USE_SAFE_TIMER_CALLBACKS = false;
-
-
-RGWRealmWatcher::RGWRealmWatcher(CephContext *cct, RGWRados *&store,
- FrontendPauser *frontends)
- : cct(cct),
- store(store),
- frontends(frontends),
- timer(cct, mutex, USE_SAFE_TIMER_CALLBACKS),
- mutex("RGWRealmWatcher"),
- reconfigure_scheduled(nullptr)
+RGWRealmWatcher::RGWRealmWatcher(CephContext* cct, RGWRealm& realm)
+ : cct(cct), watcher(nullptr)
{
// no default realm, nothing to watch
- if (store->realm.get_id().empty()) {
+ if (realm.get_id().empty()) {
ldout(cct, 4) << "No realm, disabling dynamic reconfiguration." << dendl;
return;
}
// establish the watch on RGWRealm
- int r = watch_start();
+ int r = watch_start(realm);
if (r < 0) {
lderr(cct) << "Failed to establish a watch on RGWRealm, "
"disabling dynamic reconfiguration." << dendl;
return;
}
-
- timer.init();
}
RGWRealmWatcher::~RGWRealmWatcher()
{
watch_stop();
-
- Mutex::Locker lock(mutex);
- timer.shutdown();
}
-
-void RGWRealmWatcher::reconfigure()
+void RGWRealmWatcher::set_watcher(Watcher* watcher)
{
- ldout(cct, 1) << "Pausing frontends for realm update..." << dendl;
-
- frontends->pause();
-
- // destroy the existing store
- RGWStoreManager::close_storage(store);
- store = nullptr;
-
- {
- // allow a new notify to reschedule us. it's important that we do this
- // before we start loading the new realm, or we could miss some updates
- Mutex::Locker lock(mutex);
- reconfigure_scheduled = nullptr;
- }
-
- while (!store) {
- // recreate and initialize a new store
- store = RGWStoreManager::get_storage(cct,
- cct->_conf->rgw_enable_gc_threads,
- cct->_conf->rgw_enable_quota_threads,
- cct->_conf->rgw_run_sync_thread);
-
- RGWRados *store_cleanup = nullptr;
- {
- Mutex::Locker lock(mutex);
-
- // failure to recreate RGWRados is not a recoverable error, but we
- // don't want to assert or abort the entire cluster. instead, just
- // sleep until we get another notification, and retry until we get
- // a working configuration
- if (store == nullptr) {
- lderr(cct) << "Failed to reinitialize RGWRados after a realm "
- "configuration update. Waiting for a new update." << dendl;
-
- // sleep until another event is scheduled
- while (!reconfigure_scheduled)
- cond.Wait(mutex);
-
- ldout(cct, 1) << "Woke up with a new configuration, retrying "
- "RGWRados initialization." << dendl;
- }
-
- if (reconfigure_scheduled) {
- // cancel the event; we'll handle it now
- timer.cancel_event(reconfigure_scheduled);
- reconfigure_scheduled = nullptr;
-
- // if we successfully created a store, clean it up outside of the lock,
- // then continue to loop and recreate another
- std::swap(store, store_cleanup);
- }
- }
-
- if (store_cleanup) {
- ldout(cct, 4) << "Got another notification, restarting RGWRados "
- "initialization." << dendl;
-
- RGWStoreManager::close_storage(store_cleanup);
- }
- }
-
- // finish initializing the new store
- rgw_rest_init(cct, store->get_zonegroup());
- rgw_user_init(store);
- rgw_bucket_init(store->meta_mgr);
- rgw_log_usage_init(cct, store);
-
- ldout(cct, 1) << "Resuming frontends with new realm configuration." << dendl;
-
- frontends->resume(store);
+ this->watcher = watcher;
}
-
-class RGWRealmWatcher::C_Reconfigure : public Context {
- RGWRealmWatcher *watcher;
- public:
- C_Reconfigure(RGWRealmWatcher *watcher) : watcher(watcher) {}
- void finish(int r) { watcher->reconfigure(); }
-};
-
void RGWRealmWatcher::handle_notify(uint64_t notify_id, uint64_t cookie,
uint64_t notifier_id, bufferlist& bl)
{
if (cookie != watch_handle)
return;
- // send an empty notify ack
- bufferlist reply_bl;
- pool_ctx.notify_ack(watch_oid, notify_id, cookie, reply_bl);
+ auto p = bl.begin();
+ watcher->handle_notify(p);
- Mutex::Locker lock(mutex);
- if (reconfigure_scheduled) {
- ldout(cct, 4) << "Notification on " << watch_oid << ", reconfiguration "
- "already scheduled" << dendl;
- return;
- }
-
- reconfigure_scheduled = new C_Reconfigure(this);
- cond.SignalOne(); // wake reconfigure() if it blocked on a bad configuration
-
- // schedule reconfigure() with a delay so we can batch up changes
- auto delay = cct->_conf->rgw_realm_reconfigure_delay;
- timer.add_event_after(delay, reconfigure_scheduled);
-
- ldout(cct, 4) << "Notification on " << watch_oid << ", reconfiguration "
- "scheduled in " << delay << 's' << dendl;
+ // send an empty notify ack
+ bufferlist reply;
+ pool_ctx.notify_ack(watch_oid, notify_id, cookie, reply);
}
void RGWRealmWatcher::handle_error(uint64_t cookie, int err)
}
}
-
-int RGWRealmWatcher::watch_start()
+int RGWRealmWatcher::watch_start(RGWRealm& realm)
{
// initialize a Rados client
int r = rados.init_with_context(cct);
}
// open an IoCtx for the realm's pool
- auto& realm = store->realm;
auto pool = realm.get_pool_name(cct);
r = rados.ioctx_create(pool.c_str(), pool_ctx);
if (r < 0) {
#include "common/Cond.h"
class RGWRados;
+class RGWRealm;
/**
* RGWRealmWatcher establishes a watch on the current RGWRealm's control object,
class RGWRealmWatcher : public librados::WatchCtx2 {
public:
/**
- * FrontendPauser is an interface to pause/resume frontends. Frontend
- * cooperation is required to ensure that they stop issuing requests on the
- * old RGWRados instance, and restart with the updated configuration.
- *
- * This abstraction avoids a depency on class RGWFrontend, which is only
- * defined in rgw_main.cc
+ * Watcher is an interface that allows the RGWRealmWatcher to pass
+ * notifications on to other interested objects.
*/
- class FrontendPauser {
+ class Watcher {
public:
- virtual ~FrontendPauser() = default;
+ virtual ~Watcher() = default;
- /// pause all frontends while realm reconfiguration is in progress
- virtual void pause() = 0;
- /// resume all frontends with the given RGWRados instance
- virtual void resume(RGWRados *store) = 0;
+ virtual void handle_notify(bufferlist::iterator& p) = 0;
};
- RGWRealmWatcher(CephContext *cct, RGWRados *&store,
- FrontendPauser *frontends);
+ RGWRealmWatcher(CephContext* cct, RGWRealm& realm);
~RGWRealmWatcher();
- /// respond to realm notifications by scheduling a reconfigure()
+ /// register a watcher to be notified
+ void set_watcher(Watcher* watcher);
+
+ /// respond to realm notifications by calling the appropriate watcher
void handle_notify(uint64_t notify_id, uint64_t cookie,
uint64_t notifier_id, bufferlist& bl) override;
void handle_error(uint64_t cookie, int err) override;
private:
- CephContext *cct;
- /// main()'s RGWRados pointer as a reference, modified by reconfigure()
- RGWRados *&store;
- FrontendPauser *frontends;
+ CephContext *const cct;
- /// to prevent a race between reconfigure() and another realm notify, we need
- /// to keep the watch open during reconfiguration. this means we need a
- /// separate Rados client whose lifetime is independent of RGWRados
+ /// keep a separate Rados client whose lifetime is independent of RGWRados
+ /// so that we don't miss notifications during realm reconfiguration
librados::Rados rados;
librados::IoCtx pool_ctx;
uint64_t watch_handle;
std::string watch_oid;
- int watch_start();
+ int watch_start(RGWRealm& realm);
int watch_restart();
void watch_stop();
- /// reconfigure() takes a significant amount of time, so we don't want to run
- /// it in the handle_notify() thread. we choose a timer thread, because we
- /// also want to add a delay (see rgw_realm_reconfigure_delay) so that we can
- /// batch up notifications within that window
- SafeTimer timer;
- Mutex mutex; //< protects access to timer and reconfigure_scheduled
- Cond cond; //< to signal reconfigure() after an invalid realm config
- Context *reconfigure_scheduled; //< reconfigure() context if scheduled
-
- /// pause frontends and replace the RGWRados
- void reconfigure();
-
- class C_Reconfigure; //< Context to call reconfigure()
+ Watcher* watcher;
};
#endif // RGW_REALM_WATCHER_H
projects / ceph.git / commitdiff