From 08e687fc9ea9088e8702d69f2233018126b52277 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Sat, 27 Aug 2022 10:35:45 -0400 Subject: [PATCH] rgw/main: AppMain s/rgw::InitHelper/rgw::AppMain/; Move program state from RGWLib into AppMain, preparing it to manage program state for all RGW instances (e.g., main()). Signed-off-by: Matt Benjamin --- src/rgw/librgw.cc | 85 +++++++++------------------------------------ src/rgw/rgw_lib.h | 31 +++-------------- src/rgw/rgw_main.cc | 73 ++++++++++++++++++++++++++++++++------ src/rgw/rgw_main.h | 67 +++++++++++++---------------------- 4 files changed, 107 insertions(+), 149 deletions(-) diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 98c637c06d2..a07c942e89b 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -558,18 +558,12 @@ namespace rgw { mutex.unlock(); /* stage all front-ends (before common-init-finish) */ - - rgw::InitHelper init_helper( - fes, fe_configs, fe_map, ldh, olog, rest, lua_background, - implicit_tenant_context, sched_ctx, ratelimiter, reloader, - pusher, fe_pauser, realm_watcher, rgw_pauser, store, this); - - init_helper.init_frontends1(true /* nfs */); + main.init_frontends1(true /* nfs */); common_init_finish(g_ceph_context); - init_helper.init_storage(); - if (!store) { + main.init_storage(); + if (! main.store) { mutex.lock(); init_timer.cancel_all_events(); init_timer.shutdown(); @@ -579,25 +573,25 @@ namespace rgw { return -EIO; } - init_helper.init_perfcounters(); - init_helper.init_http_clients(); - init_helper.cond_init_apis(); + main.init_perfcounters(); + main.init_http_clients(); + main.cond_init_apis(); mutex.lock(); init_timer.cancel_all_events(); init_timer.shutdown(); mutex.unlock(); - init_helper.init_ldap(); - init_helper.init_opslog(); + main.init_ldap(); + main.init_opslog(); init_async_signal_handler(); register_async_signal_handler(SIGUSR1, handle_sigterm); - init_helper.init_tracepoints(); - init_helper.init_frontends2(this /* rgwlib */); - init_helper.init_notification_endpoints(); - init_helper.init_lua(); + main.init_tracepoints(); + main.init_frontends2(this /* rgwlib */); + main.init_notification_endpoints(); + main.init_lua(); return 0; } /* RGWLib::init() */ @@ -605,57 +599,7 @@ namespace rgw { int RGWLib::stop() { derr << "shutting down" << dendl; - - if (store->get_name() == "rados") { - reloader.reset(); // stop the realm reloader - } - - for (auto& fe : fes) { - fe->stop(); - } - - for (auto& fe : fes) { - fe->join(); - delete fe; - } - - for (auto& fec : fe_configs) { - delete fec; - } - - ldh.reset(nullptr); // deletes - - unregister_async_signal_handler(SIGUSR1, handle_sigterm); - shutdown_async_signal_handler(); - - rgw_log_usage_finalize(); - - delete olog; - - if (lua_background) { - lua_background->shutdown(); - } - - StoreManager::close_storage(store); - - rgw_tools_cleanup(); - rgw_shutdown_resolver(); - rgw_http_client_cleanup(); - rgw_kmip_client_cleanup(); - rgw::curl::cleanup_curl(); - g_conf().remove_observer(implicit_tenant_context.get()); - implicit_tenant_context.reset(); // deletes -#ifdef WITH_RADOSGW_AMQP_ENDPOINT - rgw::amqp::shutdown(); -#endif -#ifdef WITH_RADOSGW_KAFKA_ENDPOINT - rgw::kafka::shutdown(); -#endif - - rgw_perf_stop(g_ceph_context); - - dout(1) << "final shutdown" << dendl; - cct.reset(); + main.shutdown(); return 0; } /* RGWLib::stop() */ @@ -761,6 +705,9 @@ void librgw_shutdown(librgw_t rgw) CephContext* cct = static_cast(rgw); rgwlib.stop(); + + dout(1) << "final shutdown" << dendl; + cct->put(); } diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h index 60be364eeba..02ed8a0393d 100644 --- a/src/rgw/rgw_lib.h +++ b/src/rgw/rgw_lib.h @@ -5,19 +5,11 @@ #define RGW_LIB_H #include -#include "include/unordered_map.h" -#include "global/global_init.h" #include "rgw_common.h" #include "rgw_client_io.h" #include "rgw_rest.h" #include "rgw_request.h" -#include "rgw_frontend.h" -#include "rgw_process.h" -#include "rgw_rest_s3.h" // RGW_Auth_S3 -#include "rgw_period_pusher.h" -#include "rgw_realm_reloader.h" #include "rgw_ldap.h" -#include "services/svc_zone_utils.h" #include "include/ceph_assert.h" #include "rgw_main.h" @@ -28,35 +20,20 @@ namespace rgw { class RGWLibFrontend; class RGWLib : public DoutPrefixProvider { + AppMain main; RGWLibFrontend* fe; - std::vector fes; - std::vector fe_configs; - std::multimap fe_map; - std::unique_ptr reloader; - std::unique_ptr pusher; - std::unique_ptr fe_pauser; - std::unique_ptr realm_watcher; - std::unique_ptr rgw_pauser; - std::unique_ptr lua_background; - OpsLogSink* olog; - std::unique_ptr implicit_tenant_context; - std::unique_ptr sched_ctx; - std::unique_ptr ratelimiter; - std::unique_ptr ldh; - RGWREST rest; - rgw::sal::Store* store; boost::intrusive_ptr cct; public: - RGWLib() : fe(nullptr), olog(nullptr), store(nullptr) + RGWLib() : main(this), fe(nullptr) {} ~RGWLib() {} - rgw::sal::Store* get_store() { return store; } + rgw::sal::Store* get_store() { return main.store; } RGWLibFrontend* get_fe() { return fe; } - rgw::LDAPHelper* get_ldh() { return ldh.get(); } + rgw::LDAPHelper* get_ldh() { return main.ldh.get(); } CephContext *get_cct() const override { return cct.get(); } unsigned get_subsys() const { return ceph_subsys_rgw; } std::ostream& gen_prefix(std::ostream& out) const { return out << "lib rgw: "; } diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 14fa4c17068..94d64d5430d 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -165,7 +165,7 @@ static int usage() return 0; } -void rgw::InitHelper::init_frontends1(bool nfs) +void rgw::AppMain::init_frontends1(bool nfs) { this->nfs = nfs; std::string fe_key = (nfs) ? "rgw_nfs_frontends" : "rgw_frontends"; @@ -230,7 +230,7 @@ void rgw::InitHelper::init_frontends1(bool nfs) ceph::crypto::init_openssl_engine_once(); } /* init_frontends1 */ -void rgw::InitHelper::init_storage() +void rgw::AppMain::init_storage() { auto run_gc = g_conf()->rgw_enable_gc_threads && @@ -259,12 +259,12 @@ void rgw::InitHelper::init_storage() } /* init_storage */ -void rgw::InitHelper::init_perfcounters() +void rgw::AppMain::init_perfcounters() { (void) rgw_perf_start(dpp->get_cct()); } /* init_perfcounters */ -void rgw::InitHelper::init_http_clients() +void rgw::AppMain::init_http_clients() { rgw_init_resolver(); rgw::curl::setup_curl(fe_map); @@ -272,7 +272,7 @@ void rgw::InitHelper::init_http_clients() rgw_kmip_client_init(*new RGWKMIPManagerImpl(dpp->get_cct())); } /* init_http_clients */ -void rgw::InitHelper::cond_init_apis() +void rgw::AppMain::cond_init_apis() { rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup()); @@ -364,7 +364,7 @@ void rgw::InitHelper::cond_init_apis() } /* have_http_frontend */ } /* init_apis */ -void rgw::InitHelper::init_ldap() +void rgw::AppMain::init_ldap() { const string &ldap_uri = store->ctx()->_conf->rgw_ldap_uri; const string &ldap_binddn = store->ctx()->_conf->rgw_ldap_binddn; @@ -379,7 +379,7 @@ void rgw::InitHelper::init_ldap() ldh->bind(); } /* init_ldap */ -void rgw::InitHelper::init_opslog() +void rgw::AppMain::init_opslog() { rgw_log_usage_init(dpp->get_cct(), store); @@ -402,7 +402,7 @@ void rgw::InitHelper::init_opslog() olog = olog_manifold; } /* init_opslog */ -int rgw::InitHelper::init_frontends2(RGWLib* rgwlib) +int rgw::AppMain::init_frontends2(RGWLib* rgwlib) { int r{0}; vector frontends_def; @@ -533,14 +533,14 @@ int rgw::InitHelper::init_frontends2(RGWLib* rgwlib) return r; } /* init_frontends2 */ -void rgw::InitHelper::init_tracepoints() +void rgw::AppMain::init_tracepoints() { TracepointProvider::initialize(dpp->get_cct()); TracepointProvider::initialize(dpp->get_cct()); tracing::rgw::tracer.init("rgw"); } /* init_tracepoints() */ -void rgw::InitHelper::init_notification_endpoints() +void rgw::AppMain::init_notification_endpoints() { #ifdef WITH_RADOSGW_AMQP_ENDPOINT if (!rgw::amqp::init(dpp->get_cct())) { @@ -554,7 +554,7 @@ void rgw::InitHelper::init_notification_endpoints() #endif } /* init_notification_endpoints */ -void rgw::InitHelper::init_lua() +void rgw::AppMain::init_lua() { int r{0}; const auto &luarocks_path = @@ -589,6 +589,57 @@ void rgw::InitHelper::init_lua() } } /* init_lua */ +void rgw::AppMain::shutdown() +{ + if (store->get_name() == "rados") { + reloader.reset(); // stop the realm reloader + } + + for (auto& fe : fes) { + fe->stop(); + } + + for (auto& fe : fes) { + fe->join(); + delete fe; + } + + for (auto& fec : fe_configs) { + delete fec; + } + + ldh.reset(nullptr); // deletes + + unregister_async_signal_handler(SIGUSR1, handle_sigterm); + shutdown_async_signal_handler(); + + rgw_log_usage_finalize(); + + delete olog; + + if (lua_background) { + lua_background->shutdown(); + } + + StoreManager::close_storage(store); + + rgw_tools_cleanup(); + rgw_shutdown_resolver(); + rgw_http_client_cleanup(); + rgw_kmip_client_cleanup(); + rgw::curl::cleanup_curl(); + g_conf().remove_observer(implicit_tenant_context.get()); + implicit_tenant_context.reset(); // deletes +#ifdef WITH_RADOSGW_AMQP_ENDPOINT + rgw::amqp::shutdown(); +#endif +#ifdef WITH_RADOSGW_KAFKA_ENDPOINT + rgw::kafka::shutdown(); +#endif + + rgw_perf_stop(g_ceph_context); +} + /* * start up the RADOS connection and then handle HTTP messages as they come in */ diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h index 305bf71adf1..432c58bb490 100644 --- a/src/rgw/rgw_main.h +++ b/src/rgw/rgw_main.h @@ -53,57 +53,40 @@ namespace rgw { class RGWLib; - class InitHelper { + class AppMain { /* several components should be initalized only if librgw is * also serving HTTP */ bool have_http_frontend{false}; bool nfs{false}; - std::vector& fes; - std::vector& fe_configs; - std::multimap& fe_map; - std::unique_ptr& ldh; - OpsLogSink*& olog; - RGWREST& rest; - std::unique_ptr& lua_background; - std::unique_ptr& implicit_tenant_context; - std::unique_ptr& sched_ctx; - std::unique_ptr& ratelimiter; + std::vector fes; + std::vector fe_configs; + std::multimap fe_map; + std::unique_ptr ldh; + OpsLogSink* olog; + RGWREST rest; + std::unique_ptr lua_background; + std::unique_ptr implicit_tenant_context; + std::unique_ptr sched_ctx; + std::unique_ptr ratelimiter; // wow, realm reloader has a lot of parts - std::unique_ptr& reloader; - std::unique_ptr& pusher; - std::unique_ptr& fe_pauser; - std::unique_ptr& realm_watcher; - std::unique_ptr& rgw_pauser; - rgw::sal::Store*& store; + std::unique_ptr reloader; + std::unique_ptr pusher; + std::unique_ptr fe_pauser; + std::unique_ptr realm_watcher; + std::unique_ptr rgw_pauser; + rgw::sal::Store* store; DoutPrefixProvider* dpp; + friend class RGWLib; + public: - InitHelper(std::vector& fes, - std::vector& fe_configs, - std::multimap& fe_map, - std::unique_ptr& ldh, - OpsLogSink*& olog, - RGWREST& rest, - std::unique_ptr& lua_background, - std::unique_ptr& implicit_tenant_context, - std::unique_ptr& sched_ctx, - std::unique_ptr& ratelimiter, - std::unique_ptr& reloader, - std::unique_ptr& pusher, - std::unique_ptr& fe_pauser, - std::unique_ptr& realm_watcher, - std::unique_ptr& rgw_pauser, - rgw::sal::Store*& store, - DoutPrefixProvider* dpp) - : fes(fes), fe_configs(fe_configs), fe_map(fe_map), ldh(ldh), olog(olog), - rest(rest), lua_background(lua_background), - implicit_tenant_context(implicit_tenant_context), sched_ctx(sched_ctx), - ratelimiter(ratelimiter), reloader(reloader), pusher(pusher), - fe_pauser(fe_pauser), realm_watcher(realm_watcher), rgw_pauser(rgw_pauser), - store(store), dpp(dpp) + AppMain(DoutPrefixProvider* dpp) + : dpp(dpp) {} - + + void shutdown(); + void init_frontends1(bool nfs = false); void init_storage(); void init_perfcounters(); @@ -119,7 +102,7 @@ namespace rgw { bool have_http() { return have_http_frontend; } - }; /* InitHelper */ + }; /* AppMain */ } // namespace rgw -- 2.39.5