From 0e584be9029dd4520e5c05a4d198bf1f9eeea40e Mon Sep 17 00:00:00 2001 From: Adam Emerson Date: Thu, 9 Nov 2023 15:23:53 -0500 Subject: [PATCH] rgw: Thread site config into RADOS driver Make this available so multisite sync doesn't have to rely on the zone service. Signed-off-by: Adam Emerson --- src/rgw/driver/rados/rgw_sal_rados.cc | 5 ++-- src/rgw/driver/rados/rgw_sal_rados.h | 7 +++-- src/rgw/driver/rados/rgw_zone.cc | 13 ++++++-- src/rgw/driver/rados/rgw_zone.h | 13 +++++++- src/rgw/rgw_admin.cc | 19 ++++++++++-- src/rgw/rgw_appmain.cc | 1 + src/rgw/rgw_object_expirer.cc | 18 ++++++++++- src/rgw/rgw_realm_reloader.cc | 1 + src/rgw/rgw_sal.cc | 14 +++++---- src/rgw/rgw_sal.h | 16 +++++++--- src/test/rgw/rgw_cr_test.cc | 16 ++++++++++ src/test/rgw/test_d4n_filter.cc | 14 +++++++-- src/test/rgw/test_rgw_common.h | 2 ++ src/test/rgw/test_rgw_iam_policy.cc | 8 ++--- src/test/rgw/test_rgw_lua.cc | 43 +++++++++++++++++++-------- 15 files changed, 153 insertions(+), 37 deletions(-) diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index 600e54d97d50a..50c49d24fc347 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -3733,10 +3733,11 @@ int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) extern "C" { -void* newRadosStore(void* io_context) +void* newRadosStore(void* io_context, const void* site_config) { rgw::sal::RadosStore* store = new rgw::sal::RadosStore( - *static_cast(io_context)); + *static_cast(io_context), + *static_cast(site_config)); if (store) { RGWRados* rados = new RGWRados(); diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index d5174f39bc8f6..d4bd19a55fe1a 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -117,6 +117,7 @@ class RadosZone : public StoreZone { class RadosStore : public StoreDriver { private: boost::asio::io_context& io_context; + const rgw::SiteConfig& site_config; RGWRados* rados; RGWUserCtl* user_ctl; std::unique_ptr zone; @@ -124,8 +125,9 @@ class RadosStore : public StoreDriver { std::string topics_oid(const std::string& tenant) const; public: - RadosStore(boost::asio::io_context& io_context) - : io_context(io_context), rados(nullptr) { + RadosStore(boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config) + : io_context(io_context), site_config(site_config), rados(nullptr) { } ~RadosStore() { delete rados; @@ -248,6 +250,7 @@ class RadosStore : public StoreDriver { void setRados(RGWRados * st) { rados = st; } RGWRados* getRados(void) { return rados; } boost::asio::io_context& get_io_context() { return io_context; } + const rgw::SiteConfig& get_siteconfig() { return site_config; } neorados::RADOS& get_neorados() { return *neorados; } RGWServices* svc() { return &rados->svc; } diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc index e6cc54d69d89d..c8ebb79f542f0 100644 --- a/src/rgw/driver/rados/rgw_zone.cc +++ b/src/rgw/driver/rados/rgw_zone.cc @@ -1165,7 +1165,7 @@ static int read_or_create_default_zonegroup(const DoutPrefixProvider* dpp, } int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore) + sal::ConfigStore* cfgstore, bool force_local_zonegroup) { // clear existing configuration zone = nullptr; @@ -1219,7 +1219,7 @@ int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y, } } - if (realm) { + if (realm && !force_local_zonegroup) { // try to load the realm's period r = load_period_zonegroup(dpp, y, cfgstore, *realm, zone_params.id); } else { @@ -1230,6 +1230,15 @@ int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y, return r; } +std::unique_ptr SiteConfig::make_fake() { + auto fake = std::make_unique(); + fake->local_zonegroup.emplace(); + fake->local_zonegroup->zones.emplace(""s, RGWZone{}); + fake->zonegroup = &*fake->local_zonegroup; + fake->zone = &fake->zonegroup->zones.begin()->second; + return fake; +} + int SiteConfig::load_period_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, sal::ConfigStore* cfgstore, diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h index 446f1efda91a6..f0dccdbc4e94e 100644 --- a/src/rgw/driver/rados/rgw_zone.h +++ b/src/rgw/driver/rados/rgw_zone.h @@ -973,7 +973,17 @@ class SiteConfig { /// Load or reload the multisite configuration from storage. This is not /// thread-safe, so requires careful coordination with the RGWRealmReloader. int load(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore); + sal::ConfigStore* cfgstore, bool force_local_zonegroup = false); + + /// Create a fake site config to be used by tests and similar, just + /// to have a site config. + /// + /// \warning Do not use this anywhere but unittests where we need to + /// bring up parts of RGW that require a SiteConfig exist, but need + /// to run without a cluster. + static std::unique_ptr make_fake(); + + virtual ~SiteConfig() = default; private: int load_period_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, @@ -991,4 +1001,5 @@ class SiteConfig { const RGWZone* zone = nullptr; }; + } // namespace rgw diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 5a3e5102bf7f6..8265852973f90 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -87,7 +87,6 @@ using namespace std; static rgw::sal::Driver* driver = NULL; static constexpr auto dout_subsys = ceph_subsys_rgw; - static const DoutPrefixProvider* dpp() { struct GlobalPrefix : public DoutPrefixProvider { CephContext *get_cct() const override { return dout_context; } @@ -4130,6 +4129,11 @@ int main(int argc, const char **argv) // not a raw op if 'period pull' needs to read zone/period configuration bool raw_period_pull = opt_cmd == OPT::PERIOD_PULL && !url.empty(); + // Before a period commit or pull, our zonegroup may not be in the + // period, causing `load_period_zonegroup` to fail. + bool localzonegroup_op = ((opt_cmd == OPT::PERIOD_UPDATE && commit) || + (opt_cmd == OPT::PERIOD_PULL && url.empty())); + std::set raw_storage_ops_list = {OPT::ZONEGROUP_ADD, OPT::ZONEGROUP_CREATE, OPT::ZONEGROUP_DELETE, OPT::ZONEGROUP_GET, OPT::ZONEGROUP_LIST, @@ -4258,14 +4262,25 @@ int main(int argc, const char **argv) return EIO; } + std::unique_ptr site; + if (raw_storage_op) { + site = rgw::SiteConfig::make_fake(); driver = DriverManager::get_raw_storage(dpp(), g_ceph_context, - cfg, context_pool); + cfg, context_pool, *site); } else { + site = std::make_unique(); + auto r = site->load(dpp(), null_yield, cfgstore.get(), localzonegroup_op); + if (r < 0) { + std::cerr << "Unable to initialize site config." << std::endl; + exit(1); + } + driver = DriverManager::get_storage(dpp(), g_ceph_context, cfg, context_pool, + *site, false, false, false, diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc index 096b7d006e1ee..b4aa1019a7398 100644 --- a/src/rgw/rgw_appmain.cc +++ b/src/rgw/rgw_appmain.cc @@ -250,6 +250,7 @@ int rgw::AppMain::init_storage() env.driver = DriverManager::get_storage(dpp, dpp->get_cct(), cfg, *context_pool, + site, run_gc, run_lc, run_quota, diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc index e3a816865f80c..58723503c5e91 100644 --- a/src/rgw/rgw_object_expirer.cc +++ b/src/rgw/rgw_object_expirer.cc @@ -31,6 +31,8 @@ #include "rgw_formats.h" #include "rgw_usage.h" #include "rgw_object_expirer_core.h" +#include "driver/rados/rgw_zone.h" +#include "rgw_sal_config.h" #define dout_subsys ceph_subsys_rgw @@ -89,7 +91,21 @@ int main(const int argc, const char **argv) DriverManager::Config cfg; cfg.store_name = "rados"; cfg.filter_name = "none"; - driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, context_pool, false, false, false, false, false, false, null_yield); + std::unique_ptr cfgstore; + auto config_store_type = g_conf().get_val("rgw_config_store"); + cfgstore = DriverManager::create_config_store(&dp, config_store_type); + if (!cfgstore) { + std::cerr << "Unable to initialize config store." << std::endl; + exit(1); + } + rgw::SiteConfig site; + auto r = site.load(&dp, null_yield, cfgstore.get()); + if (r < 0) { + std::cerr << "Unable to initialize config store." << std::endl; + exit(1); + } + + driver = DriverManager::get_storage(&dp, g_ceph_context, cfg, context_pool, site, false, false, false, false, false, false, null_yield); if (!driver) { std::cerr << "couldn't init storage provider" << std::endl; return EIO; diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc index 26dffa99cf4f7..d425793a3fa47 100644 --- a/src/rgw/rgw_realm_reloader.cc +++ b/src/rgw/rgw_realm_reloader.cc @@ -121,6 +121,7 @@ void RGWRealmReloader::reload() cfg.store_name = "rados"; cfg.filter_name = "none"; env.driver = DriverManager::get_storage(&dp, cct, cfg, io_context, + *env.site, cct->_conf->rgw_enable_gc_threads, cct->_conf->rgw_enable_lc_threads, cct->_conf->rgw_enable_quota_threads, diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index a70813435bc01..5e4603b811625 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -46,7 +46,8 @@ #define dout_subsys ceph_subsys_rgw extern "C" { -extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context); +extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context, + const rgw::SiteConfig* site_config); #ifdef WITH_RADOSGW_DBSTORE extern rgw::sal::Driver* newDBStore(CephContext *cct); #endif @@ -104,6 +105,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* CephContext* cct, const Config& cfg, boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config, bool use_gc_thread, bool use_lc_thread, bool quota_threads, @@ -116,7 +118,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* rgw::sal::Driver* driver{nullptr}; if (cfg.store_name.compare("rados") == 0) { - driver = newRadosStore(&io_context); + driver = newRadosStore(&io_context, &site_config); RGWRados* rados = static_cast(driver)->getRados(); if ((*rados).set_use_cache(use_cache) @@ -142,7 +144,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* } } else if (cfg.store_name.compare("d3n") == 0) { - driver = new rgw::sal::RadosStore(io_context); + driver = new rgw::sal::RadosStore(io_context, site_config); RGWRados* rados = new D3nRGWDataCache; dynamic_cast(driver)->setRados(rados); rados->set_store(static_cast(driver)); @@ -262,11 +264,13 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* return driver; } -rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg, boost::asio::io_context& io_context) +rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, + const Config& cfg, boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config) { rgw::sal::Driver* driver = nullptr; if (cfg.store_name.compare("rados") == 0) { - driver = newRadosStore(&io_context); + driver = newRadosStore(&io_context, &site_config); RGWRados* rados = static_cast(driver)->getRados(); rados->set_context(cct); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index fbff4f60645cb..6ee02be0b67f5 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1537,6 +1537,7 @@ public: CephContext* cct, const Config& cfg, boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config, bool use_gc_thread, bool use_lc_thread, bool quota_threads, @@ -1545,7 +1546,9 @@ public: bool run_notification_thread, optional_yield y, bool use_cache = true, bool use_gc = true) { - rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context, use_gc_thread, + rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context, + site_config, + use_gc_thread, use_lc_thread, quota_threads, run_sync_thread, @@ -1557,8 +1560,11 @@ public: /** Get a stripped down driver by service name */ static rgw::sal::Driver* get_raw_storage(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg, - boost::asio::io_context& io_context) { - rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg, io_context); + boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config) { + rgw::sal::Driver* driver = init_raw_storage_provider(dpp, cct, cfg, + io_context, + site_config); return driver; } /** Initialize a new full Driver */ @@ -1566,6 +1572,7 @@ public: CephContext* cct, const Config& cfg, boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config, bool use_gc_thread, bool use_lc_thread, bool quota_threads, @@ -1578,7 +1585,8 @@ public: static rgw::sal::Driver* init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg, - boost::asio::io_context& io_context); + boost::asio::io_context& io_context, + const rgw::SiteConfig& site_config); /** Close a Driver when it's no longer needed */ static void close_storage(rgw::sal::Driver* driver); diff --git a/src/test/rgw/rgw_cr_test.cc b/src/test/rgw/rgw_cr_test.cc index db59dd1841b38..d75f776eb7499 100644 --- a/src/test/rgw/rgw_cr_test.cc +++ b/src/test/rgw/rgw_cr_test.cc @@ -21,6 +21,8 @@ #include "rgw_cr_rados.h" #include "rgw_sal.h" #include "rgw_sal_rados.h" +#include "driver/rados/rgw_zone.h" +#include "rgw_sal_config.h" #include "gtest/gtest.h" @@ -323,12 +325,26 @@ int main(int argc, const char **argv) ceph::async::io_context_pool context_pool{cct->_conf->rgw_thread_pool_size}; DriverManager::Config cfg = DriverManager::get_config(true, g_ceph_context); + auto config_store_type = g_conf().get_val("rgw_config_store"); + std::unique_ptr cfgstore + = DriverManager::create_config_store(dpp(), config_store_type); + if (!cfgstore) { + std::cerr << "Unable to initialize config store." << std::endl; + exit(1); + } + rgw::SiteConfig site; + auto r = site.load(dpp(), null_yield, cfgstore.get()); + if (r < 0) { + std::cerr << "Unable to initialize config store." << std::endl; + exit(1); + } store = static_cast( DriverManager::get_storage(dpp(), g_ceph_context, cfg, context_pool, + site, false, false, false, diff --git a/src/test/rgw/test_d4n_filter.cc b/src/test/rgw/test_d4n_filter.cc index 00f7f24e00f80..9b24916009189 100644 --- a/src/test/rgw/test_d4n_filter.cc +++ b/src/test/rgw/test_d4n_filter.cc @@ -11,6 +11,8 @@ #include "rgw_sal.h" #include "rgw_auth.h" #include "rgw_auth_registry.h" +#include "driver/rados/rgw_zone.h" +#include "rgw_sal_config.h" #include @@ -55,16 +57,24 @@ class Environment : public ::testing::Environment { cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, CINIT_FLAG_NO_MON_CONFIG); - + dpp = new DoutPrefix(cct->get(), dout_subsys, "d4n test: "); DriverManager::Config cfg; cfg.store_name = "dbstore"; cfg.filter_name = "d4n"; - + auto config_store_type = g_conf().get_val("rgw_config_store"); + std::unique_ptr cfgstore + = DriverManager::create_config_store(dpp, config_store_type); + ASSERT_TRUE(cfgstore); + rgw::SiteConfig site; + auto r = site.load(dpp, null_yield, cfgstore.get()); + ASSERT_GT(r, 0); + driver = DriverManager::get_storage(dpp, dpp->get_cct(), cfg, ioc, + site, false, false, false, diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h index 664e0b22e8046..ece16e844037e 100644 --- a/src/test/rgw/test_rgw_common.h +++ b/src/test/rgw/test_rgw_common.h @@ -502,5 +502,7 @@ void test_rgw_populate_bucket(rgw_bucket *b, const char *t, const char *n, const void test_rgw_init_bucket(rgw_bucket *bucket, const char *name); rgw_obj test_rgw_create_obj(const rgw_bucket& bucket, const std::string& name, const std::string& instance, const std::string& ns); + + #endif diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc index b7cd66f6ea5ad..e69910395a8ff 100644 --- a/src/test/rgw/test_rgw_iam_policy.cc +++ b/src/test/rgw/test_rgw_iam_policy.cc @@ -30,14 +30,13 @@ #include "rgw_op.h" #include "rgw_process_env.h" #include "rgw_sal_rados.h" - +#include "driver/rados/rgw_zone.h" +#include "rgw_sal_config.h" using std::string; -using std::vector; using boost::container::flat_set; using boost::intrusive_ptr; -using boost::make_optional; using boost::none; using rgw::auth::Identity; @@ -914,7 +913,8 @@ TEST_F(IPPolicyTest, IPEnvironment) { // Unfortunately RGWCivetWeb is too tightly tied to civetweb to test RGWCivetWeb::init_env. RGWEnv rgw_env; ceph::async::io_context_pool context_pool(cct->_conf->rgw_thread_pool_size); \ - rgw::sal::RadosStore store(context_pool); + auto site = rgw::SiteConfig::make_fake(); + rgw::sal::RadosStore store(context_pool, *site); std::unique_ptr user = store.get_user(rgw_user()); rgw_env.set("REMOTE_ADDR", "192.168.1.1"); rgw_env.set("HTTP_HOST", "1.2.3.4"); diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index 1ae7976c7944d..8aed91bb6f5ac 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -8,6 +8,8 @@ #include "rgw_lua_request.h" #include "rgw_lua_background.h" #include "rgw_lua_data_filter.h" +#include "driver/rados/rgw_zone.h" +#include "rgw_sal_config.h" using namespace std; using namespace rgw; @@ -160,13 +162,30 @@ CctCleaner cleaner(g_cct); tracing::Tracer tracer; -#define MAKE_STORE \ - ceph::async::io_context_pool context_pool(g_cct->_conf->rgw_thread_pool_size); \ - auto store = std::unique_ptr(new sal::RadosStore(context_pool)); \ - store->setRados(new RGWRados); +inline std::unique_ptr make_store() { + auto context_pool = std::make_unique( + g_cct->_conf->rgw_thread_pool_size); + std::unique_ptr site = rgw::SiteConfig::make_fake(); + + + struct StoreBundle : public sal::RadosStore { + std::unique_ptr context_pool; + std::unique_ptr site; + StoreBundle(std::unique_ptr context_pool_, + std::unique_ptr site_) + : sal::RadosStore(*context_pool_.get(), *site_), + context_pool(std::move(context_pool_)), + site(std::move(site_)) { + setRados(new RGWRados); + } + virtual ~StoreBundle() = default; + }; + return std::make_unique(std::move(context_pool), + std::move(site)); +}; #define DEFINE_REQ_STATE RGWProcessEnv pe; \ - MAKE_STORE; \ + auto store = make_store(); \ pe.lua.manager = store->get_lua_manager(""); \ RGWEnv e; \ req_state s(g_cct, pe, &e, 0); @@ -861,7 +880,7 @@ public: TEST(TestRGWLuaBackground, Start) { - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); { // ctr and dtor without running @@ -895,7 +914,7 @@ TEST(TestRGWLuaBackground, Script) RGW[key] = value )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get()); lua_background.start(); @@ -948,7 +967,7 @@ TEST(TestRGWLuaBackground, Pause) end )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get()); lua_background.start(); @@ -974,7 +993,7 @@ TEST(TestRGWLuaBackground, PauseWhileReading) end )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get(), 2); lua_background.start(); @@ -996,7 +1015,7 @@ TEST(TestRGWLuaBackground, ReadWhilePaused) RGW[key] = value )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get()); lua_background.pause(); @@ -1020,7 +1039,7 @@ TEST(TestRGWLuaBackground, PauseResume) end )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get()); lua_background.start(); @@ -1049,7 +1068,7 @@ TEST(TestRGWLuaBackground, MultipleStarts) end )"; - MAKE_STORE; + auto store = make_store(); auto manager = store->get_lua_manager(""); TestBackground lua_background(store.get(), script, manager.get()); lua_background.start(); -- 2.39.5