From: Daniel Gryniewicz Date: Fri, 29 Jul 2022 14:04:45 +0000 (-0400) Subject: RGW - Zipper - Init cleanup X-Git-Tag: v18.0.0~267^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=3eac59420636a64cb61cf06179a816044e9d0b48;p=ceph.git RGW - Zipper - Init cleanup Cleanup getting config, replacing it with a future-extensible system, and consolidating it all into common zipper code. Signed-off-by: Daniel Gryniewicz --- diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index 48d5b3047881..d26cd49ec89b 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -537,52 +537,9 @@ namespace rgw { g_conf()->rgw_run_sync_thread && g_conf()->rgw_nfs_run_sync_thread; - bool rgw_d3n_datacache_enabled = - cct->_conf->rgw_d3n_l1_local_datacache_enabled; - if (rgw_d3n_datacache_enabled && - (cct->_conf->rgw_max_chunk_size != cct->_conf->rgw_obj_stripe_size)) { - lsubdout(cct, rgw_datacache, 0) - << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires that " - "the chunk_size equals stripe_size)" - << dendl; - rgw_d3n_datacache_enabled = false; - } - if (rgw_d3n_datacache_enabled && !cct->_conf->rgw_beast_enable_async) { - lsubdout(cct, rgw_datacache, 0) - << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires yield " - "context - rgw_beast_enable_async=true)" - << dendl; - rgw_d3n_datacache_enabled = false; - } - lsubdout(cct, rgw, 1) << "D3N datacache enabled: " - << rgw_d3n_datacache_enabled << dendl; - - std::string rgw_store = (!rgw_d3n_datacache_enabled) ? "rados" : "d3n"; - - const auto &config_store = - g_conf().get_val("rgw_backend_store"); -#ifdef WITH_RADOSGW_DBSTORE - if (config_store == "dbstore") { - rgw_store = "dbstore"; - } -#endif - -#ifdef WITH_RADOSGW_MOTR - if (config_store == "motr") { - rgw_store = "motr"; - } -#endif - - // Get the filter - std::string rgw_filter = "none"; - const auto& config_filter = g_conf().get_val("rgw_filter"); - if (config_filter == "base") { - rgw_filter = "base"; - } - + StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context); store = StoreManager::get_storage(this, g_ceph_context, - rgw_store, - rgw_filter, + cfg, run_gc, run_lc, run_quota, diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index ee7c926ed7aa..a22843ade92a 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -4352,38 +4352,16 @@ int main(int argc, const char **argv) bool need_cache = readonly_ops_list.find(opt_cmd) == readonly_ops_list.end(); bool need_gc = (gc_ops_list.find(opt_cmd) != gc_ops_list.end()) && !bypass_gc; - std::string rgw_store = "rados"; - // Get the store backend - const auto& config_store = g_conf().get_val("rgw_backend_store"); -#ifdef WITH_RADOSGW_DBSTORE - if (config_store == "dbstore") { - rgw_store = "dbstore"; - } -#endif - -#ifdef WITH_RADOSGW_MOTR - if (config_store == "motr") { - rgw_store = "motr"; - } -#endif - - // Get the filter - std::string rgw_filter = "none"; - const auto& config_filter = g_conf().get_val("rgw_filter"); - if (config_filter == "base") { - rgw_filter = "base"; - } + StoreManager::Config cfg = StoreManager::get_config(true, g_ceph_context); if (raw_storage_op) { store = StoreManager::get_raw_storage(dpp(), g_ceph_context, - rgw_store, - rgw_filter); + cfg); } else { store = StoreManager::get_storage(dpp(), g_ceph_context, - rgw_store, - rgw_filter, + cfg, false, false, false, diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 6e7509a29df2..af48d1c841d2 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -360,52 +360,11 @@ int radosgw_Main(int argc, const char **argv) rgw_http_client_init(g_ceph_context); rgw_kmip_client_init(*new RGWKMIPManagerImpl(g_ceph_context)); - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_local_datacache_enabled=" << cct->_conf->rgw_d3n_l1_local_datacache_enabled << dendl; - if (cct->_conf->rgw_d3n_l1_local_datacache_enabled) { - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_datacache_persistent_path='" << cct->_conf->rgw_d3n_l1_datacache_persistent_path << "'" << dendl; - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_datacache_size=" << cct->_conf->rgw_d3n_l1_datacache_size << dendl; - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_evict_cache_on_start=" << cct->_conf->rgw_d3n_l1_evict_cache_on_start << dendl; - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_fadvise=" << cct->_conf->rgw_d3n_l1_fadvise << dendl; - lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_eviction_policy=" << cct->_conf->rgw_d3n_l1_eviction_policy << dendl; - } - bool rgw_d3n_datacache_enabled = cct->_conf->rgw_d3n_l1_local_datacache_enabled; - if (rgw_d3n_datacache_enabled && (cct->_conf->rgw_max_chunk_size != cct->_conf->rgw_obj_stripe_size)) { - lsubdout(cct, rgw_datacache, 0) << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires that the chunk_size equals stripe_size)" << dendl; - rgw_d3n_datacache_enabled = false; - } - if (rgw_d3n_datacache_enabled && !cct->_conf->rgw_beast_enable_async) { - lsubdout(cct, rgw_datacache, 0) << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires yield context - rgw_beast_enable_async=true)" << dendl; - rgw_d3n_datacache_enabled = false; - } - lsubdout(cct, rgw, 1) << "D3N datacache enabled: " << rgw_d3n_datacache_enabled << dendl; - - std::string rgw_store = (!rgw_d3n_datacache_enabled) ? "rados" : "d3n"; - - // Get the store backend - const auto& config_store = g_conf().get_val("rgw_backend_store"); -#ifdef WITH_RADOSGW_DBSTORE - if (config_store == "dbstore") { - rgw_store = "dbstore"; - } -#endif - -#ifdef WITH_RADOSGW_MOTR - if (config_store == "motr") { - rgw_store = "motr"; - } -#endif - - // Get the filter - std::string rgw_filter = "none"; - const auto& config_filter = g_conf().get_val("rgw_filter"); - if (config_filter == "base") { - rgw_filter = "base"; - } + StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context); rgw::sal::Store* store = StoreManager::get_storage(&dp, g_ceph_context, - rgw_store, - rgw_filter, + cfg, g_conf()->rgw_enable_gc_threads, g_conf()->rgw_enable_lc_threads, g_conf()->rgw_enable_quota_threads, diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc index 79ecf7452764..2e1249b85f71 100644 --- a/src/rgw/rgw_object_expirer.cc +++ b/src/rgw/rgw_object_expirer.cc @@ -80,7 +80,10 @@ int main(const int argc, const char **argv) common_init_finish(g_ceph_context); const DoutPrefix dp(cct.get(), dout_subsys, "rgw object expirer: "); - store = StoreManager::get_storage(&dp, g_ceph_context, "rados", "none", false, false, false, false, false); + StoreManager::Config cfg; + cfg.store_name = "rados"; + cfg.filter_name = "none"; + store = StoreManager::get_storage(&dp, g_ceph_context, cfg, false, false, false, false, false); if (!store) { std::cerr << "couldn't init storage provider" << std::endl; return EIO; diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc index d27567cf9328..1bcfeb15ac82 100644 --- a/src/rgw/rgw_realm_reloader.cc +++ b/src/rgw/rgw_realm_reloader.cc @@ -105,10 +105,12 @@ void RGWRealmReloader::reload() while (!store) { // recreate and initialize a new store + StoreManager::Config cfg; + cfg.store_name = "rados"; + cfg.filter_name = "none"; store = StoreManager::get_storage(&dp, cct, - "rados", - "none", + cfg, cct->_conf->rgw_enable_gc_threads, cct->_conf->rgw_enable_lc_threads, cct->_conf->rgw_enable_quota_threads, diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 1c55c2dbe0ad..37b95cf0dbed 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -82,11 +82,20 @@ RGWObjState::RGWObjState(const RGWObjState& rhs) : obj (rhs.obj) { compressed = rhs.compressed; } -rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_cache, bool use_gc) +rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* dpp, + CephContext* cct, + const Config& cfg, + bool use_gc_thread, + bool use_lc_thread, + bool quota_threads, + bool run_sync_thread, + bool run_reshard_thread, + bool use_cache, + bool use_gc) { rgw::sal::Store* store{nullptr}; - if (svc.compare("rados") == 0) { + if (cfg.store_name.compare("rados") == 0) { store = newStore(); RGWRados* rados = static_cast(store)->getRados(); @@ -111,7 +120,7 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d return nullptr; } } - else if (svc.compare("d3n") == 0) { + else if (cfg.store_name.compare("d3n") == 0) { store = new rgw::sal::RadosStore(); RGWRados* rados = new D3nRGWDataCache; dynamic_cast(store)->setRados(rados); @@ -136,9 +145,22 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d delete store; return nullptr; } + + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_local_datacache_enabled=" << + cct->_conf->rgw_d3n_l1_local_datacache_enabled << dendl; + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_datacache_persistent_path='" << + cct->_conf->rgw_d3n_l1_datacache_persistent_path << "'" << dendl; + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_datacache_size=" << + cct->_conf->rgw_d3n_l1_datacache_size << dendl; + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_evict_cache_on_start=" << + cct->_conf->rgw_d3n_l1_evict_cache_on_start << dendl; + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_fadvise=" << + cct->_conf->rgw_d3n_l1_fadvise << dendl; + lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_eviction_policy=" << + cct->_conf->rgw_d3n_l1_eviction_policy << dendl; } #ifdef WITH_RADOSGW_DBSTORE - else if (svc.compare("dbstore") == 0) { + else if (cfg.store_name.compare("dbstore") == 0) { store = newDBStore(cct); if ((*(rgw::sal::DBStore*)store).set_run_lc_thread(use_lc_thread) @@ -150,7 +172,7 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d #endif #ifdef WITH_RADOSGW_MOTR - else if (svc.compare("motr") == 0) { + else if (cfg.store_name.compare("motr") == 0) { store = newMotrStore(cct); if (store == nullptr) { ldpp_dout(dpp, 0) << "newMotrStore() failed!" << dendl; @@ -189,7 +211,7 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d } #endif - if (filter.compare("base") == 0) { + if (cfg.filter_name.compare("base") == 0) { rgw::sal::Store* next = store; store = newBaseFilter(next); @@ -203,10 +225,10 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d return store; } -rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter) +rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const Config& cfg) { rgw::sal::Store* store = nullptr; - if (svc.compare("rados") == 0) { + if (cfg.store_name.compare("rados") == 0) { store = newStore(); RGWRados* rados = static_cast(store)->getRados(); @@ -227,7 +249,7 @@ rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvide delete store; return nullptr; } - } else if (svc.compare("dbstore") == 0) { + } else if (cfg.store_name.compare("dbstore") == 0) { #ifdef WITH_RADOSGW_DBSTORE store = newDBStore(cct); @@ -238,7 +260,7 @@ rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvide #else store = nullptr; #endif - } else if (svc.compare("motr") == 0) { + } else if (cfg.store_name.compare("motr") == 0) { #ifdef WITH_RADOSGW_MOTR store = newMotrStore(cct); #else @@ -246,7 +268,7 @@ rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvide #endif } - if (filter.compare("base") == 0) { + if (cfg.filter_name.compare("base") == 0) { rgw::sal::Store* next = store; store = newBaseFilter(next); @@ -270,6 +292,49 @@ void StoreManager::close_storage(rgw::sal::Store* store) delete store; } +StoreManager::Config StoreManager::get_config(bool admin, CephContext* cct) +{ + StoreManager::Config cfg; + + // Get the store backend + const auto& config_store = g_conf().get_val("rgw_backend_store"); + if (config_store == "rados") { + cfg.store_name = "rados"; + + /* Check to see if d3n is configured, but only for non-admin */ + const auto& d3n = g_conf().get_val("rgw_d3n_l1_local_datacache_enabled"); + if (!admin && d3n) { + if (g_conf().get_val("rgw_max_chunk_size") != + g_conf().get_val("rgw_obj_stripe_size")) { + lsubdout(cct, rgw_datacache, 0) << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires that the chunk_size equals stripe_size)" << dendl; + } else if (!g_conf().get_val("rgw_beast_enable_async")) { + lsubdout(cct, rgw_datacache, 0) << "rgw_d3n: WARNING: D3N DataCache disabling (D3N requires yield context - rgw_beast_enable_async=true)" << dendl; + } else { + cfg.store_name = "d3n"; + } + } + } +#ifdef WITH_RADOSGW_DBSTORE + else if (config_store == "dbstore") { + cfg.store_name = "dbstore"; + } +#endif +#ifdef WITH_RADOSGW_MOTR + else if (config_store == "motr") { + cfg.store_name = "motr"; + } +#endif + + // Get the filter + cfg.filter_name = "none"; + const auto& config_filter = g_conf().get_val("rgw_filter"); + if (config_filter == "base") { + cfg.filter_name = "base"; + } + + return cfg; +} + namespace rgw::sal { int Object::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) { @@ -292,4 +357,4 @@ int Object::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) } return 0; } -} +} // namespace rgw::sal diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 6b234aa16848..f15b88c376e9 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1536,26 +1536,59 @@ public: */ class StoreManager { public: + struct Config { + /** Name of store to create */ + std::string store_name; + /** Name of filter to create or "none" */ + std::string filter_name; + }; + StoreManager() {} /** Get a full store by service name */ - static rgw::sal::Store* get_storage(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter, bool use_gc_thread, bool use_lc_thread, bool quota_threads, - bool run_sync_thread, bool run_reshard_thread, bool use_cache = true, bool use_gc = true) { - rgw::sal::Store* store = init_storage_provider(dpp, cct, svc, filter, use_gc_thread, use_lc_thread, - quota_threads, run_sync_thread, run_reshard_thread, use_cache, use_gc); + static rgw::sal::Store* get_storage(const DoutPrefixProvider* dpp, + CephContext* cct, + const Config& cfg, + bool use_gc_thread, + bool use_lc_thread, + bool quota_threads, + bool run_sync_thread, + bool run_reshard_thread, + bool use_cache = true, + bool use_gc = true) { + rgw::sal::Store* store = init_storage_provider(dpp, cct, cfg, use_gc_thread, + use_lc_thread, + quota_threads, + run_sync_thread, + run_reshard_thread, + use_cache, use_gc); return store; } /** Get a stripped down store by service name */ - static rgw::sal::Store* get_raw_storage(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter) { - rgw::sal::Store* store = init_raw_storage_provider(dpp, cct, svc, filter); + static rgw::sal::Store* get_raw_storage(const DoutPrefixProvider* dpp, + CephContext* cct, const Config& cfg) { + rgw::sal::Store* store = init_raw_storage_provider(dpp, cct, cfg); return store; } /** Initialize a new full Store */ - static rgw::sal::Store* init_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_metadata_cache, bool use_gc); + static rgw::sal::Store* init_storage_provider(const DoutPrefixProvider* dpp, + CephContext* cct, + const Config& cfg, + bool use_gc_thread, + bool use_lc_thread, + bool quota_threads, + bool run_sync_thread, + bool run_reshard_thread, + bool use_metadata_cache, + bool use_gc); /** Initialize a new raw Store */ - static rgw::sal::Store* init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, const std::string filter); + static rgw::sal::Store* init_raw_storage_provider(const DoutPrefixProvider* dpp, + CephContext* cct, + const Config& cfg); /** Close a Store when it's no longer needed */ static void close_storage(rgw::sal::Store* store); + /** Get the config for stores/filters */ + static Config get_config(bool admin, CephContext* cct); }; /** @} */