From 67d30e555638d30fb0dedc595797876b45d3f3a4 Mon Sep 17 00:00:00 2001 From: Matt Benjamin Date: Thu, 18 Aug 2022 17:25:09 -0400 Subject: [PATCH] librgw: add ability to conditionally export HTTP frontends/apis Signed-off-by: Matt Benjamin --- doc/radosgw/nfs.rst | 10 +- src/common/options/rgw.yaml.in | 16 ++ src/rgw/CMakeLists.txt | 18 +- src/rgw/librgw.cc | 197 ++++++-------- src/rgw/rgw_frontend.h | 5 +- src/rgw/rgw_lib.h | 29 +- src/rgw/rgw_main.cc | 481 ++++++++++++++++++++++++++++++--- src/rgw/rgw_main.h | 141 ++++++++++ src/rgw/rgw_rest.h | 1 + 9 files changed, 727 insertions(+), 171 deletions(-) create mode 100644 src/rgw/rgw_main.h diff --git a/doc/radosgw/nfs.rst b/doc/radosgw/nfs.rst index e1a5cc1c12202..5b5732dec1166 100644 --- a/doc/radosgw/nfs.rst +++ b/doc/radosgw/nfs.rst @@ -122,13 +122,17 @@ Required ceph.conf configuration for RGW NFS includes: * valid [client.rgw.{instance-name}] section * valid values for minimal instance configuration, in particular, an installed and correct ``keyring`` -Other config variables are optional, front-end-specific and front-end -selection variables (e.g., ``rgw data`` and ``rgw frontends``) are -optional and in some cases ignored. +Other config variables (e.g., ``rgw data`` and ``rgw backend store``) are +optional. A small number of config variables (e.g., ``rgw_nfs_namespace_expire_secs``) are unique to RGW NFS. +In particular, front-end selection is handled specially by the librgw.so runtime. By default, only the +``rgw-nfs`` frontend is started. Additional frontends (e.g., ``beast``) are enabled via the +``rgw nfs frontends`` config option. It's syntax is identical to the ordinary ``rgw frontends`` option. +Default options for non-default frontends are specified via ``rgw frontend defaults`` as normal. + ganesha.conf ------------ diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index ca2db88ae2916..61bf6f07cf5ca 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -1148,6 +1148,22 @@ options: services: - rgw with_legacy: true +- name: rgw_nfs_frontends + type: str + level: basic + desc: RGW frontends configuration when running as librgw/nfs + long_desc: A comma delimited list of frontends configuration. Each configuration + contains the type of the frontend followed by an optional space delimited set + of key=value config parameters. + fmt_desc: Configures the HTTP frontend(s). The configuration for multiple + frontends can be provided in a comma-delimited list. Each frontend + configuration may include a list of options separated by spaces, + where each option is in the form "key=value" or "key". See + `HTTP Frontends`_ for more on supported options. + default: rgw-nfs + services: + - rgw + with_legacy: true - name: rgw_rados_pool_autoscale_bias type: float level: advanced diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 6206db6ff1844..c81e5f103ca43 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -346,6 +346,8 @@ target_link_libraries(rgw_schedulers set(radosgw_srcs rgw_main.cc + rgw_file.cc + librgw.cc rgw_loadgen_process.cc rgw_asio_client.cc rgw_asio_frontend.cc @@ -413,12 +415,23 @@ target_link_libraries(radosgw-object-expirer ${rgw_libs} librados install(TARGETS radosgw-object-expirer DESTINATION bin) set(librgw_srcs - librgw.cc - rgw_file.cc) + ${radosgw_srcs}) add_library(rgw SHARED ${librgw_srcs}) + +target_compile_definitions(rgw PUBLIC "-DCLS_CLIENT_HIDE_IOCTX") +target_include_directories(rgw + PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src" + PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw" + PRIVATE "${LUA_INCLUDE_DIR}") + +target_include_directories(rgw SYSTEM PUBLIC "../rapidjson/include") + target_link_libraries(rgw PRIVATE ${rgw_libs} + rgw_schedulers + kmip librados cls_rgw_client cls_otp_client @@ -429,7 +442,6 @@ target_link_libraries(rgw neorados_cls_fifo cls_version_client cls_user_client - global ${LIB_RESOLV} ${CURL_LIBRARIES} ${EXPAT_LIBRARIES} diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index d26cd49ec89b9..98c637c06d246 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -29,6 +29,8 @@ #include "common/config.h" #include "common/errno.h" #include "common/Timer.h" +#include "common/TracepointProvider.h" +#include "common/openssl_opts_handler.h" #include "common/Throttle.h" #include "common/WorkQueue.h" #include "common/ceph_argparse.h" @@ -38,7 +40,25 @@ #include "rgw_resolve.h" #include "rgw_op.h" +#include "rgw_period_pusher.h" +#include "rgw_realm_reloader.h" #include "rgw_rest.h" +#include "rgw_rest_s3.h" +#include "rgw_rest_swift.h" +#include "rgw_rest_admin.h" +#include "rgw_rest_info.h" +#include "rgw_rest_usage.h" +#include "rgw_rest_user.h" +#include "rgw_rest_bucket.h" +#include "rgw_rest_metadata.h" +#include "rgw_rest_log.h" +#include "rgw_rest_config.h" +#include "rgw_rest_realm.h" +#include "rgw_rest_sts.h" +#include "rgw_rest_ratelimit.h" +#include "rgw_swift_auth.h" +#include "rgw_log.h" +#include "rgw_tools.h" #include "rgw_frontend.h" #include "rgw_request.h" #include "rgw_process.h" @@ -51,6 +71,8 @@ #include "rgw_lib_frontend.h" #include "rgw_http_client.h" #include "rgw_http_client_curl.h" +#include "rgw_kmip_client.h" +#include "rgw_kmip_client_impl.h" #include "rgw_perf_counters.h" #ifdef WITH_RADOSGW_AMQP_ENDPOINT #include "rgw_amqp.h" @@ -58,8 +80,16 @@ #ifdef WITH_RADOSGW_KAFKA_ENDPOINT #include "rgw_kafka.h" #endif +#include "rgw_asio_frontend.h" +#include "rgw_dmclock_scheduler_ctx.h" +#include "rgw_lua.h" +#ifdef WITH_RADOSGW_DBSTORE +#include "rgw_sal_dbstore.h" +#endif +#include "rgw_lua_background.h" #include "services/svc_zone.h" +#include "rgw_main.h" #include #include @@ -70,6 +100,13 @@ using namespace std; +namespace { + TracepointProvider::Traits rgw_op_tracepoint_traits( + "librgw_op_tp.so", "rgw_op_tracing"); + TracepointProvider::Traits rgw_rados_tracepoint_traits( + "librgw_rados_tp.so", "rgw_rados_tracing"); +} + bool global_stop = false; static void handle_sigterm(int signum) @@ -484,6 +521,11 @@ namespace rgw { return 0; } + void RGWLib::set_fe(rgw::RGWLibFrontend* fe) + { + this->fe = fe; + } + int RGWLib::init() { vector args; @@ -492,13 +534,15 @@ namespace rgw { int RGWLib::init(vector& args) { - int r = 0; - /* alternative default for module */ map defaults = { { "debug_rgw", "1/5" }, { "keyring", "$rgw_data/keyring" }, - { "log_file", "/var/log/radosgw/$cluster-$name.log" } + { "log_file", "/var/log/radosgw/$cluster-$name.log" }, + { "objecter_inflight_ops", "24576" }, + // require a secure mon connection by default + { "ms_mon_client_mode", "secure" }, + { "auth_client_required", "cephx" }, }; cct = rgw_global_init(&defaults, args, @@ -513,39 +557,18 @@ namespace rgw { init_timer.add_event_after(g_conf()->rgw_init_timeout, new C_InitTimeout); mutex.unlock(); - common_init_finish(g_ceph_context); - - rgw_tools_init(this, g_ceph_context); - - rgw_init_resolver(); - rgw::curl::setup_curl(boost::none); - rgw_http_client_init(g_ceph_context); + /* stage all front-ends (before common-init-finish) */ - auto run_gc = - g_conf()->rgw_enable_gc_threads && - g_conf()->rgw_nfs_run_gc_threads; + rgw::InitHelper init_helper( + fes, fe_configs, fe_map, ldh, olog, rest, lua_background, + implicit_tenant_context, sched_ctx, ratelimiter, reloader, + pusher, fe_pauser, realm_watcher, rgw_pauser, store, this); - auto run_lc = - g_conf()->rgw_enable_lc_threads && - g_conf()->rgw_nfs_run_lc_threads; + init_helper.init_frontends1(true /* nfs */); - auto run_quota = - g_conf()->rgw_enable_quota_threads && - g_conf()->rgw_nfs_run_quota_threads; - - auto run_sync = - g_conf()->rgw_run_sync_thread && - g_conf()->rgw_nfs_run_sync_thread; - - StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context); - store = StoreManager::get_storage(this, g_ceph_context, - cfg, - run_gc, - run_lc, - run_quota, - run_sync, - g_conf().get_val("rgw_dynamic_resharding")); + common_init_finish(g_ceph_context); + init_helper.init_storage(); if (!store) { mutex.lock(); init_timer.cancel_all_events(); @@ -556,89 +579,25 @@ namespace rgw { return -EIO; } - r = rgw_perf_start(g_ceph_context); - - rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup()); + init_helper.init_perfcounters(); + init_helper.init_http_clients(); + init_helper.cond_init_apis(); mutex.lock(); init_timer.cancel_all_events(); init_timer.shutdown(); mutex.unlock(); - if (r) - return -EIO; - - const string& ldap_uri = store->ctx()->_conf->rgw_ldap_uri; - const string& ldap_binddn = store->ctx()->_conf->rgw_ldap_binddn; - const string& ldap_searchdn = store->ctx()->_conf->rgw_ldap_searchdn; - const string& ldap_searchfilter = store->ctx()->_conf->rgw_ldap_searchfilter; - const string& ldap_dnattr = - store->ctx()->_conf->rgw_ldap_dnattr; - std::string ldap_bindpw = parse_rgw_ldap_bindpw(store->ctx()); - - ldh = new rgw::LDAPHelper(ldap_uri, ldap_binddn, ldap_bindpw.c_str(), - ldap_searchdn, ldap_searchfilter, ldap_dnattr); - ldh->init(); - ldh->bind(); - - rgw_log_usage_init(g_ceph_context, store); - - // XXX ex-RGWRESTMgr_lib, mgr->set_logging(true) - - OpsLogManifold* olog_manifold = new OpsLogManifold(); - if (!g_conf()->rgw_ops_log_socket_path.empty()) { - OpsLogSocket* olog_socket = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); - olog_socket->init(g_conf()->rgw_ops_log_socket_path); - olog_manifold->add_sink(olog_socket); - } - OpsLogFile* ops_log_file; - if (!g_conf()->rgw_ops_log_file_path.empty()) { - ops_log_file = new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, g_conf()->rgw_ops_log_data_backlog); - ops_log_file->start(); - olog_manifold->add_sink(ops_log_file); - } - olog_manifold->add_sink(new OpsLogRados(store)); - olog = olog_manifold; - - int port = 80; - RGWProcessEnv env = { store, &rest, olog, port }; - - string fe_count{"0"}; - fec = new RGWFrontendConfig("rgwlib"); - fe = new RGWLibFrontend(env, fec); + init_helper.init_ldap(); + init_helper.init_opslog(); init_async_signal_handler(); register_async_signal_handler(SIGUSR1, handle_sigterm); - map service_map_meta; - service_map_meta["pid"] = stringify(getpid()); - service_map_meta["frontend_type#" + fe_count] = "rgw-nfs"; - service_map_meta["frontend_config#" + fe_count] = fec->get_config(); - - fe->init(); - if (r < 0) { - derr << "ERROR: failed initializing frontend" << dendl; - return r; - } - - fe->run(); - - r = store->register_to_service_map(this, "rgw-nfs", service_map_meta); - if (r < 0) { - derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl; - /* ignore error */ - } - -#ifdef WITH_RADOSGW_AMQP_ENDPOINT - if (!rgw::amqp::init(cct.get())) { - derr << "ERROR: failed to initialize AMQP manager" << dendl; - } -#endif -#ifdef WITH_RADOSGW_KAFKA_ENDPOINT - if (!rgw::kafka::init(cct.get())) { - derr << "ERROR: failed to initialize Kafka manager" << dendl; - } -#endif + init_helper.init_tracepoints(); + init_helper.init_frontends2(this /* rgwlib */); + init_helper.init_notification_endpoints(); + init_helper.init_lua(); return 0; } /* RGWLib::init() */ @@ -647,13 +606,24 @@ namespace rgw { { derr << "shutting down" << dendl; - fe->stop(); + if (store->get_name() == "rados") { + reloader.reset(); // stop the realm reloader + } - fe->join(); + for (auto& fe : fes) { + fe->stop(); + } - delete fe; - delete fec; - delete ldh; + for (auto& fe : fes) { + fe->join(); + delete fe; + } + + for (auto& fec : fe_configs) { + delete fec; + } + + ldh.reset(nullptr); // deletes unregister_async_signal_handler(SIGUSR1, handle_sigterm); shutdown_async_signal_handler(); @@ -662,12 +632,19 @@ namespace rgw { delete olog; + if (lua_background) { + lua_background->shutdown(); + } + StoreManager::close_storage(store); rgw_tools_cleanup(); rgw_shutdown_resolver(); rgw_http_client_cleanup(); + rgw_kmip_client_cleanup(); rgw::curl::cleanup_curl(); + g_conf().remove_observer(implicit_tenant_context.get()); + implicit_tenant_context.reset(); // deletes #ifdef WITH_RADOSGW_AMQP_ENDPOINT rgw::amqp::shutdown(); #endif diff --git a/src/rgw/rgw_frontend.h b/src/rgw/rgw_frontend.h index a648489ac910d..12f38602b3ce9 100644 --- a/src/rgw/rgw_frontend.h +++ b/src/rgw/rgw_frontend.h @@ -6,6 +6,7 @@ #include #include +#include #include "common/RWLock.h" @@ -188,12 +189,12 @@ public: // FrontendPauser implementation for RGWRealmReloader class RGWFrontendPauser : public RGWRealmReloader::Pauser { - std::list &frontends; + std::vector &frontends; RGWRealmReloader::Pauser* pauser; rgw::auth::ImplicitTenants& implicit_tenants; public: - RGWFrontendPauser(std::list &frontends, + RGWFrontendPauser(std::vector &frontends, rgw::auth::ImplicitTenants& implicit_tenants, RGWRealmReloader::Pauser* pauser = nullptr) : frontends(frontends), diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h index 7336aaf4c8fb2..60be364eebabe 100644 --- a/src/rgw/rgw_lib.h +++ b/src/rgw/rgw_lib.h @@ -14,9 +14,12 @@ #include "rgw_frontend.h" #include "rgw_process.h" #include "rgw_rest_s3.h" // RGW_Auth_S3 +#include "rgw_period_pusher.h" +#include "rgw_realm_reloader.h" #include "rgw_ldap.h" #include "services/svc_zone_utils.h" #include "include/ceph_assert.h" +#include "rgw_main.h" class OpsLogSink; @@ -25,16 +28,27 @@ namespace rgw { class RGWLibFrontend; class RGWLib : public DoutPrefixProvider { - RGWFrontendConfig* fec; RGWLibFrontend* fe; + std::vector fes; + std::vector fe_configs; + std::multimap fe_map; + std::unique_ptr reloader; + std::unique_ptr pusher; + std::unique_ptr fe_pauser; + std::unique_ptr realm_watcher; + std::unique_ptr rgw_pauser; + std::unique_ptr lua_background; OpsLogSink* olog; - rgw::LDAPHelper* ldh{nullptr}; - RGWREST rest; // XXX needed for RGWProcessEnv + std::unique_ptr implicit_tenant_context; + std::unique_ptr sched_ctx; + std::unique_ptr ratelimiter; + std::unique_ptr ldh; + RGWREST rest; rgw::sal::Store* store; boost::intrusive_ptr cct; public: - RGWLib() : fec(nullptr), fe(nullptr), olog(nullptr), store(nullptr) + RGWLib() : fe(nullptr), olog(nullptr), store(nullptr) {} ~RGWLib() {} @@ -42,12 +56,13 @@ namespace rgw { RGWLibFrontend* get_fe() { return fe; } - rgw::LDAPHelper* get_ldh() { return ldh; } - + rgw::LDAPHelper* get_ldh() { return ldh.get(); } CephContext *get_cct() const override { return cct.get(); } unsigned get_subsys() const { return ceph_subsys_rgw; } std::ostream& gen_prefix(std::ostream& out) const { return out << "lib rgw: "; } + void set_fe(RGWLibFrontend* fe); + int init(); int init(std::vector& args); int stop(); @@ -109,14 +124,12 @@ namespace rgw { }; /* RGWLibIO */ -/* XXX */ class RGWRESTMgr_Lib : public RGWRESTMgr { public: RGWRESTMgr_Lib() {} ~RGWRESTMgr_Lib() override {} }; /* RGWRESTMgr_Lib */ -/* XXX */ class RGWHandler_Lib : public RGWHandler { friend class RGWRESTMgr_Lib; public: diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 2b6e66e3806f1..14fa4c1706853 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -29,6 +29,9 @@ #include "rgw_rest_sts.h" #include "rgw_swift_auth.h" #include "rgw_log.h" +#include "rgw_lib.h" +#include "rgw_frontend.h" +#include "rgw_lib_frontend.h" #include "rgw_tools.h" #include "rgw_resolve.h" #include "rgw_request.h" @@ -51,7 +54,6 @@ #include "rgw_sal_dbstore.h" #endif #include "rgw_lua_background.h" - #include "services/svc_zone.h" #ifdef HAVE_SYS_PRCTL_H @@ -63,10 +65,10 @@ using namespace std; static constexpr auto dout_subsys = ceph_subsys_rgw; namespace { -TracepointProvider::Traits rgw_op_tracepoint_traits("librgw_op_tp.so", - "rgw_op_tracing"); -TracepointProvider::Traits rgw_rados_tracepoint_traits("librgw_rados_tp.so", - "rgw_rados_tracing"); + TracepointProvider::Traits rgw_op_tracepoint_traits( + "librgw_op_tp.so", "rgw_op_tracing"); + TracepointProvider::Traits rgw_rados_tracepoint_traits( + "librgw_rados_tp.so", "rgw_rados_tracing"); } static sig_t sighandler_alrm; @@ -138,7 +140,6 @@ static void godown_alarm(int signum) _exit(0); } - class C_InitTimeout : public Context { public: C_InitTimeout() {} @@ -164,46 +165,439 @@ static int usage() return 0; } -static RGWRESTMgr *set_logging(RGWRESTMgr *mgr) +void rgw::InitHelper::init_frontends1(bool nfs) { - mgr->set_logging(true); - return mgr; -} + this->nfs = nfs; + std::string fe_key = (nfs) ? "rgw_nfs_frontends" : "rgw_frontends"; + std::vector frontends; + std::string rgw_frontends_str = g_conf().get_val(fe_key); + g_conf().early_expand_meta(rgw_frontends_str, &cerr); + get_str_vec(rgw_frontends_str, ",", frontends); + + for (auto &f : frontends) { + if (f.find("beast") != string::npos) { + have_http_frontend = true; + if (f.find("port") != string::npos) { + // check for the most common ws problems + if ((f.find("port=") == string::npos) || + (f.find("port= ") != string::npos)) { + derr << + R"(WARNING: radosgw frontend config found unexpected spacing around 'port' + (ensure frontend port parameter has the form 'port=80' with no spaces + before or after '='))" + << dendl; + } + } + } else { + if (f.find("civetweb") != string::npos) { + have_http_frontend = true; + } + } /* fe !beast */ + + RGWFrontendConfig *config = new RGWFrontendConfig(f); + int r = config->init(); + if (r < 0) { + delete config; + cerr << "ERROR: failed to init config: " << f << std::endl; + continue; + } + + fe_configs.push_back(config); + fe_map.insert( + pair(config->get_framework(), config)); + } /* for each frontend */ + + // maintain existing region root pool for new multisite objects + if (!g_conf()->rgw_region_root_pool.empty()) { + const char *root_pool = g_conf()->rgw_region_root_pool.c_str(); + if (g_conf()->rgw_zonegroup_root_pool.empty()) { + g_conf().set_val_or_die("rgw_zonegroup_root_pool", root_pool); + } + if (g_conf()->rgw_period_root_pool.empty()) { + g_conf().set_val_or_die("rgw_period_root_pool", root_pool); + } + if (g_conf()->rgw_realm_root_pool.empty()) { + g_conf().set_val_or_die("rgw_realm_root_pool", root_pool); + } + } -static RGWRESTMgr *rest_filter(rgw::sal::Store* store, int dialect, RGWRESTMgr *orig) + // for region -> zonegroup conversion (must happen before + // common_init_finish()) + if (!g_conf()->rgw_region.empty() && g_conf()->rgw_zonegroup.empty()) { + g_conf().set_val_or_die("rgw_zonegroup", g_conf()->rgw_region.c_str()); + } + + ceph::crypto::init_openssl_engine_once(); +} /* init_frontends1 */ + +void rgw::InitHelper::init_storage() { - RGWSyncModuleInstanceRef sync_module = store->get_sync_module(); - if (sync_module) { - return sync_module->get_rest_filter(dialect, orig); - } else { - return orig; + auto run_gc = + g_conf()->rgw_enable_gc_threads && + g_conf()->rgw_nfs_run_gc_threads; + + auto run_lc = + g_conf()->rgw_enable_lc_threads && + g_conf()->rgw_nfs_run_lc_threads; + + auto run_quota = + g_conf()->rgw_enable_quota_threads && + g_conf()->rgw_nfs_run_quota_threads; + + auto run_sync = + g_conf()->rgw_run_sync_thread && + g_conf()->rgw_nfs_run_sync_thread; + + StoreManager::Config cfg = StoreManager::get_config(false, g_ceph_context); + store = StoreManager::get_storage(dpp, dpp->get_cct(), + cfg, + run_gc, + run_lc, + run_quota, + run_sync, + g_conf().get_val("rgw_dynamic_resharding")); + +} /* init_storage */ + +void rgw::InitHelper::init_perfcounters() +{ + (void) rgw_perf_start(dpp->get_cct()); +} /* init_perfcounters */ + +void rgw::InitHelper::init_http_clients() +{ + rgw_init_resolver(); + rgw::curl::setup_curl(fe_map); + rgw_http_client_init(dpp->get_cct()); + rgw_kmip_client_init(*new RGWKMIPManagerImpl(dpp->get_cct())); +} /* init_http_clients */ + +void rgw::InitHelper::cond_init_apis() +{ + rgw_rest_init(g_ceph_context, store->get_zone()->get_zonegroup()); + + if (have_http_frontend) { + std::vector apis; + get_str_vec(g_conf()->rgw_enable_apis, apis); + + std::map apis_map; + for (auto &api : apis) { + apis_map[api] = true; + } + + /* warn about insecure keystone secret config options */ + if (!(g_ceph_context->_conf->rgw_keystone_admin_token.empty() || + g_ceph_context->_conf->rgw_keystone_admin_password.empty())) { + dout(0) + << "WARNING: rgw_keystone_admin_token and " + "rgw_keystone_admin_password should be avoided as they can " + "expose secrets. Prefer the new rgw_keystone_admin_token_path " + "and rgw_keystone_admin_password_path options, which read their " + "secrets from files." + << dendl; + } + + // S3 website mode is a specialization of S3 + const bool s3website_enabled = apis_map.count("s3website") > 0; + const bool sts_enabled = apis_map.count("sts") > 0; + const bool iam_enabled = apis_map.count("iam") > 0; + const bool pubsub_enabled = + apis_map.count("pubsub") > 0 || apis_map.count("notifications") > 0; + // Swift API entrypoint could placed in the root instead of S3 + const bool swift_at_root = g_conf()->rgw_swift_url_prefix == "/"; + if (apis_map.count("s3") > 0 || s3website_enabled) { + if (!swift_at_root) { + rest.register_default_mgr(set_logging( + rest_filter(store, RGW_REST_S3, + new RGWRESTMgr_S3(s3website_enabled, sts_enabled, + iam_enabled, pubsub_enabled)))); + } else { + derr << "Cannot have the S3 or S3 Website enabled together with " + << "Swift API placed in the root of hierarchy" << dendl; + } + } + + if (apis_map.count("swift") > 0) { + RGWRESTMgr_SWIFT* const swift_resource = new RGWRESTMgr_SWIFT; + + if (! g_conf()->rgw_cross_domain_policy.empty()) { + swift_resource->register_resource("crossdomain.xml", + set_logging(new RGWRESTMgr_SWIFT_CrossDomain)); + } + + swift_resource->register_resource("healthcheck", + set_logging(new RGWRESTMgr_SWIFT_HealthCheck)); + + swift_resource->register_resource("info", + set_logging(new RGWRESTMgr_SWIFT_Info)); + + if (! swift_at_root) { + rest.register_resource(g_conf()->rgw_swift_url_prefix, + set_logging(rest_filter(store, RGW_REST_SWIFT, + swift_resource))); + } else { + if (store->get_zone()->get_zonegroup().get_zone_count() > 1) { + derr << "Placing Swift API in the root of URL hierarchy while running" + << " multi-site configuration requires another instance of RadosGW" + << " with S3 API enabled!" << dendl; + } + + rest.register_default_mgr(set_logging(swift_resource)); + } + } + + if (apis_map.count("swift_auth") > 0) { + rest.register_resource(g_conf()->rgw_swift_auth_entry, + set_logging(new RGWRESTMgr_SWIFT_Auth)); + } + + if (apis_map.count("admin") > 0) { + RGWRESTMgr_Admin *admin_resource = new RGWRESTMgr_Admin; + admin_resource->register_resource("info", new RGWRESTMgr_Info); + admin_resource->register_resource("usage", new RGWRESTMgr_Usage); + admin_resource->register_resource("user", new RGWRESTMgr_User); + + /* Register store-specific admin APIs */ + store->register_admin_apis(admin_resource); + rest.register_resource(g_conf()->rgw_admin_entry, admin_resource); + } + } /* have_http_frontend */ +} /* init_apis */ + +void rgw::InitHelper::init_ldap() +{ + const string &ldap_uri = store->ctx()->_conf->rgw_ldap_uri; + const string &ldap_binddn = store->ctx()->_conf->rgw_ldap_binddn; + const string &ldap_searchdn = store->ctx()->_conf->rgw_ldap_searchdn; + const string &ldap_searchfilter = store->ctx()->_conf->rgw_ldap_searchfilter; + const string &ldap_dnattr = store->ctx()->_conf->rgw_ldap_dnattr; + std::string ldap_bindpw = parse_rgw_ldap_bindpw(store->ctx()); + + ldh.reset(new rgw::LDAPHelper(ldap_uri, ldap_binddn, + ldap_bindpw.c_str(), ldap_searchdn, ldap_searchfilter, ldap_dnattr)); + ldh->init(); + ldh->bind(); +} /* init_ldap */ + +void rgw::InitHelper::init_opslog() +{ + rgw_log_usage_init(dpp->get_cct(), store); + + OpsLogManifold *olog_manifold = new OpsLogManifold(); + if (!g_conf()->rgw_ops_log_socket_path.empty()) { + OpsLogSocket *olog_socket = + new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); + olog_socket->init(g_conf()->rgw_ops_log_socket_path); + olog_manifold->add_sink(olog_socket); } -} + OpsLogFile *ops_log_file; + if (!g_conf()->rgw_ops_log_file_path.empty()) { + ops_log_file = + new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, + g_conf()->rgw_ops_log_data_backlog); + ops_log_file->start(); + olog_manifold->add_sink(ops_log_file); + } + olog_manifold->add_sink(new OpsLogRados(store)); + olog = olog_manifold; +} /* init_opslog */ -class RGWPauser : public RGWRealmReloader::Pauser { - std::vector pausers; +int rgw::InitHelper::init_frontends2(RGWLib* rgwlib) +{ + int r{0}; + vector frontends_def; + std::string frontend_defs_str = + g_conf().get_val("rgw_frontend_defaults"); + get_str_vec(frontend_defs_str, ",", frontends_def); -public: - ~RGWPauser() override = default; - - void add_pauser(Pauser* pauser) { - pausers.push_back(pauser); + std::map service_map_meta; + service_map_meta["pid"] = stringify(getpid()); + + std::map > fe_def_map; + for (auto& f : frontends_def) { + RGWFrontendConfig *config = new RGWFrontendConfig(f); + int r = config->init(); + if (r < 0) { + delete config; + cerr << "ERROR: failed to init default config: " << f << std::endl; + continue; + } + fe_def_map[config->get_framework()].reset(config); + } + + /* Initialize the registry of auth strategies which will coordinate + * the dynamic reconfiguration. */ + implicit_tenant_context.reset(new rgw::auth::ImplicitTenants{g_conf()}); + g_conf().add_observer(implicit_tenant_context.get()); + auto auth_registry = + rgw::auth::StrategyRegistry::create(dpp->get_cct(), *(implicit_tenant_context.get()), store); + + /* allocate a mime table (you'd never guess that from the name) */ + rgw_tools_init(dpp, dpp->get_cct()); + + /* Header custom behavior */ + rest.register_x_headers(g_conf()->rgw_log_http_headers); + + sched_ctx.reset(new rgw::dmclock::SchedulerCtx{dpp->get_cct()}); + ratelimiter.reset(new ActiveRateLimiter{dpp->get_cct()}); + ratelimiter->start(); + + int fe_count = 0; + for (multimap::iterator fiter = fe_map.begin(); + fiter != fe_map.end(); ++fiter, ++fe_count) { + RGWFrontendConfig *config = fiter->second; + string framework = config->get_framework(); + + auto def_iter = fe_def_map.find(framework); + if (def_iter != fe_def_map.end()) { + config->set_default_config(*def_iter->second); + } + + RGWFrontend* fe = nullptr; + + if (framework == "loadgen") { + int port; + config->get_val("port", 80, &port); + std::string uri_prefix; + config->get_val("prefix", "", &uri_prefix); + + RGWProcessEnv env = {store, &rest, olog, port, uri_prefix, + auth_registry, ratelimiter.get(), lua_background.get()}; + + fe = new RGWLoadGenFrontend(env, config); + } + else if (framework == "beast") { + int port; + config->get_val("port", 80, &port); + std::string uri_prefix; + config->get_val("prefix", "", &uri_prefix); + RGWProcessEnv env{store, &rest, olog, port, uri_prefix, + auth_registry, ratelimiter.get(), lua_background.get()}; + fe = new RGWAsioFrontend(env, config, *(sched_ctx.get())); + } + else if (framework == "rgw-nfs") { + int port = 80; + RGWProcessEnv env = { store, &rest, olog, port }; + fe = new RGWLibFrontend(env, config); + if (rgwlib) { + rgwlib->set_fe(static_cast(fe)); + } + } + + service_map_meta["frontend_type#" + stringify(fe_count)] = framework; + service_map_meta["frontend_config#" + stringify(fe_count)] = config->get_config(); + + if (! fe) { + dout(0) << "WARNING: skipping unknown framework: " << framework << dendl; + continue; + } + + dout(0) << "starting handler: " << fiter->first << dendl; + int r = fe->init(); + if (r < 0) { + derr << "ERROR: failed initializing frontend" << dendl; + return -r; + } + r = fe->run(); + if (r < 0) { + derr << "ERROR: failed run" << dendl; + return -r; + } + + fes.push_back(fe); } - void pause() override { - std::for_each(pausers.begin(), pausers.end(), [](Pauser* p){p->pause();}); + std::string daemon_type = (nfs) ? "rgw-nfs" : "nfs"; + r = store->register_to_service_map(dpp, daemon_type, service_map_meta); + if (r < 0) { + derr << "ERROR: failed to register to service map: " << cpp_strerror(-r) << dendl; + /* ignore error */ } - void resume(rgw::sal::Store* store) override { - std::for_each(pausers.begin(), pausers.end(), [store](Pauser* p){p->resume(store);}); + + if (store->get_name() == "rados") { + // add a watcher to respond to realm configuration changes + pusher = std::make_unique(dpp, store, null_yield); + fe_pauser = std::make_unique(fes, *(implicit_tenant_context.get()), pusher.get()); + rgw_pauser = std::make_unique(); + rgw_pauser->add_pauser(fe_pauser.get()); + if (lua_background) { + rgw_pauser->add_pauser(lua_background.get()); + } + reloader = std::make_unique(store, service_map_meta, rgw_pauser.get()); + realm_watcher = std::make_unique(dpp, g_ceph_context, + static_cast(store)->svc()->zone->get_realm()); + realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader); + realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher.get()); } -}; + return r; +} /* init_frontends2 */ + +void rgw::InitHelper::init_tracepoints() +{ + TracepointProvider::initialize(dpp->get_cct()); + TracepointProvider::initialize(dpp->get_cct()); + tracing::rgw::tracer.init("rgw"); +} /* init_tracepoints() */ + +void rgw::InitHelper::init_notification_endpoints() +{ +#ifdef WITH_RADOSGW_AMQP_ENDPOINT + if (!rgw::amqp::init(dpp->get_cct())) { + derr << "ERROR: failed to initialize AMQP manager" << dendl; + } +#endif +#ifdef WITH_RADOSGW_KAFKA_ENDPOINT + if (!rgw::kafka::init(dpp->get_cct())) { + derr << "ERROR: failed to initialize Kafka manager" << dendl; + } +#endif +} /* init_notification_endpoints */ + +void rgw::InitHelper::init_lua() +{ + int r{0}; + const auto &luarocks_path = + g_conf().get_val("rgw_luarocks_location"); + if (luarocks_path.empty()) { + store->set_luarocks_path(""); + } else { + store->set_luarocks_path(luarocks_path + "/" + g_conf()->name.to_str()); + } +#ifdef WITH_RADOSGW_LUA_PACKAGES + rgw::lua::packages_t failed_packages; + std::string output; + r = rgw::lua::install_packages(dpp, store, null_yield, failed_packages, + output); + if (r < 0) { + dout(1) << "WARNING: failed to install lua packages from allowlist" + << dendl; + } + if (!output.empty()) { + dout(10) << "INFO: lua packages installation output: \n" << output << dendl; + } + for (const auto &p : failed_packages) { + dout(5) << "WARNING: failed to install lua package: " << p + << " from allowlist" << dendl; + } +#endif + + if (store->get_name() == "rados") { /* Supported for only RadosStore */ + lua_background = std::make_unique< + rgw::lua::Background>(store, dpp->get_cct(), store->get_luarocks_path()); + lua_background->start(); + } +} /* init_lua */ /* * start up the RADOS connection and then handle HTTP messages as they come in */ int main(int argc, const char **argv) { + + multimap fe_map; + vector configs; + // dout() messages will be sent to stderr, but FCGX wants messages on stdout // Redirect stderr to stdout. TEMP_FAILURE_RETRY(close(STDERR_FILENO)); @@ -243,16 +637,14 @@ int main(int argc, const char **argv) CODE_ENVIRONMENT_DAEMON, flags); // First, let's determine which frontends are configured. - list frontends; + vector frontends; string rgw_frontends_str = g_conf().get_val("rgw_frontends"); g_conf().early_expand_meta(rgw_frontends_str, &cerr); - get_str_list(rgw_frontends_str, ",", frontends); - multimap fe_map; - list configs; + get_str_vec(rgw_frontends_str, ",", frontends); if (frontends.empty()) { frontends.push_back("beast"); } - for (list::iterator iter = frontends.begin(); iter != frontends.end(); ++iter) { + for (vector::iterator iter = frontends.begin(); iter != frontends.end(); ++iter) { string& f = *iter; if (f.find("beast") != string::npos) { @@ -279,7 +671,7 @@ int main(int argc, const char **argv) string framework = config->get_framework(); fe_map.insert(pair(framework, config)); - } + } /* for each frontend */ int numa_node = g_conf().get_val("rgw_numa_node"); size_t numa_cpu_set_size = 0; @@ -389,12 +781,11 @@ int main(int argc, const char **argv) RGWREST rest; - list apis; - - get_str_list(g_conf()->rgw_enable_apis, apis); + vector apis; + get_str_vec(g_conf()->rgw_enable_apis, apis); map apis_map; - for (list::iterator li = apis.begin(); li != apis.end(); ++li) { + for (vector::iterator li = apis.begin(); li != apis.end(); ++li) { apis_map[*li] = true; } @@ -552,12 +943,12 @@ int main(int argc, const char **argv) map service_map_meta; service_map_meta["pid"] = stringify(getpid()); - list fes; + vector fes; string frontend_defs_str = g_conf().get_val("rgw_frontend_defaults"); - list frontends_def; - get_str_list(frontend_defs_str, ",", frontends_def); + vector frontends_def; + get_str_vec(frontend_defs_str, ",", frontends_def); map > fe_def_map; for (auto& f : frontends_def) { @@ -680,20 +1071,20 @@ int main(int argc, const char **argv) reloader.reset(); // stop the realm reloader } - for (list::iterator liter = fes.begin(); liter != fes.end(); + for (vector::iterator liter = fes.begin(); liter != fes.end(); ++liter) { RGWFrontend *fe = *liter; fe->stop(); } - for (list::iterator liter = fes.begin(); liter != fes.end(); + for (vector::iterator liter = fes.begin(); liter != fes.end(); ++liter) { RGWFrontend *fe = *liter; fe->join(); delete fe; } - for (list::iterator liter = configs.begin(); + for (vector::iterator liter = configs.begin(); liter != configs.end(); ++liter) { RGWFrontendConfig *fec = *liter; delete fec; diff --git a/src/rgw/rgw_main.h b/src/rgw/rgw_main.h new file mode 100644 index 0000000000000..305bf71adf11d --- /dev/null +++ b/src/rgw/rgw_main.h @@ -0,0 +1,141 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2022 New Dream Network + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include +#include +#include +#include "rgw_common.h" +#include "rgw_rest.h" +#include "rgw_frontend.h" +#include "rgw_period_pusher.h" +#include "rgw_realm_reloader.h" +#include "rgw_ldap.h" +#include "rgw_lua.h" +#include "rgw_lua_background.h" +#include "rgw_dmclock_scheduler_ctx.h" +#include "rgw_ratelimit.h" + + +class RGWPauser : public RGWRealmReloader::Pauser { + std::vector pausers; + +public: + ~RGWPauser() override = default; + + void add_pauser(Pauser* pauser) { + pausers.push_back(pauser); + } + + void pause() override { + std::for_each(pausers.begin(), pausers.end(), [](Pauser* p){p->pause();}); + } + void resume(rgw::sal::Store* store) override { + std::for_each(pausers.begin(), pausers.end(), [store](Pauser* p){p->resume(store);}); + } + +}; + +namespace rgw { + + class RGWLib; + + class InitHelper { + /* several components should be initalized only if librgw is + * also serving HTTP */ + bool have_http_frontend{false}; + bool nfs{false}; + + std::vector& fes; + std::vector& fe_configs; + std::multimap& fe_map; + std::unique_ptr& ldh; + OpsLogSink*& olog; + RGWREST& rest; + std::unique_ptr& lua_background; + std::unique_ptr& implicit_tenant_context; + std::unique_ptr& sched_ctx; + std::unique_ptr& ratelimiter; + // wow, realm reloader has a lot of parts + std::unique_ptr& reloader; + std::unique_ptr& pusher; + std::unique_ptr& fe_pauser; + std::unique_ptr& realm_watcher; + std::unique_ptr& rgw_pauser; + rgw::sal::Store*& store; + DoutPrefixProvider* dpp; + + public: + InitHelper(std::vector& fes, + std::vector& fe_configs, + std::multimap& fe_map, + std::unique_ptr& ldh, + OpsLogSink*& olog, + RGWREST& rest, + std::unique_ptr& lua_background, + std::unique_ptr& implicit_tenant_context, + std::unique_ptr& sched_ctx, + std::unique_ptr& ratelimiter, + std::unique_ptr& reloader, + std::unique_ptr& pusher, + std::unique_ptr& fe_pauser, + std::unique_ptr& realm_watcher, + std::unique_ptr& rgw_pauser, + rgw::sal::Store*& store, + DoutPrefixProvider* dpp) + : fes(fes), fe_configs(fe_configs), fe_map(fe_map), ldh(ldh), olog(olog), + rest(rest), lua_background(lua_background), + implicit_tenant_context(implicit_tenant_context), sched_ctx(sched_ctx), + ratelimiter(ratelimiter), reloader(reloader), pusher(pusher), + fe_pauser(fe_pauser), realm_watcher(realm_watcher), rgw_pauser(rgw_pauser), + store(store), dpp(dpp) + {} + + void init_frontends1(bool nfs = false); + void init_storage(); + void init_perfcounters(); + void init_http_clients(); + void cond_init_apis(); + void init_ldap(); + void init_opslog(); + int init_frontends2(RGWLib* rgwlib = nullptr); + void init_tracepoints(); + void init_notification_endpoints(); + void init_lua(); + + bool have_http() { + return have_http_frontend; + } + }; /* InitHelper */ + +} // namespace rgw + +static inline RGWRESTMgr *set_logging(RGWRESTMgr* mgr) +{ + mgr->set_logging(true); + return mgr; +} + +static inline RGWRESTMgr *rest_filter(rgw::sal::Store* store, int dialect, RGWRESTMgr* orig) +{ + RGWSyncModuleInstanceRef sync_module = store->get_sync_module(); + if (sync_module) { + return sync_module->get_rest_filter(dialect, orig); + } else { + return orig; + } +} + diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index 926756402eab7..0b7defa2f113f 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -13,6 +13,7 @@ #include "rgw_op.h" #include "rgw_formats.h" #include "rgw_client_io.h" +#include "rgw_lua_background.h" extern std::map rgw_to_http_attrs; -- 2.39.5