From: Yehuda Sadeh Date: Thu, 4 Oct 2018 00:18:57 +0000 (-0700) Subject: rgw: svc: due to popular demand, get rid of service registry X-Git-Tag: v14.1.0~965^2~23 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4c0675bbc3f554acd052c11329287efc16b02357;p=ceph-ci.git rgw: svc: due to popular demand, get rid of service registry Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index ffc2f41a855..675dd41a112 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1644,27 +1644,6 @@ int RGWRados::init_complete() return ret; } -/* - * FIXME: in the future formattable will derive from formatter, so formattable - * could be constructed directly - */ -static bool to_formattable(CephContext *cct, JSONFormatter& f, JSONFormattable *result) -{ - stringstream ss; - f.flush(ss); - string s = ss.str(); - - JSONParser jp; - if (!jp.parse(s.c_str(), s.size())) { - ldout(cct, 0) << "failed to parse formatter string: data=" << s << dendl; - return false; - } - - result->decode_json(&jp); - - return true; -} - /** * Initialize the RADOS instance and prepare to do other ops * Returns 0 on success, -ERR# on failure. @@ -1677,57 +1656,11 @@ int RGWRados::initialize() cct->_conf.get_val("rgw_inject_notify_timeout_probability"); max_notify_retries = cct->_conf.get_val("rgw_max_notify_retries"); - svc_registry = std::make_shared(cct); - - JSONFormattable zone_svc_conf; - ret = svc_registry->get_instance("zone", zone_svc_conf, &_svc.zone); - if (ret < 0) { - return ret; - } - svc.zone = _svc.zone.get(); - - JSONFormattable zone_utils_svc_conf; - ret = svc_registry->get_instance("zone_utils", zone_utils_svc_conf, &_svc.zone_utils); - if (ret < 0) { - return ret; - } - svc.zone_utils = _svc.zone_utils.get(); - - JSONFormattable quota_svc_conf; - ret = svc_registry->get_instance("quota", quota_svc_conf, &_svc.quota); - if (ret < 0) { - return ret; - } - svc.quota = _svc.quota.get(); - - JSONFormattable sync_modules_svc_conf; - ret = svc_registry->get_instance("sync_modules", sync_modules_svc_conf, &_svc.sync_modules); - if (ret < 0) { - return ret; - } - svc.sync_modules = _svc.sync_modules.get(); - - if (use_cache) { - JSONFormattable cache_svc_conf; - ret = svc_registry->get_instance("sysobj_cache", cache_svc_conf, &_svc.cache); - if (ret < 0) { - return ret; - } - svc.cache = _svc.cache.get(); - } - - JSONFormattable sysobj_svc_conf; - - JSONFormatter f; - encode_json("cache", use_cache, &f); - if (!to_formattable(cct, f, &sysobj_svc_conf)) { - assert(0); - } - ret = svc_registry->get_instance("sysobj", sysobj_svc_conf, &_svc.sysobj); + ret = svc.init(cct, use_cache); if (ret < 0) { + ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl; return ret; } - svc.sysobj = _svc.sysobj.get(); host_id = svc.zone_utils->gen_host_id(); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 4230031c377..14013412bb8 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -26,6 +26,7 @@ #include "rgw_period_puller.h" #include "rgw_sync_module.h" #include "rgw_sync_log_trim.h" +#include "rgw_service.h" #include "services/svc_rados.h" #include "services/svc_zone.h" @@ -47,13 +48,6 @@ struct RGWZoneParams; class RGWReshard; class RGWReshardWait; -class RGWSI_Zone; -class RGWSI_ZoneUtils; -class RGWSI_Quota; -class RGWSI_SyncModules; -class RGWSI_SysObj; -class RGWSI_SysObj_Cache; - class RGWSysObjectCtx; /* flags for put_obj_meta() */ @@ -1296,7 +1290,6 @@ protected: RGWSyncModuleInstanceRef sync_module; bool writeable_zone{false}; - RGWServiceRegistryRef svc_registry; RGWIndexCompletionManager *index_completion_manager{nullptr}; bool use_cache{false}; @@ -1358,25 +1351,7 @@ public: cct = _cct; } - struct { - std::shared_ptr rados; - std::shared_ptr zone; - std::shared_ptr zone_utils; - std::shared_ptr quota; - std::shared_ptr sync_modules; - std::shared_ptr sysobj; - std::shared_ptr cache; - } _svc; - - struct { - RGWSI_RADOS *rados{nullptr}; - RGWSI_Zone *zone{nullptr}; - RGWSI_ZoneUtils *zone_utils{nullptr}; - RGWSI_Quota *quota{nullptr}; - RGWSI_SyncModules *sync_modules{nullptr}; - RGWSI_SysObj *sysobj{nullptr}; - RGWSI_SysObj_Cache *cache{nullptr}; - } svc; + RGWServices svc; /** * AmazonS3 errors contain a HostId string, but is an opaque base64 blob; we diff --git a/src/rgw/rgw_service.cc b/src/rgw/rgw_service.cc index 2e3690b2a95..f36554c076f 100644 --- a/src/rgw/rgw_service.cc +++ b/src/rgw/rgw_service.cc @@ -11,142 +11,143 @@ #include "services/svc_sys_obj_cache.h" #include "services/svc_sys_obj_core.h" +#include "common/errno.h" + #define dout_subsys ceph_subsys_rgw -RGWServiceInstance::~RGWServiceInstance() + +int RGWServices_Shared::init(CephContext *cct, + bool have_cache) { - if (svc) { - shutdown(); - svc->svc_registry->remove_instance(this); + finisher = std::make_shared(cct); + notify = std::make_shared(cct); + rados = std::make_shared(cct); + zone = std::make_shared(cct); + zone_utils = std::make_shared(cct); + quota = std::make_shared(cct); + sync_modules = std::make_shared(cct); + sysobj = std::make_shared(cct); + sysobj_core = std::make_shared(cct); + + if (have_cache) { + sysobj_cache = std::make_shared(cct); } -} -void RGWServiceRegistry::register_all(CephContext *cct) -{ - services["finisher"] = make_shared(cct); - services["notify"] = make_shared(cct); - services["rados"] = make_shared(cct); - services["zone"] = make_shared(cct); - services["zone_utils"] = make_shared(cct); - services["quota"] = make_shared(cct); - services["sync_modules"] = make_shared(cct); - services["sysobj"] = make_shared(cct); - services["sysobj_cache"] = make_shared(cct); - services["sysobj_core"] = make_shared(cct); -} + finisher->init(); + notify->init(zone, rados, finisher); + rados->init(); + zone->init(sysobj, rados, sync_modules); + zone_utils->init(rados, zone); + quota->init(zone); + sync_modules->init(); + sysobj_core->core_init(rados, zone); + if (have_cache) { + sysobj_cache->init(rados, zone, notify); + auto _cache = std::static_pointer_cast(sysobj_cache); + sysobj->init(rados, _cache); + } else { + sysobj->init(rados, sysobj_core); + } -bool RGWServiceRegistry::find(const string& name, RGWServiceRef *svc) -{ - auto iter = services.find(name); - if (iter == services.end()) { - return false; + + int r = finisher->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start finisher service (" << cpp_strerror(-r) << dendl; + return r; } - *svc = iter->second; - return true; -} + r = notify->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start notify service (" << cpp_strerror(-r) << dendl; + return r; + } -string RGWServiceRegistry::get_conf_id(const string& service_type, const string& conf) -{ - return service_type + ":" + conf; -} + r = rados->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start rados service (" << cpp_strerror(-r) << dendl; + return r; + } -int RGWServiceRegistry::do_get_instance(RGWServiceRef& svc, - const string& conf, - RGWServiceInstanceRef *ref, - vector *new_instances) -{ - RGWServiceInstanceRef instance_ref; + r = zone->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start zone service (" << cpp_strerror(-r) << dendl; + return r; + } - string conf_id = get_conf_id(svc->type(), conf); + r = zone_utils->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start zone_utils service (" << cpp_strerror(-r) << dendl; + return r; + } - auto iter = instances_by_conf.find(conf_id); - if (iter != instances_by_conf.end()) { - *ref = iter->second.ref; - return 0; + r = quota->start(); + if (r < 0) { + ldout(cct, 0) << "ERROR: failed to start quota service (" << cpp_strerror(-r) << dendl; + return r; } - int r = svc->create_instance(conf, &instance_ref); + + r = sysobj_core->start(); if (r < 0) { - ldout(cct, 0) << "ERROR: failed to create instance for service " << svc->type() << " conf=" << conf << " (r=" << r << ")" << dendl; + ldout(cct, 0) << "ERROR: failed to start sysobj_core service (" << cpp_strerror(-r) << dendl; return r; } - svc->svc_registry = shared_from_this(); - instance_ref->svc = svc; - instance_ref->svc_id = ++max_registry_id; - - map dep_refs; - - instance_info& iinfo = instances[instance_ref->svc_id]; - iinfo.conf_id = get_conf_id(svc->type(), conf); - iinfo.id = instance_ref->svc_id; - iinfo.title = instance_ref->get_title(); - iinfo.conf = conf; - iinfo.ref = instance_ref; - - instances_by_conf[iinfo.conf_id] = iinfo; - - auto deps = instance_ref->get_deps(); - for (auto iter : deps) { - auto& dep_id = iter.first; - auto& dep = iter.second; - RGWServiceInstanceRef dep_ref; - r = do_get_instance(dep.name, dep.conf, &dep_ref, new_instances); + + if (have_cache) { + r = sysobj_cache->start(); if (r < 0) { - ldout(cct, 0) << "ERROR: cannot satisfy dependency for service " << svc->type() << ": " << dep.name << dendl; + ldout(cct, 0) << "ERROR: failed to start sysobj_cache service (" << cpp_strerror(-r) << dendl; return r; } - dep_refs[dep_id] = dep_ref; } - ldout(cct, 10) << "svc: load service: " << instance_ref->get_svc()->type() << dendl; - r = instance_ref->load(conf, dep_refs); - ldout(cct, 10) << "svc: done load service: " << instance_ref->get_svc()->type() << dendl; + r = sysobj->start(); if (r < 0) { - ldout(cct, 0) << "ERROR: service instance load return error: service=" << svc->type() << " r=" << r << dendl; + ldout(cct, 0) << "ERROR: failed to start sysobj service (" << cpp_strerror(-r) << dendl; return r; } - new_instances->push_back(instance_ref); + /* cache or core services will be started by sysobj */ - if (instance_ref->svc_instance.empty()) { - char buf[32]; - snprintf(buf, sizeof(buf), "%lld", (long long)instance_ref->svc_id); - instance_ref->svc_instance = buf; + return 0; +} + + +int RGWServices::init(CephContext *cct, bool have_cache) +{ + int r = _svc.init(cct, have_cache); + if (r < 0) { + return r; } - *ref = iinfo.ref; + finisher = _svc.finisher.get(); + notify = _svc.notify.get(); + rados = _svc.rados.get(); + zone = _svc.zone.get(); + zone_utils = _svc.zone_utils.get(); + quota = _svc.quota.get(); + sync_modules = _svc.sync_modules.get(); + sysobj = _svc.sysobj.get(); + cache = _svc.sysobj_cache.get(); + core = _svc.sysobj_core.get(); return 0; } -int RGWServiceRegistry::get_instance(RGWServiceRef& svc, - const string& conf, - RGWServiceInstanceRef *ref) +int RGWServiceInstance::start() { - vector new_instances; + if (start_state != StateInit) { + return 0; + } + + start_state = StateStarting;; /* setting started prior to do_start() on purpose so that circular + references can call start() on each other */ - int r = do_get_instance(svc, conf, ref, &new_instances); + int r = do_start(); if (r < 0) { - ldout(cct, 0) << "ERROR: service instance load return error: service=" << svc->type() << " r=" << r << dendl; + return r; } - for (auto& instance_ref : new_instances) { - ldout(cct, 10) << "svc: init service: " << instance_ref->get_svc()->type() << dendl; - r = instance_ref->init(); - ldout(cct, 10) << "svc: done init service: " << instance_ref->get_svc()->type() << dendl; - if (r < 0) { - ldout(cct, 0) << "ERROR: service instance init return error: service=" << instance_ref->get_svc()->type() << " r=" << r << dendl; - return r; - } - } - return 0; -} + start_state = StateStarted; -void RGWServiceRegistry::remove_instance(RGWServiceInstance *instance) { - auto iter = instances.find(instance->svc_id); - if (iter == instances.end()) { - return; - } - instances_by_conf.erase(iter->second.conf_id); - instances.erase(iter); + return 0; } diff --git a/src/rgw/rgw_service.h b/src/rgw/rgw_service.h index e1baa616c29..ebb33f39fcc 100644 --- a/src/rgw/rgw_service.h +++ b/src/rgw/rgw_service.h @@ -8,150 +8,83 @@ #include "rgw/rgw_common.h" - -class CephContext; -class RGWServiceInstance; -class RGWServiceRegistry; - -using RGWServiceInstanceRef = std::shared_ptr; -using RGWServiceRegistryRef = std::shared_ptr; - -class RGWService -{ - friend class RGWServiceRegistry; - friend class RGWServiceInstance; - -protected: - RGWServiceRegistryRef svc_registry; - CephContext *cct; - std::string svc_type; - -public: - RGWService(CephContext *_cct, const std::string& _svc_type) : cct(_cct), - svc_type(_svc_type) {} - virtual ~RGWService() = default; - - const std::string& type() { - return svc_type; - } - virtual int create_instance(const string& conf, RGWServiceInstanceRef *instance) = 0; -}; - - -using RGWServiceRef = std::shared_ptr; - +struct RGWServices_Shared; class RGWServiceInstance { - friend class RGWServiceRegistry; + friend struct RGWServices_Shared; + protected: CephContext *cct; - std::shared_ptr svc; - string svc_instance; - uint64_t svc_id{0}; - struct dependency { - string name; - string conf; - }; + enum StartState { + StateInit = 0, + StateStarting = 1, + StateStarted = 2, + } start_state{StateInit}; - virtual std::map get_deps() { - return std::map(); - } - virtual int load(const string& conf, std::map& dep_refs) = 0; - virtual int init() { + virtual void shutdown() {} + virtual int do_start() { return 0; } - virtual void shutdown() {} public: - RGWServiceInstance(RGWService *svc, CephContext *_cct) : cct(_cct) {} + RGWServiceInstance(CephContext *_cct) : cct(_cct) {} + virtual ~RGWServiceInstance() {} - virtual ~RGWServiceInstance(); + int start(); + bool is_started() { + return (start_state == StateStarted); + } CephContext *ctx() { return cct; } - - string get_title() { - return svc->type() + ":" + svc_instance; - } - - std::shared_ptr& get_svc() { - return svc; - } }; -class RGWServiceRegistry : public std::enable_shared_from_this { - CephContext *cct; +class RGWSI_Finisher; +class RGWSI_Notify; +class RGWSI_RADOS; +class RGWSI_Zone; +class RGWSI_ZoneUtils; +class RGWSI_Quota; +class RGWSI_SyncModules; +class RGWSI_SysObj; +class RGWSI_SysObj_Core; +class RGWSI_SysObj_Cache; + +struct RGWServices_Shared +{ + std::shared_ptr finisher; + std::shared_ptr notify; + std::shared_ptr rados; + std::shared_ptr zone; + std::shared_ptr zone_utils; + std::shared_ptr quota; + std::shared_ptr sync_modules; + std::shared_ptr sysobj; + std::shared_ptr sysobj_core; + std::shared_ptr sysobj_cache; + + int init(CephContext *cct, bool have_cache); +}; - map services; - - struct instance_info { - string conf_id; - uint64_t id; - string title; - string conf; - RGWServiceInstanceRef ref; - }; - map instances; /* registry_id -> instance */ - map instances_by_conf; /* conf_id -> instance */ - - std::atomic max_registry_id; - - string get_conf_id(const string& service_type, const string& conf); - void register_all(CephContext *cct); - - int do_get_instance(RGWServiceRef& svc, - const string& conf, - RGWServiceInstanceRef *ref, - std::vector *new_instances); - template - int do_get_instance(const string& svc_name, - const string& conf, - T *ref, - std::vector *new_instances) { - auto iter = services.find(svc_name); - if (iter == services.end()) { - return -ENOENT; - } - return do_get_instance(iter->second, conf, ref, new_instances); - } -public: - RGWServiceRegistry(CephContext *_cct) : cct(_cct) { - register_all(cct); - } - bool find(const string& name, RGWServiceRef *svc); - - /* returns existing or creates a new one */ - int get_instance(RGWServiceRef& svc, - const string& conf, - RGWServiceInstanceRef *ref); /* returns existing or creates a new one */ - - /* returns existing or creates a new one */ - template - int get_instance(RGWServiceRef& svc, - const string& conf, - T *ref) { - RGWServiceInstanceRef r; - int ret = get_instance(svc, conf, &r); - if (ret < 0) { - return ret; - } - *ref = std::static_pointer_cast(r); - return 0; - } - template - int get_instance(const string& svc_name, - const string& conf, - T *ref) { - auto iter = services.find(svc_name); - if (iter == services.end()) { - return -ENOENT; - } - return get_instance(iter->second, conf, ref); - } - void remove_instance(RGWServiceInstance *instance); +struct RGWServices +{ + RGWServices_Shared _svc; + + RGWSI_Finisher *finisher{nullptr}; + RGWSI_Notify *notify{nullptr}; + RGWSI_RADOS *rados{nullptr}; + RGWSI_Zone *zone{nullptr}; + RGWSI_ZoneUtils *zone_utils{nullptr}; + RGWSI_Quota *quota{nullptr}; + RGWSI_SyncModules *sync_modules{nullptr}; + RGWSI_SysObj *sysobj{nullptr}; + RGWSI_SysObj_Cache *cache{nullptr}; + RGWSI_SysObj_Core *core{nullptr}; + + int init(CephContext *cct, bool have_cache); }; diff --git a/src/rgw/services/svc_finisher.cc b/src/rgw/services/svc_finisher.cc index dd1150deebe..d239ff3c4fa 100644 --- a/src/rgw/services/svc_finisher.cc +++ b/src/rgw/services/svc_finisher.cc @@ -2,19 +2,7 @@ #include "svc_finisher.h" -int RGWS_Finisher::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_Finisher(this, cct)); - return 0; -} - -std::map RGWSI_Finisher::get_deps() -{ - std::map dep; - return dep; -} - -int RGWSI_Finisher::init() +int RGWSI_Finisher::do_start() { finisher = new Finisher(cct); finisher->start(); diff --git a/src/rgw/services/svc_finisher.h b/src/rgw/services/svc_finisher.h index fbe4a7c52cb..7660ed14cdc 100644 --- a/src/rgw/services/svc_finisher.h +++ b/src/rgw/services/svc_finisher.h @@ -7,16 +7,9 @@ class Context; class Finisher; -class RGWS_Finisher : public RGWService -{ -public: - RGWS_Finisher(CephContext *cct) : RGWService(cct, "finisher") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_Finisher : public RGWServiceInstance { + friend struct RGWServices_Shared; public: class ShutdownCB; @@ -24,18 +17,17 @@ private: Finisher *finisher{nullptr}; bool finalized{false}; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override { - return 0; - } - int init() override; void shutdown() override; std::map shutdown_cbs; std::atomic handles_counter; +protected: + void init() {} + int do_start() override; + public: - RGWSI_Finisher(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_Finisher(CephContext *cct): RGWServiceInstance(cct) {} ~RGWSI_Finisher(); class ShutdownCB { diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc index 47f9bfbdb79..938b417c85f 100644 --- a/src/rgw/services/svc_notify.cc +++ b/src/rgw/services/svc_notify.cc @@ -12,12 +12,6 @@ static string notify_oid_prefix = "notify"; -int RGWS_Notify::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_Notify(this, cct)); - return 0; -} - class RGWWatcher : public librados::WatchCtx2 { CephContext *cct; RGWSI_Notify *svc; @@ -145,29 +139,6 @@ public: } }; -std::map RGWSI_Notify::get_deps() -{ - map deps; - deps["zone_dep"] = { .name = "zone", - .conf = "{}" }; - deps["rados_dep"] = { .name = "rados", - .conf = "{}" }; - deps["finisher_dep"] = { .name = "finisher", - .conf = "{}" }; - return deps; -} - -int RGWSI_Notify::load(const string& conf, std::map& dep_refs) -{ - zone_svc = static_pointer_cast(dep_refs["zone_dep"]); - assert(zone_svc); - rados_svc = static_pointer_cast(dep_refs["rados_dep"]); - assert(rados_svc); - finisher_svc = static_pointer_cast(dep_refs["finisher_dep"]); - assert(finisher_svc); - return 0; -} - string RGWSI_Notify::get_control_oid(int i) { char buf[notify_oid_prefix.size() + 16]; @@ -262,8 +233,24 @@ void RGWSI_Notify::finalize_watch() delete[] watchers; } -int RGWSI_Notify::init() +int RGWSI_Notify::do_start() { + int r = zone_svc->start(); + if (r < 0) { + return r; + } + + assert(zone_svc->is_started()); /* otherwise there's an ordering problem */ + + r = rados_svc->start(); + if (r < 0) { + return r; + } + r = finisher_svc->start(); + if (r < 0) { + return r; + } + control_pool = zone_svc->get_zone_params().control_pool; int ret = init_watch(); diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h index 75f716fe779..2601b8b14f8 100644 --- a/src/rgw/services/svc_notify.h +++ b/src/rgw/services/svc_notify.h @@ -11,21 +11,13 @@ class RGWSI_Zone; class RGWSI_Finisher; class RGWWatcher; - -class RGWS_Notify : public RGWService -{ -public: - RGWS_Notify(CephContext *cct) : RGWService(cct, "notify") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_Notify_ShutdownCB; class RGWSI_Notify : public RGWServiceInstance { friend class RGWWatcher; friend class RGWSI_Notify_ShutdownCB; + friend class RGWServices_Shared; public: class CB; @@ -35,9 +27,6 @@ private: std::shared_ptr rados_svc; std::shared_ptr finisher_svc; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - RWLock watchers_lock{"watchers_lock"}; rgw_pool control_pool; @@ -62,7 +51,14 @@ private: int init_watch(); void finalize_watch(); - int init() override; + void init(std::shared_ptr& _zone_svc, + std::shared_ptr& _rados_svc, + std::shared_ptr& _finisher_svc) { + zone_svc = _zone_svc; + rados_svc = _rados_svc; + finisher_svc = _finisher_svc; + } + int do_start() override; void shutdown() override; int unwatch(RGWSI_RADOS::Obj& obj, uint64_t watch_handle); @@ -80,7 +76,7 @@ private: void schedule_context(Context *c); public: - RGWSI_Notify(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_Notify(CephContext *cct): RGWServiceInstance(cct) {} ~RGWSI_Notify(); class CB { diff --git a/src/rgw/services/svc_quota.cc b/src/rgw/services/svc_quota.cc index 267c265babf..f2baac36b45 100644 --- a/src/rgw/services/svc_quota.cc +++ b/src/rgw/services/svc_quota.cc @@ -3,28 +3,6 @@ #include "rgw/rgw_zone.h" -int RGWS_Quota::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_Quota(this, cct)); - return 0; -} - -std::map RGWSI_Quota::get_deps() -{ - RGWServiceInstance::dependency dep = { .name = "zone", - .conf = "{}" }; - map deps; - deps["zone_dep"] = dep; - return deps; -} - -int RGWSI_Quota::load(const string& conf, std::map& dep_refs) -{ - zone_svc = static_pointer_cast(dep_refs["zone_dep"]); - assert(zone_svc); - return 0; -} - const RGWQuotaInfo& RGWSI_Quota::get_bucket_quota() const { return zone_svc->get_current_period().get_config().bucket_quota; diff --git a/src/rgw/services/svc_quota.h b/src/rgw/services/svc_quota.h index a1e56d2c7a7..9e063d7df90 100644 --- a/src/rgw/services/svc_quota.h +++ b/src/rgw/services/svc_quota.h @@ -5,25 +5,16 @@ #include "rgw/rgw_service.h" -class RGWSI_Zone; - -class RGWS_Quota : public RGWService -{ -public: - RGWS_Quota(CephContext *cct) : RGWService(cct, "quota") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_Quota : public RGWServiceInstance { std::shared_ptr zone_svc; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - public: - RGWSI_Quota(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_Quota(CephContext *cct): RGWServiceInstance(cct) {} + + void init(std::shared_ptr& _zone_svc) { + zone_svc = _zone_svc; + } const RGWQuotaInfo& get_bucket_quota() const; const RGWQuotaInfo& get_user_quota() const; diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc index 54bb81cf36f..5c61dcdbc0d 100644 --- a/src/rgw/services/svc_rados.cc +++ b/src/rgw/services/svc_rados.cc @@ -6,12 +6,6 @@ #define dout_subsys ceph_subsys_rgw -int RGWS_RADOS::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_RADOS(this, cct)); - return 0; -} - static int init_ioctx(CephContext *cct, librados::Rados *rados, const rgw_pool& pool, librados::IoCtx& ioctx, bool create) { int r = rados->ioctx_create(pool.name.c_str(), ioctx); @@ -47,7 +41,7 @@ static int init_ioctx(CephContext *cct, librados::Rados *rados, const rgw_pool& return 0; } -int RGWSI_RADOS::load(const string& conf, map& deps) +int RGWSI_RADOS::do_start() { auto handles = std::vector{static_cast(cct->_conf->rgw_num_rados_handles)}; diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h index 73eaa284172..ee94e81ff46 100644 --- a/src/rgw/services/svc_rados.h +++ b/src/rgw/services/svc_rados.h @@ -21,15 +21,6 @@ struct RGWAccessListFilterPrefix : public RGWAccessListFilter { } }; -class RGWS_RADOS : public RGWService -{ - std::vector get_deps(); -public: - RGWS_RADOS(CephContext *cct) : RGWService(cct, "rados") {} - - int create_instance(const string& conf, RGWServiceInstanceRef *instance) override; -}; - struct rgw_rados_ref { rgw_pool pool; string oid; @@ -88,7 +79,7 @@ class RGWSI_RADOS : public RGWServiceInstance RWLock handle_lock; std::map rados_map; - int load(const string& conf, std::map& deps) override; + int do_start() override; librados::Rados* get_rados_handle(int rados_handle); int open_pool_ctx(const rgw_pool& pool, librados::IoCtx& io_ctx, int rados_handle); @@ -99,8 +90,10 @@ class RGWSI_RADOS : public RGWServiceInstance bool *is_truncated); public: - RGWSI_RADOS(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct), - handle_lock("rados_handle_lock") {} + RGWSI_RADOS(CephContext *cct): RGWServiceInstance(cct), + handle_lock("rados_handle_lock") {} + + void init() {} uint64_t instance_id(); diff --git a/src/rgw/services/svc_sync_modules.cc b/src/rgw/services/svc_sync_modules.cc index e6239628e78..ca6a7a30350 100644 --- a/src/rgw/services/svc_sync_modules.cc +++ b/src/rgw/services/svc_sync_modules.cc @@ -2,27 +2,10 @@ #include "rgw/rgw_sync_module.h" -int RGWS_SyncModules::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_SyncModules(this, cct)); - return 0; -} - -std::map RGWSI_SyncModules::get_deps() -{ - return std::map(); -} - -int RGWSI_SyncModules::load(const string& conf, std::map& dep_refs) -{ - return 0; -} - -int RGWSI_SyncModules::init() +void RGWSI_SyncModules::init() { sync_modules_manager = new RGWSyncModulesManager(); rgw_register_sync_modules(sync_modules_manager); - return 0; } RGWSI_SyncModules::~RGWSI_SyncModules() diff --git a/src/rgw/services/svc_sync_modules.h b/src/rgw/services/svc_sync_modules.h index 6e578c8d7cf..19c4ec57f7a 100644 --- a/src/rgw/services/svc_sync_modules.h +++ b/src/rgw/services/svc_sync_modules.h @@ -7,29 +7,19 @@ class RGWSyncModulesManager; -class RGWS_SyncModules : public RGWService -{ -public: - RGWS_SyncModules(CephContext *cct) : RGWService(cct, "sync_modules") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_SyncModules : public RGWServiceInstance { - RGWSyncModulesManager *sync_modules_manager; - - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - int init() override; + RGWSyncModulesManager *sync_modules_manager{nullptr}; public: - RGWSI_SyncModules(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_SyncModules(CephContext *cct): RGWServiceInstance(cct) {} ~RGWSI_SyncModules(); RGWSyncModulesManager *get_manager() { return sync_modules_manager; } + + void init(); }; #endif diff --git a/src/rgw/services/svc_sys_obj.cc b/src/rgw/services/svc_sys_obj.cc index b00a5d685a2..8008580377f 100644 --- a/src/rgw/services/svc_sys_obj.cc +++ b/src/rgw/services/svc_sys_obj.cc @@ -15,35 +15,6 @@ RGWSI_SysObj::Obj RGWSI_SysObj::get_obj(RGWSysObjectCtx& obj_ctx, const rgw_raw_ return Obj(core_svc.get(), obj_ctx, obj); } -int RGWS_SysObj::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_SysObj(this, cct)); - return 0; -} - -std::map RGWSI_SysObj::get_deps() -{ - RGWServiceInstance::dependency dep1 = { .name = "rados", - .conf = "{}" }; - RGWServiceInstance::dependency dep2 = { .name = "sysobj_core", - .conf = "{}" }; - map deps; - deps["rados_dep"] = dep1; - deps["sysobj_core_dep"] = dep2; - return deps; -} - -int RGWSI_SysObj::load(const string& conf, std::map& dep_refs) -{ - rados_svc = static_pointer_cast(dep_refs["rados_dep"]); - assert(rados_svc); - - core_svc = static_pointer_cast(dep_refs["sysobj_core_dep"]); - assert(core_svc); - - return 0; -} - void RGWSI_SysObj::Obj::invalidate() { ctx.invalidate(obj); diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h index 0d792ea7839..8c3701f52e5 100644 --- a/src/rgw/services/svc_sys_obj.h +++ b/src/rgw/services/svc_sys_obj.h @@ -14,16 +14,10 @@ class RGWSysObjectCtx; struct rgw_cache_entry_info; -class RGWS_SysObj : public RGWService -{ -public: - RGWS_SysObj(CephContext *cct) : RGWService(cct, "sys_obj") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_SysObj : public RGWServiceInstance { + friend struct RGWServices_Shared; + public: class Obj { friend class ROp; @@ -239,11 +233,14 @@ protected: std::shared_ptr rados_svc; std::shared_ptr core_svc; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; + void init(std::shared_ptr& _rados_svc, + std::shared_ptr& _core_svc) { + rados_svc = _rados_svc; + core_svc = _core_svc; + } public: - RGWSI_SysObj(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_SysObj(CephContext *cct): RGWServiceInstance(cct) {} RGWSysObjectCtx init_obj_ctx(); Obj get_obj(RGWSysObjectCtx& obj_ctx, const rgw_raw_obj& obj); diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc index ee044ed2beb..472f2f7366a 100644 --- a/src/rgw/services/svc_sys_obj_cache.cc +++ b/src/rgw/services/svc_sys_obj_cache.cc @@ -4,12 +4,6 @@ #define dout_subsys ceph_subsys_rgw -int RGWS_SysObj_Cache::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_SysObj_Cache(this, cct)); - return 0; -} - class RGWSI_SysObj_Cache_CB : public RGWSI_Notify::CB { RGWSI_SysObj_Cache *svc; @@ -27,24 +21,19 @@ public: } }; -std::map RGWSI_SysObj_Cache::get_deps() +int RGWSI_SysObj_Cache::do_start() { - map deps = RGWSI_SysObj_Core::get_deps(); - - deps["cache_notify_dep"] = { .name = "notify", - .conf = "{}" }; - return deps; -} + int r = RGWSI_SysObj_Core::do_start(); + if (r < 0) { + return r; + } -int RGWSI_SysObj_Cache::load(const string& conf, std::map& dep_refs) -{ - int r = RGWSI_SysObj_Core::load(conf, dep_refs); + r = notify_svc->start(); if (r < 0) { return r; } - notify_svc = static_pointer_cast(dep_refs["cache_notify_dep"]); - assert(notify_svc); + assert(notify_svc->is_started()); cb.reset(new RGWSI_SysObj_Cache_CB(this)); diff --git a/src/rgw/services/svc_sys_obj_cache.h b/src/rgw/services/svc_sys_obj_cache.h index a7110607c28..2168abe0eeb 100644 --- a/src/rgw/services/svc_sys_obj_cache.h +++ b/src/rgw/services/svc_sys_obj_cache.h @@ -12,17 +12,10 @@ class RGWSI_Notify; class RGWSI_SysObj_Cache_CB; -class RGWS_SysObj_Cache : public RGWService -{ -public: - RGWS_SysObj_Cache(CephContext *cct) : RGWService(cct, "sysobj_cache") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core { friend class RGWSI_SysObj_Cache_CB; + friend class RGWServices_Shared; std::shared_ptr notify_svc; ObjectCache cache; @@ -31,8 +24,14 @@ class RGWSI_SysObj_Cache : public RGWSI_SysObj_Core void normalize_pool_and_obj(rgw_pool& src_pool, const string& src_obj, rgw_pool& dst_pool, string& dst_obj); protected: - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; + void init(std::shared_ptr& _rados_svc, + std::shared_ptr& _zone_svc, + std::shared_ptr& _notify_svc) { + core_init(_rados_svc, _zone_svc); + notify_svc = _notify_svc; + } + + int do_start() override; int raw_stat(rgw_raw_obj& obj, uint64_t *psize, real_time *pmtime, uint64_t *epoch, map *attrs, bufferlist *first_chunk, @@ -81,7 +80,7 @@ protected: void set_enabled(bool status); public: - RGWSI_SysObj_Cache(RGWService *svc, CephContext *cct) : RGWSI_SysObj_Core(svc, cct) { + RGWSI_SysObj_Cache(CephContext *cct) : RGWSI_SysObj_Core(cct) { cache.set_ctx(cct); } diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc index f19b2cf7dd5..b8699380262 100644 --- a/src/rgw/services/svc_sys_obj_core.cc +++ b/src/rgw/services/svc_sys_obj_core.cc @@ -6,12 +6,6 @@ #define dout_subsys ceph_subsys_rgw -int RGWS_SysObj_Core::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_SysObj_Core(this, cct)); - return 0; -} - int RGWSI_SysObj_Core::GetObjState::get_rados_obj(RGWSI_RADOS *rados_svc, RGWSI_Zone *zone_svc, rgw_raw_obj& obj, @@ -31,29 +25,6 @@ int RGWSI_SysObj_Core::GetObjState::get_rados_obj(RGWSI_RADOS *rados_svc, return 0; } -std::map RGWSI_SysObj_Core::get_deps() -{ - RGWServiceInstance::dependency dep1 = { .name = "rados", - .conf = "{}" }; - RGWServiceInstance::dependency dep2 = { .name = "zone", - .conf = "{}" }; - map deps; - deps["rados_dep"] = dep1; - deps["zone_dep"] = dep2; - return deps; -} - -int RGWSI_SysObj_Core::load(const string& conf, std::map& dep_refs) -{ - rados_svc = static_pointer_cast(dep_refs["rados_dep"]); - assert(rados_svc); - - zone_svc = static_pointer_cast(dep_refs["zone_dep"]); - assert(zone_svc); - - return 0; -} - int RGWSI_SysObj_Core::get_rados_obj(RGWSI_Zone *zone_svc, rgw_raw_obj& obj, RGWSI_RADOS::Obj *pobj) diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h index 7165ae29ebc..0de97011396 100644 --- a/src/rgw/services/svc_sys_obj_core.h +++ b/src/rgw/services/svc_sys_obj_core.h @@ -98,16 +98,9 @@ public: } }; -class RGWS_SysObj_Core : public RGWService -{ -public: - RGWS_SysObj_Core(CephContext *cct) : RGWService(cct, "sysobj_core") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_SysObj_Core : public RGWServiceInstance { + friend class RGWServices_Shared; friend class RGWSI_SysObj; protected: @@ -128,9 +121,11 @@ protected: }; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - + void core_init(std::shared_ptr& _rados_svc, + std::shared_ptr& _zone_svc) { + rados_svc = _rados_svc; + zone_svc = _zone_svc; + } int get_rados_obj(RGWSI_Zone *zone_svc, rgw_raw_obj& obj, RGWSI_RADOS::Obj *pobj); virtual int raw_stat(rgw_raw_obj& obj, uint64_t *psize, real_time *pmtime, uint64_t *epoch, @@ -198,7 +193,7 @@ protected: RGWObjVersionTracker *objv_tracker); public: - RGWSI_SysObj_Core(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_SysObj_Core(CephContext *cct): RGWServiceInstance(cct) {} RGWSI_Zone *get_zone_svc() { return zone_svc.get(); diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index d5fc135aaa2..63858e0cd1a 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -13,42 +13,19 @@ using namespace rgw_zone_defaults; -int RGWS_Zone::create_instance(const string& conf, RGWServiceInstanceRef *instance) +void RGWSI_Zone::init(std::shared_ptr& _sysobj_svc, + std::shared_ptr& _rados_svc, + std::shared_ptr& _sync_modules_svc) { - instance->reset(new RGWSI_Zone(this, cct)); - return 0; -} - -std::map RGWSI_Zone::get_deps() -{ - map deps; - deps["sysobj_dep"] = { .name = "sysobj", - .conf = "{}" }; - deps["rados_dep"] = { .name = "rados", - .conf = "{}" }; - deps["sync_modules_dep"] = { .name = "sync_modules", - .conf = "{}" }; - return deps; -} - -int RGWSI_Zone::load(const string& conf, std::map& dep_refs) -{ - sysobj_svc = static_pointer_cast(dep_refs["sysobj_dep"]); - assert(sysobj_svc); + sysobj_svc = _sysobj_svc; + rados_svc = _rados_svc; + sync_modules_svc = _sync_modules_svc; realm = make_shared(); zonegroup = make_shared(); zone_public_config = make_shared(); zone_params = make_shared(); current_period = make_shared(); - - rados_svc = static_pointer_cast(dep_refs["rados_dep"]); - assert(rados_svc); - - sync_modules_svc = static_pointer_cast(dep_refs["sync_modules_dep"]); - assert(sync_modules_svc); - - return 0; } bool RGWSI_Zone::zone_syncs_from(RGWZone& target_zone, RGWZone& source_zone) @@ -57,9 +34,24 @@ bool RGWSI_Zone::zone_syncs_from(RGWZone& target_zone, RGWZone& source_zone) sync_modules_svc->get_manager()->supports_data_export(source_zone.tier_type); } -int RGWSI_Zone::init() +int RGWSI_Zone::do_start() { - int ret = realm->init(cct, sysobj_svc.get()); + int ret = sysobj_svc->start(); + if (ret < 0) { + return ret; + } + + assert(sysobj_svc->start_state == StateStarted); /* if not then there's ordering issue */ + + ret = rados_svc->start(); + if (ret < 0) { + return ret; + } + ret = sync_modules_svc->start(); + if (ret < 0) { + return ret; + } + ret = realm->init(cct, sysobj_svc.get()); if (ret < 0 && ret != -ENOENT) { ldout(cct, 0) << "failed reading realm info: ret "<< ret << " " << cpp_strerror(-ret) << dendl; return ret; diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h index b26684cc2bc..76f4fac7f97 100644 --- a/src/rgw/services/svc_zone.h +++ b/src/rgw/services/svc_zone.h @@ -12,16 +12,10 @@ class RGWSI_SyncModules; class RGWRESTConn; -class RGWS_Zone : public RGWService -{ -public: - RGWS_Zone(CephContext *cct) : RGWService(cct, "zone") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_Zone : public RGWServiceInstance { + friend struct RGWServices_Shared; + std::shared_ptr sysobj_svc; std::shared_ptr rados_svc; std::shared_ptr sync_modules_svc; @@ -43,9 +37,10 @@ class RGWSI_Zone : public RGWServiceInstance map zone_id_by_name; map zone_by_id; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - int init() override; + void init(std::shared_ptr& _sysobj_svc, + std::shared_ptr& _rados_svc, + std::shared_ptr& _sync_modules_svc); + int do_start() override; void shutdown() override; int replace_region_with_zonegroup(); @@ -55,7 +50,7 @@ class RGWSI_Zone : public RGWServiceInstance int update_placement_map(); public: - RGWSI_Zone(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_Zone(CephContext *cct): RGWServiceInstance(cct) {} RGWZoneParams& get_zone_params(); RGWPeriod& get_current_period(); diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc index a498da08a78..ef9c9c88c33 100644 --- a/src/rgw/services/svc_zone_utils.cc +++ b/src/rgw/services/svc_zone_utils.cc @@ -4,36 +4,7 @@ #include "rgw/rgw_zone.h" -int RGWS_ZoneUtils::create_instance(const string& conf, RGWServiceInstanceRef *instance) -{ - instance->reset(new RGWSI_ZoneUtils(this, cct)); - return 0; -} - -std::map RGWSI_ZoneUtils::get_deps() -{ - RGWServiceInstance::dependency dep1 = { .name = "rados", - .conf = "{}" }; - RGWServiceInstance::dependency dep2 = { .name = "zone", - .conf = "{}" }; - map deps; - deps["rados_dep"] = dep1; - deps["zone_dep"] = dep2; - return deps; -} - -int RGWSI_ZoneUtils::load(const string& conf, std::map& dep_refs) -{ - rados_svc = static_pointer_cast(dep_refs["rados_dep"]); - assert(rados_svc); - - zone_svc = static_pointer_cast(dep_refs["zone_dep"]); - assert(zone_svc); - - return 0; -} - -int RGWSI_ZoneUtils::init() +int RGWSI_ZoneUtils::do_start() { init_unique_trans_id_deps(); diff --git a/src/rgw/services/svc_zone_utils.h b/src/rgw/services/svc_zone_utils.h index 79b779afe80..fda2a0dc22a 100644 --- a/src/rgw/services/svc_zone_utils.h +++ b/src/rgw/services/svc_zone_utils.h @@ -8,29 +8,27 @@ class RGWSI_RADOS; class RGWSI_Zone; -class RGWS_ZoneUtils : public RGWService -{ -public: - RGWS_ZoneUtils(CephContext *cct) : RGWService(cct, "zone_utils") {} - - int create_instance(const std::string& conf, RGWServiceInstanceRef *instance) override; -}; - class RGWSI_ZoneUtils : public RGWServiceInstance { + friend struct RGWServices_Shared; + std::shared_ptr rados_svc; std::shared_ptr zone_svc; string trans_id_suffix; - std::map get_deps() override; - int load(const std::string& conf, std::map& dep_refs) override; - int init() override; + void init(std::shared_ptr& _rados_svc, + std::shared_ptr& _zone_svc) { + rados_svc = _rados_svc; + zone_svc = _zone_svc; + } + + int do_start() override; void init_unique_trans_id_deps(); public: - RGWSI_ZoneUtils(RGWService *svc, CephContext *cct): RGWServiceInstance(svc, cct) {} + RGWSI_ZoneUtils(CephContext *cct): RGWServiceInstance(cct) {} string gen_host_id(); string unique_id(uint64_t unique_num);