From 8ab3694b214730ac65dd683ca4544f993ec68e19 Mon Sep 17 00:00:00 2001 From: Ali Masarwa Date: Wed, 19 Mar 2025 04:33:47 +0200 Subject: [PATCH] RGW/standalone: refactor RGWRealm with configstore Signed-off-by: Ali Masarwa --- src/rgw/driver/rados/rgw_rest_realm.cc | 14 +-- src/rgw/driver/rados/rgw_zone.h | 55 ++++++------ src/rgw/rgw_realm.cc | 120 +++---------------------- src/rgw/rgw_zone.cc | 17 +++- src/rgw/services/svc_zone.cc | 12 +-- 5 files changed, 72 insertions(+), 146 deletions(-) diff --git a/src/rgw/driver/rados/rgw_rest_realm.cc b/src/rgw/driver/rados/rgw_rest_realm.cc index 0d48db51817..154a5ae54bd 100644 --- a/src/rgw/driver/rados/rgw_rest_realm.cc +++ b/src/rgw/driver/rados/rgw_rest_realm.cc @@ -133,7 +133,7 @@ void RGWOp_Period_Post::execute(optional_yield y) // period that we haven't restarted with yet. we also don't want to modify // the objects in use by RGWRados RGWRealm realm(period.get_realm()); - op_ret = realm.init(this, cct, static_cast(driver)->svc()->sysobj, y); + op_ret = rgw::read_realm(this, y, cfgstore.get(), realm.get_id(), realm.get_name(), realm); if (op_ret < 0) { ldpp_dout(this, -1) << "failed to read current realm: " << cpp_strerror(-op_ret) << dendl; @@ -273,7 +273,9 @@ void RGWOp_Period_Post::send_response() if (notify_realm) { // trigger realm reload after sending the response, because reload may // race to close this connection - notify_realm->notify_new_period(this, period, s->yield); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(this, config_store_type); + (void) cfgstore->realm_notify_new_period(this, null_yield, period); } } @@ -320,7 +322,9 @@ void RGWOp_Realm_Get::execute(optional_yield y) // read realm realm.reset(new RGWRealm(id, name)); - op_ret = realm->init(this, g_ceph_context, static_cast(driver)->svc()->sysobj, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(this, config_store_type); + op_ret = rgw::read_realm(this, y, cfgstore.get(), realm->get_id(), realm->get_name(), *realm); if (op_ret < 0) ldpp_dout(this, -1) << "failed to read realm id=" << id << " name=" << name << dendl; @@ -361,8 +365,8 @@ void RGWOp_Realm_List::execute(optional_yield y) { { // read default realm - RGWRealm realm(driver->ctx(), static_cast(driver)->svc()->sysobj); - [[maybe_unused]] int ret = realm.read_default_id(this, default_id, y); + RGWRealm realm(driver->ctx()); + default_id = realm.get_default_oid(); } op_ret = static_cast(driver)->svc()->zone->list_realms(this, realms); if (op_ret < 0) diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h index a4c1856047c..7f420007b09 100644 --- a/src/rgw/driver/rados/rgw_zone.h +++ b/src/rgw/driver/rados/rgw_zone.h @@ -525,47 +525,56 @@ WRITE_CLASS_ENCODER(RGWPeriodConfig) class RGWRealm; class RGWPeriod; -class RGWRealm : public RGWSystemMetaObj +class RGWRealm { public: + std::string id; + std::string name; + + CephContext *cct{nullptr}; + std::string current_period; epoch_t epoch{0}; //< realm epoch, incremented for each new period - int create_control(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y); - int delete_control(const DoutPrefixProvider *dpp, optional_yield y); public: RGWRealm() {} - RGWRealm(const std::string& _id, const std::string& _name = "") : RGWSystemMetaObj(_id, _name) {} - RGWRealm(CephContext *_cct, RGWSI_SysObj *_sysobj_svc): RGWSystemMetaObj(_cct, _sysobj_svc) {} - RGWRealm(const std::string& _name, CephContext *_cct, RGWSI_SysObj *_sysobj_svc): RGWSystemMetaObj(_name, _cct, _sysobj_svc){} - virtual ~RGWRealm() override; + RGWRealm(const std::string& _id, const std::string& _name = "") : id(_id), name(_name) {} + RGWRealm(CephContext *_cct): cct(_cct) {} + RGWRealm(const std::string& _name, CephContext *_cct, RGWSI_SysObj *_sysobj_svc): name(_name), cct(_cct){} - void encode(bufferlist& bl) const override { + const std::string& get_name() const { return name; } + const std::string& get_id() const { return id; } + + void set_name(const std::string& _name) { name = _name;} + void set_id(const std::string& _id) { id = _id;} + void clear_id() { id.clear(); } + + virtual ~RGWRealm(); + + void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); - RGWSystemMetaObj::encode(bl); + encode(id, bl); + encode(name, bl); encode(current_period, bl); encode(epoch, bl); ENCODE_FINISH(bl); } - void decode(bufferlist::const_iterator& bl) override { + void decode(bufferlist::const_iterator& bl) { DECODE_START(1, bl); - RGWSystemMetaObj::decode(bl); + decode(id, bl); + decode(name, bl); decode(current_period, bl); decode(epoch, bl); DECODE_FINISH(bl); } - int create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive = true) override; - int delete_obj(const DoutPrefixProvider *dpp, optional_yield y); - rgw_pool get_pool(CephContext *cct) const override; - const std::string get_default_oid(bool old_format = false) const override; - const std::string& get_names_oid_prefix() const override; - const std::string& get_info_oid_prefix(bool old_format = false) const override; - std::string get_predefined_id(CephContext *cct) const override; - const std::string& get_predefined_name(CephContext *cct) const override; - - using RGWSystemMetaObj::read_id; // expose as public for radosgw-admin + rgw_pool get_pool(CephContext *cct) const; + const std::string get_default_oid(bool old_format = false) const; + const std::string& get_names_oid_prefix() const; + const std::string& get_info_oid_prefix(bool old_format = false) const; + std::string get_predefined_id(CephContext *cct) const; + const std::string& get_predefined_name(CephContext *cct) const; void dump(Formatter *f) const; void decode_json(JSONObj *obj); @@ -582,10 +591,6 @@ public: epoch_t get_epoch() const { return epoch; } std::string get_control_oid() const; - /// send a notify on the realm control object - int notify_zone(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y); - /// notify the zone of a new period - int notify_new_period(const DoutPrefixProvider *dpp, const RGWPeriod& period, optional_yield y); int find_zone(const DoutPrefixProvider *dpp, const rgw_zone_id& zid, diff --git a/src/rgw/rgw_realm.cc b/src/rgw/rgw_realm.cc index 225884e9f81..aa75e3e7e8e 100644 --- a/src/rgw/rgw_realm.cc +++ b/src/rgw/rgw_realm.cc @@ -48,84 +48,6 @@ const string& RGWRealm::get_predefined_name(CephContext *cct) const { return cct->_conf->rgw_realm; } -int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) -{ - int ret = RGWSystemMetaObj::create(dpp, y, exclusive); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR creating new realm object " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - // create the control object for watch/notify - ret = create_control(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR creating control for new realm " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - RGWPeriod period; - auto config_store_type = g_conf().get_val("rgw_config_store"); - auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); - if (current_period.empty()) { - /* create new period for the realm */ - ret = cfgstore->read_period(dpp, y, period.get_id(), period.get_epoch(), period); - if (ret < 0 ) { - return ret; - } - ret = cfgstore->create_period(dpp, y, true, period); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: creating new period for realm " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - } else { - period = RGWPeriod(current_period, 0); - ret = cfgstore->read_period(dpp, y, period.get_id(), period.get_epoch(), period); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to init period " << current_period << dendl; - return ret; - } - } - ret = set_current_period(dpp, period, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed set current period " << current_period << dendl; - return ret; - } - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - ret = set_as_default(dpp, y, true); - if (ret < 0 && ret != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default realm, ret=" << ret << dendl; - } - - return 0; -} - -int RGWRealm::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) -{ - int ret = RGWSystemMetaObj::delete_obj(dpp, y); - if (ret < 0) { - return ret; - } - return delete_control(dpp, y); -} - -int RGWRealm::create_control(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) -{ - auto pool = rgw_pool{get_pool(cct)}; - auto oid = get_control_oid(); - bufferlist bl; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); -} - -int RGWRealm::delete_control(const DoutPrefixProvider *dpp, optional_yield y) -{ - auto pool = rgw_pool{get_pool(cct)}; - auto obj = rgw_raw_obj{pool, get_control_oid()}; - auto sysobj = sysobj_svc->get_obj(obj); - return sysobj.wop().remove(dpp, y); -} - rgw_pool RGWRealm::get_pool(CephContext *cct) const { if (cct->_conf->rgw_realm_root_pool.empty()) { @@ -170,14 +92,20 @@ int RGWRealm::set_current_period(const DoutPrefixProvider *dpp, RGWPeriod& perio epoch = period.get_realm_epoch(); current_period = period.get_id(); - int ret = update(dpp, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + std::unique_ptr writer; + int ret = cfgstore->create_realm(dpp, y, false, *this, &writer); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: realm create: " << cpp_strerror(-ret) << dendl; + return ret; + } + ret = rgw::realm_set_current_period(dpp, y, cfgstore.get(), *writer, *this, period); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: period update: " << cpp_strerror(-ret) << dendl; return ret; } - auto config_store_type = g_conf().get_val("rgw_config_store"); - auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); ret = rgw::reflect_period(dpp, y, cfgstore.get(), period); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: period.reflect(): " << cpp_strerror(-ret) << dendl; @@ -192,30 +120,6 @@ string RGWRealm::get_control_oid() const return get_info_oid_prefix() + id + ".control"; } -int RGWRealm::notify_zone(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y) -{ - rgw_pool pool{get_pool(cct)}; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_control_oid()}); - int ret = sysobj.wn().notify(dpp, bl, 0, nullptr, y); - if (ret < 0) { - return ret; - } - return 0; -} - -int RGWRealm::notify_new_period(const DoutPrefixProvider *dpp, const RGWPeriod& period, optional_yield y) -{ - bufferlist bl; - using ceph::encode; - // push the period to dependent zonegroups/zones - encode(RGWRealmNotify::ZonesNeedPeriod, bl); - encode(period, bl); - // reload the gateway with the new period - encode(RGWRealmNotify::Reload, bl); - - return notify_zone(dpp, bl, y); -} - int RGWRealm::find_zone(const DoutPrefixProvider *dpp, const rgw_zone_id& zid, @@ -256,7 +160,8 @@ void RGWRealm::generate_test_instances(list &o) void RGWRealm::dump(Formatter *f) const { - RGWSystemMetaObj::dump(f); + encode_json("id", id , f); + encode_json("name", name , f); encode_json("current_period", current_period, f); encode_json("epoch", epoch, f); } @@ -264,7 +169,8 @@ void RGWRealm::dump(Formatter *f) const void RGWRealm::decode_json(JSONObj *obj) { - RGWSystemMetaObj::decode_json(obj); + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("name", name, obj); JSONDecoder::decode_json("current_period", current_period, obj); JSONDecoder::decode_json("epoch", epoch, obj); } diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index d2f5d494bca..d8726b1df04 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -6,6 +6,7 @@ #include "common/errno.h" #include "rgw_zone.h" +#include "rgw_sal.h" #include "rgw_sal_config.h" #include "rgw_sync.h" @@ -202,7 +203,9 @@ int RGWZoneGroup::read_default_id(const DoutPrefixProvider *dpp, string& default if (realm_id.empty()) { /* try using default realm */ RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = rgw::read_realm(dpp, y, cfgstore.get(), realm.get_id(), realm.get_name(), realm); // no default realm exist if (ret < 0) { return read_id(dpp, default_zonegroup_name, default_id, y); @@ -386,7 +389,9 @@ int RGWZoneParams::read_default_id(const DoutPrefixProvider *dpp, string& defaul if (realm_id.empty()) { /* try using default realm */ RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = rgw::read_realm(dpp, y, cfgstore.get(), realm.get_id(), realm.get_name(), realm); //no default realm exist if (ret < 0) { return read_id(dpp, default_zone_name, default_id, y); @@ -403,7 +408,9 @@ int RGWZoneParams::set_as_default(const DoutPrefixProvider *dpp, optional_yield if (realm_id.empty()) { /* try using default realm */ RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = rgw::read_realm(dpp, y, cfgstore.get(), realm.get_id(), realm.get_name(), realm); if (ret < 0) { ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; return -EINVAL; @@ -1204,7 +1211,9 @@ int RGWZoneGroup::set_as_default(const DoutPrefixProvider *dpp, optional_yield y if (realm_id.empty()) { /* try using default realm */ RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = rgw::read_realm(dpp, y, cfgstore.get(), realm.get_id(), realm.get_name(), realm); if (ret < 0) { ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; return -EINVAL; diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index 9fec53028dd..7e25e1d9580 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -104,7 +104,9 @@ int RGWSI_Zone::search_realm_with_zone(const DoutPrefixProvider *dpp, for (auto& realm_name : realms) { string realm_id; RGWRealm realm(realm_id, realm_name); - r = realm.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + r = rgw::read_realm(dpp, y, cfgstore.get(), realm_id, realm_name, realm); if (r < 0) { ldpp_dout(dpp, 0) << "WARNING: can't open realm " << realm_name << ": " << cpp_strerror(-r) << " ... skipping" << dendl; continue; @@ -136,14 +138,14 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp) assert(sysobj_svc->is_started()); /* if not then there's ordering issue */ - ret = realm->init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + ret = rgw::read_realm(dpp, y, cfgstore.get(), realm->get_id(), realm->get_name(), *realm); if (ret < 0 && ret != -ENOENT) { ldpp_dout(dpp, 0) << "failed reading realm info: ret "<< ret << " " << cpp_strerror(-ret) << dendl; return ret; } - auto config_store_type = g_conf().get_val("rgw_config_store"); - auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); ldpp_dout(dpp, 20) << "realm " << realm->get_name() << " " << realm->get_id() << dendl; current_period->set_realm_id(realm->get_id()); ret = cfgstore->read_period(dpp, y, current_period->get_id(), current_period->epoch, *current_period); @@ -425,7 +427,7 @@ int RGWSI_Zone::list_zones(const DoutPrefixProvider *dpp, list& zones) int RGWSI_Zone::list_realms(const DoutPrefixProvider *dpp, list& realms) { - RGWRealm realm(cct, sysobj_svc); + RGWRealm realm(cct); RGWSI_SysObj::Pool syspool = sysobj_svc->get_pool(realm.get_pool(cct)); return syspool.list_prefixed_objs(dpp, realm_names_oid_prefix, &realms); -- 2.39.5