From 976889ff3ac42d8ce677dadb4700615274d77eb7 Mon Sep 17 00:00:00 2001 From: Ali Masarwa Date: Tue, 18 Mar 2025 15:11:19 +0200 Subject: [PATCH] RGW/standalone: refactor RGWPeriod with configstore Signed-off-by: Ali Masarwa --- src/rgw/driver/dbstore/config/sqlite.cc | 81 +++++++++ src/rgw/driver/dbstore/config/sqlite.h | 4 + src/rgw/driver/immutable_config/store.cc | 13 ++ src/rgw/driver/immutable_config/store.h | 4 + src/rgw/driver/rados/config/period.cc | 30 ++-- src/rgw/driver/rados/config/store.h | 4 + src/rgw/driver/rados/rgw_period.cc | 112 +++--------- src/rgw/driver/rados/rgw_rest_realm.cc | 29 +++- src/rgw/driver/rados/rgw_zone.cc | 10 +- src/rgw/driver/rados/rgw_zone.h | 23 +-- src/rgw/rgw_period.cc | 207 +---------------------- src/rgw/rgw_period_puller.cc | 16 +- src/rgw/rgw_period_pusher.cc | 6 +- src/rgw/rgw_realm.cc | 16 +- src/rgw/rgw_sal_config.h | 5 + src/rgw/services/svc_zone.cc | 11 +- 16 files changed, 220 insertions(+), 351 deletions(-) diff --git a/src/rgw/driver/dbstore/config/sqlite.cc b/src/rgw/driver/dbstore/config/sqlite.cc index aa2d1a354ffcf..43274a91ba5d9 100644 --- a/src/rgw/driver/dbstore/config/sqlite.cc +++ b/src/rgw/driver/dbstore/config/sqlite.cc @@ -872,6 +872,87 @@ int SQLiteConfigStore::delete_period(const DoutPrefixProvider* dpp, return 0; } +int SQLiteConfigStore::read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) +{ + Prefix prefix{*dpp, "dbconfig:sqlite:read_latest_epoch "}; dpp = &prefix; + + if (period_id.empty()) { + ldpp_dout(dpp, 0) << "requires a period id" << dendl; + return -EINVAL; + } + + try { + auto conn = impl->get(dpp); + period_select_epoch(dpp, *conn, period_id, epoch, info); + } catch (const buffer::error& e) { + ldpp_dout(dpp, 20) << "period decode failed: " << e.what() << dendl; + return -EIO; + } catch (const sqlite::error& e) { + ldpp_dout(dpp, 20) << "period select failed: " << e.what() << dendl; + if (e.code() == sqlite::errc::done) { + return -ENOENT; + } else if (e.code() == sqlite::errc::busy) { + return -EBUSY; + } + return -EIO; + } + return 0; +} + +int SQLiteConfigStore::write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, + std::string_view period_id, uint32_t epoch, RGWObjVersionTracker* objv, + const RGWPeriod& info) +{ + Prefix prefix{*dpp, "dbconfig:sqlite:write_latest_epoch "}; dpp = &prefix; + + if (info.id.empty()) { + ldpp_dout(dpp, 0) << "period cannot have an empty id" << dendl; + return -EINVAL; + } + + bufferlist bl; + encode(info, bl); + const auto data = std::string_view{bl.c_str(), bl.length()}; + + try { + auto conn = impl->get(dpp); + sqlite::stmt_ptr* stmt = nullptr; + if (exclusive) { + stmt = &conn->statements["period_ins"]; + if (!*stmt) { + const std::string sql = fmt::format(schema::period_insert4, + P1, P2, P3, P4); + *stmt = sqlite::prepare_statement(dpp, conn->db.get(), sql); + } + } else { + stmt = &conn->statements["period_ups"]; + if (!*stmt) { + const std::string sql = fmt::format(schema::period_upsert4, + P1, P2, P3, P4); + *stmt = sqlite::prepare_statement(dpp, conn->db.get(), sql); + } + } + auto binding = sqlite::stmt_binding{stmt->get()}; + sqlite::bind_text(dpp, binding, P1, info.id); + sqlite::bind_int(dpp, binding, P2, info.epoch); + sqlite::bind_text(dpp, binding, P3, info.realm_id); + sqlite::bind_text(dpp, binding, P4, data); + + auto reset = sqlite::stmt_execution{stmt->get()}; + sqlite::eval0(dpp, reset); + } catch (const sqlite::error& e) { + ldpp_dout(dpp, 20) << "period insert failed: " << e.what() << dendl; + if (e.code() == sqlite::errc::foreign_key_constraint) { + return -EINVAL; // refers to nonexistent RealmID + } else if (e.code() == sqlite::errc::busy) { + return -EBUSY; + } + return -EIO; + } + return 0; +} + int SQLiteConfigStore::list_period_ids(const DoutPrefixProvider* dpp, optional_yield y, const std::string& marker, diff --git a/src/rgw/driver/dbstore/config/sqlite.h b/src/rgw/driver/dbstore/config/sqlite.h index d79e040728c26..2b4c304b22d64 100644 --- a/src/rgw/driver/dbstore/config/sqlite.h +++ b/src/rgw/driver/dbstore/config/sqlite.h @@ -79,6 +79,10 @@ class SQLiteConfigStore : public sal::ConfigStore { optional_yield y, const std::string& marker, std::span entries, sal::ListResult& result) override; + int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) override; + int write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, std::string_view period_id, + uint32_t epoch, RGWObjVersionTracker* objv, const RGWPeriod& info) override; int write_default_zonegroup_id(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, diff --git a/src/rgw/driver/immutable_config/store.cc b/src/rgw/driver/immutable_config/store.cc index 8d3e0765faa1f..9afc70efc122a 100644 --- a/src/rgw/driver/immutable_config/store.cc +++ b/src/rgw/driver/immutable_config/store.cc @@ -137,6 +137,19 @@ int ImmutableConfigStore::list_period_ids(const DoutPrefixProvider* dpp, return 0; } +int ImmutableConfigStore::read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) +{ + return -ENOENT; +} + +int ImmutableConfigStore::write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, + std::string_view period_id, uint32_t epoch, RGWObjVersionTracker* objv, + const RGWPeriod& info) +{ + return -EROFS; +} + // ZoneGroup diff --git a/src/rgw/driver/immutable_config/store.h b/src/rgw/driver/immutable_config/store.h index 9a1ac5f14432f..08b874c4e0389 100644 --- a/src/rgw/driver/immutable_config/store.h +++ b/src/rgw/driver/immutable_config/store.h @@ -78,6 +78,10 @@ class ImmutableConfigStore : public ConfigStore { optional_yield y, const std::string& marker, std::span entries, ListResult& result) override; + virtual int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) override; + virtual int write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, std::string_view period_id, + uint32_t epoch, RGWObjVersionTracker* objv, const RGWPeriod& info) override; // ZoneGroup virtual int write_default_zonegroup_id(const DoutPrefixProvider* dpp, diff --git a/src/rgw/driver/rados/config/period.cc b/src/rgw/driver/rados/config/period.cc index bc3fa27e72c0d..046e3a58d17c8 100644 --- a/src/rgw/driver/rados/config/period.cc +++ b/src/rgw/driver/rados/config/period.cc @@ -44,9 +44,8 @@ static std::string latest_epoch_oid(const ceph::common::ConfigProxy& conf, period_latest_epoch_info_oid)); } -static int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, - ConfigImpl* impl, std::string_view period_id, - uint32_t& epoch, RGWObjVersionTracker* objv) +int RadosConfigStore::read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) { const auto& pool = impl->period_pool; const auto latest_oid = latest_epoch_oid(dpp->get_cct()->_conf, period_id); @@ -58,10 +57,9 @@ static int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, return r; } -static int write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, - ConfigImpl* impl, bool exclusive, - std::string_view period_id, uint32_t epoch, - RGWObjVersionTracker* objv) +int RadosConfigStore::write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, + std::string_view period_id, uint32_t epoch, RGWObjVersionTracker* objv, + const RGWPeriod& info) { const auto& pool = impl->period_pool; const auto latest_oid = latest_epoch_oid(dpp->get_cct()->_conf, period_id); @@ -80,8 +78,8 @@ static int delete_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, } static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, - ConfigImpl* impl, std::string_view period_id, - uint32_t epoch) + RadosConfigStore& rados_config_store, std::string_view period_id, + uint32_t epoch, RGWPeriod& info) { static constexpr int MAX_RETRIES = 20; @@ -91,7 +89,7 @@ static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive = false; // read existing epoch - int r = read_latest_epoch(dpp, y, impl, period_id, existing_epoch, &objv); + int r = rados_config_store.read_latest_epoch(dpp, y, period_id, existing_epoch, &objv, info); if (r == -ENOENT) { // use an exclusive create to set the epoch atomically exclusive = true; @@ -111,7 +109,7 @@ static int update_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, << " -> " << epoch << " on period=" << period_id << dendl; } - r = write_latest_epoch(dpp, y, impl, exclusive, period_id, epoch, &objv); + r = rados_config_store.write_latest_epoch(dpp, y, exclusive, period_id, epoch, &objv, info); if (r == -EEXIST) { continue; // exclusive create raced with another update, retry } else if (r == -ECANCELED) { @@ -149,7 +147,9 @@ int RadosConfigStore::create_period(const DoutPrefixProvider* dpp, return r; } - (void) update_latest_epoch(dpp, y, impl.get(), info.get_id(), info.get_epoch()); + // non const RGWPeriod + RGWPeriod info_copy = info; + (void) update_latest_epoch(dpp, y, *this, info.get_id(), info.get_epoch(), info_copy); return 0; } @@ -162,7 +162,7 @@ int RadosConfigStore::read_period(const DoutPrefixProvider* dpp, int r = 0; if (!epoch) { epoch = 0; - r = read_latest_epoch(dpp, y, impl.get(), period_id, *epoch, nullptr); + r = read_latest_epoch(dpp, y, period_id, *epoch, nullptr, info); if (r < 0) { return r; } @@ -182,8 +182,8 @@ int RadosConfigStore::delete_period(const DoutPrefixProvider* dpp, // read the latest_epoch uint32_t latest_epoch = 0; RGWObjVersionTracker latest_objv; - int r = read_latest_epoch(dpp, y, impl.get(), period_id, - latest_epoch, &latest_objv); + RGWPeriod period; // not used in RadosConfigStore, but needed in the API + int r = read_latest_epoch(dpp, y, period_id, latest_epoch, &latest_objv, period); if (r < 0 && r != -ENOENT) { // just delete epoch=0 on ENOENT ldpp_dout(dpp, 0) << "failed to read latest epoch for period " << period_id << ": " << cpp_strerror(r) << dendl; diff --git a/src/rgw/driver/rados/config/store.h b/src/rgw/driver/rados/config/store.h index 1b93a803db3e9..0420996c93458 100644 --- a/src/rgw/driver/rados/config/store.h +++ b/src/rgw/driver/rados/config/store.h @@ -85,6 +85,10 @@ class RadosConfigStore : public sal::ConfigStore { optional_yield y, const std::string& marker, std::span entries, sal::ListResult& result) override; + virtual int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) override; + virtual int write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, std::string_view period_id, + uint32_t epoch, RGWObjVersionTracker* objv, const RGWPeriod& info)override; // ZoneGroup virtual int write_default_zonegroup_id(const DoutPrefixProvider* dpp, diff --git a/src/rgw/driver/rados/rgw_period.cc b/src/rgw/driver/rados/rgw_period.cc index aacb9b6a09af8..cdd0ab9432399 100644 --- a/src/rgw/driver/rados/rgw_period.cc +++ b/src/rgw/driver/rados/rgw_period.cc @@ -2,9 +2,13 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "rgw_sync.h" +#include "rgw_sal.h" +#include "rgw_sal_config.h" #include "services/svc_zone.h" +#define FIRST_EPOCH 1 + #define dout_subsys ceph_subsys_rgw using namespace std; @@ -31,7 +35,9 @@ int RGWPeriod::get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& latest_e { RGWPeriodLatestEpochInfo info; - int ret = read_latest_epoch(dpp, info, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = cfgstore->read_latest_epoch(dpp, y, id, info.epoch, nullptr, *this); if (ret < 0) { return ret; } @@ -41,92 +47,6 @@ int RGWPeriod::get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& latest_e return 0; } -int RGWPeriod::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) -{ - rgw_pool pool(get_pool(cct)); - - // delete the object for each period epoch - for (epoch_t e = 1; e <= epoch; e++) { - RGWPeriod p{get_id(), e}; - rgw_raw_obj oid{pool, p.get_period_oid()}; - auto sysobj = sysobj_svc->get_obj(oid); - int ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid - << ": " << cpp_strerror(-ret) << dendl; - } - } - - // delete the .latest_epoch object - rgw_raw_obj oid{pool, get_period_oid_prefix() + get_latest_epoch_oid()}; - auto sysobj = sysobj_svc->get_obj(oid); - int ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid - << ": " << cpp_strerror(-ret) << dendl; - } - return ret; -} - -int RGWPeriod::update(const DoutPrefixProvider *dpp, optional_yield y) -{ - auto zone_svc = sysobj_svc->get_zone_svc(); - ldpp_dout(dpp, 20) << __func__ << " realm " << realm_id << " period " << get_id() << dendl; - list zonegroups; - int ret = zone_svc->list_zonegroups(dpp, zonegroups); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to list zonegroups: " << cpp_strerror(-ret) << dendl; - return ret; - } - - // clear zone short ids of removed zones. period_map.update() will add the - // remaining zones back - period_map.short_zone_ids.clear(); - - for (auto& iter : zonegroups) { - RGWZoneGroup zg(string(), iter); - ret = zg.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: zg.init() failed: " << cpp_strerror(-ret) << dendl; - continue; - } - - if (zg.realm_id != realm_id) { - ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() << " zone realm id " << zg.realm_id << ", not on our realm " << realm_id << dendl; - continue; - } - - if (zg.master_zone.empty()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; - return -EINVAL; - } - - if (zg.zones.find(zg.master_zone) == zg.zones.end()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() - << " has a non existent master zone "<< dendl; - return -EINVAL; - } - - if (zg.is_master_zonegroup()) { - master_zonegroup = zg.get_id(); - master_zone = zg.master_zone; - } - - int ret = period_map.update(zg, cct); - if (ret < 0) { - return ret; - } - } - - ret = period_config.read(dpp, sysobj_svc, realm_id, y); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " - << cpp_strerror(ret) << dendl; - return ret; - } - return 0; -} - void RGWPeriod::fork() { ldout(cct, 20) << __func__ << " realm " << realm_id << " period " << id << dendl; @@ -207,12 +127,11 @@ int RGWPeriod::commit(const DoutPrefixProvider *dpp, std::ostream& error_stream, optional_yield y, bool force_if_stale) { - auto zone_svc = sysobj_svc->get_zone_svc(); ldpp_dout(dpp, 20) << __func__ << " realm " << realm.get_id() << " period " << current_period.get_id() << dendl; // gateway must be in the master zone to commit - if (master_zone != zone_svc->get_zone_params().get_id()) { + if (driver->get_zone()->get_zonegroup().is_master_zonegroup()) { error_stream << "Cannot commit period on zone " - << zone_svc->get_zone_params().get_id() << ", it must be sent to " + << driver->get_zone()->get_id() << ", it must be sent to " "the period's master zone " << master_zone << '.' << std::endl; return -EINVAL; } @@ -233,6 +152,9 @@ int RGWPeriod::commit(const DoutPrefixProvider *dpp, "and try again." << std::endl; return -EINVAL; } + + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); // did the master zone change? if (master_zone != current_period.get_master_zone()) { // store the current metadata sync status in the period @@ -243,7 +165,11 @@ int RGWPeriod::commit(const DoutPrefixProvider *dpp, return r; } // create an object with a new period id - r = create(dpp, y, true); + period_map.id = id = rgw::gen_random_uuid(); + epoch = FIRST_EPOCH; + + constexpr bool exclusive = true; + r = cfgstore->create_period(dpp, y, exclusive, *this); if (r < 0) { ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; return r; @@ -273,7 +199,7 @@ int RGWPeriod::commit(const DoutPrefixProvider *dpp, set_predecessor(current_period.get_predecessor()); realm_epoch = current_period.get_realm_epoch(); // write the period to rados - int r = store_info(dpp, false, y); + int r = cfgstore->create_period(dpp, y, false, *this); if (r < 0) { ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(-r) << dendl; return r; @@ -284,7 +210,7 @@ int RGWPeriod::commit(const DoutPrefixProvider *dpp, ldpp_dout(dpp, 0) << "failed to set latest epoch: " << cpp_strerror(-r) << dendl; return r; } - r = reflect(dpp, y); + r = rgw::reflect_period(dpp, y, cfgstore.get(), *this); if (r < 0) { ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(-r) << dendl; return r; diff --git a/src/rgw/driver/rados/rgw_rest_realm.cc b/src/rgw/driver/rados/rgw_rest_realm.cc index 881819237eb1a..0d48db51817d3 100644 --- a/src/rgw/driver/rados/rgw_rest_realm.cc +++ b/src/rgw/driver/rados/rgw_rest_realm.cc @@ -8,6 +8,7 @@ #include "rgw_rest_config.h" #include "rgw_zone.h" #include "rgw_sal_rados.h" +#include "rgw_sal_config.h" #include "services/svc_zone.h" #include "services/svc_mdlog.h" @@ -72,10 +73,15 @@ void RGWOp_Period_Get::execute(optional_yield y) RESTArgs::get_string(s, "period_id", period_id, &period_id); RESTArgs::get_uint32(s, "epoch", 0, &epoch); + + period.set_realm_id(realm_id); period.set_id(period_id); period.set_epoch(epoch); - op_ret = period.init(this, driver->ctx(), static_cast(driver)->svc()->sysobj, realm_id, y); + + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(this, config_store_type); + op_ret = cfgstore->read_period(this, y, period_id, epoch, period); if (op_ret < 0) ldpp_dout(this, 5) << "failed to read period" << dendl; } @@ -102,7 +108,9 @@ void RGWOp_Period_Post::execute(optional_yield y) auto cct = driver->ctx(); // initialize the period without reading from rados - period.init(this, cct, static_cast(driver)->svc()->sysobj, y, false); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(this, config_store_type); + cfgstore->read_period(this, y,driver->get_zone()->get_current_period_id(), std::nullopt, period); // decode the period from input const auto max_size = cct->_conf->rgw_max_put_param_size; @@ -133,7 +141,7 @@ void RGWOp_Period_Post::execute(optional_yield y) } RGWPeriod current_period; - op_ret = current_period.init(this, cct, static_cast(driver)->svc()->sysobj, realm.get_id(), y); + op_ret = cfgstore->read_period(this, y, driver->get_zone()->get_current_period_id(), std::nullopt, current_period); if (op_ret < 0) { ldpp_dout(this, -1) << "failed to read current period: " << cpp_strerror(-op_ret) << dendl; @@ -142,7 +150,16 @@ void RGWOp_Period_Post::execute(optional_yield y) // if period id is empty, handle as 'period commit' if (period.get_id().empty()) { - op_ret = period.commit(this, driver, realm, current_period, error_stream, y); +// op_ret = period.commit(this, driver, realm, current_period, error_stream, y); + std::unique_ptr realm_writer; + op_ret = rgw::read_realm(this, null_yield, cfgstore.get(), + period.realm_id, period.get_realm(), + realm, &realm_writer); + if (op_ret < 0) { + cerr << "Error initializing realm: " << cpp_strerror(-op_ret) << std::endl; + return; + } + op_ret = rgw::commit_period(this, y, cfgstore.get(), driver, realm, *realm_writer, current_period, period, error_stream, false); if (op_ret == -EEXIST) { op_ret = 0; // succeed on retries so the op is idempotent return; @@ -164,7 +181,7 @@ void RGWOp_Period_Post::execute(optional_yield y) } // write the period to rados - op_ret = period.store_info(this, false, y); + op_ret = cfgstore->create_period(this, y, false, period); if (op_ret < 0) { ldpp_dout(this, -1) << "failed to store period " << period.get_id() << dendl; return; @@ -235,7 +252,7 @@ void RGWOp_Period_Post::execute(optional_yield y) return; } // reflect the period into our local objects - op_ret = period.reflect(this, y); + op_ret = rgw::reflect_period(this, y, cfgstore.get(), period); if (op_ret < 0) { ldpp_dout(this, -1) << "failed to update local objects: " << cpp_strerror(-op_ret) << dendl; diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc index c71656838df2c..b6c65a7b20bb2 100644 --- a/src/rgw/driver/rados/rgw_zone.cc +++ b/src/rgw/driver/rados/rgw_zone.cc @@ -20,13 +20,15 @@ RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){} struct RGWAccessKey; +namespace rgw { /// Generate a random uuid for realm/period/zonegroup/zone ids -static std::string gen_random_uuid() +std::string gen_random_uuid() { uuid_d uuid; uuid.generate_random(); return uuid.to_string(); } +} void RGWDefaultZoneGroupInfo::dump(Formatter *f) const { encode_json("default_zonegroup", default_zonegroup, f); @@ -704,14 +706,12 @@ int commit_period(const DoutPrefixProvider* dpp, optional_yield y, RGWPeriod& info, std::ostream& error_stream, bool force_if_stale) { - auto zone_svc = static_cast(driver)->svc()->zone; // XXX - ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id << " period " << current_period.id << dendl; // gateway must be in the master zone to commit - if (info.master_zone != zone_svc->get_zone_params().id) { + if (driver->get_zone()->get_zonegroup().is_master_zonegroup()) { error_stream << "Cannot commit period on zone " - << zone_svc->get_zone_params().id << ", it must be sent to " + << driver->get_zone()->get_id() << ", it must be sent to " "the period's master zone " << info.master_zone << '.' << std::endl; return -EINVAL; } diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h index 5fb2b4b809664..bf0d2fb968d14 100644 --- a/src/rgw/driver/rados/rgw_zone.h +++ b/src/rgw/driver/rados/rgw_zone.h @@ -660,13 +660,6 @@ public: epoch_t realm_epoch{1}; //< realm epoch when period was made current CephContext *cct{nullptr}; - RGWSI_SysObj *sysobj_svc{nullptr}; - - int read_info(const DoutPrefixProvider *dpp, optional_yield y); - int read_latest_epoch(const DoutPrefixProvider *dpp, - RGWPeriodLatestEpochInfo& epoch_info, - optional_yield y, - RGWObjVersionTracker *objv = nullptr); int use_latest_epoch(const DoutPrefixProvider *dpp, optional_yield y); int use_current_period(); @@ -724,8 +717,6 @@ public: realm_id = _realm_id; } - int reflect(const DoutPrefixProvider *dpp, optional_yield y); - int get_zonegroup(RGWZoneGroup& zonegroup, const std::string& zonegroup_id) const; @@ -756,22 +747,10 @@ public: optional_yield y) const; int get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& epoch, optional_yield y); - int set_latest_epoch(const DoutPrefixProvider *dpp, optional_yield y, - epoch_t epoch, bool exclusive = false, - RGWObjVersionTracker *objv = nullptr); // update latest_epoch if the given epoch is higher, else return -EEXIST int update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, optional_yield y); - int init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, const std::string &period_realm_id, optional_yield y, - bool setup_obj = true); - int init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, optional_yield y, bool setup_obj = true); - - int create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive = true); - int delete_obj(const DoutPrefixProvider *dpp, optional_yield y); - int store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y); - void fork(); - int update(const DoutPrefixProvider *dpp, optional_yield y); // commit a staging period; only for use on master zone int commit(const DoutPrefixProvider *dpp, @@ -1019,4 +998,6 @@ class SiteConfig { /// Test whether all zonegroups in the realm support the given zone feature. bool all_zonegroups_support(const SiteConfig& site, std::string_view feature); +std::string gen_random_uuid(); + } // namespace rgw diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc index 67ce980395133..0a6ae19e78642 100644 --- a/src/rgw/rgw_period.cc +++ b/src/rgw/rgw_period.cc @@ -2,6 +2,8 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "rgw_sync.h" +#include "rgw_sal.h" +#include "rgw_sal_config.h" using namespace std; using namespace rgw_zone_defaults; @@ -11,54 +13,6 @@ std::string period_info_oid_prefix = "periods."; #define FIRST_EPOCH 1 -int RGWPeriod::init(const DoutPrefixProvider *dpp, - CephContext *_cct, RGWSI_SysObj *_sysobj_svc, - optional_yield y, bool setup_obj) -{ - cct = _cct; - sysobj_svc = _sysobj_svc; - - if (!setup_obj) - return 0; - - if (id.empty()) { - RGWRealm realm(realm_id); - int ret = realm.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm id " << realm_id << " : " << - cpp_strerror(-ret) << dendl; - return ret; - } - id = realm.get_current_period(); - realm_id = realm.get_id(); - } - - if (!epoch) { - int ret = use_latest_epoch(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed to use_latest_epoch period id " << id << " realm id " << realm_id - << " : " << cpp_strerror(-ret) << dendl; - return ret; - } - } - - return read_info(dpp, y); -} - -int RGWPeriod::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, - const string& period_realm_id, optional_yield y, bool setup_obj) -{ - cct = _cct; - sysobj_svc = _sysobj_svc; - - realm_id = period_realm_id; - - if (!setup_obj) - return 0; - - return init(dpp, _cct, _sysobj_svc, y, setup_obj); -} - const string& RGWPeriod::get_latest_epoch_oid() const { if (cct->_conf->rgw_period_latest_epoch_info_oid.empty()) { @@ -111,126 +65,6 @@ rgw_pool RGWPeriod::get_pool(CephContext *cct) const return rgw_pool(cct->_conf->rgw_period_root_pool); } -int RGWPeriod::set_latest_epoch(const DoutPrefixProvider *dpp, - optional_yield y, - epoch_t epoch, bool exclusive, - RGWObjVersionTracker *objv) -{ - string oid = get_period_oid_prefix() + get_latest_epoch_oid(); - - rgw_pool pool(get_pool(cct)); - bufferlist bl; - - RGWPeriodLatestEpochInfo info; - info.epoch = epoch; - - using ceph::encode; - encode(info, bl); - - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); -} - -int RGWPeriod::read_info(const DoutPrefixProvider *dpp, optional_yield y) -{ - rgw_pool pool(get_pool(cct)); - - bufferlist bl; - - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_period_oid()}); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed reading obj info from " << pool << ":" << get_period_oid() << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - - try { - using ceph::decode; - auto iter = bl.cbegin(); - decode(*this, iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << get_period_oid() << dendl; - return -EIO; - } - - return 0; -} - -int RGWPeriod::store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) -{ - rgw_pool pool(get_pool(cct)); - - string oid = get_period_oid(); - bufferlist bl; - using ceph::encode; - encode(*this, bl); - - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); -} - -int RGWPeriod::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) -{ - int ret; - - /* create unique id */ - uuid_d new_uuid; - char uuid_str[37]; - new_uuid.generate_random(); - new_uuid.print(uuid_str); - id = uuid_str; - - epoch = FIRST_EPOCH; - - period_map.id = id; - - ret = store_info(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: storing info for " << id << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - - ret = set_latest_epoch(dpp, y, epoch); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: setting latest epoch " << id << ": " << cpp_strerror(-ret) << dendl; - } - - return ret; -} - -int RGWPeriod::reflect(const DoutPrefixProvider *dpp, optional_yield y) -{ - for (auto& iter : period_map.zonegroups) { - RGWZoneGroup& zg = iter.second; - zg.reinit_instance(cct, sysobj_svc); - int r = zg.write(dpp, false, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to store zonegroup info for zonegroup=" << iter.first << ": " << cpp_strerror(-r) << dendl; - return r; - } - if (zg.is_master_zonegroup()) { - // set master as default if no default exists - r = zg.set_as_default(dpp, y, true); - if (r == 0) { - ldpp_dout(dpp, 1) << "Set the period's master zonegroup " << zg.get_id() - << " as the default" << dendl; - } - } - } - - int r = period_config.write(dpp, sysobj_svc, realm_id, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to store period config: " - << cpp_strerror(-r) << dendl; - return r; - } - return 0; -} - void RGWPeriod::dump(Formatter *f) const { encode_json("id", id, f); @@ -269,7 +103,9 @@ int RGWPeriod::update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, bool exclusive = false; // read existing epoch - int r = read_latest_epoch(dpp, info, y, &objv); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int r = cfgstore->read_latest_epoch(dpp, y, id, info.epoch, &objv, *this); if (r == -ENOENT) { // use an exclusive create to set the epoch atomically exclusive = true; @@ -288,7 +124,7 @@ int RGWPeriod::update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, << " -> " << epoch << " on period=" << id << dendl; } - r = set_latest_epoch(dpp, y, epoch, exclusive, &objv); + r = cfgstore->write_latest_epoch(dpp, y, exclusive, id, epoch, &objv, *this); if (r == -EEXIST) { continue; // exclusive create raced with another update, retry } else if (r == -ECANCELED) { @@ -304,37 +140,12 @@ int RGWPeriod::update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, return -ECANCELED; // fail after max retries } -int RGWPeriod::read_latest_epoch(const DoutPrefixProvider *dpp, - RGWPeriodLatestEpochInfo& info, - optional_yield y, - RGWObjVersionTracker *objv) -{ - string oid = get_period_oid_prefix() + get_latest_epoch_oid(); - - rgw_pool pool(get_pool(cct)); - bufferlist bl; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) { - ldpp_dout(dpp, 1) << "error read_lastest_epoch " << pool << ":" << oid << dendl; - return ret; - } - try { - auto iter = bl.cbegin(); - using ceph::decode; - decode(info, iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "error decoding data from " << pool << ":" << oid << dendl; - return -EIO; - } - - return 0; -} - int RGWPeriod::use_latest_epoch(const DoutPrefixProvider *dpp, optional_yield y) { RGWPeriodLatestEpochInfo info; - int ret = read_latest_epoch(dpp, info, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int ret = cfgstore->read_latest_epoch(dpp, y, id, info.epoch, nullptr, *this); if (ret < 0) { return ret; } diff --git a/src/rgw/rgw_period_puller.cc b/src/rgw/rgw_period_puller.cc index ea2f28e567b46..10da79a9172c0 100644 --- a/src/rgw/rgw_period_puller.cc +++ b/src/rgw/rgw_period_puller.cc @@ -6,9 +6,11 @@ #include "rgw_rest_conn.h" #include "common/ceph_json.h" #include "common/errno.h" +#include "rgw_sal_config.h" #include "services/svc_zone.h" +#define FIRST_EPOCH 1 #define dout_subsys ceph_subsys_rgw #undef dout_prefix @@ -68,9 +70,12 @@ int RGWPeriodPuller::pull(const DoutPrefixProvider *dpp, const std::string& peri optional_yield y) { // try to read the period from rados + constexpr auto zero_epoch = 0; period.set_id(period_id); - period.set_epoch(0); - int r = period.init(dpp, cct, svc.sysobj, y); + period.set_epoch(zero_epoch); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int r = cfgstore->read_period(dpp, y, period_id, zero_epoch, period); if (r < 0) { if (svc.zone->is_meta_master()) { // can't pull if we're the master @@ -88,7 +93,10 @@ int RGWPeriodPuller::pull(const DoutPrefixProvider *dpp, const std::string& peri return r; } // write the period to rados - r = period.store_info(dpp, true, y); + period.period_map.id = period.id = rgw::gen_random_uuid(); + period.epoch = FIRST_EPOCH; + constexpr bool exclusive = true; + r = cfgstore->create_period(dpp, y, exclusive, period); if (r == -EEXIST) { r = 0; } else if (r < 0) { @@ -108,7 +116,7 @@ int RGWPeriodPuller::pull(const DoutPrefixProvider *dpp, const std::string& peri } // reflect period objects if this is the latest version if (svc.zone->get_realm().get_current_period() == period_id) { - r = period.reflect(dpp, y); + r = rgw::reflect_period(dpp, y, cfgstore.get(), period); if (r < 0) { return r; } diff --git a/src/rgw/rgw_period_pusher.cc b/src/rgw/rgw_period_pusher.cc index 0fc9efa85e755..4929d94ba63d6 100644 --- a/src/rgw/rgw_period_pusher.cc +++ b/src/rgw/rgw_period_pusher.cc @@ -8,6 +8,7 @@ #include "rgw_cr_rest.h" #include "rgw_zone.h" #include "rgw_sal.h" +#include "rgw_sal_config.h" #include "rgw_sal_rados.h" #include "services/svc_zone.h" @@ -174,8 +175,9 @@ RGWPeriodPusher::RGWPeriodPusher(const DoutPrefixProvider *dpp, rgw::sal::Driver // always send out the current period on startup RGWPeriod period; - // XXX dang - int r = period.init(dpp, cct, static_cast(driver)->svc()->sysobj, realm_id, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int r = cfgstore->read_period(dpp, y, zone->get_current_period_id(), std::nullopt, period); if (r < 0) { ldpp_dout(dpp, -1) << "failed to load period for realm " << realm_id << dendl; return; diff --git a/src/rgw/rgw_realm.cc b/src/rgw/rgw_realm.cc index 2d854e7244f5d..225884e9f811d 100644 --- a/src/rgw/rgw_realm.cc +++ b/src/rgw/rgw_realm.cc @@ -62,20 +62,22 @@ int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclu return ret; } RGWPeriod period; + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); if (current_period.empty()) { /* create new period for the realm */ - ret = period.init(dpp, cct, sysobj_svc, id, y, false); + ret = cfgstore->read_period(dpp, y, period.get_id(), period.get_epoch(), period); if (ret < 0 ) { return ret; } - ret = period.create(dpp, y, true); + ret = cfgstore->create_period(dpp, y, true, period); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: creating new period for realm " << name << ": " << cpp_strerror(-ret) << dendl; return ret; } } else { period = RGWPeriod(current_period, 0); - int ret = period.init(dpp, cct, sysobj_svc, id, y); + ret = cfgstore->read_period(dpp, y, period.get_id(), period.get_epoch(), period); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: failed to init period " << current_period << dendl; return ret; @@ -174,7 +176,9 @@ int RGWRealm::set_current_period(const DoutPrefixProvider *dpp, RGWPeriod& perio return ret; } - ret = period.reflect(dpp, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + ret = rgw::reflect_period(dpp, y, cfgstore.get(), period); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: period.reflect(): " << cpp_strerror(-ret) << dendl; return ret; @@ -228,7 +232,9 @@ int RGWRealm::find_zone(const DoutPrefixProvider *dpp, epoch_t epoch = 0; RGWPeriod period(period_id, epoch); - int r = period.init(dpp, cct, sysobj_svc, get_id(), y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + int r = cfgstore->read_period(dpp, y, period_id, epoch, period); if (r < 0) { ldpp_dout(dpp, 0) << "WARNING: period init failed: " << cpp_strerror(-r) << " ... skipping" << dendl; return r; diff --git a/src/rgw/rgw_sal_config.h b/src/rgw/rgw_sal_config.h index 7050940220b59..214133520c67e 100644 --- a/src/rgw/rgw_sal_config.h +++ b/src/rgw/rgw_sal_config.h @@ -28,6 +28,7 @@ struct RGWPeriodConfig; struct RGWRealm; struct RGWZoneGroup; struct RGWZoneParams; +class RGWObjVersionTracker; namespace rgw::sal { @@ -118,6 +119,10 @@ class ConfigStore { optional_yield y, const std::string& marker, std::span entries, ListResult& result) = 0; + virtual int read_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, std::string_view period_id, + uint32_t& epoch, RGWObjVersionTracker* objv, RGWPeriod& info) = 0; + virtual int write_latest_epoch(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, std::string_view period_id, + uint32_t epoch, RGWObjVersionTracker* objv, const RGWPeriod& info) = 0; ///@} /// @group ZoneGroup diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index 97d81550058e4..c7916d7f5b006 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -9,6 +9,8 @@ #include "rgw_zone.h" #include "rgw_rest_conn.h" #include "rgw_bucket_sync.h" +#include "rgw_sal.h" +#include "rgw_sal_config.h" #include "common/errno.h" #include "include/random.h" @@ -140,8 +142,11 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp) return ret; } + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); ldpp_dout(dpp, 20) << "realm " << realm->get_name() << " " << realm->get_id() << dendl; - ret = current_period->init(dpp, cct, sysobj_svc, realm->get_id(), y); + current_period->set_realm_id(realm->get_id()); + ret = cfgstore->read_period(dpp, y, current_period->get_id(), current_period->epoch, *current_period); if (ret < 0 && ret != -ENOENT) { ldpp_dout(dpp, 0) << "failed reading current period info: " << " " << cpp_strerror(-ret) << dendl; return ret; @@ -455,7 +460,9 @@ int RGWSI_Zone::list_periods(const DoutPrefixProvider *dpp, const string& curren string period_id = current_period; while(!period_id.empty()) { RGWPeriod period(period_id); - ret = period.init(dpp, cct, sysobj_svc, y); + auto config_store_type = g_conf().get_val("rgw_config_store"); + auto cfgstore = DriverManager::create_config_store(dpp, config_store_type); + ret = cfgstore->read_period(dpp, y, period_id, std::nullopt, period); if (ret < 0) { return ret; } -- 2.39.5