From: Ali Masarwa Date: Tue, 6 May 2025 08:17:48 +0000 (+0300) Subject: RGW/standalone: Build without RadosStore X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F61850%2Fhead;p=ceph.git RGW/standalone: Build without RadosStore Signed-off-by: Ali Masarwa --- diff --git a/CMakeLists.txt b/CMakeLists.txt index 912dfee79d59..5fdd4222245c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -553,6 +553,7 @@ option(WITH_RADOSGW_MOTR "CORTX-Motr backend for RADOS Gateway" OFF) option(WITH_RADOSGW_DAOS "DAOS backend for RADOS Gateway" OFF) option(WITH_RADOSGW_D4N "D4N wrapper for RADOS Gateway" ON) option(WITH_RADOSGW_POSIX "POSIX backend for RADOS Gateway" ON) +option(WITH_RADOSGW_RADOS "RADOS backend for Rados Gateway" ON) option(WITH_RADOSGW_SELECT_PARQUET "Support for s3 select on parquet objects" ON) option(WITH_RADOSGW_ARROW_FLIGHT "Build arrow flight when not using system-provided arrow" OFF) option(WITH_RADOSGW_BACKTRACE_LOGGING "Enable backtraces in rgw logs" OFF) diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index bc85aac22ee9..25182499cd95 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -366,6 +366,9 @@ /* Backend POSIX for Rados Gateway */ #cmakedefine WITH_RADOSGW_POSIX +/* Backend RADOS for Rados Gateway */ +#cmakedefine WITH_RADOSGW_RADOS + /* Defined if std::map::merge() is supported */ #cmakedefine HAVE_STDLIB_MAP_SPLICING diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index f235f1879369..9fc368f45f24 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -36,26 +36,6 @@ endfunction() find_package(ICU 52.0 COMPONENTS uc REQUIRED) set(librgw_common_srcs - services/svc_finisher.cc - services/svc_bi_rados.cc - services/svc_bilog_rados.cc - services/svc_bucket.cc - services/svc_bucket_sobj.cc - services/svc_bucket_sync_sobj.cc - services/svc_cls.cc - services/svc_config_key_rados.cc - services/svc_mdlog.cc - services/svc_notify.cc - services/svc_quota.cc - services/svc_sync_modules.cc - services/svc_sys_obj.cc - services/svc_sys_obj_cache.cc - services/svc_sys_obj_core.cc - services/svc_tier_rados.cc - services/svc_user.cc - services/svc_user_rados.cc - services/svc_zone.cc - services/svc_zone_utils.cc spdk/crc64.c madler/crc64nvme.c madler/crc32iso_hdlc.c @@ -166,76 +146,97 @@ set(librgw_common_srcs rgw_dedup_cluster.cc rgw_data_access.cc rgw_realm_watcher.cc - driver/rados/account.cc - driver/rados/buckets.cc rgw_bucket_logging.cc rgw_rest_bucket_logging.cc - driver/rados/group.cc - driver/rados/groups.cc - driver/rados/rgw_bucket.cc - driver/rados/rgw_bucket_sync.cc - driver/rados/rgw_cr_rados.cc - driver/rados/rgw_cr_tools.cc - driver/rados/rgw_d3n_datacache.cc - driver/rados/rgw_datalog.cc - driver/rados/rgw_datalog_notify.cc - driver/rados/rgw_data_sync.cc - driver/rados/rgw_etag_verifier.cc - driver/rados/rgw_gc.cc - driver/rados/rgw_gc_log.cc - driver/rados/rgw_lc_tier.cc - driver/rados/rgw_log_backing.cc - driver/rados/rgw_metadata.cc - driver/rados/rgw_notify.cc - driver/rados/rgw_obj_manifest.cc - driver/rados/rgw_object_expirer_core.cc - driver/rados/rgw_otp.cc - driver/rados/rgw_period.cc - driver/rados/rgw_pubsub_push.cc - driver/rados/rgw_putobj_processor.cc - driver/rados/rgw_rados.cc - driver/rados/rgw_reshard.cc - driver/rados/rgw_rest_bucket.cc - driver/rados/rgw_rest_log.cc - driver/rados/rgw_rest_realm.cc - driver/rados/rgw_rest_user.cc - driver/rados/rgw_sal_rados.cc - driver/rados/rgw_service.cc - driver/rados/rgw_sync.cc - driver/rados/rgw_sync_counters.cc - driver/rados/rgw_sync_error_repo.cc - driver/rados/rgw_sync_module.cc - driver/rados/rgw_sync_module_aws.cc - driver/rados/rgw_sync_module_es.cc - driver/rados/rgw_sync_module_es_rest.cc - driver/rados/rgw_sync_module_log.cc - driver/rados/rgw_sync_trace.cc - driver/rados/rgw_tools.cc - driver/rados/rgw_trim_bilog.cc - driver/rados/rgw_trim_datalog.cc - driver/rados/rgw_trim_mdlog.cc - driver/rados/rgw_user.cc - driver/rados/rgw_zone.cc - driver/rados/role.cc - driver/rados/roles.cc - driver/rados/sync_fairness.cc - driver/rados/topic.cc - driver/rados/topic_migration.cc - driver/rados/topics.cc - driver/rados/users.cc) + rgw_bucket_sync.cc) list(APPEND librgw_common_srcs driver/immutable_config/store.cc - driver/json_config/store.cc - driver/rados/config/impl.cc - driver/rados/config/period.cc - driver/rados/config/period_config.cc - driver/rados/config/realm.cc - driver/rados/config/realm_watcher.cc - driver/rados/config/store.cc - driver/rados/config/zone.cc - driver/rados/config/zonegroup.cc) + driver/json_config/store.cc) +if(WITH_RADOSGW_RADOS) + list(APPEND librgw_common_srcs + services/svc_finisher.cc + services/svc_bi_rados.cc + services/svc_bilog_rados.cc + services/svc_bucket.cc + services/svc_bucket_sobj.cc + services/svc_bucket_sync_sobj.cc + services/svc_cls.cc + services/svc_config_key_rados.cc + services/svc_mdlog.cc + services/svc_notify.cc + services/svc_quota.cc + services/svc_sync_modules.cc + services/svc_sys_obj.cc + services/svc_sys_obj_cache.cc + services/svc_sys_obj_core.cc + services/svc_tier_rados.cc + services/svc_user.cc + services/svc_user_rados.cc + services/svc_zone.cc + services/svc_zone_utils.cc + driver/rados/account.cc + driver/rados/buckets.cc + driver/rados/group.cc + driver/rados/groups.cc + driver/rados/rgw_bucket.cc + driver/rados/rgw_cr_rados.cc + driver/rados/rgw_cr_tools.cc + driver/rados/rgw_d3n_datacache.cc + driver/rados/rgw_datalog.cc + driver/rados/rgw_datalog_notify.cc + driver/rados/rgw_data_sync.cc + driver/rados/rgw_etag_verifier.cc + driver/rados/rgw_gc.cc + driver/rados/rgw_gc_log.cc + driver/rados/rgw_lc_tier.cc + driver/rados/rgw_log_backing.cc + driver/rados/rgw_metadata.cc + driver/rados/rgw_notify.cc + driver/rados/rgw_obj_manifest.cc + driver/rados/rgw_object_expirer_core.cc + driver/rados/rgw_otp.cc + driver/rados/rgw_pubsub_push.cc + driver/rados/rgw_putobj_processor.cc + driver/rados/rgw_rados.cc + driver/rados/rgw_reshard.cc + driver/rados/rgw_rest_bucket.cc + driver/rados/rgw_rest_log.cc + driver/rados/rgw_rest_realm.cc + driver/rados/rgw_rest_user.cc + driver/rados/rgw_sal_rados.cc + driver/rados/rgw_service.cc + driver/rados/rgw_sync.cc + driver/rados/rgw_sync_counters.cc + driver/rados/rgw_sync_error_repo.cc + driver/rados/rgw_sync_module.cc + driver/rados/rgw_sync_module_aws.cc + driver/rados/rgw_sync_module_es.cc + driver/rados/rgw_sync_module_es_rest.cc + driver/rados/rgw_sync_module_log.cc + driver/rados/rgw_sync_trace.cc + driver/rados/rgw_tools.cc + driver/rados/rgw_trim_bilog.cc + driver/rados/rgw_trim_datalog.cc + driver/rados/rgw_trim_mdlog.cc + driver/rados/rgw_user.cc + driver/rados/role.cc + driver/rados/roles.cc + driver/rados/sync_fairness.cc + driver/rados/topic.cc + driver/rados/topic_migration.cc + driver/rados/topics.cc + driver/rados/users.cc + driver/rados/config/impl.cc + driver/rados/config/period.cc + driver/rados/config/period_config.cc + driver/rados/config/realm.cc + driver/rados/config/realm_watcher.cc + driver/rados/config/store.cc + driver/rados/config/zone.cc + driver/rados/config/zonegroup.cc) +endif() if(WITH_RADOSGW_AMQP_ENDPOINT) list(APPEND librgw_common_srcs rgw_amqp.cc) endif() @@ -412,10 +413,13 @@ set(rgw_a_srcs rgw_signal.cc rgw_swift_auth.cc rgw_usage.cc - rgw_sts.cc - driver/rados/rgw_rest_bucket.cc - driver/rados/rgw_rest_log.cc - driver/rados/rgw_rest_realm.cc) + rgw_sts.cc) + +if(WITH_RADOSGW_RADOS) + list(APPEND rgw_a_srcs driver/rados/rgw_rest_bucket.cc + driver/rados/rgw_rest_log.cc + driver/rados/rgw_rest_realm.cc) +endif() gperf_generate(${CMAKE_SOURCE_DIR}/src/rgw/rgw_iam_policy_keywords.gperf rgw_iam_policy_keywords.frag.cc) @@ -512,6 +516,7 @@ if(WITH_RADOSGW_ARROW_FLIGHT) list(APPEND radosgw_admin_srcs rgw_flight.cc) endif(WITH_RADOSGW_ARROW_FLIGHT) +if(WITH_RADOSGW_RADOS) add_executable(radosgw-admin ${radosgw_admin_srcs}) target_link_libraries(radosgw-admin legacy-option-headers @@ -522,6 +527,7 @@ target_link_libraries(radosgw-admin global ${LIB_RESOLV} OATH::OATH ${CURL_LIBRARIES} ${EXPAT_LIBRARIES} ${BLKID_LIBRARIES}) +endif() # this is unsatisfying and hopefully temporary; ARROW should not be # part of radosgw_admin @@ -529,7 +535,9 @@ if(WITH_RADOSGW_ARROW_FLIGHT) target_link_libraries(radosgw-admin ${ARROW_LIBRARIES} ${ARROW_FLIGHT_LIBRARIES}) endif(WITH_RADOSGW_ARROW_FLIGHT) +if(WITH_RADOSGW_RADOS) install(TARGETS radosgw-admin DESTINATION bin) +endif() set(radosgw_es_srcs rgw_es_main.cc) @@ -551,6 +559,7 @@ target_link_libraries(radosgw-token global) install(TARGETS radosgw-token DESTINATION bin) +if(WITH_RADOSGW_RADOS) set(radosgw_object_expirer_srcs rgw_object_expirer.cc) add_executable(radosgw-object-expirer ${radosgw_object_expirer_srcs}) @@ -561,6 +570,7 @@ target_link_libraries(radosgw-object-expirer ${rgw_libs} librados global ${LIB_RESOLV} ${CURL_LIBRARIES} ${EXPAT_LIBRARIES}) install(TARGETS radosgw-object-expirer DESTINATION bin) +endif() set(radosgw_polparser_srcs rgw_polparser.cc) diff --git a/src/rgw/driver/dbstore/config/sqlite.cc b/src/rgw/driver/dbstore/config/sqlite.cc index 68ae8f0f239a..d80f0cdbf8f9 100644 --- a/src/rgw/driver/dbstore/config/sqlite.cc +++ b/src/rgw/driver/dbstore/config/sqlite.cc @@ -27,7 +27,7 @@ #include "common/random_string.h" #include "rgw_realm_watcher.h" -#include "driver/rados/rgw_zone.h" // FIXME: subclass dependency +#include "rgw/rgw_zone.h" #include "common/connection_pool.h" #include "sqlite/connection.h" diff --git a/src/rgw/driver/rados/rgw_bucket_sync.cc b/src/rgw/driver/rados/rgw_bucket_sync.cc deleted file mode 100644 index 1e7316d4271d..000000000000 --- a/src/rgw/driver/rados/rgw_bucket_sync.cc +++ /dev/null @@ -1,1022 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#include "rgw_common.h" -#include "rgw_bucket_sync.h" -#include "rgw_data_sync.h" -#include "rgw_zone.h" - -#include "services/svc_zone.h" -#include "services/svc_bucket_sync.h" - -#define dout_subsys ceph_subsys_rgw - -using namespace std; - -ostream& operator<<(ostream& os, const rgw_sync_bucket_entity& e) { - os << "{b=" << rgw_sync_bucket_entities::bucket_key(e.bucket) << ",z=" << e.zone.value_or(rgw_zone_id()) << ",az=" << (int)e.all_zones << "}"; - return os; -} - -ostream& operator<<(ostream& os, const rgw_sync_bucket_pipe& pipe) { - os << "{s=" << pipe.source << ",d=" << pipe.dest << "}"; - return os; -} - -ostream& operator<<(ostream& os, const rgw_sync_bucket_entities& e) { - os << "{b=" << rgw_sync_bucket_entities::bucket_key(e.bucket) << ",z=" << e.zones.value_or(std::set()) << "}"; - return os; -} - -ostream& operator<<(ostream& os, const rgw_sync_bucket_pipes& pipe) { - os << "{id=" << pipe.id << ",s=" << pipe.source << ",d=" << pipe.dest << "}"; - return os; -} - -static std::vector filter_relevant_pipes(const std::vector& pipes, - const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone) -{ - std::vector relevant_pipes; - for (auto& p : pipes) { - if (p.source.match_zone(source_zone) && - p.dest.match_zone(dest_zone)) { - for (auto pipe : p.expand()) { - pipe.source.apply_zone(source_zone); - pipe.dest.apply_zone(dest_zone); - relevant_pipes.push_back(pipe); - } - } - } - - return relevant_pipes; -} - -static bool is_wildcard_bucket(const rgw_bucket& bucket) -{ - return bucket.name.empty(); -} - -void rgw_sync_group_pipe_map::dump(ceph::Formatter *f) const -{ - encode_json("zone", zone.id, f); - encode_json("buckets", rgw_sync_bucket_entities::bucket_key(bucket), f); - encode_json("sources", sources, f); - encode_json("dests", dests, f); -} - - -template -void rgw_sync_group_pipe_map::try_add_to_pipe_map(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - zb_pipe_map_t *pipe_map, - CB1 filter_cb, - CB2 call_filter_cb) -{ - if (!filter_cb(source_zone, nullopt, dest_zone, nullopt)) { - return; - } - auto relevant_pipes = filter_relevant_pipes(pipes, source_zone, dest_zone); - - for (auto& pipe : relevant_pipes) { - rgw_sync_bucket_entity zb; - if (!call_filter_cb(pipe, &zb)) { - continue; - } - pipe_map->insert(make_pair(zb, pipe)); - } -} - -template -void rgw_sync_group_pipe_map::try_add_source(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - CB filter_cb) -{ - return try_add_to_pipe_map(source_zone, dest_zone, pipes, - &sources, - filter_cb, - [&](const rgw_sync_bucket_pipe& pipe, rgw_sync_bucket_entity *zb) { - *zb = rgw_sync_bucket_entity{source_zone, pipe.source.get_bucket()}; - return filter_cb(source_zone, zb->bucket, dest_zone, pipe.dest.get_bucket()); - }); -} - -template -void rgw_sync_group_pipe_map::try_add_dest(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - CB filter_cb) -{ - return try_add_to_pipe_map(source_zone, dest_zone, pipes, - &dests, - filter_cb, - [&](const rgw_sync_bucket_pipe& pipe, rgw_sync_bucket_entity *zb) { - *zb = rgw_sync_bucket_entity{dest_zone, pipe.dest.get_bucket()}; - return filter_cb(source_zone, pipe.source.get_bucket(), dest_zone, zb->bucket); - }); -} - -using zb_pipe_map_t = rgw_sync_group_pipe_map::zb_pipe_map_t; - -pair rgw_sync_group_pipe_map::find_pipes(const zb_pipe_map_t& m, - const rgw_zone_id& zone, - std::optional b) const -{ - if (!b) { - return m.equal_range(rgw_sync_bucket_entity{zone, rgw_bucket()}); - } - - auto zb = rgw_sync_bucket_entity{zone, *b}; - - auto range = m.equal_range(zb); - if (range.first == range.second && - !is_wildcard_bucket(*b)) { - /* couldn't find the specific bucket, try to find by wildcard */ - zb.bucket = rgw_bucket(); - range = m.equal_range(zb); - } - - return range; -} - - -template -void rgw_sync_group_pipe_map::init(const DoutPrefixProvider *dpp, - CephContext *cct, - const rgw_zone_id& _zone, - std::optional _bucket, - const rgw_sync_policy_group& group, - rgw_sync_data_flow_group *_default_flow, - std::set *_pall_zones, - CB filter_cb) { - zone = _zone; - bucket = _bucket; - default_flow = _default_flow; - pall_zones = _pall_zones; - - rgw_sync_bucket_entity zb(zone, bucket); - - status = group.status; - - std::vector zone_pipes; - - string bucket_key = (bucket ? bucket->get_key() : "*"); - - /* only look at pipes that touch the specific zone and bucket */ - for (auto& pipe : group.pipes) { - if (pipe.contains_zone_bucket(zone, bucket)) { - ldpp_dout(dpp, 20) << __func__ << "(): pipe_map (zone=" << zone << " bucket=" << bucket_key << "): adding potential pipe: " << pipe << dendl; - zone_pipes.push_back(pipe); - } - } - - const rgw_sync_data_flow_group *pflow; - - if (!group.data_flow.empty()) { - pflow = &group.data_flow; - } else { - if (!default_flow) { - return; - } - pflow = default_flow; - } - - auto& flow = *pflow; - - pall_zones->insert(zone); - - /* symmetrical */ - for (auto& symmetrical_group : flow.symmetrical) { - if (symmetrical_group.zones.find(zone) != symmetrical_group.zones.end()) { - for (auto& z : symmetrical_group.zones) { - if (z != zone) { - pall_zones->insert(z); - try_add_source(z, zone, zone_pipes, filter_cb); - try_add_dest(zone, z, zone_pipes, filter_cb); - } - } - } - } - - /* directional */ - for (auto& rule : flow.directional) { - if (rule.source_zone == zone) { - pall_zones->insert(rule.dest_zone); - try_add_dest(zone, rule.dest_zone, zone_pipes, filter_cb); - } else if (rule.dest_zone == zone) { - pall_zones->insert(rule.source_zone); - try_add_source(rule.source_zone, zone, zone_pipes, filter_cb); - } - } -} - -/* - * find all relevant pipes in our zone that match {dest_bucket} <- {source_zone, source_bucket} - */ -vector rgw_sync_group_pipe_map::find_source_pipes(const rgw_zone_id& source_zone, - std::optional source_bucket, - std::optional dest_bucket) const { - vector result; - - auto range = find_pipes(sources, source_zone, source_bucket); - - for (auto iter = range.first; iter != range.second; ++iter) { - auto pipe = iter->second; - if (pipe.dest.match_bucket(dest_bucket)) { - result.push_back(pipe); - } - } - return result; -} - -/* - * find all relevant pipes in other zones that pull from a specific - * source bucket in out zone {source_bucket} -> {dest_zone, dest_bucket} - */ -vector rgw_sync_group_pipe_map::find_dest_pipes(std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket) const { - vector result; - - auto range = find_pipes(dests, dest_zone, dest_bucket); - - for (auto iter = range.first; iter != range.second; ++iter) { - auto pipe = iter->second; - if (pipe.source.match_bucket(source_bucket)) { - result.push_back(pipe); - } - } - - return result; -} - -/* - * find all relevant pipes from {source_zone, source_bucket} -> {dest_zone, dest_bucket} - */ -vector rgw_sync_group_pipe_map::find_pipes(const rgw_zone_id& source_zone, - std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket) const { - if (dest_zone == zone) { - return find_source_pipes(source_zone, source_bucket, dest_bucket); - } - - if (source_zone == zone) { - return find_dest_pipes(source_bucket, dest_zone, dest_bucket); - } - - return vector(); -} - -void RGWBucketSyncFlowManager::pipe_rules::insert(const rgw_sync_bucket_pipe& pipe) -{ - pipes.push_back(pipe); - - auto ppipe = &pipes.back(); - auto prefix = ppipe->params.source.filter.prefix.value_or(string()); - - prefix_refs.insert(make_pair(prefix, ppipe)); - - for (auto& t : ppipe->params.source.filter.tags) { - string tag = t.key + "=" + t.value; - auto titer = tag_refs.find(tag); - if (titer != tag_refs.end() && - ppipe->params.priority > titer->second->params.priority) { - titer->second = ppipe; - } else { - tag_refs[tag] = ppipe; - } - } -} - -bool RGWBucketSyncFlowManager::pipe_rules::find_basic_info_without_tags(const rgw_obj_key& key, - std::optional *user, - std::optional *acl_translation_owner, - std::optional *storage_class, - rgw_sync_pipe_params::Mode *mode, - bool *need_more_info) const -{ - std::optional owner; - - *need_more_info = false; - - if (prefix_refs.empty()) { - return false; - } - - auto end = prefix_refs.upper_bound(key.name); - auto iter = end; - if (iter != prefix_refs.begin()) { - --iter; - } - if (iter == prefix_refs.end()) { - return false; - } - - if (iter != prefix_refs.begin()) { - iter = prefix_refs.find(iter->first); /* prefix_refs is multimap, find first element - holding that key */ - } - - std::vector iters; - - std::optional priority; - - for (; iter != end; ++iter) { - auto& prefix = iter->first; - if (!boost::starts_with(key.name, prefix)) { - continue; - } - - auto& rule_params = iter->second->params; - auto& filter = rule_params.source.filter; - - if (rule_params.priority > priority) { - priority = rule_params.priority; - - if (!filter.has_tags()) { - iters.clear(); - } - iters.push_back(iter); - - *need_more_info = filter.has_tags(); /* if highest priority filter has tags, then - we can't be sure if it would be used. - We need to first read the info from the source object */ - } - } - - if (iters.empty()) { - return false; - } - - std::optional _user; - std::optional _acl_translation; - std::optional _storage_class; - rgw_sync_pipe_params::Mode _mode{rgw_sync_pipe_params::Mode::MODE_SYSTEM}; - - // make sure all params are the same by saving the first one - // encountered and comparing all subsequent to it - bool first_iter = true; - for (auto& iter : iters) { - const rgw_sync_pipe_params& rule_params = iter->second->params; - if (first_iter) { - _user = rule_params.user; - _acl_translation = rule_params.dest.acl_translation; - _storage_class = rule_params.dest.storage_class; - _mode = rule_params.mode; - first_iter = false; - } else { - // note: three of these == operators are comparing std::optional - // against std::optional; as one would expect they are equal a) - // if both do not contain values or b) if both do and those - // contained values are the same - const bool conflict = - !(_user == rule_params.user && - _acl_translation == rule_params.dest.acl_translation && - _storage_class == rule_params.dest.storage_class && - _mode == rule_params.mode); - if (conflict) { - *need_more_info = true; - return false; - } - } - } - - *user = _user; - if (_acl_translation) { - *acl_translation_owner = _acl_translation->owner; - } - *storage_class = _storage_class; - *mode = _mode; - - return true; -} - -bool RGWBucketSyncFlowManager::pipe_rules::find_obj_params(const rgw_obj_key& key, - const RGWObjTags::tag_map_t& tags, - rgw_sync_pipe_params *params) const -{ - if (prefix_refs.empty()) { - return false; - } - - auto iter = prefix_refs.upper_bound(key.name); - if (iter != prefix_refs.begin()) { - --iter; - } - if (iter == prefix_refs.end()) { - return false; - } - - auto end = prefix_refs.upper_bound(key.name); - auto max = end; - - std::optional priority; - - for (; iter != end; ++iter) { - /* NOTE: this is not the most efficient way to do it, - * a trie data structure would be better - */ - auto& prefix = iter->first; - if (!boost::starts_with(key.name, prefix)) { - continue; - } - - auto& rule_params = iter->second->params; - auto& filter = rule_params.source.filter; - - if (!filter.check_tags(tags)) { - continue; - } - - if (rule_params.priority > priority) { - priority = rule_params.priority; - max = iter; - } - } - - if (max == end) { - return false; - } - - *params = max->second->params; - return true; -} - -/* - * return either the current prefix for s, or the next one if s is not within a prefix - */ - -RGWBucketSyncFlowManager::pipe_rules::prefix_map_t::const_iterator RGWBucketSyncFlowManager::pipe_rules::prefix_search(const std::string& s) const -{ - if (prefix_refs.empty()) { - return prefix_refs.end(); - } - auto next = prefix_refs.upper_bound(s); - auto iter = next; - if (iter != prefix_refs.begin()) { - --iter; - } - if (!boost::starts_with(s, iter->first)) { - return next; - } - - return iter; -} - -void RGWBucketSyncFlowManager::pipe_set::insert(const rgw_sync_bucket_pipe& pipe) { - /* Ensure this pipe doesn't match with any disabled pipes */ - for (auto p: disabled_pipe_map) { - if (p.second.source.match(pipe.source) && p.second.dest.match(pipe.dest)) { - return; - } - } - pipe_map.insert(make_pair(pipe.id, pipe)); - - auto& rules_ref = rules[endpoints_pair(pipe)]; - - if (!rules_ref) { - rules_ref = make_shared(); - } - - rules_ref->insert(pipe); - - pipe_handler h(rules_ref, pipe); - - handlers.insert(h); -} - -void RGWBucketSyncFlowManager::pipe_set::remove_all() { - pipe_map.clear(); - disabled_pipe_map.clear(); - rules.clear(); - handlers.clear(); -} - -void RGWBucketSyncFlowManager::pipe_set::disable(const rgw_sync_bucket_pipe& pipe) { - /* This pipe is disabled. Add it to disabled pipes & remove any - * matching pipes already inserted - */ - disabled_pipe_map.insert(make_pair(pipe.id, pipe)); - for (auto iter_p = pipe_map.begin(); iter_p != pipe_map.end(); ) { - auto p = iter_p++; - if (p->second.source.match(pipe.source) && p->second.dest.match(pipe.dest)) { - auto& rules_ref = rules[endpoints_pair(p->second)]; - if (rules_ref) { - pipe_handler h(rules_ref, p->second); - handlers.erase(h); - } - rules.erase(endpoints_pair(p->second)); - pipe_map.erase(p); - } - } -} - -void RGWBucketSyncFlowManager::pipe_set::dump(ceph::Formatter *f) const -{ - encode_json("pipes", pipe_map, f); -} - -bool RGWBucketSyncFlowManager::allowed_data_flow(const rgw_zone_id& source_zone, - std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket, - bool check_activated) const -{ - bool found = false; - bool found_activated = false; - - for (auto m : flow_groups) { - auto& fm = m.second; - auto pipes = fm.find_pipes(source_zone, source_bucket, - dest_zone, dest_bucket); - - bool is_found = !pipes.empty(); - - if (is_found) { - switch (fm.status) { - case rgw_sync_policy_group::Status::FORBIDDEN: - return false; - case rgw_sync_policy_group::Status::ENABLED: - found = true; - found_activated = true; - break; - case rgw_sync_policy_group::Status::ALLOWED: - found = true; - break; - default: - break; /* unknown -- ignore */ - } - } - } - - if (check_activated && found_activated) { - return true; - } - - return found; -} - -void RGWBucketSyncFlowManager::init(const DoutPrefixProvider *dpp, const rgw_sync_policy_info& sync_policy) { - std::optional default_flow; - if (parent) { - default_flow.emplace(); - default_flow->init_default(parent->all_zones); - } - - for (auto& item : sync_policy.groups) { - auto& group = item.second; - auto& flow_group_map = flow_groups[group.id]; - - flow_group_map.init(dpp, cct, zone_id, bucket, group, - (default_flow ? &(*default_flow) : nullptr), - &all_zones, - [&](const rgw_zone_id& source_zone, - std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket) { - if (!parent) { - return true; - } - return parent->allowed_data_flow(source_zone, - source_bucket, - dest_zone, - dest_bucket, - false); /* just check that it's not disabled */ - }); - } -} - -/* -* These are the semantics to be followed while resolving the policy -* conflicts - -* -* ================================================== -* zonegroup bucket Result -* ================================================== -* enabled enabled enabled -* allowed enabled -* forbidden disabled -* allowed enabled enabled -* allowed disabled -* forbidden disabled -* forbidden enabled disabled -* allowed disabled -* forbidden disabled -* -* In case multiple group policies are set to reflect for any sync pair -* (, ), the following -* rules are applied in the order- -* 1) Even if one policy status is FORBIDDEN, the sync will be disabled -* 2) Atleast one policy should be ENABLED for the sync to be allowed. -* -*/ -void RGWBucketSyncFlowManager::reflect(const DoutPrefixProvider *dpp, - std::optional effective_bucket, - RGWBucketSyncFlowManager::pipe_set *source_pipes, - RGWBucketSyncFlowManager::pipe_set *dest_pipes, - bool only_enabled) const - -{ - string effective_bucket_key; - bool is_forbidden = false; - if (effective_bucket) { - effective_bucket_key = effective_bucket->get_key(); - } - if (parent) { - parent->reflect(dpp, effective_bucket, source_pipes, dest_pipes, only_enabled); - } - - for (auto& item : flow_groups) { - auto& flow_group_map = item.second; - is_forbidden = false; - - if (flow_group_map.status == rgw_sync_policy_group::Status::FORBIDDEN) { - /* FORBIDDEN takes precedence over all the other rules. - * Remove any other pipes which may allow access. - */ - is_forbidden = true; - } else if (flow_group_map.status != rgw_sync_policy_group::Status::ENABLED && - (only_enabled || flow_group_map.status != rgw_sync_policy_group::Status::ALLOWED)) { - /* only return enabled groups */ - continue; - } - - for (auto& entry : flow_group_map.sources) { - rgw_sync_bucket_pipe pipe = entry.second; - if (!pipe.dest.match_bucket(effective_bucket)) { - continue; - } - - pipe.source.apply_bucket(effective_bucket); - pipe.dest.apply_bucket(effective_bucket); - - if (is_forbidden) { - ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): removing source pipe: " << pipe << dendl; - source_pipes->disable(pipe); - } else { - ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): adding source pipe: " << pipe << dendl; - source_pipes->insert(pipe); - } - } - - for (auto& entry : flow_group_map.dests) { - rgw_sync_bucket_pipe pipe = entry.second; - - if (!pipe.source.match_bucket(effective_bucket)) { - continue; - } - - pipe.source.apply_bucket(effective_bucket); - pipe.dest.apply_bucket(effective_bucket); - - if (is_forbidden) { - ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): removing dest pipe: " << pipe << dendl; - dest_pipes->disable(pipe); - } else { - ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): adding dest pipe: " << pipe << dendl; - dest_pipes->insert(pipe); - } - } - } -} - - -RGWBucketSyncFlowManager::RGWBucketSyncFlowManager(CephContext *_cct, - const rgw_zone_id& _zone_id, - std::optional _bucket, - const RGWBucketSyncFlowManager *_parent) : cct(_cct), - zone_id(_zone_id), - bucket(_bucket), - parent(_parent) {} - - -void RGWSyncPolicyCompat::convert_old_sync_config(RGWSI_Zone *zone_svc, - RGWSI_SyncModules *sync_modules_svc, - rgw_sync_policy_info *ppolicy) -{ - bool found = false; - - rgw_sync_policy_info policy; - - auto& group = policy.groups["default"]; - auto& zonegroup = zone_svc->get_zonegroup(); - - for (const auto& ziter1 : zonegroup.zones) { - auto& id1 = ziter1.first; - const RGWZone& z1 = ziter1.second; - - for (const auto& ziter2 : zonegroup.zones) { - auto& id2 = ziter2.first; - const RGWZone& z2 = ziter2.second; - - if (id1 == id2) { - continue; - } - - if (z1.syncs_from(z2.name)) { - found = true; - rgw_sync_directional_rule *rule; - group.data_flow.find_or_create_directional(id2, - id1, - &rule); - } - } - } - - if (!found) { /* nothing syncs */ - return; - } - - rgw_sync_bucket_pipes pipes; - pipes.id = "all"; - pipes.source.all_zones = true; - pipes.dest.all_zones = true; - - group.pipes.emplace_back(std::move(pipes)); - - - group.status = rgw_sync_policy_group::Status::ENABLED; - - *ppolicy = std::move(policy); -} - -RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(RGWSI_Zone *_zone_svc, - RGWSI_SyncModules *sync_modules_svc, - RGWSI_Bucket_Sync *_bucket_sync_svc, - std::optional effective_zone) : zone_svc(_zone_svc) , - bucket_sync_svc(_bucket_sync_svc) { - zone_id = effective_zone.value_or(zone_svc->zone_id()); - flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), - zone_id, - nullopt, - nullptr)); - sync_policy = zone_svc->get_zonegroup().sync_policy; - - if (sync_policy.empty()) { - RGWSyncPolicyCompat::convert_old_sync_config(zone_svc, sync_modules_svc, &sync_policy); - legacy_config = true; - } -} - -RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, - const RGWBucketInfo& _bucket_info, - map&& _bucket_attrs) : parent(_parent), - bucket_info(_bucket_info), - bucket_attrs(std::move(_bucket_attrs)) { - if (_bucket_info.sync_policy) { - sync_policy = *_bucket_info.sync_policy; - } - legacy_config = parent->legacy_config; - bucket = _bucket_info.bucket; - zone_svc = parent->zone_svc; - bucket_sync_svc = parent->bucket_sync_svc; - flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), - parent->zone_id, - _bucket_info.bucket, - parent->flow_mgr.get())); -} - -RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, - const rgw_bucket& _bucket, - std::optional _sync_policy) : parent(_parent) { - if (_sync_policy) { - sync_policy = *_sync_policy; - } - legacy_config = parent->legacy_config; - bucket = _bucket; - zone_svc = parent->zone_svc; - bucket_sync_svc = parent->bucket_sync_svc; - flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), - parent->zone_id, - _bucket, - parent->flow_mgr.get())); -} - -RGWBucketSyncPolicyHandler *RGWBucketSyncPolicyHandler::alloc_child(const RGWBucketInfo& bucket_info, - map&& bucket_attrs) const -{ - return new RGWBucketSyncPolicyHandler(this, bucket_info, std::move(bucket_attrs)); -} - -RGWBucketSyncPolicyHandler *RGWBucketSyncPolicyHandler::alloc_child(const rgw_bucket& bucket, - std::optional sync_policy) const -{ - return new RGWBucketSyncPolicyHandler(this, bucket, sync_policy); -} - -int RGWBucketSyncPolicyHandler::init(const DoutPrefixProvider *dpp, optional_yield y) -{ - int r = bucket_sync_svc->get_bucket_sync_hints(dpp, bucket.value_or(rgw_bucket()), - &source_hints, - &target_hints, - y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to initialize bucket sync policy handler: get_bucket_sync_hints() on bucket=" - << bucket << " returned r=" << r << dendl; - return r; - } - - flow_mgr->init(dpp, sync_policy); - - reflect(dpp, &source_pipes, - &target_pipes, - &sources, - &targets, - &source_zones, - &target_zones, - true); - - return 0; -} - -void RGWBucketSyncPolicyHandler::reflect(const DoutPrefixProvider *dpp, RGWBucketSyncFlowManager::pipe_set *psource_pipes, - RGWBucketSyncFlowManager::pipe_set *ptarget_pipes, - map *psources, - map *ptargets, - std::set *psource_zones, - std::set *ptarget_zones, - bool only_enabled) const -{ - RGWBucketSyncFlowManager::pipe_set _source_pipes; - RGWBucketSyncFlowManager::pipe_set _target_pipes; - map _sources; - map _targets; - std::set _source_zones; - std::set _target_zones; - - flow_mgr->reflect(dpp, bucket, &_source_pipes, &_target_pipes, only_enabled); - - for (auto& entry : _source_pipes.pipe_map) { - auto& pipe = entry.second; - if (!pipe.source.zone) { - continue; - } - _source_zones.insert(*pipe.source.zone); - _sources[*pipe.source.zone].insert(pipe); - } - - for (auto& entry : _target_pipes.pipe_map) { - auto& pipe = entry.second; - if (!pipe.dest.zone) { - continue; - } - _target_zones.insert(*pipe.dest.zone); - _targets[*pipe.dest.zone].insert(pipe); - } - - if (psource_pipes) { - *psource_pipes = std::move(_source_pipes); - } - if (ptarget_pipes) { - *ptarget_pipes = std::move(_target_pipes); - } - if (psources) { - *psources = std::move(_sources); - } - if (ptargets) { - *ptargets = std::move(_targets); - } - if (psource_zones) { - *psource_zones = std::move(_source_zones); - } - if (ptarget_zones) { - *ptarget_zones = std::move(_target_zones); - } -} - -multimap RGWBucketSyncPolicyHandler::get_all_sources() const -{ - multimap m; - - for (auto& source_entry : sources) { - auto& zone_id = source_entry.first; - - auto& pipes = source_entry.second.pipe_map; - - for (auto& entry : pipes) { - auto& pipe = entry.second; - m.insert(make_pair(zone_id, pipe)); - } - } - - for (auto& pipe : resolved_sources) { - if (!pipe.source.zone) { - continue; - } - - m.insert(make_pair(*pipe.source.zone, pipe)); - } - - return m; -} - -multimap RGWBucketSyncPolicyHandler::get_all_dests() const -{ - multimap m; - - for (auto& dest_entry : targets) { - auto& zone_id = dest_entry.first; - - auto& pipes = dest_entry.second.pipe_map; - - for (auto& entry : pipes) { - auto& pipe = entry.second; - m.insert(make_pair(zone_id, pipe)); - } - } - - for (auto& pipe : resolved_dests) { - if (!pipe.dest.zone) { - continue; - } - - m.insert(make_pair(*pipe.dest.zone, pipe)); - } - - return m; -} - -multimap RGWBucketSyncPolicyHandler::get_all_dests_in_zone(const rgw_zone_id& zone_id) const -{ - multimap m; - - auto iter = targets.find(zone_id); - if (iter != targets.end()) { - auto& pipes = iter->second.pipe_map; - - for (auto& entry : pipes) { - auto& pipe = entry.second; - m.insert(make_pair(zone_id, pipe)); - } - } - - for (auto& pipe : resolved_dests) { - if (!pipe.dest.zone || - *pipe.dest.zone != zone_id) { - continue; - } - - m.insert(make_pair(*pipe.dest.zone, pipe)); - } - - return m; -} - -void RGWBucketSyncPolicyHandler::get_pipes(std::set *_sources, std::set *_targets, - std::optional filter_peer) { /* return raw pipes */ - for (auto& entry : source_pipes.pipe_map) { - auto& source_pipe = entry.second; - if (!filter_peer || - source_pipe.source.match(*filter_peer)) { - _sources->insert(source_pipe); - } - } - - for (auto& entry : target_pipes.pipe_map) { - auto& target_pipe = entry.second; - if (!filter_peer || - target_pipe.dest.match(*filter_peer)) { - _targets->insert(target_pipe); - } - } -} - -bool RGWBucketSyncPolicyHandler::bucket_exports_object(const std::string& obj_name, const RGWObjTags& tags) const { - if (bucket_exports_data()) { - for (auto& entry : target_pipes.pipe_map) { - auto& filter = entry.second.params.source.filter; - if (filter.check_prefix(obj_name) && filter.check_tags(tags.get_tags())) { - return true; - } - } - } - - return false; -} - -bool RGWBucketSyncPolicyHandler::bucket_exports_data() const -{ - if (!bucket) { - return false; - } - - if (!zone_svc->sync_module_exports_data()) { - return false; - } - - if (bucket_is_sync_source()) { - return true; - } - - return (zone_svc->need_to_log_data() && - bucket_info->datasync_flag_enabled()); -} - -bool RGWBucketSyncPolicyHandler::bucket_imports_data() const -{ - return bucket_is_sync_target(); -} - diff --git a/src/rgw/driver/rados/rgw_bucket_sync.h b/src/rgw/driver/rados/rgw_bucket_sync.h deleted file mode 100644 index d41bb4f3ee12..000000000000 --- a/src/rgw/driver/rados/rgw_bucket_sync.h +++ /dev/null @@ -1,445 +0,0 @@ - -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab - -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2018 Red Hat, Inc. - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#pragma once - -#include "rgw_common.h" -#include "rgw_sync_policy.h" - -class RGWSI_Zone; -class RGWSI_SyncModules; -class RGWSI_Bucket_Sync; - -struct rgw_sync_group_pipe_map; -struct rgw_sync_bucket_pipes; -struct rgw_sync_policy_info; - -struct rgw_sync_group_pipe_map { - rgw_zone_id zone; - std::optional bucket; - - rgw_sync_policy_group::Status status{rgw_sync_policy_group::Status::UNKNOWN}; - - using zb_pipe_map_t = std::multimap; - - zb_pipe_map_t sources; /* all the pipes where zone is pulling from */ - zb_pipe_map_t dests; /* all the pipes that pull from zone */ - - std::set *pall_zones{nullptr}; - rgw_sync_data_flow_group *default_flow{nullptr}; /* flow to use if policy doesn't define it, - used in the case of bucket sync policy, not at the - zonegroup level */ - - void dump(ceph::Formatter *f) const; - - template - void try_add_to_pipe_map(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - zb_pipe_map_t *pipe_map, - CB1 filter_cb, - CB2 call_filter_cb); - - template - void try_add_source(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - CB filter_cb); - - template - void try_add_dest(const rgw_zone_id& source_zone, - const rgw_zone_id& dest_zone, - const std::vector& pipes, - CB filter_cb); - - std::pair find_pipes(const zb_pipe_map_t& m, - const rgw_zone_id& zone, - std::optional b) const; - - template - void init(const DoutPrefixProvider *dpp, CephContext *cct, - const rgw_zone_id& _zone, - std::optional _bucket, - const rgw_sync_policy_group& group, - rgw_sync_data_flow_group *_default_flow, - std::set *_pall_zones, - CB filter_cb); - - /* - * find all relevant pipes in our zone that match {dest_bucket} <- {source_zone, source_bucket} - */ - std::vector find_source_pipes(const rgw_zone_id& source_zone, - std::optional source_bucket, - std::optional dest_bucket) const; - - /* - * find all relevant pipes in other zones that pull from a specific - * source bucket in out zone {source_bucket} -> {dest_zone, dest_bucket} - */ - std::vector find_dest_pipes(std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket) const; - - /* - * find all relevant pipes from {source_zone, source_bucket} -> {dest_zone, dest_bucket} - */ - std::vector find_pipes(const rgw_zone_id& source_zone, - std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket) const; -}; - -class RGWSyncPolicyCompat { -public: - static void convert_old_sync_config(RGWSI_Zone *zone_svc, - RGWSI_SyncModules *sync_modules_svc, - rgw_sync_policy_info *ppolicy); -}; - -class RGWBucketSyncFlowManager { - friend class RGWBucketSyncPolicyHandler; -public: - struct endpoints_pair { - rgw_sync_bucket_entity source; - rgw_sync_bucket_entity dest; - - endpoints_pair() {} - endpoints_pair(const rgw_sync_bucket_pipe& pipe) { - source = pipe.source; - dest = pipe.dest; - } - - bool operator<(const endpoints_pair& e) const { - if (source < e.source) { - return true; - } - if (e.source < source) { - return false; - } - return (dest < e.dest); - } - }; - - /* - * pipe_rules: deal with a set of pipes that have common endpoints_pair - */ - class pipe_rules { - std::list pipes; - - public: - using prefix_map_t = std::multimap; - - std::map tag_refs; - prefix_map_t prefix_refs; - - void insert(const rgw_sync_bucket_pipe& pipe); - - bool find_basic_info_without_tags(const rgw_obj_key& key, - std::optional *user, - std::optional *acl_translation, - std::optional *storage_class, - rgw_sync_pipe_params::Mode *mode, - bool *need_more_info) const; - bool find_obj_params(const rgw_obj_key& key, - const RGWObjTags::tag_map_t& tags, - rgw_sync_pipe_params *params) const; - - void scan_prefixes(std::vector *prefixes) const; - - prefix_map_t::const_iterator prefix_begin() const { - return prefix_refs.begin(); - } - prefix_map_t::const_iterator prefix_search(const std::string& s) const; - prefix_map_t::const_iterator prefix_end() const { - return prefix_refs.end(); - } - }; - - using pipe_rules_ref = std::shared_ptr; - - /* - * pipe_handler: extends endpoints_rule to point at the corresponding rules handler - */ - struct pipe_handler : public endpoints_pair { - pipe_rules_ref rules; - - pipe_handler() {} - pipe_handler(pipe_rules_ref& _rules, - const rgw_sync_bucket_pipe& _pipe) : endpoints_pair(_pipe), - rules(_rules) {} - bool specific() const { - return source.specific() && dest.specific(); - } - - bool find_basic_info_without_tags(const rgw_obj_key& key, - std::optional *user, - std::optional *acl_translation, - std::optional *storage_class, - rgw_sync_pipe_params::Mode *mode, - bool *need_more_info) const { - if (!rules) { - return false; - } - return rules->find_basic_info_without_tags(key, user, acl_translation, storage_class, mode, need_more_info); - } - - bool find_obj_params(const rgw_obj_key& key, - const RGWObjTags::tag_map_t& tags, - rgw_sync_pipe_params *params) const { - if (!rules) { - return false; - } - return rules->find_obj_params(key, tags, params); - } - }; - - struct pipe_set { - std::map rules; - std::multimap pipe_map; - std::multimap disabled_pipe_map; - - std::set handlers; - - using iterator = std::set::iterator; - - void clear() { - rules.clear(); - pipe_map.clear(); - disabled_pipe_map.clear(); - handlers.clear(); - } - - void insert(const rgw_sync_bucket_pipe& pipe); - void remove_all(); - void disable(const rgw_sync_bucket_pipe& pipe); - - iterator begin() const { - return handlers.begin(); - } - - iterator end() const { - return handlers.end(); - } - - void dump(ceph::Formatter *f) const; - }; - -private: - - CephContext *cct; - - rgw_zone_id zone_id; - std::optional bucket; - - const RGWBucketSyncFlowManager *parent{nullptr}; - - std::map flow_groups; - - std::set all_zones; - - bool allowed_data_flow(const rgw_zone_id& source_zone, - std::optional source_bucket, - const rgw_zone_id& dest_zone, - std::optional dest_bucket, - bool check_activated) const; - - /* - * find all the matching flows om a flow map for a specific bucket - */ - void update_flow_maps(const rgw_sync_bucket_pipes& pipe); - - void init(const DoutPrefixProvider *dpp, const rgw_sync_policy_info& sync_policy); - -public: - - RGWBucketSyncFlowManager(CephContext *_cct, - const rgw_zone_id& _zone_id, - std::optional _bucket, - const RGWBucketSyncFlowManager *_parent); - - void reflect(const DoutPrefixProvider *dpp, std::optional effective_bucket, - pipe_set *flow_by_source, - pipe_set *flow_by_dest, - bool only_enabled) const; - -}; - -static inline std::ostream& operator<<(std::ostream& os, const RGWBucketSyncFlowManager::endpoints_pair& e) { - os << e.dest << " -> " << e.source; - return os; -} - -class RGWBucketSyncPolicyHandler { - bool legacy_config{false}; - const RGWBucketSyncPolicyHandler *parent{nullptr}; - RGWSI_Zone *zone_svc; - RGWSI_Bucket_Sync *bucket_sync_svc; - rgw_zone_id zone_id; - std::optional bucket_info; - std::optional > bucket_attrs; - std::optional bucket; - std::unique_ptr flow_mgr; - rgw_sync_policy_info sync_policy; - - RGWBucketSyncFlowManager::pipe_set source_pipes; - RGWBucketSyncFlowManager::pipe_set target_pipes; - - std::map sources; /* source pipes by source zone id */ - std::map targets; /* target pipes by target zone id */ - - std::set source_zones; - std::set target_zones; - - std::set source_hints; - std::set target_hints; - std::set resolved_sources; - std::set resolved_dests; - - - bool bucket_is_sync_source() const { - return !targets.empty() || !resolved_dests.empty(); - } - - bool bucket_is_sync_target() const { - return !sources.empty() || !resolved_sources.empty(); - } - - RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, - const RGWBucketInfo& _bucket_info, - std::map&& _bucket_attrs); - - RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, - const rgw_bucket& _bucket, - std::optional _sync_policy); -public: - RGWBucketSyncPolicyHandler(RGWSI_Zone *_zone_svc, - RGWSI_SyncModules *sync_modules_svc, - RGWSI_Bucket_Sync *bucket_sync_svc, - std::optional effective_zone = std::nullopt); - - RGWBucketSyncPolicyHandler *alloc_child(const RGWBucketInfo& bucket_info, - std::map&& bucket_attrs) const; - RGWBucketSyncPolicyHandler *alloc_child(const rgw_bucket& bucket, - std::optional sync_policy) const; - - int init(const DoutPrefixProvider *dpp, optional_yield y); - - void reflect(const DoutPrefixProvider *dpp, RGWBucketSyncFlowManager::pipe_set *psource_pipes, - RGWBucketSyncFlowManager::pipe_set *ptarget_pipes, - std::map *psources, - std::map *ptargets, - std::set *psource_zones, - std::set *ptarget_zones, - bool only_enabled) const; - - void set_resolved_hints(std::set&& _resolved_sources, - std::set&& _resolved_dests) { - resolved_sources = std::move(_resolved_sources); - resolved_dests = std::move(_resolved_dests); - } - - const std::set& get_resolved_source_hints() { - return resolved_sources; - } - - const std::set& get_resolved_dest_hints() { - return resolved_dests; - } - - const std::set& get_source_zones() const { - return source_zones; - } - - const std::set& get_target_zones() const { - return target_zones; - } - - const std::map& get_sources() { - return sources; - } - - std::multimap get_all_sources() const; - std::multimap get_all_dests() const; - std::multimap get_all_dests_in_zone(const rgw_zone_id& zone_id) const; - - const std::map& get_targets() { - return targets; - } - - const std::optional& get_bucket_info() const { - return bucket_info; - } - - const std::optional >& get_bucket_attrs() const { - return bucket_attrs; - } - - void get_pipes(RGWBucketSyncFlowManager::pipe_set **_sources, RGWBucketSyncFlowManager::pipe_set **_targets) { /* return raw pipes (with zone name) */ - *_sources = &source_pipes; - *_targets = &target_pipes; - } - void get_pipes(std::set *sources, std::set *targets, - std::optional filter_peer); - - const std::set& get_source_hints() const { - return source_hints; - } - - const std::set& get_target_hints() const { - return target_hints; - } - - bool bucket_exports_object(const std::string& obj_name, const RGWObjTags& tags) const; - bool bucket_exports_data() const; - bool bucket_imports_data() const; - - const rgw_sync_policy_info& get_sync_policy() const { - return sync_policy; - } - - bool is_legacy_config() const { - return legacy_config; - } -}; - -struct rgw_bucket_sync_pair_info { - RGWBucketSyncFlowManager::pipe_handler handler; /* responsible for sync filters */ - rgw_bucket_shard source_bs; - rgw_bucket dest_bucket; -}; - -inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pair_info& p) { - if (p.source_bs.bucket == p.dest_bucket) { - return out << p.source_bs; - } - return out << p.source_bs << "->" << p.dest_bucket; -} - -struct rgw_bucket_sync_pipe { - rgw_bucket_sync_pair_info info; - RGWBucketInfo source_bucket_info; - std::map source_bucket_attrs; - RGWBucketInfo dest_bucket_info; - std::map dest_bucket_attrs; - - RGWBucketSyncFlowManager::pipe_rules_ref& get_rules() { - return info.handler.rules; - } -}; - -inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pipe& p) { - return out << p.info; -} diff --git a/src/rgw/driver/rados/rgw_period.cc b/src/rgw/driver/rados/rgw_period.cc deleted file mode 100644 index eb57c3439180..000000000000 --- a/src/rgw/driver/rados/rgw_period.cc +++ /dev/null @@ -1,104 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#include "rgw_sync.h" -#include "rgw_sal.h" -#include "rgw_sal_config.h" - -#include "services/svc_zone.h" - -#define dout_subsys ceph_subsys_rgw - -using namespace std; -using namespace rgw_zone_defaults; - -int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup, - const string& zonegroup_id) const -{ - map::const_iterator iter; - if (!zonegroup_id.empty()) { - iter = period_map.zonegroups.find(zonegroup_id); - } else { - iter = period_map.zonegroups.find("default"); - } - if (iter != period_map.zonegroups.end()) { - zonegroup = iter->second; - return 0; - } - - return -ENOENT; -} - -static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw_meta_sync_status *sync_status) -{ - rgw::sal::RadosStore* rados_store = static_cast(driver); - // initialize a sync status manager to read the status - RGWMetaSyncStatusManager mgr(rados_store, rados_store->svc()->async_processor); - int r = mgr.init(dpp); - if (r < 0) { - return r; - } - r = mgr.read_sync_status(dpp, sync_status); - mgr.stop(); - return r; -} - -int RGWPeriod::update_sync_status(const DoutPrefixProvider *dpp, - rgw::sal::Driver* driver, /* for now */ - const RGWPeriod ¤t_period, - std::ostream& error_stream, - bool force_if_stale) -{ - rgw_meta_sync_status status; - int r = read_sync_status(dpp, driver, &status); - if (r < 0) { - ldpp_dout(dpp, 0) << "period failed to read sync status: " - << cpp_strerror(-r) << dendl; - return r; - } - - std::vector markers; - - const auto current_epoch = current_period.get_realm_epoch(); - if (current_epoch != status.sync_info.realm_epoch) { - // no sync status markers for the current period - ceph_assert(current_epoch > status.sync_info.realm_epoch); - const int behind = current_epoch - status.sync_info.realm_epoch; - if (!force_if_stale && current_epoch > 1) { - error_stream << "ERROR: This zone is " << behind << " period(s) behind " - "the current master zone in metadata sync. If this zone is promoted " - "to master, any metadata changes during that time are likely to " - "be lost.\n" - "Waiting for this zone to catch up on metadata sync (see " - "'radosgw-admin sync status') is recommended.\n" - "To promote this zone to master anyway, add the flag " - "--yes-i-really-mean-it." << std::endl; - return -EINVAL; - } - // empty sync status markers - other zones will skip this period during - // incremental metadata sync - markers.resize(status.sync_info.num_shards); - } else { - markers.reserve(status.sync_info.num_shards); - for (auto& i : status.sync_markers) { - auto& marker = i.second; - // filter out markers from other periods - if (marker.realm_epoch != current_epoch) { - marker.marker.clear(); - } - markers.emplace_back(std::move(marker.marker)); - } - } - - std::swap(sync_status, markers); - return 0; -} - -void RGWPeriod::generate_test_instances(list &o) -{ - RGWPeriod *z = new RGWPeriod; - o.push_back(z); - o.push_back(new RGWPeriod); -} - - diff --git a/src/rgw/driver/rados/rgw_rest_realm.cc b/src/rgw/driver/rados/rgw_rest_realm.cc index 34234a04221c..7b941733ae89 100644 --- a/src/rgw/driver/rados/rgw_rest_realm.cc +++ b/src/rgw/driver/rados/rgw_rest_realm.cc @@ -141,7 +141,7 @@ void RGWOp_Period_Post::execute(optional_yield y) // if period id is empty, handle as 'period commit' if (period.get_id().empty()) { - op_ret = rgw::commit_period(this, y, s->penv.cfgstore, driver, realm, *realm_writer, current_period, period, error_stream, false); + op_ret = rgw::commit_period(this, y, s->penv.cfgstore, driver, realm, *realm_writer, current_period, period, error_stream, false, *s->penv.site); if (op_ret == -EEXIST) { op_ret = 0; // succeed on retries so the op is idempotent return; diff --git a/src/rgw/driver/rados/rgw_user.cc b/src/rgw/driver/rados/rgw_user.cc index cce593c6bd50..d90f2de32835 100644 --- a/src/rgw/driver/rados/rgw_user.cc +++ b/src/rgw/driver/rados/rgw_user.cc @@ -32,33 +32,6 @@ static string key_type_to_str(int key_type) { } } -static bool char_is_unreserved_url(char c) -{ - if (isalnum(c)) - return true; - - switch (c) { - case '-': - case '.': - case '_': - case '~': - return true; - default: - return false; - } -} - -static bool validate_access_key(string& key) -{ - const char *p = key.c_str(); - while (*p) { - if (!char_is_unreserved_url(*p)) - return false; - p++; - } - return true; -} - static void set_err_msg(std::string *sink, std::string msg) { if (sink && !msg.empty()) @@ -502,41 +475,6 @@ int RGWAccessKeyPool::check_op(RGWUserAdminOpState& op_state, return 0; } -void rgw_generate_secret_key(CephContext* cct, - std::string& secret_key) -{ - char secret_key_buf[SECRET_KEY_LEN + 1]; - gen_rand_alphanumeric_plain(cct, secret_key_buf, sizeof(secret_key_buf)); - secret_key = secret_key_buf; -} - -int rgw_generate_access_key(const DoutPrefixProvider* dpp, - optional_yield y, - rgw::sal::Driver* driver, - std::string& access_key_id) -{ - std::string id; - int r = 0; - - do { - id.resize(PUBLIC_ID_LEN + 1); - gen_rand_alphanumeric_upper(dpp->get_cct(), id.data(), id.size()); - id.pop_back(); // remove trailing null - - if (!validate_access_key(id)) - continue; - - std::unique_ptr duplicate_check; - r = driver->get_user_by_access_key(dpp, id, y, &duplicate_check); - } while (r == 0); - - if (r == -ENOENT) { - access_key_id = std::move(id); - return 0; - } - return r; -} - // Generate a new random key int RGWAccessKeyPool::generate_key(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) diff --git a/src/rgw/driver/rados/rgw_zone.cc b/src/rgw/driver/rados/rgw_zone.cc deleted file mode 100644 index 804a47ad03d0..000000000000 --- a/src/rgw/driver/rados/rgw_zone.cc +++ /dev/null @@ -1,1334 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#include "rgw_zone.h" -#include "rgw_realm_watcher.h" -#include "rgw_sal_config.h" -#include "rgw_sync.h" - -#include "services/svc_zone.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_rgw - -using namespace std; -using namespace rgw_zone_defaults; - -RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){} - -#define FIRST_EPOCH 1 - -struct RGWAccessKey; - -namespace rgw { -/// Generate a random uuid for realm/period/zonegroup/zone ids -std::string gen_random_uuid() -{ - uuid_d uuid; - uuid.generate_random(); - return uuid.to_string(); -} -} - -void RGWDefaultZoneGroupInfo::dump(Formatter *f) const { - encode_json("default_zonegroup", default_zonegroup, f); -} - -void RGWDefaultZoneGroupInfo::decode_json(JSONObj *obj) { - - JSONDecoder::decode_json("default_zonegroup", default_zonegroup, obj); - /* backward compatability with region */ - if (default_zonegroup.empty()) { - JSONDecoder::decode_json("default_region", default_zonegroup, obj); - } -} - -int RGWZoneGroup::equals(const string& other_zonegroup) const -{ - if (is_master && other_zonegroup.empty()) - return true; - - return (id == other_zonegroup); -} - -void RGWDefaultSystemMetaObjInfo::dump(Formatter *f) const { - encode_json("default_id", default_id, f); -} - -void RGWDefaultSystemMetaObjInfo::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("default_id", default_id, obj); -} - -const string& RGWZoneParams::get_compression_type(const rgw_placement_rule& placement_rule) const -{ - static const std::string NONE{"none"}; - auto p = placement_pools.find(placement_rule.name); - if (p == placement_pools.end()) { - return NONE; - } - const auto& type = p->second.get_compression_type(placement_rule.get_storage_class()); - return !type.empty() ? type : NONE; -} - -// run an MD5 hash on the zone_id and return the first 32 bits -static uint32_t gen_short_zone_id(const std::string zone_id) -{ - unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; - MD5 hash; - // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes - hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); - hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size()); - hash.Final(md5); - - uint32_t short_id; - memcpy((char *)&short_id, md5, sizeof(short_id)); - return std::max(short_id, 1u); -} - -int RGWPeriodMap::update(const RGWZoneGroup& zonegroup, CephContext *cct) -{ - if (zonegroup.is_master_zonegroup() && (!master_zonegroup.empty() && zonegroup.get_id() != master_zonegroup)) { - ldout(cct,0) << "Error updating periodmap, multiple master zonegroups configured "<< dendl; - ldout(cct,0) << "master zonegroup: " << master_zonegroup << " and " << zonegroup.get_id() <::iterator iter = zonegroups.find(zonegroup.get_id()); - if (iter != zonegroups.end()) { - RGWZoneGroup& old_zonegroup = iter->second; - if (!old_zonegroup.api_name.empty()) { - zonegroups_by_api.erase(old_zonegroup.api_name); - } - } - zonegroups[zonegroup.get_id()] = zonegroup; - - if (!zonegroup.api_name.empty()) { - zonegroups_by_api[zonegroup.api_name] = zonegroup; - } - - if (zonegroup.is_master_zonegroup()) { - master_zonegroup = zonegroup.get_id(); - } else if (master_zonegroup == zonegroup.get_id()) { - master_zonegroup = ""; - } - - for (auto& i : zonegroup.zones) { - auto& zone = i.second; - if (short_zone_ids.find(zone.id) != short_zone_ids.end()) { - continue; - } - // calculate the zone's short id - uint32_t short_id = gen_short_zone_id(zone.id); - - // search for an existing zone with the same short id - for (auto& s : short_zone_ids) { - if (s.second == short_id) { - ldout(cct, 0) << "New zone '" << zone.name << "' (" << zone.id - << ") generates the same short_zone_id " << short_id - << " as existing zone id " << s.first << dendl; - return -EEXIST; - } - } - - short_zone_ids[zone.id] = short_id; - } - - return 0; -} - -uint32_t RGWPeriodMap::get_zone_short_id(const string& zone_id) const -{ - auto i = short_zone_ids.find(zone_id); - if (i == short_zone_ids.end()) { - return 0; - } - return i->second; -} - -bool RGWPeriodMap::find_zone_by_name(const string& zone_name, - RGWZoneGroup *zonegroup, - RGWZone *zone) const -{ - for (auto& iter : zonegroups) { - auto& zg = iter.second; - for (auto& ziter : zg.zones) { - auto& z = ziter.second; - - if (z.name == zone_name) { - *zonegroup = zg; - *zone = z; - return true; - } - } - } - - return false; -} - -namespace rgw { - -int read_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view realm_id, - std::string_view realm_name, - RGWRealm& info, - std::unique_ptr* writer) -{ - if (!realm_id.empty()) { - return cfgstore->read_realm_by_id(dpp, y, realm_id, info, writer); - } - if (!realm_name.empty()) { - return cfgstore->read_realm_by_name(dpp, y, realm_name, info, writer); - } - return cfgstore->read_default_realm(dpp, y, info, writer); -} - -int create_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWRealm& info, - std::unique_ptr* writer_out) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a realm name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // if the realm already has a current_period, just make sure it exists - std::optional period; - if (!info.current_period.empty()) { - period.emplace(); - int r = cfgstore->read_period(dpp, y, info.current_period, - std::nullopt, *period); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to read realm's current_period=" - << info.current_period << " with " << cpp_strerror(r) << dendl; - return r; - } - } - - // create the realm - std::unique_ptr writer; - int r = cfgstore->create_realm(dpp, y, exclusive, info, &writer); - if (r < 0) { - return r; - } - - if (!period) { - // initialize and exclusive-create the initial period - period.emplace(); - period->id = gen_random_uuid(); - period->period_map.id = period->id; - period->epoch = FIRST_EPOCH; - period->realm_id = info.id; - - r = cfgstore->create_period(dpp, y, true, *period); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to create the initial period id=" - << period->id << " for realm " << info.name - << " with " << cpp_strerror(r) << dendl; - return r; - } - } - - // update the realm's current_period - r = realm_set_current_period(dpp, y, cfgstore, *writer, info, *period); - if (r < 0) { - return r; - } - - if (writer_out) { - *writer_out = std::move(writer); - } - return 0; -} - -int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWRealm& info, - bool exclusive) -{ - return cfgstore->write_default_realm_id(dpp, y, exclusive, info.id); -} - -int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - sal::RealmWriter& writer, RGWRealm& realm, - const RGWPeriod& period) -{ - // update realm epoch to match the period's - if (realm.epoch > period.realm_epoch) { - ldpp_dout(dpp, -1) << __func__ << " with old realm epoch " - << period.realm_epoch << ", current epoch=" << realm.epoch << dendl; - return -EINVAL; - } - if (realm.epoch == period.realm_epoch && realm.current_period != period.id) { - ldpp_dout(dpp, -1) << __func__ << " with same realm epoch " - << period.realm_epoch << ", but different period id " - << period.id << " != " << realm.current_period << dendl; - return -EINVAL; - } - - realm.epoch = period.realm_epoch; - realm.current_period = period.id; - - // update the realm object - int r = writer.write(dpp, y, realm); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to overwrite realm " - << realm.name << " with " << cpp_strerror(r) << dendl; - return r; - } - - // reflect the zonegroup and period config - (void) reflect_period(dpp, y, cfgstore, period); - return 0; -} - -int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWPeriod& info) -{ - // overwrite the local period config and zonegroup objects - constexpr bool exclusive = false; - - int r = cfgstore->write_period_config(dpp, y, exclusive, info.realm_id, - info.period_config); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to store period config for realm id=" - << info.realm_id << " with " << cpp_strerror(r) << dendl; - return r; - } - - for (auto& [zonegroup_id, zonegroup] : info.period_map.zonegroups) { - r = cfgstore->create_zonegroup(dpp, y, exclusive, zonegroup, nullptr); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to store zonegroup id=" - << zonegroup_id << " with " << cpp_strerror(r) << dendl; - return r; - } - if (zonegroup.is_master) { - // set master as default if no default exists - constexpr bool exclusive = true; - r = set_default_zonegroup(dpp, y, cfgstore, zonegroup, exclusive); - if (r == 0) { - ldpp_dout(dpp, 1) << "Set the period's master zonegroup " - << zonegroup.name << " as the default" << dendl; - } - } - } - return 0; -} - -std::string get_staging_period_id(std::string_view realm_id) -{ - return string_cat_reserve(realm_id, ":staging"); -} - -void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info) -{ - ldpp_dout(dpp, 20) << __func__ << " realm id=" << info.realm_id - << " period id=" << info.id << dendl; - - info.predecessor_uuid = std::move(info.id); - info.id = get_staging_period_id(info.realm_id); - info.period_map.reset(); - info.realm_epoch++; -} - -int update_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, RGWPeriod& info) -{ - // clear zone short ids of removed zones. period_map.update() will add the - // remaining zones back - info.period_map.short_zone_ids.clear(); - - // list all zonegroups in the realm - rgw::sal::ListResult listing; - std::array zonegroup_names; // list in pages of 1000 - do { - int ret = cfgstore->list_zonegroup_names(dpp, y, listing.next, - zonegroup_names, listing); - if (ret < 0) { - std::cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl; - return -ret; - } - for (const auto& name : listing.entries) { - RGWZoneGroup zg; - ret = cfgstore->read_zonegroup_by_name(dpp, y, name, zg, nullptr); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to read zonegroup " - << name << ": " << cpp_strerror(-ret) << dendl; - continue; - } - - if (zg.realm_id != info.realm_id) { - ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() - << " with realm id " << zg.realm_id - << ", not on our realm " << info.realm_id << dendl; - continue; - } - - if (zg.master_zone.empty()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; - return -EINVAL; - } - - if (zg.zones.find(zg.master_zone) == zg.zones.end()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() - << " has a non existent master zone "<< dendl; - return -EINVAL; - } - - if (zg.is_master_zonegroup()) { - info.master_zonegroup = zg.get_id(); - info.master_zone = zg.master_zone; - } - - ret = info.period_map.update(zg, dpp->get_cct()); - if (ret < 0) { - return ret; - } - } // foreach name in listing.entries - } while (!listing.next.empty()); - - // read the realm's current period config - int ret = cfgstore->read_period_config(dpp, y, info.realm_id, - info.period_config); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " - << cpp_strerror(ret) << dendl; - return ret; - } - - return 0; -} - -int commit_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, sal::Driver* driver, - RGWRealm& realm, sal::RealmWriter& realm_writer, - const RGWPeriod& current_period, - RGWPeriod& info, std::ostream& error_stream, - bool force_if_stale) -{ - ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id - << " period " << current_period.id << dendl; - auto zone_svc = static_cast(driver)->svc()->zone; // XXX - - // gateway must be in the master zone to commit - if (info.master_zone != zone_svc->get_zone_params().id) { - error_stream << "Cannot commit period on zone " - << zone_svc->get_zone_params().id << ", it must be sent to " - "the period's master zone " << info.master_zone << '.' << std::endl; - return -EINVAL; - } - // period predecessor must match current period - if (info.predecessor_uuid != current_period.id) { - error_stream << "Period predecessor " << info.predecessor_uuid - << " does not match current period " << current_period.id - << ". Use 'period pull' to get the latest period from the master, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; - } - // realm epoch must be 1 greater than current period - if (info.realm_epoch != current_period.realm_epoch + 1) { - error_stream << "Period's realm epoch " << info.realm_epoch - << " does not come directly after current realm epoch " - << current_period.realm_epoch << ". Use 'realm pull' to get the " - "latest realm and period from the master zone, reapply your changes, " - "and try again." << std::endl; - return -EINVAL; - } - // did the master zone change? - if (info.master_zone != current_period.master_zone) { - // store the current metadata sync status in the period - int r = info.update_sync_status(dpp, driver, current_period, - error_stream, force_if_stale); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update metadata sync status: " - << cpp_strerror(-r) << dendl; - return r; - } - // create an object with a new period id - info.period_map.id = info.id = gen_random_uuid(); - info.epoch = FIRST_EPOCH; - - constexpr bool exclusive = true; - r = cfgstore->create_period(dpp, y, exclusive, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; - return r; - } - // set as current period - r = realm_set_current_period(dpp, y, cfgstore, realm_writer, realm, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update realm's current period: " - << cpp_strerror(-r) << dendl; - return r; - } - ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " - << info.id << dendl; - return 0; - } - // period must be based on current epoch - if (info.epoch != current_period.epoch) { - error_stream << "Period epoch " << info.epoch << " does not match " - "predecessor epoch " << current_period.epoch << ". Use " - "'period pull' to get the latest epoch from the master zone, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; - } - // set period as next epoch - info.id = current_period.id; - info.epoch = current_period.epoch + 1; - info.predecessor_uuid = current_period.predecessor_uuid; - info.realm_epoch = current_period.realm_epoch; - // write the period - constexpr bool exclusive = true; - int r = cfgstore->create_period(dpp, y, exclusive, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(r) << dendl; - return r; - } - r = reflect_period(dpp, y, cfgstore, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(r) << dendl; - return r; - } - ldpp_dout(dpp, 4) << "Committed new epoch " << info.epoch - << " for period " << info.id << dendl; - return 0; -} - - -int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zonegroup_id, - std::string_view zonegroup_name, - RGWZoneGroup& info, - std::unique_ptr* writer) -{ - if (!zonegroup_id.empty()) { - return cfgstore->read_zonegroup_by_id(dpp, y, zonegroup_id, info, writer); - } - if (!zonegroup_name.empty()) { - return cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, info, writer); - } - - std::string realm_id; - int r = cfgstore->read_default_realm_id(dpp, y, realm_id); - if (r == -ENOENT) { - return cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, - info, writer); - } - if (r < 0) { - return r; - } - return cfgstore->read_default_zonegroup(dpp, y, realm_id, info, writer); -} - -int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneGroup& info) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zonegroup name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // insert the default placement target if it doesn't exist - constexpr std::string_view default_placement_name = "default-placement"; - - RGWZoneGroupPlacementTarget placement_target; - placement_target.name = default_placement_name; - - info.placement_targets.emplace(default_placement_name, placement_target); - if (info.default_placement.name.empty()) { - info.default_placement.name = default_placement_name; - } - - int r = cfgstore->create_zonegroup(dpp, y, exclusive, info, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create zonegroup with " - << cpp_strerror(r) << dendl; - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_default_zonegroup(dpp, y, cfgstore, info, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set zonegroup as default: " - << cpp_strerror(r) << dendl; - } - - return 0; -} - -static int create_default_zonegroup(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - bool exclusive, - const RGWZoneParams& default_zone, - RGWZoneGroup& info) -{ - info.name = default_zonegroup_name; - info.api_name = default_zonegroup_name; - info.is_master = true; - - // enable all supported features - info.enabled_features.insert(rgw::zone_features::enabled.begin(), - rgw::zone_features::enabled.end()); - - // add the zone to the zonegroup - bool is_master = true; - std::list empty_list; - rgw::zone_features::set disable_features; // empty - int r = add_zone_to_group(dpp, info, default_zone, &is_master, nullptr, - empty_list, nullptr, nullptr, empty_list, - empty_list, nullptr, std::nullopt, - info.enabled_features, disable_features); - if (r < 0) { - return r; - } - - // write the zone - return create_zonegroup(dpp, y, cfgstore, exclusive, info); -} - -int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneGroup& info, - bool exclusive) -{ - return cfgstore->write_default_zonegroup_id( - dpp, y, exclusive, info.realm_id, info.id); -} - -int remove_zone_from_group(const DoutPrefixProvider* dpp, - RGWZoneGroup& zonegroup, - const rgw_zone_id& zone_id) -{ - auto z = zonegroup.zones.find(zone_id); - if (z == zonegroup.zones.end()) { - return -ENOENT; - } - zonegroup.zones.erase(z); - - if (zonegroup.master_zone == zone_id) { - // choose a new master zone - auto m = zonegroup.zones.begin(); - if (m != zonegroup.zones.end()) { - zonegroup.master_zone = m->first; - ldpp_dout(dpp, 0) << "NOTICE: promoted " << m->second.name - << " as new master_zone of zonegroup " << zonegroup.name << dendl; - } else { - ldpp_dout(dpp, 0) << "NOTICE: removed master_zone of zonegroup " - << zonegroup.name << dendl; - } - } - - const bool log_data = zonegroup.zones.size() > 1; - for (auto& [id, zone] : zonegroup.zones) { - zone.log_data = log_data; - } - - return 0; -} - -// try to remove the given zone id from every zonegroup in the cluster -static int remove_zone_from_groups(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - const rgw_zone_id& zone_id) -{ - std::array zonegroup_names; - sal::ListResult listing; - do { - int r = cfgstore->list_zonegroup_names(dpp, y, listing.next, - zonegroup_names, listing); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to list zonegroups with " - << cpp_strerror(r) << dendl; - return r; - } - - for (const auto& name : listing.entries) { - RGWZoneGroup zonegroup; - std::unique_ptr writer; - r = cfgstore->read_zonegroup_by_name(dpp, y, name, zonegroup, &writer); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to load zonegroup " << name - << " with " << cpp_strerror(r) << dendl; - continue; - } - - r = remove_zone_from_group(dpp, zonegroup, zone_id); - if (r < 0) { - continue; - } - - // write the updated zonegroup - r = writer->write(dpp, y, zonegroup); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to write zonegroup " << name - << " with " << cpp_strerror(r) << dendl; - continue; - } - ldpp_dout(dpp, 0) << "Removed zone from zonegroup " << name << dendl; - } - } while (!listing.next.empty()); - - return 0; -} - - -int read_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zone_id, - std::string_view zone_name, - RGWZoneParams& info, - std::unique_ptr* writer) -{ - if (!zone_id.empty()) { - return cfgstore->read_zone_by_id(dpp, y, zone_id, info, writer); - } - if (!zone_name.empty()) { - return cfgstore->read_zone_by_name(dpp, y, zone_name, info, writer); - } - - std::string realm_id; - int r = cfgstore->read_default_realm_id(dpp, y, realm_id); - if (r == -ENOENT) { - return cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, writer); - } - if (r < 0) { - return r; - } - return cfgstore->read_default_zone(dpp, y, realm_id, info, writer); -} - -extern int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y, - rgw::sal::ConfigStore* cfgstore, - std::string_view my_zone_id, - std::set& pools); - -int create_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneParams& info, std::unique_ptr* writer) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // add default placement with empty pool name - RGWZonePlacementInfo placement; - rgw_pool pool; - placement.storage_classes.set_storage_class( - RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); - // don't overwrite if it already exists - info.placement_pools.emplace("default-placement", std::move(placement)); - - // build a set of all pool names used by other zones - std::set pools; - int r = get_zones_pool_set(dpp, y, cfgstore, info.id, pools); - if (r < 0) { - return r; - } - - // initialize pool names with the zone name prefix - r = init_zone_pool_names(dpp, y, pools, info); - if (r < 0) { - return r; - } - - r = cfgstore->create_zone(dpp, y, exclusive, info, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create zone with " - << cpp_strerror(r) << dendl; - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_default_zone(dpp, y, cfgstore, info, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set zone as default: " - << cpp_strerror(r) << dendl; - } - - return 0; - -} - -int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - bool exclusive) -{ - return cfgstore->write_default_zone_id( - dpp, y, exclusive, info.realm_id, info.id); -} - -int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - sal::ZoneWriter& writer) -{ - // remove this zone from any zonegroups that contain it - int r = remove_zone_from_groups(dpp, y, cfgstore, info.id); - if (r < 0) { - return r; - } - - return writer.remove(dpp, y); -} - -auto find_zone_placement(const DoutPrefixProvider* dpp, - const RGWZoneParams& info, - const rgw_placement_rule& rule) - -> const RGWZonePlacementInfo* -{ - auto i = info.placement_pools.find(rule.name); - if (i == info.placement_pools.end()) { - ldpp_dout(dpp, 0) << "ERROR: This zone does not contain placement rule " - << rule.name << dendl; - return nullptr; - } - - const std::string& storage_class = rule.get_storage_class(); - if (!i->second.storage_class_exists(storage_class)) { - ldpp_dout(dpp, 5) << "ERROR: The zone placement for rule " << rule.name - << " does not contain storage class " << storage_class << dendl; - return nullptr; - } - - return &i->second; -} - -bool all_zonegroups_support(const SiteConfig& site, std::string_view feature) -{ - const auto& period = site.get_period(); - if (!period) { - // if we're not in a realm, just check the local zonegroup - return site.get_zonegroup().supports(feature); - } - const auto& zgs = period->period_map.zonegroups; - return std::all_of(zgs.begin(), zgs.end(), [feature] (const auto& pair) { - return pair.second.supports(feature); - }); -} - -static int read_or_create_default_zone(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - RGWZoneParams& info) -{ - int r = cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, nullptr); - if (r == -ENOENT) { - info.name = default_zone_name; - constexpr bool exclusive = true; - r = create_zone(dpp, y, cfgstore, exclusive, info, nullptr); - if (r == -EEXIST) { - r = cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, nullptr); - } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create default zone: " - << cpp_strerror(r) << dendl; - return r; - } - } - return r; -} - -static int read_or_create_default_zonegroup(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - const RGWZoneParams& zone_params, - RGWZoneGroup& info) -{ - int r = cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, - info, nullptr); - if (r == -ENOENT) { - constexpr bool exclusive = true; - r = create_default_zonegroup(dpp, y, cfgstore, exclusive, - zone_params, info); - if (r == -EEXIST) { - r = cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, - info, nullptr); - } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create default zonegroup: " - << cpp_strerror(r) << dendl; - return r; - } - } - return r; -} - -int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool force_local_zonegroup) -{ - // clear existing configuration - zone = nullptr; - zonegroup = nullptr; - local_zonegroup = std::nullopt; - period = std::nullopt; - zone_params = RGWZoneParams{}; - - int r = 0; - - // try to load a realm - realm.emplace(); - std::string realm_name = dpp->get_cct()->_conf->rgw_realm; - if (!realm_name.empty()) { - r = cfgstore->read_realm_by_name(dpp, y, realm_name, *realm, nullptr); - } else { - r = cfgstore->read_default_realm(dpp, y, *realm, nullptr); - if (r == -ENOENT) { // no realm - r = 0; - realm = std::nullopt; - } - } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load realm: " << cpp_strerror(r) << dendl; - return r; - } - - // try to load the local zone params - std::string zone_name = dpp->get_cct()->_conf->rgw_zone; - if (!zone_name.empty()) { - r = cfgstore->read_zone_by_name(dpp, y, zone_name, zone_params, nullptr); - } else if (realm) { - // load the realm's default zone - r = cfgstore->read_default_zone(dpp, y, realm->id, zone_params, nullptr); - if (r == -ENOENT) { - if (realm_name.empty()) { - // rgw_realm was not specified, and we found a default realm that - // doesn't have a default zone. ignore the realm and try to load the - // global default zone - realm = std::nullopt; - r = read_or_create_default_zone(dpp, y, cfgstore, zone_params); - } else { - ldpp_dout(dpp, 0) << "No rgw_zone configured, and the selected realm \"" - << realm->name << "\" does not have a default zone." << dendl; - } - } - } else { - // load or create the "default" zone - r = read_or_create_default_zone(dpp, y, cfgstore, zone_params); - } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load zone: " << cpp_strerror(r) << dendl; - return r; - } - - if (!realm && !zone_params.realm_id.empty()) { - realm.emplace(); - r = cfgstore->read_realm_by_id(dpp, y, zone_params.realm_id, - *realm, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load realm: " << cpp_strerror(r) << dendl; - return r; - } - } - - if (realm && !force_local_zonegroup) { - // try to load the realm's period - r = load_period_zonegroup(dpp, y, cfgstore, *realm, zone_params.id); - if (r != -ENOENT) { - return r; - } - ldpp_dout(dpp, 10) << "cannot find current period zonegroup, " - "using local zonegroup configuration" << dendl; - } - - // fall back to a local zonegroup - return load_local_zonegroup(dpp, y, cfgstore, zone_params.id); -} - -std::unique_ptr SiteConfig::make_fake() { - auto fake = std::make_unique(); - fake->local_zonegroup.emplace(); - fake->local_zonegroup->zones.emplace(""s, RGWZone{}); - fake->zonegroup = &*fake->local_zonegroup; - fake->zone = &fake->zonegroup->zones.begin()->second; - return fake; -} - -int SiteConfig::load_period_zonegroup(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - const RGWRealm& realm, - const rgw_zone_id& zone_id) -{ - // load the realm's current period - period.emplace(); - int r = cfgstore->read_period(dpp, y, realm.current_period, - std::nullopt, *period); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load current period: " - << cpp_strerror(r) << dendl; - return r; - } - - // find our zone and zonegroup in the period - for (const auto& zg : period->period_map.zonegroups) { - auto z = zg.second.zones.find(zone_id); - if (z != zg.second.zones.end()) { - zone = &z->second; - zonegroup = &zg.second; - return 0; - } - } - - ldpp_dout(dpp, 0) << "ERROR: current period " << period->id - << " does not contain zone id " << zone_id << dendl; - - period = std::nullopt; - return -ENOENT; -} - -int SiteConfig::load_local_zonegroup(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - const rgw_zone_id& zone_id) -{ - int r = 0; - - // load the zonegroup - local_zonegroup.emplace(); - std::string zonegroup_name = dpp->get_cct()->_conf->rgw_zonegroup; - if (!zonegroup_name.empty()) { - r = cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, - *local_zonegroup, nullptr); - } else if (realm) { - r = cfgstore->read_default_zonegroup(dpp, y, realm->id, - *local_zonegroup, nullptr); - } else { - r = read_or_create_default_zonegroup(dpp, y, cfgstore, zone_params, - *local_zonegroup); - } - - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load zonegroup: " - << cpp_strerror(r) << dendl; - } else { - // find our zone in the zonegroup - auto z = local_zonegroup->zones.find(zone_id); - if (z != local_zonegroup->zones.end()) { - zone = &z->second; - zonegroup = &*local_zonegroup; - return 0; - } - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << local_zonegroup->id - << " does not contain zone id " << zone_id << dendl; - r = -ENOENT; - } - - local_zonegroup = std::nullopt; - return r; -} - -} // namespace rgw - -static inline int conf_to_uint64(const JSONFormattable& config, const string& key, uint64_t *pval) -{ - string sval; - if (config.find(key, &sval)) { - string err; - uint64_t val = strict_strtoll(sval.c_str(), 10, &err); - if (!err.empty()) { - return -EINVAL; - } - *pval = val; - } - return 0; -} - -int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) -{ - int r = -1; - - if (config.exists("retain_head_object")) { - string s = config["retain_head_object"]; - if (s == "true") { - retain_head_object = true; - } else { - retain_head_object = false; - } - } - if (config.exists("allow_read_through")) { - string s = config["allow_read_through"]; - if (s == "true") { - allow_read_through = true; - } else { - allow_read_through = false; - } - } - if (config.exists("read_through_restore_days")) { - r = conf_to_uint64(config, "read_through_restore_days", &read_through_restore_days); - if (r < 0) { - read_through_restore_days = DEFAULT_READ_THROUGH_RESTORE_DAYS; - } - } - - if (is_tier_type_s3()) { - r = t.s3.update_params(config); - } - - if (config.exists("restore_storage_class")) { - restore_storage_class = config["restore_storage_class"]; - } - - if (is_tier_type_s3_glacier()) { - r = s3_glacier.update_params(config); - } - return r; -} - -int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) -{ - if (config.exists("retain_head_object")) { - retain_head_object = false; - } - if (config.exists("allow_read_through")) { - allow_read_through = false; - } - if (config.exists("read_through_restore_days")) { - read_through_restore_days = DEFAULT_READ_THROUGH_RESTORE_DAYS; - } - - if (is_tier_type_s3()) { - t.s3.clear_params(config); - } - - if (config.exists("restore_storage_class")) { - restore_storage_class = RGW_STORAGE_CLASS_STANDARD; - } - - if (is_tier_type_s3_glacier()) { - s3_glacier.clear_params(config); - } - - return 0; -} - -int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config) -{ - int r = -1; - - if (config.exists("endpoint")) { - endpoint = config["endpoint"]; - } - if (config.exists("target_path")) { - target_path = config["target_path"]; - } - if (config.exists("region")) { - region = config["region"]; - } - if (config.exists("host_style")) { - string s; - s = config["host_style"]; - if (s != "virtual") { - host_style = PathStyle; - } else { - host_style = VirtualStyle; - } - } - if (config.exists("target_storage_class")) { - target_storage_class = config["target_storage_class"]; - } - if (config.exists("access_key")) { - key.id = config["access_key"]; - } - if (config.exists("secret")) { - key.key = config["secret"]; - } - if (config.exists("multipart_sync_threshold")) { - r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold); - if (r < 0) { - multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - } - - if (config.exists("multipart_min_part_size")) { - r = conf_to_uint64(config, "multipart_min_part_size", &multipart_min_part_size); - if (r < 0) { - multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - } - - if (config.exists("acls")) { - const JSONFormattable& cc = config["acls"]; - if (cc.is_array()) { - for (auto& c : cc.array()) { - RGWTierACLMapping m; - m.init(c); - if (!m.source_id.empty()) { - acl_mappings[m.source_id] = m; - } - } - } else { - RGWTierACLMapping m; - m.init(cc); - if (!m.source_id.empty()) { - acl_mappings[m.source_id] = m; - } - } - } - return 0; -} - -int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config) -{ - if (config.exists("endpoint")) { - endpoint.clear(); - } - if (config.exists("target_path")) { - target_path.clear(); - } - if (config.exists("region")) { - region.clear(); - } - if (config.exists("host_style")) { - /* default */ - host_style = PathStyle; - } - if (config.exists("target_storage_class")) { - target_storage_class.clear(); - } - if (config.exists("access_key")) { - key.id.clear(); - } - if (config.exists("secret")) { - key.key.clear(); - } - if (config.exists("multipart_sync_threshold")) { - multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - if (config.exists("multipart_min_part_size")) { - multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - if (config.exists("acls")) { - const JSONFormattable& cc = config["acls"]; - if (cc.is_array()) { - for (auto& c : cc.array()) { - RGWTierACLMapping m; - m.init(c); - acl_mappings.erase(m.source_id); - } - } else { - RGWTierACLMapping m; - m.init(cc); - acl_mappings.erase(m.source_id); - } - } - return 0; -} - -int RGWZoneGroupTierS3Glacier::update_params(const JSONFormattable& config) -{ - int r = -1; - - if (config.exists("glacier_restore_days")) { - r = conf_to_uint64(config, "glacier_restore_days", &glacier_restore_days); - if (r < 0) { - glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS; - } - } - if (config.exists("glacier_restore_tier_type")) { - string s; - s = config["glacier_restore_tier_type"]; - if (s != "Expedited") { - glacier_restore_tier_type = Standard; - } else { - glacier_restore_tier_type = Expedited; - } - } - return 0; -} - -int RGWZoneGroupTierS3Glacier::clear_params(const JSONFormattable& config) -{ - if (config.exists("glacier_restore_days")) { - glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS; - } - if (config.exists("glacier_restore_tier_type")) { - /* default */ - glacier_restore_tier_type = Standard; - } - return 0; -} - -void rgw_meta_sync_info::generate_test_instances(list& o) -{ - auto info = new rgw_meta_sync_info; - info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps; - info->period = "periodid"; - info->realm_epoch = 5; - o.push_back(info); - o.push_back(new rgw_meta_sync_info); -} - -void rgw_meta_sync_marker::generate_test_instances(list& o) -{ - auto marker = new rgw_meta_sync_marker; - marker->state = rgw_meta_sync_marker::IncrementalSync; - marker->marker = "01234"; - marker->realm_epoch = 5; - o.push_back(marker); - o.push_back(new rgw_meta_sync_marker); -} - -void rgw_meta_sync_status::generate_test_instances(list& o) -{ - o.push_back(new rgw_meta_sync_status); -} - -void RGWZoneParams::generate_test_instances(list &o) -{ - o.push_back(new RGWZoneParams); - o.push_back(new RGWZoneParams); -} - -void RGWPeriodLatestEpochInfo::generate_test_instances(list &o) -{ - RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo; - o.push_back(z); - o.push_back(new RGWPeriodLatestEpochInfo); -} - -void RGWZoneGroup::generate_test_instances(list& o) -{ - RGWZoneGroup *r = new RGWZoneGroup; - o.push_back(r); - o.push_back(new RGWZoneGroup); -} - -void RGWPeriodLatestEpochInfo::dump(Formatter *f) const { - encode_json("latest_epoch", epoch, f); -} - -void RGWPeriodLatestEpochInfo::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("latest_epoch", epoch, obj); -} - -void RGWNameToId::dump(Formatter *f) const { - encode_json("obj_id", obj_id, f); -} - -void RGWNameToId::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("obj_id", obj_id, obj); -} - -void RGWNameToId::generate_test_instances(list& o) { - RGWNameToId *n = new RGWNameToId; - n->obj_id = "id"; - o.push_back(n); - o.push_back(new RGWNameToId); -} - diff --git a/src/rgw/driver/rados/rgw_zone.h b/src/rgw/driver/rados/rgw_zone.h deleted file mode 100644 index ff344a173d9d..000000000000 --- a/src/rgw/driver/rados/rgw_zone.h +++ /dev/null @@ -1,900 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#pragma once - -#include -#include "rgw_zone_types.h" -#include "rgw_common.h" -#include "rgw_sal_fwd.h" -#include "rgw_sync_policy.h" - - -struct RGWZoneParams { - std::string id; - std::string name; - rgw_pool domain_root; - rgw_pool control_pool; - rgw_pool gc_pool; - rgw_pool lc_pool; - rgw_pool log_pool; - rgw_pool intent_log_pool; - rgw_pool usage_log_pool; - rgw_pool user_keys_pool; - rgw_pool user_email_pool; - rgw_pool user_swift_pool; - rgw_pool user_uid_pool; - rgw_pool roles_pool; - rgw_pool reshard_pool; - rgw_pool otp_pool; - rgw_pool oidc_pool; - rgw_pool notif_pool; - rgw_pool topics_pool; - rgw_pool account_pool; - rgw_pool group_pool; - - RGWAccessKey system_key; - - std::map placement_pools; - - std::string realm_id; - - JSONFormattable tier_config; - - RGWZoneParams() {} - explicit RGWZoneParams(const std::string& _name) : name(_name){} - RGWZoneParams(const rgw_zone_id& _id, const std::string& _name) : id(_id.id), name(_name) {} - RGWZoneParams(const rgw_zone_id& _id, const std::string& _name, const std::string& _realm_id) - : id(_id.id), name(_name), realm_id(_realm_id) {} - - const std::string& get_name() const { return name; } - const std::string& get_id() const { return id; } - - void set_name(const std::string& _name) { name = _name;} - void set_id(const std::string& _id) { id = _id;} - void clear_id() { id.clear(); } - - rgw_pool get_pool(CephContext *cct) const; - - const std::string& get_compression_type(const rgw_placement_rule& placement_rule) const; - - void encode(bufferlist& bl) const { - ENCODE_START(15, 1, bl); - encode(domain_root, bl); - encode(control_pool, bl); - encode(gc_pool, bl); - encode(log_pool, bl); - encode(intent_log_pool, bl); - encode(usage_log_pool, bl); - encode(user_keys_pool, bl); - encode(user_email_pool, bl); - encode(user_swift_pool, bl); - encode(user_uid_pool, bl); - { - // these used to be wrapped by RGWSystemMetaObj::encode(), - // so the extra ENCODE_START/ENCODE_FINISH are preserved - ENCODE_START(1, 1, bl); - encode(id, bl); - encode(name, bl); - ENCODE_FINISH(bl); - } - encode(system_key, bl); - encode(placement_pools, bl); - rgw_pool unused_metadata_heap; - encode(unused_metadata_heap, bl); - encode(realm_id, bl); - encode(lc_pool, bl); - std::map old_tier_config; - encode(old_tier_config, bl); - encode(roles_pool, bl); - encode(reshard_pool, bl); - encode(otp_pool, bl); - encode(tier_config, bl); - encode(oidc_pool, bl); - encode(notif_pool, bl); - encode(topics_pool, bl); - encode(account_pool, bl); - encode(group_pool, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(15, bl); - decode(domain_root, bl); - decode(control_pool, bl); - decode(gc_pool, bl); - decode(log_pool, bl); - decode(intent_log_pool, bl); - decode(usage_log_pool, bl); - decode(user_keys_pool, bl); - decode(user_email_pool, bl); - decode(user_swift_pool, bl); - decode(user_uid_pool, bl); - if (struct_v >= 6) { - { - // these used to be wrapped by RGWSystemMetaObj::decode(), - // so the extra DECODE_START/DECODE_FINISH are preserved - DECODE_START(1, bl); - decode(id, bl); - decode(name, bl); - DECODE_FINISH(bl); - } - } else if (struct_v >= 2) { - decode(name, bl); - id = name; - } - if (struct_v >= 3) - decode(system_key, bl); - if (struct_v >= 4) - decode(placement_pools, bl); - if (struct_v >= 5) { - rgw_pool unused_metadata_heap; - decode(unused_metadata_heap, bl); - } - if (struct_v >= 6) { - decode(realm_id, bl); - } - if (struct_v >= 7) { - decode(lc_pool, bl); - } else { - lc_pool = log_pool.name + ":lc"; - } - std::map old_tier_config; - if (struct_v >= 8) { - decode(old_tier_config, bl); - } - if (struct_v >= 9) { - decode(roles_pool, bl); - } else { - roles_pool = name + ".rgw.meta:roles"; - } - if (struct_v >= 10) { - decode(reshard_pool, bl); - } else { - reshard_pool = log_pool.name + ":reshard"; - } - if (struct_v >= 11) { - ::decode(otp_pool, bl); - } else { - otp_pool = name + ".rgw.otp"; - } - if (struct_v >= 12) { - ::decode(tier_config, bl); - } else { - for (auto& kv : old_tier_config) { - tier_config.set(kv.first, kv.second); - } - } - if (struct_v >= 13) { - ::decode(oidc_pool, bl); - } else { - oidc_pool = name + ".rgw.meta:oidc"; - } - if (struct_v >= 14) { - decode(notif_pool, bl); - } else { - notif_pool = log_pool.name + ":notif"; - } - if (struct_v >= 15) { - decode(topics_pool, bl); - decode(account_pool, bl); - decode(group_pool, bl); - } else { - topics_pool = name + ".rgw.meta:topics"; - account_pool = name + ".rgw.meta:accounts"; - group_pool = name + ".rgw.meta:groups"; - } - DECODE_FINISH(bl); - } - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - static void generate_test_instances(std::list& o); - - bool get_placement(const std::string& placement_id, RGWZonePlacementInfo *placement) const { - auto iter = placement_pools.find(placement_id); - if (iter == placement_pools.end()) { - return false; - } - *placement = iter->second; - return true; - } - - /* - * return data pool of the head object - */ - bool get_head_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool* pool) const { - const rgw_data_placement_target& explicit_placement = obj.bucket.explicit_placement; - if (!explicit_placement.data_pool.empty()) { - if (!obj.in_extra_data) { - *pool = explicit_placement.data_pool; - } else { - *pool = explicit_placement.get_data_extra_pool(); - } - return true; - } - if (placement_rule.empty()) { - return false; - } - auto iter = placement_pools.find(placement_rule.name); - if (iter == placement_pools.end()) { - return false; - } - if (!obj.in_extra_data) { - *pool = iter->second.get_data_pool(placement_rule.storage_class); - } else { - *pool = iter->second.get_data_extra_pool(); - } - return true; - } - - bool valid_placement(const rgw_placement_rule& rule) const { - auto iter = placement_pools.find(rule.name); - if (iter == placement_pools.end()) { - return false; - } - return iter->second.storage_class_exists(rule.storage_class); - } -}; -WRITE_CLASS_ENCODER(RGWZoneParams) - -struct RGWZoneGroup { - std::string id; - std::string name; - std::string api_name; - std::list endpoints; - bool is_master = false; - - rgw_zone_id master_zone; - std::map zones; - - std::map placement_targets; - rgw_placement_rule default_placement; - - std::list hostnames; - std::list hostnames_s3website; - // TODO: Maybe convert hostnames to a map> for - // endpoint_type->hostnames -/* -20:05 < _robbat21irssi> maybe I do something like: if (hostname_map.empty()) { populate all map keys from hostnames; }; -20:05 < _robbat21irssi> but that's a later compatability migration planning bit -20:06 < yehudasa> more like if (!hostnames.empty()) { -20:06 < yehudasa> for (std::list::iterator iter = hostnames.begin(); iter != hostnames.end(); ++iter) { -20:06 < yehudasa> hostname_map["s3"].append(iter->second); -20:07 < yehudasa> hostname_map["s3website"].append(iter->second); -20:07 < yehudasa> s/append/push_back/g -20:08 < _robbat21irssi> inner loop over APIs -20:08 < yehudasa> yeah, probably -20:08 < _robbat21irssi> s3, s3website, swift, swith_auth, swift_website -*/ - std::map > api_hostname_map; - std::map > api_endpoints_map; - - std::string realm_id; - - rgw_sync_policy_info sync_policy; - rgw::zone_features::set enabled_features; - CephContext *cct{nullptr}; - - RGWZoneGroup(): is_master(false){} - RGWZoneGroup(const std::string &_id, const std::string &_name):id(_id), name(_name) {} - explicit RGWZoneGroup(const std::string &_name):name(_name) {} - RGWZoneGroup(const std::string &_name, bool _is_master, const std::string& _realm_id, - const std::list& _endpoints) - : name(_name), endpoints(_endpoints), is_master(_is_master), realm_id(_realm_id) {} - - const std::string& get_name() const { return name; } - const std::string& get_id() const { return id; } - - void set_name(const std::string& _name) { name = _name;} - void set_id(const std::string& _id) { id = _id;} - void clear_id() { id.clear(); } - - bool is_master_zonegroup() const { return is_master;} - - void encode(bufferlist& bl) const { - ENCODE_START(6, 1, bl); - encode(name, bl); - encode(api_name, bl); - encode(is_master, bl); - encode(endpoints, bl); - encode(master_zone, bl); - encode(zones, bl); - encode(placement_targets, bl); - encode(default_placement, bl); - encode(hostnames, bl); - encode(hostnames_s3website, bl); - { - // these used to be wrapped by RGWSystemMetaObj::encode(), - // so the extra ENCODE_START/ENCODE_FINISH are preserved - ENCODE_START(1, 1, bl); - encode(id, bl); - encode(name, bl); - ENCODE_FINISH(bl); - } - encode(realm_id, bl); - encode(sync_policy, bl); - encode(enabled_features, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(6, bl); - decode(name, bl); - decode(api_name, bl); - decode(is_master, bl); - decode(endpoints, bl); - decode(master_zone, bl); - decode(zones, bl); - decode(placement_targets, bl); - decode(default_placement, bl); - if (struct_v >= 2) { - decode(hostnames, bl); - } - if (struct_v >= 3) { - decode(hostnames_s3website, bl); - } - if (struct_v >= 4) { - { - // these used to be wrapped by RGWSystemMetaObj::decode(), - // so the extra DECODE_START/DECODE_FINISH are preserved - DECODE_START(1, bl); - decode(id, bl); - decode(name, bl); - DECODE_FINISH(bl); - } - decode(realm_id, bl); - } else { - id = name; - } - if (struct_v >= 5) { - decode(sync_policy, bl); - } - if (struct_v >= 6) { - decode(enabled_features, bl); - } - DECODE_FINISH(bl); - } - - int equals(const std::string& other_zonegroup) const; - rgw_pool get_pool(CephContext *cct) const; - - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - static void generate_test_instances(std::list& o); - - bool supports(std::string_view feature) const { - return enabled_features.contains(feature); - } -}; -WRITE_CLASS_ENCODER(RGWZoneGroup) - -struct RGWPeriodMap -{ - std::string id; - std::map zonegroups; - std::map zonegroups_by_api; - std::map short_zone_ids; - - std::string master_zonegroup; - - void encode(bufferlist& bl) const; - void decode(bufferlist::const_iterator& bl); - - int update(const RGWZoneGroup& zonegroup, CephContext *cct); - - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - - void reset() { - zonegroups.clear(); - zonegroups_by_api.clear(); - master_zonegroup.clear(); - } - - uint32_t get_zone_short_id(const std::string& zone_id) const; - - bool find_zone_by_id(const rgw_zone_id& zone_id, - RGWZoneGroup *zonegroup, - RGWZone *zone) const; - bool find_zone_by_name(const std::string& zone_id, - RGWZoneGroup *zonegroup, - RGWZone *zone) const; -}; -WRITE_CLASS_ENCODER(RGWPeriodMap) - -struct RGWPeriodConfig -{ - RGWQuota quota; - RGWRateLimitInfo user_ratelimit; - RGWRateLimitInfo bucket_ratelimit; - // rate limit unauthenticated user - RGWRateLimitInfo anon_ratelimit; - - void encode(bufferlist& bl) const { - ENCODE_START(2, 1, bl); - encode(quota.bucket_quota, bl); - encode(quota.user_quota, bl); - encode(bucket_ratelimit, bl); - encode(user_ratelimit, bl); - encode(anon_ratelimit, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(2, bl); - decode(quota.bucket_quota, bl); - decode(quota.user_quota, bl); - if (struct_v >= 2) { - decode(bucket_ratelimit, bl); - decode(user_ratelimit, bl); - decode(anon_ratelimit, bl); - } - DECODE_FINISH(bl); - } - - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); -}; -WRITE_CLASS_ENCODER(RGWPeriodConfig) - -class RGWRealm; -class RGWPeriod; - -class RGWRealm -{ -public: - std::string id; - std::string name; - - std::string current_period; - epoch_t epoch{0}; //< realm epoch, incremented for each new period - -public: - RGWRealm() {} - RGWRealm(const std::string& _id, const std::string& _name = "") : id(_id), name(_name) {} - - const std::string& get_name() const { return name; } - const std::string& get_id() const { return id; } - - void set_name(const std::string& _name) { name = _name;} - void set_id(const std::string& _id) { id = _id;} - void clear_id() { id.clear(); } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - { - // these used to be wrapped by RGWSystemMetaObj::encode(), - // so the extra ENCODE_START/ENCODE_FINISH are preserved - ENCODE_START(1, 1, bl); - encode(id, bl); - encode(name, bl); - ENCODE_FINISH(bl); - } - encode(current_period, bl); - encode(epoch, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - { - // these used to be wrapped by RGWSystemMetaObj::decode(), - // so the extra DECODE_START/DECODE_FINISH are preserved - DECODE_START(1, bl); - decode(id, bl); - decode(name, bl); - DECODE_FINISH(bl); - } - decode(current_period, bl); - decode(epoch, bl); - DECODE_FINISH(bl); - } - - // TODO: use ConfigStore for watch/notify, - // After refactoring RGWRealmWatcher and RGWRealmReloader, get_pool and get_info_oid_prefix will be removed. - rgw_pool get_pool(CephContext *cct) const; - const std::string& get_info_oid_prefix(bool old_format = false) const; - - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - static void generate_test_instances(std::list& o); - - const std::string& get_current_period() const { - return current_period; - } - void clear_current_period_and_epoch() { - current_period.clear(); - epoch = 0; - } - epoch_t get_epoch() const { return epoch; } - - std::string get_control_oid() const; - - int find_zone(const DoutPrefixProvider *dpp, - const rgw_zone_id& zid, - RGWPeriod *pperiod, - RGWZoneGroup *pzonegroup, - bool *pfound, - rgw::sal::ConfigStore* cfgstore, - optional_yield y) const; -}; -WRITE_CLASS_ENCODER(RGWRealm) - -struct RGWPeriodLatestEpochInfo { - epoch_t epoch = 0; - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - encode(epoch, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(epoch, bl); - DECODE_FINISH(bl); - } - - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - static void generate_test_instances(std::list& o); -}; -WRITE_CLASS_ENCODER(RGWPeriodLatestEpochInfo) - - -/* - * The RGWPeriod object contains the entire configuration of a - * RGWRealm, including its RGWZoneGroups and RGWZones. Consistency of - * this configuration is maintained across all zones by passing around - * the RGWPeriod object in its JSON representation. - * - * If a new configuration changes which zone is the metadata master - * zone (i.e., master zone of the master zonegroup), then a new - * RGWPeriod::id (a uuid) is generated, its RGWPeriod::realm_epoch is - * incremented, and the RGWRealm object is updated to reflect that new - * current_period id and epoch. If the configuration changes BUT which - * zone is the metadata master does NOT change, then only the - * RGWPeriod::epoch is incremented (and the RGWPeriod::id remains the - * same). - * - * When a new RGWPeriod is created with a new RGWPeriod::id (uuid), it - * is linked back to its predecessor RGWPeriod through the - * RGWPeriod::predecessor_uuid field, thus creating a "linked - * list"-like structure of RGWPeriods back to the cluster's creation. - */ -class RGWPeriod -{ -public: - std::string id; //< a uuid - epoch_t epoch{0}; - std::string predecessor_uuid; - std::vector sync_status; - RGWPeriodMap period_map; - RGWPeriodConfig period_config; - std::string master_zonegroup; - rgw_zone_id master_zone; - - std::string realm_id; - epoch_t realm_epoch{1}; //< realm epoch when period was made current - - // gather the metadata sync status for each shard; only for use on master zone - int update_sync_status(const DoutPrefixProvider *dpp, - rgw::sal::Driver* driver, - const RGWPeriod ¤t_period, - std::ostream& error_stream, bool force_if_stale); - -public: - RGWPeriod() {} - - explicit RGWPeriod(const std::string& period_id, epoch_t _epoch = 0) - : id(period_id), epoch(_epoch) {} - - const std::string& get_id() const { return id; } - epoch_t get_epoch() const { return epoch; } - epoch_t get_realm_epoch() const { return realm_epoch; } - const std::string& get_predecessor() const { return predecessor_uuid; } - const rgw_zone_id& get_master_zone() const { return master_zone; } - const std::string& get_master_zonegroup() const { return master_zonegroup; } - const std::string& get_realm() const { return realm_id; } - const RGWPeriodMap& get_map() const { return period_map; } - RGWPeriodConfig& get_config() { return period_config; } - const RGWPeriodConfig& get_config() const { return period_config; } - const std::vector& get_sync_status() const { return sync_status; } - rgw_pool get_pool(CephContext *cct) const; - const std::string& get_info_oid_prefix() const; - - void set_user_quota(RGWQuotaInfo& user_quota) { - period_config.quota.user_quota = user_quota; - } - - void set_bucket_quota(RGWQuotaInfo& bucket_quota) { - period_config.quota.bucket_quota = bucket_quota; - } - - void set_id(const std::string& _id) { - this->id = _id; - period_map.id = _id; - } - void set_epoch(epoch_t epoch) { this->epoch = epoch; } - void set_realm_epoch(epoch_t epoch) { realm_epoch = epoch; } - - void set_predecessor(const std::string& predecessor) - { - predecessor_uuid = predecessor; - } - - void set_realm_id(const std::string& _realm_id) { - realm_id = _realm_id; - } - - int get_zonegroup(RGWZoneGroup& zonegroup, - const std::string& zonegroup_id) const; - - bool is_single_zonegroup() const - { - return (period_map.zonegroups.size() <= 1); - } - - /* - returns true if there are several zone groups with a least one zone - */ - bool is_multi_zonegroups_with_zones() const - { - int count = 0; - for (const auto& zg: period_map.zonegroups) { - if (zg.second.zones.size() > 0) { - if (count++ > 0) { - return true; - } - } - } - return false; - } - - bool find_zone(const DoutPrefixProvider *dpp, - const rgw_zone_id& zid, - RGWZoneGroup *pzonegroup, - optional_yield y) const; - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - encode(id, bl); - encode(epoch, bl); - encode(realm_epoch, bl); - encode(predecessor_uuid, bl); - encode(sync_status, bl); - encode(period_map, bl); - encode(master_zone, bl); - encode(master_zonegroup, bl); - encode(period_config, bl); - encode(realm_id, bl); - std::string realm_name; // removed - encode(realm_name, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(id, bl); - decode(epoch, bl); - decode(realm_epoch, bl); - decode(predecessor_uuid, bl); - decode(sync_status, bl); - decode(period_map, bl); - decode(master_zone, bl); - decode(master_zonegroup, bl); - decode(period_config, bl); - decode(realm_id, bl); - std::string realm_name; // removed - decode(realm_name, bl); - DECODE_FINISH(bl); - } - void dump(Formatter *f) const; - void decode_json(JSONObj *obj); - static void generate_test_instances(std::list& o); - - static std::string get_staging_id(const std::string& realm_id) { - return realm_id + ":staging"; - } -}; -WRITE_CLASS_ENCODER(RGWPeriod) - -namespace rgw { - -/// Look up a realm by its id. If no id is given, look it up by name. -/// If no name is given, fall back to the cluster's default realm. -int read_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view realm_id, - std::string_view realm_name, - RGWRealm& info, - std::unique_ptr* writer = nullptr); - -/// Create a realm and its initial period. If the info.id is empty, a -/// random uuid will be generated. -int create_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWRealm& info, - std::unique_ptr* writer = nullptr); - -/// Set the given realm as the cluster's default realm. -int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWRealm& info, - bool exclusive = false); - -/// Update the current_period of an existing realm. -int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - sal::RealmWriter& writer, RGWRealm& realm, - const RGWPeriod& period); - -/// Overwrite the local zonegroup and period config objects with the new -/// configuration contained in the given period. -int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWPeriod& info); - -/// Return the staging period id for the given realm. -std::string get_staging_period_id(std::string_view realm_id); - -/// Convert the given period into a separate staging period, where -/// radosgw-admin can make changes to it without effecting the running -/// configuration. -void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info); - -/// Read all zonegroups in the period's realm and add them to the period. -int update_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, RGWPeriod& info); - -/// Validates the given 'staging' period and tries to commit it as the -/// realm's new current period. -int commit_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, sal::Driver* driver, - RGWRealm& realm, sal::RealmWriter& realm_writer, - const RGWPeriod& current_period, - RGWPeriod& info, std::ostream& error_stream, - bool force_if_stale); - - -/// Look up a zonegroup by its id. If no id is given, look it up by name. -/// If no name is given, fall back to the cluster's default zonegroup. -int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zonegroup_id, - std::string_view zonegroup_name, - RGWZoneGroup& info, - std::unique_ptr* writer = nullptr); - -/// Initialize and create the given zonegroup. If the given info.id is empty, -/// a random uuid will be generated. May fail with -EEXIST. -int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneGroup& info); - -/// Set the given zonegroup as its realm's default zonegroup. -int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneGroup& info, - bool exclusive = false); - -/// Return an endpoint from the zonegroup or its master zone. -std::string get_zonegroup_endpoint(const RGWZoneGroup& info); - -/// Add a zone to the zonegroup, or update an existing zone entry. -int add_zone_to_group(const DoutPrefixProvider* dpp, - RGWZoneGroup& zonegroup, - const RGWZoneParams& zone_params, - const bool *pis_master, const bool *pread_only, - const std::list& endpoints, - const std::string *ptier_type, - const bool *psync_from_all, - const std::list& sync_from, - const std::list& sync_from_rm, - const std::string *predirect_zone, - std::optional bucket_index_max_shards, - const rgw::zone_features::set& enable_features, - const rgw::zone_features::set& disable_features); - -/// Remove a zone by id from its zonegroup, promoting a new master zone if -/// necessary. -int remove_zone_from_group(const DoutPrefixProvider* dpp, - RGWZoneGroup& info, - const rgw_zone_id& zone_id); - - -/// Look up a zone by its id. If no id is given, look it up by name. If no name -/// is given, fall back to the realm's default zone. -int read_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zone_id, - std::string_view zone_name, - RGWZoneParams& info, - std::unique_ptr* writer = nullptr); - -/// Initialize and create a new zone. If the given info.id is empty, a random -/// uuid will be generated. Pool names are initialized with the zone name as a -/// prefix. If any pool names conflict with existing zones, a random suffix is -/// added. -int create_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneParams& info, - std::unique_ptr* writer = nullptr); - -/// Initialize the zone's pool names using the zone name as a prefix. If a pool -/// name conflicts with an existing zone's pool, add a unique suffix. -int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y, - const std::set& pools, RGWZoneParams& info); - -/// Set the given zone as its realm's default zone. -int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - bool exclusive = false); - -/// Delete an existing zone and remove it from any zonegroups that contain it. -int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - sal::ZoneWriter& writer); - -/// Return the zone placement corresponding to the given rule, or nullptr. -auto find_zone_placement(const DoutPrefixProvider* dpp, - const RGWZoneParams& info, - const rgw_placement_rule& rule) - -> const RGWZonePlacementInfo*; - - -/// Global state about the site configuration. Initialized once during -/// startup and may be reinitialized by RGWRealmReloader, but is otherwise -/// immutable at runtime. -class SiteConfig { - public: - /// Return the local zone params. - const RGWZoneParams& get_zone_params() const { return zone_params; } - /// Return the current realm configuration, if a realm is present. - const std::optional& get_realm() const { return realm; } - /// Return the current period configuration, if a period is present. - const std::optional& get_period() const { return period; } - /// Return the zonegroup configuration. - const RGWZoneGroup& get_zonegroup() const { return *zonegroup; } - /// Return the public zone configuration. - const RGWZone& get_zone() const { return *zone; } - /// Return true if the local zone can write metadata. - bool is_meta_master() const { - return zonegroup->is_master && zonegroup->master_zone == zone->id; - } - - /// Load or reload the multisite configuration from storage. This is not - /// thread-safe, so requires careful coordination with the RGWRealmReloader. - int load(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool force_local_zonegroup = false); - - /// Create a fake site config to be used by tests and similar, just - /// to have a site config. - /// - /// \warning Do not use this anywhere but unittests where we need to - /// bring up parts of RGW that require a SiteConfig exist, but need - /// to run without a cluster. - static std::unique_ptr make_fake(); - - virtual ~SiteConfig() = default; - - private: - int load_period_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWRealm& realm, - const rgw_zone_id& zone_id); - int load_local_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - const rgw_zone_id& zone_id); - - RGWZoneParams zone_params; - std::optional realm; - std::optional period; - std::optional local_zonegroup; - const RGWZoneGroup* zonegroup = nullptr; - const RGWZone* zone = nullptr; -}; - - -/// Test whether all zonegroups in the realm support the given zone feature. -bool all_zonegroups_support(const SiteConfig& site, std::string_view feature); - -std::string gen_random_uuid(); - -} // namespace rgw diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index 33f33ebc76da..29927948be44 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -1988,7 +1988,7 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, RGWPeriod& period, string remote, const string& url, std::optional opt_region, const string& access, const string& secret, - bool force) + bool force, rgw::SiteConfig* site) { auto& master_zone = period.get_master_zone().id; if (master_zone.empty()) { @@ -2008,7 +2008,7 @@ static int commit_period(rgw::sal::ConfigStore* cfgstore, // the master zone can commit locally ret = rgw::commit_period(dpp(), null_yield, cfgstore, driver, realm, realm_writer, current_period, - period, cerr, force); + period, cerr, force, *site); if (ret < 0) { cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; } @@ -2095,7 +2095,7 @@ static int update_period(rgw::sal::ConfigStore* cfgstore, const string& remote, const string& url, std::optional opt_region, const string& access, const string& secret, - Formatter *formatter, bool force) + Formatter *formatter, bool force, rgw::SiteConfig* site) { RGWRealm realm; std::unique_ptr realm_writer; @@ -2133,7 +2133,7 @@ static int update_period(rgw::sal::ConfigStore* cfgstore, } if (commit) { ret = commit_period(cfgstore, realm, *realm_writer, period, remote, url, - opt_region, access, secret, force); + opt_region, access, secret, force, site); if (ret < 0) { cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; return ret; @@ -4890,7 +4890,7 @@ int main(int argc, const char **argv) int ret = update_period(cfgstore.get(), realm_id, realm_name, period_epoch, commit, remote, url, opt_region, access_key, secret_key, - formatter.get(), yes_i_really_mean_it); + formatter.get(), yes_i_really_mean_it, site.get()); if (ret < 0) { return -ret; } @@ -7040,7 +7040,7 @@ int main(int argc, const char **argv) int ret = update_period(cfgstore.get(), realm_id, realm_name, period_epoch, commit, remote, url, opt_region, access_key, secret_key, - formatter.get(), yes_i_really_mean_it); + formatter.get(), yes_i_really_mean_it, site.get()); if (ret < 0) { return -ret; } @@ -7069,7 +7069,7 @@ int main(int argc, const char **argv) } ret = commit_period(cfgstore.get(), realm, *realm_writer, period, remote, url, opt_region, access_key, secret_key, - yes_i_really_mean_it); + yes_i_really_mean_it, site.get()); if (ret < 0) { cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl; return -ret; diff --git a/src/rgw/rgw_appmain.cc b/src/rgw/rgw_appmain.cc index 3966ea6d2123..82e48c62ae3f 100644 --- a/src/rgw/rgw_appmain.cc +++ b/src/rgw/rgw_appmain.cc @@ -521,28 +521,33 @@ int rgw::AppMain::init_frontends2(RGWLib* rgwlib) /* ignore error */ } - // if we're part of a realm, add a watcher to respond to configuration changes - if (const auto& realm = env.site->get_realm(); realm) { - realm_watcher = env.cfgstore->create_realm_watcher(dpp, null_yield, *realm); - } - if (realm_watcher) { - pusher = std::make_unique(dpp, env.driver, env.cfgstore, null_yield); - realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher); - - fe_pauser = std::make_unique(fes, pusher.get()); - rgw_pauser = std::make_unique(); - rgw_pauser->add_pauser(fe_pauser.get()); - if (env.lua.background) { - rgw_pauser->add_pauser(env.lua.background); +#ifdef WITH_RADOSGW_RADOS + if (env.driver->get_name() == "rados") { + // add a watcher to respond to realm configuration changes + // if we're part of a realm, add a watcher to respond to configuration changes + if (const auto& realm = env.site->get_realm(); realm) { + realm_watcher = env.cfgstore->create_realm_watcher(dpp, null_yield, *realm); } + if (realm_watcher) { + pusher = std::make_unique(dpp, env.driver, env.cfgstore, null_yield); + realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher); + + fe_pauser = std::make_unique(fes, pusher.get()); + rgw_pauser = std::make_unique(); + rgw_pauser->add_pauser(fe_pauser.get()); + if (env.lua.background) { + rgw_pauser->add_pauser(env.lua.background); + } if (dedup_background) { rgw_pauser->add_pauser(dedup_background.get()); } - need_context_pool(); - reloader = std::make_unique( - env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), *context_pool); - realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader); + need_context_pool(); + reloader = std::make_unique( + env, *implicit_tenant_context, service_map_meta, rgw_pauser.get(), *context_pool); + realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader); + } } +#endif return r; } /* init_frontends2 */ @@ -576,6 +581,7 @@ void rgw::AppMain::init_lua() #endif env.lua.manager = env.driver->get_lua_manager(install_dir); +#ifdef WITH_RADOSGW_RADOS if (driver->get_name() == "rados") { /* Supported for only RadosStore */ lua_background = std::make_unique< rgw::lua::Background>(driver, dpp->get_cct(), env.lua.manager.get()); @@ -583,6 +589,7 @@ void rgw::AppMain::init_lua() env.lua.background = lua_background.get(); static_cast(env.lua.manager.get())->watch_reload(dpp); } +#endif } /* init_lua */ void rgw::AppMain::init_dedup() @@ -608,7 +615,7 @@ void rgw::AppMain::shutdown(std::function finalize_async_signals) realm_watcher.reset(); pusher.reset(); reloader.reset(); - +#ifdef WITH_RADOSGW_RADOS if (env.driver->get_name() == "rados") { if (g_conf().get_val("rgw_lua_enable")) static_cast(env.lua.manager.get())-> @@ -618,6 +625,7 @@ void rgw::AppMain::shutdown(std::function finalize_async_signals) dedup_background->unwatch_reload(dpp); } } +#endif for (auto& fe : fes) { fe->stop(); diff --git a/src/rgw/rgw_bucket_sync.cc b/src/rgw/rgw_bucket_sync.cc new file mode 100644 index 000000000000..1e7316d4271d --- /dev/null +++ b/src/rgw/rgw_bucket_sync.cc @@ -0,0 +1,1022 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_common.h" +#include "rgw_bucket_sync.h" +#include "rgw_data_sync.h" +#include "rgw_zone.h" + +#include "services/svc_zone.h" +#include "services/svc_bucket_sync.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +ostream& operator<<(ostream& os, const rgw_sync_bucket_entity& e) { + os << "{b=" << rgw_sync_bucket_entities::bucket_key(e.bucket) << ",z=" << e.zone.value_or(rgw_zone_id()) << ",az=" << (int)e.all_zones << "}"; + return os; +} + +ostream& operator<<(ostream& os, const rgw_sync_bucket_pipe& pipe) { + os << "{s=" << pipe.source << ",d=" << pipe.dest << "}"; + return os; +} + +ostream& operator<<(ostream& os, const rgw_sync_bucket_entities& e) { + os << "{b=" << rgw_sync_bucket_entities::bucket_key(e.bucket) << ",z=" << e.zones.value_or(std::set()) << "}"; + return os; +} + +ostream& operator<<(ostream& os, const rgw_sync_bucket_pipes& pipe) { + os << "{id=" << pipe.id << ",s=" << pipe.source << ",d=" << pipe.dest << "}"; + return os; +} + +static std::vector filter_relevant_pipes(const std::vector& pipes, + const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone) +{ + std::vector relevant_pipes; + for (auto& p : pipes) { + if (p.source.match_zone(source_zone) && + p.dest.match_zone(dest_zone)) { + for (auto pipe : p.expand()) { + pipe.source.apply_zone(source_zone); + pipe.dest.apply_zone(dest_zone); + relevant_pipes.push_back(pipe); + } + } + } + + return relevant_pipes; +} + +static bool is_wildcard_bucket(const rgw_bucket& bucket) +{ + return bucket.name.empty(); +} + +void rgw_sync_group_pipe_map::dump(ceph::Formatter *f) const +{ + encode_json("zone", zone.id, f); + encode_json("buckets", rgw_sync_bucket_entities::bucket_key(bucket), f); + encode_json("sources", sources, f); + encode_json("dests", dests, f); +} + + +template +void rgw_sync_group_pipe_map::try_add_to_pipe_map(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + zb_pipe_map_t *pipe_map, + CB1 filter_cb, + CB2 call_filter_cb) +{ + if (!filter_cb(source_zone, nullopt, dest_zone, nullopt)) { + return; + } + auto relevant_pipes = filter_relevant_pipes(pipes, source_zone, dest_zone); + + for (auto& pipe : relevant_pipes) { + rgw_sync_bucket_entity zb; + if (!call_filter_cb(pipe, &zb)) { + continue; + } + pipe_map->insert(make_pair(zb, pipe)); + } +} + +template +void rgw_sync_group_pipe_map::try_add_source(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + CB filter_cb) +{ + return try_add_to_pipe_map(source_zone, dest_zone, pipes, + &sources, + filter_cb, + [&](const rgw_sync_bucket_pipe& pipe, rgw_sync_bucket_entity *zb) { + *zb = rgw_sync_bucket_entity{source_zone, pipe.source.get_bucket()}; + return filter_cb(source_zone, zb->bucket, dest_zone, pipe.dest.get_bucket()); + }); +} + +template +void rgw_sync_group_pipe_map::try_add_dest(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + CB filter_cb) +{ + return try_add_to_pipe_map(source_zone, dest_zone, pipes, + &dests, + filter_cb, + [&](const rgw_sync_bucket_pipe& pipe, rgw_sync_bucket_entity *zb) { + *zb = rgw_sync_bucket_entity{dest_zone, pipe.dest.get_bucket()}; + return filter_cb(source_zone, pipe.source.get_bucket(), dest_zone, zb->bucket); + }); +} + +using zb_pipe_map_t = rgw_sync_group_pipe_map::zb_pipe_map_t; + +pair rgw_sync_group_pipe_map::find_pipes(const zb_pipe_map_t& m, + const rgw_zone_id& zone, + std::optional b) const +{ + if (!b) { + return m.equal_range(rgw_sync_bucket_entity{zone, rgw_bucket()}); + } + + auto zb = rgw_sync_bucket_entity{zone, *b}; + + auto range = m.equal_range(zb); + if (range.first == range.second && + !is_wildcard_bucket(*b)) { + /* couldn't find the specific bucket, try to find by wildcard */ + zb.bucket = rgw_bucket(); + range = m.equal_range(zb); + } + + return range; +} + + +template +void rgw_sync_group_pipe_map::init(const DoutPrefixProvider *dpp, + CephContext *cct, + const rgw_zone_id& _zone, + std::optional _bucket, + const rgw_sync_policy_group& group, + rgw_sync_data_flow_group *_default_flow, + std::set *_pall_zones, + CB filter_cb) { + zone = _zone; + bucket = _bucket; + default_flow = _default_flow; + pall_zones = _pall_zones; + + rgw_sync_bucket_entity zb(zone, bucket); + + status = group.status; + + std::vector zone_pipes; + + string bucket_key = (bucket ? bucket->get_key() : "*"); + + /* only look at pipes that touch the specific zone and bucket */ + for (auto& pipe : group.pipes) { + if (pipe.contains_zone_bucket(zone, bucket)) { + ldpp_dout(dpp, 20) << __func__ << "(): pipe_map (zone=" << zone << " bucket=" << bucket_key << "): adding potential pipe: " << pipe << dendl; + zone_pipes.push_back(pipe); + } + } + + const rgw_sync_data_flow_group *pflow; + + if (!group.data_flow.empty()) { + pflow = &group.data_flow; + } else { + if (!default_flow) { + return; + } + pflow = default_flow; + } + + auto& flow = *pflow; + + pall_zones->insert(zone); + + /* symmetrical */ + for (auto& symmetrical_group : flow.symmetrical) { + if (symmetrical_group.zones.find(zone) != symmetrical_group.zones.end()) { + for (auto& z : symmetrical_group.zones) { + if (z != zone) { + pall_zones->insert(z); + try_add_source(z, zone, zone_pipes, filter_cb); + try_add_dest(zone, z, zone_pipes, filter_cb); + } + } + } + } + + /* directional */ + for (auto& rule : flow.directional) { + if (rule.source_zone == zone) { + pall_zones->insert(rule.dest_zone); + try_add_dest(zone, rule.dest_zone, zone_pipes, filter_cb); + } else if (rule.dest_zone == zone) { + pall_zones->insert(rule.source_zone); + try_add_source(rule.source_zone, zone, zone_pipes, filter_cb); + } + } +} + +/* + * find all relevant pipes in our zone that match {dest_bucket} <- {source_zone, source_bucket} + */ +vector rgw_sync_group_pipe_map::find_source_pipes(const rgw_zone_id& source_zone, + std::optional source_bucket, + std::optional dest_bucket) const { + vector result; + + auto range = find_pipes(sources, source_zone, source_bucket); + + for (auto iter = range.first; iter != range.second; ++iter) { + auto pipe = iter->second; + if (pipe.dest.match_bucket(dest_bucket)) { + result.push_back(pipe); + } + } + return result; +} + +/* + * find all relevant pipes in other zones that pull from a specific + * source bucket in out zone {source_bucket} -> {dest_zone, dest_bucket} + */ +vector rgw_sync_group_pipe_map::find_dest_pipes(std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket) const { + vector result; + + auto range = find_pipes(dests, dest_zone, dest_bucket); + + for (auto iter = range.first; iter != range.second; ++iter) { + auto pipe = iter->second; + if (pipe.source.match_bucket(source_bucket)) { + result.push_back(pipe); + } + } + + return result; +} + +/* + * find all relevant pipes from {source_zone, source_bucket} -> {dest_zone, dest_bucket} + */ +vector rgw_sync_group_pipe_map::find_pipes(const rgw_zone_id& source_zone, + std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket) const { + if (dest_zone == zone) { + return find_source_pipes(source_zone, source_bucket, dest_bucket); + } + + if (source_zone == zone) { + return find_dest_pipes(source_bucket, dest_zone, dest_bucket); + } + + return vector(); +} + +void RGWBucketSyncFlowManager::pipe_rules::insert(const rgw_sync_bucket_pipe& pipe) +{ + pipes.push_back(pipe); + + auto ppipe = &pipes.back(); + auto prefix = ppipe->params.source.filter.prefix.value_or(string()); + + prefix_refs.insert(make_pair(prefix, ppipe)); + + for (auto& t : ppipe->params.source.filter.tags) { + string tag = t.key + "=" + t.value; + auto titer = tag_refs.find(tag); + if (titer != tag_refs.end() && + ppipe->params.priority > titer->second->params.priority) { + titer->second = ppipe; + } else { + tag_refs[tag] = ppipe; + } + } +} + +bool RGWBucketSyncFlowManager::pipe_rules::find_basic_info_without_tags(const rgw_obj_key& key, + std::optional *user, + std::optional *acl_translation_owner, + std::optional *storage_class, + rgw_sync_pipe_params::Mode *mode, + bool *need_more_info) const +{ + std::optional owner; + + *need_more_info = false; + + if (prefix_refs.empty()) { + return false; + } + + auto end = prefix_refs.upper_bound(key.name); + auto iter = end; + if (iter != prefix_refs.begin()) { + --iter; + } + if (iter == prefix_refs.end()) { + return false; + } + + if (iter != prefix_refs.begin()) { + iter = prefix_refs.find(iter->first); /* prefix_refs is multimap, find first element + holding that key */ + } + + std::vector iters; + + std::optional priority; + + for (; iter != end; ++iter) { + auto& prefix = iter->first; + if (!boost::starts_with(key.name, prefix)) { + continue; + } + + auto& rule_params = iter->second->params; + auto& filter = rule_params.source.filter; + + if (rule_params.priority > priority) { + priority = rule_params.priority; + + if (!filter.has_tags()) { + iters.clear(); + } + iters.push_back(iter); + + *need_more_info = filter.has_tags(); /* if highest priority filter has tags, then + we can't be sure if it would be used. + We need to first read the info from the source object */ + } + } + + if (iters.empty()) { + return false; + } + + std::optional _user; + std::optional _acl_translation; + std::optional _storage_class; + rgw_sync_pipe_params::Mode _mode{rgw_sync_pipe_params::Mode::MODE_SYSTEM}; + + // make sure all params are the same by saving the first one + // encountered and comparing all subsequent to it + bool first_iter = true; + for (auto& iter : iters) { + const rgw_sync_pipe_params& rule_params = iter->second->params; + if (first_iter) { + _user = rule_params.user; + _acl_translation = rule_params.dest.acl_translation; + _storage_class = rule_params.dest.storage_class; + _mode = rule_params.mode; + first_iter = false; + } else { + // note: three of these == operators are comparing std::optional + // against std::optional; as one would expect they are equal a) + // if both do not contain values or b) if both do and those + // contained values are the same + const bool conflict = + !(_user == rule_params.user && + _acl_translation == rule_params.dest.acl_translation && + _storage_class == rule_params.dest.storage_class && + _mode == rule_params.mode); + if (conflict) { + *need_more_info = true; + return false; + } + } + } + + *user = _user; + if (_acl_translation) { + *acl_translation_owner = _acl_translation->owner; + } + *storage_class = _storage_class; + *mode = _mode; + + return true; +} + +bool RGWBucketSyncFlowManager::pipe_rules::find_obj_params(const rgw_obj_key& key, + const RGWObjTags::tag_map_t& tags, + rgw_sync_pipe_params *params) const +{ + if (prefix_refs.empty()) { + return false; + } + + auto iter = prefix_refs.upper_bound(key.name); + if (iter != prefix_refs.begin()) { + --iter; + } + if (iter == prefix_refs.end()) { + return false; + } + + auto end = prefix_refs.upper_bound(key.name); + auto max = end; + + std::optional priority; + + for (; iter != end; ++iter) { + /* NOTE: this is not the most efficient way to do it, + * a trie data structure would be better + */ + auto& prefix = iter->first; + if (!boost::starts_with(key.name, prefix)) { + continue; + } + + auto& rule_params = iter->second->params; + auto& filter = rule_params.source.filter; + + if (!filter.check_tags(tags)) { + continue; + } + + if (rule_params.priority > priority) { + priority = rule_params.priority; + max = iter; + } + } + + if (max == end) { + return false; + } + + *params = max->second->params; + return true; +} + +/* + * return either the current prefix for s, or the next one if s is not within a prefix + */ + +RGWBucketSyncFlowManager::pipe_rules::prefix_map_t::const_iterator RGWBucketSyncFlowManager::pipe_rules::prefix_search(const std::string& s) const +{ + if (prefix_refs.empty()) { + return prefix_refs.end(); + } + auto next = prefix_refs.upper_bound(s); + auto iter = next; + if (iter != prefix_refs.begin()) { + --iter; + } + if (!boost::starts_with(s, iter->first)) { + return next; + } + + return iter; +} + +void RGWBucketSyncFlowManager::pipe_set::insert(const rgw_sync_bucket_pipe& pipe) { + /* Ensure this pipe doesn't match with any disabled pipes */ + for (auto p: disabled_pipe_map) { + if (p.second.source.match(pipe.source) && p.second.dest.match(pipe.dest)) { + return; + } + } + pipe_map.insert(make_pair(pipe.id, pipe)); + + auto& rules_ref = rules[endpoints_pair(pipe)]; + + if (!rules_ref) { + rules_ref = make_shared(); + } + + rules_ref->insert(pipe); + + pipe_handler h(rules_ref, pipe); + + handlers.insert(h); +} + +void RGWBucketSyncFlowManager::pipe_set::remove_all() { + pipe_map.clear(); + disabled_pipe_map.clear(); + rules.clear(); + handlers.clear(); +} + +void RGWBucketSyncFlowManager::pipe_set::disable(const rgw_sync_bucket_pipe& pipe) { + /* This pipe is disabled. Add it to disabled pipes & remove any + * matching pipes already inserted + */ + disabled_pipe_map.insert(make_pair(pipe.id, pipe)); + for (auto iter_p = pipe_map.begin(); iter_p != pipe_map.end(); ) { + auto p = iter_p++; + if (p->second.source.match(pipe.source) && p->second.dest.match(pipe.dest)) { + auto& rules_ref = rules[endpoints_pair(p->second)]; + if (rules_ref) { + pipe_handler h(rules_ref, p->second); + handlers.erase(h); + } + rules.erase(endpoints_pair(p->second)); + pipe_map.erase(p); + } + } +} + +void RGWBucketSyncFlowManager::pipe_set::dump(ceph::Formatter *f) const +{ + encode_json("pipes", pipe_map, f); +} + +bool RGWBucketSyncFlowManager::allowed_data_flow(const rgw_zone_id& source_zone, + std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket, + bool check_activated) const +{ + bool found = false; + bool found_activated = false; + + for (auto m : flow_groups) { + auto& fm = m.second; + auto pipes = fm.find_pipes(source_zone, source_bucket, + dest_zone, dest_bucket); + + bool is_found = !pipes.empty(); + + if (is_found) { + switch (fm.status) { + case rgw_sync_policy_group::Status::FORBIDDEN: + return false; + case rgw_sync_policy_group::Status::ENABLED: + found = true; + found_activated = true; + break; + case rgw_sync_policy_group::Status::ALLOWED: + found = true; + break; + default: + break; /* unknown -- ignore */ + } + } + } + + if (check_activated && found_activated) { + return true; + } + + return found; +} + +void RGWBucketSyncFlowManager::init(const DoutPrefixProvider *dpp, const rgw_sync_policy_info& sync_policy) { + std::optional default_flow; + if (parent) { + default_flow.emplace(); + default_flow->init_default(parent->all_zones); + } + + for (auto& item : sync_policy.groups) { + auto& group = item.second; + auto& flow_group_map = flow_groups[group.id]; + + flow_group_map.init(dpp, cct, zone_id, bucket, group, + (default_flow ? &(*default_flow) : nullptr), + &all_zones, + [&](const rgw_zone_id& source_zone, + std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket) { + if (!parent) { + return true; + } + return parent->allowed_data_flow(source_zone, + source_bucket, + dest_zone, + dest_bucket, + false); /* just check that it's not disabled */ + }); + } +} + +/* +* These are the semantics to be followed while resolving the policy +* conflicts - +* +* ================================================== +* zonegroup bucket Result +* ================================================== +* enabled enabled enabled +* allowed enabled +* forbidden disabled +* allowed enabled enabled +* allowed disabled +* forbidden disabled +* forbidden enabled disabled +* allowed disabled +* forbidden disabled +* +* In case multiple group policies are set to reflect for any sync pair +* (, ), the following +* rules are applied in the order- +* 1) Even if one policy status is FORBIDDEN, the sync will be disabled +* 2) Atleast one policy should be ENABLED for the sync to be allowed. +* +*/ +void RGWBucketSyncFlowManager::reflect(const DoutPrefixProvider *dpp, + std::optional effective_bucket, + RGWBucketSyncFlowManager::pipe_set *source_pipes, + RGWBucketSyncFlowManager::pipe_set *dest_pipes, + bool only_enabled) const + +{ + string effective_bucket_key; + bool is_forbidden = false; + if (effective_bucket) { + effective_bucket_key = effective_bucket->get_key(); + } + if (parent) { + parent->reflect(dpp, effective_bucket, source_pipes, dest_pipes, only_enabled); + } + + for (auto& item : flow_groups) { + auto& flow_group_map = item.second; + is_forbidden = false; + + if (flow_group_map.status == rgw_sync_policy_group::Status::FORBIDDEN) { + /* FORBIDDEN takes precedence over all the other rules. + * Remove any other pipes which may allow access. + */ + is_forbidden = true; + } else if (flow_group_map.status != rgw_sync_policy_group::Status::ENABLED && + (only_enabled || flow_group_map.status != rgw_sync_policy_group::Status::ALLOWED)) { + /* only return enabled groups */ + continue; + } + + for (auto& entry : flow_group_map.sources) { + rgw_sync_bucket_pipe pipe = entry.second; + if (!pipe.dest.match_bucket(effective_bucket)) { + continue; + } + + pipe.source.apply_bucket(effective_bucket); + pipe.dest.apply_bucket(effective_bucket); + + if (is_forbidden) { + ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): removing source pipe: " << pipe << dendl; + source_pipes->disable(pipe); + } else { + ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): adding source pipe: " << pipe << dendl; + source_pipes->insert(pipe); + } + } + + for (auto& entry : flow_group_map.dests) { + rgw_sync_bucket_pipe pipe = entry.second; + + if (!pipe.source.match_bucket(effective_bucket)) { + continue; + } + + pipe.source.apply_bucket(effective_bucket); + pipe.dest.apply_bucket(effective_bucket); + + if (is_forbidden) { + ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): removing dest pipe: " << pipe << dendl; + dest_pipes->disable(pipe); + } else { + ldpp_dout(dpp, 20) << __func__ << "(): flow manager (bucket=" << effective_bucket_key << "): adding dest pipe: " << pipe << dendl; + dest_pipes->insert(pipe); + } + } + } +} + + +RGWBucketSyncFlowManager::RGWBucketSyncFlowManager(CephContext *_cct, + const rgw_zone_id& _zone_id, + std::optional _bucket, + const RGWBucketSyncFlowManager *_parent) : cct(_cct), + zone_id(_zone_id), + bucket(_bucket), + parent(_parent) {} + + +void RGWSyncPolicyCompat::convert_old_sync_config(RGWSI_Zone *zone_svc, + RGWSI_SyncModules *sync_modules_svc, + rgw_sync_policy_info *ppolicy) +{ + bool found = false; + + rgw_sync_policy_info policy; + + auto& group = policy.groups["default"]; + auto& zonegroup = zone_svc->get_zonegroup(); + + for (const auto& ziter1 : zonegroup.zones) { + auto& id1 = ziter1.first; + const RGWZone& z1 = ziter1.second; + + for (const auto& ziter2 : zonegroup.zones) { + auto& id2 = ziter2.first; + const RGWZone& z2 = ziter2.second; + + if (id1 == id2) { + continue; + } + + if (z1.syncs_from(z2.name)) { + found = true; + rgw_sync_directional_rule *rule; + group.data_flow.find_or_create_directional(id2, + id1, + &rule); + } + } + } + + if (!found) { /* nothing syncs */ + return; + } + + rgw_sync_bucket_pipes pipes; + pipes.id = "all"; + pipes.source.all_zones = true; + pipes.dest.all_zones = true; + + group.pipes.emplace_back(std::move(pipes)); + + + group.status = rgw_sync_policy_group::Status::ENABLED; + + *ppolicy = std::move(policy); +} + +RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(RGWSI_Zone *_zone_svc, + RGWSI_SyncModules *sync_modules_svc, + RGWSI_Bucket_Sync *_bucket_sync_svc, + std::optional effective_zone) : zone_svc(_zone_svc) , + bucket_sync_svc(_bucket_sync_svc) { + zone_id = effective_zone.value_or(zone_svc->zone_id()); + flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), + zone_id, + nullopt, + nullptr)); + sync_policy = zone_svc->get_zonegroup().sync_policy; + + if (sync_policy.empty()) { + RGWSyncPolicyCompat::convert_old_sync_config(zone_svc, sync_modules_svc, &sync_policy); + legacy_config = true; + } +} + +RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, + const RGWBucketInfo& _bucket_info, + map&& _bucket_attrs) : parent(_parent), + bucket_info(_bucket_info), + bucket_attrs(std::move(_bucket_attrs)) { + if (_bucket_info.sync_policy) { + sync_policy = *_bucket_info.sync_policy; + } + legacy_config = parent->legacy_config; + bucket = _bucket_info.bucket; + zone_svc = parent->zone_svc; + bucket_sync_svc = parent->bucket_sync_svc; + flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), + parent->zone_id, + _bucket_info.bucket, + parent->flow_mgr.get())); +} + +RGWBucketSyncPolicyHandler::RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, + const rgw_bucket& _bucket, + std::optional _sync_policy) : parent(_parent) { + if (_sync_policy) { + sync_policy = *_sync_policy; + } + legacy_config = parent->legacy_config; + bucket = _bucket; + zone_svc = parent->zone_svc; + bucket_sync_svc = parent->bucket_sync_svc; + flow_mgr.reset(new RGWBucketSyncFlowManager(zone_svc->ctx(), + parent->zone_id, + _bucket, + parent->flow_mgr.get())); +} + +RGWBucketSyncPolicyHandler *RGWBucketSyncPolicyHandler::alloc_child(const RGWBucketInfo& bucket_info, + map&& bucket_attrs) const +{ + return new RGWBucketSyncPolicyHandler(this, bucket_info, std::move(bucket_attrs)); +} + +RGWBucketSyncPolicyHandler *RGWBucketSyncPolicyHandler::alloc_child(const rgw_bucket& bucket, + std::optional sync_policy) const +{ + return new RGWBucketSyncPolicyHandler(this, bucket, sync_policy); +} + +int RGWBucketSyncPolicyHandler::init(const DoutPrefixProvider *dpp, optional_yield y) +{ + int r = bucket_sync_svc->get_bucket_sync_hints(dpp, bucket.value_or(rgw_bucket()), + &source_hints, + &target_hints, + y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to initialize bucket sync policy handler: get_bucket_sync_hints() on bucket=" + << bucket << " returned r=" << r << dendl; + return r; + } + + flow_mgr->init(dpp, sync_policy); + + reflect(dpp, &source_pipes, + &target_pipes, + &sources, + &targets, + &source_zones, + &target_zones, + true); + + return 0; +} + +void RGWBucketSyncPolicyHandler::reflect(const DoutPrefixProvider *dpp, RGWBucketSyncFlowManager::pipe_set *psource_pipes, + RGWBucketSyncFlowManager::pipe_set *ptarget_pipes, + map *psources, + map *ptargets, + std::set *psource_zones, + std::set *ptarget_zones, + bool only_enabled) const +{ + RGWBucketSyncFlowManager::pipe_set _source_pipes; + RGWBucketSyncFlowManager::pipe_set _target_pipes; + map _sources; + map _targets; + std::set _source_zones; + std::set _target_zones; + + flow_mgr->reflect(dpp, bucket, &_source_pipes, &_target_pipes, only_enabled); + + for (auto& entry : _source_pipes.pipe_map) { + auto& pipe = entry.second; + if (!pipe.source.zone) { + continue; + } + _source_zones.insert(*pipe.source.zone); + _sources[*pipe.source.zone].insert(pipe); + } + + for (auto& entry : _target_pipes.pipe_map) { + auto& pipe = entry.second; + if (!pipe.dest.zone) { + continue; + } + _target_zones.insert(*pipe.dest.zone); + _targets[*pipe.dest.zone].insert(pipe); + } + + if (psource_pipes) { + *psource_pipes = std::move(_source_pipes); + } + if (ptarget_pipes) { + *ptarget_pipes = std::move(_target_pipes); + } + if (psources) { + *psources = std::move(_sources); + } + if (ptargets) { + *ptargets = std::move(_targets); + } + if (psource_zones) { + *psource_zones = std::move(_source_zones); + } + if (ptarget_zones) { + *ptarget_zones = std::move(_target_zones); + } +} + +multimap RGWBucketSyncPolicyHandler::get_all_sources() const +{ + multimap m; + + for (auto& source_entry : sources) { + auto& zone_id = source_entry.first; + + auto& pipes = source_entry.second.pipe_map; + + for (auto& entry : pipes) { + auto& pipe = entry.second; + m.insert(make_pair(zone_id, pipe)); + } + } + + for (auto& pipe : resolved_sources) { + if (!pipe.source.zone) { + continue; + } + + m.insert(make_pair(*pipe.source.zone, pipe)); + } + + return m; +} + +multimap RGWBucketSyncPolicyHandler::get_all_dests() const +{ + multimap m; + + for (auto& dest_entry : targets) { + auto& zone_id = dest_entry.first; + + auto& pipes = dest_entry.second.pipe_map; + + for (auto& entry : pipes) { + auto& pipe = entry.second; + m.insert(make_pair(zone_id, pipe)); + } + } + + for (auto& pipe : resolved_dests) { + if (!pipe.dest.zone) { + continue; + } + + m.insert(make_pair(*pipe.dest.zone, pipe)); + } + + return m; +} + +multimap RGWBucketSyncPolicyHandler::get_all_dests_in_zone(const rgw_zone_id& zone_id) const +{ + multimap m; + + auto iter = targets.find(zone_id); + if (iter != targets.end()) { + auto& pipes = iter->second.pipe_map; + + for (auto& entry : pipes) { + auto& pipe = entry.second; + m.insert(make_pair(zone_id, pipe)); + } + } + + for (auto& pipe : resolved_dests) { + if (!pipe.dest.zone || + *pipe.dest.zone != zone_id) { + continue; + } + + m.insert(make_pair(*pipe.dest.zone, pipe)); + } + + return m; +} + +void RGWBucketSyncPolicyHandler::get_pipes(std::set *_sources, std::set *_targets, + std::optional filter_peer) { /* return raw pipes */ + for (auto& entry : source_pipes.pipe_map) { + auto& source_pipe = entry.second; + if (!filter_peer || + source_pipe.source.match(*filter_peer)) { + _sources->insert(source_pipe); + } + } + + for (auto& entry : target_pipes.pipe_map) { + auto& target_pipe = entry.second; + if (!filter_peer || + target_pipe.dest.match(*filter_peer)) { + _targets->insert(target_pipe); + } + } +} + +bool RGWBucketSyncPolicyHandler::bucket_exports_object(const std::string& obj_name, const RGWObjTags& tags) const { + if (bucket_exports_data()) { + for (auto& entry : target_pipes.pipe_map) { + auto& filter = entry.second.params.source.filter; + if (filter.check_prefix(obj_name) && filter.check_tags(tags.get_tags())) { + return true; + } + } + } + + return false; +} + +bool RGWBucketSyncPolicyHandler::bucket_exports_data() const +{ + if (!bucket) { + return false; + } + + if (!zone_svc->sync_module_exports_data()) { + return false; + } + + if (bucket_is_sync_source()) { + return true; + } + + return (zone_svc->need_to_log_data() && + bucket_info->datasync_flag_enabled()); +} + +bool RGWBucketSyncPolicyHandler::bucket_imports_data() const +{ + return bucket_is_sync_target(); +} + diff --git a/src/rgw/rgw_bucket_sync.h b/src/rgw/rgw_bucket_sync.h new file mode 100644 index 000000000000..1ad936429435 --- /dev/null +++ b/src/rgw/rgw_bucket_sync.h @@ -0,0 +1,446 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2018 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include "rgw_common.h" +#include "rgw_sync_policy.h" +#include "rgw_zone.h" + +class RGWSI_Zone; +class RGWSI_SyncModules; +class RGWSI_Bucket_Sync; + +struct rgw_sync_group_pipe_map; +struct rgw_sync_bucket_pipes; +struct rgw_sync_policy_info; + +struct rgw_sync_group_pipe_map { + rgw_zone_id zone; + std::optional bucket; + + rgw_sync_policy_group::Status status{rgw_sync_policy_group::Status::UNKNOWN}; + + using zb_pipe_map_t = std::multimap; + + zb_pipe_map_t sources; /* all the pipes where zone is pulling from */ + zb_pipe_map_t dests; /* all the pipes that pull from zone */ + + std::set *pall_zones{nullptr}; + rgw_sync_data_flow_group *default_flow{nullptr}; /* flow to use if policy doesn't define it, + used in the case of bucket sync policy, not at the + zonegroup level */ + + void dump(ceph::Formatter *f) const; + + template + void try_add_to_pipe_map(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + zb_pipe_map_t *pipe_map, + CB1 filter_cb, + CB2 call_filter_cb); + + template + void try_add_source(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + CB filter_cb); + + template + void try_add_dest(const rgw_zone_id& source_zone, + const rgw_zone_id& dest_zone, + const std::vector& pipes, + CB filter_cb); + + std::pair find_pipes(const zb_pipe_map_t& m, + const rgw_zone_id& zone, + std::optional b) const; + + template + void init(const DoutPrefixProvider *dpp, CephContext *cct, + const rgw_zone_id& _zone, + std::optional _bucket, + const rgw_sync_policy_group& group, + rgw_sync_data_flow_group *_default_flow, + std::set *_pall_zones, + CB filter_cb); + + /* + * find all relevant pipes in our zone that match {dest_bucket} <- {source_zone, source_bucket} + */ + std::vector find_source_pipes(const rgw_zone_id& source_zone, + std::optional source_bucket, + std::optional dest_bucket) const; + + /* + * find all relevant pipes in other zones that pull from a specific + * source bucket in out zone {source_bucket} -> {dest_zone, dest_bucket} + */ + std::vector find_dest_pipes(std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket) const; + + /* + * find all relevant pipes from {source_zone, source_bucket} -> {dest_zone, dest_bucket} + */ + std::vector find_pipes(const rgw_zone_id& source_zone, + std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket) const; +}; + +class RGWSyncPolicyCompat { +public: + static void convert_old_sync_config(RGWSI_Zone *zone_svc, + RGWSI_SyncModules *sync_modules_svc, + rgw_sync_policy_info *ppolicy); +}; + +class RGWBucketSyncFlowManager { + friend class RGWBucketSyncPolicyHandler; +public: + struct endpoints_pair { + rgw_sync_bucket_entity source; + rgw_sync_bucket_entity dest; + + endpoints_pair() {} + endpoints_pair(const rgw_sync_bucket_pipe& pipe) { + source = pipe.source; + dest = pipe.dest; + } + + bool operator<(const endpoints_pair& e) const { + if (source < e.source) { + return true; + } + if (e.source < source) { + return false; + } + return (dest < e.dest); + } + }; + + /* + * pipe_rules: deal with a set of pipes that have common endpoints_pair + */ + class pipe_rules { + std::list pipes; + + public: + using prefix_map_t = std::multimap; + + std::map tag_refs; + prefix_map_t prefix_refs; + + void insert(const rgw_sync_bucket_pipe& pipe); + + bool find_basic_info_without_tags(const rgw_obj_key& key, + std::optional *user, + std::optional *acl_translation, + std::optional *storage_class, + rgw_sync_pipe_params::Mode *mode, + bool *need_more_info) const; + bool find_obj_params(const rgw_obj_key& key, + const RGWObjTags::tag_map_t& tags, + rgw_sync_pipe_params *params) const; + + void scan_prefixes(std::vector *prefixes) const; + + prefix_map_t::const_iterator prefix_begin() const { + return prefix_refs.begin(); + } + prefix_map_t::const_iterator prefix_search(const std::string& s) const; + prefix_map_t::const_iterator prefix_end() const { + return prefix_refs.end(); + } + }; + + using pipe_rules_ref = std::shared_ptr; + + /* + * pipe_handler: extends endpoints_rule to point at the corresponding rules handler + */ + struct pipe_handler : public endpoints_pair { + pipe_rules_ref rules; + + pipe_handler() {} + pipe_handler(pipe_rules_ref& _rules, + const rgw_sync_bucket_pipe& _pipe) : endpoints_pair(_pipe), + rules(_rules) {} + bool specific() const { + return source.specific() && dest.specific(); + } + + bool find_basic_info_without_tags(const rgw_obj_key& key, + std::optional *user, + std::optional *acl_translation, + std::optional *storage_class, + rgw_sync_pipe_params::Mode *mode, + bool *need_more_info) const { + if (!rules) { + return false; + } + return rules->find_basic_info_without_tags(key, user, acl_translation, storage_class, mode, need_more_info); + } + + bool find_obj_params(const rgw_obj_key& key, + const RGWObjTags::tag_map_t& tags, + rgw_sync_pipe_params *params) const { + if (!rules) { + return false; + } + return rules->find_obj_params(key, tags, params); + } + }; + + struct pipe_set { + std::map rules; + std::multimap pipe_map; + std::multimap disabled_pipe_map; + + std::set handlers; + + using iterator = std::set::iterator; + + void clear() { + rules.clear(); + pipe_map.clear(); + disabled_pipe_map.clear(); + handlers.clear(); + } + + void insert(const rgw_sync_bucket_pipe& pipe); + void remove_all(); + void disable(const rgw_sync_bucket_pipe& pipe); + + iterator begin() const { + return handlers.begin(); + } + + iterator end() const { + return handlers.end(); + } + + void dump(ceph::Formatter *f) const; + }; + +private: + + CephContext *cct; + + rgw_zone_id zone_id; + std::optional bucket; + + const RGWBucketSyncFlowManager *parent{nullptr}; + + std::map flow_groups; + + std::set all_zones; + + bool allowed_data_flow(const rgw_zone_id& source_zone, + std::optional source_bucket, + const rgw_zone_id& dest_zone, + std::optional dest_bucket, + bool check_activated) const; + + /* + * find all the matching flows om a flow map for a specific bucket + */ + void update_flow_maps(const rgw_sync_bucket_pipes& pipe); + + void init(const DoutPrefixProvider *dpp, const rgw_sync_policy_info& sync_policy); + +public: + + RGWBucketSyncFlowManager(CephContext *_cct, + const rgw_zone_id& _zone_id, + std::optional _bucket, + const RGWBucketSyncFlowManager *_parent); + + void reflect(const DoutPrefixProvider *dpp, std::optional effective_bucket, + pipe_set *flow_by_source, + pipe_set *flow_by_dest, + bool only_enabled) const; + +}; + +static inline std::ostream& operator<<(std::ostream& os, const RGWBucketSyncFlowManager::endpoints_pair& e) { + os << e.dest << " -> " << e.source; + return os; +} + +class RGWBucketSyncPolicyHandler { + bool legacy_config{false}; + const RGWBucketSyncPolicyHandler *parent{nullptr}; + RGWSI_Zone *zone_svc; + RGWSI_Bucket_Sync *bucket_sync_svc; + rgw_zone_id zone_id; + std::optional bucket_info; + std::optional > bucket_attrs; + std::optional bucket; + std::unique_ptr flow_mgr; + rgw_sync_policy_info sync_policy; + + RGWBucketSyncFlowManager::pipe_set source_pipes; + RGWBucketSyncFlowManager::pipe_set target_pipes; + + std::map sources; /* source pipes by source zone id */ + std::map targets; /* target pipes by target zone id */ + + std::set source_zones; + std::set target_zones; + + std::set source_hints; + std::set target_hints; + std::set resolved_sources; + std::set resolved_dests; + + + bool bucket_is_sync_source() const { + return !targets.empty() || !resolved_dests.empty(); + } + + bool bucket_is_sync_target() const { + return !sources.empty() || !resolved_sources.empty(); + } + + RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, + const RGWBucketInfo& _bucket_info, + std::map&& _bucket_attrs); + + RGWBucketSyncPolicyHandler(const RGWBucketSyncPolicyHandler *_parent, + const rgw_bucket& _bucket, + std::optional _sync_policy); +public: + RGWBucketSyncPolicyHandler(RGWSI_Zone *_zone_svc, + RGWSI_SyncModules *sync_modules_svc, + RGWSI_Bucket_Sync *bucket_sync_svc, + std::optional effective_zone = std::nullopt); + + RGWBucketSyncPolicyHandler *alloc_child(const RGWBucketInfo& bucket_info, + std::map&& bucket_attrs) const; + RGWBucketSyncPolicyHandler *alloc_child(const rgw_bucket& bucket, + std::optional sync_policy) const; + + int init(const DoutPrefixProvider *dpp, optional_yield y); + + void reflect(const DoutPrefixProvider *dpp, RGWBucketSyncFlowManager::pipe_set *psource_pipes, + RGWBucketSyncFlowManager::pipe_set *ptarget_pipes, + std::map *psources, + std::map *ptargets, + std::set *psource_zones, + std::set *ptarget_zones, + bool only_enabled) const; + + void set_resolved_hints(std::set&& _resolved_sources, + std::set&& _resolved_dests) { + resolved_sources = std::move(_resolved_sources); + resolved_dests = std::move(_resolved_dests); + } + + const std::set& get_resolved_source_hints() { + return resolved_sources; + } + + const std::set& get_resolved_dest_hints() { + return resolved_dests; + } + + const std::set& get_source_zones() const { + return source_zones; + } + + const std::set& get_target_zones() const { + return target_zones; + } + + const std::map& get_sources() { + return sources; + } + + std::multimap get_all_sources() const; + std::multimap get_all_dests() const; + std::multimap get_all_dests_in_zone(const rgw_zone_id& zone_id) const; + + const std::map& get_targets() { + return targets; + } + + const std::optional& get_bucket_info() const { + return bucket_info; + } + + const std::optional >& get_bucket_attrs() const { + return bucket_attrs; + } + + void get_pipes(RGWBucketSyncFlowManager::pipe_set **_sources, RGWBucketSyncFlowManager::pipe_set **_targets) { /* return raw pipes (with zone name) */ + *_sources = &source_pipes; + *_targets = &target_pipes; + } + void get_pipes(std::set *sources, std::set *targets, + std::optional filter_peer); + + const std::set& get_source_hints() const { + return source_hints; + } + + const std::set& get_target_hints() const { + return target_hints; + } + + bool bucket_exports_object(const std::string& obj_name, const RGWObjTags& tags) const; + bool bucket_exports_data() const; + bool bucket_imports_data() const; + + const rgw_sync_policy_info& get_sync_policy() const { + return sync_policy; + } + + bool is_legacy_config() const { + return legacy_config; + } +}; + +struct rgw_bucket_sync_pair_info { + RGWBucketSyncFlowManager::pipe_handler handler; /* responsible for sync filters */ + rgw_bucket_shard source_bs; + rgw_bucket dest_bucket; +}; + +inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pair_info& p) { + if (p.source_bs.bucket == p.dest_bucket) { + return out << p.source_bs; + } + return out << p.source_bs << "->" << p.dest_bucket; +} + +struct rgw_bucket_sync_pipe { + rgw_bucket_sync_pair_info info; + RGWBucketInfo source_bucket_info; + std::map source_bucket_attrs; + RGWBucketInfo dest_bucket_info; + std::map dest_bucket_attrs; + + RGWBucketSyncFlowManager::pipe_rules_ref& get_rules() { + return info.handler.rules; + } +}; + +inline std::ostream& operator<<(std::ostream& out, const rgw_bucket_sync_pipe& p) { + return out << p.info; +} diff --git a/src/rgw/rgw_object_expirer.cc b/src/rgw/rgw_object_expirer.cc index 43540c1b8d2b..d5831b93fbfc 100644 --- a/src/rgw/rgw_object_expirer.cc +++ b/src/rgw/rgw_object_expirer.cc @@ -31,7 +31,7 @@ #include "rgw_formats.h" #include "rgw_usage.h" #include "rgw_object_expirer_core.h" -#include "driver/rados/rgw_zone.h" +#include "rgw_zone.h" #include "rgw_sal_config.h" #define dout_subsys ceph_subsys_rgw @@ -88,8 +88,7 @@ int main(const int argc, const char **argv) ceph::async::io_context_pool context_pool{cct->_conf->rgw_thread_pool_size}; const DoutPrefix dp(cct.get(), dout_subsys, "rgw object expirer: "); - DriverManager::Config cfg; - cfg.store_name = "rados"; + DriverManager::Config cfg = DriverManager::get_config(false, g_ceph_context); cfg.filter_name = "none"; std::unique_ptr cfgstore; auto config_store_type = g_conf().get_val("rgw_config_store"); diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc index 242cb2eb1dfc..c3c5e2b48b52 100644 --- a/src/rgw/rgw_period.cc +++ b/src/rgw/rgw_period.cc @@ -13,6 +13,98 @@ std::string period_info_oid_prefix = "periods."; #define FIRST_EPOCH 1 +int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup, + const string& zonegroup_id) const +{ + map::const_iterator iter; + if (!zonegroup_id.empty()) { + iter = period_map.zonegroups.find(zonegroup_id); + } else { + iter = period_map.zonegroups.find("default"); + } + if (iter != period_map.zonegroups.end()) { + zonegroup = iter->second; + return 0; + } + + return -ENOENT; +} + +static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Driver* driver, rgw_meta_sync_status *sync_status) +{ + int r = -ENOTSUP; +#ifdef WITH_RADOSGW_RADOS + rgw::sal::RadosStore* rados_store = static_cast(driver); + // initialize a sync status manager to read the status + RGWMetaSyncStatusManager mgr(rados_store, rados_store->svc()->async_processor); + r = mgr.init(dpp); + if (r < 0) { + return r; + } + r = mgr.read_sync_status(dpp, sync_status); + mgr.stop(); +#endif + return r; +} + +int RGWPeriod::update_sync_status(const DoutPrefixProvider *dpp, + rgw::sal::Driver* driver, /* for now */ + const RGWPeriod ¤t_period, + std::ostream& error_stream, + bool force_if_stale) +{ + rgw_meta_sync_status status; + int r = read_sync_status(dpp, driver, &status); + if (r < 0) { + ldpp_dout(dpp, 0) << "period failed to read sync status: " + << cpp_strerror(-r) << dendl; + return r; + } + + std::vector markers; + + const auto current_epoch = current_period.get_realm_epoch(); + if (current_epoch != status.sync_info.realm_epoch) { + // no sync status markers for the current period + ceph_assert(current_epoch > status.sync_info.realm_epoch); + const int behind = current_epoch - status.sync_info.realm_epoch; + if (!force_if_stale && current_epoch > 1) { + error_stream << "ERROR: This zone is " << behind << " period(s) behind " + "the current master zone in metadata sync. If this zone is promoted " + "to master, any metadata changes during that time are likely to " + "be lost.\n" + "Waiting for this zone to catch up on metadata sync (see " + "'radosgw-admin sync status') is recommended.\n" + "To promote this zone to master anyway, add the flag " + "--yes-i-really-mean-it." << std::endl; + return -EINVAL; + } + // empty sync status markers - other zones will skip this period during + // incremental metadata sync + markers.resize(status.sync_info.num_shards); + } else { + markers.reserve(status.sync_info.num_shards); + for (auto& i : status.sync_markers) { + auto& marker = i.second; + // filter out markers from other periods + if (marker.realm_epoch != current_epoch) { + marker.marker.clear(); + } + markers.emplace_back(std::move(marker.marker)); + } + } + + std::swap(sync_status, markers); + return 0; +} + +void RGWPeriod::generate_test_instances(list &o) +{ + RGWPeriod *z = new RGWPeriod; + o.push_back(z); + o.push_back(new RGWPeriod); +} + const string& RGWPeriod::get_info_oid_prefix() const { return period_info_oid_prefix; diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc index a586b19b815c..c3c8ea78d906 100644 --- a/src/rgw/rgw_realm_reloader.cc +++ b/src/rgw/rgw_realm_reloader.cc @@ -118,8 +118,7 @@ void RGWRealmReloader::reload() ldpp_dout(&dp, 1) << "Creating new driver" << dendl; // recreate and initialize a new driver - DriverManager::Config cfg; - cfg.store_name = "rados"; + DriverManager::Config cfg = DriverManager::get_config(false, g_ceph_context); cfg.filter_name = "none"; env.driver = DriverManager::get_storage(&dp, cct, cfg, io_context, *env.site, diff --git a/src/rgw/rgw_rest_iam.cc b/src/rgw/rgw_rest_iam.cc index e074706a9dc2..70d274946d25 100644 --- a/src/rgw/rgw_rest_iam.cc +++ b/src/rgw/rgw_rest_iam.cc @@ -13,7 +13,7 @@ #include "rgw_rest_iam_group.h" #include "rgw_rest_iam_user.h" #include "rgw_rest_conn.h" -#include "driver/rados/rgw_zone.h" +#include "rgw_zone.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 7195964ae8d8..c11b5a80207a 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -48,7 +48,9 @@ //#define dout_context g_ceph_context extern "C" { +#ifdef WITH_RADOSGW_RADOS extern rgw::sal::Driver* newRadosStore(boost::asio::io_context* io_context); +#endif #ifdef WITH_RADOSGW_DBSTORE extern rgw::sal::Driver* newDBStore(CephContext *cct); #endif @@ -86,6 +88,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* rgw::sal::Driver* driver{nullptr}; if (cfg.store_name.compare("rados") == 0) { +#ifdef WITH_RADOSGW_RADOS driver = newRadosStore(&io_context); RGWRados* rados = static_cast(driver)->getRados(); @@ -110,7 +113,9 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* delete driver; return nullptr; } +#endif } +#ifdef WITH_RADOSGW_RADOS else if (cfg.store_name.compare("d3n") == 0) { driver = new rgw::sal::RadosStore(io_context); RGWRados* rados = new D3nRGWDataCache; @@ -151,6 +156,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* lsubdout(cct, rgw, 1) << "rgw_d3n: rgw_d3n_l1_eviction_policy=" << cct->_conf->rgw_d3n_l1_eviction_policy << dendl; } +#endif #ifdef WITH_RADOSGW_DBSTORE else if (cfg.store_name.compare("dbstore") == 0) { driver = newDBStore(cct); @@ -238,6 +244,7 @@ rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvi { rgw::sal::Driver* driver = nullptr; if (cfg.store_name.compare("rados") == 0) { +#ifdef WITH_RADOSGW_RADOS driver = newRadosStore(&io_context); RGWRados* rados = static_cast(driver)->getRados(); @@ -259,6 +266,7 @@ rgw::sal::Driver* DriverManager::init_raw_storage_provider(const DoutPrefixProvi delete driver; return nullptr; } +#endif } else if (cfg.store_name.compare("dbstore") == 0) { #ifdef WITH_RADOSGW_DBSTORE driver = newDBStore(cct); @@ -320,6 +328,7 @@ DriverManager::Config DriverManager::get_config(bool admin, CephContext* cct) // Get the store backend const auto& config_store = g_conf().get_val("rgw_backend_store"); if (config_store == "rados") { +#ifdef WITH_RADOSGW_RADOS cfg.store_name = "rados"; /* Check to see if d3n is configured, but only for non-admin */ @@ -334,6 +343,7 @@ DriverManager::Config DriverManager::get_config(bool admin, CephContext* cct) cfg.store_name = "d3n"; } } +#endif } #ifdef WITH_RADOSGW_DBSTORE else if (config_store == "dbstore") { @@ -373,14 +383,18 @@ auto DriverManager::create_config_store(const DoutPrefixProvider* dpp, -> std::unique_ptr { try { +#ifdef WITH_RADOSGW_RADOS if (type == "rados") { return rgw::rados::create_config_store(dpp); + } +#endif #ifdef WITH_RADOSGW_DBSTORE - } else if (type == "dbstore") { + if (type == "dbstore") { const auto uri = g_conf().get_val("dbstore_config_uri"); return rgw::dbstore::create_config_store(dpp, uri); + } #endif - } else if (type == "json") { + if (type == "json") { auto filename = g_conf().get_val("rgw_json_config"); return rgw::sal::create_json_config_store(dpp, filename); } else { diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 6636d8bea19b..ee8d101f3ecc 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -110,3 +110,65 @@ void rgw_get_anon_user(RGWUserInfo& info) info.access_keys.clear(); } +static bool char_is_unreserved_url(char c) +{ + if (isalnum(c)) + return true; + + switch (c) { + case '-': + case '.': + case '_': + case '~': + return true; + default: + return false; + } +} + +static bool validate_access_key(string& key) +{ + const char *p = key.c_str(); + while (*p) { + if (!char_is_unreserved_url(*p)) + return false; + p++; + } + return true; +} + +int rgw_generate_access_key(const DoutPrefixProvider* dpp, + optional_yield y, + rgw::sal::Driver* driver, + std::string& access_key_id) +{ + std::string id; + int r = 0; + + do { + id.resize(PUBLIC_ID_LEN + 1); + gen_rand_alphanumeric_upper(dpp->get_cct(), id.data(), id.size()); + id.pop_back(); // remove trailing null + + if (!validate_access_key(id)) + continue; + + std::unique_ptr duplicate_check; + r = driver->get_user_by_access_key(dpp, id, y, &duplicate_check); + } while (r == 0); + + if (r == -ENOENT) { + access_key_id = std::move(id); + return 0; + } + return r; +} + +void rgw_generate_secret_key(CephContext* cct, + std::string& secret_key) +{ + char secret_key_buf[SECRET_KEY_LEN + 1]; + gen_rand_alphanumeric_plain(cct, secret_key_buf, sizeof(secret_key_buf)); + secret_key = secret_key_buf; +} + diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index 9ec69c1fa2c3..4e0fddabb3cc 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -16,6 +16,12 @@ #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw +RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){} + +#define FIRST_EPOCH 1 + +struct RGWAccessKey; + namespace rgw_zone_defaults { static std::string default_bucket_index_pool_suffix = "rgw.buckets.index"; @@ -259,7 +265,148 @@ void add_zone_pools(const RGWZoneParams& info, } } +void RGWDefaultZoneGroupInfo::dump(Formatter *f) const { + encode_json("default_zonegroup", default_zonegroup, f); +} + +void RGWDefaultZoneGroupInfo::decode_json(JSONObj *obj) { + + JSONDecoder::decode_json("default_zonegroup", default_zonegroup, obj); + /* backward compatability with region */ + if (default_zonegroup.empty()) { + JSONDecoder::decode_json("default_region", default_zonegroup, obj); + } +} + +int RGWZoneGroup::equals(const string& other_zonegroup) const +{ + if (is_master && other_zonegroup.empty()) + return true; + + return (id == other_zonegroup); +} + +void RGWDefaultSystemMetaObjInfo::dump(Formatter *f) const { + encode_json("default_id", default_id, f); +} + +void RGWDefaultSystemMetaObjInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("default_id", default_id, obj); +} + +const string& RGWZoneParams::get_compression_type(const rgw_placement_rule& placement_rule) const +{ + static const std::string NONE{"none"}; + auto p = placement_pools.find(placement_rule.name); + if (p == placement_pools.end()) { + return NONE; + } + const auto& type = p->second.get_compression_type(placement_rule.get_storage_class()); + return !type.empty() ? type : NONE; +} + +// run an MD5 hash on the zone_id and return the first 32 bits +static uint32_t gen_short_zone_id(const std::string zone_id) +{ + unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; + MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size()); + hash.Final(md5); + + uint32_t short_id; + memcpy((char *)&short_id, md5, sizeof(short_id)); + return std::max(short_id, 1u); +} + +int RGWPeriodMap::update(const RGWZoneGroup& zonegroup, CephContext *cct) +{ + if (zonegroup.is_master_zonegroup() && (!master_zonegroup.empty() && zonegroup.get_id() != master_zonegroup)) { + ldout(cct,0) << "Error updating periodmap, multiple master zonegroups configured "<< dendl; + ldout(cct,0) << "master zonegroup: " << master_zonegroup << " and " << zonegroup.get_id() <::iterator iter = zonegroups.find(zonegroup.get_id()); + if (iter != zonegroups.end()) { + RGWZoneGroup& old_zonegroup = iter->second; + if (!old_zonegroup.api_name.empty()) { + zonegroups_by_api.erase(old_zonegroup.api_name); + } + } + zonegroups[zonegroup.get_id()] = zonegroup; + + if (!zonegroup.api_name.empty()) { + zonegroups_by_api[zonegroup.api_name] = zonegroup; + } + + if (zonegroup.is_master_zonegroup()) { + master_zonegroup = zonegroup.get_id(); + } else if (master_zonegroup == zonegroup.get_id()) { + master_zonegroup = ""; + } + + for (auto& i : zonegroup.zones) { + auto& zone = i.second; + if (short_zone_ids.find(zone.id) != short_zone_ids.end()) { + continue; + } + // calculate the zone's short id + uint32_t short_id = gen_short_zone_id(zone.id); + + // search for an existing zone with the same short id + for (auto& s : short_zone_ids) { + if (s.second == short_id) { + ldout(cct, 0) << "New zone '" << zone.name << "' (" << zone.id + << ") generates the same short_zone_id " << short_id + << " as existing zone id " << s.first << dendl; + return -EEXIST; + } + } + + short_zone_ids[zone.id] = short_id; + } + + return 0; +} + +uint32_t RGWPeriodMap::get_zone_short_id(const string& zone_id) const +{ + auto i = short_zone_ids.find(zone_id); + if (i == short_zone_ids.end()) { + return 0; + } + return i->second; +} + +bool RGWPeriodMap::find_zone_by_name(const string& zone_name, + RGWZoneGroup *zonegroup, + RGWZone *zone) const +{ + for (auto& iter : zonegroups) { + auto& zg = iter.second; + for (auto& ziter : zg.zones) { + auto& z = ziter.second; + + if (z.name == zone_name) { + *zonegroup = zg; + *zone = z; + return true; + } + } + } + + return false; +} + namespace rgw { +/// Generate a random uuid for realm/period/zonegroup/zone ids +std::string gen_random_uuid() +{ + uuid_d uuid; + uuid.generate_random(); + return uuid.to_string(); +} int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y, @@ -829,5 +976,1168 @@ int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup, return 0; } +int read_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view realm_id, + std::string_view realm_name, + RGWRealm& info, + std::unique_ptr* writer) +{ + if (!realm_id.empty()) { + return cfgstore->read_realm_by_id(dpp, y, realm_id, info, writer); + } + if (!realm_name.empty()) { + return cfgstore->read_realm_by_name(dpp, y, realm_name, info, writer); + } + return cfgstore->read_default_realm(dpp, y, info, writer); +} + +int create_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWRealm& info, + std::unique_ptr* writer_out) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a realm name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // if the realm already has a current_period, just make sure it exists + std::optional period; + if (!info.current_period.empty()) { + period.emplace(); + int r = cfgstore->read_period(dpp, y, info.current_period, + std::nullopt, *period); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to read realm's current_period=" + << info.current_period << " with " << cpp_strerror(r) << dendl; + return r; + } + } + + // create the realm + std::unique_ptr writer; + int r = cfgstore->create_realm(dpp, y, exclusive, info, &writer); + if (r < 0) { + return r; + } + + if (!period) { + // initialize and exclusive-create the initial period + period.emplace(); + period->id = gen_random_uuid(); + period->period_map.id = period->id; + period->epoch = FIRST_EPOCH; + period->realm_id = info.id; + + r = cfgstore->create_period(dpp, y, true, *period); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to create the initial period id=" + << period->id << " for realm " << info.name + << " with " << cpp_strerror(r) << dendl; + return r; + } + } + + // update the realm's current_period + r = realm_set_current_period(dpp, y, cfgstore, *writer, info, *period); + if (r < 0) { + return r; + } + + if (writer_out) { + *writer_out = std::move(writer); + } + return 0; +} + +int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWRealm& info, + bool exclusive) +{ + return cfgstore->write_default_realm_id(dpp, y, exclusive, info.id); +} + +int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + sal::RealmWriter& writer, RGWRealm& realm, + const RGWPeriod& period) +{ + // update realm epoch to match the period's + if (realm.epoch > period.realm_epoch) { + ldpp_dout(dpp, -1) << __func__ << " with old realm epoch " + << period.realm_epoch << ", current epoch=" << realm.epoch << dendl; + return -EINVAL; + } + if (realm.epoch == period.realm_epoch && realm.current_period != period.id) { + ldpp_dout(dpp, -1) << __func__ << " with same realm epoch " + << period.realm_epoch << ", but different period id " + << period.id << " != " << realm.current_period << dendl; + return -EINVAL; + } + + realm.epoch = period.realm_epoch; + realm.current_period = period.id; + + // update the realm object + int r = writer.write(dpp, y, realm); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to overwrite realm " + << realm.name << " with " << cpp_strerror(r) << dendl; + return r; + } + + // reflect the zonegroup and period config + (void) reflect_period(dpp, y, cfgstore, period); + return 0; +} + +int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWPeriod& info) +{ + // overwrite the local period config and zonegroup objects + constexpr bool exclusive = false; + + int r = cfgstore->write_period_config(dpp, y, exclusive, info.realm_id, + info.period_config); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to store period config for realm id=" + << info.realm_id << " with " << cpp_strerror(r) << dendl; + return r; + } + + for (auto& [zonegroup_id, zonegroup] : info.period_map.zonegroups) { + r = cfgstore->create_zonegroup(dpp, y, exclusive, zonegroup, nullptr); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to store zonegroup id=" + << zonegroup_id << " with " << cpp_strerror(r) << dendl; + return r; + } + if (zonegroup.is_master) { + // set master as default if no default exists + constexpr bool exclusive = true; + r = set_default_zonegroup(dpp, y, cfgstore, zonegroup, exclusive); + if (r == 0) { + ldpp_dout(dpp, 1) << "Set the period's master zonegroup " + << zonegroup.name << " as the default" << dendl; + } + } + } + return 0; +} + +std::string get_staging_period_id(std::string_view realm_id) +{ + return string_cat_reserve(realm_id, ":staging"); +} + +void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info) +{ + ldpp_dout(dpp, 20) << __func__ << " realm id=" << info.realm_id + << " period id=" << info.id << dendl; + + info.predecessor_uuid = std::move(info.id); + info.id = get_staging_period_id(info.realm_id); + info.period_map.reset(); + info.realm_epoch++; +} + +int update_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, RGWPeriod& info) +{ + // clear zone short ids of removed zones. period_map.update() will add the + // remaining zones back + info.period_map.short_zone_ids.clear(); + + // list all zonegroups in the realm + rgw::sal::ListResult listing; + std::array zonegroup_names; // list in pages of 1000 + do { + int ret = cfgstore->list_zonegroup_names(dpp, y, listing.next, + zonegroup_names, listing); + if (ret < 0) { + std::cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + for (const auto& name : listing.entries) { + RGWZoneGroup zg; + ret = cfgstore->read_zonegroup_by_name(dpp, y, name, zg, nullptr); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to read zonegroup " + << name << ": " << cpp_strerror(-ret) << dendl; + continue; + } + + if (zg.realm_id != info.realm_id) { + ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() + << " with realm id " << zg.realm_id + << ", not on our realm " << info.realm_id << dendl; + continue; + } + + if (zg.master_zone.empty()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; + return -EINVAL; + } + + if (zg.zones.find(zg.master_zone) == zg.zones.end()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() + << " has a non existent master zone "<< dendl; + return -EINVAL; + } + + if (zg.is_master_zonegroup()) { + info.master_zonegroup = zg.get_id(); + info.master_zone = zg.master_zone; + } + + ret = info.period_map.update(zg, dpp->get_cct()); + if (ret < 0) { + return ret; + } + } // foreach name in listing.entries + } while (!listing.next.empty()); + + // read the realm's current period config + int ret = cfgstore->read_period_config(dpp, y, info.realm_id, + info.period_config); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " + << cpp_strerror(ret) << dendl; + return ret; + } + + return 0; +} + +int commit_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, sal::Driver* driver, + RGWRealm& realm, sal::RealmWriter& realm_writer, + const RGWPeriod& current_period, + RGWPeriod& info, std::ostream& error_stream, + bool force_if_stale, const rgw::SiteConfig& site) +{ + ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id + << " period " << current_period.id << dendl; + + // gateway must be in the master zone to commit + if (info.master_zone != site.get_zone_params().id) { + error_stream << "Cannot commit period on zone " + << site.get_zone_params().id << ", it must be sent to " + "the period's master zone " << info.master_zone << '.' << std::endl; + return -EINVAL; + } + // period predecessor must match current period + if (info.predecessor_uuid != current_period.id) { + error_stream << "Period predecessor " << info.predecessor_uuid + << " does not match current period " << current_period.id + << ". Use 'period pull' to get the latest period from the master, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // realm epoch must be 1 greater than current period + if (info.realm_epoch != current_period.realm_epoch + 1) { + error_stream << "Period's realm epoch " << info.realm_epoch + << " does not come directly after current realm epoch " + << current_period.realm_epoch << ". Use 'realm pull' to get the " + "latest realm and period from the master zone, reapply your changes, " + "and try again." << std::endl; + return -EINVAL; + } + // did the master zone change? + if (info.master_zone != current_period.master_zone) { + // store the current metadata sync status in the period + int r = info.update_sync_status(dpp, driver, current_period, + error_stream, force_if_stale); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update metadata sync status: " + << cpp_strerror(-r) << dendl; + return r; + } + // create an object with a new period id + info.period_map.id = info.id = gen_random_uuid(); + info.epoch = FIRST_EPOCH; + + constexpr bool exclusive = true; + r = cfgstore->create_period(dpp, y, exclusive, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; + return r; + } + // set as current period + r = realm_set_current_period(dpp, y, cfgstore, realm_writer, realm, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update realm's current period: " + << cpp_strerror(-r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " + << info.id << dendl; + return 0; + } + // period must be based on current epoch + if (info.epoch != current_period.epoch) { + error_stream << "Period epoch " << info.epoch << " does not match " + "predecessor epoch " << current_period.epoch << ". Use " + "'period pull' to get the latest epoch from the master zone, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // set period as next epoch + info.id = current_period.id; + info.epoch = current_period.epoch + 1; + info.predecessor_uuid = current_period.predecessor_uuid; + info.realm_epoch = current_period.realm_epoch; + // write the period + constexpr bool exclusive = true; + int r = cfgstore->create_period(dpp, y, exclusive, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(r) << dendl; + return r; + } + r = reflect_period(dpp, y, cfgstore, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Committed new epoch " << info.epoch + << " for period " << info.id << dendl; + return 0; +} + + +int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zonegroup_id, + std::string_view zonegroup_name, + RGWZoneGroup& info, + std::unique_ptr* writer) +{ + if (!zonegroup_id.empty()) { + return cfgstore->read_zonegroup_by_id(dpp, y, zonegroup_id, info, writer); + } + if (!zonegroup_name.empty()) { + return cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, info, writer); + } + + std::string realm_id; + int r = cfgstore->read_default_realm_id(dpp, y, realm_id); + if (r == -ENOENT) { + return cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, + info, writer); + } + if (r < 0) { + return r; + } + return cfgstore->read_default_zonegroup(dpp, y, realm_id, info, writer); +} + +int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneGroup& info) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zonegroup name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // insert the default placement target if it doesn't exist + constexpr std::string_view default_placement_name = "default-placement"; + + RGWZoneGroupPlacementTarget placement_target; + placement_target.name = default_placement_name; + + info.placement_targets.emplace(default_placement_name, placement_target); + if (info.default_placement.name.empty()) { + info.default_placement.name = default_placement_name; + } + + int r = cfgstore->create_zonegroup(dpp, y, exclusive, info, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create zonegroup with " + << cpp_strerror(r) << dendl; + return r; + } + + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_default_zonegroup(dpp, y, cfgstore, info, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set zonegroup as default: " + << cpp_strerror(r) << dendl; + } + + return 0; +} + +static int create_default_zonegroup(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + bool exclusive, + const RGWZoneParams& default_zone, + RGWZoneGroup& info) +{ + info.name = default_zonegroup_name; + info.api_name = default_zonegroup_name; + info.is_master = true; + + // enable all supported features + info.enabled_features.insert(rgw::zone_features::enabled.begin(), + rgw::zone_features::enabled.end()); + + // add the zone to the zonegroup + bool is_master = true; + std::list empty_list; + rgw::zone_features::set disable_features; // empty + int r = add_zone_to_group(dpp, info, default_zone, &is_master, nullptr, + empty_list, nullptr, nullptr, empty_list, + empty_list, nullptr, std::nullopt, + info.enabled_features, disable_features); + if (r < 0) { + return r; + } + + // write the zone + return create_zonegroup(dpp, y, cfgstore, exclusive, info); +} + +int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneGroup& info, + bool exclusive) +{ + return cfgstore->write_default_zonegroup_id( + dpp, y, exclusive, info.realm_id, info.id); +} + +int remove_zone_from_group(const DoutPrefixProvider* dpp, + RGWZoneGroup& zonegroup, + const rgw_zone_id& zone_id) +{ + auto z = zonegroup.zones.find(zone_id); + if (z == zonegroup.zones.end()) { + return -ENOENT; + } + zonegroup.zones.erase(z); + + if (zonegroup.master_zone == zone_id) { + // choose a new master zone + auto m = zonegroup.zones.begin(); + if (m != zonegroup.zones.end()) { + zonegroup.master_zone = m->first; + ldpp_dout(dpp, 0) << "NOTICE: promoted " << m->second.name + << " as new master_zone of zonegroup " << zonegroup.name << dendl; + } else { + ldpp_dout(dpp, 0) << "NOTICE: removed master_zone of zonegroup " + << zonegroup.name << dendl; + } + } + + const bool log_data = zonegroup.zones.size() > 1; + for (auto& [id, zone] : zonegroup.zones) { + zone.log_data = log_data; + } + + return 0; +} + +// try to remove the given zone id from every zonegroup in the cluster +static int remove_zone_from_groups(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + const rgw_zone_id& zone_id) +{ + std::array zonegroup_names; + sal::ListResult listing; + do { + int r = cfgstore->list_zonegroup_names(dpp, y, listing.next, + zonegroup_names, listing); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to list zonegroups with " + << cpp_strerror(r) << dendl; + return r; + } + + for (const auto& name : listing.entries) { + RGWZoneGroup zonegroup; + std::unique_ptr writer; + r = cfgstore->read_zonegroup_by_name(dpp, y, name, zonegroup, &writer); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to load zonegroup " << name + << " with " << cpp_strerror(r) << dendl; + continue; + } + + r = remove_zone_from_group(dpp, zonegroup, zone_id); + if (r < 0) { + continue; + } + + // write the updated zonegroup + r = writer->write(dpp, y, zonegroup); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to write zonegroup " << name + << " with " << cpp_strerror(r) << dendl; + continue; + } + ldpp_dout(dpp, 0) << "Removed zone from zonegroup " << name << dendl; + } + } while (!listing.next.empty()); + + return 0; +} + + +int read_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zone_id, + std::string_view zone_name, + RGWZoneParams& info, + std::unique_ptr* writer) +{ + if (!zone_id.empty()) { + return cfgstore->read_zone_by_id(dpp, y, zone_id, info, writer); + } + if (!zone_name.empty()) { + return cfgstore->read_zone_by_name(dpp, y, zone_name, info, writer); + } + + std::string realm_id; + int r = cfgstore->read_default_realm_id(dpp, y, realm_id); + if (r == -ENOENT) { + return cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, writer); + } + if (r < 0) { + return r; + } + return cfgstore->read_default_zone(dpp, y, realm_id, info, writer); +} + +extern int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y, + rgw::sal::ConfigStore* cfgstore, + std::string_view my_zone_id, + std::set& pools); + +int create_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneParams& info, std::unique_ptr* writer) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // add default placement with empty pool name + RGWZonePlacementInfo placement; + rgw_pool pool; + placement.storage_classes.set_storage_class( + RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); + // don't overwrite if it already exists + info.placement_pools.emplace("default-placement", std::move(placement)); + + // build a set of all pool names used by other zones + std::set pools; + int r = get_zones_pool_set(dpp, y, cfgstore, info.id, pools); + if (r < 0) { + return r; + } + + // initialize pool names with the zone name prefix + r = init_zone_pool_names(dpp, y, pools, info); + if (r < 0) { + return r; + } + + r = cfgstore->create_zone(dpp, y, exclusive, info, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create zone with " + << cpp_strerror(r) << dendl; + return r; + } + + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_default_zone(dpp, y, cfgstore, info, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set zone as default: " + << cpp_strerror(r) << dendl; + } + + return 0; + +} + +int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + bool exclusive) +{ + return cfgstore->write_default_zone_id( + dpp, y, exclusive, info.realm_id, info.id); +} + +int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + sal::ZoneWriter& writer) +{ + // remove this zone from any zonegroups that contain it + int r = remove_zone_from_groups(dpp, y, cfgstore, info.id); + if (r < 0) { + return r; + } + + return writer.remove(dpp, y); +} + +auto find_zone_placement(const DoutPrefixProvider* dpp, + const RGWZoneParams& info, + const rgw_placement_rule& rule) + -> const RGWZonePlacementInfo* +{ + auto i = info.placement_pools.find(rule.name); + if (i == info.placement_pools.end()) { + ldpp_dout(dpp, 0) << "ERROR: This zone does not contain placement rule " + << rule.name << dendl; + return nullptr; + } + + const std::string& storage_class = rule.get_storage_class(); + if (!i->second.storage_class_exists(storage_class)) { + ldpp_dout(dpp, 5) << "ERROR: The zone placement for rule " << rule.name + << " does not contain storage class " << storage_class << dendl; + return nullptr; + } + + return &i->second; +} + +bool all_zonegroups_support(const SiteConfig& site, std::string_view feature) +{ + const auto& period = site.get_period(); + if (!period) { + // if we're not in a realm, just check the local zonegroup + return site.get_zonegroup().supports(feature); + } + const auto& zgs = period->period_map.zonegroups; + return std::all_of(zgs.begin(), zgs.end(), [feature] (const auto& pair) { + return pair.second.supports(feature); + }); +} + +static int read_or_create_default_zone(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + RGWZoneParams& info) +{ + int r = cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, nullptr); + if (r == -ENOENT) { + info.name = default_zone_name; + constexpr bool exclusive = true; + r = create_zone(dpp, y, cfgstore, exclusive, info, nullptr); + if (r == -EEXIST) { + r = cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, nullptr); + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create default zone: " + << cpp_strerror(r) << dendl; + return r; + } + } + return r; +} + +static int read_or_create_default_zonegroup(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + const RGWZoneParams& zone_params, + RGWZoneGroup& info) +{ + int r = cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, + info, nullptr); + if (r == -ENOENT) { + constexpr bool exclusive = true; + r = create_default_zonegroup(dpp, y, cfgstore, exclusive, + zone_params, info); + if (r == -EEXIST) { + r = cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, + info, nullptr); + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create default zonegroup: " + << cpp_strerror(r) << dendl; + return r; + } + } + return r; +} + +int SiteConfig::load(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool force_local_zonegroup) +{ + // clear existing configuration + zone = nullptr; + zonegroup = nullptr; + local_zonegroup = std::nullopt; + period = std::nullopt; + zone_params = RGWZoneParams{}; + + int r = 0; + + // try to load a realm + realm.emplace(); + std::string realm_name = dpp->get_cct()->_conf->rgw_realm; + if (!realm_name.empty()) { + r = cfgstore->read_realm_by_name(dpp, y, realm_name, *realm, nullptr); + } else { + r = cfgstore->read_default_realm(dpp, y, *realm, nullptr); + if (r == -ENOENT) { // no realm + r = 0; + realm = std::nullopt; + } + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load realm: " << cpp_strerror(r) << dendl; + return r; + } + + // try to load the local zone params + std::string zone_name = dpp->get_cct()->_conf->rgw_zone; + if (!zone_name.empty()) { + r = cfgstore->read_zone_by_name(dpp, y, zone_name, zone_params, nullptr); + } else if (realm) { + // load the realm's default zone + r = cfgstore->read_default_zone(dpp, y, realm->id, zone_params, nullptr); + if (r == -ENOENT) { + if (realm_name.empty()) { + // rgw_realm was not specified, and we found a default realm that + // doesn't have a default zone. ignore the realm and try to load the + // global default zone + realm = std::nullopt; + r = read_or_create_default_zone(dpp, y, cfgstore, zone_params); + } else { + ldpp_dout(dpp, 0) << "No rgw_zone configured, and the selected realm \"" + << realm->name << "\" does not have a default zone." << dendl; + } + } + } else { + // load or create the "default" zone + r = read_or_create_default_zone(dpp, y, cfgstore, zone_params); + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load zone: " << cpp_strerror(r) << dendl; + return r; + } + + if (!realm && !zone_params.realm_id.empty()) { + realm.emplace(); + r = cfgstore->read_realm_by_id(dpp, y, zone_params.realm_id, + *realm, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load realm: " << cpp_strerror(r) << dendl; + return r; + } + } + + if (realm && !force_local_zonegroup) { + // try to load the realm's period + r = load_period_zonegroup(dpp, y, cfgstore, *realm, zone_params.id); + if (r != -ENOENT) { + return r; + } + ldpp_dout(dpp, 10) << "cannot find current period zonegroup, " + "using local zonegroup configuration" << dendl; + } + + // fall back to a local zonegroup + return load_local_zonegroup(dpp, y, cfgstore, zone_params.id); +} + +std::unique_ptr SiteConfig::make_fake() { + auto fake = std::make_unique(); + fake->local_zonegroup.emplace(); + fake->local_zonegroup->zones.emplace(""s, RGWZone{}); + fake->zonegroup = &*fake->local_zonegroup; + fake->zone = &fake->zonegroup->zones.begin()->second; + return fake; +} + +int SiteConfig::load_period_zonegroup(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + const RGWRealm& realm, + const rgw_zone_id& zone_id) +{ + // load the realm's current period + period.emplace(); + int r = cfgstore->read_period(dpp, y, realm.current_period, + std::nullopt, *period); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load current period: " + << cpp_strerror(r) << dendl; + return r; + } + + // find our zone and zonegroup in the period + for (const auto& zg : period->period_map.zonegroups) { + auto z = zg.second.zones.find(zone_id); + if (z != zg.second.zones.end()) { + zone = &z->second; + zonegroup = &zg.second; + return 0; + } + } + + ldpp_dout(dpp, 0) << "ERROR: current period " << period->id + << " does not contain zone id " << zone_id << dendl; + + period = std::nullopt; + return -ENOENT; +} + +int SiteConfig::load_local_zonegroup(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + const rgw_zone_id& zone_id) +{ + int r = 0; + + // load the zonegroup + local_zonegroup.emplace(); + std::string zonegroup_name = dpp->get_cct()->_conf->rgw_zonegroup; + if (!zonegroup_name.empty()) { + r = cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, + *local_zonegroup, nullptr); + } else if (realm) { + r = cfgstore->read_default_zonegroup(dpp, y, realm->id, + *local_zonegroup, nullptr); + } else { + r = read_or_create_default_zonegroup(dpp, y, cfgstore, zone_params, + *local_zonegroup); + } + + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load zonegroup: " + << cpp_strerror(r) << dendl; + } else { + // find our zone in the zonegroup + auto z = local_zonegroup->zones.find(zone_id); + if (z != local_zonegroup->zones.end()) { + zone = &z->second; + zonegroup = &*local_zonegroup; + return 0; + } + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << local_zonegroup->id + << " does not contain zone id " << zone_id << dendl; + r = -ENOENT; + } + + local_zonegroup = std::nullopt; + return r; +} + } // namespace rgw +static inline int conf_to_uint64(const JSONFormattable& config, const string& key, uint64_t *pval) +{ + string sval; + if (config.find(key, &sval)) { + string err; + uint64_t val = strict_strtoll(sval.c_str(), 10, &err); + if (!err.empty()) { + return -EINVAL; + } + *pval = val; + } + return 0; +} + +int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) +{ + int r = -1; + + if (config.exists("retain_head_object")) { + string s = config["retain_head_object"]; + if (s == "true") { + retain_head_object = true; + } else { + retain_head_object = false; + } + } + if (config.exists("allow_read_through")) { + string s = config["allow_read_through"]; + if (s == "true") { + allow_read_through = true; + } else { + allow_read_through = false; + } + } + if (config.exists("read_through_restore_days")) { + r = conf_to_uint64(config, "read_through_restore_days", &read_through_restore_days); + if (r < 0) { + read_through_restore_days = DEFAULT_READ_THROUGH_RESTORE_DAYS; + } + } + + if (is_tier_type_s3()) { + r = t.s3.update_params(config); + } + + if (config.exists("restore_storage_class")) { + restore_storage_class = config["restore_storage_class"]; + } + + if (is_tier_type_s3_glacier()) { + r = s3_glacier.update_params(config); + } + return r; +} + +int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) +{ + if (config.exists("retain_head_object")) { + retain_head_object = false; + } + if (config.exists("allow_read_through")) { + allow_read_through = false; + } + if (config.exists("read_through_restore_days")) { + read_through_restore_days = DEFAULT_READ_THROUGH_RESTORE_DAYS; + } + + if (is_tier_type_s3()) { + t.s3.clear_params(config); + } + + if (config.exists("restore_storage_class")) { + restore_storage_class = RGW_STORAGE_CLASS_STANDARD; + } + + if (is_tier_type_s3_glacier()) { + s3_glacier.clear_params(config); + } + + return 0; +} + +int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config) +{ + int r = -1; + + if (config.exists("endpoint")) { + endpoint = config["endpoint"]; + } + if (config.exists("target_path")) { + target_path = config["target_path"]; + } + if (config.exists("region")) { + region = config["region"]; + } + if (config.exists("host_style")) { + string s; + s = config["host_style"]; + if (s != "virtual") { + host_style = PathStyle; + } else { + host_style = VirtualStyle; + } + } + if (config.exists("target_storage_class")) { + target_storage_class = config["target_storage_class"]; + } + if (config.exists("access_key")) { + key.id = config["access_key"]; + } + if (config.exists("secret")) { + key.key = config["secret"]; + } + if (config.exists("multipart_sync_threshold")) { + r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold); + if (r < 0) { + multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + } + + if (config.exists("multipart_min_part_size")) { + r = conf_to_uint64(config, "multipart_min_part_size", &multipart_min_part_size); + if (r < 0) { + multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + } + + if (config.exists("acls")) { + const JSONFormattable& cc = config["acls"]; + if (cc.is_array()) { + for (auto& c : cc.array()) { + RGWTierACLMapping m; + m.init(c); + if (!m.source_id.empty()) { + acl_mappings[m.source_id] = m; + } + } + } else { + RGWTierACLMapping m; + m.init(cc); + if (!m.source_id.empty()) { + acl_mappings[m.source_id] = m; + } + } + } + return 0; +} + +int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config) +{ + if (config.exists("endpoint")) { + endpoint.clear(); + } + if (config.exists("target_path")) { + target_path.clear(); + } + if (config.exists("region")) { + region.clear(); + } + if (config.exists("host_style")) { + /* default */ + host_style = PathStyle; + } + if (config.exists("target_storage_class")) { + target_storage_class.clear(); + } + if (config.exists("access_key")) { + key.id.clear(); + } + if (config.exists("secret")) { + key.key.clear(); + } + if (config.exists("multipart_sync_threshold")) { + multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + if (config.exists("multipart_min_part_size")) { + multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + if (config.exists("acls")) { + const JSONFormattable& cc = config["acls"]; + if (cc.is_array()) { + for (auto& c : cc.array()) { + RGWTierACLMapping m; + m.init(c); + acl_mappings.erase(m.source_id); + } + } else { + RGWTierACLMapping m; + m.init(cc); + acl_mappings.erase(m.source_id); + } + } + return 0; +} + +int RGWZoneGroupTierS3Glacier::update_params(const JSONFormattable& config) +{ + int r = -1; + + if (config.exists("glacier_restore_days")) { + r = conf_to_uint64(config, "glacier_restore_days", &glacier_restore_days); + if (r < 0) { + glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS; + } + } + if (config.exists("glacier_restore_tier_type")) { + string s; + s = config["glacier_restore_tier_type"]; + if (s != "Expedited") { + glacier_restore_tier_type = Standard; + } else { + glacier_restore_tier_type = Expedited; + } + } + return 0; +} + +int RGWZoneGroupTierS3Glacier::clear_params(const JSONFormattable& config) +{ + if (config.exists("glacier_restore_days")) { + glacier_restore_days = DEFAULT_GLACIER_RESTORE_DAYS; + } + if (config.exists("glacier_restore_tier_type")) { + /* default */ + glacier_restore_tier_type = Standard; + } + return 0; +} + +void rgw_meta_sync_info::generate_test_instances(list& o) +{ + auto info = new rgw_meta_sync_info; + info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps; + info->period = "periodid"; + info->realm_epoch = 5; + o.push_back(info); + o.push_back(new rgw_meta_sync_info); +} + +void rgw_meta_sync_marker::generate_test_instances(list& o) +{ + auto marker = new rgw_meta_sync_marker; + marker->state = rgw_meta_sync_marker::IncrementalSync; + marker->marker = "01234"; + marker->realm_epoch = 5; + o.push_back(marker); + o.push_back(new rgw_meta_sync_marker); +} + +void rgw_meta_sync_status::generate_test_instances(list& o) +{ + o.push_back(new rgw_meta_sync_status); +} + +void RGWZoneParams::generate_test_instances(list &o) +{ + o.push_back(new RGWZoneParams); + o.push_back(new RGWZoneParams); +} + +void RGWPeriodLatestEpochInfo::generate_test_instances(list &o) +{ + RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo; + o.push_back(z); + o.push_back(new RGWPeriodLatestEpochInfo); +} + +void RGWZoneGroup::generate_test_instances(list& o) +{ + RGWZoneGroup *r = new RGWZoneGroup; + o.push_back(r); + o.push_back(new RGWZoneGroup); +} + +void RGWPeriodLatestEpochInfo::dump(Formatter *f) const { + encode_json("latest_epoch", epoch, f); +} + +void RGWPeriodLatestEpochInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("latest_epoch", epoch, obj); +} + +void RGWNameToId::dump(Formatter *f) const { + encode_json("obj_id", obj_id, f); +} + +void RGWNameToId::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("obj_id", obj_id, obj); +} + +void RGWNameToId::generate_test_instances(list& o) { + RGWNameToId *n = new RGWNameToId; + n->obj_id = "id"; + o.push_back(n); + o.push_back(new RGWNameToId); +} + diff --git a/src/rgw/rgw_zone.h b/src/rgw/rgw_zone.h new file mode 100644 index 000000000000..418214c32cbc --- /dev/null +++ b/src/rgw/rgw_zone.h @@ -0,0 +1,901 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include +#include "rgw_zone_types.h" +#include "rgw_common.h" +#include "rgw_sal_fwd.h" +#include "rgw_sync_policy.h" + + +struct RGWZoneParams { + std::string id; + std::string name; + rgw_pool domain_root; + rgw_pool control_pool; + rgw_pool gc_pool; + rgw_pool lc_pool; + rgw_pool log_pool; + rgw_pool intent_log_pool; + rgw_pool usage_log_pool; + rgw_pool user_keys_pool; + rgw_pool user_email_pool; + rgw_pool user_swift_pool; + rgw_pool user_uid_pool; + rgw_pool roles_pool; + rgw_pool reshard_pool; + rgw_pool otp_pool; + rgw_pool oidc_pool; + rgw_pool notif_pool; + rgw_pool topics_pool; + rgw_pool account_pool; + rgw_pool group_pool; + + RGWAccessKey system_key; + + std::map placement_pools; + + std::string realm_id; + + JSONFormattable tier_config; + + RGWZoneParams() {} + explicit RGWZoneParams(const std::string& _name) : name(_name){} + RGWZoneParams(const rgw_zone_id& _id, const std::string& _name) : id(_id.id), name(_name) {} + RGWZoneParams(const rgw_zone_id& _id, const std::string& _name, const std::string& _realm_id) + : id(_id.id), name(_name), realm_id(_realm_id) {} + + const std::string& get_name() const { return name; } + const std::string& get_id() const { return id; } + + void set_name(const std::string& _name) { name = _name;} + void set_id(const std::string& _id) { id = _id;} + void clear_id() { id.clear(); } + + rgw_pool get_pool(CephContext *cct) const; + + const std::string& get_compression_type(const rgw_placement_rule& placement_rule) const; + + void encode(bufferlist& bl) const { + ENCODE_START(15, 1, bl); + encode(domain_root, bl); + encode(control_pool, bl); + encode(gc_pool, bl); + encode(log_pool, bl); + encode(intent_log_pool, bl); + encode(usage_log_pool, bl); + encode(user_keys_pool, bl); + encode(user_email_pool, bl); + encode(user_swift_pool, bl); + encode(user_uid_pool, bl); + { + // these used to be wrapped by RGWSystemMetaObj::encode(), + // so the extra ENCODE_START/ENCODE_FINISH are preserved + ENCODE_START(1, 1, bl); + encode(id, bl); + encode(name, bl); + ENCODE_FINISH(bl); + } + encode(system_key, bl); + encode(placement_pools, bl); + rgw_pool unused_metadata_heap; + encode(unused_metadata_heap, bl); + encode(realm_id, bl); + encode(lc_pool, bl); + std::map old_tier_config; + encode(old_tier_config, bl); + encode(roles_pool, bl); + encode(reshard_pool, bl); + encode(otp_pool, bl); + encode(tier_config, bl); + encode(oidc_pool, bl); + encode(notif_pool, bl); + encode(topics_pool, bl); + encode(account_pool, bl); + encode(group_pool, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(15, bl); + decode(domain_root, bl); + decode(control_pool, bl); + decode(gc_pool, bl); + decode(log_pool, bl); + decode(intent_log_pool, bl); + decode(usage_log_pool, bl); + decode(user_keys_pool, bl); + decode(user_email_pool, bl); + decode(user_swift_pool, bl); + decode(user_uid_pool, bl); + if (struct_v >= 6) { + { + // these used to be wrapped by RGWSystemMetaObj::decode(), + // so the extra DECODE_START/DECODE_FINISH are preserved + DECODE_START(1, bl); + decode(id, bl); + decode(name, bl); + DECODE_FINISH(bl); + } + } else if (struct_v >= 2) { + decode(name, bl); + id = name; + } + if (struct_v >= 3) + decode(system_key, bl); + if (struct_v >= 4) + decode(placement_pools, bl); + if (struct_v >= 5) { + rgw_pool unused_metadata_heap; + decode(unused_metadata_heap, bl); + } + if (struct_v >= 6) { + decode(realm_id, bl); + } + if (struct_v >= 7) { + decode(lc_pool, bl); + } else { + lc_pool = log_pool.name + ":lc"; + } + std::map old_tier_config; + if (struct_v >= 8) { + decode(old_tier_config, bl); + } + if (struct_v >= 9) { + decode(roles_pool, bl); + } else { + roles_pool = name + ".rgw.meta:roles"; + } + if (struct_v >= 10) { + decode(reshard_pool, bl); + } else { + reshard_pool = log_pool.name + ":reshard"; + } + if (struct_v >= 11) { + ::decode(otp_pool, bl); + } else { + otp_pool = name + ".rgw.otp"; + } + if (struct_v >= 12) { + ::decode(tier_config, bl); + } else { + for (auto& kv : old_tier_config) { + tier_config.set(kv.first, kv.second); + } + } + if (struct_v >= 13) { + ::decode(oidc_pool, bl); + } else { + oidc_pool = name + ".rgw.meta:oidc"; + } + if (struct_v >= 14) { + decode(notif_pool, bl); + } else { + notif_pool = log_pool.name + ":notif"; + } + if (struct_v >= 15) { + decode(topics_pool, bl); + decode(account_pool, bl); + decode(group_pool, bl); + } else { + topics_pool = name + ".rgw.meta:topics"; + account_pool = name + ".rgw.meta:accounts"; + group_pool = name + ".rgw.meta:groups"; + } + DECODE_FINISH(bl); + } + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); + + bool get_placement(const std::string& placement_id, RGWZonePlacementInfo *placement) const { + auto iter = placement_pools.find(placement_id); + if (iter == placement_pools.end()) { + return false; + } + *placement = iter->second; + return true; + } + + /* + * return data pool of the head object + */ + bool get_head_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool* pool) const { + const rgw_data_placement_target& explicit_placement = obj.bucket.explicit_placement; + if (!explicit_placement.data_pool.empty()) { + if (!obj.in_extra_data) { + *pool = explicit_placement.data_pool; + } else { + *pool = explicit_placement.get_data_extra_pool(); + } + return true; + } + if (placement_rule.empty()) { + return false; + } + auto iter = placement_pools.find(placement_rule.name); + if (iter == placement_pools.end()) { + return false; + } + if (!obj.in_extra_data) { + *pool = iter->second.get_data_pool(placement_rule.storage_class); + } else { + *pool = iter->second.get_data_extra_pool(); + } + return true; + } + + bool valid_placement(const rgw_placement_rule& rule) const { + auto iter = placement_pools.find(rule.name); + if (iter == placement_pools.end()) { + return false; + } + return iter->second.storage_class_exists(rule.storage_class); + } +}; +WRITE_CLASS_ENCODER(RGWZoneParams) + +struct RGWZoneGroup { + std::string id; + std::string name; + std::string api_name; + std::list endpoints; + bool is_master = false; + + rgw_zone_id master_zone; + std::map zones; + + std::map placement_targets; + rgw_placement_rule default_placement; + + std::list hostnames; + std::list hostnames_s3website; + // TODO: Maybe convert hostnames to a map> for + // endpoint_type->hostnames +/* +20:05 < _robbat21irssi> maybe I do something like: if (hostname_map.empty()) { populate all map keys from hostnames; }; +20:05 < _robbat21irssi> but that's a later compatability migration planning bit +20:06 < yehudasa> more like if (!hostnames.empty()) { +20:06 < yehudasa> for (std::list::iterator iter = hostnames.begin(); iter != hostnames.end(); ++iter) { +20:06 < yehudasa> hostname_map["s3"].append(iter->second); +20:07 < yehudasa> hostname_map["s3website"].append(iter->second); +20:07 < yehudasa> s/append/push_back/g +20:08 < _robbat21irssi> inner loop over APIs +20:08 < yehudasa> yeah, probably +20:08 < _robbat21irssi> s3, s3website, swift, swith_auth, swift_website +*/ + std::map > api_hostname_map; + std::map > api_endpoints_map; + + std::string realm_id; + + rgw_sync_policy_info sync_policy; + rgw::zone_features::set enabled_features; + CephContext *cct{nullptr}; + + RGWZoneGroup(): is_master(false){} + RGWZoneGroup(const std::string &_id, const std::string &_name):id(_id), name(_name) {} + explicit RGWZoneGroup(const std::string &_name):name(_name) {} + RGWZoneGroup(const std::string &_name, bool _is_master, const std::string& _realm_id, + const std::list& _endpoints) + : name(_name), endpoints(_endpoints), is_master(_is_master), realm_id(_realm_id) {} + + const std::string& get_name() const { return name; } + const std::string& get_id() const { return id; } + + void set_name(const std::string& _name) { name = _name;} + void set_id(const std::string& _id) { id = _id;} + void clear_id() { id.clear(); } + + bool is_master_zonegroup() const { return is_master;} + + void encode(bufferlist& bl) const { + ENCODE_START(6, 1, bl); + encode(name, bl); + encode(api_name, bl); + encode(is_master, bl); + encode(endpoints, bl); + encode(master_zone, bl); + encode(zones, bl); + encode(placement_targets, bl); + encode(default_placement, bl); + encode(hostnames, bl); + encode(hostnames_s3website, bl); + { + // these used to be wrapped by RGWSystemMetaObj::encode(), + // so the extra ENCODE_START/ENCODE_FINISH are preserved + ENCODE_START(1, 1, bl); + encode(id, bl); + encode(name, bl); + ENCODE_FINISH(bl); + } + encode(realm_id, bl); + encode(sync_policy, bl); + encode(enabled_features, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(6, bl); + decode(name, bl); + decode(api_name, bl); + decode(is_master, bl); + decode(endpoints, bl); + decode(master_zone, bl); + decode(zones, bl); + decode(placement_targets, bl); + decode(default_placement, bl); + if (struct_v >= 2) { + decode(hostnames, bl); + } + if (struct_v >= 3) { + decode(hostnames_s3website, bl); + } + if (struct_v >= 4) { + { + // these used to be wrapped by RGWSystemMetaObj::decode(), + // so the extra DECODE_START/DECODE_FINISH are preserved + DECODE_START(1, bl); + decode(id, bl); + decode(name, bl); + DECODE_FINISH(bl); + } + decode(realm_id, bl); + } else { + id = name; + } + if (struct_v >= 5) { + decode(sync_policy, bl); + } + if (struct_v >= 6) { + decode(enabled_features, bl); + } + DECODE_FINISH(bl); + } + + int equals(const std::string& other_zonegroup) const; + rgw_pool get_pool(CephContext *cct) const; + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); + + bool supports(std::string_view feature) const { + return enabled_features.contains(feature); + } +}; +WRITE_CLASS_ENCODER(RGWZoneGroup) + +struct RGWPeriodMap +{ + std::string id; + std::map zonegroups; + std::map zonegroups_by_api; + std::map short_zone_ids; + + std::string master_zonegroup; + + void encode(bufferlist& bl) const; + void decode(bufferlist::const_iterator& bl); + + int update(const RGWZoneGroup& zonegroup, CephContext *cct); + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + + void reset() { + zonegroups.clear(); + zonegroups_by_api.clear(); + master_zonegroup.clear(); + } + + uint32_t get_zone_short_id(const std::string& zone_id) const; + + bool find_zone_by_id(const rgw_zone_id& zone_id, + RGWZoneGroup *zonegroup, + RGWZone *zone) const; + bool find_zone_by_name(const std::string& zone_id, + RGWZoneGroup *zonegroup, + RGWZone *zone) const; +}; +WRITE_CLASS_ENCODER(RGWPeriodMap) + +struct RGWPeriodConfig +{ + RGWQuota quota; + RGWRateLimitInfo user_ratelimit; + RGWRateLimitInfo bucket_ratelimit; + // rate limit unauthenticated user + RGWRateLimitInfo anon_ratelimit; + + void encode(bufferlist& bl) const { + ENCODE_START(2, 1, bl); + encode(quota.bucket_quota, bl); + encode(quota.user_quota, bl); + encode(bucket_ratelimit, bl); + encode(user_ratelimit, bl); + encode(anon_ratelimit, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(2, bl); + decode(quota.bucket_quota, bl); + decode(quota.user_quota, bl); + if (struct_v >= 2) { + decode(bucket_ratelimit, bl); + decode(user_ratelimit, bl); + decode(anon_ratelimit, bl); + } + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; +WRITE_CLASS_ENCODER(RGWPeriodConfig) + +class RGWRealm; +class RGWPeriod; +namespace rgw {class SiteConfig;} + +class RGWRealm +{ +public: + std::string id; + std::string name; + + std::string current_period; + epoch_t epoch{0}; //< realm epoch, incremented for each new period + +public: + RGWRealm() {} + RGWRealm(const std::string& _id, const std::string& _name = "") : id(_id), name(_name) {} + + const std::string& get_name() const { return name; } + const std::string& get_id() const { return id; } + + void set_name(const std::string& _name) { name = _name;} + void set_id(const std::string& _id) { id = _id;} + void clear_id() { id.clear(); } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + { + // these used to be wrapped by RGWSystemMetaObj::encode(), + // so the extra ENCODE_START/ENCODE_FINISH are preserved + ENCODE_START(1, 1, bl); + encode(id, bl); + encode(name, bl); + ENCODE_FINISH(bl); + } + encode(current_period, bl); + encode(epoch, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + { + // these used to be wrapped by RGWSystemMetaObj::decode(), + // so the extra DECODE_START/DECODE_FINISH are preserved + DECODE_START(1, bl); + decode(id, bl); + decode(name, bl); + DECODE_FINISH(bl); + } + decode(current_period, bl); + decode(epoch, bl); + DECODE_FINISH(bl); + } + + // TODO: use ConfigStore for watch/notify, + // After refactoring RGWRealmWatcher and RGWRealmReloader, get_pool and get_info_oid_prefix will be removed. + rgw_pool get_pool(CephContext *cct) const; + const std::string& get_info_oid_prefix(bool old_format = false) const; + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); + + const std::string& get_current_period() const { + return current_period; + } + void clear_current_period_and_epoch() { + current_period.clear(); + epoch = 0; + } + epoch_t get_epoch() const { return epoch; } + + std::string get_control_oid() const; + + int find_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWPeriod *pperiod, + RGWZoneGroup *pzonegroup, + bool *pfound, + rgw::sal::ConfigStore* cfgstore, + optional_yield y) const; +}; +WRITE_CLASS_ENCODER(RGWRealm) + +struct RGWPeriodLatestEpochInfo { + epoch_t epoch = 0; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(epoch, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(epoch, bl); + DECODE_FINISH(bl); + } + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); +}; +WRITE_CLASS_ENCODER(RGWPeriodLatestEpochInfo) + + +/* + * The RGWPeriod object contains the entire configuration of a + * RGWRealm, including its RGWZoneGroups and RGWZones. Consistency of + * this configuration is maintained across all zones by passing around + * the RGWPeriod object in its JSON representation. + * + * If a new configuration changes which zone is the metadata master + * zone (i.e., master zone of the master zonegroup), then a new + * RGWPeriod::id (a uuid) is generated, its RGWPeriod::realm_epoch is + * incremented, and the RGWRealm object is updated to reflect that new + * current_period id and epoch. If the configuration changes BUT which + * zone is the metadata master does NOT change, then only the + * RGWPeriod::epoch is incremented (and the RGWPeriod::id remains the + * same). + * + * When a new RGWPeriod is created with a new RGWPeriod::id (uuid), it + * is linked back to its predecessor RGWPeriod through the + * RGWPeriod::predecessor_uuid field, thus creating a "linked + * list"-like structure of RGWPeriods back to the cluster's creation. + */ +class RGWPeriod +{ +public: + std::string id; //< a uuid + epoch_t epoch{0}; + std::string predecessor_uuid; + std::vector sync_status; + RGWPeriodMap period_map; + RGWPeriodConfig period_config; + std::string master_zonegroup; + rgw_zone_id master_zone; + + std::string realm_id; + epoch_t realm_epoch{1}; //< realm epoch when period was made current + + // gather the metadata sync status for each shard; only for use on master zone + int update_sync_status(const DoutPrefixProvider *dpp, + rgw::sal::Driver* driver, + const RGWPeriod ¤t_period, + std::ostream& error_stream, bool force_if_stale); + +public: + RGWPeriod() {} + + explicit RGWPeriod(const std::string& period_id, epoch_t _epoch = 0) + : id(period_id), epoch(_epoch) {} + + const std::string& get_id() const { return id; } + epoch_t get_epoch() const { return epoch; } + epoch_t get_realm_epoch() const { return realm_epoch; } + const std::string& get_predecessor() const { return predecessor_uuid; } + const rgw_zone_id& get_master_zone() const { return master_zone; } + const std::string& get_master_zonegroup() const { return master_zonegroup; } + const std::string& get_realm() const { return realm_id; } + const RGWPeriodMap& get_map() const { return period_map; } + RGWPeriodConfig& get_config() { return period_config; } + const RGWPeriodConfig& get_config() const { return period_config; } + const std::vector& get_sync_status() const { return sync_status; } + rgw_pool get_pool(CephContext *cct) const; + const std::string& get_info_oid_prefix() const; + + void set_user_quota(RGWQuotaInfo& user_quota) { + period_config.quota.user_quota = user_quota; + } + + void set_bucket_quota(RGWQuotaInfo& bucket_quota) { + period_config.quota.bucket_quota = bucket_quota; + } + + void set_id(const std::string& _id) { + this->id = _id; + period_map.id = _id; + } + void set_epoch(epoch_t epoch) { this->epoch = epoch; } + void set_realm_epoch(epoch_t epoch) { realm_epoch = epoch; } + + void set_predecessor(const std::string& predecessor) + { + predecessor_uuid = predecessor; + } + + void set_realm_id(const std::string& _realm_id) { + realm_id = _realm_id; + } + + int get_zonegroup(RGWZoneGroup& zonegroup, + const std::string& zonegroup_id) const; + + bool is_single_zonegroup() const + { + return (period_map.zonegroups.size() <= 1); + } + + /* + returns true if there are several zone groups with a least one zone + */ + bool is_multi_zonegroups_with_zones() const + { + int count = 0; + for (const auto& zg: period_map.zonegroups) { + if (zg.second.zones.size() > 0) { + if (count++ > 0) { + return true; + } + } + } + return false; + } + + bool find_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWZoneGroup *pzonegroup, + optional_yield y) const; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(id, bl); + encode(epoch, bl); + encode(realm_epoch, bl); + encode(predecessor_uuid, bl); + encode(sync_status, bl); + encode(period_map, bl); + encode(master_zone, bl); + encode(master_zonegroup, bl); + encode(period_config, bl); + encode(realm_id, bl); + std::string realm_name; // removed + encode(realm_name, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(id, bl); + decode(epoch, bl); + decode(realm_epoch, bl); + decode(predecessor_uuid, bl); + decode(sync_status, bl); + decode(period_map, bl); + decode(master_zone, bl); + decode(master_zonegroup, bl); + decode(period_config, bl); + decode(realm_id, bl); + std::string realm_name; // removed + decode(realm_name, bl); + DECODE_FINISH(bl); + } + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); + static void generate_test_instances(std::list& o); + + static std::string get_staging_id(const std::string& realm_id) { + return realm_id + ":staging"; + } +}; +WRITE_CLASS_ENCODER(RGWPeriod) + +namespace rgw { + +/// Look up a realm by its id. If no id is given, look it up by name. +/// If no name is given, fall back to the cluster's default realm. +int read_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view realm_id, + std::string_view realm_name, + RGWRealm& info, + std::unique_ptr* writer = nullptr); + +/// Create a realm and its initial period. If the info.id is empty, a +/// random uuid will be generated. +int create_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWRealm& info, + std::unique_ptr* writer = nullptr); + +/// Set the given realm as the cluster's default realm. +int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWRealm& info, + bool exclusive = false); + +/// Update the current_period of an existing realm. +int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + sal::RealmWriter& writer, RGWRealm& realm, + const RGWPeriod& period); + +/// Overwrite the local zonegroup and period config objects with the new +/// configuration contained in the given period. +int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWPeriod& info); + +/// Return the staging period id for the given realm. +std::string get_staging_period_id(std::string_view realm_id); + +/// Convert the given period into a separate staging period, where +/// radosgw-admin can make changes to it without effecting the running +/// configuration. +void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info); + +/// Read all zonegroups in the period's realm and add them to the period. +int update_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, RGWPeriod& info); + +/// Validates the given 'staging' period and tries to commit it as the +/// realm's new current period. +int commit_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, sal::Driver* driver, + RGWRealm& realm, sal::RealmWriter& realm_writer, + const RGWPeriod& current_period, + RGWPeriod& info, std::ostream& error_stream, + bool force_if_stale, const rgw::SiteConfig& site); + + +/// Look up a zonegroup by its id. If no id is given, look it up by name. +/// If no name is given, fall back to the cluster's default zonegroup. +int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zonegroup_id, + std::string_view zonegroup_name, + RGWZoneGroup& info, + std::unique_ptr* writer = nullptr); + +/// Initialize and create the given zonegroup. If the given info.id is empty, +/// a random uuid will be generated. May fail with -EEXIST. +int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneGroup& info); + +/// Set the given zonegroup as its realm's default zonegroup. +int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneGroup& info, + bool exclusive = false); + +/// Return an endpoint from the zonegroup or its master zone. +std::string get_zonegroup_endpoint(const RGWZoneGroup& info); + +/// Add a zone to the zonegroup, or update an existing zone entry. +int add_zone_to_group(const DoutPrefixProvider* dpp, + RGWZoneGroup& zonegroup, + const RGWZoneParams& zone_params, + const bool *pis_master, const bool *pread_only, + const std::list& endpoints, + const std::string *ptier_type, + const bool *psync_from_all, + const std::list& sync_from, + const std::list& sync_from_rm, + const std::string *predirect_zone, + std::optional bucket_index_max_shards, + const rgw::zone_features::set& enable_features, + const rgw::zone_features::set& disable_features); + +/// Remove a zone by id from its zonegroup, promoting a new master zone if +/// necessary. +int remove_zone_from_group(const DoutPrefixProvider* dpp, + RGWZoneGroup& info, + const rgw_zone_id& zone_id); + + +/// Look up a zone by its id. If no id is given, look it up by name. If no name +/// is given, fall back to the realm's default zone. +int read_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zone_id, + std::string_view zone_name, + RGWZoneParams& info, + std::unique_ptr* writer = nullptr); + +/// Initialize and create a new zone. If the given info.id is empty, a random +/// uuid will be generated. Pool names are initialized with the zone name as a +/// prefix. If any pool names conflict with existing zones, a random suffix is +/// added. +int create_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneParams& info, + std::unique_ptr* writer = nullptr); + +/// Initialize the zone's pool names using the zone name as a prefix. If a pool +/// name conflicts with an existing zone's pool, add a unique suffix. +int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y, + const std::set& pools, RGWZoneParams& info); + +/// Set the given zone as its realm's default zone. +int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + bool exclusive = false); + +/// Delete an existing zone and remove it from any zonegroups that contain it. +int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + sal::ZoneWriter& writer); + +/// Return the zone placement corresponding to the given rule, or nullptr. +auto find_zone_placement(const DoutPrefixProvider* dpp, + const RGWZoneParams& info, + const rgw_placement_rule& rule) + -> const RGWZonePlacementInfo*; + + +/// Global state about the site configuration. Initialized once during +/// startup and may be reinitialized by RGWRealmReloader, but is otherwise +/// immutable at runtime. +class SiteConfig { + public: + /// Return the local zone params. + const RGWZoneParams& get_zone_params() const { return zone_params; } + /// Return the current realm configuration, if a realm is present. + const std::optional& get_realm() const { return realm; } + /// Return the current period configuration, if a period is present. + const std::optional& get_period() const { return period; } + /// Return the zonegroup configuration. + const RGWZoneGroup& get_zonegroup() const { return *zonegroup; } + /// Return the public zone configuration. + const RGWZone& get_zone() const { return *zone; } + /// Return true if the local zone can write metadata. + bool is_meta_master() const { + return zonegroup->is_master && zonegroup->master_zone == zone->id; + } + + /// Load or reload the multisite configuration from storage. This is not + /// thread-safe, so requires careful coordination with the RGWRealmReloader. + int load(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool force_local_zonegroup = false); + + /// Create a fake site config to be used by tests and similar, just + /// to have a site config. + /// + /// \warning Do not use this anywhere but unittests where we need to + /// bring up parts of RGW that require a SiteConfig exist, but need + /// to run without a cluster. + static std::unique_ptr make_fake(); + + virtual ~SiteConfig() = default; + + private: + int load_period_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWRealm& realm, + const rgw_zone_id& zone_id); + int load_local_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + const rgw_zone_id& zone_id); + + RGWZoneParams zone_params; + std::optional realm; + std::optional period; + std::optional local_zonegroup; + const RGWZoneGroup* zonegroup = nullptr; + const RGWZone* zone = nullptr; +}; + + +/// Test whether all zonegroups in the realm support the given zone feature. +bool all_zonegroups_support(const SiteConfig& site, std::string_view feature); + +std::string gen_random_uuid(); + +} // namespace rgw diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc index b8f86c61a01e..89a5ec44fb82 100644 --- a/src/rgw/services/svc_mdlog.cc +++ b/src/rgw/services/svc_mdlog.cc @@ -12,7 +12,7 @@ #include "rgw_coroutine.h" #include "rgw_cr_rados.h" -#include "driver/rados/rgw_zone.h" // FIXME: subclass dependency +#include "rgw/rgw_zone.h" #include "common/errno.h" diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index 6cd8c0a28af9..07654e832d8f 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -427,11 +427,6 @@ const RGWZone& RGWSI_Zone::get_zone() const return *zone_public_config; } -const RGWZoneGroup& RGWSI_Zone::get_zonegroup() const -{ - return *zonegroup; -} - int RGWSI_Zone::get_zonegroup(const string& id, RGWZoneGroup& zg) const { int ret = 0; @@ -516,11 +511,6 @@ bool RGWSI_Zone::need_to_sync() const current_period->get_id().empty()); } -bool RGWSI_Zone::need_to_log_data() const -{ - return (zone_public_config->log_data && sync_module_exports_data()); -} - bool RGWSI_Zone::is_meta_master() const { if (!zonegroup->is_master_zonegroup()) { diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h index 7256aae37286..de56cde6e5f5 100644 --- a/src/rgw/services/svc_zone.h +++ b/src/rgw/services/svc_zone.h @@ -75,7 +75,7 @@ public: const RGWZoneParams& get_zone_params() const; const RGWPeriod& get_current_period() const; const RGWRealm& get_realm() const; - const RGWZoneGroup& get_zonegroup() const; + const RGWZoneGroup& get_zonegroup() const { return *zonegroup; }; int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) const; const RGWZone& get_zone() const; @@ -134,7 +134,7 @@ public: bool is_meta_master() const; bool need_to_sync() const; - bool need_to_log_data() const; + bool need_to_log_data() const { return (zone_public_config->log_data && sync_module_exports_data()); }; bool need_to_log_metadata() const; bool can_reshard() const; bool is_syncing_bucket_meta() const; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a94826f7ad76..7a21f2b9817b 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -614,7 +614,7 @@ if(WITH_RBD) add_dependencies(tests unittest_librbd rbd-mirror) endif(WITH_RBD_MIRROR) endif(WITH_RBD) -if(WITH_RADOSGW) +if(WITH_RADOSGW AND WITH_RADOSGW_RADOS) add_dependencies(tests radosgw radosgw-admin) endif() #add dependency from fio just to ensure the plugin build isn't failing diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index 560503173140..f9305c5afc62 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -143,6 +143,7 @@ add_executable(unittest_rgw_ratelimit test_rgw_ratelimit.cc $) target_include_directories(ceph_test_rgw_gc_log SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(ceph_test_rgw_gc_log ${rgw_libs} radostest-cxx) install(TARGETS ceph_test_rgw_gc_log DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() add_executable(unittest_rgw_shard_io test_rgw_shard_io.cc) add_ceph_unittest(unittest_rgw_shard_io) @@ -325,6 +331,7 @@ target_link_libraries(unittest_rgw_shard_io ${rgw_libs} unit-main ${UNITTEST_LIB add_ceph_test(test-ceph-diff-sorted.sh ${CMAKE_CURRENT_SOURCE_DIR}/test-ceph-diff-sorted.sh) +if(WITH_RADOSGW_RADOS) # unittest_log_backing add_executable(unittest_log_backing test_log_backing.cc) target_include_directories(unittest_log_backing @@ -335,6 +342,7 @@ target_link_libraries(unittest_log_backing neoradostest-support ${UNITTEST_LIBS} ${rgw_libs}) +endif() add_executable(unittest_rgw_lua test_rgw_lua.cc) add_ceph_unittest(unittest_rgw_lua) @@ -342,6 +350,7 @@ target_include_directories(unittest_rgw_lua SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(unittest_rgw_lua unit-main ${rgw_libs}) +if(WITH_RADOSGW_RADOS) add_executable(radosgw-cr-test rgw_cr_test.cc) target_link_libraries(radosgw-cr-test ${rgw_libs} librados cls_rgw_client cls_otp_client cls_lock_client cls_refcount_client @@ -351,6 +360,7 @@ target_link_libraries(radosgw-cr-test ${rgw_libs} librados OATH::OATH ${CURL_LIBRARIES} ${EXPAT_LIBRARIES} ${BLKID_LIBRARIES} GTest::GTest) +endif() if(WITH_RADOSGW_POSIX) # unittest_posix_bucket_cache @@ -376,6 +386,7 @@ target_link_libraries(unittest_rgw_posix_driver ${UNITTEST_LIBS} ${rgw_libs} ${LMDB_LIBRARIES}) endif(WITH_RADOSGW_POSIX) +if(WITH_RADOSGW_RADOS) # ceph_test_datalog add_executable(ceph_test_datalog test_datalog.cc) target_include_directories(ceph_test_datalog @@ -387,6 +398,7 @@ target_link_libraries(ceph_test_datalog ${UNITTEST_LIBS} ${rgw_libs}) install(TARGETS ceph_test_datalog DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() if(WITH_CATCH2) add_executable(unittest_rgw_hex diff --git a/src/test/rgw/rgw_cr_test.cc b/src/test/rgw/rgw_cr_test.cc index 78b6fc9413b9..f75f79132ebe 100644 --- a/src/test/rgw/rgw_cr_test.cc +++ b/src/test/rgw/rgw_cr_test.cc @@ -21,7 +21,7 @@ #include "rgw_cr_rados.h" #include "rgw_sal.h" #include "rgw_sal_rados.h" -#include "driver/rados/rgw_zone.h" +#include "rgw/rgw_zone.h" #include "rgw_sal_config.h" #include "gtest/gtest.h" diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc index 8e9be5c11e5e..057bcab362b6 100644 --- a/src/test/rgw/test_rgw_iam_policy.cc +++ b/src/test/rgw/test_rgw_iam_policy.cc @@ -32,7 +32,7 @@ #include "rgw_op.h" #include "rgw_process_env.h" #include "rgw_sal_rados.h" -#include "driver/rados/rgw_zone.h" +#include "rgw_zone.h" #include "rgw_sal_config.h" using std::string;