From 92252d22c837f8b71008439d561ba0248f851314 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Tue, 22 Mar 2022 13:22:15 -0400 Subject: [PATCH] RGW: Zipper - zonegroup, placement tier Add ZoneGroup and PlacementTier to the Zipper API. This cleaned up a lot of issues in the Module branch. In addition, add cloud transition. Signed-off-by: Daniel Gryniewicz --- src/rgw/rgw_admin.cc | 47 +++-- src/rgw/rgw_asio_frontend.cc | 4 +- src/rgw/rgw_bucket.cc | 31 ---- src/rgw/rgw_bucket.h | 2 - src/rgw/rgw_common.cc | 57 ++++-- src/rgw/rgw_common.h | 5 +- src/rgw/rgw_file.cc | 3 +- src/rgw/rgw_lc.cc | 167 +---------------- src/rgw/rgw_lc_tier.cc | 4 +- src/rgw/rgw_main.cc | 30 +-- src/rgw/rgw_notify.cc | 2 +- src/rgw/rgw_obj_manifest.cc | 5 - src/rgw/rgw_obj_manifest.h | 1 - src/rgw/rgw_object_expirer_core.cc | 6 +- src/rgw/rgw_object_expirer_core.h | 9 +- src/rgw/rgw_op.cc | 56 +++--- src/rgw/rgw_orphan.cc | 2 +- src/rgw/rgw_period_pusher.cc | 10 +- src/rgw/rgw_putobj_processor.cc | 3 +- src/rgw/rgw_rest.cc | 9 +- src/rgw/rgw_rest.h | 2 +- src/rgw/rgw_rest_config.cc | 2 +- src/rgw/rgw_rest_conn.cc | 48 +---- src/rgw/rgw_rest_conn.h | 36 +--- src/rgw/rgw_rest_ratelimit.cc | 6 +- src/rgw/rgw_rest_realm.cc | 6 +- src/rgw/rgw_rest_s3.cc | 22 +-- src/rgw/rgw_rest_swift.cc | 19 +- src/rgw/rgw_rest_user.cc | 4 +- src/rgw/rgw_sal.cc | 24 ++- src/rgw/rgw_sal.h | 70 ++++++- src/rgw/rgw_sal_dbstore.cc | 77 +++++++- src/rgw/rgw_sal_dbstore.h | 93 +++++++++- src/rgw/rgw_sal_motr.cc | 91 +++++++-- src/rgw/rgw_sal_motr.h | 97 +++++++++- src/rgw/rgw_sal_rados.cc | 285 ++++++++++++++++++++++++++--- src/rgw/rgw_sal_rados.h | 94 ++++++++-- src/rgw/rgw_sync_module_aws.cc | 4 +- src/rgw/rgw_sync_module_es.cc | 2 +- src/rgw/rgw_tools.cc | 2 +- src/rgw/rgw_tools.h | 4 + src/rgw/rgw_trim_bilog.cc | 2 +- src/rgw/rgw_trim_mdlog.cc | 2 +- src/rgw/services/svc_zone.cc | 8 +- 44 files changed, 944 insertions(+), 509 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index a128f8be5f6..08054c9f77c 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1762,12 +1762,12 @@ static boost::optional get_remote_conn(rgw::sal::RadosStore* store, { boost::optional conn; if (remote == zonegroup.get_id()) { - conn.emplace(store->ctx(), store->svc()->zone, remote, zonegroup.endpoints, zonegroup.api_name); + conn.emplace(store->ctx(), store, remote, zonegroup.endpoints, zonegroup.api_name); } else { for (const auto& z : zonegroup.zones) { const auto& zone = z.second; if (remote == zone.id) { - conn.emplace(store->ctx(), store->svc()->zone, remote, zone.endpoints, zonegroup.api_name); + conn.emplace(store->ctx(), store, remote, zone.endpoints, zonegroup.api_name); break; } } @@ -2416,15 +2416,14 @@ static void tab_dump(const string& header, int width, const list& entrie static void sync_status(Formatter *formatter) { - const RGWRealm& realm = store->get_zone()->get_realm(); - const RGWZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); - const RGWZone& zone = static_cast(store)->svc()->zone->get_zone(); + const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); + rgw::sal::Zone* zone = store->get_zone(); int width = 15; - cout << std::setw(width) << "realm" << std::setw(1) << " " << realm.get_id() << " (" << realm.get_name() << ")" << std::endl; + cout << std::setw(width) << "realm" << std::setw(1) << " " << zone->get_realm_id() << " (" << zone->get_realm_name() << ")" << std::endl; cout << std::setw(width) << "zonegroup" << std::setw(1) << " " << zonegroup.get_id() << " (" << zonegroup.get_name() << ")" << std::endl; - cout << std::setw(width) << "zone" << std::setw(1) << " " << zone.id << " (" << zone.name << ")" << std::endl; + cout << std::setw(width) << "zone" << std::setw(1) << " " << zone->get_id() << " (" << zone->get_name() << ")" << std::endl; list md_status; @@ -2726,14 +2725,13 @@ static int sync_info(std::optional opt_target_zone, std::optionalget_zone()->get_realm(); - const RGWZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); - const RGWZone& zone = store->svc()->zone->get_zone(); + const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); + rgw::sal::Zone* zone = store->get_zone(); constexpr int width = 15; - out << indented{width, "realm"} << realm.get_id() << " (" << realm.get_name() << ")\n"; + out << indented{width, "realm"} << zone->get_realm_id() << " (" << zone->get_realm_name() << ")\n"; out << indented{width, "zonegroup"} << zonegroup.get_id() << " (" << zonegroup.get_name() << ")\n"; - out << indented{width, "zone"} << zone.id << " (" << zone.name << ")\n"; + out << indented{width, "zone"} << zone->get_id() << " (" << zone->get_name() << ")\n"; out << indented{width, "bucket"} << info.bucket << "\n\n"; if (!static_cast(store)->ctl()->bucket->bucket_imports_data(info.bucket, null_yield, dpp())) { @@ -2767,14 +2765,13 @@ static int bucket_sync_status(rgw::sal::RadosStore* store, const RGWBucketInfo& std::optional& opt_source_bucket, std::ostream& out) { - const RGWRealm& realm = store->get_zone()->get_realm(); - const RGWZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); - const RGWZone& zone = static_cast(store)->svc()->zone->get_zone(); + const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); + rgw::sal::Zone* zone = store->get_zone(); constexpr int width = 15; - out << indented{width, "realm"} << realm.get_id() << " (" << realm.get_name() << ")\n"; + out << indented{width, "realm"} << zone->get_realm_id() << " (" << zone->get_realm_name() << ")\n"; out << indented{width, "zonegroup"} << zonegroup.get_id() << " (" << zonegroup.get_name() << ")\n"; - out << indented{width, "zone"} << zone.id << " (" << zone.name << ")\n"; + out << indented{width, "zone"} << zone->get_id() << " (" << zone->get_name() << ")\n"; out << indented{width, "bucket"} << info.bucket << "\n\n"; if (!static_cast(store)->ctl()->bucket->bucket_imports_data(info.bucket, null_yield, dpp())) { @@ -2796,8 +2793,8 @@ static int bucket_sync_status(rgw::sal::RadosStore* store, const RGWBucketInfo& set zone_ids; if (!source_zone_id.empty()) { - auto z = zonegroup.zones.find(source_zone_id); - if (z == zonegroup.zones.end()) { + auto z = static_cast(store)->svc()->zone->get_zonegroup().zones.find(source_zone_id); + if (z == static_cast(store)->svc()->zone->get_zonegroup().zones.end()) { ldpp_dout(dpp(), -1) << "Source zone not found in zonegroup " << zonegroup.get_name() << dendl; return -EINVAL; @@ -2809,14 +2806,14 @@ static int bucket_sync_status(rgw::sal::RadosStore* store, const RGWBucketInfo& } zone_ids.insert(source_zone_id); } else { - for (const auto& entry : zonegroup.zones) { + for (const auto& entry : static_cast(store)->svc()->zone->get_zonegroup().zones) { zone_ids.insert(entry.second.id); } } for (auto& zone_id : zone_ids) { - auto z = zonegroup.zones.find(zone_id.id); - if (z == zonegroup.zones.end()) { /* should't happen */ + auto z = static_cast(store)->svc()->zone->get_zonegroup().zones.find(zone_id.id); + if (z == static_cast(store)->svc()->zone->get_zonegroup().zones.end()) { /* should't happen */ continue; } auto c = zone_conn_map.find(zone_id.id); @@ -2831,7 +2828,7 @@ static int bucket_sync_status(rgw::sal::RadosStore* store, const RGWBucketInfo& continue; } if (pipe.source.zone.value_or(rgw_zone_id()) == z->second.id) { - bucket_source_sync_status(dpp(), store, zone, z->second, + bucket_source_sync_status(dpp(), store, static_cast(store)->svc()->zone->get_zone(), z->second, c->second, info, pipe, width, out); @@ -6155,7 +6152,7 @@ int main(int argc, const char **argv) rgw_placement_rule target_rule; target_rule.name = placement_id; target_rule.storage_class = *opt_storage_class; - if (!store->get_zone()->get_params().valid_placement(target_rule)) { + if (!store->valid_placement(target_rule)) { cerr << "NOTICE: invalid dest placement: " << target_rule.to_str() << std::endl; return EINVAL; } @@ -8565,7 +8562,7 @@ next: RGWSyncModuleInstanceRef sync_module; int ret = static_cast(store)->svc()->sync_modules->get_manager()->create_instance(dpp(), g_ceph_context, static_cast(store)->svc()->zone->get_zone().tier_type, - store->get_zone()->get_params().tier_config, &sync_module); + static_cast(store)->svc()->zone->get_zone_params().tier_config, &sync_module); if (ret < 0) { ldpp_dout(dpp(), -1) << "ERROR: failed to init sync module instance, ret=" << ret << dendl; return ret; diff --git a/src/rgw/rgw_asio_frontend.cc b/src/rgw/rgw_asio_frontend.cc index d6001a67f16..c368b719ef6 100644 --- a/src/rgw/rgw_asio_frontend.cc +++ b/src/rgw/rgw_asio_frontend.cc @@ -671,8 +671,8 @@ class ExpandMetaVar { public: ExpandMetaVar(rgw::sal::Zone* zone_svc) { - meta_map["realm"] = zone_svc->get_realm().get_name(); - meta_map["realm_id"] = zone_svc->get_realm().get_id(); + meta_map["realm"] = zone_svc->get_realm_name(); + meta_map["realm_id"] = zone_svc->get_realm_id(); meta_map["zonegroup"] = zone_svc->get_zonegroup().get_name(); meta_map["zonegroup_id"] = zone_svc->get_zonegroup().get_id(); meta_map["zone"] = zone_svc->get_name(); diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 5a6eb774992..673f527e384 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -159,37 +159,6 @@ void rgw_parse_url_bucket(const string &bucket, const string& auth_tenant, } } -int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id) -{ - auto pos = bucket_instance.rfind(':'); - if (pos == string::npos) { - return -EINVAL; - } - - string first = bucket_instance.substr(0, pos); - string second = bucket_instance.substr(pos + 1); - - pos = first.find(':'); - - if (pos == string::npos) { - *shard_id = -1; - *bucket_name = first; - *bucket_id = second; - return 0; - } - - *bucket_name = first.substr(0, pos); - *bucket_id = first.substr(pos + 1); - - string err; - *shard_id = strict_strtol(second.c_str(), 10, &err); - if (!err.empty()) { - return -EINVAL; - } - - return 0; -} - // parse key in format: [tenant/]name:instance[:shard_id] int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, rgw_bucket *bucket, int *shard_id) diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 0bc24db7e40..28c315e6ab2 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -39,8 +39,6 @@ class RGWZone; struct RGWZoneParams; extern void init_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id); - -extern int rgw_bucket_parse_bucket_instance(const std::string& bucket_instance, std::string *bucket_name, std::string *bucket_id, int *shard_id); extern int rgw_bucket_parse_bucket_key(CephContext *cct, const std::string& key, rgw_bucket* bucket, int *shard_id); diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 5e30c14f3cc..d2f1ed918a1 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -1828,6 +1828,25 @@ static struct rgw_name_to_flag cap_names[] = { {"*", RGW_CAP_ALL}, {"write", RGW_CAP_WRITE}, {NULL, 0} }; +static int rgw_parse_list_of_flags(struct rgw_name_to_flag *mapping, + const string& str, uint32_t *perm) +{ + list strs; + get_str_list(str, strs); + list::iterator iter; + uint32_t v = 0; + for (iter = strs.begin(); iter != strs.end(); ++iter) { + string& s = *iter; + for (int i = 0; mapping[i].type_name; i++) { + if (s.compare(mapping[i].type_name) == 0) + v |= mapping[i].flag; + } + } + + *perm = v; + return 0; +} + int RGWUserCaps::parse_cap_perm(const string& str, uint32_t *perm) { return rgw_parse_list_of_flags(cap_names, str, perm); @@ -2920,21 +2939,33 @@ void rgw_obj::dump(Formatter *f) const encode_json("key", key, f); } -int rgw_parse_list_of_flags(struct rgw_name_to_flag *mapping, - const string& str, uint32_t *perm) +int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *bucket_name, string *bucket_id, int *shard_id) { - list strs; - get_str_list(str, strs); - list::iterator iter; - uint32_t v = 0; - for (iter = strs.begin(); iter != strs.end(); ++iter) { - string& s = *iter; - for (int i = 0; mapping[i].type_name; i++) { - if (s.compare(mapping[i].type_name) == 0) - v |= mapping[i].flag; - } + auto pos = bucket_instance.rfind(':'); + if (pos == string::npos) { + return -EINVAL; + } + + string first = bucket_instance.substr(0, pos); + string second = bucket_instance.substr(pos + 1); + + pos = first.find(':'); + + if (pos == string::npos) { + *shard_id = -1; + *bucket_name = first; + *bucket_id = second; + return 0; + } + + *bucket_name = first.substr(0, pos); + *bucket_id = first.substr(pos + 1); + + string err; + *shard_id = strict_strtol(second.c_str(), 10, &err); + if (!err.empty()) { + return -EINVAL; } - *perm = v; return 0; } diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index ddb455f041c..702ce96512a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -2071,9 +2071,6 @@ struct rgw_name_to_flag { uint32_t flag; }; -extern int rgw_parse_list_of_flags(struct rgw_name_to_flag *mapping, - const std::string& str, uint32_t *perm); - /** time parsing */ extern int parse_time(const char *time_str, real_time *time); extern bool parse_rfc2616(const char *s, struct tm *t); @@ -2413,3 +2410,5 @@ int decode_bl(bufferlist& bl, T& t) } return 0; } + +extern int rgw_bucket_parse_bucket_instance(const std::string& bucket_instance, std::string *bucket_name, std::string *bucket_id, int *shard_id); diff --git a/src/rgw/rgw_file.cc b/src/rgw/rgw_file.cc index d705c402005..0fe9fc09e27 100644 --- a/src/rgw/rgw_file.cc +++ b/src/rgw/rgw_file.cc @@ -1826,8 +1826,7 @@ namespace rgw { state->object->set_bucket(state->bucket.get()); auto compression_type = - get_store()->get_zone()->get_params().get_compression_type( - state->bucket->get_placement_rule()); + get_store()->get_compression_type(state->bucket->get_placement_rule()); /* not obviously supportable */ ceph_assert(! dlo_manifest); diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index fbfe4d70cc5..da2bd841e58 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -487,7 +487,7 @@ struct lc_op_ctx { const DoutPrefixProvider *dpp; WorkQ* wq; - RGWZoneGroupPlacementTier tier = {}; + std::unique_ptr tier; lc_op_ctx(op_env& env, rgw_bucket_dir_entry& o, boost::optional next_key_name, @@ -1250,26 +1250,6 @@ public: return need_to_process; } - /* find out if the the storage class is remote cloud */ - int get_tier_target(const RGWZoneGroup &zonegroup, const rgw_placement_rule& rule, - RGWZoneGroupPlacementTier &tier) { - std::map::const_iterator titer; - titer = zonegroup.placement_targets.find(rule.name); - if (titer == zonegroup.placement_targets.end()) { - return -ENOENT; - } - - const auto& target_rule = titer->second; - std::map::const_iterator ttier; - ttier = target_rule.tier_targets.find(rule.storage_class); - if (ttier != target_rule.tier_targets.end()) { - tier = ttier->second; - } else { // not found - return -ENOENT; - } - return 0; - } - int delete_tier_obj(lc_op_ctx& oc) { int ret = 0; @@ -1285,137 +1265,15 @@ public: return ret; } - int update_tier_obj(lc_op_ctx& oc, RGWLCCloudTierCtx& tier_ctx) { - - map attrs; - int r = 0; - - real_time read_mtime; - - std::unique_ptr read_op(oc.obj->get_read_op()); - - read_op->params.lastmod = &read_mtime; - - r = read_op->prepare(null_yield, oc.dpp); - if (r < 0) { - return r; - } - - if (read_mtime != tier_ctx.o.meta.mtime) { - /* raced */ - return -ECANCELED; - } - - attrs = oc.obj->get_attrs(); - - /* TODO dang - need to replace this with something zipper */ - rgw::sal::RadosStore *rados = static_cast(oc.store); - RGWRados::Object op_target(rados->getRados(), oc.bucket, oc.rctx, oc.obj.get()); - RGWRados::Object::Write obj_op(&op_target); - - obj_op.meta.modify_tail = true; - obj_op.meta.flags = PUT_OBJ_CREATE; - obj_op.meta.category = RGWObjCategory::CloudTiered; - obj_op.meta.delete_at = real_time(); - bufferlist blo; - obj_op.meta.data = &blo; - obj_op.meta.if_match = NULL; - obj_op.meta.if_nomatch = NULL; - obj_op.meta.user_data = NULL; - obj_op.meta.zones_trace = NULL; - obj_op.meta.delete_at = real_time(); - obj_op.meta.olh_epoch = tier_ctx.o.versioned_epoch; - - RGWObjManifest *pmanifest; - RGWObjManifest manifest; - - pmanifest = &manifest; - RGWObjTier tier_config; - tier_config.name = oc.tier.storage_class; - tier_config.tier_placement = oc.tier; - tier_config.is_multipart_upload = tier_ctx.is_multipart_upload; - - pmanifest->set_tier_type("cloud-s3"); - pmanifest->set_tier_config(tier_config); - - /* check if its necessary */ - rgw_placement_rule target_placement; - target_placement.inherit_from(tier_ctx.bucket_info.placement_rule); - target_placement.storage_class = oc.tier.storage_class; - pmanifest->set_head(target_placement, tier_ctx.obj->get_obj(), 0); - - pmanifest->set_tail_placement(target_placement, tier_ctx.obj->get_obj().bucket); - - pmanifest->set_obj_size(0); - - obj_op.meta.manifest = pmanifest; - - /* update storage class */ - bufferlist bl; - bl.append(oc.tier.storage_class); - attrs[RGW_ATTR_STORAGE_CLASS] = bl; - - attrs.erase(RGW_ATTR_ID_TAG); - attrs.erase(RGW_ATTR_TAIL_TAG); - - r = obj_op.write_meta(oc.dpp, 0, 0, attrs, null_yield); - if (r < 0) { - return r; - } - - return 0; - } - int transition_obj_to_cloud(lc_op_ctx& oc) { - /* init */ - string id = "cloudid"; - string endpoint = oc.tier.t.s3.endpoint; - RGWAccessKey key = oc.tier.t.s3.key; - string region = oc.tier.t.s3.region; - HostStyle host_style = oc.tier.t.s3.host_style; - string bucket_name = oc.tier.t.s3.target_path; - const RGWZoneGroup& zonegroup = oc.store->get_zone()->get_zonegroup(); - bool delete_object; - /* If CurrentVersion object, remove it & create delete marker */ - delete_object = (!oc.tier.retain_head_object || + bool delete_object = (!oc.tier->retain_head_object() || (oc.o.is_current() && oc.bucket->versioned())); - if (bucket_name.empty()) { - bucket_name = "rgwx-" + zonegroup.get_name() + "-" + oc.tier.storage_class + - "-cloud-bucket"; - boost::algorithm::to_lower(bucket_name); - } - - /* Create RGW REST connection */ - S3RESTConn conn(oc.cct, oc.store, id, { endpoint }, key, region, host_style); - - RGWLCCloudTierCtx tier_ctx(oc.cct, oc.dpp, oc.o, oc.store, oc.bucket->get_info(), - oc.obj.get(), conn, bucket_name, - oc.tier.t.s3.target_storage_class); - tier_ctx.acl_mappings = oc.tier.t.s3.acl_mappings; - tier_ctx.multipart_min_part_size = oc.tier.t.s3.multipart_min_part_size; - tier_ctx.multipart_sync_threshold = oc.tier.t.s3.multipart_sync_threshold; - tier_ctx.storage_class = oc.tier.storage_class; - - // check if target_path is already created - std::set& cloud_targets = oc.env.worker->get_cloud_targets(); - std::pair::iterator, bool> it; - - it = cloud_targets.insert(bucket_name); - tier_ctx.target_bucket_created = !(it.second); - - ldpp_dout(oc.dpp, 0) << "Transitioning object(" << oc.o.key << ") to the cloud endpoint(" << endpoint << ")" << dendl; - - /* Transition object to cloud end point */ - int ret = rgw_cloud_tier_transfer_object(tier_ctx); - + int ret = oc.obj->transition_to_cloud(oc.bucket, oc.tier.get(), oc.o, + oc.env.worker->get_cloud_targets(), oc.cct, + !delete_object, oc.dpp, null_yield); if (ret < 0) { - ldpp_dout(oc.dpp, 0) << "ERROR: failed to transfer object(" << oc.o.key << ") to the cloud endpoint(" << endpoint << ") ret=" << ret << dendl; - - if (!tier_ctx.target_bucket_created) { - cloud_targets.erase(it.first); - } return ret; } @@ -1425,12 +1283,6 @@ public: ldpp_dout(oc.dpp, 0) << "ERROR: Deleting tier object(" << oc.o.key << ") failed ret=" << ret << dendl; return ret; } - } else { - ret = update_tier_obj(oc, tier_ctx); - if (ret < 0) { - ldpp_dout(oc.dpp, 0) << "ERROR: Updating tier object(" << oc.o.key << ") failed ret=" << ret << dendl; - return ret; - } } return 0; @@ -1447,15 +1299,15 @@ public: } std::string tier_type = ""; - const RGWZoneGroup& zonegroup = oc.store->get_zone()->get_zonegroup(); + rgw::sal::ZoneGroup& zonegroup = oc.store->get_zone()->get_zonegroup(); rgw_placement_rule target_placement; target_placement.inherit_from(oc.bucket->get_placement_rule()); target_placement.storage_class = transition.storage_class; - r = get_tier_target(zonegroup, target_placement, oc.tier); + r = zonegroup.get_placement_tier(target_placement, &oc.tier); - if (!r && oc.tier.tier_type == "cloud-s3") { + if (!r && oc.tier->get_tier_type() == "cloud-s3") { ldpp_dout(oc.dpp, 30) << "Found cloud s3 tier: " << target_placement.storage_class << dendl; if (!oc.o.is_current() && !pass_object_lock_check(oc.store, oc.obj.get(), oc.dpp)) { @@ -1479,8 +1331,7 @@ public: return r; } } else { - if (!oc.store->get_zone()->get_params(). - valid_placement(target_placement)) { + if (!oc.store->valid_placement(target_placement)) { ldpp_dout(oc.dpp, 0) << "ERROR: non existent dest placement: " << target_placement << " bucket="<< oc.bucket diff --git a/src/rgw/rgw_lc_tier.cc b/src/rgw/rgw_lc_tier.cc index 82cd913e993..ba84ef836eb 100644 --- a/src/rgw/rgw_lc_tier.cc +++ b/src/rgw/rgw_lc_tier.cc @@ -1104,8 +1104,8 @@ static int cloud_tier_multipart_transfer(RGWLCCloudTierCtx& tier_ctx) { } dest_obj.init(target_bucket, target_obj_name); - status_obj = rgw_raw_obj(tier_ctx.store->get_zone()->get_params().log_pool, - "lc_multipart_" + tier_ctx.obj->get_oid()); + rgw_pool pool = static_cast(tier_ctx.store)->svc()->zone->get_zone_params().log_pool; + status_obj = rgw_raw_obj(pool, "lc_multipart_" + tier_ctx.obj->get_oid()); ret = read_upload_status(tier_ctx.dpp, tier_ctx.store, &status_obj, &status); diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index 2ea898c72d1..24930bb896d 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -16,7 +16,7 @@ #include "include/str_list.h" #include "include/stringify.h" #include "rgw_common.h" -#include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "rgw_period_pusher.h" #include "rgw_realm_reloader.h" #include "rgw_rest.h" @@ -498,7 +498,7 @@ int radosgw_Main(int argc, const char **argv) set_logging(rest_filter(store, RGW_REST_SWIFT, swift_resource))); } else { - if (store->get_zone()->get_zonegroup().zones.size() > 1) { + if (store->get_zone()->get_zonegroup().get_zone_count() > 1) { derr << "Placing Swift API in the root of URL hierarchy while running" << " multi-site configuration requires another instance of RadosGW" << " with S3 API enabled!" << dendl; @@ -666,15 +666,21 @@ int radosgw_Main(int argc, const char **argv) } - // add a watcher to respond to realm configuration changes - RGWPeriodPusher pusher(&dp, store, null_yield); - RGWFrontendPauser pauser(fes, implicit_tenant_context, &pusher); - auto reloader = std::make_unique(store, - service_map_meta, &pauser); + std::unique_ptr reloader; + std::unique_ptr pusher; + std::unique_ptr pauser; + std::unique_ptr realm_watcher; + if (store->get_name() == "rados") { + // add a watcher to respond to realm configuration changes + pusher = std::make_unique(&dp, store, null_yield); + pauser = std::make_unique(fes, implicit_tenant_context, pusher.get()); + reloader = std::make_unique(store, service_map_meta, pauser.get()); - RGWRealmWatcher realm_watcher(&dp, g_ceph_context, store->get_zone()->get_realm()); - realm_watcher.add_watcher(RGWRealmNotify::Reload, *reloader); - realm_watcher.add_watcher(RGWRealmNotify::ZonesNeedPeriod, pusher); + realm_watcher = std::make_unique(&dp, g_ceph_context, + static_cast(store)->svc()->zone->get_realm()); + realm_watcher->add_watcher(RGWRealmNotify::Reload, *reloader); + realm_watcher->add_watcher(RGWRealmNotify::ZonesNeedPeriod, *pusher.get()); + } #if defined(HAVE_SYS_PRCTL_H) if (prctl(PR_SET_DUMPABLE, 1) == -1) { @@ -686,7 +692,9 @@ int radosgw_Main(int argc, const char **argv) derr << "shutting down" << dendl; - reloader.reset(); // stop the realm reloader + if (store->get_name() == "rados") { + reloader.reset(); // stop the realm reloader + } for (list::iterator liter = fes.begin(); liter != fes.end(); ++liter) { diff --git a/src/rgw/rgw_notify.cc b/src/rgw/rgw_notify.cc index aa02cf85aac..0edcb384945 100644 --- a/src/rgw/rgw_notify.cc +++ b/src/rgw/rgw_notify.cc @@ -692,7 +692,7 @@ static inline void populate_event(reservation_t& res, event.object_size = size; event.object_etag = etag; event.object_versionId = version; - event.awsRegion = res.store->get_zone()->get_zonegroup().api_name; + event.awsRegion = res.store->get_zone()->get_zonegroup().get_api_name(); // use timestamp as per key sequence id (hex encoded) const utime_t ts(real_clock::now()); boost::algorithm::hex((const char*)&ts, (const char*)&ts + sizeof(utime_t), diff --git a/src/rgw/rgw_obj_manifest.cc b/src/rgw/rgw_obj_manifest.cc index 5af0b12a8da..61eeb83bbe9 100644 --- a/src/rgw/rgw_obj_manifest.cc +++ b/src/rgw/rgw_obj_manifest.cc @@ -120,11 +120,6 @@ int RGWObjManifest::append(const DoutPrefixProvider *dpp, RGWObjManifest& m, con return 0; } -int RGWObjManifest::append(const DoutPrefixProvider *dpp, RGWObjManifest& m, rgw::sal::Zone* zone_svc) -{ - return append(dpp, m, zone_svc->get_zonegroup(), zone_svc->get_params()); -} - void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter, string *override_prefix) { diff --git a/src/rgw/rgw_obj_manifest.h b/src/rgw/rgw_obj_manifest.h index 0c285e97784..ac73359305e 100644 --- a/src/rgw/rgw_obj_manifest.h +++ b/src/rgw/rgw_obj_manifest.h @@ -358,7 +358,6 @@ public: int append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params); - int append(const DoutPrefixProvider *dpp, RGWObjManifest& m, rgw::sal::Zone* zone); bool get_rule(uint64_t ofs, RGWObjManifestRule *rule); diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc index 45a2feff2e4..f8c88c226fc 100644 --- a/src/rgw/rgw_object_expirer_core.cc +++ b/src/rgw/rgw_object_expirer_core.cc @@ -108,7 +108,7 @@ int RGWObjExpStore::objexp_hint_add(const DoutPrefixProvider *dpp, cls_timeindex_add(op, utime_t(delete_at), keyext, hebl); string shard_name = objexp_hint_get_shardname(objexp_key_shard(obj_key, cct->_conf->rgw_objexp_hints_num_shards)); - auto obj = rados_svc->obj(rgw_raw_obj(zone_svc->get_params().log_pool, shard_name)); + auto obj = rados_svc->obj(rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, shard_name)); int r = obj.open(dpp); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl; @@ -131,7 +131,7 @@ int RGWObjExpStore::objexp_hint_list(const DoutPrefixProvider *dpp, cls_timeindex_list(op, utime_t(start_time), utime_t(end_time), marker, max_entries, entries, out_marker, truncated); - auto obj = rados_svc->obj(rgw_raw_obj(zone_svc->get_params().log_pool, oid)); + auto obj = rados_svc->obj(rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, oid)); int r = obj.open(dpp); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl; @@ -180,7 +180,7 @@ int RGWObjExpStore::objexp_hint_trim(const DoutPrefixProvider *dpp, const string& from_marker, const string& to_marker) { - auto obj = rados_svc->obj(rgw_raw_obj(zone_svc->get_params().log_pool, oid)); + auto obj = rados_svc->obj(rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, oid)); int r = obj.open(dpp); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl; diff --git a/src/rgw/rgw_object_expirer_core.h b/src/rgw/rgw_object_expirer_core.h index c02ca5ea9df..90261cdb223 100644 --- a/src/rgw/rgw_object_expirer_core.h +++ b/src/rgw/rgw_object_expirer_core.h @@ -29,7 +29,6 @@ #include "include/utime.h" #include "include/str_list.h" -#include "rgw_sal.h" #include "rgw_sal_rados.h" class RGWSI_RADOS; @@ -40,11 +39,11 @@ class cls_timeindex_entry; class RGWObjExpStore { CephContext *cct; RGWSI_RADOS *rados_svc; - rgw::sal::Zone* zone_svc; + rgw::sal::RadosStore* store; public: - RGWObjExpStore(CephContext *_cct, RGWSI_RADOS *_rados_svc, rgw::sal::Zone* _zone_svc) : cct(_cct), + RGWObjExpStore(CephContext *_cct, RGWSI_RADOS *_rados_svc, rgw::sal::RadosStore* _store) : cct(_cct), rados_svc(_rados_svc), - zone_svc(_zone_svc) {} + store(_store) {} int objexp_hint_add(const DoutPrefixProvider *dpp, const ceph::real_time& delete_at, @@ -103,7 +102,7 @@ protected: public: explicit RGWObjectExpirer(rgw::sal::Store* _store) : store(_store), - exp_store(_store->ctx(), static_cast(store)->svc()->rados, store->get_zone()), + exp_store(_store->ctx(), static_cast(store)->svc()->rados, static_cast(store)), worker(NULL) { } ~RGWObjectExpirer() { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 961d3aec8e0..6f45b353ce3 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -531,19 +531,11 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Store* st s->bucket_owner = s->bucket_acl->get_owner(); - RGWZoneGroup zonegroup; - int r = store->get_zone()->get_zonegroup(s->bucket->get_info().zonegroup, zonegroup); + std::unique_ptr zonegroup; + int r = store->get_zone()->get_zonegroup(s->bucket->get_info().zonegroup, &zonegroup); if (!r) { - if (!zonegroup.endpoints.empty()) { - s->zonegroup_endpoint = zonegroup.endpoints.front(); - } else { - // use zonegroup's master zone endpoints - auto z = zonegroup.zones.find(zonegroup.master_zone); - if (z != zonegroup.zones.end() && !z->second.endpoints.empty()) { - s->zonegroup_endpoint = z->second.endpoints.front(); - } - } - s->zonegroup_name = zonegroup.get_name(); + s->zonegroup_endpoint = zonegroup->get_endpoint(); + s->zonegroup_name = zonegroup->get_name(); } if (r < 0 && ret == 0) { ret = r; @@ -571,7 +563,7 @@ int rgw_build_bucket_policies(const DoutPrefixProvider *dpp, rgw::sal::Store* st s->dest_placement.storage_class = s->info.storage_class; s->dest_placement.inherit_from(s->bucket->get_placement_rule()); - if (!store->get_zone()->get_params().valid_placement(s->dest_placement)) { + if (!store->valid_placement(s->dest_placement)) { ldpp_dout(dpp, 0) << "NOTICE: invalid dest placement: " << s->dest_placement.to_str() << dendl; return -EINVAL; } @@ -2420,9 +2412,11 @@ void RGWListBuckets::execute(optional_yield y) /* We need to have stats for all our policies - even if a given policy * isn't actually used in a given account. In such situation its usage * stats would be simply full of zeros. */ - for (const auto& policy : store->get_zone()->get_zonegroup().placement_targets) { - policies_stats.emplace(policy.second.name, - decltype(policies_stats)::mapped_type()); + std::set targets; + if (store->get_zone()->get_zonegroup().get_placement_target_names(targets)) { + for (const auto& policy : targets) { + policies_stats.emplace(policy, decltype(policies_stats)::mapped_type()); + } } std::map>& m = buckets.get_buckets(); @@ -2561,9 +2555,10 @@ void RGWStatAccount::execute(optional_yield y) /* We need to have stats for all our policies - even if a given policy * isn't actually used in a given account. In such situation its usage * stats would be simply full of zeros. */ - for (const auto& policy : store->get_zone()->get_zonegroup().placement_targets) { - policies_stats.emplace(policy.second.name, - decltype(policies_stats)::mapped_type()); + std::set names; + store->get_zone()->get_zonegroup().get_placement_target_names(names); + for (const auto& policy : names) { + policies_stats.emplace(policy, decltype(policies_stats)::mapped_type()); } std::map>& m = buckets.get_buckets(); @@ -3187,7 +3182,6 @@ void RGWCreateBucket::execute(optional_yield y) buffer::list aclbl; buffer::list corsbl; string bucket_name = rgw_make_bucket_entry_name(s->bucket_tenant, s->bucket_name); - rgw_raw_obj obj(store->get_zone()->get_params().domain_root, bucket_name); op_ret = get_params(y); if (op_ret < 0) @@ -3204,21 +3198,22 @@ void RGWCreateBucket::execute(optional_yield y) } if (!relaxed_region_enforcement && !store->get_zone()->get_zonegroup().is_master_zonegroup() && !location_constraint.empty() && - store->get_zone()->get_zonegroup().api_name != location_constraint) { + store->get_zone()->get_zonegroup().get_api_name() != location_constraint) { ldpp_dout(this, 0) << "location constraint (" << location_constraint << ")" - << " doesn't match zonegroup" << " (" << store->get_zone()->get_zonegroup().api_name << ")" + << " doesn't match zonegroup" << " (" << store->get_zone()->get_zonegroup().get_api_name() << ")" << dendl; op_ret = -ERR_INVALID_LOCATION_CONSTRAINT; s->err.message = "The specified location-constraint is not valid"; return; } - const auto& zonegroup = store->get_zone()->get_zonegroup(); + std::set names; + store->get_zone()->get_zonegroup().get_placement_target_names(names); if (!placement_rule.name.empty() && - !zonegroup.placement_targets.count(placement_rule.name)) { + !names.count(placement_rule.name)) { ldpp_dout(this, 0) << "placement target (" << placement_rule.name << ")" << " doesn't exist in the placement targets of zonegroup" - << " (" << store->get_zone()->get_zonegroup().api_name << ")" << dendl; + << " (" << store->get_zone()->get_zonegroup().get_api_name() << ")" << dendl; op_ret = -ERR_INVALID_LOCATION_CONSTRAINT; s->err.message = "The specified placement target does not exist"; return; @@ -4036,7 +4031,7 @@ void RGWPutObj::execute(optional_yield y) // no filters by default rgw::sal::DataProcessor *filter = processor.get(); - const auto& compression_type = store->get_zone()->get_params().get_compression_type(*pdest_placement); + const auto& compression_type = store->get_compression_type(*pdest_placement); CompressorRef plugin; boost::optional compressor; @@ -4393,8 +4388,7 @@ void RGWPostObj::execute(optional_yield y) if (encrypt != nullptr) { filter = encrypt.get(); } else { - const auto& compression_type = store->get_zone()->get_params().get_compression_type( - s->dest_placement); + const auto& compression_type = store->get_compression_type(s->dest_placement); if (compression_type != "none") { plugin = Compressor::create(s->cct, compression_type); if (!plugin) { @@ -7275,9 +7269,6 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y) rgw_obj_key object_junk; std::tie(bucket_name, object_junk) = *parse_path(path); - rgw_raw_obj obj(store->get_zone()->get_params().domain_root, - rgw_make_bucket_entry_name(s->bucket_tenant, bucket_name)); - /* we need to make sure we read bucket info, it's not read before for this * specific request */ std::unique_ptr bucket; @@ -7446,8 +7437,7 @@ int RGWBulkUploadOp::handle_file(const std::string_view path, /* No filters by default. */ rgw::sal::DataProcessor *filter = processor.get(); - const auto& compression_type = store->get_zone()->get_params().get_compression_type( - dest_placement); + const auto& compression_type = store->get_compression_type(dest_placement); CompressorRef plugin; boost::optional compressor; if (compression_type != "none") { diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 6c942df8013..94eac61c6f6 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -150,7 +150,7 @@ int RGWOrphanStore::list_jobs(map & job_list) int RGWOrphanStore::init(const DoutPrefixProvider *dpp) { - const rgw_pool& log_pool = store->get_zone()->get_params().log_pool; + const rgw_pool& log_pool = static_cast(store)->svc()->zone->get_zone_params().log_pool; int r = rgw_init_ioctx(dpp, static_cast(store)->getRados()->get_rados_handle(), log_pool, ioctx); if (r < 0) { cerr << "ERROR: failed to open log pool (" << log_pool << " ret=" << r << std::endl; diff --git a/src/rgw/rgw_period_pusher.cc b/src/rgw/rgw_period_pusher.cc index f9963ee485c..3d60c42d838 100644 --- a/src/rgw/rgw_period_pusher.cc +++ b/src/rgw/rgw_period_pusher.cc @@ -167,15 +167,15 @@ RGWPeriodPusher::RGWPeriodPusher(const DoutPrefixProvider *dpp, rgw::sal::Store* optional_yield y) : cct(store->ctx()), store(store) { - const auto& realm = store->get_zone()->get_realm(); - auto& realm_id = realm.get_id(); + rgw::sal::Zone* zone = store->get_zone(); + auto& realm_id = zone->get_realm_id(); if (realm_id.empty()) // no realm configuration return; // always send out the current period on startup RGWPeriod period; // XXX dang - int r = period.init(dpp, cct, static_cast(store)->svc()->sysobj, realm_id, y, realm.get_name()); + int r = period.init(dpp, cct, static_cast(store)->svc()->sysobj, realm_id, y, zone->get_realm_name()); if (r < 0) { ldpp_dout(dpp, -1) << "failed to load period for realm " << realm_id << dendl; return; @@ -238,7 +238,7 @@ void RGWPeriodPusher::handle_notify(RGWZonesNeedPeriod&& period) auto& my_zonegroup = i->second; // if we're not a master zone, we're not responsible for pushing any updates - if (my_zonegroup.master_zone != store->get_zone()->get_params().get_id()) + if (my_zonegroup.master_zone != store->get_zone()->get_id()) return; // construct a map of the zones that need this period. the map uses the same @@ -266,7 +266,7 @@ void RGWPeriodPusher::handle_notify(RGWZonesNeedPeriod&& period) // update other zone endpoints for (auto& z : my_zonegroup.zones) { auto& zone = z.second; - if (zone.id == store->get_zone()->get_params().get_id()) + if (zone.id == store->get_zone()->get_id().id) continue; if (zone.endpoints.empty()) continue; diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 8da8d2564d4..0b4579169c5 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -18,6 +18,7 @@ #include "rgw_multi.h" #include "rgw_compression.h" #include "services/svc_sys_obj.h" +#include "services/svc_zone.h" #include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw @@ -630,7 +631,7 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c //For Append obj, disable versioning op_target.set_versioning_disabled(true); if (cur_manifest) { - cur_manifest->append(dpp, manifest, store->get_zone()); + cur_manifest->append(dpp, manifest, store->svc()->zone->get_zonegroup(), store->svc()->zone->get_zone_params()); obj_op.meta.manifest = cur_manifest; } else { obj_op.meta.manifest = &manifest; diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index fd2b781fa20..0e898dee9d1 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -180,7 +180,7 @@ string uppercase_underscore_http_attr(const string& orig) static set hostnames_set; static set hostnames_s3website_set; -void rgw_rest_init(CephContext *cct, const RGWZoneGroup& zone_group) +void rgw_rest_init(CephContext *cct, const rgw::sal::ZoneGroup& zone_group) { for (const auto& rgw2http : base_rgw_to_http_attrs) { rgw_to_http_attrs[rgw2http.rgw_attr] = rgw2http.http_attr; @@ -209,9 +209,11 @@ void rgw_rest_init(CephContext *cct, const RGWZoneGroup& zone_group) for (const struct rgw_http_status_code *h = http_codes; h->code; h++) { http_status_names[h->code] = h->name; } + std::list names; + zone_group.get_hostnames(names); hostnames_set.insert(cct->_conf->rgw_dns_name); - hostnames_set.insert(zone_group.hostnames.begin(), zone_group.hostnames.end()); + hostnames_set.insert(names.begin(), names.end()); hostnames_set.erase(""); // filter out empty hostnames ldout(cct, 20) << "RGW hostnames: " << hostnames_set << dendl; /* TODO: We should have a sanity check that no hostname matches the end of @@ -224,8 +226,9 @@ void rgw_rest_init(CephContext *cct, const RGWZoneGroup& zone_group) * X.B.A ambigously splits to both {X, B.A} and {X.B, A} */ + zone_group.get_s3website_hostnames(names); hostnames_s3website_set.insert(cct->_conf->rgw_dns_s3website_name); - hostnames_s3website_set.insert(zone_group.hostnames_s3website.begin(), zone_group.hostnames_s3website.end()); + hostnames_s3website_set.insert(names.begin(), names.end()); hostnames_s3website_set.erase(""); // filter out empty hostnames ldout(cct, 20) << "RGW S3website hostnames: " << hostnames_s3website_set << dendl; /* TODO: we should repeat the hostnames_set sanity check here diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index d12c4e80edb..4a0142fbaa0 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -16,7 +16,7 @@ extern std::map rgw_to_http_attrs; -extern void rgw_rest_init(CephContext *cct, const RGWZoneGroup& zone_group); +extern void rgw_rest_init(CephContext *cct, const rgw::sal::ZoneGroup& zone_group); extern void rgw_flush_formatter_and_reset(struct req_state *s, ceph::Formatter *formatter); diff --git a/src/rgw/rgw_rest_config.cc b/src/rgw/rgw_rest_config.cc index f0b7c60bce3..d28637cbd54 100644 --- a/src/rgw/rgw_rest_config.cc +++ b/src/rgw/rgw_rest_config.cc @@ -61,7 +61,7 @@ void RGWOp_ZoneGroupMap_Get::send_response() { } void RGWOp_ZoneConfig_Get::send_response() { - const RGWZoneParams& zone_params = store->get_zone()->get_params(); + const RGWZoneParams& zone_params = static_cast(store)->svc()->zone->get_zone_params(); set_req_state_err(s, op_ret); dump_errno(s); diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc index afcc86db11c..a93e855e99d 100644 --- a/src/rgw/rgw_rest_conn.cc +++ b/src/rgw/rgw_rest_conn.cc @@ -6,29 +6,10 @@ #include "rgw_sal.h" #include "rgw_rados.h" -#include "services/svc_zone.h" - #define dout_subsys ceph_subsys_rgw using namespace std; -RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, - const string& _remote_id, - const list& remote_endpoints, - std::optional _api_name, - HostStyle _host_style) - : cct(_cct), - endpoints(remote_endpoints.begin(), remote_endpoints.end()), - remote_id(_remote_id), - api_name(_api_name), - host_style(_host_style) -{ - if (zone_svc) { - key = zone_svc->get_zone_params().system_key; - self_zone_group = zone_svc->get_zonegroup().get_id(); - } -} - RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Store* store, const string& _remote_id, const list& remote_endpoints, @@ -41,45 +22,26 @@ RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Store* store, host_style(_host_style) { if (store) { - key = store->get_zone()->get_params().system_key; + key = store->get_zone()->get_system_key(); self_zone_group = store->get_zone()->get_zonegroup().get_id(); } } -RGWRESTConn::RGWRESTConn(CephContext *_cct, RGWSI_Zone *zone_svc, - const string& _remote_id, - const list& remote_endpoints, - RGWAccessKey _cred, - std::optional _api_name, - HostStyle _host_style) - : cct(_cct), - endpoints(remote_endpoints.begin(), remote_endpoints.end()), - key(std::move(_cred)), - remote_id(_remote_id), - api_name(_api_name), - host_style(_host_style) -{ - if (zone_svc) { - self_zone_group = zone_svc->get_zonegroup().get_id(); - } -} - -RGWRESTConn::RGWRESTConn(CephContext *_cct, rgw::sal::Store* store, +RGWRESTConn::RGWRESTConn(CephContext *_cct, const string& _remote_id, const list& remote_endpoints, RGWAccessKey _cred, + std::string _zone_group, std::optional _api_name, HostStyle _host_style) : cct(_cct), endpoints(remote_endpoints.begin(), remote_endpoints.end()), - key(std::move(_cred)), + key(_cred), + self_zone_group(_zone_group), remote_id(_remote_id), api_name(_api_name), host_style(_host_style) { - if (store) { - self_zone_group = store->get_zone()->get_zonegroup().get_id(); - } } RGWRESTConn::RGWRESTConn(RGWRESTConn&& other) diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h index cae8120705c..95d3a897dca 100644 --- a/src/rgw/rgw_rest_conn.h +++ b/src/rgw/rgw_rest_conn.h @@ -81,12 +81,6 @@ class RGWRESTConn public: - RGWRESTConn(CephContext *_cct, - RGWSI_Zone *zone_svc, - const std::string& _remote_id, - const std::list& endpoints, - std::optional _api_name, - HostStyle _host_style = PathStyle); RGWRESTConn(CephContext *_cct, rgw::sal::Store* store, const std::string& _remote_id, @@ -94,19 +88,12 @@ public: std::optional _api_name, HostStyle _host_style = PathStyle); RGWRESTConn(CephContext *_cct, - RGWSI_Zone *zone_svc, - const std::string& _remote_id, - const std::list& endpoints, - RGWAccessKey _cred, - std::optional _api_name, - HostStyle _host_style = PathStyle); - RGWRESTConn(CephContext *_cct, - rgw::sal::Store* store, - const std::string& _remote_id, - const std::list& endpoints, - RGWAccessKey _cred, - std::optional _api_name, - HostStyle _host_style = PathStyle); + const std::string& _remote_id, + const std::list& endpoints, + RGWAccessKey _cred, + std::string _zone_group, + std::optional _api_name, + HostStyle _host_style = PathStyle); // custom move needed for atomic RGWRESTConn(RGWRESTConn&& other); @@ -240,17 +227,10 @@ class S3RESTConn : public RGWRESTConn { public: - S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const std::string& _remote_id, const std::list& endpoints, std::optional _api_name, HostStyle _host_style = PathStyle) : - RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _api_name, _host_style) {} - S3RESTConn(CephContext *_cct, rgw::sal::Store* store, const std::string& _remote_id, const std::list& endpoints, std::optional _api_name, HostStyle _host_style = PathStyle) : RGWRESTConn(_cct, store, _remote_id, endpoints, _api_name, _host_style) {} - - S3RESTConn(CephContext *_cct, RGWSI_Zone *svc_zone, const std::string& _remote_id, const std::list& endpoints, RGWAccessKey _cred, std::optional _api_name, HostStyle _host_style = PathStyle): - RGWRESTConn(_cct, svc_zone, _remote_id, endpoints, _cred, _api_name, _host_style) {} - - S3RESTConn(CephContext *_cct, rgw::sal::Store* store, const std::string& _remote_id, const std::list& endpoints, RGWAccessKey _cred, std::optional _api_name, HostStyle _host_style = PathStyle): - RGWRESTConn(_cct, store, _remote_id, endpoints, _cred, _api_name, _host_style) {} + S3RESTConn(CephContext *_cct, const std::string& _remote_id, const std::list& endpoints, RGWAccessKey _cred, std::string _zone_group, std::optional _api_name, HostStyle _host_style = PathStyle): + RGWRESTConn(_cct, _remote_id, endpoints, _cred, _zone_group, _api_name, _host_style) {} ~S3RESTConn() override = default; void populate_params(param_vec_t& params, const rgw_user *uid, const std::string& zonegroup) override { diff --git a/src/rgw/rgw_rest_ratelimit.cc b/src/rgw/rgw_rest_ratelimit.cc index 5820ab71abd..df6ba0f44ff 100644 --- a/src/rgw/rgw_rest_ratelimit.cc +++ b/src/rgw/rgw_rest_ratelimit.cc @@ -98,7 +98,7 @@ void RGWOp_Ratelimit_Info::execute(optional_yield y) flusher.flush(); } if (global) { - std::string realm_id = store->get_zone()->get_realm().get_id(); + std::string realm_id = store->get_zone()->get_realm_id(); RGWPeriodConfig period_config; op_ret = period_config.read(this, static_cast(store)->svc()->sysobj, realm_id, y); if (op_ret && op_ret != -ENOENT) { @@ -301,7 +301,7 @@ void RGWOp_Ratelimit_Set::execute(optional_yield y) return; } if (global) { - std::string realm_id = store->get_zone()->get_realm().get_id(); + std::string realm_id = store->get_zone()->get_realm_id(); RGWPeriodConfig period_config; op_ret = period_config.read(s, static_cast(store)->svc()->sysobj, realm_id, y); if (op_ret && op_ret != -ENOENT) { @@ -346,4 +346,4 @@ RGWOp* RGWHandler_Ratelimit::op_get() RGWOp* RGWHandler_Ratelimit::op_post() { return new RGWOp_Ratelimit_Set; -} \ No newline at end of file +} diff --git a/src/rgw/rgw_rest_realm.cc b/src/rgw/rgw_rest_realm.cc index 6f4ebee8002..9e318ef400a 100644 --- a/src/rgw/rgw_rest_realm.cc +++ b/src/rgw/rgw_rest_realm.cc @@ -111,9 +111,9 @@ void RGWOp_Period_Post::execute(optional_yield y) } // require period.realm_id to match our realm - if (period.get_realm() != store->get_zone()->get_realm().get_id()) { + if (period.get_realm() != static_cast(store)->svc()->zone->get_realm().get_id()) { error_stream << "period with realm id " << period.get_realm() - << " doesn't match current realm " << store->get_zone()->get_realm().get_id() << std::endl; + << " doesn't match current realm " << static_cast(store)->svc()->zone->get_realm().get_id() << std::endl; op_ret = -EINVAL; return; } @@ -147,7 +147,7 @@ void RGWOp_Period_Post::execute(optional_yield y) } // if it's not period commit, nobody is allowed to push to the master zone - if (period.get_master_zone() == store->get_zone()->get_params().get_id()) { + if (period.get_master_zone() == static_cast(store)->svc()->zone->get_zone_params().get_id()) { ldpp_dout(this, 10) << "master zone rejecting period id=" << period.get_id() << " epoch=" << period.get_epoch() << dendl; op_ret = -EINVAL; // XXX: error code diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index f2bdfa39b04..b28c1cf5d4b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -936,9 +936,9 @@ struct ReplicationConfiguration { set ids; for (auto& name : zone_names) { - rgw_zone_id id; - if (static_cast(store)->svc()->zone->find_zone_id_by_name(name, &id)) { - ids.insert(std::move(id)); + rgw_zone_id id; + if (static_cast(store)->svc()->zone->find_zone_id_by_name(name, &id)) { + ids.insert(std::move(id)); } } @@ -950,10 +950,10 @@ struct ReplicationConfiguration { vector names; for (auto& id : zone_ids) { - RGWZone *zone; - if (static_cast(store)->svc()->zone->find_zone(id, &zone)) { - names.emplace_back(zone->name); - } + RGWZone *zone; + if (static_cast(store)->svc()->zone->find_zone(id, &zone)) { + names.emplace_back(zone->name); + } } return names; @@ -1900,12 +1900,12 @@ void RGWGetBucketLocation_ObjStore_S3::send_response() end_header(s, this); dump_start(s); - RGWZoneGroup zonegroup; + std::unique_ptr zonegroup; string api_name; - int ret = store->get_zone()->get_zonegroup(s->bucket->get_info().zonegroup, zonegroup); + int ret = store->get_zone()->get_zonegroup(s->bucket->get_info().zonegroup, &zonegroup); if (ret >= 0) { - api_name = zonegroup.api_name; + api_name = zonegroup->get_api_name(); } else { if (s->bucket->get_info().zonegroup != "default") { api_name = s->bucket->get_info().zonegroup; @@ -2725,7 +2725,7 @@ int RGWPostObj_ObjStore_S3::get_params(optional_yield y) if (! storage_class.empty()) { s->dest_placement.storage_class = storage_class; - if (!store->get_zone()->get_params().valid_placement(s->dest_placement)) { + if (!store->valid_placement(s->dest_placement)) { ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << s->dest_placement.to_str() << dendl; err_msg = "The storage class you specified is not valid"; return -EINVAL; diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index c403ab70006..6d533931a7a 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -714,7 +714,7 @@ int RGWCreateBucket_ObjStore_SWIFT::get_params(optional_yield y) policy.create_default(s->user->get_id(), s->user->get_display_name()); } - location_constraint = store->get_zone()->get_zonegroup().api_name; + location_constraint = store->get_zone()->get_zonegroup().get_api_name(); get_rmattrs_from_headers(s, CONT_PUT_ATTR_PREFIX, CONT_REMOVE_ATTR_PREFIX, rmattr_names); placement_rule.init(s->info.env->get("HTTP_X_STORAGE_POLICY", ""), s->info.storage_class); @@ -1899,14 +1899,17 @@ void RGWInfo_ObjStore_SWIFT::list_swift_data(Formatter& formatter, } formatter.open_array_section("policies"); - const RGWZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); + const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); - for (const auto& placement_targets : zonegroup.placement_targets) { - formatter.open_object_section("policy"); - if (placement_targets.second.name.compare(zonegroup.default_placement.name) == 0) - formatter.dump_bool("default", true); - formatter.dump_string("name", placement_targets.second.name.c_str()); - formatter.close_section(); + std::set targets; + if (zonegroup.get_placement_target_names(targets)) { + for (const auto& placement_targets : targets) { + formatter.open_object_section("policy"); + if (placement_targets.compare(zonegroup.get_default_placement_name()) == 0) + formatter.dump_bool("default", true); + formatter.dump_string("name", placement_targets.c_str()); + formatter.close_section(); + } } formatter.close_section(); diff --git a/src/rgw/rgw_rest_user.cc b/src/rgw/rgw_rest_user.cc index e715752142d..e5fd846132c 100644 --- a/src/rgw/rgw_rest_user.cc +++ b/src/rgw/rgw_rest_user.cc @@ -212,7 +212,7 @@ void RGWOp_User_Create::execute(optional_yield y) if (!default_placement_str.empty()) { rgw_placement_rule target_rule; target_rule.from_str(default_placement_str); - if (!store->get_zone()->get_params().valid_placement(target_rule)) { + if (!store->valid_placement(target_rule)) { ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << target_rule.to_str() << dendl; op_ret = -EINVAL; return; @@ -351,7 +351,7 @@ void RGWOp_User_Modify::execute(optional_yield y) if (!default_placement_str.empty()) { rgw_placement_rule target_rule; target_rule.from_str(default_placement_str); - if (!store->get_zone()->get_params().valid_placement(target_rule)) { + if (!store->valid_placement(target_rule)) { ldpp_dout(this, 0) << "NOTICE: invalid dest placement: " << target_rule.to_str() << dendl; op_ret = -EINVAL; return; diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index d08fbcc8713..988b231decc 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -96,7 +96,12 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d .set_run_sync_thread(run_sync_thread) .set_run_reshard_thread(run_reshard_thread) .initialize(cct, dpp) < 0) { - delete store; store = nullptr; + delete store; + return nullptr; + } + if (store->initialize(cct, dpp) < 0) { + delete store; + return nullptr; } return store; } @@ -114,7 +119,8 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d .set_run_sync_thread(run_sync_thread) .set_run_reshard_thread(run_reshard_thread) .initialize(cct, dpp) < 0) { - delete store; store = nullptr; + delete store; + return nullptr; } return store; } @@ -125,7 +131,8 @@ rgw::sal::Store* StoreManager::init_storage_provider(const DoutPrefixProvider* d if ((*(rgw::sal::DBStore*)store).set_run_lc_thread(use_lc_thread) .initialize(cct, dpp) < 0) { - delete store; store = nullptr; + delete store; + return nullptr; } /* XXX: temporary - create testid user */ @@ -202,12 +209,17 @@ rgw::sal::Store* StoreManager::init_raw_storage_provider(const DoutPrefixProvide int ret = rados->init_svc(true, dpp); if (ret < 0) { ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl; - delete store; store = nullptr; - return store; + delete store; + return nullptr; } if (rados->init_rados() < 0) { - delete store; store = nullptr; + delete store; + return nullptr; + } + if (store->initialize(cct, dpp) < 0) { + delete store; + return nullptr; } } diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 38ef3bf4b95..db7b0f586e3 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -22,9 +22,6 @@ class RGWGetDataCB; class RGWAccessListFilter; class RGWLC; -struct RGWZoneGroup; -struct RGWZoneParams; -struct RGWRealm; struct RGWCtl; struct rgw_user_bucket; class RGWUsageBatch; @@ -212,6 +209,7 @@ class Zone; class LuaScriptManager; class RGWOIDCProvider; class RGWRole; +class PlacementTier; enum AttrsMod { ATTRSMOD_NONE = 0, @@ -286,8 +284,10 @@ class Store { Store() {} virtual ~Store() = default; + /** Post-creation initialization of store */ + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) = 0; /** Name of this store provider (e.g., "rados") */ - virtual const char* get_name() const = 0; + virtual const std::string get_name() const = 0; /** Get cluster unique identifier */ virtual std::string get_cluster_id(const DoutPrefixProvider* dpp, optional_yield y) = 0; /** Get a User from a rgw_user. Does not query store for user info, so quick */ @@ -437,6 +437,11 @@ class Store { uint64_t olh_epoch, const std::string& unique_tag) = 0; + /** Get the compression type of a placement rule */ + virtual const std::string& get_compression_type(const rgw_placement_rule& rule) = 0; + /** Check to see if this placement rule is valid */ + virtual bool valid_placement(const rgw_placement_rule& rule) = 0; + /** Clean up a store for termination */ virtual void finalize(void) = 0; @@ -1073,6 +1078,15 @@ class Object { uint64_t olh_epoch, const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Move an object to the cloud */ + virtual int transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) = 0; /** Check to see if two placement rules match */ virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) = 0; /** Dump store-specific object layout info in JSON */ @@ -1441,6 +1455,40 @@ public: optional_yield y) = 0; }; +class PlacementTier { +public: + virtual ~PlacementTier() = default; + + /** Get the type of this tier */ + virtual const std::string& get_tier_type() = 0; + /** Get the storage class of this tier */ + virtual const std::string& get_storage_class() = 0; + /** Should we retain the head object when transitioning */ + virtual bool retain_head_object() = 0; + /** Get the placement rule associated with this tier */ +}; + +class ZoneGroup { +public: + virtual ~ZoneGroup() = default; + virtual const std::string& get_id() const = 0; + virtual const std::string& get_name() const = 0; + virtual int equals(const std::string& other_zonegroup) const = 0; + /** Get the endpoint from zonegroup, or from master zone if not set */ + virtual const std::string& get_endpoint() const = 0; + virtual bool placement_target_exists(std::string& target) const = 0; + virtual bool is_master_zonegroup() const = 0; + virtual const std::string& get_api_name() const = 0; + virtual int get_placement_target_names(std::set& names) const = 0; + virtual const std::string& get_default_placement_name() const = 0; + virtual int get_hostnames(std::list& names) const = 0; + virtual int get_s3website_hostnames(std::list& names) const = 0; + /** Get the number of zones in this zonegroup */ + virtual int get_zone_count() const = 0; + /** Get the placement tier associated with the rule */ + virtual int get_placement_tier(const rgw_placement_rule& rule, std::unique_ptr* tier) = 0; +}; + /** * @brief Abstraction of a Zone * @@ -1452,15 +1500,11 @@ class Zone { virtual ~Zone() = default; /** Get info about the zonegroup containing this zone */ - virtual const RGWZoneGroup& get_zonegroup() = 0; + virtual ZoneGroup& get_zonegroup() = 0; /** Get info about a zonegroup by ID */ - virtual int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) = 0; - /** Get the parameters of this zone */ - virtual const RGWZoneParams& get_params() = 0; + virtual int get_zonegroup(const std::string& id, std::unique_ptr* zonegroup) = 0; /** Get the ID of this zone */ virtual const rgw_zone_id& get_id() = 0; - /** Get info about the realm containing this zone */ - virtual const RGWRealm& get_realm() = 0; /** Get the name of this zone */ virtual const std::string& get_name() const = 0; /** True if this zone is writable */ @@ -1471,6 +1515,12 @@ class Zone { virtual bool has_zonegroup_api(const std::string& api) const = 0; /** Get the current period ID for this zone */ virtual const std::string& get_current_period_id() = 0; + /** Get thes system access key for this zone */ + virtual const RGWAccessKey& get_system_key() = 0; + /** Get the name of the realm containing this zone */ + virtual const std::string& get_realm_name() = 0; + /** Get the ID of the realm containing this zone */ + virtual const std::string& get_realm_id() = 0; }; /** diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index e7415b9907b..e0b0aa3224a 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -508,19 +508,48 @@ namespace rgw::sal { dbsm->destroyAllHandles(); } - const RGWZoneGroup& DBZone::get_zonegroup() + const std::string& DBZoneGroup::get_endpoint() const { + if (!group->endpoints.empty()) { + return group->endpoints.front(); + } else { + // use zonegroup's master zone endpoints + auto z = group->zones.find(group->master_zone); + if (z != group->zones.end() && !z->second.endpoints.empty()) { + return z->second.endpoints.front(); + } + } + return empty; + } + + bool DBZoneGroup::placement_target_exists(std::string& target) const { + return !!group->placement_targets.count(target); + } + + int DBZoneGroup::get_placement_target_names(std::set& names) const { + for (const auto& target : group->placement_targets) { + names.emplace(target.second.name); + } + + return 0; + } + + ZoneGroup& DBZone::get_zonegroup() { return *zonegroup; } - int DBZone::get_zonegroup(const std::string& id, RGWZoneGroup& zg) + int DBZone::get_zonegroup(const std::string& id, std::unique_ptr* zg) { /* XXX: for now only one zonegroup supported */ - zg = *zonegroup; + ZoneGroup* group = new DBZoneGroup(store, std::make_unique()); + if (!group) + return -ENOMEM; + + zg->reset(group); return 0; } - const RGWZoneParams& DBZone::get_params() + const RGWZoneParams& DBZone::get_rgw_params() { return *zone_params; } @@ -530,10 +559,6 @@ namespace rgw::sal { return cur_zone_id; } - const RGWRealm& DBZone::get_realm() - { - return *realm; - } const std::string& DBZone::get_name() const { @@ -560,6 +585,21 @@ namespace rgw::sal { return current_period->get_id(); } + const RGWAccessKey& DBZone::get_system_key() + { + return zone_params->system_key; + } + + const std::string& DBZone::get_realm_name() + { + return realm->get_name(); + } + + const std::string& DBZone::get_realm_id() + { + return realm->get_id(); + } + std::unique_ptr DBStore::get_lua_script_manager() { return std::make_unique(this); @@ -695,6 +735,18 @@ namespace rgw::sal { return 0; } + int DBObject::transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) + { + return 0; + } + bool DBObject::placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) { /* XXX: support single default zone and zonegroup for now */ @@ -1543,6 +1595,15 @@ namespace rgw::sal { ptail_placement_rule, olh_epoch, unique_tag); } + const std::string& DBStore::get_compression_type(const rgw_placement_rule& rule) { + return zone.get_rgw_params().get_compression_type(rule); + } + + bool DBStore::valid_placement(const rgw_placement_rule& rule) + { + return zone.get_rgw_params().valid_placement(rule); + } + std::unique_ptr DBStore::get_user(const rgw_user &u) { return std::make_unique(this, u); diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index b7451dc91ca..5cd3fbc8015 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -232,11 +232,65 @@ protected: friend class DBStore; }; + class DBPlacementTier: public PlacementTier { + DBStore* store; + RGWZoneGroupPlacementTier tier; + public: + DBPlacementTier(DBStore* _store, const RGWZoneGroupPlacementTier& _tier) : store(_store), tier(_tier) {} + virtual ~DBPlacementTier() = default; + + virtual const std::string& get_tier_type() { return tier.tier_type; } + virtual const std::string& get_storage_class() { return tier.storage_class; } + virtual bool retain_head_object() { return tier.retain_head_object; } + RGWZoneGroupPlacementTier& get_rt() { return tier; } + }; + + class DBZoneGroup : public ZoneGroup { + DBStore* store; + std::unique_ptr group; + std::string empty; + public: + DBZoneGroup(DBStore* _store, std::unique_ptr _group) : store(_store), group(std::move(_group)) {} + virtual ~DBZoneGroup() = default; + + virtual const std::string& get_id() const override { return group->get_id(); }; + virtual const std::string& get_name() const override { return group->get_name(); }; + virtual int equals(const std::string& other_zonegroup) const override { + return group->equals(other_zonegroup); + }; + /** Get the endpoint from zonegroup, or from master zone if not set */ + virtual const std::string& get_endpoint() const override; + virtual bool placement_target_exists(std::string& target) const override; + virtual bool is_master_zonegroup() const override { + return group->is_master_zonegroup(); + }; + virtual const std::string& get_api_name() const override { return group->api_name; }; + virtual int get_placement_target_names(std::set& names) const override; + virtual const std::string& get_default_placement_name() const override { + return group->default_placement.name; }; + virtual int get_hostnames(std::list& names) const override { + names = group->hostnames; + return 0; + }; + virtual int get_s3website_hostnames(std::list& names) const override { + names = group->hostnames_s3website; + return 0; + }; + virtual int get_zone_count() const override { + /* currently only 1 zone supported */ + return 1; + } + virtual int get_placement_tier(const rgw_placement_rule& rule, + std::unique_ptr* tier) { + return -1; + } + }; + class DBZone : public Zone { protected: DBStore* store; RGWRealm *realm{nullptr}; - RGWZoneGroup *zonegroup{nullptr}; + DBZoneGroup *zonegroup{nullptr}; RGWZone *zone_public_config{nullptr}; /* external zone params, e.g., entrypoints, log flags, etc. */ RGWZoneParams *zone_params{nullptr}; /* internal zone params, e.g., rados pools */ RGWPeriod *current_period{nullptr}; @@ -244,8 +298,8 @@ protected: public: DBZone(DBStore* _store) : store(_store) { - realm = new RGWRealm(); - zonegroup = new RGWZoneGroup(); + realm = new RGWRealm(); + zonegroup = new DBZoneGroup(store, std::make_unique()); zone_public_config = new RGWZone(); zone_params = new RGWZoneParams(); current_period = new RGWPeriod(); @@ -258,18 +312,26 @@ protected: info.storage_classes = sc; zone_params->placement_pools["default"] = info; } - ~DBZone() = default; + ~DBZone() { + delete realm; + delete zonegroup; + delete zone_public_config; + delete zone_params; + delete current_period; + } - virtual const RGWZoneGroup& get_zonegroup() override; - virtual int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) override; - virtual const RGWZoneParams& get_params() override; + virtual ZoneGroup& get_zonegroup() override; + virtual int get_zonegroup(const std::string& id, std::unique_ptr* zonegroup) override; + const RGWZoneParams& get_rgw_params(); virtual const rgw_zone_id& get_id() override; - virtual const RGWRealm& get_realm() override; virtual const std::string& get_name() const override; virtual bool is_writeable() override; virtual bool get_redirect_endpoint(std::string* endpoint) override; virtual bool has_zonegroup_api(const std::string& api) const override; virtual const std::string& get_current_period_id() override; + virtual const RGWAccessKey& get_system_key() override; + virtual const std::string& get_realm_name() override; + virtual const std::string& get_realm_id() override; }; class DBLuaScriptManager : public LuaScriptManager { @@ -517,6 +579,14 @@ protected: uint64_t olh_epoch, const DoutPrefixProvider* dpp, optional_yield y) override; + virtual int transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) override; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override; @@ -675,9 +745,9 @@ public: return *this; } - int initialize(CephContext *cct, const DoutPrefixProvider *dpp); + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override; - virtual const char* get_name() const override { + virtual const std::string get_name() const override { return "dbstore"; } @@ -779,6 +849,9 @@ public: uint64_t olh_epoch, const std::string& unique_tag) override; + virtual const std::string& get_compression_type(const rgw_placement_rule& rule) override; + virtual bool valid_placement(const rgw_placement_rule& rule) override; + virtual void finalize(void) override; virtual CephContext *ctx(void) override { diff --git a/src/rgw/rgw_sal_motr.cc b/src/rgw/rgw_sal_motr.cc index 200ea53525f..9e0668c33aa 100644 --- a/src/rgw/rgw_sal_motr.cc +++ b/src/rgw/rgw_sal_motr.cc @@ -837,31 +837,78 @@ void MotrStore::finalize(void) { } -const RGWZoneGroup& MotrZone::get_zonegroup() +const std::string& MotrZoneGroup::get_endpoint() const { - return *zonegroup; + if (!group.endpoints.empty()) { + return group.endpoints.front(); + } else { + // use zonegroup's master zone endpoints + auto z = group.zones.find(group.master_zone); + if (z != group.zones.end() && !z->second.endpoints.empty()) { + return z->second.endpoints.front(); + } + } + return empty; } -int MotrZone::get_zonegroup(const std::string& id, RGWZoneGroup& zg) +bool MotrZoneGroup::placement_target_exists(std::string& target) const { - /* XXX: for now only one zonegroup supported */ - zg = *zonegroup; + return !!group.placement_targets.count(target); +} + +int MotrZoneGroup::get_placement_target_names(std::set& names) const +{ + for (const auto& target : group.placement_targets) { + names.emplace(target.second.name); + } + return 0; } -const RGWZoneParams& MotrZone::get_params() +int MotrZoneGroup::get_placement_tier(const rgw_placement_rule& rule, + std::unique_ptr* tier) { - return *zone_params; + std::map::const_iterator titer; + titer = group.placement_targets.find(rule.name); + if (titer == group.placement_targets.end()) { + return -ENOENT; + } + + const auto& target_rule = titer->second; + std::map::const_iterator ttier; + ttier = target_rule.tier_targets.find(rule.storage_class); + if (ttier == target_rule.tier_targets.end()) { + // not found + return -ENOENT; + } + + PlacementTier* t; + t = new MotrPlacementTier(store, ttier->second); + if (!t) + return -ENOMEM; + + tier->reset(t); + return 0; } -const rgw_zone_id& MotrZone::get_id() +ZoneGroup& MotrZone::get_zonegroup() { - return cur_zone_id; + return zonegroup; +} + +int MotrZone::get_zonegroup(const std::string& id, std::unique_ptr* group) +{ + /* XXX: for now only one zonegroup supported */ + ZoneGroup* zg; + zg = new MotrZoneGroup(store, zonegroup.get_group()); + + group->reset(zg); + return 0; } -const RGWRealm& MotrZone::get_realm() +const rgw_zone_id& MotrZone::get_id() { - return *realm; + return cur_zone_id; } const std::string& MotrZone::get_name() const @@ -1075,6 +1122,18 @@ int MotrObject::transition(Bucket* bucket, return 0; } +int MotrObject::transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) +{ + return 0; +} + bool MotrObject::placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) { /* XXX: support single default zone and zonegroup for now */ @@ -2761,6 +2820,16 @@ std::unique_ptr MotrStore::get_atomic_writer(const DoutPrefixProvider *d ptail_placement_rule, olh_epoch, unique_tag); } +const std::string& MotrStore::get_compression_type(const rgw_placement_rule& rule) +{ + return zone.zone_params->get_compression_type(rule); +} + +bool MotrStore::valid_placement(const rgw_placement_rule& rule) +{ + return zone.zone_params->valid_placement(rule); +} + std::unique_ptr MotrStore::get_user(const rgw_user &u) { ldout(cctx, 20) << "bucket's user: " << u.to_str() << dendl; diff --git a/src/rgw/rgw_sal_motr.h b/src/rgw/rgw_sal_motr.h index 6e87aabdcb2..aa9b86ed5f0 100644 --- a/src/rgw/rgw_sal_motr.h +++ b/src/rgw/rgw_sal_motr.h @@ -1,4 +1,4 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- + // vim: ts=2 sw=2 expandtab ft=cpp /* @@ -350,20 +350,85 @@ class MotrBucket : public Bucket { friend class MotrStore; }; +class MotrPlacementTier: public PlacementTier { + MotrStore* store; + RGWZoneGroupPlacementTier tier; +public: + MotrPlacementTier(MotrStore* _store, const RGWZoneGroupPlacementTier& _tier) : store(_store), tier(_tier) {} + virtual ~MotrPlacementTier() = default; + + virtual const std::string& get_tier_type() { return tier.tier_type; } + virtual const std::string& get_storage_class() { return tier.storage_class; } + virtual bool retain_head_object() { return tier.retain_head_object; } + RGWZoneGroupPlacementTier& get_rt() { return tier; } +}; + +class MotrZoneGroup : public ZoneGroup { + MotrStore* store; + const RGWZoneGroup group; + std::string empty; +public: + MotrZoneGroup(MotrStore* _store) : store(_store), group() {} + MotrZoneGroup(MotrStore* _store, const RGWZoneGroup& _group) : store(_store), group(_group) {} + virtual ~MotrZoneGroup() = default; + + virtual const std::string& get_id() const override { return group.get_id(); }; + virtual const std::string& get_name() const override { return group.get_name(); }; + virtual int equals(const std::string& other_zonegroup) const override { + return group.equals(other_zonegroup); + }; + /** Get the endpoint from zonegroup, or from master zone if not set */ + virtual const std::string& get_endpoint() const override; + virtual bool placement_target_exists(std::string& target) const override; + virtual bool is_master_zonegroup() const override { + return group.is_master_zonegroup(); + }; + virtual const std::string& get_api_name() const override { return group.api_name; }; + virtual int get_placement_target_names(std::set& names) const override; + virtual const std::string& get_default_placement_name() const override { + return group.default_placement.name; }; + virtual int get_hostnames(std::list& names) const override { + names = group.hostnames; + return 0; + }; + virtual int get_s3website_hostnames(std::list& names) const override { + names = group.hostnames_s3website; + return 0; + }; + virtual int get_zone_count() const override { + return group.zones.size(); + } + virtual int get_placement_tier(const rgw_placement_rule& rule, std::unique_ptr* tier); + const RGWZoneGroup& get_group() { return group; } +}; + class MotrZone : public Zone { protected: MotrStore* store; RGWRealm *realm{nullptr}; - RGWZoneGroup *zonegroup{nullptr}; + MotrZoneGroup zonegroup; RGWZone *zone_public_config{nullptr}; /* external zone params, e.g., entrypoints, log flags, etc. */ RGWZoneParams *zone_params{nullptr}; /* internal zone params, e.g., rados pools */ RGWPeriod *current_period{nullptr}; rgw_zone_id cur_zone_id; public: - MotrZone(MotrStore* _store) : store(_store) { + MotrZone(MotrStore* _store) : store(_store), zonegroup(_store) { + realm = new RGWRealm(); + zone_public_config = new RGWZone(); + zone_params = new RGWZoneParams(); + current_period = new RGWPeriod(); + cur_zone_id = rgw_zone_id(zone_params->get_id()); + + // XXX: only default and STANDARD supported for now + RGWZonePlacementInfo info; + RGWZoneStorageClasses sc; + sc.set_storage_class("STANDARD", nullptr, nullptr); + info.storage_classes = sc; + zone_params->placement_pools["default"] = info; + } + MotrZone(MotrStore* _store, MotrZoneGroup _zg) : store(_store), zonegroup(_zg) { realm = new RGWRealm(); - zonegroup = new RGWZoneGroup(); zone_public_config = new RGWZone(); zone_params = new RGWZoneParams(); current_period = new RGWPeriod(); @@ -378,16 +443,19 @@ class MotrZone : public Zone { } ~MotrZone() = default; - virtual const RGWZoneGroup& get_zonegroup() override; - virtual int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) override; - virtual const RGWZoneParams& get_params() override; + virtual ZoneGroup& get_zonegroup() override; + virtual int get_zonegroup(const std::string& id, std::unique_ptr* zonegroup) override; virtual const rgw_zone_id& get_id() override; - virtual const RGWRealm& get_realm() override; virtual const std::string& get_name() const override; virtual bool is_writeable() override; virtual bool get_redirect_endpoint(std::string* endpoint) override; virtual bool has_zonegroup_api(const std::string& api) const override; virtual const std::string& get_current_period_id() override; + virtual const RGWAccessKey& get_system_key() { return zone_params->system_key; } + virtual const std::string& get_realm_name() { return realm->get_name(); } + virtual const std::string& get_realm_id() { return realm->get_id(); } + + friend class MotrStore; }; class MotrLuaScriptManager : public LuaScriptManager { @@ -545,6 +613,14 @@ class MotrObject : public Object { uint64_t olh_epoch, const DoutPrefixProvider* dpp, optional_yield y) override; + virtual int transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) override; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override; @@ -817,7 +893,8 @@ class MotrStore : public Store { delete bucket_inst_cache; } - virtual const char* get_name() const override { + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) { return 0; } + virtual const std::string get_name() const override { return "motr"; } @@ -912,6 +989,8 @@ class MotrStore : public Store { const rgw_placement_rule *ptail_placement_rule, uint64_t olh_epoch, const std::string& unique_tag) override; + virtual const std::string& get_compression_type(const rgw_placement_rule& rule) override; + virtual bool valid_placement(const rgw_placement_rule& rule) override; virtual void finalize(void) override; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index be2ecf90481..0e9c3e2648a 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -35,6 +35,7 @@ #include "rgw_rest_conn.h" #include "rgw_service.h" #include "rgw_lc.h" +#include "rgw_lc_tier.h" #include "services/svc_sys_obj.h" #include "services/svc_zone.h" #include "services/svc_tier_rados.h" @@ -1193,6 +1194,13 @@ int RadosStore::get_raw_chunk_size(const DoutPrefixProvider* dpp, const rgw_raw_ return rados->get_max_chunk_size(obj.pool, chunk_size, dpp); } +int RadosStore::initialize(CephContext *cct, const DoutPrefixProvider *dpp) +{ + RadosZoneGroup zg(this, svc()->zone->get_zonegroup()); + zone = make_unique(this, zg); + return 0; +} + int RadosStore::log_usage(const DoutPrefixProvider *dpp, map& usage_info) { return rados->log_usage(dpp, usage_info); @@ -1334,7 +1342,7 @@ int RadosStore::get_roles(const DoutPrefixProvider *dpp, const std::string& tenant, vector>& roles) { - auto pool = get_zone()->get_params().roles_pool; + auto pool = svc()->zone->get_zone_params().roles_pool; std::string prefix; // List all roles if path prefix is empty @@ -1398,7 +1406,7 @@ int RadosStore::get_oidc_providers(const DoutPrefixProvider *dpp, vector>& providers) { std::string prefix = tenant + RGWOIDCProvider::oidc_url_oid_prefix; - auto pool = zone.get_params().oidc_pool; + auto pool = svc()->zone->get_zone_params().oidc_pool; auto obj_ctx = svc()->sysobj->init_obj_ctx(); //Get the filtered objects @@ -1473,6 +1481,16 @@ std::unique_ptr RadosStore::get_atomic_writer(const DoutPrefixProvider * olh_epoch, unique_tag); } +const std::string& RadosStore::get_compression_type(const rgw_placement_rule& rule) +{ + return svc()->zone->get_zone_params().get_compression_type(rule); +} + +bool RadosStore::valid_placement(const rgw_placement_rule& rule) +{ + return svc()->zone->get_zone_params().valid_placement(rule); +} + int RadosStore::get_obj_head_ioctx(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx* ioctx) { return rados->get_obj_head_ioctx(dpp, bucket_info, obj, ioctx); @@ -1670,6 +1688,148 @@ int RadosObject::transition(Bucket* bucket, return store->getRados()->transition_obj(*rados_ctx, bucket, *this, placement_rule, mtime, olh_epoch, dpp, y); } +int RadosObject::transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) +{ + /* init */ + rgw::sal::RadosPlacementTier* rtier = static_cast(tier); + string id = "cloudid"; + string endpoint = rtier->get_rt().t.s3.endpoint; + RGWAccessKey key = rtier->get_rt().t.s3.key; + string region = rtier->get_rt().t.s3.region; + HostStyle host_style = rtier->get_rt().t.s3.host_style; + string bucket_name = rtier->get_rt().t.s3.target_path; + const rgw::sal::ZoneGroup& zonegroup = store->get_zone()->get_zonegroup(); + + if (bucket_name.empty()) { + bucket_name = "rgwx-" + zonegroup.get_name() + "-" + tier->get_storage_class() + + "-cloud-bucket"; + boost::algorithm::to_lower(bucket_name); + } + + /* Create RGW REST connection */ + S3RESTConn conn(cct, id, { endpoint }, key, zonegroup.get_id(), region, host_style); + + RGWLCCloudTierCtx tier_ctx(cct, dpp, o, store, bucket->get_info(), + this, conn, bucket_name, + rtier->get_rt().t.s3.target_storage_class); + tier_ctx.acl_mappings = rtier->get_rt().t.s3.acl_mappings; + tier_ctx.multipart_min_part_size = rtier->get_rt().t.s3.multipart_min_part_size; + tier_ctx.multipart_sync_threshold = rtier->get_rt().t.s3.multipart_sync_threshold; + tier_ctx.storage_class = tier->get_storage_class(); + + // check if target_path is already created + std::pair::iterator, bool> it; + + it = cloud_targets.insert(bucket_name); + tier_ctx.target_bucket_created = !(it.second); + + ldpp_dout(dpp, 0) << "Transitioning object(" << o.key << ") to the cloud endpoint(" << endpoint << ")" << dendl; + + /* Transition object to cloud end point */ + int ret = rgw_cloud_tier_transfer_object(tier_ctx); + + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to transfer object(" << o.key << ") to the cloud endpoint(" << endpoint << ") ret=" << ret << dendl; + return ret; + + if (!tier_ctx.target_bucket_created) { + cloud_targets.erase(it.first); + } + } + + if (update_object) { + real_time read_mtime; + + std::unique_ptr read_op(get_read_op()); + read_op->params.lastmod = &read_mtime; + + ret = read_op->prepare(null_yield, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: Updating tier object(" << o.key << ") failed ret=" << ret << dendl; + return ret; + } + + if (read_mtime != tier_ctx.o.meta.mtime) { + /* raced */ + ldpp_dout(dpp, 0) << "ERROR: Updating tier object(" << o.key << ") failed ret=" << -ECANCELED << dendl; + return -ECANCELED; + } + + rgw_placement_rule target_placement; + target_placement.inherit_from(tier_ctx.bucket_info.placement_rule); + target_placement.storage_class = tier->get_storage_class(); + + ret = write_cloud_tier(dpp, null_yield, tier_ctx.o.versioned_epoch, + tier, tier_ctx.is_multipart_upload, + target_placement, tier_ctx.obj); + + } + + return ret; +} + +int RadosObject::write_cloud_tier(const DoutPrefixProvider* dpp, + optional_yield y, + uint64_t olh_epoch, + PlacementTier* tier, + bool is_multipart_upload, + rgw_placement_rule& target_placement, + Object* head_obj) +{ + rgw::sal::RadosPlacementTier* rtier = static_cast(tier); + map attrs = get_attrs(); + RGWRados::Object op_target(store->getRados(), bucket, *rados_ctx, this); + RGWRados::Object::Write obj_op(&op_target); + + obj_op.meta.modify_tail = true; + obj_op.meta.flags = PUT_OBJ_CREATE; + obj_op.meta.category = RGWObjCategory::CloudTiered; + obj_op.meta.delete_at = real_time(); + bufferlist blo; + obj_op.meta.data = &blo; + obj_op.meta.if_match = NULL; + obj_op.meta.if_nomatch = NULL; + obj_op.meta.user_data = NULL; + obj_op.meta.zones_trace = NULL; + obj_op.meta.delete_at = real_time(); + obj_op.meta.olh_epoch = olh_epoch; + + RGWObjManifest *pmanifest; + RGWObjManifest manifest; + + pmanifest = &manifest; + RGWObjTier tier_config; + tier_config.name = tier->get_storage_class(); + tier_config.tier_placement = rtier->get_rt(); + tier_config.is_multipart_upload = is_multipart_upload; + + pmanifest->set_tier_type("cloud-s3"); + pmanifest->set_tier_config(tier_config); + + /* check if its necessary */ + pmanifest->set_head(target_placement, head_obj->get_obj(), 0); + pmanifest->set_tail_placement(target_placement, head_obj->get_obj().bucket); + pmanifest->set_obj_size(0); + obj_op.meta.manifest = pmanifest; + + /* update storage class */ + bufferlist bl; + bl.append(tier->get_storage_class()); + attrs[RGW_ATTR_STORAGE_CLASS] = bl; + + attrs.erase(RGW_ATTR_ID_TAG); + attrs.erase(RGW_ATTR_TAIL_TAG); + + return obj_op.write_meta(dpp, 0, 0, attrs, y); +} + int RadosObject::get_max_chunk_size(const DoutPrefixProvider* dpp, rgw_placement_rule placement_rule, uint64_t* max_chunk_size, uint64_t* alignment) { return store->getRados()->get_max_chunk_size(placement_rule, get_obj(), max_chunk_size, dpp, alignment); @@ -2270,7 +2430,7 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, ret = -ERR_INVALID_PART; return ret; } else { - manifest.append(dpp, obj_part.manifest, store->get_zone()); + manifest.append(dpp, obj_part.manifest, store->svc()->zone->get_zonegroup(), store->svc()->zone->get_zone_params()); } bool part_compressed = (obj_part.cs_info.compression_type != "none"); @@ -2662,29 +2822,84 @@ int RadosMultipartWriter::complete(size_t accounted_size, const std::string& eta if_match, if_nomatch, user_data, zones_trace, canceled, y); } -const RGWZoneGroup& RadosZone::get_zonegroup() +const std::string& RadosZoneGroup::get_endpoint() const { - return store->svc()->zone->get_zonegroup(); + if (!group.endpoints.empty()) { + return group.endpoints.front(); + } else { + // use zonegroup's master zone endpoints + auto z = group.zones.find(group.master_zone); + if (z != group.zones.end() && !z->second.endpoints.empty()) { + return z->second.endpoints.front(); + } + } + return empty; } -int RadosZone::get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) +bool RadosZoneGroup::placement_target_exists(std::string& target) const { - return store->svc()->zone->get_zonegroup(id, zonegroup); + return !!group.placement_targets.count(target); } -const RGWZoneParams& RadosZone::get_params() +int RadosZoneGroup::get_placement_target_names(std::set& names) const { - return store->svc()->zone->get_zone_params(); + for (const auto& target : group.placement_targets) { + names.emplace(target.second.name); + } + + return 0; } -const rgw_zone_id& RadosZone::get_id() +int RadosZoneGroup::get_placement_tier(const rgw_placement_rule& rule, + std::unique_ptr* tier) { - return store->svc()->zone->zone_id(); + std::map::const_iterator titer; + titer = group.placement_targets.find(rule.name); + if (titer == group.placement_targets.end()) { + return -ENOENT; + } + + const auto& target_rule = titer->second; + std::map::const_iterator ttier; + ttier = target_rule.tier_targets.find(rule.storage_class); + if (ttier == target_rule.tier_targets.end()) { + // not found + return -ENOENT; + } + + PlacementTier* t; + t = new RadosPlacementTier(store, ttier->second); + if (!t) + return -ENOMEM; + + tier->reset(t); + return 0; +} + +ZoneGroup& RadosZone::get_zonegroup() +{ + return group; } -const RGWRealm& RadosZone::get_realm() +int RadosZone::get_zonegroup(const std::string& id, std::unique_ptr* zonegroup) { - return store->svc()->zone->get_realm(); + ZoneGroup* zg; + RGWZoneGroup rzg; + int r = store->svc()->zone->get_zonegroup(id, rzg); + if (r < 0) + return r; + + zg = new RadosZoneGroup(store, rzg); + if (!zg) + return -ENOMEM; + + zonegroup->reset(zg); + return 0; +} + +const rgw_zone_id& RadosZone::get_id() +{ + return store->svc()->zone->zone_id(); } const std::string& RadosZone::get_name() const @@ -2712,6 +2927,26 @@ const std::string& RadosZone::get_current_period_id() return store->svc()->zone->get_current_period_id(); } +const RGWAccessKey& RadosZone::get_system_key() +{ + return store->svc()->zone->get_zone_params().system_key; +} + +const std::string& RadosZone::get_realm_name() +{ + return store->svc()->zone->get_realm().get_name(); +} + +const std::string& RadosZone::get_realm_id() +{ + return store->svc()->zone->get_realm().get_id(); +} + +RadosLuaScriptManager::RadosLuaScriptManager(RadosStore* _s) : store(_s) +{ + pool = store->svc()->zone->get_zone_params().log_pool; +} + int RadosLuaScriptManager::get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) { auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); @@ -2764,13 +2999,13 @@ int RadosOIDCProvider::store_url(const DoutPrefixProvider *dpp, const std::strin bufferlist bl; using ceph::encode; encode(*this, bl); - return rgw_put_system_obj(dpp, obj_ctx, store->get_zone()->get_params().oidc_pool, oid, bl, exclusive, nullptr, real_time(), y); + return rgw_put_system_obj(dpp, obj_ctx, store->svc()->zone->get_zone_params().oidc_pool, oid, bl, exclusive, nullptr, real_time(), y); } int RadosOIDCProvider::read_url(const DoutPrefixProvider *dpp, const std::string& url, const std::string& tenant) { auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - auto& pool = store->get_zone()->get_params().oidc_pool; + auto& pool = store->svc()->zone->get_zone_params().oidc_pool; std::string oid = tenant + get_url_oid_prefix() + url; bufferlist bl; @@ -2794,7 +3029,7 @@ int RadosOIDCProvider::read_url(const DoutPrefixProvider *dpp, const std::string int RadosOIDCProvider::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) { - auto& pool = store->get_zone()->get_params().oidc_pool; + auto& pool = store->svc()->zone->get_zone_params().oidc_pool; std::string url, tenant; auto ret = get_tenant_url_from_arn(tenant, url); @@ -2834,10 +3069,10 @@ int RadosRole::store_info(const DoutPrefixProvider *dpp, bool exclusive, optiona encode(this->tags, bl_tags); map attrs; attrs.emplace("tagging", bl_tags); - return rgw_put_system_obj(dpp, obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y, &attrs); + return rgw_put_system_obj(dpp, obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y, &attrs); } - return rgw_put_system_obj(dpp, obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); + return rgw_put_system_obj(dpp, obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); } int RadosRole::store_name(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) @@ -2852,7 +3087,7 @@ int RadosRole::store_name(const DoutPrefixProvider *dpp, bool exclusive, optiona using ceph::encode; encode(nameToId, bl); - return rgw_put_system_obj(dpp, obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); + return rgw_put_system_obj(dpp, obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); } int RadosRole::store_path(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) @@ -2862,7 +3097,7 @@ int RadosRole::store_path(const DoutPrefixProvider *dpp, bool exclusive, optiona bufferlist bl; - return rgw_put_system_obj(dpp, obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); + return rgw_put_system_obj(dpp, obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, exclusive, nullptr, real_time(), y); } int RadosRole::read_id(const DoutPrefixProvider *dpp, const std::string& role_name, const std::string& tenant, std::string& role_id, optional_yield y) @@ -2871,7 +3106,7 @@ int RadosRole::read_id(const DoutPrefixProvider *dpp, const std::string& role_na std::string oid = tenant + get_names_oid_prefix() + role_name; bufferlist bl; - int ret = rgw_get_system_obj(obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp); + int ret = rgw_get_system_obj(obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp); if (ret < 0) { return ret; } @@ -2895,7 +3130,7 @@ int RadosRole::read_name(const DoutPrefixProvider *dpp, optional_yield y) std::string oid = tenant + get_names_oid_prefix() + name; bufferlist bl; - int ret = rgw_get_system_obj(obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp); + int ret = rgw_get_system_obj(obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: failed reading role name from Role pool: " << name << ": " << cpp_strerror(-ret) << dendl; @@ -2922,7 +3157,7 @@ int RadosRole::read_info(const DoutPrefixProvider *dpp, optional_yield y) bufferlist bl; map attrs; - int ret = rgw_get_system_obj(obj_ctx, store->get_zone()->get_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp, &attrs, nullptr, boost::none, true); + int ret = rgw_get_system_obj(obj_ctx, store->svc()->zone->get_zone_params().roles_pool, oid, bl, nullptr, nullptr, null_yield, dpp, &attrs, nullptr, boost::none, true); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: failed reading role info from Role pool: " << id << ": " << cpp_strerror(-ret) << dendl; return ret; @@ -2996,7 +3231,7 @@ int RadosRole::create(const DoutPrefixProvider *dpp, bool exclusive, optional_yi sprintf(buf + strlen(buf),".%dZ",(int)tv.tv_usec/1000); creation_date.assign(buf, strlen(buf)); - auto& pool = store->get_zone()->get_params().roles_pool; + auto& pool = store->svc()->zone->get_zone_params().roles_pool; ret = store_info(dpp, exclusive, y); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: storing role info in Role pool: " @@ -3044,7 +3279,7 @@ int RadosRole::create(const DoutPrefixProvider *dpp, bool exclusive, optional_yi int RadosRole::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) { - auto& pool = store->get_zone()->get_params().roles_pool; + auto& pool = store->svc()->zone->get_zone_params().roles_pool; int ret = read_name(dpp, y); if (ret < 0) { diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 8d61af2e7fc..a0aea9ce03e 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -37,23 +37,76 @@ class RadosCompletions : public Completions { virtual int drain() override; }; +class RadosPlacementTier: public PlacementTier { + RadosStore* store; + RGWZoneGroupPlacementTier tier; +public: + RadosPlacementTier(RadosStore* _store, const RGWZoneGroupPlacementTier& _tier) : store(_store), tier(_tier) {} + virtual ~RadosPlacementTier() = default; + + virtual const std::string& get_tier_type() { return tier.tier_type; } + virtual const std::string& get_storage_class() { return tier.storage_class; } + virtual bool retain_head_object() { return tier.retain_head_object; } + RGWZoneGroupPlacementTier& get_rt() { return tier; } +}; + +class RadosZoneGroup : public ZoneGroup { + RadosStore* store; + const RGWZoneGroup group; + std::string empty; +public: + RadosZoneGroup(RadosStore* _store, const RGWZoneGroup& _group) : store(_store), group(_group) {} + virtual ~RadosZoneGroup() = default; + + virtual const std::string& get_id() const override { return group.get_id(); }; + virtual const std::string& get_name() const override { return group.get_name(); }; + virtual int equals(const std::string& other_zonegroup) const override { + return group.equals(other_zonegroup); + }; + /** Get the endpoint from zonegroup, or from master zone if not set */ + virtual const std::string& get_endpoint() const override; + virtual bool placement_target_exists(std::string& target) const override; + virtual bool is_master_zonegroup() const override { + return group.is_master_zonegroup(); + }; + virtual const std::string& get_api_name() const override { return group.api_name; }; + virtual int get_placement_target_names(std::set& names) const override; + virtual const std::string& get_default_placement_name() const override { + return group.default_placement.name; }; + virtual int get_hostnames(std::list& names) const override { + names = group.hostnames; + return 0; + }; + virtual int get_s3website_hostnames(std::list& names) const override { + names = group.hostnames_s3website; + return 0; + }; + virtual int get_zone_count() const override { + return group.zones.size(); + } + virtual int get_placement_tier(const rgw_placement_rule& rule, std::unique_ptr* tier); + const RGWZoneGroup& get_group() { return group; } +}; + class RadosZone : public Zone { protected: RadosStore* store; + RadosZoneGroup group; public: - RadosZone(RadosStore* _store) : store(_store) {} + RadosZone(RadosStore* _store, RadosZoneGroup _zg) : store(_store), group(_zg) {} ~RadosZone() = default; - virtual const RGWZoneGroup& get_zonegroup() override; - virtual int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) override; - virtual const RGWZoneParams& get_params() override; + virtual ZoneGroup& get_zonegroup() override; + virtual int get_zonegroup(const std::string& id, std::unique_ptr* zonegroup) override; virtual const rgw_zone_id& get_id() override; - virtual const RGWRealm& get_realm() override; virtual const std::string& get_name() const override; virtual bool is_writeable() override; virtual bool get_redirect_endpoint(std::string* endpoint) override; virtual bool has_zonegroup_api(const std::string& api) const override; virtual const std::string& get_current_period_id() override; + virtual const RGWAccessKey& get_system_key() override; + virtual const std::string& get_realm_name() override; + virtual const std::string& get_realm_id() override; }; class RadosStore : public Store { @@ -61,17 +114,18 @@ class RadosStore : public Store { RGWRados* rados; RGWUserCtl* user_ctl; std::string luarocks_path; - RadosZone zone; + std::unique_ptr zone; public: RadosStore() - : rados(nullptr), zone(this) { + : rados(nullptr) { } ~RadosStore() { delete rados; } - virtual const char* get_name() const override { + virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override; + virtual const std::string get_name() const override { return "rados"; } virtual std::string get_cluster_id(const DoutPrefixProvider* dpp, optional_yield y) override; @@ -87,7 +141,7 @@ class RadosStore : public Store { virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, bufferlist& in_data, JSONParser* jp, req_info& info, optional_yield y) override; - virtual Zone* get_zone() { return &zone; } + virtual Zone* get_zone() { return zone.get(); } virtual std::string zone_unique_id(uint64_t unique_num) override; virtual std::string zone_unique_trans_id(const uint64_t unique_num) override; virtual int cluster_stat(RGWClusterStat& stats) override; @@ -163,6 +217,8 @@ class RadosStore : public Store { const rgw_placement_rule *ptail_placement_rule, uint64_t olh_epoch, const std::string& unique_tag) override; + virtual const std::string& get_compression_type(const rgw_placement_rule& rule) override; + virtual bool valid_placement(const rgw_placement_rule& rule) override; virtual void finalize(void) override; @@ -364,6 +420,14 @@ class RadosObject : public Object { uint64_t olh_epoch, const DoutPrefixProvider* dpp, optional_yield y) override; + virtual int transition_to_cloud(Bucket* bucket, + rgw::sal::PlacementTier* tier, + rgw_bucket_dir_entry& o, + std::set& cloud_targets, + CephContext* cct, + bool update_object, + const DoutPrefixProvider* dpp, + optional_yield y) override; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f) override; @@ -396,6 +460,13 @@ class RadosObject : public Object { uint64_t* alignment = nullptr); void get_max_aligned_size(uint64_t size, uint64_t alignment, uint64_t* max_size); void raw_obj_to_obj(const rgw_raw_obj& raw_obj); + int write_cloud_tier(const DoutPrefixProvider* dpp, + optional_yield y, + uint64_t olh_epoch, + rgw::sal::PlacementTier* tier, + bool is_multipart_upload, + rgw_placement_rule& target_placement, + Object* head_obj); RGWObjManifest* get_manifest() { return manifest; } RGWObjectCtx& get_ctx() { return *rados_ctx; } @@ -795,10 +866,7 @@ class RadosLuaScriptManager : public LuaScriptManager { rgw_pool pool; public: - RadosLuaScriptManager(RadosStore* _s) : store(_s) - { - pool = store->get_zone()->get_params().log_pool; - } + RadosLuaScriptManager(RadosStore* _s); virtual ~RadosLuaScriptManager() = default; virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) override; diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/rgw_sync_module_aws.cc index 3ea5b4d8a63..c7090e937f9 100644 --- a/src/rgw/rgw_sync_module_aws.cc +++ b/src/rgw/rgw_sync_module_aws.cc @@ -655,10 +655,10 @@ struct AWSSyncConfig { auto& root_conf = root_profile->conn_conf; root_profile->conn.reset(new S3RESTConn(sc->cct, - sync_env->svc->zone, id, { root_conf->endpoint }, root_conf->key, + sync_env->svc->zone->get_zonegroup().get_id(), root_conf->region, root_conf->host_style)); @@ -666,10 +666,10 @@ struct AWSSyncConfig { auto& c = i.second; c->conn.reset(new S3RESTConn(sc->cct, - sync_env->svc->zone, id, { c->conn_conf->endpoint }, c->conn_conf->key, + sync_env->svc->zone->get_zonegroup().get_id(), c->conn_conf->region, c->conn_conf->host_style)); } diff --git a/src/rgw/rgw_sync_module_es.cc b/src/rgw/rgw_sync_module_es.cc index 7f65637b3e9..bba54660894 100644 --- a/src/rgw/rgw_sync_module_es.cc +++ b/src/rgw/rgw_sync_module_es.cc @@ -178,7 +178,7 @@ struct ElasticConfig { void init(CephContext *cct, const JSONFormattable& config) { string elastic_endpoint = config["endpoint"]; id = string("elastic:") + elastic_endpoint; - conn.reset(new RGWRESTConn(cct, (RGWSI_Zone*)nullptr, id, { elastic_endpoint }, nullopt /* region */ )); + conn.reset(new RGWRESTConn(cct, (rgw::sal::Store*)nullptr, id, { elastic_endpoint }, nullopt /* region */ )); explicit_custom_meta = config["explicit_custom_meta"](true); index_buckets.init(config["index_buckets_list"], true); /* approve all buckets by default */ allow_owners.init(config["approved_owners_list"], true); /* approve all bucket owners by default */ diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 0f3345a3f28..a6ada43fd38 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -447,7 +447,7 @@ int RGWDataAccess::Object::put(bufferlist& data, CompressorRef plugin; boost::optional compressor; - const auto& compression_type = store->get_zone()->get_params().get_compression_type(bucket_info.placement_rule); + const auto& compression_type = store->get_compression_type(bucket_info.placement_rule); if (compression_type != "none") { plugin = Compressor::create(store->ctx(), compression_type); if (!plugin) { diff --git a/src/rgw/rgw_tools.h b/src/rgw/rgw_tools.h index d2ae2548972..28ff91835ee 100644 --- a/src/rgw/rgw_tools.h +++ b/src/rgw/rgw_tools.h @@ -60,6 +60,10 @@ inline int rgw_shard_id(const std::string& key, int max_shards) max_shards); } +void rgw_shard_name(const std::string& prefix, unsigned max_shards, const std::string& key, std::string& name, int *shard_id); +void rgw_shard_name(const std::string& prefix, unsigned max_shards, const std::string& section, const std::string& key, std::string& name); +void rgw_shard_name(const std::string& prefix, unsigned shard_id, std::string& name); + int rgw_put_system_obj(const DoutPrefixProvider *dpp, RGWSysObjectCtx& obj_ctx, const rgw_pool& pool, const std::string& oid, bufferlist& data, bool exclusive, RGWObjVersionTracker *objv_tracker, real_time set_mtime, optional_yield y, std::map *pattrs = NULL); int rgw_get_system_obj(RGWSysObjectCtx& obj_ctx, const rgw_pool& pool, const std::string& key, bufferlist& bl, diff --git a/src/rgw/rgw_trim_bilog.cc b/src/rgw/rgw_trim_bilog.cc index 1756d7f7a16..05fbe4bbbfb 100644 --- a/src/rgw/rgw_trim_bilog.cc +++ b/src/rgw/rgw_trim_bilog.cc @@ -1085,7 +1085,7 @@ class BucketTrimManager::Impl : public TrimCounters::Server, Impl(rgw::sal::RadosStore* store, const BucketTrimConfig& config) : store(store), config(config), - status_obj(store->get_zone()->get_params().log_pool, BucketTrimStatus::oid), + status_obj(store->svc()->zone->get_zone_params().log_pool, BucketTrimStatus::oid), counter(config.counter_size), trimmed(config.recent_size, config.recent_duration), watcher(store, status_obj, this) diff --git a/src/rgw/rgw_trim_mdlog.cc b/src/rgw/rgw_trim_mdlog.cc index 4ddde03363d..852d0221a75 100644 --- a/src/rgw/rgw_trim_mdlog.cc +++ b/src/rgw/rgw_trim_mdlog.cc @@ -144,7 +144,7 @@ connection_map make_peer_connections(rgw::sal::RadosStore* store, for (auto& g : zonegroups) { for (auto& z : g.second.zones) { std::unique_ptr conn{ - new RGWRESTConn(store->ctx(), store->svc()->zone, z.first.id, z.second.endpoints, g.second.api_name)}; + new RGWRESTConn(store->ctx(), store, z.first.id, z.second.endpoints, g.second.api_name)}; connections.emplace(z.first.id, std::move(conn)); } } diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index bea9726f342..5280371ad23 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -358,7 +358,7 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp) continue; } ldpp_dout(dpp, 20) << "generating connection object for zone " << z.name << " id " << z.id << dendl; - RGWRESTConn *conn = new RGWRESTConn(cct, this, z.id, z.endpoints, zonegroup->api_name); + RGWRESTConn *conn = new RGWRESTConn(cct, z.id, z.endpoints, zone_params->system_key, zonegroup->get_id(), zonegroup->api_name); zone_conn_map[id] = conn; bool zone_is_source = source_zones.find(z.id) != source_zones.end(); @@ -805,10 +805,10 @@ int RGWSI_Zone::init_zg_from_period(const DoutPrefixProvider *dpp, optional_yiel } } const auto& endpoints = master->second.endpoints; - add_new_connection_to_map(zonegroup_conn_map, zg, new RGWRESTConn(cct, this, zg.get_id(), endpoints, zg.api_name)); + add_new_connection_to_map(zonegroup_conn_map, zg, new RGWRESTConn(cct, zg.get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zg.api_name)); if (!current_period->get_master_zonegroup().empty() && zg.get_id() == current_period->get_master_zonegroup()) { - rest_master_conn = new RGWRESTConn(cct, this, zg.get_id(), endpoints, zg.api_name); + rest_master_conn = new RGWRESTConn(cct, zg.get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zg.api_name); } } @@ -872,7 +872,7 @@ int RGWSI_Zone::init_zg_from_local(const DoutPrefixProvider *dpp, optional_yield } } const auto& endpoints = master->second.endpoints; - rest_master_conn = new RGWRESTConn(cct, this, zonegroup->get_id(), endpoints, zonegroup->api_name); + rest_master_conn = new RGWRESTConn(cct, zonegroup->get_id(), endpoints, zone_params->system_key, zonegroup->get_id(), zonegroup->api_name); } return 0; -- 2.47.3