From 728f13d8c622628fdbae9d858228c766c8b5bacb Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Mon, 11 Jan 2021 03:02:17 +0530 Subject: [PATCH] rgw/CloudTransition: handle versioned objects If the object is versioned, to avoid objects getting overwritten post transition to cloud, append object versionID to the target object name Signed-off-by: Soumya Koduri --- doc/radosgw/cloud-transition.rst | 16 ++- src/rgw/rgw_acl.cc | 30 +++++ src/rgw/rgw_acl.h | 3 + src/rgw/rgw_admin.cc | 203 ++++++++++++++++--------------- src/rgw/rgw_json_enc.cc | 6 +- src/rgw/rgw_lc.cc | 82 +++++++------ src/rgw/rgw_lc_tier.cc | 34 ++++-- src/rgw/rgw_lc_tier.h | 10 +- src/rgw/rgw_obj_manifest.h | 4 +- src/rgw/rgw_op.cc | 1 + src/rgw/rgw_zone.h | 36 ++---- 11 files changed, 231 insertions(+), 194 deletions(-) diff --git a/doc/radosgw/cloud-transition.rst b/doc/radosgw/cloud-transition.rst index 9b756b8ee8c29..64453aac36ec9 100644 --- a/doc/radosgw/cloud-transition.rst +++ b/doc/radosgw/cloud-transition.rst @@ -116,7 +116,7 @@ How to Configure See `Adding a Storage Class `__ for how to configure storage-class for a zonegroup. The cloud transition requires a creation of a special storage class with tier type defined as ``cloud`` -Note: Once a storage class is created of ``-tier-type=cloud``, it cannot be later modified to any other storage class type. +Note: Once a storage class is created of ``--tier-type=cloud``, it cannot be later modified to any other storage class type. :: @@ -277,6 +277,13 @@ The cloud storage class once configured can then be used like any other storage Since the transition is unidirectional, while configuring S3 lifecycle rules, the cloud storage class should be specified last among all the storage classes the object transitions to. Subsequent rules (if any) do not apply post transition to the cloud. +To avoid object names collision across various buckets, source bucket name is prepended to the target object name. If the object is versioned, object versionid is appended to the end. + +Below is the sample object name format: +:: + + s3:////(:) + Due to API limitations there is no way to preserve original object modification time and ETag but they get stored as metadata attributes on the destination objects, as shown below: :: @@ -287,7 +294,6 @@ Due to API limitations there is no way to preserve original object modification x-amz-meta-rgwx-source-mtime: 1608546349.757100363 x-amz-meta-rgwx-versioned-epoch: 0 - By default, post transition, the source object gets deleted. But it is possible to retain its metadata but with updated values (like storage-class and object-size) by setting config option 'retain_object' to true. However GET on those objects shall still fail with 'InvalidObjectState' error. For example, @@ -311,13 +317,11 @@ For example, ERROR: S3 error: 403 (InvalidObjectState) -Since all the objects are transitioned to a single target bucket, bucket versioning is not enabled on the remote endpoint. That means multiple versions of the same object (if any) shall get overwritten based on the order of their transition. - Future Work ----------- +* Send presigned redirect or read-through the objects transitioned to cloud + * Support s3:RestoreObject operation on cloud transitioned objects. * Federation between RGW and Cloud services. - -* Support Object versioning for the transitioned objects diff --git a/src/rgw/rgw_acl.cc b/src/rgw/rgw_acl.cc index 413b88a9b76d9..86a7a96fd70a7 100644 --- a/src/rgw/rgw_acl.cc +++ b/src/rgw/rgw_acl.cc @@ -76,6 +76,36 @@ bool operator!=(const RGWAccessControlPolicy& lhs, return !(lhs == rhs); } +string get_acl_type_str (const ACLGranteeTypeEnum& type) { + string s; + switch (type) { + case ACL_TYPE_EMAIL_USER: + s = "email"; + break; + case ACL_TYPE_GROUP: + s = "uri"; + break; + default: + s = "id"; + break; + } + + return s; +} + +ACLGranteeTypeEnum get_acl_type (const string& t) { + ACLGranteeTypeEnum type; + if (t == "email") { + type = ACL_TYPE_EMAIL_USER; + } else if (t == "uri") { + type = ACL_TYPE_GROUP; + } else { + type = ACL_TYPE_CANON_USER; + } + + return type; +} + void RGWAccessControlList::_add_grant(ACLGrant *grant) { ACLPermission& perm = grant->get_permission(); diff --git a/src/rgw/rgw_acl.h b/src/rgw/rgw_acl.h index 6fc16c7800326..b5d9676906bb1 100644 --- a/src/rgw/rgw_acl.h +++ b/src/rgw/rgw_acl.h @@ -46,6 +46,9 @@ enum ACLGroupTypeEnum { ACL_GROUP_AUTHENTICATED_USERS = 2, }; +string get_acl_type_str (const ACLGranteeTypeEnum& type); +ACLGranteeTypeEnum get_acl_type (const string& t); + class ACLPermission { protected: diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index b8c7675247caf..ddf9b10d7e679 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -5009,21 +5009,21 @@ int main(int argc, const char **argv) case OPT::ZONEGROUP_PLACEMENT_RM: case OPT::ZONEGROUP_PLACEMENT_DEFAULT: { - if (placement_id.empty()) { - cerr << "ERROR: --placement-id not specified" << std::endl; - return EINVAL; - } + if (placement_id.empty()) { + cerr << "ERROR: --placement-id not specified" << std::endl; + return EINVAL; + } - rgw_placement_rule rule; - rule.from_str(placement_id); + rgw_placement_rule rule; + rule.from_str(placement_id); - if (!rule.storage_class.empty() && opt_storage_class && - rule.storage_class != *opt_storage_class) { - cerr << "ERROR: provided contradicting storage class configuration" << std::endl; - return EINVAL; - } else if (rule.storage_class.empty()) { - rule.storage_class = opt_storage_class.value_or(string()); - } + if (!rule.storage_class.empty() && opt_storage_class && + rule.storage_class != *opt_storage_class) { + cerr << "ERROR: provided contradicting storage class configuration" << std::endl; + return EINVAL; + } else if (rule.storage_class.empty()) { + rule.storage_class = opt_storage_class.value_or(string()); + } RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name); int ret = zonegroup.init(dpp(), g_ceph_context, static_cast(store)->svc()->sysobj, null_yield); @@ -5032,108 +5032,113 @@ int main(int argc, const char **argv) return -ret; } - if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_ADD || - opt_cmd == OPT::ZONEGROUP_PLACEMENT_MODIFY) { - RGWZoneGroupPlacementTarget& target = zonegroup.placement_targets[placement_id]; - if (!tags.empty()) { - target.tags.clear(); - for (auto& t : tags) { - target.tags.insert(t); - } - } - target.name = placement_id; - for (auto& t : tags_rm) { - target.tags.erase(t); - } - for (auto& t : tags_add) { - target.tags.insert(t); - } - target.storage_classes.insert(rule.get_storage_class()); + if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_ADD || + opt_cmd == OPT::ZONEGROUP_PLACEMENT_MODIFY) { + RGWZoneGroupPlacementTarget& target = zonegroup.placement_targets[placement_id]; + if (!tags.empty()) { + target.tags.clear(); + for (auto& t : tags) { + target.tags.insert(t); + } + } - /* Tier options */ - bool tier_class = false; - std::string storage_class = rule.get_storage_class(); - RGWZoneGroupPlacementTier t{storage_class}; - RGWZoneGroupPlacementTier *pt = &t; + target.name = placement_id; + for (auto& t : tags_rm) { + target.tags.erase(t); + } + for (auto& t : tags_add) { + target.tags.insert(t); + } + target.storage_classes.insert(rule.get_storage_class()); + + /* Tier options */ + bool tier_class = false; + std::string storage_class = rule.get_storage_class(); + RGWZoneGroupPlacementTier t{storage_class}; + RGWZoneGroupPlacementTier *pt = &t; auto ptiter = target.tier_targets.find(storage_class); if (ptiter != target.tier_targets.end()) { - pt = &ptiter->second; - tier_class = true; - } else if (tier_type_specified && tier_type == "cloud") { - /* we support only cloud tier-type for now. - * Once set cant be reset. */ - tier_class = true; - pt->tier_type = tier_type; - } + pt = &ptiter->second; + tier_class = true; + } else if (tier_type_specified) { + if (tier_type == "cloud") { + /* we support only cloud tier-type for now. + * Once set cant be reset. */ + tier_class = true; + pt->tier_type = tier_type; + } else { + cerr << "ERROR: Invalid tier-type specified" << std::endl; + return EINVAL; + } + } - if (tier_class) { - if (tier_config_add.size() > 0) { - JSONFormattable tconfig; - for (auto add : tier_config_add) { - int r = tconfig.set(add.first, add.second); - if (r < 0) { - cerr << "ERROR: failed to set configurable: " << add << std::endl; - return EINVAL; - } - } - int r = pt->update_params(tconfig); - if (r < 0) { - cerr << "ERROR: failed to update tier_config options"<< std::endl; - } + if (tier_class) { + if (tier_config_add.size() > 0) { + JSONFormattable tconfig; + for (auto add : tier_config_add) { + int r = tconfig.set(add.first, add.second); + if (r < 0) { + cerr << "ERROR: failed to set configurable: " << add << std::endl; + return EINVAL; } - if (tier_config_rm.size() > 0) { - JSONFormattable tconfig; - for (auto add : tier_config_rm) { - int r = tconfig.set(add.first, add.second); - if (r < 0) { - cerr << "ERROR: failed to set configurable: " << add << std::endl; - return EINVAL; - } - } - int r = pt->clear_params(tconfig); - if (r < 0) { - cerr << "ERROR: failed to update tier_config options"<< std::endl; - } + } + int r = pt->update_params(tconfig); + if (r < 0) { + cerr << "ERROR: failed to update tier_config options"<< std::endl; + } + } + if (tier_config_rm.size() > 0) { + JSONFormattable tconfig; + for (auto add : tier_config_rm) { + int r = tconfig.set(add.first, add.second); + if (r < 0) { + cerr << "ERROR: failed to set configurable: " << add << std::endl; + return EINVAL; } - - target.tier_targets.emplace(std::make_pair(storage_class, *pt)); } + int r = pt->clear_params(tconfig); + if (r < 0) { + cerr << "ERROR: failed to update tier_config options"<< std::endl; + } + } - } else if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_RM) { - if (!opt_storage_class || - opt_storage_class->empty()) { - zonegroup.placement_targets.erase(placement_id); - } else { - auto iter = zonegroup.placement_targets.find(placement_id); - if (iter != zonegroup.placement_targets.end()) { - RGWZoneGroupPlacementTarget& info = zonegroup.placement_targets[placement_id]; - info.storage_classes.erase(*opt_storage_class); + target.tier_targets.emplace(std::make_pair(storage_class, *pt)); + } + + } else if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_RM) { + if (!opt_storage_class || opt_storage_class->empty()) { + zonegroup.placement_targets.erase(placement_id); + } else { + auto iter = zonegroup.placement_targets.find(placement_id); + if (iter != zonegroup.placement_targets.end()) { + RGWZoneGroupPlacementTarget& info = zonegroup.placement_targets[placement_id]; + info.storage_classes.erase(*opt_storage_class); auto ptiter = info.tier_targets.find(*opt_storage_class); if (ptiter != info.tier_targets.end()) { - info.tier_targets.erase(ptiter); + info.tier_targets.erase(ptiter); } - } - } - } else if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_DEFAULT) { - if (!zonegroup.placement_targets.count(placement_id)) { - cerr << "failed to find a zonegroup placement target named '" - << placement_id << "'" << std::endl; - return -ENOENT; - } - zonegroup.default_placement = rule; } + } + } else if (opt_cmd == OPT::ZONEGROUP_PLACEMENT_DEFAULT) { + if (!zonegroup.placement_targets.count(placement_id)) { + cerr << "failed to find a zonegroup placement target named '" + << placement_id << "'" << std::endl; + return -ENOENT; + } + zonegroup.default_placement = rule; + } - zonegroup.post_process_params(dpp(), null_yield); - ret = zonegroup.update(dpp(), null_yield); - if (ret < 0) { - cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl; - return -ret; - } + zonegroup.post_process_params(dpp(), null_yield); + ret = zonegroup.update(dpp(), null_yield); + if (ret < 0) { + cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl; + return -ret; + } - encode_json("placement_targets", zonegroup.placement_targets, formatter.get()); - formatter->flush(cout); + encode_json("placement_targets", zonegroup.placement_targets, formatter.get()); + formatter->flush(cout); } break; case OPT::ZONE_CREATE: diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index a2397bc35473c..913ad956d07f8 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -101,9 +101,9 @@ void RGWObjManifestRule::dump(Formatter *f) const void RGWObjTier::dump(Formatter *f) const { - f->dump_string("name", name); - f->dump_object("tier_placement", tier_placement); - f->dump_bool("is_multipart_upload", is_multipart_upload); + encode_json("name", name, f); + encode_json("tier_placement", tier_placement, f); + encode_json("is_multipart_upload", is_multipart_upload, f); } void rgw_bucket_placement::dump(Formatter *f) const diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 7e837b705a153..e3930c0361a32 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -1251,36 +1251,34 @@ public: return need_to_process; } - /* find out if the the storage class is remote cloud */ - int get_tier_target(const RGWZoneGroup &zonegroup, rgw_placement_rule& rule, - string& storage_class, RGWZoneGroupPlacementTier &tier) { - std::map::const_iterator titer; - titer = zonegroup.placement_targets.find(rule.name); - if (titer == zonegroup.placement_targets.end()) { - return -1; - } - - if (storage_class.empty()) { - storage_class = rule.storage_class; - } - - const auto& target_rule = titer->second; - std::map::const_iterator ttier; - ttier = target_rule.tier_targets.find(storage_class); - if (ttier != target_rule.tier_targets.end()) { - tier = ttier->second; - } - - return 0; - } + /* find out if the the storage class is remote cloud */ + int get_tier_target(const RGWZoneGroup &zonegroup, rgw_placement_rule& rule, + string& storage_class, RGWZoneGroupPlacementTier &tier) { + std::map::const_iterator titer; + titer = zonegroup.placement_targets.find(rule.name); + if (titer == zonegroup.placement_targets.end()) { + return -ENOENT; + } + + if (storage_class.empty()) { + storage_class = rule.storage_class; + } + + const auto& target_rule = titer->second; + std::map::const_iterator ttier; + ttier = target_rule.tier_targets.find(storage_class); + if (ttier != target_rule.tier_targets.end()) { + tier = ttier->second; + } + return 0; + } int delete_tier_obj(lc_op_ctx& oc, RGWLCCloudTierCtx& tier_ctx) { - int ret = -1; + int ret = 0; - /* XXX: do we need to check for retention/versioning attributes - * as done in RGWDeleteObj? + /* XXX: do we need to check for retention attributes * as done in RGWDeleteObj? */ - ret = oc.store->getRados()->delete_obj(oc.rctx, oc.bucket->get_info(), oc.obj->get_obj(), tier_ctx.bucket_info.versioning_status()); + ret = oc.store->getRados()->delete_obj(oc.dpp, oc.rctx, oc.bucket->get_info(), oc.obj->get_obj(), tier_ctx.bucket_info.versioning_status()); return ret; } @@ -1298,7 +1296,7 @@ public: read_op.params.attrs = &attrs; read_op.params.lastmod = &read_mtime; - int r = read_op.prepare(null_yield); + int r = read_op.prepare(null_yield, oc.dpp); if (r < 0) { return r; } @@ -1365,7 +1363,7 @@ public: attrs.erase(RGW_ATTR_ID_TAG); attrs.erase(RGW_ATTR_TAIL_TAG); - obj_op.write_meta(tier_ctx.o.meta.size, 0, attrs, null_yield); + obj_op.write_meta(oc.dpp, tier_ctx.o.meta.size, 0, attrs, null_yield); if (r < 0) { return r; } @@ -1403,15 +1401,20 @@ public: return ret; } - RGWLCCloudTierCtx tier_ctx(oc.cct, oc.o, oc.store, oc.bucket->get_info(), - oc.obj->get_obj(), oc.rctx, conn, bucket_name, oc.tier.target_storage_class, - &http_manager); + RGWLCCloudTierCtx tier_ctx(oc.cct, oc.dpp, oc.o, oc.store, oc.bucket->get_info(), + oc.obj->get_obj(), oc.rctx, conn, bucket_name, + oc.tier.target_storage_class, &http_manager); tier_ctx.acl_mappings = oc.tier.acl_mappings; tier_ctx.multipart_min_part_size = oc.tier.multipart_min_part_size; tier_ctx.multipart_sync_threshold = oc.tier.multipart_sync_threshold; tier_ctx.storage_class = oc.tier.storage_class; bool al_tiered = false; + + /* Since multiple zones may try to transition the same object to the cloud, + * verify if the object is already transitioned. And since its just a best + * effort, do not bail out in case of any errors. + */ ret = crs.run(new RGWLCCloudCheckCR(tier_ctx, &al_tiered)); if (ret < 0) { @@ -1419,10 +1422,10 @@ public: } if (!al_tiered) { - ldout(tier_ctx.cct, 20) << "is_already_tiered false" << dendl; + ldout(tier_ctx.cct, 20) << "is already tiered false" << dendl; ret = crs.run(new RGWLCCloudTierCR(tier_ctx)); } else { - ldout(tier_ctx.cct, 20) << "is_already_tiered true" << dendl; + ldout(tier_ctx.cct, 20) << "is already tiered true" << dendl; } http_manager.stop(); @@ -1458,16 +1461,15 @@ public: target_placement.inherit_from(oc.bucket->get_placement_rule()); target_placement.storage_class = transition.storage_class; - ldpp_dout(oc.dpp, 0) << "XXXXXXXXXXX ERROR: in lifecycle::process" << dendl; r = get_tier_target(zonegroup, target_placement, target_placement.storage_class, oc.tier); if (!r && oc.tier.tier_type == "cloud") { - ldpp_dout(oc.dpp, 0) << "Found cloud tier: " << target_placement.storage_class << dendl; - r = transition_obj_to_cloud(oc); - if (r < 0) { - ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj to cloud (r=" << r << ")" - << dendl; - } + ldpp_dout(oc.dpp, 20) << "Found cloud tier: " << target_placement.storage_class << dendl; + r = transition_obj_to_cloud(oc); + if (r < 0) { + ldpp_dout(oc.dpp, 0) << "ERROR: failed to transition obj to cloud (r=" << r << ")" + << dendl; + } } else { if (!oc.store->get_zone()->get_params(). valid_placement(target_placement)) { diff --git a/src/rgw/rgw_lc_tier.cc b/src/rgw/rgw_lc_tier.cc index 32a846e8b5f7a..b6844f8c6e3a4 100644 --- a/src/rgw/rgw_lc_tier.cc +++ b/src/rgw/rgw_lc_tier.cc @@ -26,6 +26,15 @@ using namespace std; +static string get_key_instance(const rgw_obj_key& key) +{ + if (!key.instance.empty() && + !key.have_null_instance()) { + return "-" + key.instance; + } + return ""; +} + static string get_key_oid(const rgw_obj_key& key) { string oid = key.name; @@ -178,6 +187,7 @@ class RGWLCStreamGetCRF : public RGWStreamReadHTTPResourceCRF class RGWLCStreamReadCRF : public RGWStreamReadCRF { CephContext *cct; + const DoutPrefixProvider *dpp; map attrs; uint64_t obj_size; rgw_obj& obj; @@ -189,10 +199,11 @@ class RGWLCStreamReadCRF : public RGWStreamReadCRF off_t m_part_end; public: - RGWLCStreamReadCRF(CephContext *_cct, RGWRados* rados, RGWBucketInfo& bucket_info, + RGWLCStreamReadCRF(CephContext *_cct, const DoutPrefixProvider *_dpp, + RGWRados* rados, RGWBucketInfo& bucket_info, RGWObjectCtx& obj_ctx, rgw_obj& _obj, const real_time &_mtime) : RGWStreamReadCRF(rados, bucket_info, obj_ctx, _obj), cct(_cct), - obj(_obj), mtime(_mtime) {} + dpp(_dpp), obj(_obj), mtime(_mtime) {} ~RGWLCStreamReadCRF() {}; @@ -211,7 +222,7 @@ class RGWLCStreamReadCRF : public RGWStreamReadCRF read_op.params.lastmod = &read_mtime; read_op.params.obj_size = &obj_size; - int ret = read_op.prepare(y); + int ret = read_op.prepare(y, dpp); if (ret < 0) { ldout(cct, 0) << "ERROR: fail to prepare read_op, ret = " << ret << dendl; return ret; @@ -279,7 +290,7 @@ class RGWLCStreamReadCRF : public RGWStreamReadCRF int read(off_t ofs, off_t end, bufferlist &bl) { optional_yield y = null_yield; - return read_op.read(ofs, end, bl, y); + return read_op.read(ofs, end, bl, y, dpp); } }; @@ -524,7 +535,7 @@ class RGWLCStreamObjToCloudPlainCR : public RGWCoroutine { target_bucket.name = tier_ctx.target_bucket_name; target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + - tier_ctx.obj.key.name; + tier_ctx.obj.key.name + get_key_instance(tier_ctx.obj.key); dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); @@ -537,7 +548,8 @@ class RGWLCStreamObjToCloudPlainCR : public RGWCoroutine { // tier_ctx.obj.set_atomic(&tier_ctx.rctx); -- might need when updated to zipper SAL /* Prepare Read from source */ - in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.store->getRados(), tier_ctx.bucket_info, + in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.dpp, + tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); out_crf.reset(new RGWLCStreamPutCRF((CephContext *)(tier_ctx.cct), get_env(), this, @@ -586,7 +598,7 @@ class RGWLCStreamObjToCloudMultipartPartCR : public RGWCoroutine { target_bucket.name = tier_ctx.target_bucket_name; target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + - tier_ctx.obj.key.name; + tier_ctx.obj.key.name + get_key_instance(tier_ctx.obj.key); dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); @@ -597,7 +609,7 @@ class RGWLCStreamObjToCloudMultipartPartCR : public RGWCoroutine { // tier_ctx.obj.set_atomic(&tier_ctx.rctx); -- might need when updated to zipper SAL /* Prepare Read from source */ - in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.store->getRados(), + in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.dpp, tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); end = part_info.ofs + part_info.size - 1; @@ -931,7 +943,7 @@ class RGWLCStreamObjToCloudMultipartCR : public RGWCoroutine { string target_obj_name; target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + - tier_ctx.obj.key.name; + tier_ctx.obj.key.name + get_key_instance(tier_ctx.obj.key); rgw_obj dest_obj(target_bucket, target_obj_name); std::shared_ptr in_crf; rgw_rest_obj rest_obj; @@ -958,7 +970,7 @@ class RGWLCStreamObjToCloudMultipartCR : public RGWCoroutine { } if (retcode == -ENOENT) { - in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); + in_crf.reset(new RGWLCStreamReadCRF(tier_ctx.cct, tier_ctx.dpp, tier_ctx.store->getRados(), tier_ctx.bucket_info, tier_ctx.rctx, tier_ctx.obj, tier_ctx.o.meta.mtime)); in_crf->init(); @@ -1051,7 +1063,7 @@ int RGWLCCloudCheckCR::operate() { target_bucket.name = tier_ctx.target_bucket_name; target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + - tier_ctx.obj.key.name; + tier_ctx.obj.key.name + get_key_instance(tier_ctx.obj.key); std::shared_ptr dest_bucket; dest_bucket.reset(new rgw::sal::RGWRadosBucket(tier_ctx.store, target_bucket)); diff --git a/src/rgw/rgw_lc_tier.h b/src/rgw/rgw_lc_tier.h index 0e6477f9512ab..6f0fb331c09a6 100644 --- a/src/rgw/rgw_lc_tier.h +++ b/src/rgw/rgw_lc_tier.h @@ -17,6 +17,7 @@ struct RGWLCCloudTierCtx { CephContext *cct; + const DoutPrefixProvider *dpp; /* Source */ rgw_bucket_dir_entry& o; @@ -39,11 +40,12 @@ struct RGWLCCloudTierCtx { bool is_multipart_upload{false}; - RGWLCCloudTierCtx(CephContext* _cct, rgw_bucket_dir_entry& _o, - rgw::sal::RGWRadosStore* _store, RGWBucketInfo &_binfo, rgw_obj _obj, - RGWObjectCtx& _rctx, std::shared_ptr _conn, string _bucket, + RGWLCCloudTierCtx(CephContext* _cct, const DoutPrefixProvider *_dpp, + rgw_bucket_dir_entry& _o, rgw::sal::RGWRadosStore* _store, + RGWBucketInfo &_binfo, rgw_obj _obj, RGWObjectCtx& _rctx, + std::shared_ptr _conn, string _bucket, string _storage_class, RGWHTTPManager *_http) - : cct(_cct), o(_o), store(_store), bucket_info(_binfo), + : cct(_cct), dpp(_dpp), o(_o), store(_store), bucket_info(_binfo), obj(_obj), rctx(_rctx), conn(_conn), target_bucket_name(_bucket), target_storage_class(_storage_class), http_manager(_http) {} }; diff --git a/src/rgw/rgw_obj_manifest.h b/src/rgw/rgw_obj_manifest.h index b5a31b5f7be9b..c54fd36f367ba 100644 --- a/src/rgw/rgw_obj_manifest.h +++ b/src/rgw/rgw_obj_manifest.h @@ -449,7 +449,7 @@ public: return max_head_size; } - string get_tier_type() { + const string get_tier_type() { return tier_type; } @@ -472,7 +472,7 @@ public: tier_config.is_multipart_upload = t.is_multipart_upload; } - void get_tier_config(RGWObjTier* t) { + const void get_tier_config(RGWObjTier* t) { if (tier_type != "cloud") return; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f5ab3e5b273d2..751b3e72f5133 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2211,6 +2211,7 @@ void RGWGetObj::execute(optional_yield y) RGWObjManifest m; decode(m, attr_iter->second); if (m.get_tier_type() == "cloud") { + /* XXX: Instead send presigned redirect or read-through */ op_ret = -ERR_INVALID_OBJECT_STATE; ldpp_dout(this, 0) << "ERROR: Cannot get cloud tiered object. Failing with " << op_ret << dendl; diff --git a/src/rgw/rgw_zone.h b/src/rgw/rgw_zone.h index c75a192b39136..78ff3bb663b92 100644 --- a/src/rgw/rgw_zone.h +++ b/src/rgw/rgw_zone.h @@ -695,32 +695,15 @@ struct RGWTierACLMapping { void init(const JSONFormattable& config) { const string& t = config["type"]; - if (t == "email") { - type = ACL_TYPE_EMAIL_USER; - } else if (t == "uri") { - type = ACL_TYPE_GROUP; - } else { - type = ACL_TYPE_CANON_USER; - } - + type = get_acl_type(t); source_id = config["source_id"]; dest_id = config["dest_id"]; } void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); - string s; - switch (type) { - case ACL_TYPE_EMAIL_USER: - s = "email"; - break; - case ACL_TYPE_GROUP: - s = "uri"; - break; - default: - s = "id"; - break; - } + + string s = get_acl_type_str(type); encode(s, bl); encode(source_id, bl); encode(dest_id, bl); @@ -729,15 +712,10 @@ struct RGWTierACLMapping { void decode(bufferlist::const_iterator& bl) { DECODE_START(1, bl); - string s; - decode(s, bl); - if (s == "email") { - type = ACL_TYPE_EMAIL_USER; - } else if (s == "uri") { - type = ACL_TYPE_GROUP; - } else { - type = ACL_TYPE_CANON_USER; - } + string s; + decode(s, bl); + + type = get_acl_type(s); decode(source_id, bl); decode(dest_id, bl); DECODE_FINISH(bl); -- 2.39.5