From 9c3264fd943bf6ecdc093255063b9caa6ab43d84 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Mon, 13 Feb 2023 20:30:21 +0530 Subject: [PATCH] rgw/cloud: Add custom headers for objects transitioned to cloud Some of the cloud services, (like MCG Noobaa/Azure Namespace store), may not be able to map and store the objects which contain s3 style metadata keys. To help such services determine if the objects being transitioned are from RGW and whether or not ignore such attrs, added below two headers for the objects being copied/transitioned from RGW - 1) x-rgw-cloud : true/false (set to "true" if the object is being transitioned/synced from RGW) 2) x-rgw-cloud-keep-attrs : true/false - if set to default "true" , the cloud service should store all the x-amz-meta-* attrs. If cannot be mapped/stored, the operation should fail - if set to "false", the destination cloud can ignore such attrs and just store the object data being sent. Also fixed a bug in the cloudtier module wherein the user-defined attrs were not being copied to the cloud endpoint as part of transition Fixes: https://tracker.ceph.com/issues/57980 Fixes: https://tracker.ceph.com/issues/58796 Signed-off-by: Soumya Koduri --- doc/radosgw/cloud-transition.rst | 12 ++++++++++++ src/rgw/driver/rados/rgw_lc_tier.cc | 11 +++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/doc/radosgw/cloud-transition.rst b/doc/radosgw/cloud-transition.rst index bc5b3ea2c69..c00ad790b17 100644 --- a/doc/radosgw/cloud-transition.rst +++ b/doc/radosgw/cloud-transition.rst @@ -302,6 +302,18 @@ Due to API limitations there is no way to preserve original object modification x-amz-meta-rgwx-source-mtime: 1608546349.757100363 x-amz-meta-rgwx-versioned-epoch: 0 +In order to allow some cloud services detect the source and map the user-defined 'x-amz-meta-' attributes, below two additional new attributes are added to the objects being transitioned + +:: + + x-rgw-cloud : true/false + (set to "true", by default, if the object is being transitioned from RGW) + + x-rgw-cloud-keep-attrs : true/false + (if set to default value "true", the cloud service should map and store all the x-amz-meta-* attributes. If it cannot, then the operation should fail. + if set to "false", the cloud service can ignore such attributes and just store the object data being sent.) + + By default, post transition, the source object gets deleted. But it is possible to retain its metadata but with updated values (like storage-class and object-size) by setting config option 'retain_head_object' to true. However GET on those objects shall still fail with 'InvalidObjectState' error. For example, diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc index 0ad21693123..59a3106f065 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.cc +++ b/src/rgw/driver/rados/rgw_lc_tier.cc @@ -514,8 +514,7 @@ int RGWLCCloudStreamPut::init() { } bool RGWLCCloudStreamPut::keep_attr(const string& h) { - return (keep_headers.find(h) != keep_headers.end() || - boost::algorithm::starts_with(h, "X_AMZ_")); + return (keep_headers.find(h) != keep_headers.end()); } void RGWLCCloudStreamPut::init_send_attrs(const DoutPrefixProvider *dpp, @@ -531,6 +530,12 @@ void RGWLCCloudStreamPut::init_send_attrs(const DoutPrefixProvider *dpp, for (auto& hi : rest_obj.attrs) { if (keep_attr(hi.first)) { attrs.insert(hi); + } else { + std::string s1 = boost::algorithm::to_lower_copy(hi.first); + const char* k = std::strstr(s1.c_str(), "x-amz"); + if (k) { + attrs[k] = hi.second; + } } } @@ -633,6 +638,8 @@ void RGWLCCloudStreamPut::init_send_attrs(const DoutPrefixProvider *dpp, /* New attribute to specify its transitioned from RGW */ attrs["x-amz-meta-rgwx-source"] = "rgw"; + attrs["x-rgw-cloud"] = "true"; + attrs["x-rgw-cloud-keep-attrs"] = "true"; char buf[32]; snprintf(buf, sizeof(buf), "%llu", (long long)obj_properties.versioned_epoch); -- 2.47.3