From: Soumya Koduri Date: Sat, 1 Mar 2025 07:05:51 +0000 (+0530) Subject: rgw/cloudrestore: Add Restore support from Glacier/Tape cloud endpoints X-Git-Tag: v20.3.0~299^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=300f6435095001258c11300937230c399802f46e;p=ceph.git rgw/cloudrestore: Add Restore support from Glacier/Tape cloud endpoints Unlike regular S3 cloud services, restoring objects from S3/Tape or AWS Glacier services would require special handling. We need to first restore the object using Glacier RestoreObject API and then download it using GET. https://docs.aws.amazon.com/cli/latest/reference/s3api/restore-object.html This PR adds that support for "Expedited" tier retrieval type. That means the restore would be quick and the object can be downloaded soon. TODO: "Standard" tier-type support. Need to handle the case where in restore from cloud endpoint could take a longer time and need to be monitored periodically in the background. Signed-off-by: Soumya Koduri --- diff --git a/src/rgw/driver/rados/.rgw_lc_tier.cc.swm b/src/rgw/driver/rados/.rgw_lc_tier.cc.swm deleted file mode 100644 index 990e1367cbe02..0000000000000 Binary files a/src/rgw/driver/rados/.rgw_lc_tier.cc.swm and /dev/null differ diff --git a/src/rgw/driver/rados/rgw_lc_tier.cc b/src/rgw/driver/rados/rgw_lc_tier.cc index 1b576a3184952..58fb86b0af1f3 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.cc +++ b/src/rgw/driver/rados/rgw_lc_tier.cc @@ -252,6 +252,70 @@ static const struct generic_attr generic_attrs[] = { { "ETAG", RGW_ATTR_ETAG }, }; +/* Restore object from remote endpoint. + */ +int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx, + std::map& headers, + real_time* pset_mtime, std::string& etag, + uint64_t& accounted_size, rgw::sal::Attrs& attrs, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params, + void* cb) { + RGWRESTConn::get_obj_params req_params; + std::string target_obj_name; + int ret = 0; + rgw_lc_obj_properties obj_properties(tier_ctx.o.meta.mtime, tier_ctx.o.meta.etag, + tier_ctx.o.versioned_epoch, tier_ctx.acl_mappings, + tier_ctx.target_storage_class); + + rgw_bucket dest_bucket; + dest_bucket.name = tier_ctx.target_bucket_name; + target_obj_name = tier_ctx.bucket_info.bucket.name + "/" + + tier_ctx.obj->get_name(); + if (!tier_ctx.o.is_current()) { + target_obj_name += get_key_instance(tier_ctx.obj->get_key()); + } + + if (glacier_params.glacier_restore_tier_type != GlacierRestoreTierType::Expedited) { + //XXX: Supporting STANDARD tier type is still in WIP + ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: Only Expedited tier_type is supported " << dendl; + return -1; + } + + rgw_obj dest_obj(dest_bucket, rgw_obj_key(target_obj_name)); + + ret = cloud_tier_restore(tier_ctx.dpp, tier_ctx.conn, dest_obj, days, glacier_params); + + ldpp_dout(tier_ctx.dpp, 20) << __func__ << "Restoring object=" << dest_obj << "returned ret = " << ret << dendl; + + if (ret < 0 ) { + ldpp_dout(tier_ctx.dpp, -1) << __func__ << "ERROR: failed to restore object=" << dest_obj << "; ret = " << ret << dendl; + return ret; + } + + // now send HEAD request and verify if restore is complete on glacier/tape endpoint + bool restore_in_progress = false; + do { + ret = rgw_cloud_tier_get_object(tier_ctx, true, headers, nullptr, etag, + accounted_size, attrs, nullptr); + + if (ret < 0) { + ldpp_dout(tier_ctx.dpp, 0) << __func__ << "ERROR: failed to fetch HEAD from cloud for obj=" << tier_ctx.obj << " , ret = " << ret << dendl; + return ret; + } + + restore_in_progress = is_restore_in_progress(tier_ctx.dpp, headers); + } while(restore_in_progress); + + // now do the actual GET + ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, pset_mtime, etag, + accounted_size, attrs, cb); + + ldpp_dout(tier_ctx.dpp, 20) << __func__ << "(): fetching object from cloud bucket:" << dest_bucket << ", object: " << target_obj_name << " returned ret:" << ret << dendl; + + return ret; +} + /* Read object or just head from remote endpoint. */ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head, @@ -367,6 +431,30 @@ static bool is_already_tiered(const DoutPrefixProvider *dpp, return 0; } +bool is_restore_in_progress(const DoutPrefixProvider *dpp, + std::map& headers) { + map attrs = headers; + + for (const auto& a : attrs) { + ldpp_dout(dpp, 20) << "GetCrf attr[" << a.first << "] = " << a.second < days, + RGWZoneGroupTierS3Glacier& glacier_params) { + rgw_http_param_pair params[] = {{"restore", nullptr}, {nullptr, nullptr}}; + // XXX: include versionId=VersionId in the params above + + stringstream ss; + XMLFormatter formatter; + int ret; + + bufferlist bl, out_bl; + string resource = obj_to_aws_path(dest_obj); + + const std::string tier_v = (glacier_params.glacier_restore_tier_type == GlacierRestoreTierType::Expedited) ? "Expedited" : "Standard"; + + struct RestoreRequest { + std::optional days; + std::optional tier; + + explicit RestoreRequest(std::optional _days, std::optional _tier) : days(_days), tier(_tier) {} + + void dump_xml(Formatter *f) const { + encode_xml("Days", days, f); + if (tier) { + f->open_object_section("GlacierJobParameters"); + encode_xml("Tier", tier, f); + f->close_section(); + }; + } + } req_enc(days, tier_v); + + struct RestoreResult { + std::string code; + + void decode_xml(XMLObj *obj) { + RGWXMLDecoder::decode_xml("Code", code, obj); + } + } result; + + req_enc.days = glacier_params.glacier_restore_days; + req_enc.tier = tier_v; + encode_xml("RestoreRequest", req_enc, &formatter); + + formatter.flush(ss); + bl.append(ss.str()); + + ret = dest_conn.send_resource(dpp, "POST", resource, params, nullptr, + out_bl, &bl, nullptr, null_yield); + + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << "ERROR: failed to send Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl; + } else { + ldpp_dout(dpp, 0) << __func__ << "Sent Restore request to cloud for obj=" << dest_obj << " , ret = " << ret << dendl; + } + + if (out_bl.length() > 0) { + RGWXMLDecoder::XMLParser parser; + if (!parser.init()) { + ldpp_dout(dpp, 0) << "ERROR: failed to initialize xml parser for parsing restore request response from server" << dendl; + return -EIO; + } + + if (!parser.parse(out_bl.c_str(), out_bl.length(), 1)) { + string str(out_bl.c_str(), out_bl.length()); + ldpp_dout(dpp, 5) << "ERROR: failed to parse xml restore: " << str << dendl; + return -EIO; + } + + try { + RGWXMLDecoder::decode_xml("Error", result, &parser, true); + } catch (RGWXMLDecoder::err& err) { + string str(out_bl.c_str(), out_bl.length()); + ldpp_dout(dpp, 5) << "ERROR: unexpected xml: " << str << dendl; + return -EIO; + } + + ldpp_dout(dpp, 0) << "ERROR: Restore request received result : " << result.code << dendl; + if (result.code != "RestoreAlreadyInProgress") { + return -EIO; + } else { // treat as success + return 0; + } + + ldpp_dout(dpp, 0) << "ERROR: restore req failed with error: " << result.code << dendl; + } + + return ret; +} + static int cloud_tier_abort_multipart(const DoutPrefixProvider *dpp, RGWRESTConn& dest_conn, const rgw_obj& dest_obj, const std::string& upload_id) { diff --git a/src/rgw/driver/rados/rgw_lc_tier.h b/src/rgw/driver/rados/rgw_lc_tier.h index 44ebaddfd9c86..60006da89149d 100644 --- a/src/rgw/driver/rados/rgw_lc_tier.h +++ b/src/rgw/driver/rados/rgw_lc_tier.h @@ -57,3 +57,18 @@ int rgw_cloud_tier_get_object(RGWLCCloudTierCtx& tier_ctx, bool head, real_time* pset_mtime, std::string& etag, uint64_t& accounted_size, rgw::sal::Attrs& attrs, void* cb); +int rgw_cloud_tier_restore_object(RGWLCCloudTierCtx& tier_ctx, + std::map& headers, + real_time* pset_mtime, std::string& etag, + uint64_t& accounted_size, rgw::sal::Attrs& attrs, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params, + void* cb); + +int cloud_tier_restore(const DoutPrefixProvider *dpp, + RGWRESTConn& dest_conn, const rgw_obj& dest_obj, + std::optional days, + RGWZoneGroupTierS3Glacier& glacier_params); + +bool is_restore_in_progress(const DoutPrefixProvider *dpp, + std::map& headers); diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 94861390c3d69..f34667901ff87 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5290,14 +5290,8 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx, dest_placement.storage_class = tier_ctx.restore_storage_class; RGWRadosPutObj cb(dpp, cct, plugin, compressor, &processor, progress_cb, progress_data, [&](map obj_attrs) { - // XXX: do we need filter() lke in fetch_remote_obj() cb + // XXX: do we need filter() like in fetch_remote_obj() cb dest_placement.inherit_from(dest_bucket_info.placement_rule); - /* For now we always restore to STANDARD storage-class. - * Later we will add support to take restore-target-storage-class - * for permanent restore - */ - // dest_placement.storage_class = RGW_STORAGE_CLASS_STANDARD; - processor.set_tail_placement(dest_placement); ret = processor.prepare(rctx.y); @@ -5326,9 +5320,18 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx, real_time set_mtime; std::map headers; ldpp_dout(dpp, 20) << "Fetching from cloud, object:" << dest_obj << dendl; - ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, + if (tier_config.tier_placement.tier_type == "cloud-s3-glacier") { + ldpp_dout(dpp, 20) << "Restoring object:" << dest_obj << " from the cloud" << dendl; + RGWZoneGroupTierS3Glacier& glacier_params = tier_config.tier_placement.s3_glacier; + ret = rgw_cloud_tier_restore_object(tier_ctx, headers, + &set_mtime, etag, accounted_size, + attrs, days, glacier_params, &cb); + } else { + ldpp_dout(dpp, 20) << "Fetching object:" << dest_obj << "from the cloud" << dendl; + ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, &set_mtime, etag, accounted_size, attrs, &cb); + } if (ret < 0) { ldpp_dout(dpp, 20) << "Fetching from cloud failed, object:" << dest_obj << dendl; diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 5ecb2a1e17d83..0ea8140073ff1 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -139,6 +139,7 @@ rgw_http_errors rgw_http_s3_errors({ { ERR_NO_SUCH_BUCKET_ENCRYPTION_CONFIGURATION, {404, "ServerSideEncryptionConfigurationNotFoundError"}}, { ERR_NO_SUCH_PUBLIC_ACCESS_BLOCK_CONFIGURATION, {404, "NoSuchPublicAccessBlockConfiguration"}}, { ERR_ACCOUNT_EXISTS, {409, "AccountAlreadyExists"}}, + { ERR_RESTORE_ALREADY_IN_PROGRESS, {409, "RestoreAlreadyInProgress"}}, { ECANCELED, {409, "ConcurrentModification"}}, }); diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index ab261d173fa27..44ac625dd5a16 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -355,6 +355,8 @@ inline constexpr const char* RGW_REST_STS_XMLNS = #define ERR_NO_SUCH_TAG_SET 2402 #define ERR_ACCOUNT_EXISTS 2403 +#define ERR_RESTORE_ALREADY_IN_PROGRESS 2500 + #ifndef UINT32_MAX #define UINT32_MAX (0xffffffffu) #endif diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 97b0be351524d..4df987c970a64 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3614,8 +3614,9 @@ void RGWRestoreObj_ObjStore_S3::send_response() if (restore_ret == 0) { s->err.http_ret = 202; // OK } else if (restore_ret == 1) { - s->err.http_ret = 409; // Conflict - dump_header(s, "x-amz-restore", "on-going-request=\"true\""); + restore_ret = -ERR_RESTORE_ALREADY_IN_PROGRESS; + set_req_state_err(s, restore_ret); + dump_header(s, "x-amz-restore", "ongoing-request=\"true\""); } else if (restore_ret == 2) { rgw::sal::Attrs attrs; ceph::real_time expiration_date; diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 68442e741d983..45749c228f749 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1719,7 +1719,7 @@ public: /** Get the type of this tier */ virtual const std::string& get_tier_type() = 0; - /** Is the type of this tier cloud-s3/clous-s3-glacier */ + /** Is the type of this tier cloud-s3/cloud-s3-glacier */ virtual bool is_tier_type_s3() = 0; /** Get the storage class of this tier */ virtual const std::string& get_storage_class() = 0;