From 71882f61d5b5b6d46ad74a888097cd4ebd6b7464 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Sun, 10 Aug 2025 17:43:11 +0530 Subject: [PATCH] rgw/restore: Persistently store the restore state for cloud-s3 tier In order to resume IN_PROGRESS restore operations post RGW service restarts, store the entries of the objects being restored from `cloud-s3` tier persistently. This is already being done for `cloud-s3-glacier` tier and now the same will be applied to `cloud-s3` tier too. With this change, when `restore-object` is performed on any object, it will be marked RESTORE_ALREADY_IN_PROGRESS and added to a restore FIFO queue. This queue is later processed by Restore worker thread which will try to fetch the objects from Cloud or Glacier/Tape S3 services. Hence all the restore operations are now handled asynchronously (for both `cloud-s3`, `cloud-s3-glacier` tiers). Signed-off-by: Soumya Koduri --- src/rgw/driver/rados/rgw_rados.cc | 1 + src/rgw/rgw_restore.cc | 59 ++++++++++++------------------- 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 736750e15fe80..fea334ad4621f 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -5554,6 +5554,7 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx, ret = rgw_cloud_tier_get_object(tier_ctx, false, headers, &set_mtime, etag, accounted_size, attrs, &cb); + in_progress = false; } if (ret < 0) { diff --git a/src/rgw/rgw_restore.cc b/src/rgw/rgw_restore.cc index d0255a0007559..87f805259ee72 100644 --- a/src/rgw/rgw_restore.cc +++ b/src/rgw/rgw_restore.cc @@ -439,6 +439,7 @@ int Restore::process_restore_entry(RestoreEntry& entry, optional_yield y) using ceph::decode; decode(restore_status, iter); } + // check if its still in Progress state if (restore_status != rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) { ldpp_dout(this, 5) << __PRETTY_FUNCTION__ << ": Restore of object " << obj->get_key() << " not in progress state" << dendl; @@ -625,52 +626,36 @@ int Restore::restore_obj_from_cloud(rgw::sal::Bucket* pbucket, return ret; } - // now go ahead with restoring object - bool in_progress = false; - ret = pobj->restore_obj_from_cloud(pbucket, tier, cct, days, in_progress, dpp, y); + // now add the entry to the restore list to be processed by Restore worker thread + // asynchronoudly + RestoreEntry entry; + entry.bucket = pbucket->get_key(); + entry.obj_key = pobj->get_key(); + entry.status = rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress; + entry.days = days; + entry.zone_id = driver->get_zone()->get_id(); + + ldpp_dout(this, 10) << "Restore:: Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << dendl; + + int index = choose_oid(entry); + ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << ", to shard:" << obj_names[index] << dendl; + + std::vector r_entries; + r_entries.push_back(entry); + ret = sal_restore->add_entries(this, y, index, r_entries); if (ret < 0) { - ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: object " << pobj->get_key() << " fetching failed" << ret << dendl; - auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed); + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl; + auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed); if (reset_ret < 0) { - ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status to RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; + ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; } return ret; } - if (in_progress) { - // add restore entry to the list - RestoreEntry entry; - entry.bucket = pbucket->get_key(); - entry.obj_key = pobj->get_key(); - entry.status = rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress; - entry.days = days; - entry.zone_id = driver->get_zone()->get_id(); - - ldpp_dout(this, 10) << "Restore:: Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << dendl; - - int index = choose_oid(entry); - ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << ", to shard:" << obj_names[index] << dendl; - - std::vector r_entries; - r_entries.push_back(entry); - ret = sal_restore->add_entries(this, y, index, r_entries); - - if (ret < 0) { - ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl; - - auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed); - if (reset_ret < 0) { - ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl; - } - - return ret; - } - } - - ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Restore of object " << pobj->get_key() << (in_progress ? " is in progress" : " succeeded") << dendl; + ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Restore of object " << pobj->get_key() << " is in progress." << dendl; return ret; } -- 2.39.5