rgw/restore: Persistently store the restore state for cloud-s3 tier

In order to resume IN_PROGRESS restore operations post RGW service
restarts, store the entries of the objects being restored from `cloud-s3`
tier persistently. This is already being done for `cloud-s3-glacier`
tier and now the same will be applied to `cloud-s3` tier too.

With this change, when `restore-object` is performed on any object,
it will be marked RESTORE_ALREADY_IN_PROGRESS and added to a restore FIFO queue.
This queue is later processed by Restore worker thread which will try to
fetch the objects from Cloud or Glacier/Tape S3 services. Hence all the
restore operations are now handled asynchronously (for both `cloud-s3`,
`cloud-s3-glacier` tiers).

Resolves: rhbz#2312933
Signed-off-by: Soumya Koduri <skoduri@redhat.com>

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 968f064a672ef8c58ee2efb0e7aa5e94bf56c611..3b2f93b362396fb42609f1842133a1252fe53b4e 100644 (file)
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -5563,6 +5563,7 @@ int RGWRados::restore_obj_from_cloud(RGWLCCloudTierCtx& tier_ctx,
     ret = rgw_cloud_tier_get_object(tier_ctx, false,  headers,
                                 &set_mtime, etag, accounted_size,
                                 attrs, &cb);
+    in_progress = false;
   }
 
   if (ret < 0) { 

diff --git a/src/rgw/rgw_restore.cc b/src/rgw/rgw_restore.cc
index 9017fa068977dea7484cbc749ddc3d80fbc9e812..97eaacce0d0d96b10b243fe9b2c390d80a606f77 100644 (file)
--- a/src/rgw/rgw_restore.cc
+++ b/src/rgw/rgw_restore.cc
@@ -424,6 +424,7 @@ int Restore::process_restore_entry(RestoreEntry& entry, optional_yield y)
     using ceph::decode;
     decode(restore_status, iter);
   }
+  // check if its still in Progress state
   if (restore_status != rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) {
     ldpp_dout(this, 5) << __PRETTY_FUNCTION__ << ": Restore of object " << obj->get_key()
                       << " not in progress state" << dendl;
@@ -610,52 +611,36 @@ int Restore::restore_obj_from_cloud(rgw::sal::Bucket* pbucket,
     return ret;
   }
 
-  // now go ahead with restoring object
-  bool in_progress = false;
-  ret = pobj->restore_obj_from_cloud(pbucket, tier, cct, days, in_progress, dpp, y);
+  // now add the entry to the restore list to be processed by Restore worker thread
+  // asynchronoudly
+  RestoreEntry entry;
+  entry.bucket = pbucket->get_key();
+  entry.obj_key = pobj->get_key();
+  entry.status = rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress;
+  entry.days = days;
+  entry.zone_id = driver->get_zone()->get_id(); 
+ 
+  ldpp_dout(this, 10) << "Restore:: Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << dendl;
+
+  int index = choose_oid(entry);
+  ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << ", to shard:" << obj_names[index] << dendl;
+
+  std::vector<rgw::restore::RestoreEntry> r_entries;
+  r_entries.push_back(entry);
+  ret = sal_restore->add_entries(this, y, index, r_entries);
 
   if (ret < 0) {
-   ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: object " << pobj->get_key() << " fetching failed" << ret << dendl;    
-    auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed);
+    ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl;     
 
+    auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed);
     if (reset_ret < 0) {
-      ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status to RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl;       
+      ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl;         
     }
 
     return ret;
   }
 
-  if (in_progress) {
-    // add restore entry to the list
-    RestoreEntry entry;
-    entry.bucket = pbucket->get_key();
-    entry.obj_key = pobj->get_key();
-    entry.status = rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress;
-    entry.days = days;
-    entry.zone_id = driver->get_zone()->get_id(); 
-
-    ldpp_dout(this, 10) << "Restore:: Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << dendl;
-
-    int index = choose_oid(entry);
-    ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Adding restore entry of object(" << pobj->get_key() << ") entry: " << entry << ", to shard:" << obj_names[index] << dendl;
-
-    std::vector<rgw::restore::RestoreEntry> r_entries;
-    r_entries.push_back(entry);
-    ret = sal_restore->add_entries(this, y, index, r_entries);
-
-    if (ret < 0) {
-      ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": ERROR: Adding restore entry of object(" << pobj->get_key() << ") failed" << ret << dendl;           
-
-      auto reset_ret = set_cloud_restore_status(this, pobj, y, rgw::sal::RGWRestoreStatus::RestoreFailed);
-      if (reset_ret < 0) {
-        ldpp_dout(this, -1) << __PRETTY_FUNCTION__ << ": Setting restore status as RestoreFailed failed for object(" << pobj->get_key() << ") " << reset_ret << dendl;       
-      }
-
-      return ret;
-    }
-  }
-
-  ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Restore of object " << pobj->get_key() << (in_progress ? " is in progress" : " succeeded") << dendl;  
+  ldpp_dout(this, 10) << __PRETTY_FUNCTION__ << ": Restore of object " << pobj->get_key() << " is in progress." << dendl;  
   return ret;
 }