rgw/multisite: retry on detecting bucket instance metadata racing writes

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index e444521d395c94c68f17eef8a1f360fbc9373e9f..9d02f21377a0e13b989d0d50c65953b8f6227ea3 100644 (file)
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -2874,6 +2874,14 @@ int RGWBucketInstanceMetadataHandler::put_prepare(
     /* existing bucket, keep its placement */
     bci.info.bucket.explicit_placement = old_bci->info.bucket.explicit_placement;
     bci.info.placement_rule = old_bci->info.placement_rule;
+
+    //if the bucket is being deleted, create and store a special log type for
+    //bucket instance cleanup in multisite setup
+    const auto& log = bci.info.layout.logs.back();
+    if (bci.info.bucket_deleted() && log.layout.type != rgw::BucketLogType::Deleted) {
+      bci.info.layout.logs.push_back({0, {rgw::BucketLogType::Deleted}});
+      ldpp_dout(dpp, 10) << "store log layout type: " <<  bci.info.layout.logs.back().layout.type << dendl;
+    }
   }
 
   //always keep bucket versioning enabled on archive zone
@@ -2884,7 +2892,7 @@ int RGWBucketInstanceMetadataHandler::put_prepare(
   /* record the read version (if any), store the new version */
   bci.info.objv_tracker.read_version = objv_tracker.read_version;
   bci.info.objv_tracker.write_version = objv_tracker.write_version;
-
+  
   return 0;
 }
 

diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index 07192a7aa44b973232386b2fcd6f0fc9099606f7..659a3073ff9b72a2424fe78bfd04beea0f291bfe 100644 (file)
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -5614,15 +5614,30 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, std::map<std::string, bu
                                       cct->_conf->rgw_bucket_index_max_aio)();
   } else {
     bucket_info.flags |= BUCKET_DELETED;
+    static constexpr auto max_retries = 10;
+    int retries = 0;
+    do {
+      r = ctl.bucket->store_bucket_instance_info(bucket, bucket_info, y, dpp, RGWBucketCtl::BucketInstance::PutParams()
+                                                                      .set_exclusive(false)
+                                                                      .set_mtime(real_time())
+                                                                      .set_attrs(&attrs)
+                                                                      .set_orig_info(&bucket_info));
+      if (r == -ECANCELED) {
+        //racing write. re-read bucket info
+        map<string, bufferlist> attrs;
+        int ret = get_bucket_instance_info(bucket, bucket_info, nullptr, &attrs, y, dpp);
+        if (ret < 0) {
+          ldpp_dout(dpp, 5) << "ERROR: failed to get bucket instance info for bucket=" << bucket.name << " ret=" << ret << dendl;
+          r = ret;
+          break;
+        }
+      }
+    } while (r == -ECANCELED && ++retries < max_retries);
 
-    r = ctl.bucket->store_bucket_instance_info(bucket, bucket_info, y, dpp, RGWBucketCtl::BucketInstance::PutParams()
-                                                                    .set_exclusive(false)
-                                                                    .set_mtime(real_time())
-                                                                    .set_attrs(&attrs)
-                                                                    .set_orig_info(&bucket_info));
     if (r < 0) {
-      ldpp_dout(dpp, 0) << "ERROR: failed to store bucket instance info for bucket=" << bucket.name << " ret=" << r << dendl;
-      return r;
+      ldpp_dout(dpp, 0) << "WARNING: failed to store bucket instance info for bucket: " << bucket.name << " r=" << r << dendl;
+    } else {
+      ldpp_dout(dpp, 20) << "INFO: setting bucket info flag to deleted for bucket: " << bucket.name << dendl;
     }
   }