From: Shilpa Jagannath <smanjara@redhat.com>
Date: Wed, 26 Jun 2024 07:04:08 +0000 (-0400)
Subject: rgw/multisite: in a multisite env with bucket sync policies configured,
X-Git-Tag: v20.3.0~161^2~12
X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=c268e64b45bb5c0598a3ea56ec7e70c1420edc35;p=ceph.git

rgw/multisite: in a multisite env with bucket sync policies configured,
we may end up orphaning objects on remote zones when a delete bucket
is issued on metadata master. to avoid this, list the buckets on remote
zones and delete bucket only when empty. if a zone is unreachable we
drop that zone and continue with bucket deletion. such zones might have
orphaned objects that will have to be cleaned up using radosgw-admin tool

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>
---

diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc
index 332006c02fd45..1ac07306b09c0 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.cc
+++ b/src/rgw/driver/rados/rgw_cr_rados.cc
@@ -1204,3 +1204,56 @@ int RGWDataPostNotifyCR::operate(const DoutPrefixProvider* dpp)
   }
   return 0;
 }
+  
+RGWStatRemoteBucketCR::RGWStatRemoteBucketCR(const DoutPrefixProvider *dpp,
+				    rgw::sal::RadosStore* const store,
+            const rgw_zone_id source_zone,
+            const rgw_bucket& bucket,
+            RGWHTTPManager* http,
+            std::vector<rgw_zone_id> zids,
+            std::vector<bucket_unordered_list_result>& peer_result)
+      : RGWCoroutine(store->ctx()), dpp(dpp), store(store),
+      source_zone(source_zone), bucket(bucket), http(http),
+      zids(zids), peer_result(peer_result) {}
+  
+int RGWStatRemoteBucketCR::operate(const DoutPrefixProvider *dpp) {
+  reenter(this) {
+    yield {
+      auto result = peer_result.begin();
+      for (auto& zid : zids) {
+        auto& zone_conn_map = store->getRados()->svc.zone->get_zone_conn_map();
+        auto ziter = zone_conn_map.find(zid);
+        if (ziter == zone_conn_map.end()) {
+          ldpp_dout(dpp, 0) << "WARNING: no connection to zone " << ziter->first << dendl;
+          continue;
+        }
+        ldpp_dout(dpp, 20) << "query bucket from: " << ziter->first << dendl;
+        RGWRESTConn *conn = ziter->second;
+
+        rgw_http_param_pair pairs[] = { { "versions" , NULL },
+					{ "format" , "json" },
+					{ "objs-container" , "true" },
+          { "max-keys", "1" },
+          { "allow-unordered", "true"},
+          { "key-marker" , NULL },
+					{ "version-id-marker" , NULL },
+	                                { NULL, NULL } };
+        string p = string("/") + bucket.get_key(':', 0);
+        spawn(new RGWReadRESTResourceCR<bucket_unordered_list_result>(store->ctx(), &*conn, &*http, p, pairs, &*result), false);
+        ++result;
+      }
+    }
+
+    while (num_spawned()) {
+      yield wait_for_child();
+      collect(&child_ret, nullptr);
+      if (child_ret < 0) {
+        drain_all();
+        return set_cr_error(child_ret);
+      }
+    }
+
+    return set_cr_done();
+  }
+  return 0;
+}
diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h
index 09b364a7e17bf..6a5ef9851f3a4 100644
--- a/src/rgw/driver/rados/rgw_cr_rados.h
+++ b/src/rgw/driver/rados/rgw_cr_rados.h
@@ -16,6 +16,8 @@
 
 #include "services/svc_sys_obj.h"
 #include "services/svc_bucket.h"
+#include <string_view>
+using namespace std;
 
 struct rgw_http_param_pair;
 class RGWRESTConn;
@@ -1708,3 +1710,105 @@ public:
   int operate(const DoutPrefixProvider* dpp) override;
 };
 
+struct rgw_bucket_entry_owner {
+  std::string id;
+  std::string display_name;
+
+  rgw_bucket_entry_owner() {}
+  rgw_bucket_entry_owner(const std::string& _id, const std::string& _display_name) : id(_id), display_name(_display_name) {}
+
+  void decode_json(JSONObj *obj);
+};
+
+struct bucket_list_entry {
+  bool delete_marker;
+  rgw_obj_key key;
+  bool is_latest;
+  real_time mtime;
+  string etag;
+  uint64_t size;
+  string storage_class;
+  rgw_bucket_entry_owner owner;
+  uint64_t versioned_epoch;
+  string rgw_tag;
+
+  bucket_list_entry() : delete_marker(false), is_latest(false), size(0), versioned_epoch(0) {}
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("IsDeleteMarker", delete_marker, obj);
+    JSONDecoder::decode_json("Key", key.name, obj);
+    JSONDecoder::decode_json("VersionId", key.instance, obj);
+    JSONDecoder::decode_json("IsLatest", is_latest, obj);
+    string mtime_str;
+    JSONDecoder::decode_json("RgwxMtime", mtime_str, obj);
+
+    struct tm t;
+    uint32_t nsec;
+    if (parse_iso8601(mtime_str.c_str(), &t, &nsec)) {
+      ceph_timespec ts;
+      ts.tv_sec = (uint64_t)internal_timegm(&t);
+      ts.tv_nsec = nsec;
+      mtime = real_clock::from_ceph_timespec(ts);
+    }
+    JSONDecoder::decode_json("ETag", etag, obj);
+    JSONDecoder::decode_json("Size", size, obj);
+    JSONDecoder::decode_json("StorageClass", storage_class, obj);
+    JSONDecoder::decode_json("Owner", owner, obj);
+    JSONDecoder::decode_json("VersionedEpoch", versioned_epoch, obj);
+    JSONDecoder::decode_json("RgwxTag", rgw_tag, obj);
+    if (key.instance == "null" && !versioned_epoch) {
+      key.instance.clear();
+    }
+  }
+
+  RGWModifyOp get_modify_op() const {
+    if (delete_marker) {
+      return CLS_RGW_OP_LINK_OLH_DM;
+    } else if (!key.instance.empty() && key.instance != "null") {
+      return CLS_RGW_OP_LINK_OLH;
+    } else {
+      return CLS_RGW_OP_ADD;
+    }
+  }
+};
+
+struct bucket_unordered_list_result {
+  string name;
+  string prefix;
+  int max_keys;
+  bool is_truncated;
+  bool allow_unordered;
+  list<bucket_list_entry> entries;
+
+  bucket_unordered_list_result() : max_keys(0), is_truncated(false) {}
+
+  void decode_json(JSONObj *obj) {
+    JSONDecoder::decode_json("Name", name, obj);
+    JSONDecoder::decode_json("Prefix", prefix, obj);
+    JSONDecoder::decode_json("MaxKeys", max_keys, obj);
+    JSONDecoder::decode_json("IsTruncated", is_truncated, obj);
+    JSONDecoder::decode_json("allow-unordered", allow_unordered, obj);
+    JSONDecoder::decode_json("Entries", entries, obj);
+  }
+};
+
+class RGWStatRemoteBucketCR: public RGWCoroutine {
+  int child_ret = 0;
+  const DoutPrefixProvider *dpp;
+  rgw::sal::RadosStore* const store;
+  const rgw_zone_id source_zone;
+  const rgw_bucket& bucket;
+  RGWHTTPManager* http;
+  std::vector<rgw_zone_id> zids;
+  vector<bucket_unordered_list_result>& peer_result;
+
+public:
+  RGWStatRemoteBucketCR(const DoutPrefixProvider *dpp,
+				    rgw::sal::RadosStore* const store,
+            const rgw_zone_id source_zone,
+            const rgw_bucket& bucket,
+            RGWHTTPManager* http, std::vector<rgw_zone_id> zids,
+            std::vector<bucket_unordered_list_result>& peer_result);
+
+  int operate(const DoutPrefixProvider *dpp) override;
+};
diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc
index 7eb8db380889c..6b947cf76f521 100644
--- a/src/rgw/driver/rados/rgw_data_sync.cc
+++ b/src/rgw/driver/rados/rgw_data_sync.cc
@@ -4063,58 +4063,6 @@ void rgw_bucket_entry_owner::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("DisplayName", display_name, obj);
 }
 
-struct bucket_list_entry {
-  bool delete_marker;
-  rgw_obj_key key;
-  bool is_latest;
-  real_time mtime;
-  string etag;
-  uint64_t size;
-  string storage_class;
-  rgw_bucket_entry_owner owner;
-  uint64_t versioned_epoch;
-  string rgw_tag;
-
-  bucket_list_entry() : delete_marker(false), is_latest(false), size(0), versioned_epoch(0) {}
-
-  void decode_json(JSONObj *obj) {
-    JSONDecoder::decode_json("IsDeleteMarker", delete_marker, obj);
-    JSONDecoder::decode_json("Key", key.name, obj);
-    JSONDecoder::decode_json("VersionId", key.instance, obj);
-    JSONDecoder::decode_json("IsLatest", is_latest, obj);
-    string mtime_str;
-    JSONDecoder::decode_json("RgwxMtime", mtime_str, obj);
-
-    struct tm t;
-    uint32_t nsec;
-    if (parse_iso8601(mtime_str.c_str(), &t, &nsec)) {
-      ceph_timespec ts;
-      ts.tv_sec = (uint64_t)internal_timegm(&t);
-      ts.tv_nsec = nsec;
-      mtime = real_clock::from_ceph_timespec(ts);
-    }
-    JSONDecoder::decode_json("ETag", etag, obj);
-    JSONDecoder::decode_json("Size", size, obj);
-    JSONDecoder::decode_json("StorageClass", storage_class, obj);
-    JSONDecoder::decode_json("Owner", owner, obj);
-    JSONDecoder::decode_json("VersionedEpoch", versioned_epoch, obj);
-    JSONDecoder::decode_json("RgwxTag", rgw_tag, obj);
-    if (key.instance == "null" && !versioned_epoch) {
-      key.instance.clear();
-    }
-  }
-
-  RGWModifyOp get_modify_op() const {
-    if (delete_marker) {
-      return CLS_RGW_OP_LINK_OLH_DM;
-    } else if (!key.instance.empty() && key.instance != "null") {
-      return CLS_RGW_OP_LINK_OLH;
-    } else {
-      return CLS_RGW_OP_ADD;
-    }
-  }
-};
-
 struct bucket_list_result {
   string name;
   string prefix;
diff --git a/src/rgw/driver/rados/rgw_data_sync.h b/src/rgw/driver/rados/rgw_data_sync.h
index 4e160f14cc486..9daf2b739bde3 100644
--- a/src/rgw/driver/rados/rgw_data_sync.h
+++ b/src/rgw/driver/rados/rgw_data_sync.h
@@ -285,17 +285,6 @@ struct rgw_datalog_shard_data {
 
 class RGWAsyncRadosProcessor;
 class RGWDataSyncControlCR;
-
-struct rgw_bucket_entry_owner {
-  std::string id;
-  std::string display_name;
-
-  rgw_bucket_entry_owner() {}
-  rgw_bucket_entry_owner(const std::string& _id, const std::string& _display_name) : id(_id), display_name(_display_name) {}
-
-  void decode_json(JSONObj *obj);
-};
-
 class RGWSyncErrorLogger;
 class RGWRESTConn;
 class RGWServices;
diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc
index e50305d991f6b..3fb9a4eb09e50 100644
--- a/src/rgw/driver/rados/rgw_rados.cc
+++ b/src/rgw/driver/rados/rgw_rados.cc
@@ -5612,6 +5612,64 @@ int RGWRados::delete_bucket(RGWBucketInfo& bucket_info, std::map<std::string, bu
     }
   }
 
+  // delete_bucket checks for objects in the bucket on other zones,
+  // if there is bucket sync policy configured, by doing unordered
+  // listing with max_key=1. we don't prevent users from
+  // deleting a bucket if there are any errors in this block
+  // except if we found objects in another zone.
+  if (svc.zone->is_syncing_bucket_meta(bucket)) {
+    auto bs_policy = bucket_info.sync_policy;
+    if (bs_policy) {
+      ldpp_dout(dpp, 10) << "bucket policy exists" << dendl;
+      const rgw_zone_id source_zone = svc.zone->get_zone_params().get_id();
+      RGWBucketSyncPolicyHandlerRef source_handler;
+      int ret = driver->get_sync_policy_handler(dpp, source_zone, bucket, &source_handler, y);
+      if (ret < 0) {
+        ldpp_dout(dpp, 10) << "could not get bucket sync policy handler (r=" << ret << ")" << dendl;
+      }
+
+      auto all_dests = source_handler->get_all_dests();
+
+      std::vector<rgw_zone_id> zids;
+      rgw_zone_id last_zid;
+      for (auto& diter : all_dests) {
+        const auto& zid = diter.first;
+        if (zid == last_zid) {
+          continue;
+        }
+        last_zid = zid;
+        zids.push_back(zid);
+      }
+
+      std::vector<bucket_unordered_list_result> peer_status;
+      peer_status.resize(zids.size());
+
+      RGWCoroutinesManager crs(driver->ctx(), driver->getRados()->get_cr_registry());
+      RGWHTTPManager http(driver->ctx(), crs.get_completion_mgr());
+      ret = http.start();
+      if (ret < 0) {
+        ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl;
+      }
+      ret = crs.run(dpp, new RGWStatRemoteBucketCR(dpp, driver, source_zone, bucket, &http, zids, peer_status));
+      if (ret < 0) {
+        ldpp_dout(dpp, 0) << "failed to fetch remote bucket stats " << cpp_strerror(ret) << dendl;
+      }
+
+      for (const auto& list_result: peer_status) {
+        auto entries_iter = list_result.entries.begin();
+        for (; entries_iter != list_result.entries.end(); ++entries_iter) {
+          std::string ns;
+          rgw_obj_key obj;
+
+          if (rgw_obj_key::oid_to_key_in_ns(entries_iter->key.name, &obj, ns)) {
+            ldpp_dout(dpp, 0) << "cannot delete bucket. objects exist in the bucket in another zone " << dendl;
+            return -ENOTEMPTY;
+          }
+        }
+      }
+    }
+  }
+
   bool remove_ep = true;
 
   if (objv_tracker.read_version.empty()) {
diff --git a/src/rgw/driver/rados/rgw_sync_module_aws.h b/src/rgw/driver/rados/rgw_sync_module_aws.h
index 92532ff00e7fa..ce6e1d27ab018 100644
--- a/src/rgw/driver/rados/rgw_sync_module_aws.h
+++ b/src/rgw/driver/rados/rgw_sync_module_aws.h
@@ -5,6 +5,8 @@
 
 #include "rgw_sync_module.h"
 
+struct rgw_bucket_entry_owner;
+
 struct rgw_sync_aws_multipart_part_info {
   int part_num{0};
   uint64_t ofs{0};
diff --git a/src/rgw/driver/rados/rgw_sync_module_es.cc b/src/rgw/driver/rados/rgw_sync_module_es.cc
index 414fbeac4c990..5fb4b11d2fbf2 100644
--- a/src/rgw/driver/rados/rgw_sync_module_es.cc
+++ b/src/rgw/driver/rados/rgw_sync_module_es.cc
@@ -9,6 +9,7 @@
 #include "rgw_sync_module_es.h"
 #include "rgw_sync_module_es_rest.h"
 #include "rgw_rest_conn.h"
+#include "rgw_cr_rados.h"
 #include "rgw_cr_rest.h"
 #include "rgw_op.h"
 #include "rgw_es_query.h"