rgw-admin: sync info shows and resolve bucket sync hints

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 11ceacaaaa8e5cec5eb33310736db36efba1d312..ea708b5945376918f77ab3efc4cbf68e56f9947c 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2341,8 +2341,33 @@ static std::vector<string> convert_bucket_set_to_str_vec(const std::set<rgw_buck
   return std::move(result);
 }
 
+static void get_hint_entities(const std::set<string>& zone_names, const std::set<rgw_bucket>& buckets,
+                             std::set<rgw_sync_bucket_entity> *hint_entities)
+{
+  for (auto& zone : zone_names) {
+    for (auto& b : buckets) {
+      string zid;
+      if (!store->svc()->zone->find_zone_id_by_name(zone, &zid)) {
+       cerr << "WARNING: cannot find zone id for zone=" << zone << ", skippping" << std::endl;
+       continue;
+      }
+
+      RGWBucketInfo hint_bucket_info;
+      rgw_bucket hint_bucket;
+      int ret = init_bucket(b, hint_bucket_info, hint_bucket);
+      if (ret < 0) {
+       ldout(store->ctx(), 20) << "could not init bucket info for hint bucket=" << b << " ... skipping" << dendl;
+       continue;
+      }
+
+      hint_entities->insert(rgw_sync_bucket_entity(zid, hint_bucket));
+    }
+  }
+}
+
 static int sync_info(std::optional<string> opt_target_zone, std::optional<rgw_bucket> opt_bucket, Formatter *formatter)
 {
+  string zone_name;
   std::optional<string> zone_id;
 
   if (opt_target_zone) {
@@ -2351,7 +2376,10 @@ static int sync_info(std::optional<string> opt_target_zone, std::optional<rgw_bu
       cerr << "WARNING: cannot find zone id for zone=" << *opt_target_zone << std::endl;
       return -ENOENT;
     }
+    zone_name = *opt_target_zone;
     zone_id = zid;
+  } else {
+    zone_name = store->svc ()->zone->zone_name();
   }
 
   auto zone_policy_handler = store->svc()->zone->get_sync_policy_handler(zone_id);
@@ -2396,21 +2424,45 @@ static int sync_info(std::optional<string> opt_target_zone, std::optional<rgw_bu
   auto source_hints_vec = convert_bucket_set_to_str_vec(handler->get_source_hints());
   auto target_hints_vec = convert_bucket_set_to_str_vec(handler->get_target_hints());
 
-  RGWBucketSyncFlowManager::pipe_set *resolved_sources;
-  RGWBucketSyncFlowManager::pipe_set *resolved_dests;
+  RGWBucketSyncFlowManager::pipe_set resolved_sources;
+  RGWBucketSyncFlowManager::pipe_set resolved_dests;
 
-  for (auto& b : handler->get_source_hints()) {
-    RGWBucketInfo hint_bucket_info;
-    rgw_bucket hint_bucket;
-    int ret = init_bucket(b, hint_bucket_info, hint_bucket);
-    if (ret < 0) {
-      ldout(cct, 20) << "could not init bucket info for hint bucket=" << b << " ... skipping" << dendl;
-      continue;
+  rgw_sync_bucket_entity self_entity(zone_name, opt_bucket);
+
+  set<string> source_zones;
+  set<string> target_zones;
+
+  zone_policy_handler->reflect(nullptr, nullptr,
+                               nullptr, nullptr,
+                               &source_zones,
+                               &target_zones,
+                               false); /* relaxed: also get all zones that we allow to sync to/from */
+
+  std::set<rgw_sync_bucket_entity> hint_entities;
+
+  get_hint_entities(source_zones, handler->get_source_hints(), &hint_entities);
+  get_hint_entities(target_zones, handler->get_target_hints(), &hint_entities);
+
+  for (auto& hint_entity : hint_entities) {
+    if (!hint_entity.zone ||
+       !hint_entity.bucket) {
+      continue; /* shouldn't really happen */
     }
 
+    auto& zid = *hint_entity.zone;
+    auto& hint_bucket = *hint_entity.bucket;
+
     RGWBucketSyncPolicyHandlerRef hint_bucket_handler;
-    hint_bucket_handler.reset(handler->alloc_child(hint_bucket_indo));
+    int r = store->ctl()->bucket->get_sync_policy_handler(zid, hint_bucket, &hint_bucket_handler, null_yield);
+    if (r < 0) {
+      ldout(store->ctx(), 20) << "could not get bucket sync policy handler for hint bucket=" << hint_bucket << " ... skipping" << dendl;
+      continue;
+    }
 
+    hint_bucket_handler->get_pipes(&resolved_dests,
+                                  &resolved_sources,
+                                  self_entity); /* flipping resolved dests and sources as these are
+                                                   relative to the remote entity */
   }
 
   {
@@ -2424,8 +2476,8 @@ static int sync_info(std::optional<string> opt_target_zone, std::optional<rgw_bu
     }
     {
       Formatter::ObjectSection resolved_hints_section(*formatter, "resolved-hints");
-      encode_json("resolved-hints", *sources, formatter);
-      encode_json("dests", *dests, formatter);
+      encode_json("sources", resolved_sources, formatter);
+      encode_json("dests", resolved_dests, formatter);
     }
   }
 

diff --git a/src/rgw/rgw_bucket_sync.cc b/src/rgw/rgw_bucket_sync.cc
index d6e07df72d5e2114da670dc91dbd7f2382909fb5..e20d26da19e1b99dc29238f4ccd439015186343d 100644 (file)
--- a/src/rgw/rgw_bucket_sync.cc
+++ b/src/rgw/rgw_bucket_sync.cc
@@ -345,7 +345,6 @@ void RGWBucketSyncFlowManager::init(const rgw_sync_policy_info& sync_policy) {
 void RGWBucketSyncFlowManager::reflect(std::optional<rgw_bucket> effective_bucket,
                                        RGWBucketSyncFlowManager::pipe_set *source_pipes,
                                        RGWBucketSyncFlowManager::pipe_set *dest_pipes,
-                                      std::optional<rgw_bucket> filter_peer_bucket,
                                        bool only_enabled) const
 
 {
@@ -354,7 +353,7 @@ void RGWBucketSyncFlowManager::reflect(std::optional<rgw_bucket> effective_bucke
   entity.bucket = effective_bucket.value_or(rgw_bucket());
 
   if (parent) {
-    parent->reflect(effective_bucket, source_pipes, dest_pipes, filter_peer_bucket, only_enabled);
+    parent->reflect(effective_bucket, source_pipes, dest_pipes, only_enabled);
   }
 
   for (auto& item : flow_groups) {
@@ -371,9 +370,6 @@ void RGWBucketSyncFlowManager::reflect(std::optional<rgw_bucket> effective_bucke
       if (!pipe.dest.match_bucket(effective_bucket)) {
         continue;
       }
-      if (!pipe.source.match_bucket(filter_peer_bucket)) {
-       continue;
-      }
 
       pipe.source.apply_bucket(effective_bucket);
       pipe.dest.apply_bucket(effective_bucket);
@@ -387,9 +383,6 @@ void RGWBucketSyncFlowManager::reflect(std::optional<rgw_bucket> effective_bucke
       if (!pipe.source.match_bucket(effective_bucket)) {
         continue;
       }
-      if (!pipe.dest.match_bucket(filter_peer_bucket)) {
-       continue;
-      }
 
       pipe.source.apply_bucket(effective_bucket);
       pipe.dest.apply_bucket(effective_bucket);
@@ -510,8 +503,7 @@ RGWBucketSyncPolicyHandler *RGWBucketSyncPolicyHandler::alloc_child(const rgw_bu
   return new RGWBucketSyncPolicyHandler(this, bucket, sync_policy);
 }
 
-int RGWBucketSyncPolicyHandler::init(std::optional<rgw_bucket> filter_peer_bucket,
-                                    optional_yield y)
+int RGWBucketSyncPolicyHandler::init(optional_yield y)
 {
   int r = bucket_sync_svc->get_bucket_sync_hints(bucket.value_or(rgw_bucket()),
                                                 &source_hints,
@@ -531,7 +523,6 @@ int RGWBucketSyncPolicyHandler::init(std::optional<rgw_bucket> filter_peer_bucke
           &targets,
           &source_zones,
           &target_zones,
-         filter_peer_bucket,
           true);
 
   return 0;
@@ -543,7 +534,6 @@ void RGWBucketSyncPolicyHandler::reflect(RGWBucketSyncFlowManager::pipe_set *pso
                                          map<string, RGWBucketSyncFlowManager::pipe_set> *ptargets,
                                          std::set<string> *psource_zones,
                                          std::set<string> *ptarget_zones,
-                                        std::optional<rgw_bucket> filter_peer_bucket,
                                          bool only_enabled) const
 {
   RGWBucketSyncFlowManager::pipe_set _sources_by_name;
@@ -553,7 +543,7 @@ void RGWBucketSyncPolicyHandler::reflect(RGWBucketSyncFlowManager::pipe_set *pso
   std::set<string> _source_zones;
   std::set<string> _target_zones;
 
-  flow_mgr->reflect(bucket, &_sources_by_name, &_targets_by_name, filter_peer_bucket, only_enabled);
+  flow_mgr->reflect(bucket, &_sources_by_name, &_targets_by_name, only_enabled);
 
   for (auto& pipe : _sources_by_name.pipes) {
     if (!pipe.source.zone) {
@@ -601,6 +591,29 @@ void RGWBucketSyncPolicyHandler::reflect(RGWBucketSyncFlowManager::pipe_set *pso
   }
 }
 
+void RGWBucketSyncPolicyHandler::get_pipes(RGWBucketSyncFlowManager::pipe_set *sources, RGWBucketSyncFlowManager::pipe_set *targets,
+                                          std::optional<rgw_sync_bucket_entity> filter_peer) { /* return raw pipes (with zone name) */
+  if (!filter_peer) {
+    *sources = sources_by_name;
+    *targets = targets_by_name;
+    return;
+  }
+
+  auto& filter = *filter_peer;
+
+  for (auto& source_pipe : sources_by_name.pipes) {
+    if (source_pipe.source.match(filter)) {
+      sources->pipes.insert(source_pipe);
+    }
+  }
+
+  for (auto& target_pipe : targets_by_name.pipes) {
+    if (target_pipe.dest.match(filter)) {
+      targets->pipes.insert(target_pipe);
+    }
+  }
+}
+
 bool RGWBucketSyncPolicyHandler::bucket_exports_data() const
 {
   if (!bucket) {

diff --git a/src/rgw/rgw_bucket_sync.h b/src/rgw/rgw_bucket_sync.h
index 4be4dda26c6384a84c0dedf574b828ff92d6044c..d413ab04e50396967d40c6a1dd753dd2fa9d36d7 100644 (file)
--- a/src/rgw/rgw_bucket_sync.h
+++ b/src/rgw/rgw_bucket_sync.h
@@ -168,7 +168,6 @@ public:
   void reflect(std::optional<rgw_bucket> effective_bucket,
                pipe_set *flow_by_source,
                pipe_set *flow_by_dest,  
-              std::optional<rgw_bucket> filter_peer_bucket,
                bool only_enabled) const;
 
 };
@@ -219,7 +218,7 @@ public:
   RGWBucketSyncPolicyHandler *alloc_child(const rgw_bucket& bucket,
                                           std::optional<rgw_sync_policy_info> sync_policy) const;
 
-  int init(std::optional<rgw_bucket> filter_peer_bucket, optional_yield y);
+  int init(optional_yield y);
 
   void reflect(RGWBucketSyncFlowManager::pipe_set *psources_by_name,
                RGWBucketSyncFlowManager::pipe_set *ptargets_by_name,
@@ -227,7 +226,6 @@ public:
                map<string, RGWBucketSyncFlowManager::pipe_set> *ptargets,
                std::set<string> *psource_zones,
                std::set<string> *ptarget_zones,
-              std::optional<rgw_bucket> filter_peer_bucket,
                bool only_enabled) const;
 
   const std::set<string>& get_source_zones() const {
@@ -254,6 +252,8 @@ public:
     *sources = &sources_by_name;
     *targets = &targets_by_name;
   }
+  void get_pipes(RGWBucketSyncFlowManager::pipe_set *sources, RGWBucketSyncFlowManager::pipe_set *targets,
+                std::optional<rgw_sync_bucket_entity> filter_peer);
 
   const std::set<rgw_bucket>& get_source_hints() const {
     return source_hints;

diff --git a/src/rgw/rgw_sync_policy.h b/src/rgw/rgw_sync_policy.h
index 1233084e2ff6780f7c5e366c0a178dfd03232aab..a869a88acdb191d937f2fe9024de24415123082b 100644 (file)
--- a/src/rgw/rgw_sync_policy.h
+++ b/src/rgw/rgw_sync_policy.h
@@ -131,6 +131,10 @@ struct rgw_sync_bucket_entity {
     zone = z;
   }
 
+  static bool match_bucket_id(const string& bid1, const string& bid2) {
+    return (bid1.empty() || bid2.empty() || (bid1 == bid2));
+  }
+
   bool match_bucket(std::optional<rgw_bucket> b) const {
     if (!b) {
       return true;
@@ -142,7 +146,14 @@ struct rgw_sync_bucket_entity {
 
     return (match_str(bucket->tenant, b->tenant) &&
             match_str(bucket->name, b->name) &&
-            match_str(bucket->bucket_id, b->bucket_id));
+            match_bucket_id(bucket->bucket_id, b->bucket_id));
+  }
+
+  bool match(const rgw_sync_bucket_entity& entity) const {
+    if (!entity.zone) {
+      return match_bucket(entity.bucket);
+    }
+    return (match_zone(*entity.zone) && match_bucket(entity.bucket));
   }
 
   const bool operator<(const rgw_sync_bucket_entity& e) const {

diff --git a/src/rgw/services/svc_bucket_sync_sobj.cc b/src/rgw/services/svc_bucket_sync_sobj.cc
index c849011e236c68ac5e9031521bcf928af0d26cb3..0244b57c95b7e75672fe649c597bc24a5156453b 100644 (file)
--- a/src/rgw/services/svc_bucket_sync_sobj.cc
+++ b/src/rgw/services/svc_bucket_sync_sobj.cc
@@ -687,12 +687,12 @@ int RGWSI_Bucket_Sync_SObj::get_bucket_sync_hints(const rgw_bucket& bucket,
       return r;
     }
 
-    index.get_entities(bucket, dests);
+    index.get_entities(bucket, sources);
 
     if (!bucket.bucket_id.empty()) {
       rgw_bucket b = bucket;
       b.bucket_id.clear();
-      index.get_entities(bucket, dests);
+      index.get_entities(b, sources);
     }
   }
 
@@ -710,7 +710,7 @@ int RGWSI_Bucket_Sync_SObj::get_bucket_sync_hints(const rgw_bucket& bucket,
     if (!bucket.bucket_id.empty()) {
       rgw_bucket b = bucket;
       b.bucket_id.clear();
-      index.get_entities(bucket, dests);
+      index.get_entities(b, dests);
     }
   }