rgw-admin: bucket sync status related fixes and improvements

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 1811d937a1c159fe66a41ce494b56e3ee706a557..4f0ff82954fcdfa4ed99264b5fb7eb799d5d8323 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2259,20 +2259,23 @@ static int bucket_source_sync_status(rgw::sal::RGWRadosStore *store, const RGWZo
                                      const rgw_sync_bucket_pipe& pipe,
                                      int width, std::ostream& out)
 {
-  out << indented{width, "source zone"} << source.id << " (" << source.name << ")\n";
+  out << indented{width, "source zone"} << source.id << " (" << source.name << ")" << std::endl;
 
   // syncing from this zone?
   if (!zone.syncs_from(source.name)) {
     out << indented{width} << "does not sync from zone\n";
     return 0;
   }
+  RGWBucketInfo source_bucket_info;
   std::vector<rgw_bucket_shard_sync_info> status;
-  int r = rgw_bucket_sync_status(dpp(), store, pipe, bucket_info, &status);
+  int r = rgw_bucket_sync_status(dpp(), store, pipe, bucket_info, &source_bucket_info, &status);
   if (r < 0) {
     lderr(store->ctx()) << "failed to read bucket sync status: " << cpp_strerror(r) << dendl;
     return r;
   }
 
+  out << indented{width, "source bucket"} << source_bucket_info.bucket.get_key() << std::endl;
+
   int num_full = 0;
   int num_inc = 0;
   uint64_t full_complete = 0;
@@ -2296,7 +2299,7 @@ static int bucket_source_sync_status(rgw::sal::RGWRadosStore *store, const RGWZo
   out << indented{width} << "incremental sync: " << num_inc << "/" << total_shards << " shards\n";
 
   BucketIndexShardsManager remote_markers;
-  r = remote_bilog_markers(store, source, conn, bucket_info, &remote_markers);
+  r = remote_bilog_markers(store, source, conn, source_bucket_info, &remote_markers);
   if (r < 0) {
     lderr(store->ctx()) << "failed to read remote log: " << cpp_strerror(r) << dendl;
     return r;
@@ -2537,6 +2540,7 @@ static int bucket_sync_info(rgw::sal::RGWRadosStore *store, const RGWBucketInfo&
 
 static int bucket_sync_status(rgw::sal::RGWRadosStore *store, const RGWBucketInfo& info,
                               const rgw_zone_id& source_zone_id,
+                             std::optional<rgw_bucket>& opt_source_bucket,
                               std::ostream& out)
 {
   const RGWRealm& realm = store->svc()->zone->get_realm();
@@ -2550,7 +2554,7 @@ static int bucket_sync_status(rgw::sal::RGWRadosStore *store, const RGWBucketInf
   out << indented{width, "bucket"} << info.bucket << "\n\n";
 
   if (!store->ctl()->bucket->bucket_imports_data(info.bucket, null_yield)) {
-    out << "Sync is disabled for bucket " << info.bucket.name << '\n';
+    out << "Sync is disabled for bucket " << info.bucket.name << " or bucket has no sync sources" << std::endl;
     return 0;
   }
 
@@ -2562,7 +2566,7 @@ static int bucket_sync_status(rgw::sal::RGWRadosStore *store, const RGWBucketInf
     return r;
   }
 
-  auto& sources = handler->get_sources();
+  auto sources = handler->get_all_sources();
 
   auto& zone_conn_map = store->svc()->zone->get_zone_conn_map();
   set<rgw_zone_id> zone_ids;
@@ -2582,11 +2586,7 @@ static int bucket_sync_status(rgw::sal::RGWRadosStore *store, const RGWBucketInf
     zone_ids.insert(source_zone_id);
   } else {
     for (const auto& entry : zonegroup.zones) {
-      auto c = zone_conn_map.find(entry.second.id);
-      if (c == zone_conn_map.end()) {
-        continue;
-      }
-      zone_ids.insert(entry.second.name);
+      zone_ids.insert(entry.second.id);
     }
   }
 
@@ -2595,20 +2595,22 @@ static int bucket_sync_status(rgw::sal::RGWRadosStore *store, const RGWBucketInf
     if (z == zonegroup.zones.end()) { /* should't happen */
       continue;
     }
-    auto c = zone_conn_map.find(source_zone_id);
+    auto c = zone_conn_map.find(zone_id.id);
     if (c == zone_conn_map.end()) { /* should't happen */
       continue;
     }
 
-    for (auto& m : sources) {
-      for (auto& entry : m.second.pipe_map) {
-        auto& pipe = entry.second;
-        if (pipe.source.zone.value_or(rgw_zone_id()) == z->second.id) {
-          bucket_source_sync_status(store, zone, z->second,
-                                    c->second,
-                                    info, pipe,
-                                    width, out);
-        }
+    for (auto& entry : sources) {
+      auto& pipe = entry.second;
+      if (opt_source_bucket &&
+         pipe.source.bucket != opt_source_bucket) {
+       continue;
+      }
+      if (pipe.source.zone.value_or(rgw_zone_id()) == z->second.id) {
+       bucket_source_sync_status(store, zone, z->second,
+                                 c->second,
+                                 info, pipe,
+                                 width, out);
       }
     }
   }
@@ -7864,7 +7866,7 @@ next:
     if (ret < 0) {
       return -ret;
     }
-    bucket_sync_status(store, bucket_info, source_zone, std::cout);
+    bucket_sync_status(store, bucket_info, source_zone, opt_source_bucket, std::cout);
   }
 
   if (opt_cmd == OPT::BUCKET_SYNC_MARKERS) {

diff --git a/src/rgw/rgw_bucket_sync.cc b/src/rgw/rgw_bucket_sync.cc
index f54e3476ab569b43f290c12b908d9d7def9cf9eb..4095804baf03216b987ea699d7b4bffed9b9e412 100644 (file)
--- a/src/rgw/rgw_bucket_sync.cc
+++ b/src/rgw/rgw_bucket_sync.cc
@@ -801,6 +801,32 @@ void RGWBucketSyncPolicyHandler::reflect(RGWBucketSyncFlowManager::pipe_set *pso
   }
 }
 
+multimap<rgw_zone_id, rgw_sync_bucket_pipe> RGWBucketSyncPolicyHandler::get_all_sources()
+{
+  multimap<rgw_zone_id, rgw_sync_bucket_pipe> m;
+
+  for (auto& source_entry : sources) {
+    auto& zone_id = source_entry.first;
+
+    auto& pipes = source_entry.second.pipe_map;
+
+    for (auto& entry : pipes) {
+      auto& pipe = entry.second;
+      m.insert(make_pair(zone_id, pipe));
+    }
+  }
+
+  for (auto& pipe : resolved_sources) {
+    if (!pipe.source.zone) {
+      continue;
+    }
+
+    m.insert(make_pair(*pipe.source.zone, pipe));
+  }
+
+  return std::move(m);
+}
+
 void RGWBucketSyncPolicyHandler::get_pipes(std::set<rgw_sync_bucket_pipe> *_sources, std::set<rgw_sync_bucket_pipe> *_targets,
                                            std::optional<rgw_sync_bucket_entity> filter_peer) { /* return raw pipes */
   for (auto& entry : source_pipes.pipe_map) {

diff --git a/src/rgw/rgw_bucket_sync.h b/src/rgw/rgw_bucket_sync.h
index 27d5f8bb1bc9db06a109b8df5be02591365626be..6f7cdbf00e517eee6cf8810ca5559e060810bc73 100644 (file)
--- a/src/rgw/rgw_bucket_sync.h
+++ b/src/rgw/rgw_bucket_sync.h
@@ -362,6 +362,8 @@ public:
     return sources;
   }
 
+  multimap<rgw_zone_id, rgw_sync_bucket_pipe> get_all_sources();
+
   const  map<rgw_zone_id, RGWBucketSyncFlowManager::pipe_set>& get_targets() {
     return targets;
   }

diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index a029c090d638ea4277fd24b7b892aa30b7d23dae..91fc9e1b252cff92b5127d4630665333e4c79649 100644 (file)
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -4307,6 +4307,10 @@ int RGWRunBucketSourcesSyncCR::operate()
       ldpp_dout(sync_env->dpp, 20) << __func__ << "(): num shards=" << num_shards << " cur_shard=" << cur_shard << dendl;
 
       for (; num_shards > 0; --num_shards, ++cur_shard) {
+        /*
+         * use a negatvie shard_id for backward compatibility,
+         * this affects the crafted status oid
+         */
         sync_pair.source_bs.shard_id = (source_num_shards > 0 ? cur_shard : -1);
         if (source_num_shards == target_num_shards) {
           sync_pair.dest_bs.shard_id = sync_pair.source_bs.shard_id;
@@ -4891,6 +4895,7 @@ int rgw_bucket_sync_status(const DoutPrefixProvider *dpp,
                            rgw::sal::RGWRadosStore *store,
                            const rgw_sync_bucket_pipe& pipe,
                            const RGWBucketInfo& dest_bucket_info,
+                           RGWBucketInfo *psource_bucket_info,
                            std::vector<rgw_bucket_shard_sync_info> *status)
 {
   if (!pipe.source.zone ||
@@ -4909,14 +4914,18 @@ int rgw_bucket_sync_status(const DoutPrefixProvider *dpp,
 
   RGWBucketInfo source_bucket_info;
 
-  auto obj_ctx = store->svc()->sysobj->init_obj_ctx();
+  auto& bucket_ctl = store->getRados()->ctl.bucket;
 
-  int ret = store->getRados()->get_bucket_instance_info(obj_ctx, source_bucket, source_bucket_info, nullptr, nullptr, null_yield);
+  int ret = bucket_ctl->read_bucket_info(source_bucket, &source_bucket_info, null_yield);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: failed to get bucket instance info: bucket=" << source_bucket << ": " << cpp_strerror(-ret) << dendl;
     return ret;
   }
 
+  if (psource_bucket_info) {
+    *psource_bucket_info = source_bucket_info;
+  }
+
   RGWDataSyncEnv env;
   RGWSyncModuleInstanceRef module; // null sync module
   env.init(dpp, store->ctx(), store, store->svc(), store->svc()->rados->get_async_processor(),

diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/rgw_data_sync.h
index abf112e36c08e5ff0eccd7fe6a4b2407510e7a65..6e841ca8a5ab5915baf159a455987e5b55392f42 100644 (file)
--- a/src/rgw/rgw_data_sync.h
+++ b/src/rgw/rgw_data_sync.h
@@ -650,6 +650,7 @@ int rgw_bucket_sync_status(const DoutPrefixProvider *dpp,
                            rgw::sal::RGWRadosStore *store,
                            const rgw_sync_bucket_pipe& pipe,
                            const RGWBucketInfo& dest_bucket_info,
+                           RGWBucketInfo *psource_bucket_info,
                            std::vector<rgw_bucket_shard_sync_info> *status);
 
 class RGWDefaultSyncModule : public RGWSyncModule {

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 82256e32ec581befeffe5e9389cb6ca425a5b9c7..53ac3f386536b97a9a7689e0b5e3c4dc34590e71 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -862,7 +862,7 @@ void RGWOp_BILog_Status::execute()
   pipe.source.bucket = info.bucket;
   pipe.dest.zone = store->svc()->zone->zone_id();
   pipe.dest.bucket = info.bucket;
-  http_ret = rgw_bucket_sync_status(this, store, pipe, info, &status);
+  http_ret = rgw_bucket_sync_status(this, store, pipe, info, nullptr, &status);
 }
 
 void RGWOp_BILog_Status::send_response()