rgw: add sync_info to BILog_Status output

Needed so we can get the incremental generation.

Guard this behind a version check and return the original output if
less than 2.

Signed-off-by: Adam C. Emerson <aemerson@redhat.com>

diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index 0a11e5109136e96c9992705481c8bd704eb898cf..ef72d35992557481ae904469dae9c39844661d73 100644 (file)
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -5716,12 +5716,6 @@ void rgw_data_sync_status::generate_test_instances(list<rgw_data_sync_status*>&
   o.push_back(new rgw_data_sync_status);
 }
 
-void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj)
-{
-  JSONDecoder::decode_json("position", position, obj);
-  JSONDecoder::decode_json("count", count, obj);
-}
-
 void rgw_bucket_shard_full_sync_marker::dump(Formatter *f) const
 {
   encode_json("position", position, f);
@@ -5756,6 +5750,12 @@ void rgw_bucket_shard_sync_info::decode_json(JSONObj *obj)
   JSONDecoder::decode_json("inc_marker", inc_marker, obj);
 }
 
+void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("position", position, obj);
+  JSONDecoder::decode_json("count", count, obj);
+}
+
 void rgw_bucket_shard_sync_info::dump(Formatter *f) const
 {
   const char *s{nullptr};
@@ -5780,3 +5780,77 @@ void rgw_bucket_shard_sync_info::dump(Formatter *f) const
   encode_json("inc_marker", inc_marker, f);
 }
 
+void rgw_bucket_full_sync_status::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("position", position, obj);
+  JSONDecoder::decode_json("count", count, obj);
+}
+
+void rgw_bucket_full_sync_status::dump(Formatter *f) const
+{
+  encode_json("position", position, f);
+  encode_json("count", count, f);
+}
+
+void encode_json(const char *name, BucketSyncState state, Formatter *f)
+{
+  switch (state) {
+  case BucketSyncState::Init:
+    encode_json(name, "init", f);
+    break;
+  case BucketSyncState::Full:
+    encode_json(name, "full-sync", f);
+    break;
+  case BucketSyncState::Incremental:
+    encode_json(name, "incremental-sync", f);
+    break;
+  case BucketSyncState::Stopped:
+    encode_json(name, "stopped", f);
+    break;
+  default:
+    encode_json(name, "unknown", f);
+    break;
+  }
+}
+
+void decode_json_obj(BucketSyncState& state, JSONObj *obj)
+{
+  std::string s;
+  decode_json_obj(s, obj);
+  if (s == "full-sync") {
+    state = BucketSyncState::Full;
+  } else if (s == "incremental-sync") {
+    state = BucketSyncState::Incremental;
+  } else if (s == "stopped") {
+    state = BucketSyncState::Stopped;
+  } else {
+    state = BucketSyncState::Init;
+  }
+}
+
+void rgw_bucket_sync_status::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("state", state, obj);
+  JSONDecoder::decode_json("full", full, obj);
+  JSONDecoder::decode_json("incremental_gen", incremental_gen, obj);
+}
+
+void rgw_bucket_sync_status::dump(Formatter *f) const
+{
+  encode_json("state", state, f);
+  encode_json("full", full, f);
+  encode_json("incremental_gen", incremental_gen, f);
+}
+
+
+void bilog_status_v2::dump(Formatter *f) const
+{
+  encode_json("sync_status", sync_status, f);
+  encode_json("inc_status", inc_status, f);
+}
+
+void bilog_status_v2::decode_json(JSONObj *obj)
+{
+  JSONDecoder::decode_json("sync_status", sync_status, obj);
+  JSONDecoder::decode_json("inc_status", inc_status, obj);
+}

diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/rgw_data_sync.h
index 82e1172ee66b1940e542a80385c85050854820e2..a440635ec1aefc15961f2fc90a3859fa3d2d6264 100644 (file)
--- a/src/rgw/rgw_data_sync.h
+++ b/src/rgw/rgw_data_sync.h
@@ -639,6 +639,14 @@ struct rgw_bucket_sync_status {
 };
 WRITE_CLASS_ENCODER(rgw_bucket_sync_status)
 
+struct bilog_status_v2 {
+  rgw_bucket_sync_status sync_status;
+  std::vector<rgw_bucket_shard_sync_info> inc_status;
+
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+
 struct rgw_bucket_index_marker_info {
   std::string bucket_ver;
   std::string master_ver;

diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc
index b8fe07d3e45c404e3a96fe29d8383700b2ef4668..b940eb081c695f04de6596e8b24881fbb1c0afad 100644 (file)
--- a/src/rgw/rgw_json_enc.cc
+++ b/src/rgw/rgw_json_enc.cc
@@ -38,12 +38,6 @@ void RGWOLHInfo::dump(Formatter *f) const
   encode_json("target", target, f);
 }
 
-void RGWOLHPendingInfo::dump(Formatter *f) const
-{
-  utime_t ut(time);
-  encode_json("time", ut, f);
-}
-
 void RGWObjManifestPart::dump(Formatter *f) const
 {
   f->open_object_section("loc");
@@ -247,26 +241,12 @@ void RGWCacheNotifyInfo::dump(Formatter *f) const
   encode_json("ns", ns, f);
 }
 
-void RGWAccessKey::dump(Formatter *f) const
-{
-  encode_json("access_key", id, f);
-  encode_json("secret_key", key, f);
-  encode_json("subuser", subuser, f);
-}
-
 void RGWAccessKey::dump_plain(Formatter *f) const
 {
   encode_json("access_key", id, f);
   encode_json("secret_key", key, f);
 }
 
-void encode_json_plain(const char *name, const RGWAccessKey& val, Formatter *f)
-{
-  f->open_object_section(name);
-  val.dump_plain(f);
-  f->close_section();
-}
-
 void RGWAccessKey::dump(Formatter *f, const string& user, bool swift) const
 {
   string u = user;
@@ -365,11 +345,6 @@ static struct rgw_flags_desc op_type_flags[] = {
  { 0, NULL }
 };
 
-extern void op_type_to_str(uint32_t mask, char *buf, int len)
-{
-  return mask_to_str(op_type_flags, mask, buf, len);
-}
-
 void RGWSubUser::dump(Formatter *f) const
 {
   encode_json("id", name, f);
@@ -590,36 +565,13 @@ void rgw_data_placement_target::dump(Formatter *f) const
   encode_json("data_extra_pool", data_extra_pool, f);
   encode_json("index_pool", index_pool, f);
 }
-  
+
 void rgw_data_placement_target::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("data_pool", data_pool, obj);
   JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj);
   JSONDecoder::decode_json("index_pool", index_pool, obj);
 }
 
-void rgw_bucket::dump(Formatter *f) const
-{
-  encode_json("name", name, f);
-  encode_json("marker", marker, f);
-  encode_json("bucket_id", bucket_id, f);
-  encode_json("tenant", tenant, f);
-  encode_json("explicit_placement", explicit_placement, f);
-}
-
-void rgw_bucket::decode_json(JSONObj *obj) {
-  JSONDecoder::decode_json("name", name, obj);
-  JSONDecoder::decode_json("marker", marker, obj);
-  JSONDecoder::decode_json("bucket_id", bucket_id, obj);
-  JSONDecoder::decode_json("tenant", tenant, obj);
-  JSONDecoder::decode_json("explicit_placement", explicit_placement, obj);
-  if (explicit_placement.data_pool.empty()) {
-    /* decoding old format */
-    JSONDecoder::decode_json("pool", explicit_placement.data_pool, obj);
-    JSONDecoder::decode_json("data_extra_pool", explicit_placement.data_extra_pool, obj);
-    JSONDecoder::decode_json("index_pool", explicit_placement.index_pool, obj);
-  }
-}
-
 void RGWBucketEntryPoint::dump(Formatter *f) const
 {
   encode_json("bucket", bucket, f);
@@ -1727,131 +1679,6 @@ void rgw_sync_error_info::dump(Formatter *f) const {
   encode_json("error_code", error_code, f);
   encode_json("message", message, f);
 }
-
-void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj)
-{
-  JSONDecoder::decode_json("position", position, obj);
-  JSONDecoder::decode_json("count", count, obj);
-}
-
-void rgw_bucket_shard_full_sync_marker::dump(Formatter *f) const
-{
-  encode_json("position", position, f);
-  encode_json("count", count, f);
-}
-
-void rgw_bucket_shard_inc_sync_marker::decode_json(JSONObj *obj)
-{
-  JSONDecoder::decode_json("position", position, obj);
-  JSONDecoder::decode_json("timestamp", timestamp, obj);
-}
-
-void rgw_bucket_shard_inc_sync_marker::dump(Formatter *f) const
-{
-  encode_json("position", position, f);
-  encode_json("timestamp", timestamp, f);
-}
-
-void rgw_bucket_shard_sync_info::decode_json(JSONObj *obj)
-{
-  std::string s;
-  JSONDecoder::decode_json("status", s, obj);
-  if (s == "full-sync") {
-    state = StateFullSync;
-  } else if (s == "incremental-sync") {
-    state = StateIncrementalSync;
-  } else if (s == "stopped") {
-    state = StateStopped;
-  } else {
-    state = StateInit;
-  }
-  JSONDecoder::decode_json("inc_marker", inc_marker, obj);
-}
-
-void rgw_bucket_shard_sync_info::dump(Formatter *f) const
-{
-  const char *s{nullptr};
-  switch ((SyncState)state) {
-    case StateInit:
-    s = "init";
-    break;
-  case StateFullSync:
-    s = "full-sync";
-    break;
-  case StateIncrementalSync:
-    s = "incremental-sync";
-    break;
-  case StateStopped:
-    s = "stopped";
-    break;
-  default:
-    s = "unknown";
-    break;
-  }
-  encode_json("status", s, f);
-  encode_json("inc_marker", inc_marker, f);
-}
-
-void rgw_bucket_full_sync_status::decode_json(JSONObj *obj)
-{
-  JSONDecoder::decode_json("position", position, obj);
-  JSONDecoder::decode_json("count", count, obj);
-}
-
-void rgw_bucket_full_sync_status::dump(Formatter *f) const
-{
-  encode_json("position", position, f);
-  encode_json("count", count, f);
-}
-
-void encode_json(const char *name, BucketSyncState state, Formatter *f)
-{
-  switch (state) {
-  case BucketSyncState::Init:
-    encode_json(name, "init", f);
-    break;
-  case BucketSyncState::Full:
-    encode_json(name, "full-sync", f);
-    break;
-  case BucketSyncState::Incremental:
-    encode_json(name, "incremental-sync", f);
-    break;
-  case BucketSyncState::Stopped:
-    encode_json(name, "stopped", f);
-    break;
-  default:
-    encode_json(name, "unknown", f);
-    break;
-  }
-}
-
-void decode_json_obj(BucketSyncState& state, JSONObj *obj)
-{
-  std::string s;
-  decode_json_obj(s, obj);
-  if (s == "full-sync") {
-    state = BucketSyncState::Full;
-  } else if (s == "incremental-sync") {
-    state = BucketSyncState::Incremental;
-  } else if (s == "stopped") {
-    state = BucketSyncState::Stopped;
-  } else {
-    state = BucketSyncState::Init;
-  }
-}
-
-void rgw_bucket_sync_status::decode_json(JSONObj *obj)
-{
-  JSONDecoder::decode_json("state", state, obj);
-  JSONDecoder::decode_json("full", full, obj);
-}
-
-void rgw_bucket_sync_status::dump(Formatter *f) const
-{
-  encode_json("state", state, f);
-  encode_json("full", full, f);
-}
-
 /* This utility function shouldn't conflict with the overload of std::to_string
  * provided by string_ref since Boost 1.54 as it's defined outside of the std
  * namespace. I hope we'll remove it soon - just after merging the Matt's PR
@@ -1909,18 +1736,6 @@ void rgw::keystone::AdminTokenRequestVer3::dump(Formatter* const f) const
 }
 
 
-void rgw::keystone::BarbicanTokenRequestVer2::dump(Formatter* const f) const
-{
-  f->open_object_section("token_request");
-    f->open_object_section("auth");
-      f->open_object_section("passwordCredentials");
-        encode_json("username", cct->_conf->rgw_keystone_barbican_user, f);
-        encode_json("password", cct->_conf->rgw_keystone_barbican_password, f);
-      f->close_section();
-      encode_json("tenantName", cct->_conf->rgw_keystone_barbican_tenant, f);
-    f->close_section();
-  f->close_section();
-}
 
 void rgw::keystone::BarbicanTokenRequestVer3::dump(Formatter* const f) const
 {

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index e42328a3cda705c389b095be3e670574de7d5aee..7f3302de8c88a504fc3066e50d39d35ec6f18f7f 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -26,6 +26,7 @@
 #include "rgw_zone.h"
 #include "rgw_mdlog.h"
 #include "rgw_datalog_notify.h"
+#include "rgw_trim_bilog.h"
 
 #include "services/svc_zone.h"
 #include "services/svc_mdlog.h"
@@ -919,7 +920,8 @@ void RGWOp_MDLog_Status::send_response()
 
 // not in header to avoid pulling in rgw_data_sync.h
 class RGWOp_BILog_Status : public RGWRESTOp {
-  std::vector<rgw_bucket_shard_sync_info> status;
+  bilog_status_v2 status;
+  int version = 1;
 public:
   int check_caps(const RGWUserCaps& caps) override {
     return caps.check_cap("bilog", RGW_CAP_READ);
@@ -939,6 +941,8 @@ void RGWOp_BILog_Status::execute(optional_yield y)
   const auto source_zone = s->info.args.get("source-zone");
   const auto source_key = s->info.args.get("source-bucket");
   auto key = s->info.args.get("bucket");
+  op_ret = s->info.args.get_int("version", &version, 1);
+
   if (key.empty()) {
     key = source_key;
   }
@@ -989,11 +993,28 @@ void RGWOp_BILog_Status::execute(optional_yield y)
 
     ldpp_dout(this, 20) << "RGWOp_BILog_Status::execute(optional_yield y): getting sync status for pipe=" << pipe << dendl;
 
-    op_ret = rgw_read_bucket_inc_sync_status(this, static_cast<rgw::sal::RadosStore*>(store),
-                                            pipe, bucket->get_info(), nullptr, &status);
+    if (version > 1) {
+      op_ret = rgw_read_bucket_full_sync_status(
+       this,
+       static_cast<rgw::sal::RadosStore*>(store),
+       pipe,
+       &status.sync_status,
+       s->yield);
+      if (op_ret < 0) {
+       ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_full_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl;
+       return;
+      }
+    }
 
+    op_ret = rgw_read_bucket_inc_sync_status(
+      this,
+      static_cast<rgw::sal::RadosStore*>(store),
+      pipe,
+      bucket->get_info(),
+      nullptr,
+      &status.inc_status);
     if (op_ret < 0) {
-      ldpp_dout(this, -1) << "ERROR: rgw_bucket_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl;
+      ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_inc_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl;
     }
     return;
   }
@@ -1039,24 +1060,39 @@ void RGWOp_BILog_Status::execute(optional_yield y)
       pipe.dest.bucket = pinfo->bucket;
     }
 
+    if (version > 1) {
+      op_ret = rgw_read_bucket_full_sync_status(
+       this,
+       static_cast<rgw::sal::RadosStore*>(store),
+       pipe,
+       &status.sync_status,
+       s->yield);
+      if (op_ret < 0) {
+       ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_full_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl;
+       return;
+      }
+    }
     int r = rgw_read_bucket_inc_sync_status(this, static_cast<rgw::sal::RadosStore*>(store),
                                            pipe, *pinfo, &bucket->get_info(), &current_status);
     if (r < 0) {
-      ldpp_dout(this, -1) << "ERROR: rgw_bucket_sync_status() on pipe=" << pipe << " returned ret=" << r << dendl;
+      ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_inc_sync_status() on pipe=" << pipe << " returned ret=" << r << dendl;
       op_ret = r;
       return;
     }
 
-    if (status.empty()) {
-      status = std::move(current_status);
+    if (status.inc_status.empty()) {
+      status.inc_status = std::move(current_status);
     } else {
-      if (current_status.size() !=
-          status.size()) {
+      if (current_status.size() != status.inc_status.size()) {
         op_ret = -EINVAL;
-        ldpp_dout(this, -1) << "ERROR: different number of shards for sync status of buckets syncing from the same source: status.size()= " << status.size() << " current_status.size()=" << current_status.size() << dendl;
-        return;
+        ldpp_dout(this, -1) << "ERROR: different number of shards for sync status of buckets "
+         "syncing from the same source: status.size()= "
+                           << status.inc_status.size()
+                           << " current_status.size()="
+                           << current_status.size() << dendl;
+       return;
       }
-      auto m = status.begin();
+      auto m = status.inc_status.begin();
       for (auto& cur_shard_status : current_status) {
         auto& result_shard_status = *m++;
         // always take the first marker, or any later marker that's smaller
@@ -1075,7 +1111,11 @@ void RGWOp_BILog_Status::send_response()
   end_header(s);
 
   if (op_ret >= 0) {
-    encode_json("status", status, s->formatter);
+    if (version < 2) {
+      encode_json("status", status.inc_status, s->formatter);
+    } else {
+      encode_json("status", status, s->formatter);
+    }
   }
   flusher.flush();
 }