From e016790880ed6d9f802addf97d83508a640e6f6f Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Thu, 13 May 2021 17:11:52 -0400 Subject: [PATCH] rgw: add sync_info to BILog_Status output Needed so we can get the incremental generation. Guard this behind a version check and return the original output if less than 2. Signed-off-by: Adam C. Emerson --- src/rgw/rgw_data_sync.cc | 86 ++++++++++++++++-- src/rgw/rgw_data_sync.h | 8 ++ src/rgw/rgw_json_enc.cc | 187 +-------------------------------------- src/rgw/rgw_rest_log.cc | 66 +++++++++++--- 4 files changed, 142 insertions(+), 205 deletions(-) diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index 0a11e5109136e..ef72d35992557 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -5716,12 +5716,6 @@ void rgw_data_sync_status::generate_test_instances(list& o.push_back(new rgw_data_sync_status); } -void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("position", position, obj); - JSONDecoder::decode_json("count", count, obj); -} - void rgw_bucket_shard_full_sync_marker::dump(Formatter *f) const { encode_json("position", position, f); @@ -5756,6 +5750,12 @@ void rgw_bucket_shard_sync_info::decode_json(JSONObj *obj) JSONDecoder::decode_json("inc_marker", inc_marker, obj); } +void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("position", position, obj); + JSONDecoder::decode_json("count", count, obj); +} + void rgw_bucket_shard_sync_info::dump(Formatter *f) const { const char *s{nullptr}; @@ -5780,3 +5780,77 @@ void rgw_bucket_shard_sync_info::dump(Formatter *f) const encode_json("inc_marker", inc_marker, f); } +void rgw_bucket_full_sync_status::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("position", position, obj); + JSONDecoder::decode_json("count", count, obj); +} + +void rgw_bucket_full_sync_status::dump(Formatter *f) const +{ + encode_json("position", position, f); + encode_json("count", count, f); +} + +void encode_json(const char *name, BucketSyncState state, Formatter *f) +{ + switch (state) { + case BucketSyncState::Init: + encode_json(name, "init", f); + break; + case BucketSyncState::Full: + encode_json(name, "full-sync", f); + break; + case BucketSyncState::Incremental: + encode_json(name, "incremental-sync", f); + break; + case BucketSyncState::Stopped: + encode_json(name, "stopped", f); + break; + default: + encode_json(name, "unknown", f); + break; + } +} + +void decode_json_obj(BucketSyncState& state, JSONObj *obj) +{ + std::string s; + decode_json_obj(s, obj); + if (s == "full-sync") { + state = BucketSyncState::Full; + } else if (s == "incremental-sync") { + state = BucketSyncState::Incremental; + } else if (s == "stopped") { + state = BucketSyncState::Stopped; + } else { + state = BucketSyncState::Init; + } +} + +void rgw_bucket_sync_status::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("state", state, obj); + JSONDecoder::decode_json("full", full, obj); + JSONDecoder::decode_json("incremental_gen", incremental_gen, obj); +} + +void rgw_bucket_sync_status::dump(Formatter *f) const +{ + encode_json("state", state, f); + encode_json("full", full, f); + encode_json("incremental_gen", incremental_gen, f); +} + + +void bilog_status_v2::dump(Formatter *f) const +{ + encode_json("sync_status", sync_status, f); + encode_json("inc_status", inc_status, f); +} + +void bilog_status_v2::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("sync_status", sync_status, obj); + JSONDecoder::decode_json("inc_status", inc_status, obj); +} diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/rgw_data_sync.h index 82e1172ee66b1..a440635ec1aef 100644 --- a/src/rgw/rgw_data_sync.h +++ b/src/rgw/rgw_data_sync.h @@ -639,6 +639,14 @@ struct rgw_bucket_sync_status { }; WRITE_CLASS_ENCODER(rgw_bucket_sync_status) +struct bilog_status_v2 { + rgw_bucket_sync_status sync_status; + std::vector inc_status; + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; + struct rgw_bucket_index_marker_info { std::string bucket_ver; std::string master_ver; diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index b8fe07d3e45c4..b940eb081c695 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -38,12 +38,6 @@ void RGWOLHInfo::dump(Formatter *f) const encode_json("target", target, f); } -void RGWOLHPendingInfo::dump(Formatter *f) const -{ - utime_t ut(time); - encode_json("time", ut, f); -} - void RGWObjManifestPart::dump(Formatter *f) const { f->open_object_section("loc"); @@ -247,26 +241,12 @@ void RGWCacheNotifyInfo::dump(Formatter *f) const encode_json("ns", ns, f); } -void RGWAccessKey::dump(Formatter *f) const -{ - encode_json("access_key", id, f); - encode_json("secret_key", key, f); - encode_json("subuser", subuser, f); -} - void RGWAccessKey::dump_plain(Formatter *f) const { encode_json("access_key", id, f); encode_json("secret_key", key, f); } -void encode_json_plain(const char *name, const RGWAccessKey& val, Formatter *f) -{ - f->open_object_section(name); - val.dump_plain(f); - f->close_section(); -} - void RGWAccessKey::dump(Formatter *f, const string& user, bool swift) const { string u = user; @@ -365,11 +345,6 @@ static struct rgw_flags_desc op_type_flags[] = { { 0, NULL } }; -extern void op_type_to_str(uint32_t mask, char *buf, int len) -{ - return mask_to_str(op_type_flags, mask, buf, len); -} - void RGWSubUser::dump(Formatter *f) const { encode_json("id", name, f); @@ -590,36 +565,13 @@ void rgw_data_placement_target::dump(Formatter *f) const encode_json("data_extra_pool", data_extra_pool, f); encode_json("index_pool", index_pool, f); } - + void rgw_data_placement_target::decode_json(JSONObj *obj) { JSONDecoder::decode_json("data_pool", data_pool, obj); JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj); JSONDecoder::decode_json("index_pool", index_pool, obj); } -void rgw_bucket::dump(Formatter *f) const -{ - encode_json("name", name, f); - encode_json("marker", marker, f); - encode_json("bucket_id", bucket_id, f); - encode_json("tenant", tenant, f); - encode_json("explicit_placement", explicit_placement, f); -} - -void rgw_bucket::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("name", name, obj); - JSONDecoder::decode_json("marker", marker, obj); - JSONDecoder::decode_json("bucket_id", bucket_id, obj); - JSONDecoder::decode_json("tenant", tenant, obj); - JSONDecoder::decode_json("explicit_placement", explicit_placement, obj); - if (explicit_placement.data_pool.empty()) { - /* decoding old format */ - JSONDecoder::decode_json("pool", explicit_placement.data_pool, obj); - JSONDecoder::decode_json("data_extra_pool", explicit_placement.data_extra_pool, obj); - JSONDecoder::decode_json("index_pool", explicit_placement.index_pool, obj); - } -} - void RGWBucketEntryPoint::dump(Formatter *f) const { encode_json("bucket", bucket, f); @@ -1727,131 +1679,6 @@ void rgw_sync_error_info::dump(Formatter *f) const { encode_json("error_code", error_code, f); encode_json("message", message, f); } - -void rgw_bucket_shard_full_sync_marker::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("position", position, obj); - JSONDecoder::decode_json("count", count, obj); -} - -void rgw_bucket_shard_full_sync_marker::dump(Formatter *f) const -{ - encode_json("position", position, f); - encode_json("count", count, f); -} - -void rgw_bucket_shard_inc_sync_marker::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("position", position, obj); - JSONDecoder::decode_json("timestamp", timestamp, obj); -} - -void rgw_bucket_shard_inc_sync_marker::dump(Formatter *f) const -{ - encode_json("position", position, f); - encode_json("timestamp", timestamp, f); -} - -void rgw_bucket_shard_sync_info::decode_json(JSONObj *obj) -{ - std::string s; - JSONDecoder::decode_json("status", s, obj); - if (s == "full-sync") { - state = StateFullSync; - } else if (s == "incremental-sync") { - state = StateIncrementalSync; - } else if (s == "stopped") { - state = StateStopped; - } else { - state = StateInit; - } - JSONDecoder::decode_json("inc_marker", inc_marker, obj); -} - -void rgw_bucket_shard_sync_info::dump(Formatter *f) const -{ - const char *s{nullptr}; - switch ((SyncState)state) { - case StateInit: - s = "init"; - break; - case StateFullSync: - s = "full-sync"; - break; - case StateIncrementalSync: - s = "incremental-sync"; - break; - case StateStopped: - s = "stopped"; - break; - default: - s = "unknown"; - break; - } - encode_json("status", s, f); - encode_json("inc_marker", inc_marker, f); -} - -void rgw_bucket_full_sync_status::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("position", position, obj); - JSONDecoder::decode_json("count", count, obj); -} - -void rgw_bucket_full_sync_status::dump(Formatter *f) const -{ - encode_json("position", position, f); - encode_json("count", count, f); -} - -void encode_json(const char *name, BucketSyncState state, Formatter *f) -{ - switch (state) { - case BucketSyncState::Init: - encode_json(name, "init", f); - break; - case BucketSyncState::Full: - encode_json(name, "full-sync", f); - break; - case BucketSyncState::Incremental: - encode_json(name, "incremental-sync", f); - break; - case BucketSyncState::Stopped: - encode_json(name, "stopped", f); - break; - default: - encode_json(name, "unknown", f); - break; - } -} - -void decode_json_obj(BucketSyncState& state, JSONObj *obj) -{ - std::string s; - decode_json_obj(s, obj); - if (s == "full-sync") { - state = BucketSyncState::Full; - } else if (s == "incremental-sync") { - state = BucketSyncState::Incremental; - } else if (s == "stopped") { - state = BucketSyncState::Stopped; - } else { - state = BucketSyncState::Init; - } -} - -void rgw_bucket_sync_status::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("state", state, obj); - JSONDecoder::decode_json("full", full, obj); -} - -void rgw_bucket_sync_status::dump(Formatter *f) const -{ - encode_json("state", state, f); - encode_json("full", full, f); -} - /* This utility function shouldn't conflict with the overload of std::to_string * provided by string_ref since Boost 1.54 as it's defined outside of the std * namespace. I hope we'll remove it soon - just after merging the Matt's PR @@ -1909,18 +1736,6 @@ void rgw::keystone::AdminTokenRequestVer3::dump(Formatter* const f) const } -void rgw::keystone::BarbicanTokenRequestVer2::dump(Formatter* const f) const -{ - f->open_object_section("token_request"); - f->open_object_section("auth"); - f->open_object_section("passwordCredentials"); - encode_json("username", cct->_conf->rgw_keystone_barbican_user, f); - encode_json("password", cct->_conf->rgw_keystone_barbican_password, f); - f->close_section(); - encode_json("tenantName", cct->_conf->rgw_keystone_barbican_tenant, f); - f->close_section(); - f->close_section(); -} void rgw::keystone::BarbicanTokenRequestVer3::dump(Formatter* const f) const { diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index e42328a3cda70..7f3302de8c88a 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -26,6 +26,7 @@ #include "rgw_zone.h" #include "rgw_mdlog.h" #include "rgw_datalog_notify.h" +#include "rgw_trim_bilog.h" #include "services/svc_zone.h" #include "services/svc_mdlog.h" @@ -919,7 +920,8 @@ void RGWOp_MDLog_Status::send_response() // not in header to avoid pulling in rgw_data_sync.h class RGWOp_BILog_Status : public RGWRESTOp { - std::vector status; + bilog_status_v2 status; + int version = 1; public: int check_caps(const RGWUserCaps& caps) override { return caps.check_cap("bilog", RGW_CAP_READ); @@ -939,6 +941,8 @@ void RGWOp_BILog_Status::execute(optional_yield y) const auto source_zone = s->info.args.get("source-zone"); const auto source_key = s->info.args.get("source-bucket"); auto key = s->info.args.get("bucket"); + op_ret = s->info.args.get_int("version", &version, 1); + if (key.empty()) { key = source_key; } @@ -989,11 +993,28 @@ void RGWOp_BILog_Status::execute(optional_yield y) ldpp_dout(this, 20) << "RGWOp_BILog_Status::execute(optional_yield y): getting sync status for pipe=" << pipe << dendl; - op_ret = rgw_read_bucket_inc_sync_status(this, static_cast(store), - pipe, bucket->get_info(), nullptr, &status); + if (version > 1) { + op_ret = rgw_read_bucket_full_sync_status( + this, + static_cast(store), + pipe, + &status.sync_status, + s->yield); + if (op_ret < 0) { + ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_full_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl; + return; + } + } + op_ret = rgw_read_bucket_inc_sync_status( + this, + static_cast(store), + pipe, + bucket->get_info(), + nullptr, + &status.inc_status); if (op_ret < 0) { - ldpp_dout(this, -1) << "ERROR: rgw_bucket_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl; + ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_inc_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl; } return; } @@ -1039,24 +1060,39 @@ void RGWOp_BILog_Status::execute(optional_yield y) pipe.dest.bucket = pinfo->bucket; } + if (version > 1) { + op_ret = rgw_read_bucket_full_sync_status( + this, + static_cast(store), + pipe, + &status.sync_status, + s->yield); + if (op_ret < 0) { + ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_full_sync_status() on pipe=" << pipe << " returned ret=" << op_ret << dendl; + return; + } + } int r = rgw_read_bucket_inc_sync_status(this, static_cast(store), pipe, *pinfo, &bucket->get_info(), ¤t_status); if (r < 0) { - ldpp_dout(this, -1) << "ERROR: rgw_bucket_sync_status() on pipe=" << pipe << " returned ret=" << r << dendl; + ldpp_dout(this, -1) << "ERROR: rgw_read_bucket_inc_sync_status() on pipe=" << pipe << " returned ret=" << r << dendl; op_ret = r; return; } - if (status.empty()) { - status = std::move(current_status); + if (status.inc_status.empty()) { + status.inc_status = std::move(current_status); } else { - if (current_status.size() != - status.size()) { + if (current_status.size() != status.inc_status.size()) { op_ret = -EINVAL; - ldpp_dout(this, -1) << "ERROR: different number of shards for sync status of buckets syncing from the same source: status.size()= " << status.size() << " current_status.size()=" << current_status.size() << dendl; - return; + ldpp_dout(this, -1) << "ERROR: different number of shards for sync status of buckets " + "syncing from the same source: status.size()= " + << status.inc_status.size() + << " current_status.size()=" + << current_status.size() << dendl; + return; } - auto m = status.begin(); + auto m = status.inc_status.begin(); for (auto& cur_shard_status : current_status) { auto& result_shard_status = *m++; // always take the first marker, or any later marker that's smaller @@ -1075,7 +1111,11 @@ void RGWOp_BILog_Status::send_response() end_header(s); if (op_ret >= 0) { - encode_json("status", status, s->formatter); + if (version < 2) { + encode_json("status", status.inc_status, s->formatter); + } else { + encode_json("status", status, s->formatter); + } } flusher.flush(); } -- 2.39.5