From eb93cc36d1fdd9dfae71a1a6146e3fe861afc27e Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Mon, 18 Nov 2019 10:38:25 -0800 Subject: [PATCH] rgw: sync policy: reorganize a bit Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_admin.cc | 62 +++++++++++++++++++++++++++++--------- src/rgw/rgw_bucket_sync.cc | 6 ++-- src/rgw/rgw_common.cc | 2 +- src/rgw/rgw_common.h | 2 +- src/rgw/rgw_json_enc.cc | 18 ++++++++--- src/rgw/rgw_sync_policy.h | 34 ++++++++++++++++----- 6 files changed, 94 insertions(+), 30 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 43768e79686..89e5f0db8e3 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -2961,6 +2961,37 @@ static vector zone_ids_from_str(const string& val) return result; } +class JSONFormatter_PrettyZone : public JSONFormatter { + class Handler : public JSONEncodeFilter::Handler { + void encode_json(const char *name, const void *pval, ceph::Formatter *f) const override { + auto zone_id = *(static_cast(pval)); + string zone_name; + RGWZone *zone; + if (store->svc()->zone->find_zone(zone_id, &zone)) { + zone_name = zone->name; + } else { + cerr << "WARNING: cannot find zone name for id=" << zone_id << std::endl; + zone_name = zone_id.id; + } + + ::encode_json(name, zone_name, f); + } + } zone_id_type_handler; + + JSONEncodeFilter encode_filter; +public: + JSONFormatter_PrettyZone(bool pretty_format) : JSONFormatter(pretty_format) { + encode_filter.register_type(&zone_id_type_handler); + } + + void *get_external_feature_handler(const std::string& feature) override { + if (feature != "JSONEncodeFilter") { + return nullptr; + } + return &encode_filter; + } +}; + int main(int argc, const char **argv) { vector args; @@ -3030,6 +3061,7 @@ int main(int argc, const char **argv) string bucket_id; string new_bucket_name; Formatter *formatter = NULL; + Formatter *zone_formatter = nullptr; int purge_data = false; int pretty_format = false; int show_log_entries = true; @@ -3692,6 +3724,8 @@ int main(int argc, const char **argv) exit(1); } + zone_formatter = new JSONFormatter_PrettyZone(pretty_format); + realm_name = g_conf()->rgw_realm; zone_name = g_conf()->rgw_zone; zonegroup_name = g_conf()->rgw_zonegroup; @@ -7548,7 +7582,7 @@ next: } if (opt_cmd == OPT::SYNC_INFO) { - sync_info(opt_effective_zone_id, opt_bucket, formatter); + sync_info(opt_effective_zone_id, opt_bucket, zone_formatter); } if (opt_cmd == OPT::SYNC_STATUS) { @@ -8059,7 +8093,7 @@ next: return -ret; } - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::SYNC_GROUP_GET) { @@ -8073,7 +8107,7 @@ next: auto& groups = sync_policy.groups; if (!opt_group_id) { - show_result(groups, formatter, cout); + show_result(groups, zone_formatter, cout); } else { auto iter = sync_policy.groups.find(*opt_group_id); if (iter == sync_policy.groups.end()) { @@ -8081,7 +8115,7 @@ next: return ENOENT; } - show_result(iter->second, formatter, cout); + show_result(iter->second, zone_formatter, cout); } } @@ -8103,11 +8137,11 @@ next: } { - Formatter::ObjectSection os(*formatter, "result"); - encode_json("sync_policy", sync_policy, formatter); + Formatter::ObjectSection os(*zone_formatter, "result"); + encode_json("sync_policy", sync_policy, zone_formatter); } - formatter->flush(cout); + zone_formatter->flush(cout); } if (opt_cmd == OPT::SYNC_GROUP_FLOW_CREATE) { @@ -8157,7 +8191,7 @@ next: return -ret; } - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::SYNC_GROUP_FLOW_REMOVE) { @@ -8197,7 +8231,7 @@ next: return -ret; } - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::SYNC_GROUP_PIPE_CREATE || @@ -8244,8 +8278,8 @@ next: opt_dest_bucket_name, opt_dest_bucket_id); - pipe->params.filter.set_prefix(opt_prefix, !!opt_prefix_rm); - pipe->params.filter.set_tags(tags_add, tags_rm); + pipe->params.source.filter.set_prefix(opt_prefix, !!opt_prefix_rm); + pipe->params.source.filter.set_tags(tags_add, tags_rm); if (opt_priority) { pipe->params.priority = *opt_priority; } @@ -8255,7 +8289,7 @@ next: return -ret; } - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::SYNC_GROUP_PIPE_REMOVE) { @@ -8314,7 +8348,7 @@ next: return -ret; } - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::SYNC_POLICY_GET) { @@ -8325,7 +8359,7 @@ next: } auto& sync_policy = sync_policy_ctx.get_policy(); - show_result(sync_policy, formatter, cout); + show_result(sync_policy, zone_formatter, cout); } if (opt_cmd == OPT::BILOG_TRIM) { diff --git a/src/rgw/rgw_bucket_sync.cc b/src/rgw/rgw_bucket_sync.cc index fe0d5eb7e6f..ba3fbc3518b 100644 --- a/src/rgw/rgw_bucket_sync.cc +++ b/src/rgw/rgw_bucket_sync.cc @@ -272,11 +272,11 @@ void RGWBucketSyncFlowManager::pipe_rules::insert(const rgw_sync_bucket_pipe& pi pipes.push_back(pipe); auto ppipe = &pipes.back(); - auto prefix = ppipe->params.filter.prefix.value_or(string()); + auto prefix = ppipe->params.source.filter.prefix.value_or(string()); prefix_refs.insert(make_pair(prefix, ppipe)); - for (auto& t : ppipe->params.filter.tags) { + for (auto& t : ppipe->params.source.filter.tags) { string tag = t.key + "=" + t.value; auto titer = tag_refs.find(tag); if (titer != tag_refs.end() && @@ -309,7 +309,7 @@ bool RGWBucketSyncFlowManager::pipe_rules::find_obj_params(const rgw_obj_key& ke } auto& rule_params = iter->second->params; - auto& filter = rule_params.filter; + auto& filter = rule_params.source.filter; if (!filter.check_tags(tags)) { continue; diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 7e9cfb8e0a5..327f4f91116 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -2065,7 +2065,7 @@ bool RGWBucketInfo::empty_sync_policy() const return sync_policy->empty(); } -void encode_json(const char *name, const rgw_zone_id& zid, Formatter *f) +void encode_json_impl(const char *name, const rgw_zone_id& zid, Formatter *f) { encode_json(name, zid.id, f); } diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 2aa72159ba5..7316a0465ff 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -2725,7 +2725,7 @@ static inline ostream& operator<<(ostream& os, const rgw_zone_id& zid) { return os; } -void encode_json(const char *name, const rgw_zone_id& zid, Formatter *f); +void encode_json_impl(const char *name, const rgw_zone_id& zid, Formatter *f); void decode_json_obj(rgw_zone_id& zid, JSONObj *obj); #endif diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 54ef0baa992..269f5f3c545 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -920,6 +920,16 @@ void rgw_sync_pipe_acl_translation::decode_json(JSONObj *obj) JSONDecoder::decode_json("owner", owner, obj); } +void rgw_sync_pipe_source_params::dump(Formatter *f) const +{ + encode_json("filter", filter, f); +} + +void rgw_sync_pipe_source_params::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("filter", filter, obj); +} + void rgw_sync_pipe_dest_params::dump(Formatter *f) const { encode_json("acl_translation", acl_translation, f); @@ -934,15 +944,15 @@ void rgw_sync_pipe_dest_params::decode_json(JSONObj *obj) void rgw_sync_pipe_params::dump(Formatter *f) const { - encode_json("filter", filter, f); - encode_json("dest_params", dest_params, f); + encode_json("source", source, f); + encode_json("dest", dest, f); encode_json("priority", priority, f); } void rgw_sync_pipe_params::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("filter", filter, obj); - JSONDecoder::decode_json("dest_params", dest_params, obj); + JSONDecoder::decode_json("source", source, obj); + JSONDecoder::decode_json("dest", dest, obj); JSONDecoder::decode_json("priority", priority, obj); } diff --git a/src/rgw/rgw_sync_policy.h b/src/rgw/rgw_sync_policy.h index 12353777464..ae778212929 100644 --- a/src/rgw/rgw_sync_policy.h +++ b/src/rgw/rgw_sync_policy.h @@ -255,6 +255,26 @@ struct rgw_sync_pipe_acl_translation { }; WRITE_CLASS_ENCODER(rgw_sync_pipe_acl_translation) +struct rgw_sync_pipe_source_params { + rgw_sync_pipe_filter filter; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(filter, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(filter, bl); + DECODE_FINISH(bl); + } + + void dump(ceph::Formatter *f) const; + void decode_json(JSONObj *obj); +}; +WRITE_CLASS_ENCODER(rgw_sync_pipe_source_params) + struct rgw_sync_pipe_dest_params { std::optional acl_translation; std::optional storage_class; @@ -278,23 +298,23 @@ struct rgw_sync_pipe_dest_params { }; WRITE_CLASS_ENCODER(rgw_sync_pipe_dest_params) -struct rgw_sync_pipe_params { - rgw_sync_pipe_filter filter; - rgw_sync_pipe_dest_params dest_params; +struct rgw_sync_pipe_params { + rgw_sync_pipe_source_params source; + rgw_sync_pipe_dest_params dest; int32_t priority{0}; void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); - encode(filter, bl); - encode(dest_params, bl); + encode(source, bl); + encode(dest, bl); encode(priority, bl); ENCODE_FINISH(bl); } void decode(bufferlist::const_iterator& bl) { DECODE_START(1, bl); - decode(filter, bl); - decode(dest_params, bl); + decode(source, bl); + decode(dest, bl); decode(priority, bl); DECODE_FINISH(bl); } -- 2.39.5