From: Casey Bodley Date: Tue, 23 Feb 2021 19:55:50 +0000 (-0500) Subject: rgw: add custom json encode/decode for the v1 notify API X-Git-Tag: v18.0.0~787^2~110 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=cf160da17849ef12b01c27b8ac9718d70c62e048;p=ceph-ci.git rgw: add custom json encode/decode for the v1 notify API this adds wrapper structs rgw_data_notify_v1_encoder and rgw_data_notify_v1_decoder that can encode/decode the v1 json format directly on the v2 data structure Signed-off-by: Casey Bodley --- diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h index 6349a85966c..f33f293e246 100644 --- a/src/common/ceph_json.h +++ b/src/common/ceph_json.h @@ -339,7 +339,9 @@ bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool manda std::string s = "missing mandatory field " + std::string(name); throw err(s); } - val = T(); + if constexpr (std::is_default_constructible_v) { + val = T(); + } return false; } diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 3f569deb20b..15c63537627 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -158,6 +158,7 @@ set(librgw_common_srcs rgw_url.cc rgw_oidc_provider.cc rgw_datalog.cc + rgw_datalog_notify.cc cls_fifo_legacy.cc rgw_lua_utils.cc rgw_lua.cc diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index c70b70cd056..6e7a99161be 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -5299,7 +5299,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp) } } - if (sync_pair.source_bs.shard_id >= bucket_status.shards_done_with_gen.size()) { + if (size_t(sync_pair.source_bs.shard_id) >= bucket_status.shards_done_with_gen.size()) { tn->log(1, SSTR("bucket shard " << sync_pair.source_bs << " index out of bounds")); return set_cr_done(); // return success so we don't retry } diff --git a/src/rgw/rgw_datalog.h b/src/rgw/rgw_datalog.h index 565f530270b..35822e1c7f4 100644 --- a/src/rgw/rgw_datalog.h +++ b/src/rgw/rgw_datalog.h @@ -129,6 +129,26 @@ struct RGWDataChangesLogMarker { class RGWDataChangesLog; +struct rgw_data_notify_entry { + std::string key; + uint64_t gen = 0; + + void dump(ceph::Formatter* f) const; + void decode_json(JSONObj* obj); + + rgw_data_notify_entry& operator=(const rgw_data_notify_entry&) = default; + + bool operator<(const rgw_data_notify_entry& d) const { + if (key < d.key) { + return true; + } + if (d.key < key) { + return false; + } + return gen < d.gen; + } +}; + class RGWDataChangesBE; class DataLogBackends final diff --git a/src/rgw/rgw_datalog_notify.cc b/src/rgw/rgw_datalog_notify.cc new file mode 100644 index 00000000000..12cdc532f3c --- /dev/null +++ b/src/rgw/rgw_datalog_notify.cc @@ -0,0 +1,76 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_datalog_notify.h" +#include "rgw_datalog.h" + +// custom encoding for v1 notify API +struct EntryEncoderV1 { + const rgw_data_notify_entry& entry; +}; +struct SetEncoderV1 { + const bc::flat_set& entries; +}; + +// encode rgw_data_notify_entry as string +void encode_json(const char *name, const EntryEncoderV1& e, Formatter *f) +{ + f->dump_string(name, e.entry.key); // encode the key only +} +// encode set as set +void encode_json(const char *name, const SetEncoderV1& e, Formatter *f) +{ + f->open_array_section(name); + for (auto& entry : e.entries) { + encode_json("obj", EntryEncoderV1{entry}, f); + } + f->close_section(); +} +// encode map> as map> +void encode_json(const char *name, const rgw_data_notify_v1_encoder& e, Formatter *f) +{ + f->open_array_section(name); + for (auto& [key, val] : e.shards) { + f->open_object_section("entry"); + encode_json("key", key, f); + encode_json("val", SetEncoderV1{val}, f); + f->close_section(); + } + f->close_section(); +} + +struct EntryDecoderV1 { + rgw_data_notify_entry& entry; +}; +struct SetDecoderV1 { + bc::flat_set& entries; +}; + +// decode string into rgw_data_notify_entry +void decode_json_obj(EntryDecoderV1& d, JSONObj *obj) +{ + decode_json_obj(d.entry.key, obj); + d.entry.gen = 0; +} +// decode set into set +void decode_json_obj(SetDecoderV1& d, JSONObj *obj) +{ + for (JSONObjIter o = obj->find_first(); !o.end(); ++o) { + rgw_data_notify_entry val; + auto decoder = EntryDecoderV1{val}; + decode_json_obj(decoder, *o); + d.entries.insert(std::move(val)); + } +} +// decode map> into map> +void decode_json_obj(rgw_data_notify_v1_decoder& d, JSONObj *obj) +{ + for (JSONObjIter o = obj->find_first(); !o.end(); ++o) { + int shard_id = 0; + JSONDecoder::decode_json("key", shard_id, *o); + bc::flat_set val; + SetDecoderV1 decoder{val}; + JSONDecoder::decode_json("val", decoder, *o); + d.shards[shard_id] = std::move(val); + } +} diff --git a/src/rgw/rgw_datalog_notify.h b/src/rgw/rgw_datalog_notify.h new file mode 100644 index 00000000000..d32dc79b171 --- /dev/null +++ b/src/rgw/rgw_datalog_notify.h @@ -0,0 +1,31 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#pragma once + +#include +#include + +#include "rgw/rgw_datalog.h" + +namespace bc = boost::container; + +namespace ceph { class Formatter; } +class JSONObj; + +class RGWCoroutine; +class RGWHTTPManager; +class RGWRESTConn; + +struct rgw_data_notify_entry; + +// json encoder and decoder for notify v1 API +struct rgw_data_notify_v1_encoder { + const bc::flat_map>& shards; +}; +void encode_json(const char *name, const rgw_data_notify_v1_encoder& e, + ceph::Formatter *f); +struct rgw_data_notify_v1_decoder { + bc::flat_map>& shards; +}; +void decode_json_obj(rgw_data_notify_v1_decoder& d, JSONObj *obj);