rgw: add custom json encode/decode for the v1 notify API

this adds wrapper structs rgw_data_notify_v1_encoder and
rgw_data_notify_v1_decoder that can encode/decode the v1 json format
directly on the v2 data structure

Signed-off-by: Casey Bodley <cbodley@redhat.com>

diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h
index 6349a85966c79ee5c6f5b9c33556bc568fa9539e..f33f293e246b0200be85dc0b665b7fab11ce8524 100644 (file)
--- a/src/common/ceph_json.h
+++ b/src/common/ceph_json.h
@@ -339,7 +339,9 @@ bool JSONDecoder::decode_json(const char *name, T& val, JSONObj *obj, bool manda
       std::string s = "missing mandatory field " + std::string(name);
       throw err(s);
     }
-    val = T();
+    if constexpr (std::is_default_constructible_v<T>) {
+      val = T();
+    }
     return false;
   }
 

diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt
index fb0c7e476ea33fe86fa9e4466fb56ec493b21155..453d9e69eece06bb8cf13e42cfbfceddb032181c 100644 (file)
--- a/src/rgw/CMakeLists.txt
+++ b/src/rgw/CMakeLists.txt
@@ -159,6 +159,7 @@ set(librgw_common_srcs
   rgw_url.cc
   rgw_oidc_provider.cc
   rgw_datalog.cc
+  rgw_datalog_notify.cc
   cls_fifo_legacy.cc
   rgw_lua_utils.cc
   rgw_lua.cc

diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index fa220e3812072a21c5d60ea8423ae42b2d399238..1eec788974f36811e57ff0ff730c5d8afdf517a1 100644 (file)
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -5314,7 +5314,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp)
           }
         }
 
-        if (sync_pair.source_bs.shard_id >= bucket_status.shards_done_with_gen.size()) {
+        if (size_t(sync_pair.source_bs.shard_id) >= bucket_status.shards_done_with_gen.size()) {
           tn->log(1, SSTR("bucket shard " << sync_pair.source_bs << " index out of bounds"));
           return set_cr_done(); // return success so we don't retry
         }

diff --git a/src/rgw/rgw_datalog.h b/src/rgw/rgw_datalog.h
index 565f530270b813eb3b14c4be0daf9bfef8d1dd0b..35822e1c7f42ff8130ca37644626b51fbc39e40a 100644 (file)
--- a/src/rgw/rgw_datalog.h
+++ b/src/rgw/rgw_datalog.h
@@ -129,6 +129,26 @@ struct RGWDataChangesLogMarker {
 
 class RGWDataChangesLog;
 
+struct rgw_data_notify_entry {
+  std::string key;
+  uint64_t gen = 0;
+
+  void dump(ceph::Formatter* f) const;
+  void decode_json(JSONObj* obj);
+
+  rgw_data_notify_entry& operator=(const rgw_data_notify_entry&) = default;
+
+  bool operator<(const rgw_data_notify_entry& d) const {
+    if (key < d.key) {
+      return true;
+    }
+    if (d.key < key) {
+      return false;
+    }
+    return gen < d.gen;
+  }
+};
+
 class RGWDataChangesBE;
 
 class DataLogBackends final

diff --git a/src/rgw/rgw_datalog_notify.cc b/src/rgw/rgw_datalog_notify.cc
new file mode 100644 (file)
index 0000000..12cdc53
--- /dev/null
+++ b/src/rgw/rgw_datalog_notify.cc
@@ -0,0 +1,76 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "rgw_datalog_notify.h"
+#include "rgw_datalog.h"
+
+// custom encoding for v1 notify API
+struct EntryEncoderV1 {
+  const rgw_data_notify_entry& entry;
+};
+struct SetEncoderV1 {
+  const bc::flat_set<rgw_data_notify_entry>& entries;
+};
+
+// encode rgw_data_notify_entry as string
+void encode_json(const char *name, const EntryEncoderV1& e, Formatter *f)
+{
+  f->dump_string(name, e.entry.key); // encode the key only
+}
+// encode set<rgw_data_notify_entry> as set<string>
+void encode_json(const char *name, const SetEncoderV1& e, Formatter *f)
+{
+  f->open_array_section(name);
+  for (auto& entry : e.entries) {
+    encode_json("obj", EntryEncoderV1{entry}, f);
+  }
+  f->close_section();
+}
+// encode map<int, set<rgw_data_notify_entry>> as map<int, set<string>>
+void encode_json(const char *name, const rgw_data_notify_v1_encoder& e, Formatter *f)
+{
+  f->open_array_section(name);
+  for (auto& [key, val] : e.shards) {
+    f->open_object_section("entry");
+    encode_json("key", key, f);
+    encode_json("val", SetEncoderV1{val}, f);
+    f->close_section();
+  }
+  f->close_section();
+}
+
+struct EntryDecoderV1 {
+  rgw_data_notify_entry& entry;
+};
+struct SetDecoderV1 {
+  bc::flat_set<rgw_data_notify_entry>& entries;
+};
+
+// decode string into rgw_data_notify_entry
+void decode_json_obj(EntryDecoderV1& d, JSONObj *obj)
+{
+  decode_json_obj(d.entry.key, obj);
+  d.entry.gen = 0;
+}
+// decode set<string> into set<rgw_data_notify_entry>
+void decode_json_obj(SetDecoderV1& d, JSONObj *obj)
+{
+  for (JSONObjIter o = obj->find_first(); !o.end(); ++o) {
+    rgw_data_notify_entry val;
+    auto decoder = EntryDecoderV1{val};
+    decode_json_obj(decoder, *o);
+    d.entries.insert(std::move(val));
+  }
+}
+// decode map<int, set<string>> into map<int, set<rgw_data_notify_entry>>
+void decode_json_obj(rgw_data_notify_v1_decoder& d, JSONObj *obj)
+{
+  for (JSONObjIter o = obj->find_first(); !o.end(); ++o) {
+    int shard_id = 0;
+    JSONDecoder::decode_json("key", shard_id, *o);
+    bc::flat_set<rgw_data_notify_entry> val;
+    SetDecoderV1 decoder{val};
+    JSONDecoder::decode_json("val", decoder, *o);
+    d.shards[shard_id] = std::move(val);
+  }
+}

diff --git a/src/rgw/rgw_datalog_notify.h b/src/rgw/rgw_datalog_notify.h
new file mode 100644 (file)
index 0000000..d32dc79
--- /dev/null
+++ b/src/rgw/rgw_datalog_notify.h
@@ -0,0 +1,31 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#pragma once
+
+#include <boost/container/flat_map.hpp>
+#include <boost/container/flat_set.hpp>
+
+#include "rgw/rgw_datalog.h"
+
+namespace bc = boost::container;
+
+namespace ceph { class Formatter; }
+class JSONObj;
+
+class RGWCoroutine;
+class RGWHTTPManager;
+class RGWRESTConn;
+
+struct rgw_data_notify_entry;
+
+// json encoder and decoder for notify v1 API
+struct rgw_data_notify_v1_encoder {
+  const bc::flat_map<int, bc::flat_set<rgw_data_notify_entry>>& shards;
+};
+void encode_json(const char *name, const rgw_data_notify_v1_encoder& e,
+                 ceph::Formatter *f);
+struct rgw_data_notify_v1_decoder {
+  bc::flat_map<int, bc::flat_set<rgw_data_notify_entry>>& shards;
+};
+void decode_json_obj(rgw_data_notify_v1_decoder& d, JSONObj *obj);