From: Yehuda Sadeh <yehuda@redhat.com>
Date: Tue, 12 Dec 2017 13:03:54 +0000 (-0800)
Subject: rgw: sync modules, use JSONFormattable
X-Git-Tag: v13.1.0~270^2~49
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d8dc2be393d77c30cf43a94e99a42568dcb00bbc;p=ceph.git

rgw: sync modules, use JSONFormattable

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---

diff --git a/src/cls/refcount/cls_refcount_ops.h b/src/cls/refcount/cls_refcount_ops.h
index 24aa28482cae..ee560c52f230 100644
--- a/src/cls/refcount/cls_refcount_ops.h
+++ b/src/cls/refcount/cls_refcount_ops.h
@@ -6,8 +6,6 @@
 
 #include "include/types.h"
 
-class Formatter;
-
 struct cls_refcount_get_op {
   string tag;
   bool implicit_ref;
diff --git a/src/common/ceph_json.cc b/src/common/ceph_json.cc
index 29b551e8fb06..102de1768758 100644
--- a/src/common/ceph_json.cc
+++ b/src/common/ceph_json.cc
@@ -5,10 +5,16 @@
 #include <fstream>
 #include <include/types.h>
 
+#include <boost/algorithm/string.hpp>
+
 using namespace json_spirit;
 
 #define dout_subsys ceph_subsys_rgw
 
+
+static JSONFormattable default_formattable;
+
+
 JSONObjIter::JSONObjIter()
 {
 }
@@ -515,3 +521,80 @@ void encode_json(const char *name, const bufferlist& bl, Formatter *f)
   encode_json(name, s, f);
 }
 
+
+
+/* JSONFormattable */
+
+const JSONFormattable& JSONFormattable::operator[](const string& name) const
+{
+  auto i = obj.find(name);
+  if (i == obj.end()) {
+    return default_formattable;
+  }
+  return i->second;
+}
+
+bool JSONFormattable::find(const string& name, string *val) const
+{
+  auto i = obj.find(name);
+  if (i == obj.end()) {
+    return false;
+  }
+  *val = i->second;
+  return true;
+}
+
+string JSONFormattable::get(const string& name, const string& def_val) const
+{
+  string s;
+  if (find(name, &s)) {
+    return s;
+  }
+
+  return def_val;
+}
+
+int JSONFormattable::get_int(const string& name, int def_val) const
+{
+  string s;
+  if (find(name, &s)) {
+    return atoi(s.c_str());
+  }
+
+  return def_val;
+}
+
+bool JSONFormattable::get_bool(const string& name, bool def_val) const
+{
+  string s;
+  if (find(name, &s)) {
+    return (boost::iequals(s, "true") ||
+            boost::iequals(s, "on") ||
+            boost::iequals(s, "yes") ||
+            boost::iequals(s, "1"));
+  }
+
+  return def_val;
+}
+
+void encode_json(const char *name, const JSONFormattable& v, Formatter *f)
+{
+  switch (v.type) {
+    case JSONFormattable::FMT_STRING:
+      encode_json(name, v.str, f);
+      break;
+    case JSONFormattable::FMT_ARRAY:
+      encode_json(name, v.arr, f);
+      break;
+    case JSONFormattable::FMT_OBJ:
+      f->open_object_section(name);
+      for (auto iter : v.obj) {
+        encode_json(iter.first.c_str(), iter.second, f);
+      }
+      f->close_section();
+      break;
+    case JSONFormattable::FMT_NONE:
+      break;
+  }
+}
+
diff --git a/src/common/ceph_json.h b/src/common/ceph_json.h
index 08ec5e59a288..3299d4792ce9 100644
--- a/src/common/ceph_json.h
+++ b/src/common/ceph_json.h
@@ -467,4 +467,75 @@ void encode_json_map(const char *name, const char *index_name, const char *value
   encode_json_map<K, V>(name, index_name, NULL, value_name, NULL, NULL, m, f);
 }
 
+struct JSONFormattable {
+  enum Type {
+    FMT_NONE,
+    FMT_STRING,
+    FMT_ARRAY,
+    FMT_OBJ,
+  } type{FMT_NONE};
+  string str;
+  vector<JSONFormattable> arr;
+  map<string, JSONFormattable> obj;
+
+  void decode_json(JSONObj *jo) {
+    if (jo->is_array()) {
+      type = JSONFormattable::FMT_ARRAY;
+      decode_json_obj(arr, jo);
+    } else if (jo->is_object()) {
+      type = JSONFormattable::FMT_OBJ;
+      auto iter = jo->find_first();
+      while (!iter.end()) {
+        JSONObj *field = *iter;
+        decode_json_obj(obj[field->get_name()], field);
+      }
+    } else {
+      type = JSONFormattable::FMT_STRING;
+      decode_json_obj(str, jo);
+    }
+  }
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    encode((uint8_t)type, bl);
+    encode(str, bl);
+    encode(arr, bl);
+    encode(obj, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    uint8_t t;
+    decode(t, bl);
+    type = (Type)t;
+    decode(str, bl);
+    decode(arr, bl);
+    decode(obj, bl);
+    DECODE_FINISH(bl);
+  }
+  const string& val() const {
+    return str;
+  }
+
+  const vector<JSONFormattable>& array() const {
+    return arr;
+  }
+
+  const JSONFormattable& operator[](const string& name) const;
+
+  operator string() const {
+    return val();
+  }
+
+  bool find(const string& name, string *val) const;
+
+  string get(const string& name, const string& def_val) const;
+  int get_int(const string& name, int def_val) const;
+  bool get_bool(const string& name, bool def_val) const;
+};
+WRITE_CLASS_ENCODER(JSONFormattable)
+
+void encode_json(const char *name, const JSONFormattable& v, Formatter *f);
+
 #endif
diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 37546ddfef71..7b97e77a18a0 100644
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2448,77 +2448,6 @@ static int scan_totp(CephContext *cct, ceph::real_time& now, rados::cls::otp::ot
   return -ENOENT;
 }
 
-struct Formattable {
-  enum Type {
-    FMT_NONE,
-    FMT_STRING,
-    FMT_ARRAY,
-    FMT_OBJ,
-  } type{FMT_NONE};
-  string str;
-  vector<Formattable> arr;
-  map<string, Formattable> obj;
-
-  void decode_json(JSONObj *jo) {
-    if (jo->is_array()) {
-      type = Formattable::FMT_ARRAY;
-      decode_json_obj(arr, jo);
-    } else if (jo->is_object()) {
-      type = Formattable::FMT_OBJ;
-      auto iter = jo->find_first();
-      while (!iter.end()) {
-        JSONObj *field;
-        decode_json_obj(obj[field->get_name()], field);
-      }
-    } else {
-      type = Formattable::FMT_STRING;
-      decode_json_obj(str, jo);
-    }
-  }
-
-  const string& val() const {
-    return str;
-  }
-
-  const vector<Formattable>& array() const {
-    return arr;
-  }
-
-  const Formattable& operator[](const string& name) const;
-};
-
-static Formattable default_formattable;
-
-const Formattable& Formattable::operator[](const string& name) const {
-  auto i = obj.find(name);
-  if (i == obj.end()) {
-    return default_formattable;
-  }
-  return i->second;
-}
-
-void encode_json(const char *name, const Formattable& v, Formatter *f)
-{
-  switch (v.type) {
-    case Formattable::FMT_STRING:
-      encode_json(name, v.str, f);
-      break;
-    case Formattable::FMT_ARRAY:
-      encode_json(name, v.arr, f);
-      break;
-    case Formattable::FMT_OBJ:
-      f->open_object_section(name);
-      for (auto iter : v.obj) {
-        encode_json(iter.first.c_str(), iter.second, f);
-      }
-      f->close_section();
-      break;
-    case Formattable::FMT_NONE:
-      break;
-  }
-}
-
-
 #ifdef BUILDING_FOR_EMBEDDED
 extern "C" int cephd_rgw_admin(int argc, const char **argv)
 #else
diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index 807f1d81a004..a2312e8d6b65 100644
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -1721,7 +1721,7 @@ public:
   }
 };
 
-int RGWDefaultSyncModule::create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance)
+int RGWDefaultSyncModule::create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance)
 {
   instance->reset(new RGWDefaultSyncModuleInstance());
   return 0;
diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/rgw_data_sync.h
index 088486e13c4f..a1442d35b059 100644
--- a/src/rgw/rgw_data_sync.h
+++ b/src/rgw/rgw_data_sync.h
@@ -546,7 +546,7 @@ class RGWDefaultSyncModule : public RGWSyncModule {
 public:
   RGWDefaultSyncModule() {}
   bool supports_data_export() override { return true; }
-  int create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) override;
+  int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) override;
 };
 
 // DataLogTrimCR factory function
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 003c09e6ab1d..c82b77883e20 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -13,6 +13,7 @@
 #include "common/RWLock.h"
 #include "common/ceph_time.h"
 #include "common/lru_map.h"
+#include "common/ceph_json.h"
 #include "rgw_common.h"
 #include "cls/rgw/cls_rgw_types.h"
 #include "cls/version/cls_version_types.h"
@@ -1179,7 +1180,7 @@ struct RGWZoneParams : RGWSystemMetaObj {
 
   string realm_id;
 
-  map<string, string, ltstr_nocase> tier_config;
+  JSONFormattable tier_config;
 
   RGWZoneParams() : RGWSystemMetaObj() {}
   RGWZoneParams(const string& name) : RGWSystemMetaObj(name){}
diff --git a/src/rgw/rgw_sync_module.h b/src/rgw/rgw_sync_module.h
index 9925d677eafd..8b06767c0512 100644
--- a/src/rgw/rgw_sync_module.h
+++ b/src/rgw/rgw_sync_module.h
@@ -43,6 +43,8 @@ public:
 
 typedef std::shared_ptr<RGWSyncModuleInstance> RGWSyncModuleInstanceRef;
 
+class JSONFormattable;
+
 class RGWSyncModule {
 
 public:
@@ -50,7 +52,7 @@ public:
   virtual ~RGWSyncModule() {}
 
   virtual bool supports_data_export() = 0;
-  virtual int create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) = 0;
+  virtual int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) = 0;
 };
 
 typedef std::shared_ptr<RGWSyncModule> RGWSyncModuleRef;
@@ -93,7 +95,7 @@ public:
     return module.get()->supports_data_export();
   }
 
-  int create_instance(CephContext *cct, const string& name, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) {
+  int create_instance(CephContext *cct, const string& name, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) {
     RGWSyncModuleRef module;
     if (!get_module(name, &module)) {
       return -ENOENT;
diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/rgw_sync_module_aws.cc
index 162b12d7e2d7..30d3aa74d952 100644
--- a/src/rgw/rgw_sync_module_aws.cc
+++ b/src/rgw/rgw_sync_module_aws.cc
@@ -951,14 +951,14 @@ public:
   }
 };
 
-static int conf_to_uint64(CephContext *cct, const map<string, string, ltstr_nocase>& config, const string& key, uint64_t *pval)
+static int conf_to_uint64(CephContext *cct, const JSONFormattable& config, const string& key, uint64_t *pval)
 {
-  auto i = config.find(key);
-  if (i != config.end()) {
+  string sval;
+  if (config.find(key, &sval)) {
     string err;
-    uint64_t val = strict_strtoll(i->second.c_str(), 10, &err);
+    uint64_t val = strict_strtoll(sval.c_str(), 10, &err);
     if (!err.empty()) {
-      ldout(cct, 0) << "ERROR: could not parse configurable value for cloud sync module: " << i->first << ": " << i->second << dendl;
+      ldout(cct, 0) << "ERROR: could not parse configurable value for cloud sync module: " << key << ": " << sval << dendl;
       return -EINVAL;
     }
     *pval = val;
@@ -966,22 +966,13 @@ static int conf_to_uint64(CephContext *cct, const map<string, string, ltstr_noca
   return 0;
 }
 
-int RGWAWSSyncModule::create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config,  RGWSyncModuleInstanceRef *instance){
+int RGWAWSSyncModule::create_instance(CephContext *cct, const JSONFormattable& config,  RGWSyncModuleInstanceRef *instance){
   AWSSyncConfig conf;
-  auto i = config.find("s3_endpoint");
-  if (i != config.end())
-    conf.s3_endpoint = i->second;
 
-  string access_key;
-  string secret;
+  conf.s3_endpoint = config["s3_endpoint"];
 
-  i = config.find("access_key");
-  if (i != config.end())
-    access_key = i->second;
-
-  i = config.find("secret");
-  if (i != config.end())
-    secret = i->second;
+  string access_key = config["access_key"];
+  string secret = config["secret"];
 
   conf.key = RGWAccessKey(access_key, secret);
 
diff --git a/src/rgw/rgw_sync_module_aws.h b/src/rgw/rgw_sync_module_aws.h
index 14749d514b1e..350c14e32b10 100644
--- a/src/rgw/rgw_sync_module_aws.h
+++ b/src/rgw/rgw_sync_module_aws.h
@@ -99,7 +99,7 @@ class RGWAWSSyncModule : public RGWSyncModule {
  public:
   RGWAWSSyncModule() {}
   bool supports_data_export() override { return false;}
-  int create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) override;
+  int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) override;
 };
 
 #endif /* RGW_SYNC_MODULE_AWS_H */
diff --git a/src/rgw/rgw_sync_module_es.cc b/src/rgw/rgw_sync_module_es.cc
index ab6e9fea1818..cd219fffc2c5 100644
--- a/src/rgw/rgw_sync_module_es.cc
+++ b/src/rgw/rgw_sync_module_es.cc
@@ -103,6 +103,7 @@ public:
 #define ES_NUM_SHARDS_DEFAULT 16
 #define ES_NUM_REPLICAS_DEFAULT 1
 
+
 struct ElasticConfig {
   uint64_t sync_instance{0};
   string id;
@@ -115,19 +116,19 @@ struct ElasticConfig {
   uint32_t num_shards{0};
   uint32_t num_replicas{0};
 
-  void init(CephContext *cct, const map<string, string, ltstr_nocase>& config) {
-    string elastic_endpoint = rgw_conf_get(config, "endpoint", "");
+  void init(CephContext *cct, const JSONFormattable& config) {
+    string elastic_endpoint = config.get("endpoint", "");
     id = string("elastic:") + elastic_endpoint;
     conn.reset(new RGWRESTConn(cct, nullptr, id, { elastic_endpoint }));
-    explicit_custom_meta = rgw_conf_get_bool(config, "explicit_custom_meta", true);
-    index_buckets.init(rgw_conf_get(config, "index_buckets_list", ""), true); /* approve all buckets by default */
-    allow_owners.init(rgw_conf_get(config, "approved_owners_list", ""), true); /* approve all bucket owners by default */
-    override_index_path = rgw_conf_get(config, "override_index_path", "");
-    num_shards = rgw_conf_get_int(config, "num_shards", ES_NUM_SHARDS_DEFAULT);
+    explicit_custom_meta = config.get_bool("explicit_custom_meta", true);
+    index_buckets.init(config.get("index_buckets_list", ""), true); /* approve all buckets by default */
+    allow_owners.init(config.get("approved_owners_list", ""), true); /* approve all bucket owners by default */
+    override_index_path = config.get("override_index_path", "");
+    num_shards = config.get_int("num_shards", ES_NUM_SHARDS_DEFAULT);
     if (num_shards < ES_NUM_SHARDS_MIN) {
       num_shards = ES_NUM_SHARDS_MIN;
     }
-    num_replicas = rgw_conf_get_int(config, "num_replicas", ES_NUM_REPLICAS_DEFAULT);
+    num_replicas = config.get_int("num_replicas", ES_NUM_REPLICAS_DEFAULT);
   }
 
   void init_instance(RGWRealm& realm, uint64_t instance_id) {
@@ -534,7 +535,7 @@ public:
 class RGWElasticDataSyncModule : public RGWDataSyncModule {
   ElasticConfigRef conf;
 public:
-  RGWElasticDataSyncModule(CephContext *cct, const map<string, string, ltstr_nocase>& config) : conf(std::make_shared<ElasticConfig>()) {
+  RGWElasticDataSyncModule(CephContext *cct, const JSONFormattable& config) : conf(std::make_shared<ElasticConfig>()) {
     conf->init(cct, config);
   }
   ~RGWElasticDataSyncModule() override {}
@@ -580,7 +581,7 @@ public:
   }
 };
 
-RGWElasticSyncModuleInstance::RGWElasticSyncModuleInstance(CephContext *cct, const map<string, string, ltstr_nocase>& config)
+RGWElasticSyncModuleInstance::RGWElasticSyncModuleInstance(CephContext *cct, const JSONFormattable& config)
 {
   data_handler = std::unique_ptr<RGWElasticDataSyncModule>(new RGWElasticDataSyncModule(cct, config));
 }
@@ -607,12 +608,8 @@ RGWRESTMgr *RGWElasticSyncModuleInstance::get_rest_filter(int dialect, RGWRESTMg
   return new RGWRESTMgr_MDSearch_S3();
 }
 
-int RGWElasticSyncModule::create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) {
-  string endpoint;
-  auto i = config.find("endpoint");
-  if (i != config.end()) {
-    endpoint = i->second;
-  }
+int RGWElasticSyncModule::create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) {
+  string endpoint = config["endpoint"];
   instance->reset(new RGWElasticSyncModuleInstance(cct, config));
   return 0;
 }
diff --git a/src/rgw/rgw_sync_module_es.h b/src/rgw/rgw_sync_module_es.h
index 43e591e42fc3..7dcda39cbf72 100644
--- a/src/rgw/rgw_sync_module_es.h
+++ b/src/rgw/rgw_sync_module_es.h
@@ -9,7 +9,7 @@ public:
   bool supports_data_export() override {
     return false;
   }
-  int create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) override;
+  int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) override;
 };
 
 class RGWElasticDataSyncModule;
@@ -18,7 +18,7 @@ class RGWRESTConn;
 class RGWElasticSyncModuleInstance : public RGWSyncModuleInstance {
   std::unique_ptr<RGWElasticDataSyncModule> data_handler;
 public:
-  RGWElasticSyncModuleInstance(CephContext *cct, const std::map<std::string, std::string, ltstr_nocase>& config);
+  RGWElasticSyncModuleInstance(CephContext *cct, const JSONFormattable& config);
   RGWDataSyncModule *get_data_handler() override;
   RGWRESTMgr *get_rest_filter(int dialect, RGWRESTMgr *orig) override;
   RGWRESTConn *get_rest_conn();
diff --git a/src/rgw/rgw_sync_module_log.cc b/src/rgw/rgw_sync_module_log.cc
index 93a853f56000..0378c040b6af 100644
--- a/src/rgw/rgw_sync_module_log.cc
+++ b/src/rgw/rgw_sync_module_log.cc
@@ -63,12 +63,8 @@ public:
   }
 };
 
-int RGWLogSyncModule::create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) {
-  string prefix;
-  auto i = config.find("prefix");
-  if (i != config.end()) {
-    prefix = i->second;
-  }
+int RGWLogSyncModule::create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) {
+  string prefix = config["prefix"];
   instance->reset(new RGWLogSyncModuleInstance(prefix));
   return 0;
 }
diff --git a/src/rgw/rgw_sync_module_log.h b/src/rgw/rgw_sync_module_log.h
index 9afc9108e678..d64fb9d826de 100644
--- a/src/rgw/rgw_sync_module_log.h
+++ b/src/rgw/rgw_sync_module_log.h
@@ -9,7 +9,7 @@ public:
   bool supports_data_export() override {
     return false;
   }
-  int create_instance(CephContext *cct, map<string, string, ltstr_nocase>& config, RGWSyncModuleInstanceRef *instance) override;
+  int create_instance(CephContext *cct, const JSONFormattable& config, RGWSyncModuleInstanceRef *instance) override;
 };
 
 #endif