From: Yehuda Sadeh Date: Fri, 25 Jan 2019 23:45:51 +0000 (-0800) Subject: rgw: es: fix v5+ index map settings X-Git-Tag: v14.1.0~210^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1c3583c5adfd54c5a06064319c04379963e9e865;p=ceph.git rgw: es: fix v5+ index map settings and rework the way it's being done Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_sync_module_es.cc b/src/rgw/rgw_sync_module_es.cc index fce97658ebb..0f4e6b770e2 100644 --- a/src/rgw/rgw_sync_module_es.cc +++ b/src/rgw/rgw_sync_module_es.cc @@ -223,7 +223,7 @@ struct ElasticConfig { using ElasticConfigRef = std::shared_ptr; -std::optional es_type_to_str(const ESType& t) { +static const char *es_type_to_str(const ESType& t) { switch (t) { case ESType::String: return "string"; case ESType::Text: return "text"; @@ -245,79 +245,124 @@ std::optional es_type_to_str(const ESType& t) { case ESType::Geo_Point: return "geo_point"; case ESType::Ip: return "ip"; default: - return std::nullopt; + return ""; } - } -struct es_dump_type { - const char *type; - const char *format; - bool analyzed; - std::string default_type; +struct es_type_v2 { + ESType estype; + const char *format{nullptr}; + std::optional analyzed; + + es_type_v2(ESType et) : estype(et) {} + + void dump(Formatter *f) const { + const char *type_str = es_type_to_str(estype); + encode_json("type", type_str, f); + if (format) { + encode_json("format", format, f); + } + + auto is_analyzed = analyzed; - es_dump_type(const char *t, const char *f = nullptr, bool a = false) : type(t), format(f), analyzed(a) {} + if (estype == ESType::String && + !is_analyzed) { + is_analyzed = false; + } - es_dump_type(ESType t, const char *f = nullptr, bool a = false, - std::string default_type="text"): format(f), analyzed(a), - default_type(default_type) { - type = es_type_to_str(t).value_or(default_type.c_str()); + if (is_analyzed) { + encode_json("index", (is_analyzed.value() ? "analyzed" : "not_analyzed"), f); + } } +}; + +struct es_type_v5 { + ESType estype; + const char *format{nullptr}; + std::optional analyzed; + std::optional index; + + es_type_v5(ESType et) : estype(et) {} void dump(Formatter *f) const { - encode_json("type", type, f); + ESType new_estype; + if (estype != ESType::String) { + new_estype = estype; + } else { + bool is_analyzed = analyzed.value_or(false); + new_estype = (is_analyzed ? ESType::Text : ESType::Keyword); + /* index = true; ... Not setting index=true, because that's the default, + * and dumping a boolean value *might* be a problem when backporting this + * because value might get quoted + */ + } + + const char *type_str = es_type_to_str(new_estype); + encode_json("type", type_str, f); if (format) { encode_json("format", format, f); } - if (!analyzed && strcmp(type, "string") == 0) { - encode_json("index", "not_analyzed", f); + if (index) { + encode_json("index", index.value(), f); } } }; +template +struct es_type : public T { + es_type(T t) : T(t) {} + es_type& set_format(const char *f) { + T::format = f; + return *this; + } + + es_type& set_analyzed(bool a) { + T::analyzed = a; + return *this; + } +}; + +template struct es_index_mappings { ESType string_type {ESType::String}; - void dump_custom(Formatter *f, const char *section, const char *type, const char *format) const { + es_type est(ESType t) const { + return es_type(t); + } + + void dump_custom(const char *section, ESType type, const char *format, Formatter *f) const { f->open_object_section(section); ::encode_json("type", "nested", f); f->open_object_section("properties"); - encode_json("name", es_dump_type(string_type), f); - encode_json("value", es_dump_type(type, format), f); + encode_json("name", est(string_type), f); + encode_json("value", est(type).set_format(format), f); f->close_section(); // entry f->close_section(); // custom-string } - void dump_custom(Formatter *f, const char* section, ESType t, const char *format) const { - const auto s = es_type_to_str(t).value_or("text"); - dump_custom(f,section,s,format); - } - void dump(Formatter *f) const { f->open_object_section("object"); f->open_object_section("properties"); - encode_json("bucket", es_dump_type(string_type), f); - encode_json("name", es_dump_type(string_type), f); - encode_json("instance", es_dump_type(string_type), f); - encode_json("versioned_epoch", es_dump_type(ESType::Date), f); + encode_json("bucket", est(string_type), f); + encode_json("name", est(string_type), f); + encode_json("instance", est(string_type), f); + encode_json("versioned_epoch", est(ESType::Long), f); f->open_object_section("meta"); f->open_object_section("properties"); - encode_json("cache_control", es_dump_type(string_type), f); - encode_json("content_disposition", es_dump_type(string_type), f); - encode_json("content_encoding", es_dump_type(string_type), f); - encode_json("content_language", es_dump_type(string_type), f); - encode_json("content_type", es_dump_type(string_type), f); - encode_json("storage_class", es_dump_type(string_type), f); - encode_json("etag", es_dump_type(string_type), f); - encode_json("expires", es_dump_type(string_type), f); - f->open_object_section("mtime"); - ::encode_json("type", "date", f); - ::encode_json("format", "strict_date_optional_time||epoch_millis", f); - f->close_section(); // mtime - encode_json("size", es_dump_type(ESType::Long), f); - dump_custom(f, "custom-string", string_type, nullptr); - dump_custom(f, "custom-int", "long", nullptr); - dump_custom(f, "custom-date", "date", "strict_date_optional_time||epoch_millis"); + encode_json("cache_control", est(string_type), f); + encode_json("content_disposition", est(string_type), f); + encode_json("content_encoding", est(string_type), f); + encode_json("content_language", est(string_type), f); + encode_json("content_type", est(string_type), f); + encode_json("storage_class", est(string_type), f); + encode_json("etag", est(string_type), f); + encode_json("expires", est(string_type), f); + encode_json("mtime", est(ESType::Date) + .set_format("strict_date_optional_time||epoch_millis"), f); + encode_json("size", est(ESType::Long), f); + dump_custom("custom-string", string_type, nullptr, f); + dump_custom("custom-int", ESType::Long, nullptr, f); + dump_custom("custom-date", ESType::Date, "strict_date_optional_time||epoch_millis", f); f->close_section(); // properties f->close_section(); // meta f->close_section(); // properties @@ -337,11 +382,17 @@ struct es_index_settings { } }; -struct es_index_config { +struct es_index_config_base { + virtual ~es_index_config_base() {} + virtual void dump(Formatter *f) const = 0; +}; + +template +struct es_index_config : public es_index_config_base { es_index_settings settings; - es_index_mappings mappings; + es_index_mappings mappings; - es_index_config(es_index_settings& _s, es_index_mappings& _m) : settings(_s), mappings(_m) {} + es_index_config(es_index_settings& _s) : settings(_s) {} void dump(Formatter *f) const { encode_json("settings", settings, f); @@ -598,19 +649,22 @@ public: ldout(sync_env->cct, 5) << "got elastic version=" << es_info.get_version_str() << dendl; es_index_settings settings(conf->num_replicas, conf->num_shards); - es_index_mappings mappings; + + std::unique_ptr index_conf; + if (es_info.version >= ES_V5) { - ldout(sync_env->cct, 0) << "elasticsearch: using text type for string index mappings " << dendl; - mappings.string_type = ESType::Text; + ldout(sync_env->cct, 0) << "elasticsearch: index mapping: version >= 5" << dendl; + index_conf.reset(new es_index_config(settings)); + } else { + ldout(sync_env->cct, 0) << "elasticsearch: index mapping: version < 5" << dendl; + index_conf.reset(new es_index_config(settings)); } - - es_index_config index_conf(settings, mappings); - call(new RGWPutRESTResourceCR (sync_env->cct, + call(new RGWPutRESTResourceCR (sync_env->cct, conf->conn.get(), sync_env->http_manager, path, nullptr /*params*/, &(conf->default_headers), - index_conf, nullptr)); + *index_conf, nullptr)); } if (retcode < 0) { return set_cr_error(retcode);