From 42746ac9d2b4c38b78608750f445dcd1d3284eb4 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 28 Mar 2017 17:06:32 -0700 Subject: [PATCH] rgw: support for generic entities types in es Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_common.cc | 2 +- src/rgw/rgw_es_main.cc | 20 +++++ src/rgw/rgw_es_query.cc | 191 +++++++++++++++++++++++++++++----------- src/rgw/rgw_es_query.h | 16 +++- 4 files changed, 173 insertions(+), 56 deletions(-) diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index ddbd8fae70871..365cd86cb4abf 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -550,7 +550,7 @@ bool parse_iso8601(const char *s, struct tm *t, uint32_t *pns, bool extended_for trim_whitespace(p, str); int len = str.size(); - if (len == 1 && str[0] == 'Z') + if (len == 0 || (len == 1 && str[0] == 'Z')) return true; if (str[0] != '.' || diff --git a/src/rgw/rgw_es_main.cc b/src/rgw/rgw_es_main.cc index e6972b62bbc97..fb705193624e1 100644 --- a/src/rgw/rgw_es_main.cc +++ b/src/rgw/rgw_es_main.cc @@ -2,6 +2,10 @@ #include #include +#include "global/global_init.h" +#include "global/global_context.h" + +#include "common/ceph_argparse.h" #include "common/ceph_json.h" #include "rgw_es_query.h" @@ -9,6 +13,15 @@ using namespace std; int main(int argc, char *argv[]) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + env_to_vec(args); + + auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, + CODE_ENVIRONMENT_UTILITY, 0); + + common_init_finish(g_ceph_context); + list infix; string expr; @@ -21,6 +34,13 @@ int main(int argc, char *argv[]) ESQueryCompiler es_query(expr, nullptr, "x-amz-meta-"); + map generic_map = { {"key", ESEntityTypeMap::ES_ENTITY_STR}, + {"instance", ESEntityTypeMap::ES_ENTITY_STR}, + {"lastmodified", ESEntityTypeMap::ES_ENTITY_DATE}, + {"size", ESEntityTypeMap::ES_ENTITY_DATE} }; + ESEntityTypeMap gm(generic_map); + es_query.set_generic_type_map(&gm); + map custom_map = { {"str", ESEntityTypeMap::ES_ENTITY_STR}, {"int", ESEntityTypeMap::ES_ENTITY_INT}, {"date", ESEntityTypeMap::ES_ENTITY_DATE} }; diff --git a/src/rgw/rgw_es_query.cc b/src/rgw/rgw_es_query.cc index bd6bb2fe2dd45..58c2b5eb77d5b 100644 --- a/src/rgw/rgw_es_query.cc +++ b/src/rgw/rgw_es_query.cc @@ -113,15 +113,9 @@ public: ESQueryNode(ESQueryCompiler *_compiler) : compiler(_compiler) {} virtual ~ESQueryNode() {} - virtual bool init(ESQueryStack *s) = 0; + virtual bool init(ESQueryStack *s, ESQueryNode **pnode) = 0; virtual void dump(Formatter *f) const = 0; - - virtual bool leaf_field_name(string *name) { - return false; - } - - virtual void leaf_field_rename(const string& new_name) {} }; static bool alloc_node(ESQueryCompiler *compiler, ESQueryStack *s, ESQueryNode **pnode); @@ -133,7 +127,7 @@ class ESQueryNode_Bool : public ESQueryNode { public: ESQueryNode_Bool(ESQueryCompiler *compiler) : ESQueryNode(compiler) {} ESQueryNode_Bool(ESQueryCompiler *compiler, const string& _op, ESQueryNode *_first, ESQueryNode *_second) :ESQueryNode(compiler), op(_op), first(_first), second(_second) {} - bool init(ESQueryStack *s) { + bool init(ESQueryStack *s, ESQueryNode **pnode) override { bool valid = s->pop(&op); if (!valid) { return false; @@ -143,6 +137,7 @@ public: if (!valid) { return false; } + *pnode = this; return true; } virtual ~ESQueryNode_Bool() { @@ -162,46 +157,129 @@ public: }; +class ESQueryNodeLeafVal { +public: + ESQueryNodeLeafVal() = default; + virtual ~ESQueryNodeLeafVal() {} + + virtual bool init(const string& str_val) = 0; + virtual void encode_json(const string& field, Formatter *f) const = 0; +}; + +class ESQueryNodeLeafVal_Str : public ESQueryNodeLeafVal { + string val; +public: + ESQueryNodeLeafVal_Str() {} + bool init(const string& str_val) override { + val = str_val; + return true; + } + void encode_json(const string& field, Formatter *f) const { + ::encode_json(field.c_str(), val.c_str(), f); + } +}; + +class ESQueryNodeLeafVal_Int : public ESQueryNodeLeafVal { + int64_t val; +public: + ESQueryNodeLeafVal_Int() {} + bool init(const string& str_val) override { + string err; + val = strict_strtoll(str_val.c_str(), 10, &err); + if (!err.empty()) { + return false; + } + return true; + } + void encode_json(const string& field, Formatter *f) const { + ::encode_json(field.c_str(), val, f); + } +}; + +class ESQueryNodeLeafVal_Date : public ESQueryNodeLeafVal { + ceph::real_time val; +public: + ESQueryNodeLeafVal_Date() {} + bool init(const string& str_val) override { + if (parse_time(str_val.c_str(), &val) < 0) { + return false; + } + return true; + } + void encode_json(const string& field, Formatter *f) const { + utime_t ut(val); + ::encode_json(field.c_str(), ut, f); + } +}; + class ESQueryNode_Op : public ESQueryNode { protected: string op; string field; - string val; + ESQueryNodeLeafVal *val{nullptr}; + ESEntityTypeMap::EntityType entity_type{ESEntityTypeMap::ES_ENTITY_NONE}; + + bool val_from_str(const string& str_val) { + switch (entity_type) { + case ESEntityTypeMap::ES_ENTITY_DATE: + val = new ESQueryNodeLeafVal_Date; + break; + case ESEntityTypeMap::ES_ENTITY_INT: + val = new ESQueryNodeLeafVal_Int; + break; + default: + val = new ESQueryNodeLeafVal_Str; + } + return val->init(str_val); + } public: ESQueryNode_Op(ESQueryCompiler *compiler) : ESQueryNode(compiler) {} - bool init(ESQueryStack *s) { + ~ESQueryNode_Op() { + delete val; + } + virtual bool init(ESQueryStack *s, ESQueryNode **pnode) override { + string str_val; bool valid = s->pop(&op) && - s->pop(&val) && + s->pop(&str_val) && s->pop(&field); if (!valid) { return false; } + ESQueryNode *effective_node; + if (!handle_nested(&effective_node)) { + return false; + } + if (!val_from_str(str_val)) { + return false; + } + *pnode = effective_node; return true; } + bool handle_nested(ESQueryNode **pnode); virtual void dump(Formatter *f) const = 0; - bool leaf_field_name(string *name) override { - *name = field; - return true; - } - - void leaf_field_rename(const string& new_name) override { - field = new_name; - } }; class ESQueryNode_Op_Equal : public ESQueryNode_Op { + string str_val; public: ESQueryNode_Op_Equal(ESQueryCompiler *compiler) : ESQueryNode_Op(compiler) {} ESQueryNode_Op_Equal(ESQueryCompiler *compiler, const string& f, const string& v) : ESQueryNode_Op(compiler) { op = "=="; field = f; - val = v; + str_val = v; + } + + bool init(ESQueryStack *s, ESQueryNode **pnode) override { + if (op.empty()) { + return ESQueryNode_Op::init(s, pnode); + } + return val_from_str(str_val); } virtual void dump(Formatter *f) const { f->open_object_section("term"); - encode_json(field.c_str(), val.c_str(), f); + val->encode_json(field, f); f->close_section(); } }; @@ -214,7 +292,7 @@ public: virtual void dump(Formatter *f) const { f->open_object_section("range"); f->open_object_section(field.c_str()); - encode_json(range_str.c_str(), val.c_str(), f); + val->encode_json(range_str, f); f->close_section(); f->close_section(); } @@ -279,6 +357,44 @@ string ESQueryNode_Op_Nested::type_str() const { return "date"; } +bool ESQueryNode_Op::handle_nested(ESQueryNode **pnode) +{ + string field_name = field; + const string& custom_prefix = compiler->get_custom_prefix(); + if (!boost::algorithm::starts_with(field_name, custom_prefix)) { + *pnode = this; + auto m = compiler->get_generic_type_map(); + if (m) { + return m->find(field_name, &entity_type); + } + return false; + } + + field_name = field_name.substr(custom_prefix.size()); + auto m = compiler->get_custom_type_map(); + if (m) { + m->find(field_name, &entity_type); + /* ignoring returned bool, for now just treat it as string */ + } + + ESQueryNode_Op_Nested_Parent *new_node; + switch (entity_type) { + case ESEntityTypeMap::ES_ENTITY_INT: + new_node = new ESQueryNode_Op_Nested(compiler, field_name, this); + break; + case ESEntityTypeMap::ES_ENTITY_DATE: + new_node = new ESQueryNode_Op_Nested(compiler, field_name, this); + break; + default: + new_node = new ESQueryNode_Op_Nested(compiler, field_name, this); + } + + field = new_node->get_custom_leaf_field_name(); + *pnode = new_node; + + return true; +} + static bool is_bool_op(const string& str) { return (str == "or" || str == "and"); @@ -314,39 +430,10 @@ static bool alloc_node(ESQueryCompiler *compiler, ESQueryStack *s, ESQueryNode * node = new ESQueryNode_Op_Range(compiler, iter->second); } - if (!node->init(s)) { + if (!node->init(s, pnode)) { delete node; return false; } - string field_name; - string custom_prefix = compiler->get_custom_prefix(); - if (node->leaf_field_name(&field_name) && - boost::algorithm::starts_with(field_name, custom_prefix)) { - ESQueryNode *leaf_node = node; - field_name = field_name.substr(custom_prefix.size()); - ESEntityTypeMap::EntityType entity_type = ESEntityTypeMap::ES_ENTITY_NONE; - auto m = compiler->get_custom_type_map(); - if (m) { - entity_type = m->find(field_name); - } - - ESQueryNode_Op_Nested_Parent *new_node; - switch (entity_type) { - case ESEntityTypeMap::ES_ENTITY_INT: - new_node = new ESQueryNode_Op_Nested(compiler, field_name, node); - break; - case ESEntityTypeMap::ES_ENTITY_DATE: - new_node = new ESQueryNode_Op_Nested(compiler, field_name, node); - break; - default: - new_node = new ESQueryNode_Op_Nested(compiler, field_name, node); - } - - leaf_node->leaf_field_rename(new_node->get_custom_leaf_field_name()); - - node = new_node; - } - *pnode = node; return true; } diff --git a/src/rgw/rgw_es_query.h b/src/rgw/rgw_es_query.h index 79b221fd469ec..913c95e097ca6 100644 --- a/src/rgw/rgw_es_query.h +++ b/src/rgw/rgw_es_query.h @@ -74,13 +74,15 @@ struct ESEntityTypeMap { ESEntityTypeMap(map& _m) : m(_m) {} - EntityType find(const string& entity) { + bool find(const string& entity, EntityType *ptype) { auto i = m.find(entity); if (i != m.end()) { - return i->second; + *ptype = i->second; + return true; } - return ES_ENTITY_NONE; + *ptype = ES_ENTITY_NONE; + return false; } }; @@ -95,6 +97,7 @@ class ESQueryCompiler { list > eq_conds; + ESEntityTypeMap *generic_type_map{nullptr}; ESEntityTypeMap *custom_type_map{nullptr}; public: @@ -108,6 +111,13 @@ public: bool compile(); void dump(Formatter *f) const; + void set_generic_type_map(ESEntityTypeMap *entity_map) { + generic_type_map = entity_map; + } + + ESEntityTypeMap *get_generic_type_map() { + return generic_type_map; + } const string& get_custom_prefix() { return custom_prefix; } void set_custom_type_map(ESEntityTypeMap *entity_map) { -- 2.39.5