From: Yehuda Sadeh Date: Sat, 18 Mar 2017 20:27:42 +0000 (-0700) Subject: rgw: rename rgw_rest_es.cc to rgw_es_query.cc X-Git-Tag: ses5-milestone6~9^2~3^2~63 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=42130df362075e326994ff75c6b309572f989c22;p=ceph.git rgw: rename rgw_rest_es.cc to rgw_es_query.cc Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index d8e3c564b60..fc7aa1ee451 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -195,7 +195,7 @@ target_link_libraries(radosgw-admin rgw_a librados install(TARGETS radosgw-admin DESTINATION bin) set(radosgw_es_srcs - rgw_rest_es.cc) + rgw_es_query.cc) add_executable(radosgw-es ${radosgw_es_srcs}) target_link_libraries(radosgw-es rgw_a librados cls_rgw_client cls_lock_client cls_refcount_client diff --git a/src/rgw/rgw_es_query.cc b/src/rgw/rgw_es_query.cc new file mode 100644 index 00000000000..fc35786dfa4 --- /dev/null +++ b/src/rgw/rgw_es_query.cc @@ -0,0 +1,559 @@ +#include +#include +#include +#include +#include + +#include "common/ceph_json.h" +#include "rgw_common.h" + +using namespace std; + +bool pop_front(list& l, string *s) +{ + if (l.empty()) { + return false; + } + *s = l.front(); + l.pop_front(); + return true; +} + +map operator_map = { + { "or", 1 }, + { "and", 2 }, + { "<", 3 }, + { "<=", 3 }, + { "==", 3 }, + { ">=", 3 }, + { ">", 3 }, +}; + +bool is_operator(const string& s) +{ + return (operator_map.find(s) != operator_map.end()); +} + +int operand_value(const string& op) +{ + auto i = operator_map.find(op); + if (i == operator_map.end()) { + return 0; + } + + return i->second; +} + +int check_precedence(const string& op1, const string& op2) +{ + return operand_value(op1) - operand_value(op2); +} + +static bool infix_to_prefix(list& source, list *out) +{ + list operator_stack; + list operand_stack; + + operator_stack.push_front("("); + source.push_back(")"); + + for (string& entity : source) { + if (entity == "(") { + operator_stack.push_front(entity); + } else if (entity == ")") { + string popped_operator; + if (!pop_front(operator_stack, &popped_operator)) { + return false; + } + + while (popped_operator != "(") { + operand_stack.push_front(popped_operator); + if (!pop_front(operator_stack, &popped_operator)) { + return false; + } + } + + } else if (is_operator(entity)) { + string popped_operator; + if (!pop_front(operator_stack, &popped_operator)) { + return false; + } + + int precedence = check_precedence(popped_operator, entity); + + while (precedence >= 0) { + operand_stack.push_front(popped_operator); + if (!pop_front(operator_stack, &popped_operator)) { + return false; + } + precedence = check_precedence(popped_operator, entity); + } + + operator_stack.push_front(popped_operator); + operator_stack.push_front(entity); + } else { + operand_stack.push_front(entity); + } + + } + + if (!operator_stack.empty()) { + return false; + } + + out->swap(operand_stack); + return true; +} + +class ESQueryStack { + list l; + list::iterator iter; + +public: + ESQueryStack(list& src) { + assign(src); + } + + ESQueryStack() {} + + void assign(list& src) { + l.swap(src); + iter = l.begin(); + } + + bool peek(string *dest) { + if (done()) { + return false; + } + *dest = *iter; + return true; + } + + bool pop(string *dest) { + bool valid = peek(dest); + if (!valid) { + return false; + } + ++iter; + return true; + } + + bool done() { + return (iter == l.end()); + } +}; + +class ESQueryNode { +public: + ESQueryNode() {} + virtual ~ESQueryNode() {} + + virtual bool init(ESQueryStack *s) = 0; + + virtual void dump(Formatter *f) const = 0; + + virtual bool leaf_field_name(string *name) { + return false; + } + + virtual void leaf_field_rename(const string& new_name) {} +}; + +static bool alloc_node(ESQueryStack *s, ESQueryNode **pnode); + +class ESQueryNode_Bool : public ESQueryNode { + string op; + ESQueryNode *first{nullptr}; + ESQueryNode *second{nullptr}; +public: + ESQueryNode_Bool() {} + bool init(ESQueryStack *s) { + bool valid = s->pop(&op); + if (!valid) { + return false; + } + valid = alloc_node(s, &first) && + alloc_node(s, &second); + if (!valid) { + return false; + } + return true; + } + virtual ~ESQueryNode_Bool() { + delete first; + delete second; + } + + void dump(Formatter *f) const { + f->open_object_section("bool"); + const char *section = (op == "and" ? "must" : "should"); + f->open_array_section(section); + encode_json("entry", *first, f); + encode_json("entry", *second, f); + f->close_section(); + f->close_section(); + } + +}; + +class ESQueryNode_Op : public ESQueryNode { +protected: + string op; + string field; + string val; +public: + ESQueryNode_Op() {} + bool init(ESQueryStack *s) { + bool valid = s->pop(&op) && + s->pop(&val) && + s->pop(&field); + if (!valid) { + return false; + } + return true; + } + + virtual void dump(Formatter *f) const = 0; + bool leaf_field_name(string *name) override { + *name = field; + return true; + } + + void leaf_field_rename(const string& new_name) override { + field = new_name; + } +}; + +class ESQueryNode_Op_Equal : public ESQueryNode_Op { +public: + ESQueryNode_Op_Equal() {} + + virtual void dump(Formatter *f) const { + f->open_object_section("term"); + encode_json(field.c_str(), val.c_str(), f); + f->close_section(); + } +}; + +class ESQueryNode_Op_Range : public ESQueryNode_Op { + string range_str; +public: + ESQueryNode_Op_Range(const string& rs) : range_str(rs) {} + + virtual void dump(Formatter *f) const { + f->open_object_section("range"); + f->open_object_section(field.c_str()); + encode_json(range_str.c_str(), val.c_str(), f); + f->close_section(); + f->close_section(); + } +}; + +class ESQueryNode_Op_Nested : public ESQueryNode_Op { + string name; + ESQueryNode *next; +public: + ESQueryNode_Op_Nested(const string& _name, ESQueryNode *_next) : name(_name), next(_next) {} + ~ESQueryNode_Op_Nested() { + delete next; + } + + virtual void dump(Formatter *f) const { + f->open_object_section("nested"); + encode_json("path", "meta.custom-string", f); + f->open_object_section("query"); + f->open_object_section("bool"); + f->open_array_section("must"); + f->open_object_section("entry"); + f->open_object_section("match"); + encode_json("meta.custom-string.name", name.c_str(), f); + f->close_section(); + f->close_section(); + encode_json("entry", *next, f); + f->close_section(); + f->close_section(); + f->close_section(); + f->close_section(); + } +}; + +static bool is_bool_op(const string& str) +{ + return (str == "or" || str == "and"); +} + +static bool alloc_node(ESQueryStack *s, ESQueryNode **pnode) +{ + string op; + bool valid = s->peek(&op); + if (!valid) { + return false; + } + + ESQueryNode *node; + + if (is_bool_op(op)) { + node = new ESQueryNode_Bool(); + } else if (op == "==") { + node = new ESQueryNode_Op_Equal(); + } else { + static map range_op_map = { + { "<", "lt"}, + { "<=", "lte"}, + { ">=", "gte"}, + { ">", "gt"}, + }; + + auto iter = range_op_map.find(op); + if (iter == range_op_map.end()) { + return false; + } + + node = new ESQueryNode_Op_Range(iter->second); + } + + if (!node->init(s)) { + delete node; + return false; + } + string field_name; + if (node->leaf_field_name(&field_name) && + boost::algorithm::starts_with(field_name, "meta.custom.")) { + node->leaf_field_rename("meta.custom-string.value"); + field_name = field_name.substr(sizeof("meta.custom.")-1); + node = new ESQueryNode_Op_Nested(field_name, node); + + } + *pnode = node; + return true; +} + + +bool is_key_char(char c) +{ + switch (c) { + case '(': + case ')': + case '<': + case '>': + case '@': + case ',': + case ';': + case ':': + case '\\': + case '"': + case '/': + case '[': + case ']': + case '?': + case '=': + case '{': + case '}': + case ' ': + case '\t': + return false; + }; + return (isascii(c) > 0); +} + +static bool is_op_char(char c) +{ + switch (c) { + case '<': + case '=': + case '>': + return true; + }; + return false; +} + +static bool is_val_char(char c) +{ + if (isspace(c)) { + return false; + } + return (c != ')'); +} + +class ESInfixQueryParser { + string query; + int size; + const char *str; + int pos{0}; + list args; + + void skip_whitespace(const char *str, int size, int& pos) { + while (pos < size && isspace(str[pos])) { + ++pos; + } + } + + bool get_next_token(bool (*filter)(char)) { + skip_whitespace(str, size, pos); + int token_start = pos; + while (pos < size && filter(str[pos])) { + ++pos; + } + if (pos == token_start) { + return false; + } + string token = string(str + token_start, pos - token_start); + args.push_back(token); + return true; + } + + bool parse_condition() { + /* + * condition: + * + * whereas key: needs to conform to http header field restrictions + * operator: one of the following: < <= == >= > + * val: ascii, terminated by either space or ')' (or end of string) + */ + + /* parse key */ + bool valid = get_next_token(is_key_char) && + get_next_token(is_op_char) && + get_next_token(is_val_char); + + if (!valid) { + return false; + } + + return true; + } + + bool parse_and_or() { + skip_whitespace(str, size, pos); + if (pos + 3 <= size && strncmp(str + pos, "and", 3) == 0) { + pos += 3; + args.push_back("and"); + return true; + } + + if (pos + 2 <= size && strncmp(str + pos, "or", 2) == 0) { + pos += 2; + args.push_back("or"); + return true; + } + + return false; + } + + bool parse_specific_char(const char *pchar) { + skip_whitespace(str, size, pos); + if (pos >= size) { + return false; + } + if (str[pos] != *pchar) { + return false; + } + + args.push_back(pchar); + ++pos; + return true; + } + + bool parse_open_bracket() { + return parse_specific_char("("); + } + + bool parse_close_bracket() { + return parse_specific_char(")"); + } + +public: + ESInfixQueryParser(const string& _query) : query(_query), size(query.size()), str(query.c_str()) {} + bool parse(list *result) { + /* + * expression: [(][[and/or]][)][and/or]... + */ + + while (pos < size) { + parse_open_bracket(); + if (!parse_condition()) { + return false; + } + parse_close_bracket(); + parse_and_or(); + } + + result->swap(args); + + return true; + } +}; + +class ESQueryCompiler { + ESInfixQueryParser parser; + ESQueryStack stack; + ESQueryNode *query_root{nullptr}; + + bool convert(list& infix) { + list prefix; + if (!infix_to_prefix(infix, &prefix)) { + return false; + } + stack.assign(prefix); + if (!alloc_node(&stack, &query_root)) { + return false; + } + if (!stack.done()) { + return false; + } + return true; + } + +public: + ESQueryCompiler(const string& query) : parser(query) {} + ~ESQueryCompiler() { + delete query_root; + } + + bool compile() { + list infix; + if (!parser.parse(&infix)) { + return false; + } + + if (!convert(infix)) { + return false; + } + + return true; + } + + void dump(Formatter *f) const { + encode_json("query", *query_root, f); + } +}; + + +int main(int argc, char *argv[]) +{ + list infix; + + string expr; + + if (argc > 1) { + expr = argv[1]; + } else { + expr = "age >= 30"; + } + + ESQueryCompiler es_query(expr); + + bool valid = es_query.compile(); + if (!valid) { + cout << "invalid query, failed generating request json" << std::endl; + return EINVAL; + } + + JSONFormatter f; + encode_json("root", es_query, &f); + + f.flush(cout); + + return 0; +} + diff --git a/src/rgw/rgw_rest_es.cc b/src/rgw/rgw_rest_es.cc deleted file mode 100644 index fc35786dfa4..00000000000 --- a/src/rgw/rgw_rest_es.cc +++ /dev/null @@ -1,559 +0,0 @@ -#include -#include -#include -#include -#include - -#include "common/ceph_json.h" -#include "rgw_common.h" - -using namespace std; - -bool pop_front(list& l, string *s) -{ - if (l.empty()) { - return false; - } - *s = l.front(); - l.pop_front(); - return true; -} - -map operator_map = { - { "or", 1 }, - { "and", 2 }, - { "<", 3 }, - { "<=", 3 }, - { "==", 3 }, - { ">=", 3 }, - { ">", 3 }, -}; - -bool is_operator(const string& s) -{ - return (operator_map.find(s) != operator_map.end()); -} - -int operand_value(const string& op) -{ - auto i = operator_map.find(op); - if (i == operator_map.end()) { - return 0; - } - - return i->second; -} - -int check_precedence(const string& op1, const string& op2) -{ - return operand_value(op1) - operand_value(op2); -} - -static bool infix_to_prefix(list& source, list *out) -{ - list operator_stack; - list operand_stack; - - operator_stack.push_front("("); - source.push_back(")"); - - for (string& entity : source) { - if (entity == "(") { - operator_stack.push_front(entity); - } else if (entity == ")") { - string popped_operator; - if (!pop_front(operator_stack, &popped_operator)) { - return false; - } - - while (popped_operator != "(") { - operand_stack.push_front(popped_operator); - if (!pop_front(operator_stack, &popped_operator)) { - return false; - } - } - - } else if (is_operator(entity)) { - string popped_operator; - if (!pop_front(operator_stack, &popped_operator)) { - return false; - } - - int precedence = check_precedence(popped_operator, entity); - - while (precedence >= 0) { - operand_stack.push_front(popped_operator); - if (!pop_front(operator_stack, &popped_operator)) { - return false; - } - precedence = check_precedence(popped_operator, entity); - } - - operator_stack.push_front(popped_operator); - operator_stack.push_front(entity); - } else { - operand_stack.push_front(entity); - } - - } - - if (!operator_stack.empty()) { - return false; - } - - out->swap(operand_stack); - return true; -} - -class ESQueryStack { - list l; - list::iterator iter; - -public: - ESQueryStack(list& src) { - assign(src); - } - - ESQueryStack() {} - - void assign(list& src) { - l.swap(src); - iter = l.begin(); - } - - bool peek(string *dest) { - if (done()) { - return false; - } - *dest = *iter; - return true; - } - - bool pop(string *dest) { - bool valid = peek(dest); - if (!valid) { - return false; - } - ++iter; - return true; - } - - bool done() { - return (iter == l.end()); - } -}; - -class ESQueryNode { -public: - ESQueryNode() {} - virtual ~ESQueryNode() {} - - virtual bool init(ESQueryStack *s) = 0; - - virtual void dump(Formatter *f) const = 0; - - virtual bool leaf_field_name(string *name) { - return false; - } - - virtual void leaf_field_rename(const string& new_name) {} -}; - -static bool alloc_node(ESQueryStack *s, ESQueryNode **pnode); - -class ESQueryNode_Bool : public ESQueryNode { - string op; - ESQueryNode *first{nullptr}; - ESQueryNode *second{nullptr}; -public: - ESQueryNode_Bool() {} - bool init(ESQueryStack *s) { - bool valid = s->pop(&op); - if (!valid) { - return false; - } - valid = alloc_node(s, &first) && - alloc_node(s, &second); - if (!valid) { - return false; - } - return true; - } - virtual ~ESQueryNode_Bool() { - delete first; - delete second; - } - - void dump(Formatter *f) const { - f->open_object_section("bool"); - const char *section = (op == "and" ? "must" : "should"); - f->open_array_section(section); - encode_json("entry", *first, f); - encode_json("entry", *second, f); - f->close_section(); - f->close_section(); - } - -}; - -class ESQueryNode_Op : public ESQueryNode { -protected: - string op; - string field; - string val; -public: - ESQueryNode_Op() {} - bool init(ESQueryStack *s) { - bool valid = s->pop(&op) && - s->pop(&val) && - s->pop(&field); - if (!valid) { - return false; - } - return true; - } - - virtual void dump(Formatter *f) const = 0; - bool leaf_field_name(string *name) override { - *name = field; - return true; - } - - void leaf_field_rename(const string& new_name) override { - field = new_name; - } -}; - -class ESQueryNode_Op_Equal : public ESQueryNode_Op { -public: - ESQueryNode_Op_Equal() {} - - virtual void dump(Formatter *f) const { - f->open_object_section("term"); - encode_json(field.c_str(), val.c_str(), f); - f->close_section(); - } -}; - -class ESQueryNode_Op_Range : public ESQueryNode_Op { - string range_str; -public: - ESQueryNode_Op_Range(const string& rs) : range_str(rs) {} - - virtual void dump(Formatter *f) const { - f->open_object_section("range"); - f->open_object_section(field.c_str()); - encode_json(range_str.c_str(), val.c_str(), f); - f->close_section(); - f->close_section(); - } -}; - -class ESQueryNode_Op_Nested : public ESQueryNode_Op { - string name; - ESQueryNode *next; -public: - ESQueryNode_Op_Nested(const string& _name, ESQueryNode *_next) : name(_name), next(_next) {} - ~ESQueryNode_Op_Nested() { - delete next; - } - - virtual void dump(Formatter *f) const { - f->open_object_section("nested"); - encode_json("path", "meta.custom-string", f); - f->open_object_section("query"); - f->open_object_section("bool"); - f->open_array_section("must"); - f->open_object_section("entry"); - f->open_object_section("match"); - encode_json("meta.custom-string.name", name.c_str(), f); - f->close_section(); - f->close_section(); - encode_json("entry", *next, f); - f->close_section(); - f->close_section(); - f->close_section(); - f->close_section(); - } -}; - -static bool is_bool_op(const string& str) -{ - return (str == "or" || str == "and"); -} - -static bool alloc_node(ESQueryStack *s, ESQueryNode **pnode) -{ - string op; - bool valid = s->peek(&op); - if (!valid) { - return false; - } - - ESQueryNode *node; - - if (is_bool_op(op)) { - node = new ESQueryNode_Bool(); - } else if (op == "==") { - node = new ESQueryNode_Op_Equal(); - } else { - static map range_op_map = { - { "<", "lt"}, - { "<=", "lte"}, - { ">=", "gte"}, - { ">", "gt"}, - }; - - auto iter = range_op_map.find(op); - if (iter == range_op_map.end()) { - return false; - } - - node = new ESQueryNode_Op_Range(iter->second); - } - - if (!node->init(s)) { - delete node; - return false; - } - string field_name; - if (node->leaf_field_name(&field_name) && - boost::algorithm::starts_with(field_name, "meta.custom.")) { - node->leaf_field_rename("meta.custom-string.value"); - field_name = field_name.substr(sizeof("meta.custom.")-1); - node = new ESQueryNode_Op_Nested(field_name, node); - - } - *pnode = node; - return true; -} - - -bool is_key_char(char c) -{ - switch (c) { - case '(': - case ')': - case '<': - case '>': - case '@': - case ',': - case ';': - case ':': - case '\\': - case '"': - case '/': - case '[': - case ']': - case '?': - case '=': - case '{': - case '}': - case ' ': - case '\t': - return false; - }; - return (isascii(c) > 0); -} - -static bool is_op_char(char c) -{ - switch (c) { - case '<': - case '=': - case '>': - return true; - }; - return false; -} - -static bool is_val_char(char c) -{ - if (isspace(c)) { - return false; - } - return (c != ')'); -} - -class ESInfixQueryParser { - string query; - int size; - const char *str; - int pos{0}; - list args; - - void skip_whitespace(const char *str, int size, int& pos) { - while (pos < size && isspace(str[pos])) { - ++pos; - } - } - - bool get_next_token(bool (*filter)(char)) { - skip_whitespace(str, size, pos); - int token_start = pos; - while (pos < size && filter(str[pos])) { - ++pos; - } - if (pos == token_start) { - return false; - } - string token = string(str + token_start, pos - token_start); - args.push_back(token); - return true; - } - - bool parse_condition() { - /* - * condition: - * - * whereas key: needs to conform to http header field restrictions - * operator: one of the following: < <= == >= > - * val: ascii, terminated by either space or ')' (or end of string) - */ - - /* parse key */ - bool valid = get_next_token(is_key_char) && - get_next_token(is_op_char) && - get_next_token(is_val_char); - - if (!valid) { - return false; - } - - return true; - } - - bool parse_and_or() { - skip_whitespace(str, size, pos); - if (pos + 3 <= size && strncmp(str + pos, "and", 3) == 0) { - pos += 3; - args.push_back("and"); - return true; - } - - if (pos + 2 <= size && strncmp(str + pos, "or", 2) == 0) { - pos += 2; - args.push_back("or"); - return true; - } - - return false; - } - - bool parse_specific_char(const char *pchar) { - skip_whitespace(str, size, pos); - if (pos >= size) { - return false; - } - if (str[pos] != *pchar) { - return false; - } - - args.push_back(pchar); - ++pos; - return true; - } - - bool parse_open_bracket() { - return parse_specific_char("("); - } - - bool parse_close_bracket() { - return parse_specific_char(")"); - } - -public: - ESInfixQueryParser(const string& _query) : query(_query), size(query.size()), str(query.c_str()) {} - bool parse(list *result) { - /* - * expression: [(][[and/or]][)][and/or]... - */ - - while (pos < size) { - parse_open_bracket(); - if (!parse_condition()) { - return false; - } - parse_close_bracket(); - parse_and_or(); - } - - result->swap(args); - - return true; - } -}; - -class ESQueryCompiler { - ESInfixQueryParser parser; - ESQueryStack stack; - ESQueryNode *query_root{nullptr}; - - bool convert(list& infix) { - list prefix; - if (!infix_to_prefix(infix, &prefix)) { - return false; - } - stack.assign(prefix); - if (!alloc_node(&stack, &query_root)) { - return false; - } - if (!stack.done()) { - return false; - } - return true; - } - -public: - ESQueryCompiler(const string& query) : parser(query) {} - ~ESQueryCompiler() { - delete query_root; - } - - bool compile() { - list infix; - if (!parser.parse(&infix)) { - return false; - } - - if (!convert(infix)) { - return false; - } - - return true; - } - - void dump(Formatter *f) const { - encode_json("query", *query_root, f); - } -}; - - -int main(int argc, char *argv[]) -{ - list infix; - - string expr; - - if (argc > 1) { - expr = argv[1]; - } else { - expr = "age >= 30"; - } - - ESQueryCompiler es_query(expr); - - bool valid = es_query.compile(); - if (!valid) { - cout << "invalid query, failed generating request json" << std::endl; - return EINVAL; - } - - JSONFormatter f; - encode_json("root", es_query, &f); - - f.flush(cout); - - return 0; -} -