From: Yehuda Sadeh Date: Tue, 21 Mar 2017 02:07:21 +0000 (-0700) Subject: rgw: work on REST handler for es module X-Git-Tag: ses5-milestone6~9^2~3^2~62 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bb46f19e44918044d8068b54e1b3ef7c43e7fac8;p=ceph.git rgw: work on REST handler for es module Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index fc7aa1ee4516..3fa369956664 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -73,6 +73,7 @@ set(rgw_a_srcs rgw_data_sync.cc rgw_sync_module.cc rgw_sync_module_es.cc + rgw_sync_module_es_rest.cc rgw_sync_module_log.cc rgw_period_history.cc rgw_period_puller.cc diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index bc049346e732..529dfebe0984 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -464,8 +464,8 @@ enum RGWOpType { /* rgw specific */ RGW_OP_ADMIN_SET_METADATA, RGW_OP_GET_OBJ_LAYOUT, - - RGW_OP_BULK_UPLOAD + RGW_OP_BULK_UPLOAD, + RGW_OP_METADATA_SEARCH, }; class RGWAccessControlPolicy; diff --git a/src/rgw/rgw_es_query.cc b/src/rgw/rgw_es_query.cc index fc35786dfa4e..e89d261d1d67 100644 --- a/src/rgw/rgw_es_query.cc +++ b/src/rgw/rgw_es_query.cc @@ -6,6 +6,7 @@ #include "common/ceph_json.h" #include "rgw_common.h" +#include "rgw_es_query.h" using namespace std; @@ -105,44 +106,6 @@ static bool infix_to_prefix(list& source, list *out) return true; } -class ESQueryStack { - list l; - list::iterator iter; - -public: - ESQueryStack(list& src) { - assign(src); - } - - ESQueryStack() {} - - void assign(list& src) { - l.swap(src); - iter = l.begin(); - } - - bool peek(string *dest) { - if (done()) { - return false; - } - *dest = *iter; - return true; - } - - bool pop(string *dest) { - bool valid = peek(dest); - if (!valid) { - return false; - } - ++iter; - return true; - } - - bool done() { - return (iter == l.end()); - } -}; - class ESQueryNode { public: ESQueryNode() {} @@ -375,158 +338,140 @@ static bool is_val_char(char c) return (c != ')'); } -class ESInfixQueryParser { - string query; - int size; - const char *str; - int pos{0}; - list args; - - void skip_whitespace(const char *str, int size, int& pos) { - while (pos < size && isspace(str[pos])) { - ++pos; - } +void ESInfixQueryParser::skip_whitespace(const char *str, int size, int& pos) { + while (pos < size && isspace(str[pos])) { + ++pos; } +} - bool get_next_token(bool (*filter)(char)) { - skip_whitespace(str, size, pos); - int token_start = pos; - while (pos < size && filter(str[pos])) { - ++pos; - } - if (pos == token_start) { - return false; - } - string token = string(str + token_start, pos - token_start); - args.push_back(token); - return true; +bool ESInfixQueryParser::get_next_token(bool (*filter)(char)) { + skip_whitespace(str, size, pos); + int token_start = pos; + while (pos < size && filter(str[pos])) { + ++pos; } - - bool parse_condition() { - /* - * condition: - * - * whereas key: needs to conform to http header field restrictions - * operator: one of the following: < <= == >= > - * val: ascii, terminated by either space or ')' (or end of string) - */ - - /* parse key */ - bool valid = get_next_token(is_key_char) && - get_next_token(is_op_char) && - get_next_token(is_val_char); - - if (!valid) { - return false; - } - - return true; + if (pos == token_start) { + return false; } + string token = string(str + token_start, pos - token_start); + args.push_back(token); + return true; +} - bool parse_and_or() { - skip_whitespace(str, size, pos); - if (pos + 3 <= size && strncmp(str + pos, "and", 3) == 0) { - pos += 3; - args.push_back("and"); - return true; - } +bool ESInfixQueryParser::parse_condition() { + /* + * condition: + * + * whereas key: needs to conform to http header field restrictions + * operator: one of the following: < <= == >= > + * val: ascii, terminated by either space or ')' (or end of string) + */ - if (pos + 2 <= size && strncmp(str + pos, "or", 2) == 0) { - pos += 2; - args.push_back("or"); - return true; - } + /* parse key */ + bool valid = get_next_token(is_key_char) && + get_next_token(is_op_char) && + get_next_token(is_val_char); + if (!valid) { return false; } - bool parse_specific_char(const char *pchar) { - skip_whitespace(str, size, pos); - if (pos >= size) { - return false; - } - if (str[pos] != *pchar) { - return false; - } + return true; +} - args.push_back(pchar); - ++pos; +bool ESInfixQueryParser::parse_and_or() { + skip_whitespace(str, size, pos); + if (pos + 3 <= size && strncmp(str + pos, "and", 3) == 0) { + pos += 3; + args.push_back("and"); return true; } - bool parse_open_bracket() { - return parse_specific_char("("); + if (pos + 2 <= size && strncmp(str + pos, "or", 2) == 0) { + pos += 2; + args.push_back("or"); + return true; } - bool parse_close_bracket() { - return parse_specific_char(")"); + return false; +} + +bool ESInfixQueryParser::parse_specific_char(const char *pchar) { + skip_whitespace(str, size, pos); + if (pos >= size) { + return false; + } + if (str[pos] != *pchar) { + return false; } -public: - ESInfixQueryParser(const string& _query) : query(_query), size(query.size()), str(query.c_str()) {} - bool parse(list *result) { - /* - * expression: [(][[and/or]][)][and/or]... - */ - - while (pos < size) { - parse_open_bracket(); - if (!parse_condition()) { - return false; - } - parse_close_bracket(); - parse_and_or(); - } + args.push_back(pchar); + ++pos; + return true; +} - result->swap(args); +bool ESInfixQueryParser::parse_open_bracket() { + return parse_specific_char("("); +} - return true; - } -}; +bool ESInfixQueryParser::parse_close_bracket() { + return parse_specific_char(")"); +} -class ESQueryCompiler { - ESInfixQueryParser parser; - ESQueryStack stack; - ESQueryNode *query_root{nullptr}; +bool ESInfixQueryParser::parse(list *result) { + /* + * expression: [(][[and/or]][)][and/or]... + */ - bool convert(list& infix) { - list prefix; - if (!infix_to_prefix(infix, &prefix)) { - return false; - } - stack.assign(prefix); - if (!alloc_node(&stack, &query_root)) { + while (pos < size) { + parse_open_bracket(); + if (!parse_condition()) { return false; } - if (!stack.done()) { - return false; - } - return true; + parse_close_bracket(); + parse_and_or(); } -public: - ESQueryCompiler(const string& query) : parser(query) {} - ~ESQueryCompiler() { - delete query_root; - } + result->swap(args); - bool compile() { - list infix; - if (!parser.parse(&infix)) { - return false; - } + return true; +} - if (!convert(infix)) { - return false; - } +bool ESQueryCompiler::convert(list& infix) { + list prefix; + if (!infix_to_prefix(infix, &prefix)) { + return false; + } + stack.assign(prefix); + if (!alloc_node(&stack, &query_root)) { + return false; + } + if (!stack.done()) { + return false; + } + return true; +} - return true; +ESQueryCompiler::~ESQueryCompiler() { + delete query_root; +} + +bool ESQueryCompiler::compile() { + list infix; + if (!parser.parse(&infix)) { + return false; } - void dump(Formatter *f) const { - encode_json("query", *query_root, f); + if (!convert(infix)) { + return false; } -}; + + return true; +} + +void ESQueryCompiler::dump(Formatter *f) const { + encode_json("query", *query_root, f); +} int main(int argc, char *argv[]) diff --git a/src/rgw/rgw_es_query.h b/src/rgw/rgw_es_query.h new file mode 100644 index 000000000000..c8b83759db6b --- /dev/null +++ b/src/rgw/rgw_es_query.h @@ -0,0 +1,81 @@ +#ifndef CEPH_RGW_ES_QUERY_H +#define CEPH_RGW_ES_QUERY_H + +class ESQueryStack { + list l; + list::iterator iter; + +public: + ESQueryStack(list& src) { + assign(src); + } + + ESQueryStack() {} + + void assign(list& src) { + l.swap(src); + iter = l.begin(); + } + + bool peek(string *dest) { + if (done()) { + return false; + } + *dest = *iter; + return true; + } + + bool pop(string *dest) { + bool valid = peek(dest); + if (!valid) { + return false; + } + ++iter; + return true; + } + + bool done() { + return (iter == l.end()); + } +}; + +class ESInfixQueryParser { + string query; + int size; + const char *str; + int pos{0}; + list args; + + void skip_whitespace(const char *str, int size, int& pos); + bool get_next_token(bool (*filter)(char)); + + bool parse_condition(); + bool parse_and_or(); + bool parse_specific_char(const char *pchar); + bool parse_open_bracket(); + bool parse_close_bracket(); + +public: + ESInfixQueryParser(const string& _query) : query(_query), size(query.size()), str(query.c_str()) {} + bool parse(list *result); +}; + +class ESQueryNode; + +class ESQueryCompiler { + ESInfixQueryParser parser; + ESQueryStack stack; + ESQueryNode *query_root{nullptr}; + + bool convert(list& infix); + +public: + ESQueryCompiler(const string& query) : parser(query) {} + ~ESQueryCompiler(); + + bool compile(); + void dump(Formatter *f) const; +}; + + +#endif diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index f018faba8d62..5c3dbb0582b6 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -182,6 +182,12 @@ static RGWRESTMgr *set_logging(RGWRESTMgr *mgr) return mgr; } +static RGWRESTMgr *rest_filter(RGWRados *store, int dialect, RGWRESTMgr *orig) +{ + RGWSyncModuleInstanceRef sync_module = store->get_sync_module(); + return sync_module->get_rest_filter(dialect, orig); +} + RGWRealmReloader *preloader = NULL; static void reloader_handler(int signum) @@ -377,7 +383,8 @@ int main(int argc, const char **argv) const bool swift_at_root = g_conf->rgw_swift_url_prefix == "/"; if (apis_map.count("s3") > 0 || s3website_enabled) { if (! swift_at_root) { - rest.register_default_mgr(set_logging(new RGWRESTMgr_S3(s3website_enabled))); + rest.register_default_mgr(set_logging(rest_filter(store, RGW_REST_S3, + new RGWRESTMgr_S3(s3website_enabled)))); } else { derr << "Cannot have the S3 or S3 Website enabled together with " << "Swift API placed in the root of hierarchy" << dendl; @@ -401,7 +408,8 @@ int main(int argc, const char **argv) if (! swift_at_root) { rest.register_resource(g_conf->rgw_swift_url_prefix, - set_logging(swift_resource)); + set_logging(rest_filter(store, RGW_REST_SWIFT, + swift_resource))); } else { if (store->get_zonegroup().zones.size() > 1) { derr << "Placing Swift API in the root of URL hierarchy while running" diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ac025c6e7d93..6caa57c06743 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -631,7 +631,7 @@ rgw::IAM::Environment rgw_build_iam_environment(RGWRados* store, return e; } -static void rgw_bucket_object_pre_exec(struct req_state *s) +void rgw_bucket_object_pre_exec(struct req_state *s) { if (s->expect_cont) dump_continue(s); diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 1b4f02f1b7f5..ed7812cf80b5 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -97,6 +97,9 @@ public: }; + +void rgw_bucket_object_pre_exec(struct req_state *s); + /** * Provide the base class for all ops. */ diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 7174a0a4e202..c8a3f8371fa8 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -2756,10 +2756,11 @@ RGWOp *RGWHandler_REST_Service_S3::op_post() RGWOp *RGWHandler_REST_Bucket_S3::get_obj_op(bool get_data) { // Non-website mode - if (get_data) + if (get_data) { return new RGWListBucket_ObjStore_S3; - else + } else { return new RGWStatBucket_ObjStore_S3; + } } RGWOp *RGWHandler_REST_Bucket_S3::op_get() diff --git a/src/rgw/rgw_sync_module.h b/src/rgw/rgw_sync_module.h index 07b30a0201bf..55b39b775e11 100644 --- a/src/rgw/rgw_sync_module.h +++ b/src/rgw/rgw_sync_module.h @@ -27,11 +27,16 @@ public: rgw_bucket_entry_owner& owner, bool versioned, uint64_t versioned_epoch, rgw_zone_set *zones_trace) = 0; }; +class RGWRESTMgr; + class RGWSyncModuleInstance { public: RGWSyncModuleInstance() {} virtual ~RGWSyncModuleInstance() {} virtual RGWDataSyncModule *get_data_handler() = 0; + virtual RGWRESTMgr *get_rest_filter(int dialect, RGWRESTMgr *orig) { + return orig; + } }; typedef std::shared_ptr RGWSyncModuleInstanceRef; diff --git a/src/rgw/rgw_sync_module_es.cc b/src/rgw/rgw_sync_module_es.cc index f04e26c9144a..7c2ff99731ec 100644 --- a/src/rgw/rgw_sync_module_es.cc +++ b/src/rgw/rgw_sync_module_es.cc @@ -4,8 +4,10 @@ #include "rgw_data_sync.h" #include "rgw_boost_asio_yield.h" #include "rgw_sync_module_es.h" +#include "rgw_sync_module_es_rest.h" #include "rgw_rest_conn.h" #include "rgw_cr_rest.h" +#include "rgw_op.h" #define dout_subsys ceph_subsys_rgw @@ -311,16 +313,32 @@ public: << " versioned=" << versioned << " versioned_epoch=" << versioned_epoch << dendl; return NULL; } + RGWRESTConn *get_rest_conn() { + return conf.conn; + } }; -class RGWElasticSyncModuleInstance : public RGWSyncModuleInstance { - RGWElasticDataSyncModule data_handler; -public: - RGWElasticSyncModuleInstance(CephContext *cct, const string& endpoint) : data_handler(cct, endpoint) {} - RGWDataSyncModule *get_data_handler() override { - return &data_handler; +RGWElasticSyncModuleInstance::RGWElasticSyncModuleInstance(CephContext *cct, const string& endpoint) +{ + data_handler = std::unique_ptr(new RGWElasticDataSyncModule(cct, endpoint)); +} + +RGWDataSyncModule *RGWElasticSyncModuleInstance::get_data_handler() +{ + return data_handler.get(); +} + +RGWRESTConn *RGWElasticSyncModuleInstance::get_rest_conn() +{ + return data_handler->get_rest_conn(); +} + +RGWRESTMgr *RGWElasticSyncModuleInstance::get_rest_filter(int dialect, RGWRESTMgr *orig) { + if (dialect != RGW_REST_S3) { + return orig; } -}; + return new RGWRESTMgr_MDSearch_S3(this); +} int RGWElasticSyncModule::create_instance(CephContext *cct, map& config, RGWSyncModuleInstanceRef *instance) { string endpoint; diff --git a/src/rgw/rgw_sync_module_es.h b/src/rgw/rgw_sync_module_es.h index 73c8368571d8..28b11716fd0c 100644 --- a/src/rgw/rgw_sync_module_es.h +++ b/src/rgw/rgw_sync_module_es.h @@ -12,4 +12,16 @@ public: int create_instance(CephContext *cct, map& config, RGWSyncModuleInstanceRef *instance) override; }; +class RGWElasticDataSyncModule; +class RGWRESTConn; + +class RGWElasticSyncModuleInstance : public RGWSyncModuleInstance { + std::unique_ptr data_handler; +public: + RGWElasticSyncModuleInstance(CephContext *cct, const string& endpoint); + RGWDataSyncModule *get_data_handler() override; + RGWRESTMgr *get_rest_filter(int dialect, RGWRESTMgr *orig) override; + RGWRESTConn *get_rest_conn(); +}; + #endif diff --git a/src/rgw/rgw_sync_module_es_rest.cc b/src/rgw/rgw_sync_module_es_rest.cc new file mode 100644 index 000000000000..f0c9d3062a4e --- /dev/null +++ b/src/rgw/rgw_sync_module_es_rest.cc @@ -0,0 +1,118 @@ +#include "rgw_sync_module_es.h" +#include "rgw_sync_module_es_rest.h" +#include "rgw_es_query.h" +#include "rgw_op.h" +#include "rgw_rest.h" +#include "rgw_rest_s3.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +class RGWMetadataSearch : public RGWOp { +protected: + string expression; + +public: + RGWMetadataSearch() {} + + int verify_permission() { + return 0; + } + virtual int get_params() = 0; + void pre_exec(); + void execute(); + + virtual void send_response() = 0; + virtual const string name() { return "metadata_search"; } + virtual RGWOpType get_type() { return RGW_OP_METADATA_SEARCH; } + virtual uint32_t op_mask() { return RGW_OP_TYPE_READ; } +}; + +void RGWMetadataSearch::pre_exec() +{ + rgw_bucket_object_pre_exec(s); +} + + +void RGWMetadataSearch::execute() +{ + op_ret = get_params(); + if (op_ret < 0) + return; + + ESQueryCompiler es_query(expression); + + bool valid = es_query.compile(); + if (!valid) { + ldout(s->cct, 10) << "invalid query, failed generating request json" << dendl; + op_ret = -EINVAL; + return; + } + +} + +class RGWMetadataSearch_ObjStore_S3 : public RGWMetadataSearch { +public: + RGWMetadataSearch_ObjStore_S3() {} + + int get_params() override { + expression = s->info.args.get("query"); + return 0; + } + void send_response() override { + if (op_ret) + set_req_state_err(s, op_ret); + dump_errno(s); + end_header(s, this, "application/xml"); + + if (op_ret < 0) { + return; + } + + // TODO + + } +}; + +class RGWHandler_REST_MDSearch_S3 : public RGWHandler_REST_S3 { +protected: + RGWOp *op_get() { + if (!s->info.args.exists("query")) { + return nullptr; + } + return new RGWMetadataSearch_ObjStore_S3; + } + RGWOp *op_head() { + return nullptr; + } + RGWOp *op_post() { + return nullptr; + } +public: + RGWHandler_REST_MDSearch_S3() {} + virtual ~RGWHandler_REST_MDSearch_S3() {} +}; + + +RGWHandler_REST* RGWRESTMgr_MDSearch_S3::get_handler(struct req_state* const s, + const rgw::auth::StrategyRegistry& auth_registry, + const std::string& frontend_prefix) +{ + int ret = + RGWHandler_REST_S3::init_from_header(s, + RGW_FORMAT_XML, true); + if (ret < 0) { + return nullptr; + } + + if (!s->object.empty()) { + return nullptr; + } + + RGWHandler_REST *handler = new RGWHandler_REST_MDSearch_S3; + + ldout(s->cct, 20) << __func__ << " handler=" << typeid(*handler).name() + << dendl; + return handler; +} + diff --git a/src/rgw/rgw_sync_module_es_rest.h b/src/rgw/rgw_sync_module_es_rest.h new file mode 100644 index 000000000000..7578b81273f0 --- /dev/null +++ b/src/rgw/rgw_sync_module_es_rest.h @@ -0,0 +1,18 @@ +#ifndef CEPH_RGW_SYNC_MODULE_ES_REST_H +#define CEPH_RGW_SYNC_MODULE_ES_REST_H + +#include "rgw_rest.h" + +class RGWElasticSyncModuleInstance; + +class RGWRESTMgr_MDSearch_S3 : public RGWRESTMgr { + RGWElasticSyncModuleInstance *es_module; +public: + explicit RGWRESTMgr_MDSearch_S3(RGWElasticSyncModuleInstance *_es_module) : es_module(_es_module) {} + + RGWHandler_REST *get_handler(struct req_state* s, + const rgw::auth::StrategyRegistry& auth_registry, + const std::string& frontend_prefix) override; +}; + +#endif