From: Yehuda Sadeh Date: Fri, 31 May 2013 19:22:09 +0000 (-0700) Subject: Merge branch 'wip-rgw-geo-enovance' into wip-rgw-geo-2 X-Git-Tag: v0.67-rc1~128^2~91 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e46d7d680eaf279829f784918feceeb1302f9b88;p=ceph.git Merge branch 'wip-rgw-geo-enovance' into wip-rgw-geo-2 Conflicts: src/Makefile.am src/rgw/rgw_admin.cc Signed-off-by: Yehuda Sadeh --- e46d7d680eaf279829f784918feceeb1302f9b88 diff --cc src/Makefile.am index 7762cbb50fc,c8493340816..0e36f8fcc5e --- a/src/Makefile.am +++ b/src/Makefile.am @@@ -2082,10 -2085,9 +2091,11 @@@ noinst_HEADERS = rgw/rgw_rest_usage.h\ rgw/rgw_rest_user.h\ rgw/rgw_rest_bucket.h\ + rgw/rgw_rest_client.h\ + rgw/rgw_rest_conn.h\ rgw/rgw_tools.h\ rgw/rgw_rest_metadata.h\ + rgw/rgw_rest_log.h\ rgw/rgw_usage.h\ rgw/rgw_user.h\ rgw/rgw_bucket.h\ diff --cc src/rgw/rgw_admin.cc index 12f08aadd91,9cbd4405b54..e62f65019bc --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@@ -642,8 -642,7 +642,9 @@@ int main(int argc, char **argv string start_marker; string end_marker; int max_entries = -1; + int system = false; + bool system_specified = false; + int shard_id = 0; std::string val; std::ostringstream errs; diff --cc src/rgw/rgw_rest_log.cc index 00000000000,97af495e652..97c4b288d27 mode 000000,100644..100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@@ -1,0 -1,359 +1,359 @@@ + // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- + // vim: ts=8 sw=2 smarttab + /* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 eNovance SAS + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + #include "common/ceph_json.h" + #include "common/strtol.h" + #include "rgw_rest.h" + #include "rgw_op.h" + #include "rgw_rest_s3.h" + #include "rgw_rest_log.h" + #include "rgw_client_io.h" + #include "common/errno.h" + + #define dout_subsys ceph_subsys_rgw + + static int parse_date_str(string& in, utime_t& out) { + uint64_t epoch = 0; + + if (!in.empty()) { + if (parse_date(in, &epoch) < 0) { + dout(5) << "Error parsing date " << in << dendl; + return -EINVAL; + } + } + out = utime_t(epoch, 0); + return 0; + } + + void RGWOp_MDLog_List::execute() { - string shard = s->args.get("id"); ++ string shard = s->info.args.get("id"); + - string st = s->args.get("start-time"), - et = s->args.get("end-time"), ++ string st = s->info.args.get("start-time"), ++ et = s->info.args.get("end-time"), + err; + utime_t ut_st, + ut_et; + void *handle; + int shard_id; + + shard_id = strict_strtol(shard.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing shard_id " << shard << dendl; + http_ret = -EINVAL; + return; + } + + if (parse_date_str(st, ut_st) < 0) { + http_ret = -EINVAL; + return; + } + + if (parse_date_str(et, ut_et) < 0) { + http_ret = -EINVAL; + return; + } + + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); + + meta_log->init_list_entries(shard_id, ut_st, ut_et, &handle); + + bool truncated; + + http_ret = meta_log->list_entries(handle, 1000, entries, &truncated); + } + + void RGWOp_MDLog_List::send_response() { + set_req_state_err(s, http_ret); + dump_errno(s); + end_header(s); + + if (http_ret < 0) + return; + + s->formatter->open_array_section("entries"); + for (list::iterator iter = entries.begin(); + iter != entries.end(); ++iter) { + cls_log_entry& entry = *iter; + store->meta_mgr->dump_log_entry(entry, s->formatter); + flusher.flush(); + } + s->formatter->close_section(); + flusher.flush(); + } + + void RGWOp_MDLog_GetShardsInfo::execute() { + num_objects = s->cct->_conf->rgw_md_log_max_shards; + http_ret = 0; + } + + void RGWOp_MDLog_GetShardsInfo::send_response() { + set_req_state_err(s, http_ret); + dump_errno(s); + end_header(s); + + s->formatter->open_object_section("num_objects"); + s->formatter->dump_unsigned("num_objects", num_objects); + s->formatter->close_section(); + flusher.flush(); + } + + void RGWOp_MDLog_Delete::execute() { - string st = s->args.get("start-time"), - et = s->args.get("end-time"), - shard = s->args.get("id"), ++ string st = s->info.args.get("start-time"), ++ et = s->info.args.get("end-time"), ++ shard = s->info.args.get("id"), + err; + utime_t ut_st, + ut_et; + int shard_id; + + http_ret = 0; + + shard_id = strict_strtol(shard.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing shard_id " << shard << dendl; + http_ret = -EINVAL; + return; + } + if (st.empty() || et.empty()) { + http_ret = -EINVAL; + return; + } + + if (parse_date_str(st, ut_st) < 0) { + http_ret = -EINVAL; + return; + } + + if (parse_date_str(et, ut_et) < 0) { + http_ret = -EINVAL; + return; + } + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); + + http_ret = meta_log->trim(shard_id, ut_st, ut_et); + } + + int RGWOp_MDLog_Post::check_caps(RGWUserCaps& caps) { + if (caps.check_cap("mdlog", RGW_CAP_READ) && + caps.check_cap("mdlog", RGW_CAP_WRITE)) { + return -EPERM; + } + return 0; + } + + const char *RGWOp_MDLog_Post::name() { + int pt = get_post_type(); + if (pt == MDLOG_POST_LOCK) + return "lock mdlog object"; + else if (pt == MDLOG_POST_UNLOCK) + return "unlock mdlog object"; + return NULL; + } + + void RGWOp_MDLog_Post::execute() { + string shard_id_str, duration_str, lock_id; + int shard_id; + int pt = get_post_type(); + + http_ret = 0; + - shard_id_str = s->args.get("id"); ++ shard_id_str = s->info.args.get("id"); + if (pt == MDLOG_POST_LOCK) - duration_str = s->args.get("length"); - lock_id = s->args.get("lock_id"); ++ duration_str = s->info.args.get("length"); ++ lock_id = s->info.args.get("lock_id"); + + if (shard_id_str.empty() || + (pt == MDLOG_POST_LOCK && duration_str.empty()) || + lock_id.empty()) { + dout(5) << "Error invalid parameter list" << dendl; + http_ret = -EINVAL; + return; + } + + string err; + shard_id = strict_strtol(shard_id_str.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing shard_id param " << shard_id_str << dendl; + http_ret = -EINVAL; + return; + } + + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); + if (pt == MDLOG_POST_LOCK) { + int dur; + dur = strict_strtol(duration_str.c_str(), 10, &err); + if (!err.empty() || dur <= 0) { + dout(5) << "invalid length param " << duration_str << dendl; + http_ret = -EINVAL; + return; + } + utime_t time(dur, 0); + http_ret = meta_log->lock_exclusive(shard_id, time, lock_id); + } else if (pt == MDLOG_POST_UNLOCK) { + http_ret = meta_log->unlock(shard_id, lock_id); + } else + http_ret = -EINVAL; + } + + void RGWOp_BILog_List::execute() { - string bucket_name = s->args.get("bucket"), - marker = s->args.get("marker"), - max_entries_str = s->args.get("max-entries"); ++ string bucket_name = s->info.args.get("bucket"), ++ marker = s->info.args.get("marker"), ++ max_entries_str = s->info.args.get("max-entries"); + RGWBucketInfo bucket_info; + int max_entries; + + if (bucket_name.empty()) { + dout(5) << "ERROR: bucket not specified" << dendl; + http_ret = -EINVAL; + return; + } + + http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL); + if (http_ret < 0) { + dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl; + return; + } + + bool truncated; + int count = 0; + string err; + + max_entries = strict_strtol(max_entries_str.c_str(), 10, &err); + if (!err.empty()) + max_entries = 1000; + + send_response(); + do { + list entries; + int ret = store->list_bi_log_entries(bucket_info.bucket, + marker, max_entries - count, + entries, &truncated); + if (ret < 0) { + dout(5) << "ERROR: list_bi_log_entries()" << dendl; + return; + } + + count += entries.size(); + + send_response(entries, marker); + } while (truncated && count < max_entries); + + send_response_end(); + } + + void RGWOp_BILog_List::send_response() { + if (sent_header) + return; + + set_req_state_err(s, http_ret); + dump_errno(s); + end_header(s); + + sent_header = true; + + if (http_ret < 0) + return; + + s->formatter->open_array_section("entries"); + } + + void RGWOp_BILog_List::send_response(list& entries, string& marker) + { + for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) { + rgw_bi_log_entry& entry = *iter; + encode_json("entry", entry, s->formatter); + + marker = entry.id; + flusher.flush(); + } + } + + void RGWOp_BILog_List::send_response_end() { + s->formatter->close_section(); + flusher.flush(); + } + + void RGWOp_BILog_Delete::execute() { - string bucket_name = s->args.get("bucket"), - start_marker = s->args.get("start-marker"), - end_marker = s->args.get("end-marker"); ++ string bucket_name = s->info.args.get("bucket"), ++ start_marker = s->info.args.get("start-marker"), ++ end_marker = s->info.args.get("end-marker"); + RGWBucketInfo bucket_info; + + http_ret = 0; + if (bucket_name.empty() || + start_marker.empty() || + end_marker.empty()) { + dout(5) << "ERROR: bucket, start-marker, end-marker are mandatory" << dendl; + http_ret = -EINVAL; + return; + } + http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL); + if (http_ret < 0) { + dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl; + return; + } + http_ret = store->trim_bi_log_entries(bucket_info.bucket, start_marker, end_marker); + if (http_ret < 0) { + dout(5) << "ERROR: trim_bi_log_entries() " << dendl; + } + return; + } + + RGWOp *RGWHandler_Log::op_get() { + bool exists; - string type = s->args.get("type", &exists); ++ string type = s->info.args.get("type", &exists); + + if (!exists) { + return NULL; + } + + if (type.compare("metadata") == 0) { - if (s->args.exists("id")) { ++ if (s->info.args.exists("id")) { + return new RGWOp_MDLog_List; + } else { + return new RGWOp_MDLog_GetShardsInfo; + } + } else if (type.compare("bucket-index") == 0) { + return new RGWOp_BILog_List; + } + return NULL; + } + + RGWOp *RGWHandler_Log::op_delete() { + bool exists; - string type = s->args.get("type", &exists); ++ string type = s->info.args.get("type", &exists); + + if (!exists) { + return NULL; + } + + if (type.compare("metadata") == 0) + return new RGWOp_MDLog_Delete; + else if (type.compare("bucket-index") == 0) + return new RGWOp_BILog_Delete; + return NULL; + } + + RGWOp *RGWHandler_Log::op_post() { + bool exists; - string type = s->args.get("type", &exists); ++ string type = s->info.args.get("type", &exists); + + if (!exists) { + return NULL; + } + + if (type.compare("metadata") == 0) + return new RGWOp_MDLog_Post; + return NULL; + } + diff --cc src/rgw/rgw_rest_log.h index 00000000000,d21540044ce..635758d77cb mode 000000,100644..100644 --- a/src/rgw/rgw_rest_log.h +++ b/src/rgw/rgw_rest_log.h @@@ -1,0 -1,157 +1,157 @@@ + // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- + // vim: ts=8 sw=2 smarttab + /* + * Ceph - scalable distributed file system + * + * Copyright (C) 2013 eNovance SAS + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + #ifndef CEPH_RGW_REST_LOG_H + #define CEPH_RGW_REST_LOG_H + + class RGWOp_BILog_List : public RGWRESTOp { + int http_ret; + bool sent_header; + public: + RGWOp_BILog_List() : http_ret(0), sent_header(false) {} + ~RGWOp_BILog_List() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("bilog", RGW_CAP_READ); + } + int verify_permission() { + return check_caps(s->user.caps); + } + virtual void send_response(); + virtual void send_response(list& entries, string& marker); + virtual void send_response_end(); + void execute(); + virtual const char *name() { + return "list bucket index log"; + } + }; + + class RGWOp_BILog_Delete : public RGWRESTOp { + public: + RGWOp_BILog_Delete() {} + ~RGWOp_BILog_Delete() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("bilog", RGW_CAP_WRITE); + } + void execute(); + virtual const char *name() { + return "trim_bucket_index_log"; + } + }; + + class RGWOp_MDLog_List : public RGWRESTOp { + list entries; + int http_ret; + public: + RGWOp_MDLog_List() : http_ret(0) {} + ~RGWOp_MDLog_List() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("mdlog", RGW_CAP_READ); + } + int verify_permission() { + return check_caps(s->user.caps); + } + void execute(); + virtual void send_response(); + virtual const char *name() { + return "list_metadata_log"; + } + }; + + class RGWOp_MDLog_GetShardsInfo : public RGWRESTOp { + unsigned num_objects; + int http_ret; + public: + RGWOp_MDLog_GetShardsInfo() : num_objects(0), http_ret(0) {} + ~RGWOp_MDLog_GetShardsInfo() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("mdlog", RGW_CAP_READ); + } + int verify_permission() { + return check_caps(s->user.caps); + } + void execute(); + virtual void send_response(); + virtual const char *name() { + return "get_metadata_log_shards_info"; + } + }; + + class RGWOp_MDLog_Post : public RGWRESTOp { + enum { + MDLOG_POST_INVALID = 0, + MDLOG_POST_LOCK, + MDLOG_POST_UNLOCK + }; + int get_post_type() { + bool exists; - s->args.get("lock", &exists); ++ s->info.args.get("lock", &exists); + if (exists) + return MDLOG_POST_LOCK; - s->args.get("unlock", &exists); ++ s->info.args.get("unlock", &exists); + if (exists) + return MDLOG_POST_UNLOCK; + return MDLOG_POST_INVALID; + } + public: + RGWOp_MDLog_Post() {} + ~RGWOp_MDLog_Post() {} + + int check_caps(RGWUserCaps& caps); + void execute(); + virtual const char *name(); + }; + + class RGWOp_MDLog_Delete : public RGWRESTOp { + public: + RGWOp_MDLog_Delete() {} + ~RGWOp_MDLog_Delete() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("mdlog", RGW_CAP_WRITE); + } + void execute(); + virtual const char *name() { + return "trim_metadata_log"; + } + }; + + class RGWHandler_Log : public RGWHandler_Auth_S3 { + protected: + RGWOp *op_get(); + RGWOp *op_delete(); + RGWOp *op_post(); + + int read_permissions(RGWOp*) { + return 0; + } + public: + RGWHandler_Log() : RGWHandler_Auth_S3() {} + virtual ~RGWHandler_Log() {} + }; + + class RGWRESTMgr_Log : public RGWRESTMgr { + public: + RGWRESTMgr_Log() {} + virtual ~RGWRESTMgr_Log() {} + + virtual RGWHandler *get_handler(struct req_state *s){ + return new RGWHandler_Log; + } + }; + + #endif + diff --cc src/rgw/rgw_rest_s3.cc index 143ca6ee993,9f06158c266..07cd55718fd --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@@ -366,12 -367,9 +366,12 @@@ int RGWCreateBucket_ObjStore_S3::get_pa char *data; #define CREATE_BUCKET_MAX_REQ_LEN (512 * 1024) /* this is way more than enough */ ret = rgw_rest_read_all_input(s, &data, &len, CREATE_BUCKET_MAX_REQ_LEN); - if (ret < 0) + if ((ret < 0) && (ret != -ERR_LENGTH_REQUIRED)) return ret; + bufferptr in_ptr(data, len); + in_data.append(in_ptr); + if (len) { RGWCreateBucketParser parser;