From fcc423d659dd7c3b4113f81825eb23bea0fe3f44 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 2 Jul 2013 16:15:42 -0700 Subject: [PATCH] rgw: RESTful api for shards info For both mdlog, datalog Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_bucket.cc | 22 +++++++++++++++ src/rgw/rgw_bucket.h | 9 +++++++ src/rgw/rgw_json_enc.cc | 25 +++++++++++++++++ src/rgw/rgw_metadata.cc | 17 ++++++++++++ src/rgw/rgw_metadata.h | 9 +++++++ src/rgw/rgw_rados.cc | 21 +++++++++++++++ src/rgw/rgw_rados.h | 1 + src/rgw/rgw_rest_log.cc | 60 ++++++++++++++++++++++++++++++++++++----- src/rgw/rgw_rest_log.h | 56 ++++++++++++++++++++++++++++++++------ 9 files changed, 206 insertions(+), 14 deletions(-) diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 2e94423be859b..f7efd266047fb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1219,11 +1219,33 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int return 0; } +int RGWDataChangesLog::get_info(int shard_id, RGWDataChangesLogInfo *info) +{ + if (shard_id > num_shards) + return -EINVAL; + + string oid = oids[shard_id]; + + cls_log_header header; + + int ret = store->time_log_info(oid, &header); + if ((ret < 0) && (ret != -ENOENT)) + return ret; + + info->marker = header.max_marker; + info->last_update = header.max_time; + + return 0; +} + int RGWDataChangesLog::trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time, const string& start_marker, const string& end_marker) { int ret; + if (shard_id > num_shards) + return -EINVAL; + ret = store->time_log_trim(oids[shard_id], start_time, end_time, start_marker, end_marker); if (ret == -ENOENT) diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 50d51444eb0ec..d5f023f7bd9fa 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -263,6 +263,14 @@ struct rgw_data_change { }; WRITE_CLASS_ENCODER(rgw_data_change) +struct RGWDataChangesLogInfo { + string marker; + utime_t last_update; + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; + class RGWDataChangesLog { CephContext *cct; RGWRados *store; @@ -349,6 +357,7 @@ public: const string& start_marker, const string& end_marker); int trim_entries(const utime_t& start_time, const utime_t& end_time, const string& start_marker, const string& end_marker); + int get_info(int shard_id, RGWDataChangesLogInfo *info); int lock_exclusive(int shard_id, utime_t& duration, string& zone_id, string& owner_id) { return store->lock_exclusive(store->zone.log_pool, oids[shard_id], duration, zone_id, owner_id); } diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index c3d1e7baba8fe..face8534eb581 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -5,6 +5,7 @@ #include "rgw_acl.h" #include "rgw_acl_s3.h" #include "rgw_cache.h" +#include "rgw_bucket.h" #include "common/ceph_json.h" #include "common/Formatter.h" @@ -652,3 +653,27 @@ void RGWRegionMap::decode_json(JSONObj *obj) JSONDecoder::decode_json("master_region", master_region, obj); } +void RGWMetadataLogInfo::dump(Formatter *f) const +{ + encode_json("marker", marker, f); + encode_json("last_update", last_update, f); +} + +void RGWMetadataLogInfo::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("marker", marker, obj); + JSONDecoder::decode_json("last_update", last_update, obj); +} + +void RGWDataChangesLogInfo::dump(Formatter *f) const +{ + encode_json("marker", marker, f); + encode_json("last_update", last_update, f); +} + +void RGWDataChangesLogInfo::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("marker", marker, obj); + JSONDecoder::decode_json("last_update", last_update, obj); +} + diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 5735396e33cfd..c370addc293f6 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -130,6 +130,23 @@ int RGWMetadataLog::list_entries(void *handle, return 0; } +int RGWMetadataLog::get_info(int shard_id, RGWMetadataLogInfo *info) +{ + string oid; + get_shard_oid(shard_id, oid); + + cls_log_header header; + + int ret = store->time_log_info(oid, &header); + if ((ret < 0) && (ret != -ENOENT)) + return ret; + + info->marker = header.max_marker; + info->last_update = header.max_time; + + return 0; +} + int RGWMetadataLog::trim(int shard_id, const utime_t& from_time, const utime_t& end_time, const string& start_marker, const string& end_marker) { diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index ff0e35f94386e..2cc9110191aab 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -66,6 +66,14 @@ public: #define META_LOG_OBJ_PREFIX "meta.log." +struct RGWMetadataLogInfo { + string marker; + utime_t last_update; + + void dump(Formatter *f) const; + void decode_json(JSONObj *obj); +}; + class RGWMetadataLog { CephContext *cct; RGWRados *store; @@ -104,6 +112,7 @@ public: list& entries, bool *truncated); int trim(int shard_id, const utime_t& from_time, const utime_t& end_time, const string& start_marker, const string& end_marker); + int get_info(int shard_id, RGWMetadataLogInfo *info); int lock_exclusive(int shard_id, utime_t& duration, string&zone_id, string& owner_id); int unlock(int shard_id, string& zone_id, string& owner_id); }; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index be75db65fe7e0..40867432fe43b 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1524,6 +1524,27 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end return 0; } +int RGWRados::time_log_info(const string& oid, cls_log_header *header) +{ + librados::IoCtx io_ctx; + + const char *log_pool = zone.log_pool.name.c_str(); + int r = rados->ioctx_create(log_pool, io_ctx); + if (r < 0) + return r; + librados::ObjectReadOperation op; + + cls_log_info(op, header); + + bufferlist obl; + + int ret = io_ctx.operate(oid, &op, &obl); + if (ret < 0) + return ret; + + return 0; +} + int RGWRados::time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time, const string& from_marker, const string& to_marker) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index c8d031557d727..3c91153db773f 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1340,6 +1340,7 @@ public: int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl); int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time, int max_entries, list& entries, string& marker, bool *truncated); + int time_log_info(const string& oid, cls_log_header *header); int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time, const string& from_marker, const string& to_marker); int lock_exclusive(rgw_bucket& pool, const string& oid, utime_t& duration, string& zone_id, string& owner_id); diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index 2dc6071321bd3..1c54710f5beb7 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -109,12 +109,12 @@ void RGWOp_MDLog_List::send_response() { flusher.flush(); } -void RGWOp_MDLog_GetShardsInfo::execute() { +void RGWOp_MDLog_Info::execute() { num_objects = s->cct->_conf->rgw_md_log_max_shards; http_ret = 0; } -void RGWOp_MDLog_GetShardsInfo::send_response() { +void RGWOp_MDLog_Info::send_response() { set_req_state_err(s, http_ret); dump_errno(s); end_header(s); @@ -125,6 +125,31 @@ void RGWOp_MDLog_GetShardsInfo::send_response() { flusher.flush(); } +void RGWOp_MDLog_ShardInfo::execute() { + string shard = s->info.args.get("id"); + string err; + + unsigned shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing shard_id " << shard << dendl; + http_ret = -EINVAL; + return; + } + + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); + + http_ret = meta_log->get_info(shard_id, &info); +} + +void RGWOp_MDLog_ShardInfo::send_response() { + set_req_state_err(s, http_ret); + dump_errno(s); + end_header(s); + + encode_json("info", info, s->formatter); + flusher.flush(); +} + void RGWOp_MDLog_Delete::execute() { string st = s->info.args.get("start-time"), et = s->info.args.get("end-time"), @@ -425,12 +450,12 @@ void RGWOp_DATALog_List::send_response() { } -void RGWOp_DATALog_GetShardsInfo::execute() { +void RGWOp_DATALog_Info::execute() { num_objects = s->cct->_conf->rgw_data_log_num_shards; http_ret = 0; } -void RGWOp_DATALog_GetShardsInfo::send_response() { +void RGWOp_DATALog_Info::send_response() { set_req_state_err(s, http_ret); dump_errno(s); end_header(s); @@ -441,6 +466,29 @@ void RGWOp_DATALog_GetShardsInfo::send_response() { flusher.flush(); } +void RGWOp_DATALog_ShardInfo::execute() { + string shard = s->info.args.get("id"); + string err; + + unsigned shard_id = (unsigned)strict_strtol(shard.c_str(), 10, &err); + if (!err.empty()) { + dout(5) << "Error parsing shard_id " << shard << dendl; + http_ret = -EINVAL; + return; + } + + http_ret = store->data_log->get_info(shard_id, &info); +} + +void RGWOp_DATALog_ShardInfo::send_response() { + set_req_state_err(s, http_ret); + dump_errno(s); + end_header(s); + + encode_json("info", info, s->formatter); + flusher.flush(); +} + void RGWOp_DATALog_Lock::execute() { string shard_id_str, duration_str, locker_id, zone_id; unsigned shard_id; @@ -558,7 +606,7 @@ RGWOp *RGWHandler_Log::op_get() { if (s->info.args.exists("id")) { return new RGWOp_MDLog_List; } else { - return new RGWOp_MDLog_GetShardsInfo; + return new RGWOp_MDLog_Info; } } else if (type.compare("bucket-index") == 0) { return new RGWOp_BILog_List; @@ -566,7 +614,7 @@ RGWOp *RGWHandler_Log::op_get() { if (s->info.args.exists("id")) { return new RGWOp_DATALog_List; } else { - return new RGWOp_DATALog_GetShardsInfo; + return new RGWOp_DATALog_Info; } } return NULL; diff --git a/src/rgw/rgw_rest_log.h b/src/rgw/rgw_rest_log.h index 58207c59c32ae..ec340d66482af 100644 --- a/src/rgw/rgw_rest_log.h +++ b/src/rgw/rgw_rest_log.h @@ -14,6 +14,8 @@ #ifndef CEPH_RGW_REST_LOG_H #define CEPH_RGW_REST_LOG_H +#include "rgw_metadata.h" + class RGWOp_BILog_List : public RGWRESTOp { int http_ret; bool sent_header; @@ -70,12 +72,31 @@ public: } }; -class RGWOp_MDLog_GetShardsInfo : public RGWRESTOp { +class RGWOp_MDLog_Info : public RGWRESTOp { unsigned num_objects; int http_ret; public: - RGWOp_MDLog_GetShardsInfo() : num_objects(0), http_ret(0) {} - ~RGWOp_MDLog_GetShardsInfo() {} + RGWOp_MDLog_Info() : num_objects(0), http_ret(0) {} + ~RGWOp_MDLog_Info() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("mdlog", RGW_CAP_READ); + } + int verify_permission() { + return check_caps(s->user.caps); + } + void execute(); + virtual void send_response(); + virtual const char *name() { + return "get_metadata_log_info"; + } +}; + +class RGWOp_MDLog_ShardInfo : public RGWRESTOp { + RGWMetadataLogInfo info; +public: + RGWOp_MDLog_ShardInfo() {} + ~RGWOp_MDLog_ShardInfo() {} int check_caps(RGWUserCaps& caps) { return caps.check_cap("mdlog", RGW_CAP_READ); @@ -86,7 +107,7 @@ public: void execute(); virtual void send_response(); virtual const char *name() { - return "get_metadata_log_shards_info"; + return "get_metadata_log_shard_info"; } }; @@ -152,12 +173,31 @@ public: } }; -class RGWOp_DATALog_GetShardsInfo : public RGWRESTOp { +class RGWOp_DATALog_Info : public RGWRESTOp { unsigned num_objects; int http_ret; public: - RGWOp_DATALog_GetShardsInfo() : num_objects(0), http_ret(0) {} - ~RGWOp_DATALog_GetShardsInfo() {} + RGWOp_DATALog_Info() : num_objects(0), http_ret(0) {} + ~RGWOp_DATALog_Info() {} + + int check_caps(RGWUserCaps& caps) { + return caps.check_cap("datalog", RGW_CAP_READ); + } + int verify_permission() { + return check_caps(s->user.caps); + } + void execute(); + virtual void send_response(); + virtual const char *name() { + return "get_data_changes_log_info"; + } +}; + +class RGWOp_DATALog_ShardInfo : public RGWRESTOp { + RGWDataChangesLogInfo info; +public: + RGWOp_DATALog_ShardInfo() {} + ~RGWOp_DATALog_ShardInfo() {} int check_caps(RGWUserCaps& caps) { return caps.check_cap("datalog", RGW_CAP_READ); @@ -168,7 +208,7 @@ public: void execute(); virtual void send_response(); virtual const char *name() { - return "get_data_changes_log_shards_info"; + return "get_data_changes_log_shard_info"; } }; -- 2.39.5