From: Josh Durgin Date: Thu, 24 Oct 2013 15:26:19 +0000 (-0700) Subject: rgw: include marker and truncated flag in data log list api X-Git-Tag: v0.67.5~20^2~10 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8916648521e2362a54eed0f5ebe078a269e4d703;p=ceph.git rgw: include marker and truncated flag in data log list api Consumers of this api need to know their position in the log. It's readily available when fetching the log, so return it. Without the marker in this call, a client could not easily or efficiently figure out its position in the log, since it would require getting the global last marker in the log, and then reading all the log entries. This would be slow for large logs, and would be subject to races that would cause potentially very expensive duplicate work. Returning this atomically while fetching the log entries simplifies all of this. Fixes: #6615 Backport: dumpling Signed-off-by: Josh Durgin (cherry picked from commit c275912509255f8bb4c854e181318b45ab0f8564) --- diff --git a/src/cls/log/cls_log_client.cc b/src/cls/log/cls_log_client.cc index ea8adf11145d..e5b47bf81f27 100644 --- a/src/cls/log/cls_log_client.cc +++ b/src/cls/log/cls_log_client.cc @@ -107,7 +107,8 @@ public: }; void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to, - string& in_marker, int max_entries, list& entries, + const string& in_marker, int max_entries, + list& entries, string *out_marker, bool *truncated) { bufferlist inbl; diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h index 3b4c96d1d3a1..16229c992b92 100644 --- a/src/cls/log/cls_log_client.h +++ b/src/cls/log/cls_log_client.h @@ -18,7 +18,8 @@ void cls_log_add(librados::ObjectWriteOperation& op, const utime_t& timestamp, const string& section, const string& name, bufferlist& bl); void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to, - string& in_marker, int max_entries, list& entries, + const string& in_marker, int max_entries, + list& entries, string *out_marker, bool *truncated); void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time, diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 1e523f332cf3..4c53dfe6eb1c 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1186,12 +1186,16 @@ int RGWDataChangesLog::add_entry(rgw_bucket& bucket) { } int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries, - list& entries, string& marker, bool *truncated) { + list& entries, + const string& marker, + string *out_marker, + bool *truncated) { list log_entries; int ret = store->time_log_list(oids[shard], start_time, end_time, - max_entries, log_entries, marker, truncated); + max_entries, log_entries, marker, + out_marker, truncated); if (ret < 0) return ret; @@ -1219,7 +1223,7 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int for (; marker.shard < num_shards && (int)entries.size() < max_entries; marker.shard++, marker.marker.clear()) { int ret = list_entries(marker.shard, start_time, end_time, max_entries - entries.size(), entries, - marker.marker, &truncated); + marker.marker, NULL, &truncated); if (ret == -ENOENT) { continue; } diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 5ee6a9b41cd3..47795403dc6c 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -352,7 +352,10 @@ public: int add_entry(rgw_bucket& bucket); int renew_entries(); int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries, - list& entries, string& marker, bool *truncated); + list& entries, + const string& marker, + string *out_marker, + bool *truncated); int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time, const string& start_marker, const string& end_marker); int trim_entries(const utime_t& start_time, const utime_t& end_time, diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 3db7c719a826..eb6efa25235d 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1543,7 +1543,10 @@ int RGWRados::time_log_add(const string& oid, list& entries) } int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end_time, - int max_entries, list& entries, string& marker, bool *truncated) + int max_entries, list& entries, + const string& marker, + string *out_marker, + bool *truncated) { librados::IoCtx io_ctx; @@ -1553,7 +1556,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end return r; librados::ObjectReadOperation op; - cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated); + cls_log_list(op, start_time, end_time, marker, max_entries, entries, + out_marker, truncated); bufferlist obl; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 65765c414aaa..fa5954c9019c 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1352,7 +1352,8 @@ public: int time_log_add(const string& oid, list& entries); int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl); int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time, - int max_entries, list& entries, string& marker, bool *truncated); + int max_entries, list& entries, + const string& marker, string *out_marker, bool *truncated); int time_log_info(const string& oid, cls_log_header *header); int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time, const string& from_marker, const string& to_marker); diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index 74e3c445ee93..18f0697b0b26 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -471,10 +471,12 @@ void RGWOp_DATALog_List::execute() { } } - bool truncated; do { + // Note that last_marker is updated to be the marker of the last + // entry listed http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, - max_entries, entries, marker, &truncated); + max_entries, entries, marker, + &last_marker, &truncated); if (http_ret < 0) break; @@ -491,12 +493,18 @@ void RGWOp_DATALog_List::send_response() { if (http_ret < 0) return; - s->formatter->open_array_section("entries"); - for (list::iterator iter = entries.begin(); - iter != entries.end(); ++iter) { - rgw_data_change& entry = *iter; - encode_json("entry", entry, s->formatter); - flusher.flush(); + s->formatter->open_object_section("log_entries"); + s->formatter->dump_string("marker", last_marker); + s->formatter->dump_bool("truncated", truncated); + { + s->formatter->open_array_section("entries"); + for (list::iterator iter = entries.begin(); + iter != entries.end(); ++iter) { + rgw_data_change& entry = *iter; + encode_json("entry", entry, s->formatter); + flusher.flush(); + } + s->formatter->close_section(); } s->formatter->close_section(); flusher.flush(); diff --git a/src/rgw/rgw_rest_log.h b/src/rgw/rgw_rest_log.h index 2d60e289b84a..404db84aa7bf 100644 --- a/src/rgw/rgw_rest_log.h +++ b/src/rgw/rgw_rest_log.h @@ -175,9 +175,11 @@ public: class RGWOp_DATALog_List : public RGWRESTOp { list entries; + string last_marker; + bool truncated; int http_ret; public: - RGWOp_DATALog_List() : http_ret(0) {} + RGWOp_DATALog_List() : truncated(false), http_ret(0) {} ~RGWOp_DATALog_List() {} int check_caps(RGWUserCaps& caps) {