rgw: include marker and truncated flag in data log list api

Consumers of this api need to know their position in the log. It's
readily available when fetching the log, so return it.  Without the
marker in this call, a client could not easily or efficiently figure
out its position in the log, since it would require getting the global
last marker in the log, and then reading all the log entries.

This would be slow for large logs, and would be subject to races that
would cause potentially very expensive duplicate work.

Returning this atomically while fetching the log entries simplifies
all of this.

Fixes: #6615
Backport: dumpling
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>
(cherry picked from commit c275912509255f8bb4c854e181318b45ab0f8564)

diff --git a/src/cls/log/cls_log_client.cc b/src/cls/log/cls_log_client.cc
index ea8adf11145de52d293d2d3d9929df9a64b86ffb..e5b47bf81f278012d821f44fab77daac2a0a8938 100644 (file)
--- a/src/cls/log/cls_log_client.cc
+++ b/src/cls/log/cls_log_client.cc
@@ -107,7 +107,8 @@ public:
 };
 
 void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
-                  string& in_marker, int max_entries, list<cls_log_entry>& entries,
+                  const string& in_marker, int max_entries,
+                 list<cls_log_entry>& entries,
                   string *out_marker, bool *truncated)
 {
   bufferlist inbl;

diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h
index 3b4c96d1d3a13701517fd04e6a861308a93ee302..16229c992b92a70c69a1d7c3183a040eaeb041b5 100644 (file)
--- a/src/cls/log/cls_log_client.h
+++ b/src/cls/log/cls_log_client.h
@@ -18,7 +18,8 @@ void cls_log_add(librados::ObjectWriteOperation& op, const utime_t& timestamp,
                  const string& section, const string& name, bufferlist& bl);
 
 void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
-                  string& in_marker, int max_entries, list<cls_log_entry>& entries,
+                  const string& in_marker, int max_entries,
+                 list<cls_log_entry>& entries,
                   string *out_marker, bool *truncated);
 
 void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 1e523f332cf3265ef314414a5d4ca865e3ee3cd7..4c53dfe6eb1c23c721c6eb0740f3843948c309a6 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1186,12 +1186,16 @@ int RGWDataChangesLog::add_entry(rgw_bucket& bucket) {
 }
 
 int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
-             list<rgw_data_change>& entries, string& marker, bool *truncated) {
+                                   list<rgw_data_change>& entries,
+                                   const string& marker,
+                                   string *out_marker,
+                                   bool *truncated) {
 
   list<cls_log_entry> log_entries;
 
   int ret = store->time_log_list(oids[shard], start_time, end_time,
-                                 max_entries, log_entries, marker, truncated); 
+                                max_entries, log_entries, marker,
+                                out_marker, truncated);
   if (ret < 0)
     return ret;
 
@@ -1219,7 +1223,7 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int
   for (; marker.shard < num_shards && (int)entries.size() < max_entries;
        marker.shard++, marker.marker.clear()) {
     int ret = list_entries(marker.shard, start_time, end_time, max_entries - entries.size(), entries,
-                       marker.marker, &truncated);
+                          marker.marker, NULL, &truncated);
     if (ret == -ENOENT) {
       continue;
     }

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 5ee6a9b41cd3de36fc855fbb6eb7ed7f995ee42f..47795403dc6c0c36498d933608cb6e20c322efc2 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -352,7 +352,10 @@ public:
   int add_entry(rgw_bucket& bucket);
   int renew_entries();
   int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
-               list<rgw_data_change>& entries, string& marker, bool *truncated);
+                  list<rgw_data_change>& entries,
+                  const string& marker,
+                  string *out_marker,
+                  bool *truncated);
   int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
                    const string& start_marker, const string& end_marker);
   int trim_entries(const utime_t& start_time, const utime_t& end_time,

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 3db7c719a826114aa35a9c56e44b34628f3e594b..eb6efa25235db980979fc1e330a8cb79ab1fb714 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1543,7 +1543,10 @@ int RGWRados::time_log_add(const string& oid, list<cls_log_entry>& entries)
 }
 
 int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
-                            int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated)
+                            int max_entries, list<cls_log_entry>& entries,
+                           const string& marker,
+                           string *out_marker,
+                           bool *truncated)
 {
   librados::IoCtx io_ctx;
 
@@ -1553,7 +1556,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end
     return r;
   librados::ObjectReadOperation op;
 
-  cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated);
+  cls_log_list(op, start_time, end_time, marker, max_entries, entries,
+              out_marker, truncated);
 
   bufferlist obl;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 65765c414aaa8dd64889956af5dd074e345a40d6..fa5954c9019cdf6e30f13fbe2637dcf602834f3e 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1352,7 +1352,8 @@ public:
   int time_log_add(const string& oid, list<cls_log_entry>& entries);
   int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl);
   int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
-                    int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated);
+                    int max_entries, list<cls_log_entry>& entries,
+                   const string& marker, string *out_marker, bool *truncated);
   int time_log_info(const string& oid, cls_log_header *header);
   int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
                     const string& from_marker, const string& to_marker);

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 74e3c445ee935f6052942090c9dce0e0ab21abc3..18f0697b0b266b3a3ce0e2c0c3c387beeae693d0 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -471,10 +471,12 @@ void RGWOp_DATALog_List::execute() {
     }
   } 
   
-  bool truncated;
   do {
+    // Note that last_marker is updated to be the marker of the last
+    // entry listed
     http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, 
-                               max_entries, entries, marker, &truncated);
+                                            max_entries, entries, marker,
+                                            &last_marker, &truncated);
     if (http_ret < 0) 
       break;
 
@@ -491,12 +493,18 @@ void RGWOp_DATALog_List::send_response() {
   if (http_ret < 0)
     return;
 
-  s->formatter->open_array_section("entries");
-  for (list<rgw_data_change>::iterator iter = entries.begin(); 
-       iter != entries.end(); ++iter) {
-    rgw_data_change& entry = *iter;
-    encode_json("entry", entry, s->formatter);
-    flusher.flush();
+  s->formatter->open_object_section("log_entries");
+  s->formatter->dump_string("marker", last_marker);
+  s->formatter->dump_bool("truncated", truncated);
+  {
+    s->formatter->open_array_section("entries");
+    for (list<rgw_data_change>::iterator iter = entries.begin();
+        iter != entries.end(); ++iter) {
+      rgw_data_change& entry = *iter;
+      encode_json("entry", entry, s->formatter);
+      flusher.flush();
+    }
+    s->formatter->close_section();
   }
   s->formatter->close_section();
   flusher.flush();

diff --git a/src/rgw/rgw_rest_log.h b/src/rgw/rgw_rest_log.h
index 2d60e289b84a85b8022411e17c0c253bb7f1248a..404db84aa7bf3f49086b126b360cf144cc0dc936 100644 (file)
--- a/src/rgw/rgw_rest_log.h
+++ b/src/rgw/rgw_rest_log.h
@@ -175,9 +175,11 @@ public:
 
 class RGWOp_DATALog_List : public RGWRESTOp {
   list<rgw_data_change> entries;
+  string last_marker;
+  bool truncated;
   int http_ret;
 public:
-  RGWOp_DATALog_List() : http_ret(0) {}
+  RGWOp_DATALog_List() : truncated(false), http_ret(0) {}
   ~RGWOp_DATALog_List() {}
 
   int check_caps(RGWUserCaps& caps) {