rgw: include marker and truncated flag in data log list api

Consumers of this api need to know their position in the log. It's
readily available when fetching the log, so return it.  Without the
marker in this call, a client could not easily or efficiently figure
out its position in the log, since it would require getting the global
last marker in the log, and then reading all the log entries.

This would be slow for large logs, and would be subject to races that
would cause potentially very expensive duplicate work.

Returning this atomically while fetching the log entries simplifies
all of this.

Fixes: #6615
Backport: dumpling
Signed-off-by: Josh Durgin <josh.durgin@inktank.com>

diff --git a/src/cls/log/cls_log_client.cc b/src/cls/log/cls_log_client.cc
index ea8adf11145de52d293d2d3d9929df9a64b86ffb..e5b47bf81f278012d821f44fab77daac2a0a8938 100644 (file)
--- a/src/cls/log/cls_log_client.cc
+++ b/src/cls/log/cls_log_client.cc
@@ -107,7 +107,8 @@ public:
 };
 
 void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
-                  string& in_marker, int max_entries, list<cls_log_entry>& entries,
+                  const string& in_marker, int max_entries,
+                 list<cls_log_entry>& entries,
                   string *out_marker, bool *truncated)
 {
   bufferlist inbl;

diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h
index 3b4c96d1d3a13701517fd04e6a861308a93ee302..16229c992b92a70c69a1d7c3183a040eaeb041b5 100644 (file)
--- a/src/cls/log/cls_log_client.h
+++ b/src/cls/log/cls_log_client.h
@@ -18,7 +18,8 @@ void cls_log_add(librados::ObjectWriteOperation& op, const utime_t& timestamp,
                  const string& section, const string& name, bufferlist& bl);
 
 void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to,
-                  string& in_marker, int max_entries, list<cls_log_entry>& entries,
+                  const string& in_marker, int max_entries,
+                 list<cls_log_entry>& entries,
                   string *out_marker, bool *truncated);
 
 void cls_log_trim(librados::ObjectWriteOperation& op, const utime_t& from_time, const utime_t& to_time,

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 3267bc519481c06e78cfcd74f558291b7db87bd2..31afebe9ea33d98cfa9e325cc7743b6f496b5c7b 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1187,12 +1187,16 @@ int RGWDataChangesLog::add_entry(rgw_bucket& bucket) {
 }
 
 int RGWDataChangesLog::list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
-             list<rgw_data_change>& entries, string& marker, bool *truncated) {
+                                   list<rgw_data_change>& entries,
+                                   const string& marker,
+                                   string *out_marker,
+                                   bool *truncated) {
 
   list<cls_log_entry> log_entries;
 
   int ret = store->time_log_list(oids[shard], start_time, end_time,
-                                 max_entries, log_entries, marker, truncated); 
+                                max_entries, log_entries, marker,
+                                out_marker, truncated);
   if (ret < 0)
     return ret;
 
@@ -1220,7 +1224,7 @@ int RGWDataChangesLog::list_entries(utime_t& start_time, utime_t& end_time, int
   for (; marker.shard < num_shards && (int)entries.size() < max_entries;
        marker.shard++, marker.marker.clear()) {
     int ret = list_entries(marker.shard, start_time, end_time, max_entries - entries.size(), entries,
-                       marker.marker, &truncated);
+                          marker.marker, NULL, &truncated);
     if (ret == -ENOENT) {
       continue;
     }

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 5ee6a9b41cd3de36fc855fbb6eb7ed7f995ee42f..47795403dc6c0c36498d933608cb6e20c322efc2 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -352,7 +352,10 @@ public:
   int add_entry(rgw_bucket& bucket);
   int renew_entries();
   int list_entries(int shard, utime_t& start_time, utime_t& end_time, int max_entries,
-               list<rgw_data_change>& entries, string& marker, bool *truncated);
+                  list<rgw_data_change>& entries,
+                  const string& marker,
+                  string *out_marker,
+                  bool *truncated);
   int trim_entries(int shard_id, const utime_t& start_time, const utime_t& end_time,
                    const string& start_marker, const string& end_marker);
   int trim_entries(const utime_t& start_time, const utime_t& end_time,

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 20ca8d8eb8fb2a173058a09e331aa43006ee6486..4d6f8ef45301ad21d16c8a4fbb5c62a3f511a871 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -1586,7 +1586,10 @@ int RGWRados::time_log_add(const string& oid, list<cls_log_entry>& entries)
 }
 
 int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
-                            int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated)
+                            int max_entries, list<cls_log_entry>& entries,
+                           const string& marker,
+                           string *out_marker,
+                           bool *truncated)
 {
   librados::IoCtx io_ctx;
 
@@ -1596,7 +1599,8 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end
     return r;
   librados::ObjectReadOperation op;
 
-  cls_log_list(op, start_time, end_time, marker, max_entries, entries, &marker, truncated);
+  cls_log_list(op, start_time, end_time, marker, max_entries, entries,
+              out_marker, truncated);
 
   bufferlist obl;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index b37652d9f3fc6a737131fde854d00ffe38df3a6e..874492ffe692da7e6a0477246f160458482d879f 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1382,7 +1382,8 @@ public:
   int time_log_add(const string& oid, list<cls_log_entry>& entries);
   int time_log_add(const string& oid, const utime_t& ut, const string& section, const string& key, bufferlist& bl);
   int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time,
-                    int max_entries, list<cls_log_entry>& entries, string& marker, bool *truncated);
+                    int max_entries, list<cls_log_entry>& entries,
+                   const string& marker, string *out_marker, bool *truncated);
   int time_log_info(const string& oid, cls_log_header *header);
   int time_log_trim(const string& oid, const utime_t& start_time, const utime_t& end_time,
                     const string& from_marker, const string& to_marker);

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 74e3c445ee935f6052942090c9dce0e0ab21abc3..18f0697b0b266b3a3ce0e2c0c3c387beeae693d0 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -471,10 +471,12 @@ void RGWOp_DATALog_List::execute() {
     }
   } 
   
-  bool truncated;
   do {
+    // Note that last_marker is updated to be the marker of the last
+    // entry listed
     http_ret = store->data_log->list_entries(shard_id, ut_st, ut_et, 
-                               max_entries, entries, marker, &truncated);
+                                            max_entries, entries, marker,
+                                            &last_marker, &truncated);
     if (http_ret < 0) 
       break;
 
@@ -491,12 +493,18 @@ void RGWOp_DATALog_List::send_response() {
   if (http_ret < 0)
     return;
 
-  s->formatter->open_array_section("entries");
-  for (list<rgw_data_change>::iterator iter = entries.begin(); 
-       iter != entries.end(); ++iter) {
-    rgw_data_change& entry = *iter;
-    encode_json("entry", entry, s->formatter);
-    flusher.flush();
+  s->formatter->open_object_section("log_entries");
+  s->formatter->dump_string("marker", last_marker);
+  s->formatter->dump_bool("truncated", truncated);
+  {
+    s->formatter->open_array_section("entries");
+    for (list<rgw_data_change>::iterator iter = entries.begin();
+        iter != entries.end(); ++iter) {
+      rgw_data_change& entry = *iter;
+      encode_json("entry", entry, s->formatter);
+      flusher.flush();
+    }
+    s->formatter->close_section();
   }
   s->formatter->close_section();
   flusher.flush();

diff --git a/src/rgw/rgw_rest_log.h b/src/rgw/rgw_rest_log.h
index 2d60e289b84a85b8022411e17c0c253bb7f1248a..404db84aa7bf3f49086b126b360cf144cc0dc936 100644 (file)
--- a/src/rgw/rgw_rest_log.h
+++ b/src/rgw/rgw_rest_log.h
@@ -175,9 +175,11 @@ public:
 
 class RGWOp_DATALog_List : public RGWRESTOp {
   list<rgw_data_change> entries;
+  string last_marker;
+  bool truncated;
   int http_ret;
 public:
-  RGWOp_DATALog_List() : http_ret(0) {}
+  RGWOp_DATALog_List() : truncated(false), http_ret(0) {}
   ~RGWOp_DATALog_List() {}
 
   int check_caps(RGWUserCaps& caps) {