rgw: add support for "end_marker" on listing Swift account.

Fixes: #10682
Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>

diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc
index 3a9111294176209e036c1d864c150389c76a7b0c..d168e7cb8a84df86721083a2a222f8dd3a70901a 100644 (file)
--- a/src/cls/user/cls_user.cc
+++ b/src/cls/user/cls_user.cc
@@ -289,7 +289,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
 
   map<string, bufferlist> keys;
 
-  string from_index = op.marker;
+  const string& from_index = op.marker;
+  const string& to_index = op.end_marker;
+  const bool to_index_valid = !to_index.empty();
 
 #define MAX_ENTRIES 1000
   size_t max_entries = op.max_entries;
@@ -302,7 +304,10 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
   if (rc < 0)
     return rc;
 
-  CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str());
+  CLS_LOG(20, "from_index=%s to_index=%s match_prefix=%s",
+          from_index.c_str(),
+          to_index.c_str(),
+          match_prefix.c_str());
   cls_user_list_buckets_ret ret;
 
   list<cls_user_bucket_entry>& entries = ret.entries;
@@ -316,6 +321,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
     const string& index = iter->first;
     marker = index;
 
+    if (to_index_valid && to_index.compare(index) <= 0)
+      break;
+
     bufferlist& bl = iter->second;
     bufferlist::iterator biter = bl.begin();
     try {

diff --git a/src/cls/user/cls_user_client.cc b/src/cls/user/cls_user_client.cc
index d86c5bfc3cedd4c49a6b762b4237f94cb21cc20f..279117e9a31c5cfbd448408e45b569a3b9bdbf27 100644 (file)
--- a/src/cls/user/cls_user_client.cc
+++ b/src/cls/user/cls_user_client.cc
@@ -72,12 +72,18 @@ public:
 };
 
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries, list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated, int *pret)
+                          const string& in_marker,
+                          const string& end_marker,
+                          int max_entries,
+                          list<cls_user_bucket_entry>& entries,
+                          string *out_marker,
+                          bool *truncated,
+                          int *pret)
 {
   bufferlist inbl;
   cls_user_list_buckets_op call;
   call.marker = in_marker;
+  call.end_marker = end_marker;
   call.max_entries = max_entries;
 
   ::encode(call, inbl);

diff --git a/src/cls/user/cls_user_client.h b/src/cls/user/cls_user_client.h
index dcfdab6f6c4f276c1b94d16981d271cf71e1f907..5a1d9441200ab4a36a486a439413cd8a50fb5dcf 100644 (file)
--- a/src/cls/user/cls_user_client.h
+++ b/src/cls/user/cls_user_client.h
@@ -23,9 +23,12 @@ void cls_user_set_buckets(librados::ObjectWriteOperation& op, list<cls_user_buck
 void cls_user_complete_stats_sync(librados::ObjectWriteOperation& op);
 void cls_user_remove_bucket(librados::ObjectWriteOperation& op,  const cls_user_bucket& bucket);
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries,
+                       const string& in_marker,
+                       const string& end_marker,
+                       int max_entries,
                        list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated,
+                       string *out_marker,
+                       bool *truncated,
                        int *pret);
 void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret);
 int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx);

diff --git a/src/cls/user/cls_user_ops.h b/src/cls/user/cls_user_ops.h
index e1a169b6ce26c48067cb829b35f6f8cb321ab8d6..9b3686ec946838c59c7e2166801aa01103d2fb13 100644 (file)
--- a/src/cls/user/cls_user_ops.h
+++ b/src/cls/user/cls_user_ops.h
@@ -59,6 +59,7 @@ WRITE_CLASS_ENCODER(cls_user_remove_bucket_op)
 
 struct cls_user_list_buckets_op {
   string marker;
+  string end_marker;
   int max_entries; /* upperbound to returned num of entries
                       might return less than that and still be truncated */
 
@@ -66,16 +67,20 @@ struct cls_user_list_buckets_op {
     : max_entries(0) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(marker, bl);
     ::encode(max_entries, bl);
+    ::encode(end_marker, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     ::decode(marker, bl);
     ::decode(max_entries, bl);
+    if (struct_v >= 2) {
+      ::decode(end_marker, bl);
+    }
     DECODE_FINISH(bl);
   }
 

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index f4a245f536cbf50973f74a047f7086032c86186b..3224f9acd3cb6529f68112e04942c250cfd440ef 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -90,7 +90,8 @@ int rgw_read_user_buckets(RGWRados * store,
                           const string& marker,
                           uint64_t max,
                           bool need_stats,
-                          uint64_t default_amount)
+                          uint64_t default_amount,
+                          string end_marker)
 {
   int ret;
   buckets.clear();
@@ -111,7 +112,7 @@ int rgw_read_user_buckets(RGWRados * store,
   }
 
   do {
-    ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated);
+    ret = store->cls_user_list_buckets(obj, m, end_marker, max - total, entries, &m, &truncated);
     if (ret == -ENOENT)
       ret = 0;
 

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 9191fbe8651ccc97353ab781c6b3f31d1236901a..1a3b2454d7165f5ac3586efb8ba352dfdb812958 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -121,7 +121,8 @@ extern int rgw_read_user_buckets(RGWRados *store,
                                  const string& marker,
                                  uint64_t max,
                                  bool need_stats,
-                                 uint64_t default_amount = 1000);
+                                 uint64_t default_amount = 1000,
+                                 string end_marker = string());
 
 extern int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);
 extern int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id,

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 5025988038039efe00950572526ccd2208fe6d14..bdda5efbc6105f598edd6b9b29746b94429a30ba 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1087,7 +1087,7 @@ void RGWListBuckets::execute()
     }
 
     ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
-                                marker, read_count, should_get_stats(), 0);
+                                marker, read_count, should_get_stats(), 0, end_marker);
 
     if (ret < 0) {
       /* hmm.. something wrong here.. the user was authenticated, so it

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 0252ff8a65d2f882586f5010d7f5f45a58e2ab21..2b3c1be7908c66e5cc5d77868a6d81cdff9e35fb 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -195,7 +195,8 @@ protected:
   int ret;
   bool sent_data;
   string marker;
-  int64_t limit;
+  string end_marker;
+  uint64_t limit;
   uint64_t limit_max;
   uint32_t buckets_count;
   uint64_t buckets_objcount;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 982d7851ec98b8fbbea99fbdc015dc903a6f6428..b33be532f0a046293e91b0e821353155458fa2e3 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -8769,9 +8769,12 @@ int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket
 }
 
 int RGWRados::cls_user_list_buckets(rgw_obj& obj,
-                                    const string& in_marker, int max_entries,
+                                    const string& in_marker,
+                                    const string& end_marker,
+                                    const int max_entries,
                                     list<cls_user_bucket_entry>& entries,
-                                    string *out_marker, bool *truncated)
+                                    string * const out_marker,
+                                    bool * const truncated)
 {
   rgw_rados_ref ref;
   rgw_bucket bucket;
@@ -8783,7 +8786,7 @@ int RGWRados::cls_user_list_buckets(rgw_obj& obj,
   librados::ObjectReadOperation op;
   int rc;
 
-  cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc);
+  cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc);
   bufferlist ibl;
   r = ref.ioctx.operate(ref.oid, &op, &ibl);
   if (r < 0)

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 67adc60e4c73bf12ee9e6dd4017c9ddd0a7df9d8..f0f11b4d872a0a1ad4a7395a8bfc90b7077c3fb8 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2182,9 +2182,12 @@ public:
   int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
   int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
   int cls_user_list_buckets(rgw_obj& obj,
-                            const string& in_marker, int max_entries,
+                            const string& in_marker,
+                            const string& end_marker,
+                            int max_entries,
                             list<cls_user_bucket_entry>& entries,
-                            string *out_marker, bool *truncated);
+                            string *out_marker,
+                            bool *truncated);
   int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry);
   int cls_user_update_buckets(rgw_obj& obj, list<cls_user_bucket_entry>& entries, bool add);
   int cls_user_complete_stats_sync(rgw_obj& obj);

diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc
index 06817ad8a5376442c69eb260427be51a385e5ba6..de7fd6c6eabe5e9f7a3c4b7cbe7478d4c4583414 100644 (file)
--- a/src/rgw/rgw_rest_swift.cc
+++ b/src/rgw/rgw_rest_swift.cc
@@ -17,6 +17,7 @@
 int RGWListBuckets_ObjStore_SWIFT::get_params()
 {
   marker = s->info.args.get("marker");
+  end_marker = s->info.args.get("end_marker");
 
   string limit_str = s->info.args.get("limit");
   if (!limit_str.empty()) {