rgw: add marker to metadata api keys listing

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
(cherry picked from commit 84ece81bc3590b08e5565c5b96bcbb4cb97a4100)

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 41b7d12315066a5ffd7188bd4a9e275e6806069c..904d62e11a66b2acfca056499d68b1f8934d2cd2 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -2164,12 +2164,17 @@ public:
     pool = store->get_zone_params().domain_root;
   }
 
-  int list_keys_init(RGWRados *store, void **phandle) override
+  int list_keys_init(RGWRados *store, const string& marker, void **phandle)
   {
     list_keys_info *info = new list_keys_info;
 
     info->store = store;
 
+    int ret = store->list_raw_objects_init(store->get_zone_params().domain_root, marker,
+                                           &info->ctx);
+    if (ret < 0) {
+      return ret;
+    }
     *phandle = (void *)info;
 
     return 0;
@@ -2186,8 +2191,8 @@ public:
 
     list<string> unfiltered_keys;
 
-    int ret = store->list_raw_objects(store->get_zone_params().domain_root, no_filter,
-                                      max, info->ctx, unfiltered_keys, truncated);
+    int ret = store->list_raw_objects_next(no_filter, max, info->ctx,
+                                           unfiltered_keys, truncated);
     if (ret < 0 && ret != -ENOENT)
       return ret;
     if (ret == -ENOENT) {
@@ -2213,6 +2218,11 @@ public:
     list_keys_info *info = static_cast<list_keys_info *>(handle);
     delete info;
   }
+
+  string get_marker(void *handle) {
+    list_keys_info *info = static_cast<list_keys_info *>(handle);
+    return info->store->list_raw_objs_get_cursor(info->ctx);
+  }
 };
 
 class RGWBucketInstanceMetadataHandler : public RGWMetadataHandler {
@@ -2354,12 +2364,16 @@ public:
     pool = store->get_zone_params().domain_root;
   }
 
-  int list_keys_init(RGWRados *store, void **phandle) override
-  {
+  int list_keys_init(RGWRados *store, const string& marker, void **phandle) override {
     list_keys_info *info = new list_keys_info;
 
     info->store = store;
 
+    int ret = store->list_raw_objects_init(store->get_zone_params().domain_root, marker,
+                                           &info->ctx);
+    if (ret < 0) {
+      return ret;
+    }
     *phandle = (void *)info;
 
     return 0;
@@ -2376,8 +2390,8 @@ public:
 
     list<string> unfiltered_keys;
 
-    int ret = store->list_raw_objects(store->get_zone_params().domain_root, no_filter,
-                                      max, info->ctx, unfiltered_keys, truncated);
+    int ret = store->list_raw_objects_next(no_filter, max, info->ctx,
+                                           unfiltered_keys, truncated);
     if (ret < 0 && ret != -ENOENT)
       return ret;
     if (ret == -ENOENT) {
@@ -2407,6 +2421,11 @@ public:
     delete info;
   }
 
+  string get_marker(void *handle) {
+    list_keys_info *info = static_cast<list_keys_info *>(handle);
+    return info->store->list_raw_objs_get_cursor(info->ctx);
+  }
+
   /*
    * hash entry for mdlog placement. Use the same hash key we'd have for the bucket entry
    * point, so that the log entries end up at the same log shard, so that we process them

diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index f81c4490ae82160d605346f72706b1e05145a824..6c554810e98a81b181503bc0b81e2aa85609a84f 100644 (file)
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -273,8 +273,8 @@ obj_version& RGWMetadataObject::get_version()
 
 class RGWMetadataTopHandler : public RGWMetadataHandler {
   struct iter_data {
-    list<string> sections;
-    list<string>::iterator iter;
+    set<string> sections;
+    set<string>::iterator iter;
   };
 
 public:
@@ -290,10 +290,14 @@ public:
 
   int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) override { return -ENOTSUP; }
 
-  int list_keys_init(RGWRados *store, void **phandle) override {
+  int list_keys_init(RGWRados *store, const string& marker, void **phandle) override {
     iter_data *data = new iter_data;
-    store->meta_mgr->get_sections(data->sections);
-    data->iter = data->sections.begin();
+    list<string> sections;
+    store->meta_mgr->get_sections(sections);
+    for (auto& s : sections) {
+      data->sections.insert(s);
+    }
+    data->iter = data->sections.lower_bound(marker);
 
     *phandle = data;
 
@@ -314,6 +318,16 @@ public:
 
     delete data;
   }
+
+  virtual string get_marker(void *handle) {
+    iter_data *data = static_cast<iter_data *>(handle);
+
+    if (data->iter != data->sections.end()) {
+      return *(data->iter);
+    }
+
+    return string();
+  }
 };
 
 static RGWMetadataTopHandler md_top_handler;
@@ -830,8 +844,12 @@ struct list_keys_handle {
   RGWMetadataHandler *handler;
 };
 
-
 int RGWMetadataManager::list_keys_init(string& section, void **handle)
+{
+  return list_keys_init(section, string(), handle);
+}
+
+int RGWMetadataManager::list_keys_init(string& section, const string& marker, void **handle)
 {
   string entry;
   RGWMetadataHandler *handler;
@@ -845,7 +863,7 @@ int RGWMetadataManager::list_keys_init(string& section, void **handle)
 
   list_keys_handle *h = new list_keys_handle;
   h->handler = handler;
-  ret = handler->list_keys_init(store, &h->handle);
+  ret = handler->list_keys_init(store, marker, &h->handle);
   if (ret < 0) {
     delete h;
     return ret;
@@ -865,7 +883,6 @@ int RGWMetadataManager::list_keys_next(void *handle, int max, list<string>& keys
   return handler->list_keys_next(h->handle, max, keys, truncated);
 }
 
-
 void RGWMetadataManager::list_keys_complete(void *handle)
 {
   list_keys_handle *h = static_cast<list_keys_handle *>(handle);
@@ -876,6 +893,13 @@ void RGWMetadataManager::list_keys_complete(void *handle)
   delete h;
 }
 
+string RGWMetadataManager::get_marker(void *handle)
+{
+  list_keys_handle *h = static_cast<list_keys_handle *>(handle);
+
+  return h->handler->get_marker(h->handle);
+}
+
 void RGWMetadataManager::dump_log_entry(cls_log_entry& entry, Formatter *f)
 {
   f->open_object_section("entry");

diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 4d077e8f888cac395dfa6d7b9dae0170275582bd..f6dc2db03bc3be4cb85e034f4acd648506ac5b27 100644 (file)
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -79,10 +79,12 @@ public:
                   real_time mtime, JSONObj *obj, sync_type_t type) = 0;
   virtual int remove(RGWRados *store, string& entry, RGWObjVersionTracker& objv_tracker) = 0;
 
-  virtual int list_keys_init(RGWRados *store, void **phandle) = 0;
+  virtual int list_keys_init(RGWRados *store, const string& marker, void **phandle) = 0;
   virtual int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated) = 0;
   virtual void list_keys_complete(void *handle) = 0;
 
+  virtual string get_marker(void *handle) = 0;
+
   /* key to use for hashing entries for log shard placement */
   virtual void get_hash_key(const string& section, const string& key, string& hash_key) {
     hash_key = section + ":" + key;
@@ -352,9 +354,12 @@ public:
   int remove(string& metadata_key);
 
   int list_keys_init(string& section, void **phandle);
+  int list_keys_init(string& section, const string& marker, void **phandle);
   int list_keys_next(void *handle, int max, list<string>& keys, bool *truncated);
   void list_keys_complete(void *handle);
 
+  string get_marker(void *handle);
+
   void dump_log_entry(cls_log_entry& entry, Formatter *f);
 
   void get_sections(list<string>& sections);

diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index 41a5492b93089129c22db4f626d12d7f3c4e76b4..0fa690b590ff4e37e875c3ee5cea670cd8f4ef71 100644 (file)
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -2772,12 +2772,18 @@ public:
     pool = store->get_zone_params().user_uid_pool;
   }
 
-  int list_keys_init(RGWRados *store, void **phandle) override
+  int list_keys_init(RGWRados *store, const string& marker, void **phandle) override
   {
     list_keys_info *info = new list_keys_info;
 
     info->store = store;
 
+    int ret = store->list_raw_objects_init(store->get_zone_params().user_uid_pool, marker,
+                                           &info->ctx);
+    if (ret < 0) {
+      return ret;
+    }
+
     *phandle = (void *)info;
 
     return 0;
@@ -2794,8 +2800,8 @@ public:
 
     list<string> unfiltered_keys;
 
-    int ret = store->list_raw_objects(store->get_zone_params().user_uid_pool, no_filter,
-                                      max, info->ctx, unfiltered_keys, truncated);
+    int ret = store->list_raw_objects_next(no_filter, max, info->ctx,
+                                           unfiltered_keys, truncated);
     if (ret < 0 && ret != -ENOENT)
       return ret;                      
     if (ret == -ENOENT) {
@@ -2821,6 +2827,11 @@ public:
     list_keys_info *info = static_cast<list_keys_info *>(handle);
     delete info;
   }
+
+  string get_marker(void *handle) {
+    list_keys_info *info = static_cast<list_keys_info *>(handle);
+    return info->store->list_raw_objs_get_cursor(info->ctx);
+  }
 };
 
 void rgw_user_init(RGWRados *store)