From 7068d8595ff462323dedd9b56bce28a76405585a Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 17 Jul 2014 11:24:51 -0700 Subject: [PATCH] rgw: add NextMarker param for bucket listing Partially fixes #8858. Signed-off-by: Yehuda Sadeh (cherry picked from commit 924686f0b6593deffcd1d4e80ab06b1e7af00dcb) --- src/rgw/rgw_admin.cc | 2 +- src/rgw/rgw_bucket.cc | 6 +++--- src/rgw/rgw_op.cc | 8 +++++--- src/rgw/rgw_op.h | 1 + src/rgw/rgw_rados.cc | 7 ++++++- src/rgw/rgw_rados.h | 5 +++-- src/rgw/rgw_rest_s3.cc | 2 ++ 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 2d6b6e8504f2a..16209eaa10817 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -1358,7 +1358,7 @@ int main(int argc, char **argv) do { list entries; ret = store->list_objects(bucket, max_entries - count, prefix, delim, - marker, result, common_prefixes, true, + marker, NULL, result, common_prefixes, true, ns, false, &truncated, NULL); if (ret < 0) { cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 3e57b934fef77..79442e8947088 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -333,7 +333,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children) if (delete_children) { int max = 1000; - ret = store->list_objects(bucket, max, prefix, delim, marker, + ret = store->list_objects(bucket, max, prefix, delim, marker, NULL, objs, common_prefixes, false, ns, true, NULL, NULL); @@ -349,7 +349,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children) } objs.clear(); - ret = store->list_objects(bucket, max, prefix, delim, marker, objs, common_prefixes, + ret = store->list_objects(bucket, max, prefix, delim, marker, NULL, objs, common_prefixes, false, ns, true, NULL, NULL); if (ret < 0) return ret; @@ -597,7 +597,7 @@ int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, do { vector result; - int r = store->list_objects(bucket, max, prefix, delim, marker, + int r = store->list_objects(bucket, max, prefix, delim, marker, NULL, result, common_prefixes, false, ns, true, &is_truncated, NULL); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ac1e11c86ec64..7c56955269ead 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -640,7 +640,7 @@ int RGWGetObj::iterate_user_manifest_parts(rgw_bucket& bucket, string& obj_prefi do { #define MAX_LIST_OBJS 100 - int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker, + int r = store->list_objects(bucket, MAX_LIST_OBJS, obj_prefix, delim, marker, NULL, objs, common_prefixes, true, no_ns, true, &is_truncated, NULL); if (r < 0) @@ -1016,7 +1016,9 @@ void RGWListBucket::execute() if (ret < 0) return; - ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, objs, common_prefixes, + string *pnext_marker = (delimiter.empty() ? NULL : &next_marker); + + ret = store->list_objects(s->bucket, max, prefix, delimiter, marker, pnext_marker, objs, common_prefixes, !!(s->prot_flags & RGW_REST_SWIFT), no_ns, true, &is_truncated, NULL); } @@ -2520,7 +2522,7 @@ void RGWListBucketMultiparts::execute() } } marker_meta = marker.get_meta(); - ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, objs, common_prefixes, + ret = store->list_objects(s->bucket, max_uploads, prefix, delimiter, marker_meta, NULL, objs, common_prefixes, !!(s->prot_flags & RGW_REST_SWIFT), mp_ns, true, &is_truncated, &mp_filter); if (!objs.empty()) { vector::iterator iter; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 2c9ab0d899b1d..911800cd5f4bb 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -187,6 +187,7 @@ class RGWListBucket : public RGWOp { protected: string prefix; string marker; + string next_marker; string max_keys; string delimiter; int max; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 33fc4c436cada..6492f25a4ffd1 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1714,7 +1714,8 @@ int rgw_policy_from_attrset(CephContext *cct, map& attrset, * here. */ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string& delim, - string& marker, vector& result, map& common_prefixes, + string& marker, string *next_marker, vector& result, + map& common_prefixes, bool get_content_type, string& ns, bool enforce_ns, bool *is_truncated, RGWAccessListFilter *filter) { @@ -1781,6 +1782,10 @@ int RGWRados::list_objects(rgw_bucket& bucket, int max, string& prefix, string& continue; } + if (next_marker) { + *next_marker = obj; + } + if (filter && !filter->filter(obj, key)) continue; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index ea5e53429233a..7fdac57e1be18 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -968,8 +968,9 @@ public: * here. */ virtual int list_objects(rgw_bucket& bucket, int max, std::string& prefix, std::string& delim, - std::string& marker, std::vector& result, map& common_prefixes, - bool get_content_type, string& ns, bool enforce_ns, bool *is_truncated, RGWAccessListFilter *filter); + std::string& marker, std::string *next_marker, std::vector& result, + map& common_prefixes, bool get_content_type, string& ns, bool enforce_ns, + bool *is_truncated, RGWAccessListFilter *filter); virtual int create_pool(rgw_bucket& bucket); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index c6eacce29f6ac..3811f6c1a5c76 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -254,6 +254,8 @@ void RGWListBucket_ObjStore_S3::send_response() if (!prefix.empty()) s->formatter->dump_string("Prefix", prefix); s->formatter->dump_string("Marker", marker); + if (is_truncated && !next_marker.empty()) + s->formatter->dump_string("NextMarker", next_marker); s->formatter->dump_int("MaxKeys", max); if (!delimiter.empty()) s->formatter->dump_string("Delimiter", delimiter); -- 2.39.5