From: Radoslaw Zarzynski Date: Thu, 19 Nov 2015 16:45:04 +0000 (+0100) Subject: rgw: add support for "end_marker" on listing Swift account. X-Git-Tag: v10.0.3~105^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=58adddaa2c9ba2bedd0f40031e4b0d093e8f324d;p=ceph.git rgw: add support for "end_marker" on listing Swift account. Fixes: #10682 Signed-off-by: Radoslaw Zarzynski --- diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc index 3a9111294176..d168e7cb8a84 100644 --- a/src/cls/user/cls_user.cc +++ b/src/cls/user/cls_user.cc @@ -289,7 +289,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff map keys; - string from_index = op.marker; + const string& from_index = op.marker; + const string& to_index = op.end_marker; + const bool to_index_valid = !to_index.empty(); #define MAX_ENTRIES 1000 size_t max_entries = op.max_entries; @@ -302,7 +304,10 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff if (rc < 0) return rc; - CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str()); + CLS_LOG(20, "from_index=%s to_index=%s match_prefix=%s", + from_index.c_str(), + to_index.c_str(), + match_prefix.c_str()); cls_user_list_buckets_ret ret; list& entries = ret.entries; @@ -316,6 +321,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff const string& index = iter->first; marker = index; + if (to_index_valid && to_index.compare(index) <= 0) + break; + bufferlist& bl = iter->second; bufferlist::iterator biter = bl.begin(); try { diff --git a/src/cls/user/cls_user_client.cc b/src/cls/user/cls_user_client.cc index d86c5bfc3ced..279117e9a31c 100644 --- a/src/cls/user/cls_user_client.cc +++ b/src/cls/user/cls_user_client.cc @@ -72,12 +72,18 @@ public: }; void cls_user_bucket_list(librados::ObjectReadOperation& op, - const string& in_marker, int max_entries, list& entries, - string *out_marker, bool *truncated, int *pret) + const string& in_marker, + const string& end_marker, + int max_entries, + list& entries, + string *out_marker, + bool *truncated, + int *pret) { bufferlist inbl; cls_user_list_buckets_op call; call.marker = in_marker; + call.end_marker = end_marker; call.max_entries = max_entries; ::encode(call, inbl); diff --git a/src/cls/user/cls_user_client.h b/src/cls/user/cls_user_client.h index dcfdab6f6c4f..5a1d9441200a 100644 --- a/src/cls/user/cls_user_client.h +++ b/src/cls/user/cls_user_client.h @@ -23,9 +23,12 @@ void cls_user_set_buckets(librados::ObjectWriteOperation& op, list& entries, - string *out_marker, bool *truncated, + string *out_marker, + bool *truncated, int *pret); void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret); int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx); diff --git a/src/cls/user/cls_user_ops.h b/src/cls/user/cls_user_ops.h index e1a169b6ce26..9b3686ec9468 100644 --- a/src/cls/user/cls_user_ops.h +++ b/src/cls/user/cls_user_ops.h @@ -59,6 +59,7 @@ WRITE_CLASS_ENCODER(cls_user_remove_bucket_op) struct cls_user_list_buckets_op { string marker; + string end_marker; int max_entries; /* upperbound to returned num of entries might return less than that and still be truncated */ @@ -66,16 +67,20 @@ struct cls_user_list_buckets_op { : max_entries(0) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(marker, bl); ::encode(max_entries, bl); + ::encode(end_marker, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(marker, bl); ::decode(max_entries, bl); + if (struct_v >= 2) { + ::decode(end_marker, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index f4a245f536cb..3224f9acd3cb 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -90,7 +90,8 @@ int rgw_read_user_buckets(RGWRados * store, const string& marker, uint64_t max, bool need_stats, - uint64_t default_amount) + uint64_t default_amount, + string end_marker) { int ret; buckets.clear(); @@ -111,7 +112,7 @@ int rgw_read_user_buckets(RGWRados * store, } do { - ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated); + ret = store->cls_user_list_buckets(obj, m, end_marker, max - total, entries, &m, &truncated); if (ret == -ENOENT) ret = 0; diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 9191fbe8651c..1a3b2454d716 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -121,7 +121,8 @@ extern int rgw_read_user_buckets(RGWRados *store, const string& marker, uint64_t max, bool need_stats, - uint64_t default_amount = 1000); + uint64_t default_amount = 1000, + string end_marker = string()); extern int rgw_link_bucket(RGWRados *store, const rgw_user& user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true); extern int rgw_unlink_bucket(RGWRados *store, const rgw_user& user_id, diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 502598803803..bdda5efbc610 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1087,7 +1087,7 @@ void RGWListBuckets::execute() } ret = rgw_read_user_buckets(store, s->user.user_id, buckets, - marker, read_count, should_get_stats(), 0); + marker, read_count, should_get_stats(), 0, end_marker); if (ret < 0) { /* hmm.. something wrong here.. the user was authenticated, so it diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 0252ff8a65d2..2b3c1be7908c 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -195,7 +195,8 @@ protected: int ret; bool sent_data; string marker; - int64_t limit; + string end_marker; + uint64_t limit; uint64_t limit_max; uint32_t buckets_count; uint64_t buckets_objcount; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 982d7851ec98..b33be532f0a0 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -8769,9 +8769,12 @@ int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket } int RGWRados::cls_user_list_buckets(rgw_obj& obj, - const string& in_marker, int max_entries, + const string& in_marker, + const string& end_marker, + const int max_entries, list& entries, - string *out_marker, bool *truncated) + string * const out_marker, + bool * const truncated) { rgw_rados_ref ref; rgw_bucket bucket; @@ -8783,7 +8786,7 @@ int RGWRados::cls_user_list_buckets(rgw_obj& obj, librados::ObjectReadOperation op; int rc; - cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc); + cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc); bufferlist ibl; r = ref.ioctx.operate(ref.oid, &op, &ibl); if (r < 0) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 67adc60e4c73..f0f11b4d872a 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2182,9 +2182,12 @@ public: int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket); int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats); int cls_user_list_buckets(rgw_obj& obj, - const string& in_marker, int max_entries, + const string& in_marker, + const string& end_marker, + int max_entries, list& entries, - string *out_marker, bool *truncated); + string *out_marker, + bool *truncated); int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry); int cls_user_update_buckets(rgw_obj& obj, list& entries, bool add); int cls_user_complete_stats_sync(rgw_obj& obj); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 06817ad8a537..de7fd6c6eabe 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -17,6 +17,7 @@ int RGWListBuckets_ObjStore_SWIFT::get_params() { marker = s->info.args.get("marker"); + end_marker = s->info.args.get("end_marker"); string limit_str = s->info.args.get("limit"); if (!limit_str.empty()) {