From 9f5f70d05f7144c9f7cef76263b640ca5cbfc245 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Wed, 1 Jul 2020 12:10:29 -0400 Subject: [PATCH] Zipper - implement RGWBucket::List Create a list op for RGWBucket, and use it. Signed-off-by: Daniel Gryniewicz --- src/rgw/rgw_op.cc | 76 ++++++++++++++++++++++------------------------ src/rgw/rgw_sal.cc | 53 ++++++++++++++++++++------------ src/rgw/rgw_sal.h | 23 +++++++++++++- 3 files changed, 92 insertions(+), 60 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ae8aa11835814..bcf7af8bc662e 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1640,7 +1640,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, rgw::sal::RGWRadosStore * const store, const off_t ofs, const off_t end, - RGWBucketInfo *pbucket_info, + rgw::sal::RGWBucket* bucket, const string& obj_prefix, RGWAccessControlPolicy * const bucket_acl, const boost::optional& bucket_policy, @@ -1657,30 +1657,26 @@ static int iterate_user_manifest_parts(CephContext * const cct, bool swift_slo), void * const cb_param) { - rgw_bucket& bucket = pbucket_info->bucket; uint64_t obj_ofs = 0, len_count = 0; bool found_start = false, found_end = false, handled_end = false; string delim; - bool is_truncated; - vector objs; utime_t start_time = ceph_clock_now(); - RGWRados::Bucket target(store->getRados(), *pbucket_info); - RGWRados::Bucket::List list_op(&target); - - list_op.params.prefix = obj_prefix; - list_op.params.delim = delim; + rgw::sal::RGWBucket::ListParams params; + params.prefix = obj_prefix; + params.delim = delim; + rgw::sal::RGWBucket::ListResults results; MD5 etag_sum; do { #define MAX_LIST_OBJS 100 - int r = list_op.list_objects(MAX_LIST_OBJS, &objs, NULL, &is_truncated, null_yield); + int r = bucket->list(params, MAX_LIST_OBJS, results, null_yield); if (r < 0) { return r; } - for (rgw_bucket_dir_entry& ent : objs) { + for (rgw_bucket_dir_entry& ent : results.objs) { const uint64_t cur_total_len = obj_ofs; const uint64_t obj_size = ent.meta.accounted_size; uint64_t start_ofs = 0, end_ofs = obj_size; @@ -1708,7 +1704,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, len_count += end_ofs - start_ofs; if (cb) { - r = cb(bucket, ent, bucket_acl, bucket_policy, start_ofs, end_ofs, + r = cb(bucket->get_bi(), ent, bucket_acl, bucket_policy, start_ofs, end_ofs, cb_param, false /* swift_slo */); if (r < 0) { return r; @@ -1719,7 +1715,7 @@ static int iterate_user_manifest_parts(CephContext * const cct, handled_end = found_end; start_time = ceph_clock_now(); } - } while (is_truncated); + } while (results.is_truncated); if (ptotal_len) { *ptotal_len = len_count; @@ -1856,31 +1852,30 @@ int RGWGetObj::handle_user_manifest(const char *prefix) boost::optional _bucket_policy; boost::optional* bucket_policy; RGWBucketInfo bucket_info; - RGWBucketInfo *pbucket_info; + std::unique_ptr ubucket; + rgw::sal::RGWBucket *pbucket = NULL; + int r = 0; if (bucket_name.compare(s->bucket->get_name()) != 0) { map bucket_attrs; - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - int r = store->getRados()->get_bucket_info(store->svc(), s->user->get_tenant(), - bucket_name, bucket_info, NULL, - s->yield, &bucket_attrs); + r = store->get_bucket(s->user, s->user->get_tenant(), bucket_name, &ubucket); if (r < 0) { ldpp_dout(this, 0) << "could not get bucket info for bucket=" << bucket_name << dendl; return r; } - pbucket_info = &bucket_info; bucket_acl = &_bucket_acl; - r = read_bucket_policy(store, s, bucket_info, bucket_attrs, bucket_acl, bucket_info.bucket); + r = read_bucket_policy(store, s, ubucket->get_info(), bucket_attrs, bucket_acl, ubucket->get_bi()); if (r < 0) { ldpp_dout(this, 0) << "failed to read bucket policy" << dendl; return r; } _bucket_policy = get_iam_policy_from_attr(s->cct, store, bucket_attrs, - bucket_info.bucket.tenant); + s->user->get_tenant()); bucket_policy = &_bucket_policy; + pbucket = ubucket.get(); } else { - pbucket_info = &s->bucket->get_info(); + pbucket = s->bucket.get(); bucket_acl = s->bucket_acl.get(); bucket_policy = &s->iam_policy; } @@ -1889,8 +1884,8 @@ int RGWGetObj::handle_user_manifest(const char *prefix) * - total length (of the parts we are going to send to client), * - overall DLO's content size, * - md5 sum of overall DLO's content (for etag of Swift API). */ - int r = iterate_user_manifest_parts(s->cct, store, ofs, end, - pbucket_info, obj_prefix, bucket_acl, *bucket_policy, + r = iterate_user_manifest_parts(s->cct, store, ofs, end, + pbucket, obj_prefix, bucket_acl, *bucket_policy, nullptr, &s->obj_size, &lo_etag, nullptr /* cb */, nullptr /* cb arg */); if (r < 0) { @@ -1904,7 +1899,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) } r = iterate_user_manifest_parts(s->cct, store, ofs, end, - pbucket_info, obj_prefix, bucket_acl, *bucket_policy, + pbucket, obj_prefix, bucket_acl, *bucket_policy, &total_len, nullptr, nullptr, nullptr, nullptr); if (r < 0) { @@ -1918,7 +1913,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) } r = iterate_user_manifest_parts(s->cct, store, ofs, end, - pbucket_info, obj_prefix, bucket_acl, *bucket_policy, + pbucket, obj_prefix, bucket_acl, *bucket_policy, nullptr, nullptr, nullptr, get_obj_user_manifest_iterate_cb, (void *)this); if (r < 0) { @@ -1930,7 +1925,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix) send_response_data(bl, 0, 0); } - return 0; + return r; } int RGWGetObj::handle_slo_manifest(bufferlist& bl) @@ -2885,22 +2880,23 @@ void RGWListBucket::execute() op_ret = s->bucket->update_container_stats(); } - RGWRados::Bucket target(store->getRados(), s->bucket->get_info()); - if (shard_id >= 0) { - target.set_shard_id(shard_id); - } - RGWRados::Bucket::List list_op(&target); + rgw::sal::RGWBucket::ListParams params; + params.prefix = prefix; + params.delim = delimiter; + params.marker = marker; + params.end_marker = end_marker; + params.list_versions = list_versions; + params.allow_unordered = allow_unordered; + params.shard_id = shard_id; - list_op.params.prefix = prefix; - list_op.params.delim = delimiter; - list_op.params.marker = marker; - list_op.params.end_marker = end_marker; - list_op.params.list_versions = list_versions; - list_op.params.allow_unordered = allow_unordered; + rgw::sal::RGWBucket::ListResults results; - op_ret = list_op.list_objects(max, &objs, &common_prefixes, &is_truncated, s->yield); + op_ret = s->bucket->list(params, max, results, s->yield); if (op_ret >= 0) { - next_marker = list_op.get_next_marker(); + next_marker = results.next_marker; + is_truncated = results.is_truncated; + objs = std::move(results.objs); + common_prefixes = std::move(results.common_prefixes); } } diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 8b9d95edc285d..6c7f53b8ea9a8 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -84,41 +84,33 @@ RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) int RGWRadosBucket::remove_bucket(bool delete_children, std::string prefix, std::string delimiter, optional_yield y) { int ret; - map stats; - std::vector objs; - map common_prefixes; - string bucket_ver, master_ver; + // Refresh info ret = get_bucket_info(y); if (ret < 0) return ret; - ret = get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver, &master_ver, stats); - if (ret < 0) - return ret; + ListParams params; + params.list_versions = true; + params.allow_unordered = true; - RGWRados::Bucket target(store->getRados(), info); - RGWRados::Bucket::List list_op(&target); - int max = 1000; - - list_op.params.list_versions = true; - list_op.params.allow_unordered = true; + ListResults results; bool is_truncated = false; do { - objs.clear(); + results.objs.clear(); - ret = list_op.list_objects(max, &objs, &common_prefixes, &is_truncated, null_yield); - if (ret < 0) - return ret; + ret = list(params, 1000, results, y); + if (ret < 0) + return ret; - if (!objs.empty() && !delete_children) { + if (!results.objs.empty() && !delete_children) { lderr(store->ctx()) << "ERROR: could not remove non-empty bucket " << info.bucket.name << dendl; return -ENOTEMPTY; } - for (const auto& obj : objs) { + for (const auto& obj : results.objs) { rgw_obj_key key(obj.key); /* xxx dang */ ret = rgw_remove_object(store, info, info.bucket, key); @@ -305,6 +297,29 @@ std::unique_ptr RGWRadosBucket::get_object(const rgw_obj_key& k) return std::unique_ptr(new RGWRadosObject(this->store, k, this)); } +int RGWRadosBucket::list(ListParams& params, int max, ListResults& results, optional_yield y) +{ + RGWRados::Bucket target(store->getRados(), get_info()); + if (params.shard_id >= 0) { + target.set_shard_id(params.shard_id); + } + RGWRados::Bucket::List list_op(&target); + + list_op.params.prefix = params.prefix; + list_op.params.delim = params.delim; + list_op.params.marker = params.marker; + list_op.params.end_marker = params.end_marker; + list_op.params.list_versions = params.list_versions; + list_op.params.allow_unordered = params.allow_unordered; + + int ret = list_op.list_objects(max, &results.objs, &results.common_prefixes, &results.is_truncated, y); + if (ret >= 0) { + results.next_marker = list_op.get_next_marker(); + } + + return ret; +} + std::unique_ptr RGWRadosStore::get_user(const rgw_user &u) { return std::unique_ptr(new RGWRadosUser(this, u)); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 7b27b3c3d4629..98953cc208655 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -112,6 +112,26 @@ class RGWBucket { ceph::real_time mtime; public: + + struct ListParams { + std::string prefix; + std::string delim; + rgw_obj_key marker; + rgw_obj_key end_marker; + std::string ns; + bool enforce_ns{true}; + RGWAccessListFilter *filter{nullptr}; + bool list_versions{false}; + bool allow_unordered{false}; + int shard_id{0}; + }; + struct ListResults { + vector objs; + map common_prefixes; + bool is_truncated; + rgw_obj_key next_marker; + }; + RGWBucket() : ent(), info(), owner(nullptr), attrs(), bucket_version() {} RGWBucket(const rgw_bucket& _b) : ent(), info(), owner(nullptr), attrs(), bucket_version() { ent.bucket = _b; info.bucket = _b; } @@ -129,7 +149,7 @@ class RGWBucket { virtual int load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) = 0; virtual std::unique_ptr get_object(const rgw_obj_key& key) = 0; - virtual RGWBucketList* list(void) = 0; + virtual int list(ListParams&, int, ListResults&, optional_yield y) = 0; virtual RGWObject* create_object(const rgw_obj_key& key /* Attributes */) = 0; virtual RGWAttrs& get_attrs(void) { return attrs; } virtual int set_attrs(RGWAttrs a) { attrs = a; return 0; } @@ -441,6 +461,7 @@ class RGWRadosBucket : public RGWBucket { virtual int load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) override; virtual std::unique_ptr get_object(const rgw_obj_key& k) override; RGWBucketList* list(void) { return new RGWBucketList(); } + virtual int list(ListParams&, int, ListResults&, optional_yield y) override; RGWObject* create_object(const rgw_obj_key& key /* Attributes */) override; virtual int remove_bucket(bool delete_children, std::string prefix, std::string delimiter, optional_yield y) override; RGWAccessControlPolicy& get_acl(void) { return acls; } -- 2.39.5