From: Casey Bodley Date: Mon, 20 Mar 2023 22:40:05 +0000 (-0400) Subject: rgw: remove Bucket::update_container_stats() X-Git-Tag: v19.0.0~427^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=d9dfd382c1556b15fdba00d47ced212556e83f30;p=ceph.git rgw: remove Bucket::update_container_stats() callers use Bucket::read_stats() to load bucket stats Signed-off-by: Casey Bodley --- diff --git a/src/rgw/driver/daos/rgw_sal_daos.cc b/src/rgw/driver/daos/rgw_sal_daos.cc index 862eb86315cf..e25977c1068e 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.cc +++ b/src/rgw/driver/daos/rgw_sal_daos.cc @@ -509,10 +509,6 @@ int DaosBucket::sync_user_stats(const DoutPrefixProvider* dpp, return DAOS_NOT_IMPLEMENTED_LOG(dpp); } -int DaosBucket::update_container_stats(const DoutPrefixProvider* dpp) { - return DAOS_NOT_IMPLEMENTED_LOG(dpp); -} - int DaosBucket::check_bucket_shards(const DoutPrefixProvider* dpp) { return DAOS_NOT_IMPLEMENTED_LOG(dpp); } diff --git a/src/rgw/driver/daos/rgw_sal_daos.h b/src/rgw/driver/daos/rgw_sal_daos.h index 3a8f34ce91c9..e27cc02cb6c8 100644 --- a/src/rgw/driver/daos/rgw_sal_daos.h +++ b/src/rgw/driver/daos/rgw_sal_daos.h @@ -314,7 +314,6 @@ class DaosBucket : public StoreBucket { RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider* dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider* dpp) override; virtual int check_bucket_shards(const DoutPrefixProvider* dpp) override; virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override; diff --git a/src/rgw/driver/motr/rgw_sal_motr.cc b/src/rgw/driver/motr/rgw_sal_motr.cc index b951d0080e4d..bd0fce07ff72 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.cc +++ b/src/rgw/driver/motr/rgw_sal_motr.cc @@ -830,11 +830,6 @@ int MotrBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) return 0; } -int MotrBucket::update_container_stats(const DoutPrefixProvider *dpp) -{ - return 0; -} - int MotrBucket::check_bucket_shards(const DoutPrefixProvider *dpp) { return 0; diff --git a/src/rgw/driver/motr/rgw_sal_motr.h b/src/rgw/driver/motr/rgw_sal_motr.h index a14f8e30f8ba..79a22cc7d828 100644 --- a/src/rgw/driver/motr/rgw_sal_motr.h +++ b/src/rgw/driver/motr/rgw_sal_motr.h @@ -374,7 +374,6 @@ class MotrBucket : public StoreBucket { const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider *dpp) override; virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override; virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override; virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override; diff --git a/src/rgw/driver/posix/rgw_sal_posix.cc b/src/rgw/driver/posix/rgw_sal_posix.cc index 036d02cf58dc..bb7617e8b9b4 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.cc +++ b/src/rgw/driver/posix/rgw_sal_posix.cc @@ -1049,46 +1049,17 @@ int POSIXBucket::read_stats(const DoutPrefixProvider *dpp, std::map& stats, std::string* max_marker, bool* syncstopped) { - return 0; -} - -int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp, - const bucket_index_layout_generation& idx_layout, - int shard_id, RGWGetBucketStats_CB* ctx) -{ - return 0; -} - -int POSIXBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) -{ - return 0; -} - -int POSIXBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) -{ - /* Force re-stat */ - stat_done = false; - int ret = stat(dpp); - if (ret < 0) { - return ret; - } - - bucket_statx_save(stx, ent, mtime); - info.creation_time = ent.creation_time; - ent.count = 0; - ent.size = 0; - - // TODO dang: store size/count in attributes - ret = for_each(dpp, [this, &dpp](const char* name) { - int ret; - struct statx lstx; + auto& main = stats[RGWObjCategory::Main]; + // TODO: bucket stats shouldn't have to list all objects + return for_each(dpp, [this, dpp, &main] (const char* name) { if (name[0] == '.') { /* Skip dotfiles */ return 0; } - ret = statx(dir_fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &lstx); + struct statx lstx; + int ret = statx(dir_fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &lstx); if (ret < 0) { ret = errno; ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": " @@ -1097,13 +1068,25 @@ int POSIXBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_ } if (S_ISREG(lstx.stx_mode) || S_ISDIR(lstx.stx_mode)) { - ent.count++; - ent.size += lstx.stx_size; + main.num_objects++; + main.size += lstx.stx_size; + main.size_rounded += lstx.stx_size; + main.size_utilized += lstx.stx_size; } return 0; }); +} +int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp, + const bucket_index_layout_generation& idx_layout, + int shard_id, RGWGetBucketStats_CB* ctx) +{ + return 0; +} + +int POSIXBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) +{ return 0; } @@ -1199,14 +1182,9 @@ int POSIXBucket::check_quota(const DoutPrefixProvider *dpp, RGWQuota& quota, uin int POSIXBucket::try_refresh_info(const DoutPrefixProvider* dpp, ceph::real_time* pmtime, optional_yield y) { - int ret = update_container_stats(dpp, y); - if (ret < 0) { - return ret; - } - *pmtime = mtime; - ret = open(dpp); + int ret = open(dpp); if (ret < 0) { return ret; } diff --git a/src/rgw/driver/posix/rgw_sal_posix.h b/src/rgw/driver/posix/rgw_sal_posix.h index bd1285c3551a..34f415bb18fe 100644 --- a/src/rgw/driver/posix/rgw_sal_posix.h +++ b/src/rgw/driver/posix/rgw_sal_posix.h @@ -224,7 +224,6 @@ public: const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override; virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 207ab44380a4..bd505e3b0bda 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -8602,48 +8602,6 @@ int RGWRados::put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, real_t return 0; } -int RGWRados::update_containers_stats(map& m, const DoutPrefixProvider *dpp, optional_yield y) -{ - map::iterator iter; - for (iter = m.begin(); iter != m.end(); ++iter) { - RGWBucketEnt& ent = iter->second; - rgw_bucket& bucket = ent.bucket; - ent.count = 0; - ent.size = 0; - ent.size_rounded = 0; - - vector headers; - - RGWBucketInfo bucket_info; - int ret = get_bucket_instance_info(bucket, bucket_info, NULL, NULL, y, dpp); - if (ret < 0) { - return ret; - } - - int r = cls_bucket_head(dpp, bucket_info, bucket_info.layout.current_index, RGW_NO_SHARD, headers); - if (r < 0) - return r; - - auto hiter = headers.begin(); - for (; hiter != headers.end(); ++hiter) { - RGWObjCategory category = main_category; - auto iter = (hiter->stats).find(category); - if (iter != hiter->stats.end()) { - struct rgw_bucket_category_stats& stats = iter->second; - ent.count += stats.num_entries; - ent.size += stats.total_size; - ent.size_rounded += stats.total_size_rounded; - } - } - - // fill in placement_rule from the bucket instance for use in swift's - // per-storage policy statistics - ent.placement_rule = std::move(bucket_info.placement_rule); - } - - return m.size(); -} - int RGWRados::append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl) { rgw_rados_ref ref; diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h index 95ee0374db53..23b69a8259f8 100644 --- a/src/rgw/driver/rados/rgw_rados.h +++ b/src/rgw/driver/rados/rgw_rados.h @@ -1376,7 +1376,6 @@ public: void gen_rand_obj_instance_name(rgw_obj_key *target_key); void gen_rand_obj_instance_name(rgw_obj *target); - int update_containers_stats(std::map& m, const DoutPrefixProvider *dpp, optional_yield y); int append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl); public: diff --git a/src/rgw/driver/rados/rgw_sal_rados.cc b/src/rgw/driver/rados/rgw_sal_rados.cc index ebac3bb8cf9b..8eafadc905c3 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.cc +++ b/src/rgw/driver/rados/rgw_sal_rados.cc @@ -690,34 +690,6 @@ int RadosBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y return store->ctl()->bucket->sync_user_stats(dpp, owner->get_id(), info, y, &ent); } -int RadosBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) -{ - int ret; - map m; - - m[info.bucket.name] = ent; - ret = store->getRados()->update_containers_stats(m, dpp, y); - if (!ret) - return -EEXIST; - if (ret < 0) - return ret; - - map::iterator iter = m.find(info.bucket.name); - if (iter == m.end()) - return -EINVAL; - - ent.count = iter->second.count; - ent.size = iter->second.size; - ent.size_rounded = iter->second.size_rounded; - ent.creation_time = iter->second.creation_time; - ent.placement_rule = std::move(iter->second.placement_rule); - - info.creation_time = ent.creation_time; - info.placement_rule = ent.placement_rule; - - return 0; -} - int RadosBucket::check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) { return store->getRados()->check_bucket_shards(info, info.bucket, get_count(), dpp, y); diff --git a/src/rgw/driver/rados/rgw_sal_rados.h b/src/rgw/driver/rados/rgw_sal_rados.h index c84a18ee45f3..750460460e75 100644 --- a/src/rgw/driver/rados/rgw_sal_rados.h +++ b/src/rgw/driver/rados/rgw_sal_rados.h @@ -567,7 +567,6 @@ class RadosBucket : public StoreBucket { const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override; virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override; diff --git a/src/rgw/rgw_file_int.h b/src/rgw/rgw_file_int.h index 65ec3dd15d33..6f71f754ebe4 100644 --- a/src/rgw/rgw_file_int.h +++ b/src/rgw/rgw_file_int.h @@ -2352,10 +2352,10 @@ public: void send_response() override { bucket->get_creation_time() = get_state()->bucket->get_info().creation_time; - bs.size = bucket->get_size(); - bs.size_rounded = bucket->get_size_rounded(); + bs.size = stats.size; + bs.size_rounded = stats.size_rounded; bs.creation_time = bucket->get_creation_time(); - bs.num_entries = bucket->get_count(); + bs.num_entries = stats.num_objects; std::swap(attrs, get_state()->bucket_attrs); } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 5de07969deb2..585c37f3ab53 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -2955,6 +2955,22 @@ void RGWStatBucket::pre_exec() rgw_bucket_object_pre_exec(s); } +// read the bucket's stats for RGWObjCategory::Main +static int load_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y, + rgw::sal::Bucket& bucket, RGWStorageStats& stats) +{ + const auto& index = bucket.get_info().layout.current_index; + std::string bver, mver; // ignored + std::map categories; + + int r = bucket.read_stats(dpp, index, -1, &bver, &mver, categories); + if (r < 0) { + return r; + } + stats = categories[RGWObjCategory::Main]; + return 0; +} + void RGWStatBucket::execute(optional_yield y) { if (!s->bucket_exists) { @@ -2966,7 +2982,8 @@ void RGWStatBucket::execute(optional_yield y) if (op_ret) { return; } - op_ret = bucket->update_container_stats(s, y); + + op_ret = load_bucket_stats(this, y, *s->bucket, stats); } int RGWListBucket::verify_permission(optional_yield y) @@ -3029,7 +3046,10 @@ void RGWListBucket::execute(optional_yield y) } if (need_container_stats()) { - op_ret = s->bucket->update_container_stats(s, y); + stats.emplace(); + if (int ret = load_bucket_stats(this, y, *s->bucket, *stats); ret < 0) { + stats = std::nullopt; // just don't return stats + } } rgw::sal::Bucket::ListParams params; diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 63d5f673ac1d..0dcdd456d7d9 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -910,23 +910,21 @@ protected: std::string max_keys; std::string delimiter; std::string encoding_type; - bool list_versions; - int max; + bool list_versions{false}; + int max{0}; std::vector objs; std::map common_prefixes; + std::optional stats; // initialized if need_container_stats() - int default_max; - bool is_truncated; - bool allow_unordered; + int default_max{0}; + bool is_truncated{false}; + bool allow_unordered{false}; - int shard_id; + int shard_id{-1}; int parse_max_keys(); public: - RGWListBucket() : list_versions(false), max(0), - default_max(0), is_truncated(false), - allow_unordered(false), shard_id(-1) {} int verify_permission(optional_yield y) override; void pre_exec() override; void execute(optional_yield y) override; @@ -1063,6 +1061,7 @@ public: class RGWStatBucket : public RGWOp { protected: std::unique_ptr bucket; + RGWStorageStats stats; public: int verify_permission(optional_yield y) override; diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index ba2099f38ad2..ff04b98a9934 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -353,6 +353,7 @@ int RGWListBucket_ObjStore_SWIFT::get_params(optional_yield y) static void dump_container_metadata(req_state *, const rgw::sal::Bucket*, + const std::optional& stats, const RGWQuotaInfo&, const RGWBucketWebsiteConf&); @@ -362,7 +363,7 @@ void RGWListBucket_ObjStore_SWIFT::send_response() map::iterator pref_iter = common_prefixes.begin(); dump_start(s); - dump_container_metadata(s, s->bucket.get(), quota.bucket_quota, + dump_container_metadata(s, s->bucket.get(), stats, quota.bucket_quota, s->bucket->get_info().website_conf); s->formatter->open_array_section_with_attrs("container", @@ -463,15 +464,18 @@ next: static void dump_container_metadata(req_state *s, const rgw::sal::Bucket* bucket, + const std::optional& stats, const RGWQuotaInfo& quota, const RGWBucketWebsiteConf& ws_conf) { /* Adding X-Timestamp to keep align with Swift API */ dump_header(s, "X-Timestamp", utime_t(s->bucket->get_info().creation_time)); - dump_header(s, "X-Container-Object-Count", bucket->get_count()); - dump_header(s, "X-Container-Bytes-Used", bucket->get_size()); - dump_header(s, "X-Container-Bytes-Used-Actual", bucket->get_size_rounded()); + if (stats) { + dump_header(s, "X-Container-Object-Count", stats->num_objects); + dump_header(s, "X-Container-Bytes-Used", stats->size); + dump_header(s, "X-Container-Bytes-Used-Actual", stats->size_rounded); + } if (rgw::sal::Object::empty(s->object.get())) { auto swift_policy = \ @@ -586,7 +590,7 @@ void RGWStatBucket_ObjStore_SWIFT::send_response() { if (op_ret >= 0) { op_ret = STATUS_NO_CONTENT; - dump_container_metadata(s, bucket.get(), quota.bucket_quota, + dump_container_metadata(s, bucket.get(), stats, quota.bucket_quota, s->bucket->get_info().website_conf); } @@ -2509,7 +2513,7 @@ RGWOp* RGWSwiftWebsiteHandler::get_ws_listing_op() /* Generate the header now. */ set_req_state_err(s, op_ret); dump_errno(s); - dump_container_metadata(s, s->bucket.get(), quota.bucket_quota, + dump_container_metadata(s, s->bucket.get(), stats, quota.bucket_quota, s->bucket->get_info().website_conf); end_header(s, this, "text/html"); if (op_ret < 0) { diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 253d36cb3b28..6754324ea7e4 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -23,6 +23,7 @@ #include "rgw_datalog_notify.h" #include "include/random.h" +struct RGWBucketEnt; class RGWRESTMgr; class RGWAccessListFilter; class RGWLC; @@ -676,8 +677,6 @@ class Bucket { int shard_id, RGWGetBucketStats_CB* ctx) = 0; /** Sync this bucket's stats to the owning user's stats in the backing store */ virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0; - /** Refresh the metadata stats (size, count, and so on) from the backing store */ - virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) = 0; /** Check if this bucket needs resharding, and schedule it if it does */ virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) = 0; /** Change the owner of this bucket in the backing store. Current owner must be set. Does not diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 143830fda51a..2465b03e5868 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -311,11 +311,6 @@ namespace rgw::sal { return 0; } - int DBBucket::update_container_stats(const DoutPrefixProvider *dpp, optional_yield y) - { - return 0; - } - int DBBucket::check_bucket_shards(const DoutPrefixProvider *dpp, optional_yield y) { return 0; diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index a6e140380e5c..274dd6e032f4 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -200,7 +200,6 @@ protected: bool *syncstopped = nullptr) override; virtual int read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider *dpp, optional_yield y) override; virtual int check_bucket_shards(const DoutPrefixProvider *dpp, optional_yield y) override; virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override; virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override; diff --git a/src/rgw/rgw_sal_filter.cc b/src/rgw/rgw_sal_filter.cc index 923f9990a28d..786047740038 100644 --- a/src/rgw/rgw_sal_filter.cc +++ b/src/rgw/rgw_sal_filter.cc @@ -711,11 +711,6 @@ int FilterBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield return next->sync_user_stats(dpp, y); } -int FilterBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) -{ - return next->update_container_stats(dpp, y); -} - int FilterBucket::check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) { return next->check_bucket_shards(dpp, y); diff --git a/src/rgw/rgw_sal_filter.h b/src/rgw/rgw_sal_filter.h index 9776cd5c228d..3076d7d30d88 100644 --- a/src/rgw/rgw_sal_filter.h +++ b/src/rgw/rgw_sal_filter.h @@ -436,7 +436,6 @@ public: const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override;