return DAOS_NOT_IMPLEMENTED_LOG(dpp);
}
-int DaosBucket::update_container_stats(const DoutPrefixProvider* dpp) {
- return DAOS_NOT_IMPLEMENTED_LOG(dpp);
-}
-
int DaosBucket::check_bucket_shards(const DoutPrefixProvider* dpp) {
return DAOS_NOT_IMPLEMENTED_LOG(dpp);
}
RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider* dpp,
optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider* dpp) override;
virtual int check_bucket_shards(const DoutPrefixProvider* dpp) override;
virtual int chown(const DoutPrefixProvider* dpp, User& new_user,
optional_yield y) override;
return 0;
}
-int MotrBucket::update_container_stats(const DoutPrefixProvider *dpp)
-{
- return 0;
-}
-
int MotrBucket::check_bucket_shards(const DoutPrefixProvider *dpp)
{
return 0;
const bucket_index_layout_generation& idx_layout,
int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider *dpp) override;
virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override;
virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override;
virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override;
std::map<RGWObjCategory, RGWStorageStats>& stats,
std::string* max_marker, bool* syncstopped)
{
- return 0;
-}
-
-int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp,
- const bucket_index_layout_generation& idx_layout,
- int shard_id, RGWGetBucketStats_CB* ctx)
-{
- return 0;
-}
-
-int POSIXBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y)
-{
- return 0;
-}
-
-int POSIXBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y)
-{
- /* Force re-stat */
- stat_done = false;
- int ret = stat(dpp);
- if (ret < 0) {
- return ret;
- }
-
- bucket_statx_save(stx, ent, mtime);
- info.creation_time = ent.creation_time;
- ent.count = 0;
- ent.size = 0;
-
- // TODO dang: store size/count in attributes
- ret = for_each(dpp, [this, &dpp](const char* name) {
- int ret;
- struct statx lstx;
+ auto& main = stats[RGWObjCategory::Main];
+ // TODO: bucket stats shouldn't have to list all objects
+ return for_each(dpp, [this, dpp, &main] (const char* name) {
if (name[0] == '.') {
/* Skip dotfiles */
return 0;
}
- ret = statx(dir_fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &lstx);
+ struct statx lstx;
+ int ret = statx(dir_fd, name, AT_SYMLINK_NOFOLLOW, STATX_ALL, &lstx);
if (ret < 0) {
ret = errno;
ldpp_dout(dpp, 0) << "ERROR: could not stat object " << name << ": "
}
if (S_ISREG(lstx.stx_mode) || S_ISDIR(lstx.stx_mode)) {
- ent.count++;
- ent.size += lstx.stx_size;
+ main.num_objects++;
+ main.size += lstx.stx_size;
+ main.size_rounded += lstx.stx_size;
+ main.size_utilized += lstx.stx_size;
}
return 0;
});
+}
+int POSIXBucket::read_stats_async(const DoutPrefixProvider *dpp,
+ const bucket_index_layout_generation& idx_layout,
+ int shard_id, RGWGetBucketStats_CB* ctx)
+{
+ return 0;
+}
+
+int POSIXBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y)
+{
return 0;
}
int POSIXBucket::try_refresh_info(const DoutPrefixProvider* dpp, ceph::real_time* pmtime, optional_yield y)
{
- int ret = update_container_stats(dpp, y);
- if (ret < 0) {
- return ret;
- }
-
*pmtime = mtime;
- ret = open(dpp);
+ int ret = open(dpp);
if (ret < 0) {
return ret;
}
const bucket_index_layout_generation& idx_layout,
int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override;
virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive,
return 0;
}
-int RGWRados::update_containers_stats(map<string, RGWBucketEnt>& m, const DoutPrefixProvider *dpp, optional_yield y)
-{
- map<string, RGWBucketEnt>::iterator iter;
- for (iter = m.begin(); iter != m.end(); ++iter) {
- RGWBucketEnt& ent = iter->second;
- rgw_bucket& bucket = ent.bucket;
- ent.count = 0;
- ent.size = 0;
- ent.size_rounded = 0;
-
- vector<rgw_bucket_dir_header> headers;
-
- RGWBucketInfo bucket_info;
- int ret = get_bucket_instance_info(bucket, bucket_info, NULL, NULL, y, dpp);
- if (ret < 0) {
- return ret;
- }
-
- int r = cls_bucket_head(dpp, bucket_info, bucket_info.layout.current_index, RGW_NO_SHARD, headers);
- if (r < 0)
- return r;
-
- auto hiter = headers.begin();
- for (; hiter != headers.end(); ++hiter) {
- RGWObjCategory category = main_category;
- auto iter = (hiter->stats).find(category);
- if (iter != hiter->stats.end()) {
- struct rgw_bucket_category_stats& stats = iter->second;
- ent.count += stats.num_entries;
- ent.size += stats.total_size;
- ent.size_rounded += stats.total_size_rounded;
- }
- }
-
- // fill in placement_rule from the bucket instance for use in swift's
- // per-storage policy statistics
- ent.placement_rule = std::move(bucket_info.placement_rule);
- }
-
- return m.size();
-}
-
int RGWRados::append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl)
{
rgw_rados_ref ref;
void gen_rand_obj_instance_name(rgw_obj_key *target_key);
void gen_rand_obj_instance_name(rgw_obj *target);
- int update_containers_stats(std::map<std::string, RGWBucketEnt>& m, const DoutPrefixProvider *dpp, optional_yield y);
int append_async(const DoutPrefixProvider *dpp, rgw_raw_obj& obj, size_t size, bufferlist& bl);
public:
return store->ctl()->bucket->sync_user_stats(dpp, owner->get_id(), info, y, &ent);
}
-int RadosBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y)
-{
- int ret;
- map<std::string, RGWBucketEnt> m;
-
- m[info.bucket.name] = ent;
- ret = store->getRados()->update_containers_stats(m, dpp, y);
- if (!ret)
- return -EEXIST;
- if (ret < 0)
- return ret;
-
- map<std::string, RGWBucketEnt>::iterator iter = m.find(info.bucket.name);
- if (iter == m.end())
- return -EINVAL;
-
- ent.count = iter->second.count;
- ent.size = iter->second.size;
- ent.size_rounded = iter->second.size_rounded;
- ent.creation_time = iter->second.creation_time;
- ent.placement_rule = std::move(iter->second.placement_rule);
-
- info.creation_time = ent.creation_time;
- info.placement_rule = ent.placement_rule;
-
- return 0;
-}
-
int RadosBucket::check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y)
{
return store->getRados()->check_bucket_shards(info, info.bucket, get_count(), dpp, y);
const bucket_index_layout_generation& idx_layout,
int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int chown(const DoutPrefixProvider* dpp, User& new_user, optional_yield y) override;
virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override;
void send_response() override {
bucket->get_creation_time() = get_state()->bucket->get_info().creation_time;
- bs.size = bucket->get_size();
- bs.size_rounded = bucket->get_size_rounded();
+ bs.size = stats.size;
+ bs.size_rounded = stats.size_rounded;
bs.creation_time = bucket->get_creation_time();
- bs.num_entries = bucket->get_count();
+ bs.num_entries = stats.num_objects;
std::swap(attrs, get_state()->bucket_attrs);
}
rgw_bucket_object_pre_exec(s);
}
+// read the bucket's stats for RGWObjCategory::Main
+static int load_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y,
+ rgw::sal::Bucket& bucket, RGWStorageStats& stats)
+{
+ const auto& index = bucket.get_info().layout.current_index;
+ std::string bver, mver; // ignored
+ std::map<RGWObjCategory, RGWStorageStats> categories;
+
+ int r = bucket.read_stats(dpp, index, -1, &bver, &mver, categories);
+ if (r < 0) {
+ return r;
+ }
+ stats = categories[RGWObjCategory::Main];
+ return 0;
+}
+
void RGWStatBucket::execute(optional_yield y)
{
if (!s->bucket_exists) {
if (op_ret) {
return;
}
- op_ret = bucket->update_container_stats(s, y);
+
+ op_ret = load_bucket_stats(this, y, *s->bucket, stats);
}
int RGWListBucket::verify_permission(optional_yield y)
}
if (need_container_stats()) {
- op_ret = s->bucket->update_container_stats(s, y);
+ stats.emplace();
+ if (int ret = load_bucket_stats(this, y, *s->bucket, *stats); ret < 0) {
+ stats = std::nullopt; // just don't return stats
+ }
}
rgw::sal::Bucket::ListParams params;
std::string max_keys;
std::string delimiter;
std::string encoding_type;
- bool list_versions;
- int max;
+ bool list_versions{false};
+ int max{0};
std::vector<rgw_bucket_dir_entry> objs;
std::map<std::string, bool> common_prefixes;
+ std::optional<RGWStorageStats> stats; // initialized if need_container_stats()
- int default_max;
- bool is_truncated;
- bool allow_unordered;
+ int default_max{0};
+ bool is_truncated{false};
+ bool allow_unordered{false};
- int shard_id;
+ int shard_id{-1};
int parse_max_keys();
public:
- RGWListBucket() : list_versions(false), max(0),
- default_max(0), is_truncated(false),
- allow_unordered(false), shard_id(-1) {}
int verify_permission(optional_yield y) override;
void pre_exec() override;
void execute(optional_yield y) override;
class RGWStatBucket : public RGWOp {
protected:
std::unique_ptr<rgw::sal::Bucket> bucket;
+ RGWStorageStats stats;
public:
int verify_permission(optional_yield y) override;
static void dump_container_metadata(req_state *,
const rgw::sal::Bucket*,
+ const std::optional<RGWStorageStats>& stats,
const RGWQuotaInfo&,
const RGWBucketWebsiteConf&);
map<string, bool>::iterator pref_iter = common_prefixes.begin();
dump_start(s);
- dump_container_metadata(s, s->bucket.get(), quota.bucket_quota,
+ dump_container_metadata(s, s->bucket.get(), stats, quota.bucket_quota,
s->bucket->get_info().website_conf);
s->formatter->open_array_section_with_attrs("container",
static void dump_container_metadata(req_state *s,
const rgw::sal::Bucket* bucket,
+ const std::optional<RGWStorageStats>& stats,
const RGWQuotaInfo& quota,
const RGWBucketWebsiteConf& ws_conf)
{
/* Adding X-Timestamp to keep align with Swift API */
dump_header(s, "X-Timestamp", utime_t(s->bucket->get_info().creation_time));
- dump_header(s, "X-Container-Object-Count", bucket->get_count());
- dump_header(s, "X-Container-Bytes-Used", bucket->get_size());
- dump_header(s, "X-Container-Bytes-Used-Actual", bucket->get_size_rounded());
+ if (stats) {
+ dump_header(s, "X-Container-Object-Count", stats->num_objects);
+ dump_header(s, "X-Container-Bytes-Used", stats->size);
+ dump_header(s, "X-Container-Bytes-Used-Actual", stats->size_rounded);
+ }
if (rgw::sal::Object::empty(s->object.get())) {
auto swift_policy = \
{
if (op_ret >= 0) {
op_ret = STATUS_NO_CONTENT;
- dump_container_metadata(s, bucket.get(), quota.bucket_quota,
+ dump_container_metadata(s, bucket.get(), stats, quota.bucket_quota,
s->bucket->get_info().website_conf);
}
/* Generate the header now. */
set_req_state_err(s, op_ret);
dump_errno(s);
- dump_container_metadata(s, s->bucket.get(), quota.bucket_quota,
+ dump_container_metadata(s, s->bucket.get(), stats, quota.bucket_quota,
s->bucket->get_info().website_conf);
end_header(s, this, "text/html");
if (op_ret < 0) {
#include "rgw_datalog_notify.h"
#include "include/random.h"
+struct RGWBucketEnt;
class RGWRESTMgr;
class RGWAccessListFilter;
class RGWLC;
int shard_id, RGWGetBucketStats_CB* ctx) = 0;
/** Sync this bucket's stats to the owning user's stats in the backing store */
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0;
- /** Refresh the metadata stats (size, count, and so on) from the backing store */
- virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) = 0;
/** Check if this bucket needs resharding, and schedule it if it does */
virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) = 0;
/** Change the owner of this bucket in the backing store. Current owner must be set. Does not
return 0;
}
- int DBBucket::update_container_stats(const DoutPrefixProvider *dpp, optional_yield y)
- {
- return 0;
- }
-
int DBBucket::check_bucket_shards(const DoutPrefixProvider *dpp, optional_yield y)
{
return 0;
bool *syncstopped = nullptr) override;
virtual int read_stats_async(const DoutPrefixProvider *dpp, const bucket_index_layout_generation& idx_layout, int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int check_bucket_shards(const DoutPrefixProvider *dpp, optional_yield y) override;
virtual int chown(const DoutPrefixProvider *dpp, User& new_user, optional_yield y) override;
virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime, optional_yield y) override;
return next->sync_user_stats(dpp, y);
}
-int FilterBucket::update_container_stats(const DoutPrefixProvider* dpp, optional_yield y)
-{
- return next->update_container_stats(dpp, y);
-}
-
int FilterBucket::check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y)
{
return next->check_bucket_shards(dpp, y);
const bucket_index_layout_generation& idx_layout,
int shard_id, RGWGetBucketStats_CB* ctx) override;
virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override;
- virtual int update_container_stats(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int check_bucket_shards(const DoutPrefixProvider* dpp, optional_yield y) override;
virtual int chown(const DoutPrefixProvider* dpp, User& new_user,
optional_yield y) override;