From d48bebeed74e584268b81d169e3c8f69d79d7005 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 9 Aug 2011 17:12:23 -0700 Subject: [PATCH] librados, rgw: integrate stats categories --- src/include/rados/librados.hpp | 7 +++- src/librados.cc | 65 ++++++++++++++++++++++++++-------- src/osd/ReplicatedPG.cc | 14 ++++---- src/rados.cc | 51 +++++++++++++++++--------- src/rgw/rgw_access.h | 19 ++++++---- src/rgw/rgw_common.cc | 5 +++ src/rgw/rgw_common.h | 5 +++ src/rgw/rgw_fs.cc | 4 ++- src/rgw/rgw_fs.h | 3 +- src/rgw/rgw_op.cc | 14 ++++---- src/rgw/rgw_rados.cc | 58 +++++++++++++++++++----------- src/rgw/rgw_rados.h | 13 ++++--- 12 files changed, 181 insertions(+), 77 deletions(-) diff --git a/src/include/rados/librados.hpp b/src/include/rados/librados.hpp index f1decdfce0670..40573a1a63f0c 100644 --- a/src/include/rados/librados.hpp +++ b/src/include/rados/librados.hpp @@ -46,6 +46,8 @@ namespace librados uint64_t num_rd, num_rd_kb, num_wr, num_wr_kb; }; + typedef std::map stats_map; + typedef void *completion_t; typedef void (*callback_t)(completion_t cb, void *arg); @@ -353,7 +355,10 @@ namespace librados /* listing objects */ int pool_list(std::list& v); int get_pool_stats(std::list& v, - std::map& stats); + std::map& stats); + int get_pool_stats(std::list& v, + std::string& category, + std::map& stats); int cluster_stat(cluster_stat_t& result); /* pool aio */ diff --git a/src/librados.cc b/src/librados.cc index d761f803e06e4..b19a4f0718160 100644 --- a/src/librados.cc +++ b/src/librados.cc @@ -3268,26 +3268,63 @@ ioctx_create(const char *name, IoCtx &io) } int librados::Rados:: -get_pool_stats(std::list& v, std::map& result) +get_pool_stats(std::list& v, std::map& result) +{ + string category; + return get_pool_stats(v, category, result); +} + +int librados::Rados:: +get_pool_stats(std::list& v, string& category, std::map& result) { map rawresult; int r = client->get_pool_stats(v, rawresult); for (map::iterator p = rawresult.begin(); p != rawresult.end(); p++) { - pool_stat_t& v = result[p->first]; - v.num_kb = p->second.stats.sum.num_kb; - v.num_bytes = p->second.stats.sum.num_bytes; - v.num_objects = p->second.stats.sum.num_objects; - v.num_object_clones = p->second.stats.sum.num_object_clones; - v.num_object_copies = p->second.stats.sum.num_object_copies; - v.num_objects_missing_on_primary = p->second.stats.sum.num_objects_missing_on_primary; - v.num_objects_unfound = p->second.stats.sum.num_objects_unfound; - v.num_objects_degraded = p->second.stats.sum.num_objects_degraded; - v.num_rd = p->second.stats.sum.num_rd; - v.num_rd_kb = p->second.stats.sum.num_rd_kb; - v.num_wr = p->second.stats.sum.num_wr; - v.num_wr_kb = p->second.stats.sum.num_wr_kb; + stats_map& c = result[p->first]; + + string cat; + vector cats; + + if (!category.size()) { + cats.push_back(cat); + map::iterator iter; + for (iter = p->second.stats.cat_sum.begin(); iter != p->second.stats.cat_sum.end(); ++iter) { + cats.push_back(iter->first); + } + } else { + cats.push_back(category); + } + + vector::iterator cat_iter; + for (cat_iter = cats.begin(); cat_iter != cats.end(); ++cat_iter) { + string& cur_category = *cat_iter; + object_stat_sum_t *sum; + + if (!cur_category.size()) { + sum = &p->second.stats.sum; + } else { + map::iterator iter = p->second.stats.cat_sum.find(cur_category); + if (iter == p->second.stats.cat_sum.end()) + continue; + sum = &iter->second; + } + + pool_stat_t& v = c[cur_category]; + v.num_kb = sum->num_kb; + v.num_bytes = sum->num_bytes; + v.num_objects = sum->num_objects; + v.num_object_clones = sum->num_object_clones; + v.num_object_copies = sum->num_object_copies; + v.num_objects_missing_on_primary = sum->num_objects_missing_on_primary; + v.num_objects_unfound = sum->num_objects_unfound; + v.num_objects_degraded = sum->num_objects_degraded; + v.num_rd = sum->num_rd; + v.num_rd_kb = sum->num_rd_kb; + v.num_wr = sum->num_wr; + v.num_wr_kb = sum->num_wr_kb; + } } return r; } diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 6a2c0599c04d9..0a929e2798125 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -1592,12 +1592,14 @@ int ReplicatedPG::do_osd_ops(OpContext *ctx, vector& ops, bufferlist::iterator p = osd_op.data.begin(); string category; ::decode(category, p); - if (obs.exists) { - if (obs.oi.category != category) - result = -EEXIST; - } else { - obs.oi.category = category; - } + if (category.size()) { + if (obs.exists) { + if (obs.oi.category != category) + result = -EEXIST; + } else { + obs.oi.category = category; + } + } } } break; diff --git a/src/rados.cc b/src/rados.cc index a5368a76de356..db19c1732193b 100644 --- a/src/rados.cc +++ b/src/rados.cc @@ -561,6 +561,7 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, const char *snapname = NULL; snap_t snapid = CEPH_NOSNAP; std::map::const_iterator i; + std::string category; uint64_t min_obj_len = 0; uint64_t max_obj_len = 0; @@ -580,6 +581,10 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, if (i != opts.end()) { pool_name = i->second.c_str(); } + i = opts.find("category"); + if (i != opts.end()) { + category = i->second; + } i = opts.find("concurrent-ios"); if (i != opts.end()) { concurrent_ios = strtol(i->second.c_str(), NULL, 10); @@ -705,29 +710,41 @@ static int rados_tool_common(const std::map < std::string, std::string > &opts, else if (strcmp(nargs[0], "df") == 0) { // pools list vec; - rados.pool_list(vec); - map stats; - rados.get_pool_stats(vec, stats); + if (!pool_name) + rados.pool_list(vec); + else + vec.push_back(pool_name); + + map > stats; + rados.get_pool_stats(vec, category, stats); - printf("%-15s " + printf("%-15s %-15s" "%12s %12s %12s %12s " "%12s %12s %12s %12s %12s\n", "pool name", + "category", "KB", "objects", "clones", "degraded", "unfound", "rd", "rd KB", "wr", "wr KB"); - for (map::iterator i = stats.begin(); i != stats.end(); ++i) { - printf("%-15s " - "%12lld %12lld %12lld %12lld" - "%12lld %12lld %12lld %12lld %12lld\n", - i->first.c_str(), - (long long)i->second.num_kb, - (long long)i->second.num_objects, - (long long)i->second.num_object_clones, - (long long)i->second.num_objects_degraded, - (long long)i->second.num_objects_unfound, - (long long)i->second.num_rd, (long long)i->second.num_rd_kb, - (long long)i->second.num_wr, (long long)i->second.num_wr_kb); + for (map::iterator c = stats.begin(); c != stats.end(); ++c) { + const string& pool_name = c->first; + stats_map& m = c->second; + for (stats_map::iterator i = m.begin(); i != m.end(); ++i) { + const char *category = (i->first.size() ? i->first.c_str() : "-"); + printf("%-15s " + "%-15s " + "%12lld %12lld %12lld %12lld" + "%12lld %12lld %12lld %12lld %12lld\n", + pool_name.c_str(), + category, + (long long)i->second.num_kb, + (long long)i->second.num_objects, + (long long)i->second.num_object_clones, + (long long)i->second.num_objects_degraded, + (long long)i->second.num_objects_unfound, + (long long)i->second.num_rd, (long long)i->second.num_rd_kb, + (long long)i->second.num_wr, (long long)i->second.num_wr_kb); + } } // total @@ -1170,6 +1187,8 @@ int main(int argc, const char **argv) opts["create"] = "true"; } else if (ceph_argparse_witharg(args, i, &val, "-p", "--pool", (char*)NULL)) { opts["pool"] = val; + } else if (ceph_argparse_witharg(args, i, &val, "--category", (char*)NULL)) { + opts["category"] = val; } else if (ceph_argparse_witharg(args, i, &val, "-t", "--concurrent-ios", (char*)NULL)) { opts["concurrent-ios"] = val; } else if (ceph_argparse_witharg(args, i, &val, "--block-size", (char*)NULL)) { diff --git a/src/rgw/rgw_access.h b/src/rgw/rgw_access.h index 0c9b517dbd862..7b78a91131d74 100644 --- a/src/rgw/rgw_access.h +++ b/src/rgw/rgw_access.h @@ -61,17 +61,20 @@ public: /** write an object to the storage device in the appropriate pool with the given stats */ virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, - map& attrs, bool exclusive) = 0; + map& attrs, string& category, bool exclusive) = 0; virtual int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len) = 0; virtual int aio_put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len, void **handle) { return -ENOTSUP; } + + /* note that put_obj doesn't set category on an object, only use it for none user objects */ int put_obj(void *ctx, std::string& id, rgw_obj& obj, const char *data, size_t len, time_t *mtime, map& attrs) { int ret = put_obj_data(ctx, id, obj, data, -1, len); if (ret >= 0) { - ret = put_obj_meta(ctx, id, obj, mtime, attrs, false); + string category; + ret = put_obj_meta(ctx, id, obj, mtime, attrs, category, false); } return ret; } @@ -103,7 +106,8 @@ public: const time_t *unmod_ptr, const char *if_match, const char *if_nomatch, - map& attrs, + map& attrs, + string& category, struct rgw_err *err) = 0; /** * Delete a bucket. @@ -174,7 +178,8 @@ public: virtual int clone_obj(void *ctx, rgw_obj& dst_obj, off_t dst_ofs, rgw_obj& src_obj, off_t src_ofs, uint64_t size, time_t *pmtime, - map attrs) { + map attrs, + string& category) { RGWCloneRangeInfo info; vector v; info.src = src_obj; @@ -182,12 +187,14 @@ public: info.dst_ofs = dst_ofs; info.len = size; v.push_back(info); - return clone_objs(ctx, dst_obj, v, attrs, pmtime, true); + return clone_objs(ctx, dst_obj, v, attrs, category, pmtime, true); } virtual int clone_objs(void *ctx, rgw_obj& dst_obj, vector& ranges, - map attrs, time_t *pmtime, + map attrs, + string& category, + time_t *pmtime, bool truncate_dest) { return -ENOTSUP; } /** * a simple object read without keeping state diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 4574f4a7b734d..1947ec315cdc3 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -14,6 +14,11 @@ using namespace ceph::crypto; +string rgw_obj_category_main = "rgw.main"; +string rgw_obj_category_shadow = "rgw.shadow"; +string rgw_obj_category_multimeta = "rgw.multimeta"; +string rgw_obj_category_none; + rgw_err:: rgw_err() { diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 9ce90f40da4bd..850f4857d26bd 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -36,6 +36,11 @@ using ceph::crypto::MD5; extern string rgw_root_bucket; +extern string rgw_obj_category_main; +extern string rgw_obj_category_shadow; +extern string rgw_obj_category_multimeta; +extern string rgw_obj_category_none; + #define RGW_ROOT_BUCKET ".rgw" #define RGW_CONTROL_BUCKET ".rgw.control" diff --git a/src/rgw/rgw_fs.cc b/src/rgw/rgw_fs.cc index d584f990b9afe..2ba7249843df3 100644 --- a/src/rgw/rgw_fs.cc +++ b/src/rgw/rgw_fs.cc @@ -204,7 +204,8 @@ int RGWFS::create_bucket(std::string& id, std::string& bucket, map& attrs, bool exclusive) + time_t *mtime, map& attrs, + string& category, bool exclusive) { std::string& bucket = obj.bucket; std::string& oid = obj.object; @@ -299,6 +300,7 @@ int RGWFS::copy_obj(void *ctx, std::string& id, rgw_obj& dest_obj, const char *if_match, const char *if_nomatch, map& attrs, + string& category, struct rgw_err *err) { int ret; diff --git a/src/rgw/rgw_fs.h b/src/rgw/rgw_fs.h index 945e263a09276..95a1525c1f145 100644 --- a/src/rgw/rgw_fs.h +++ b/src/rgw/rgw_fs.h @@ -20,7 +20,7 @@ public: int create_bucket(std::string& id, std::string& bucket, map& attrs, bool exclusive, uint64_t auid=0); int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, - map& attrs, bool exclusive); + map& attrs, string& category, bool exclusive); int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t size); int copy_obj(void *ctx, std::string& id, rgw_obj& dest_obj, @@ -31,6 +31,7 @@ public: const char *if_match, const char *if_nomatch, map& attrs, + string& category, struct rgw_err *err); int delete_bucket(std::string& id, std::string& bucket); int delete_obj(void *ctx, std::string& id, rgw_obj& obj, bool sync); diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index e7dbb806efa59..3c3c6e57130e9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -693,7 +693,7 @@ void RGWPutObj::execute() if (!multipart) { rgw_obj dst_obj(s->bucket_str, s->object_str); rgwstore->set_atomic(s->obj_ctx, dst_obj); - ret = rgwstore->clone_obj(s->obj_ctx, dst_obj, 0, obj, 0, s->obj_size, NULL, attrs); + ret = rgwstore->clone_obj(s->obj_ctx, dst_obj, 0, obj, 0, s->obj_size, NULL, attrs, rgw_obj_category_main); if (ret < 0) goto done_err; if (created_obj) { @@ -702,7 +702,7 @@ void RGWPutObj::execute() goto done; } } else { - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, false); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, rgw_obj_category_main, false); if (ret < 0) goto done_err; @@ -720,7 +720,7 @@ void RGWPutObj::execute() rgw_obj meta_obj(s->bucket_str, multipart_meta_obj, s->object_str, mp_ns); - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, meta_obj, NULL, meta_attrs, false); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, meta_obj, NULL, meta_attrs, rgw_obj_category_multimeta, false); } } done: @@ -866,7 +866,7 @@ void RGWCopyObj::execute() unmod_ptr, if_match, if_nomatch, - attrs, &s->err); + attrs, rgw_obj_category_main, &s->err); done: send_response(); @@ -1130,7 +1130,7 @@ void RGWInitMultipart::execute() tmp_obj_name = mp.get_meta(); obj.init(s->bucket_str, tmp_obj_name, s->object_str, mp_ns); - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, true); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, obj, NULL, attrs, rgw_obj_category_multimeta, true); } while (ret == -EEXIST); done: send_response(); @@ -1271,7 +1271,7 @@ void RGWCompleteMultipart::execute() target_obj.init(s->bucket_str, s->object_str); rgwstore->set_atomic(s->obj_ctx, target_obj); - ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, target_obj, NULL, attrs, false); + ret = rgwstore->put_obj_meta(s->obj_ctx, s->user.user_id, target_obj, NULL, attrs, rgw_obj_category_main, false); if (ret < 0) goto done; @@ -1288,7 +1288,7 @@ void RGWCompleteMultipart::execute() ofs += obj_iter->second.size; } - ret = rgwstore->clone_objs(s->obj_ctx, target_obj, ranges, attrs, NULL, true); + ret = rgwstore->clone_objs(s->obj_ctx, target_obj, ranges, attrs, rgw_obj_category_main, NULL, true); if (ret < 0) goto done; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 574d2295d2b9b..4adee9b442f6d 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -22,6 +22,9 @@ Rados *rados = NULL; static string notify_oid = "notify"; static string shadow_ns = "shadow"; +static string shadow_category = "rgw.shadow"; +static string main_category = "rgw.main"; + class RGWWatcher : public librados::WatchCtx { RGWRados *rados; public: @@ -336,7 +339,7 @@ int RGWRados::create_bucket(std::string& id, std::string& bucket, map& attrs, bool exclusive) + time_t *mtime, map& attrs, string& category, bool exclusive) { std::string& bucket = obj.bucket; std::string& oid = obj.object; @@ -350,9 +353,10 @@ int RGWRados::put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, ObjectWriteOperation op; - if (exclusive) { - op.create(true); - } + if (category.size()) + op.create(exclusive, category); + else + op.create(exclusive); map::iterator iter; for (iter = attrs.begin(); iter != attrs.end(); ++iter) { @@ -465,7 +469,8 @@ bool RGWRados::aio_completed(void *handle) * err: stores any errors resulting from the get of the original object * Returns: 0 on success, -ERR# otherwise. */ -int RGWRados::copy_obj(void *ctx, std::string& id, rgw_obj& dest_obj, +int RGWRados::copy_obj(void *ctx, std::string& id, + rgw_obj& dest_obj, rgw_obj& src_obj, time_t *mtime, const time_t *mod_ptr, @@ -473,6 +478,7 @@ int RGWRados::copy_obj(void *ctx, std::string& id, rgw_obj& dest_obj, const char *if_match, const char *if_nomatch, map& attrs, /* in/out */ + string& category, struct rgw_err *err) { int ret, r; @@ -522,7 +528,7 @@ int RGWRados::copy_obj(void *ctx, std::string& id, rgw_obj& dest_obj, } attrs = attrset; - ret = clone_obj(ctx, dest_obj, 0, tmp_obj, 0, end + 1, NULL, attrs); + ret = clone_obj(ctx, dest_obj, 0, tmp_obj, 0, end + 1, NULL, attrs, category); if (mtime) obj_stat(ctx, tmp_obj, NULL, mtime); @@ -855,7 +861,7 @@ int RGWRados::prepare_atomic_for_write(RGWRadosCtx *rctx, rgw_obj& obj, librados /* FIXME: clone obj should be conditional, should check src object id-tag */ pair cond(RGW_ATTR_ID_TAG, state->obj_tag); - r = clone_obj_cond(NULL, dest_obj, 0, obj, 0, state->size, state->attrset, &state->mtime, &cond); + r = clone_obj_cond(NULL, dest_obj, 0, obj, 0, state->size, state->attrset, shadow_category, &state->mtime, &cond); if (r == -ECANCELED) { /* we lost in a race here, original object was replaced, we assume it was cloned as required */ @@ -1097,6 +1103,7 @@ done_err: int RGWRados::clone_objs(void *ctx, rgw_obj& dst_obj, vector& ranges, map attrs, + string& category, time_t *pmtime, bool truncate_dest, pair *xattr_cond) @@ -1112,9 +1119,15 @@ int RGWRados::clone_objs(void *ctx, rgw_obj& dst_obj, io_ctx.locator_set_key(dst_obj.key); ObjectWriteOperation op; - op.create(false); + + if (category.size()) + op.create(false, category); + else + op.create(false); + if (truncate_dest) op.truncate(0); + map::iterator iter; for (iter = attrs.begin(); iter != attrs.end(); ++iter) { const string& name = iter->first; @@ -1373,23 +1386,26 @@ int RGWRados::update_containers_stats(map& m) string bucket_name = iter->first; buckets_list.push_back(bucket_name); } - map stats; - int r = rados->get_pool_stats(buckets_list, stats); + map sm; + int r = rados->get_pool_stats(buckets_list, rgw_obj_category_main, sm); if (r < 0) return r; - map::iterator stats_iter = stats.begin(); + map::iterator miter; - for (iter = m.begin(); iter != m.end(); ++iter) { - string bucket_name = iter->first; - if (stats_iter->first.compare(bucket_name) == 0) { - RGWBucketEnt& ent = iter->second; - pool_stat_t stat = stats_iter->second; - ent.count = stat.num_objects; - ent.size = stat.num_bytes; - stats_iter++; - count++; - } + for (miter = sm.begin(), iter = m.begin(); miter != sm.end(), iter != m.end(); ++iter, ++miter) { + stats_map stats = miter->second; + stats_map::iterator stats_iter = stats.begin(); + + string bucket_name = miter->first; + RGWBucketEnt& ent = iter->second; + if (bucket_name.compare(ent.name) != 0) + continue; + + pool_stat_t stat = stats_iter->second; + ent.count = stat.num_objects; + ent.size = stat.num_bytes; + count++; } return count; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 3463aa75def61..8e794ff9e6e67 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -103,7 +103,7 @@ public: /** Write/overwrite an object to the bucket storage. */ virtual int put_obj_meta(void *ctx, std::string& id, rgw_obj& obj, time_t *mtime, - map& attrs, bool exclusive); + map& attrs, string& category, bool exclusive); virtual int put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, off_t ofs, size_t len); virtual int aio_put_obj_data(void *ctx, std::string& id, rgw_obj& obj, const char *data, @@ -112,13 +112,16 @@ public: virtual bool aio_completed(void *handle); virtual int clone_objs(void *ctx, rgw_obj& dst_obj, vector& ranges, - map attrs, time_t *pmtime, bool truncate_dest) { - return clone_objs(ctx, dst_obj, ranges, attrs, pmtime, truncate_dest, NULL); + map attrs, + string& category, + time_t *pmtime, bool truncate_dest) { + return clone_objs(ctx, dst_obj, ranges, attrs, category, pmtime, truncate_dest, NULL); } int clone_objs(void *ctx, rgw_obj& dst_obj, vector& ranges, map attrs, + string& category, time_t *pmtime, bool truncate_dest, pair *cmp_xattr); @@ -126,6 +129,7 @@ public: int clone_obj_cond(void *ctx, rgw_obj& dst_obj, off_t dst_ofs, rgw_obj& src_obj, off_t src_ofs, uint64_t size, map attrs, + string& category, time_t *pmtime, pair *xattr_cond) { RGWCloneRangeInfo info; @@ -135,7 +139,7 @@ public: info.dst_ofs = dst_ofs; info.len = size; v.push_back(info); - return clone_objs(ctx, dst_obj, v, attrs, pmtime, true, xattr_cond); + return clone_objs(ctx, dst_obj, v, attrs, category, pmtime, true, xattr_cond); } /** Copy an object, with many extra options */ @@ -147,6 +151,7 @@ public: const char *if_match, const char *if_nomatch, map& attrs, + string& category, struct rgw_err *err); /** delete a bucket*/ virtual int delete_bucket(std::string& id, std::string& bucket); -- 2.39.5