From ddaac56c84263f51ea0b790e69fe663652cdb632 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 6 Sep 2012 13:17:09 -0700 Subject: [PATCH] rgw: usage now account categories Instead of just keeping a flat usage info per bucket, we now maintain a list of categories for which requests usage is aggregated in. Ops are put in categories based on their names. Signed-off-by: Yehuda Sadeh --- src/cls/rgw/cls_rgw_types.h | 78 ++++++++++++++++++++++++++++--------- src/rgw/rgw_admin.cc | 29 +++++++++----- src/rgw/rgw_log.cc | 16 +++++--- src/rgw/rgw_log.h | 2 +- src/rgw/rgw_main.cc | 2 +- 5 files changed, 91 insertions(+), 36 deletions(-) diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h index f587bc95bc63a..b6a8f399c7384 100644 --- a/src/cls/rgw/cls_rgw_types.h +++ b/src/cls/rgw/cls_rgw_types.h @@ -198,24 +198,17 @@ struct rgw_bucket_dir { }; WRITE_CLASS_ENCODER(rgw_bucket_dir) - -struct rgw_usage_log_entry { - string owner; - string bucket; - uint64_t epoch; +struct rgw_usage_data { uint64_t bytes_sent; uint64_t bytes_received; uint64_t ops; uint64_t successful_ops; - rgw_usage_log_entry() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {} - rgw_usage_log_entry(string& o, string& b, uint64_t s, uint64_t r) : owner(o), bucket(b), bytes_sent(s), bytes_received(r), ops(0), successful_ops(0) {} + rgw_usage_data() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {} + rgw_usage_data(uint64_t sent, uint64_t received) : bytes_sent(sent), bytes_received(received), ops(0), successful_ops(0) {} void encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); - ::encode(owner, bl); - ::encode(bucket, bl); - ::encode(epoch, bl); ::encode(bytes_sent, bl); ::encode(bytes_received, bl); ::encode(ops, bl); @@ -223,12 +216,8 @@ struct rgw_usage_log_entry { ENCODE_FINISH(bl); } - - void decode(bufferlist::iterator& bl) { + void decode(bufferlist::iterator& bl) { DECODE_START(1, bl); - ::decode(owner, bl); - ::decode(bucket, bl); - ::decode(epoch, bl); ::decode(bytes_sent, bl); ::decode(bytes_received, bl); ::decode(ops, bl); @@ -236,16 +225,67 @@ struct rgw_usage_log_entry { DECODE_FINISH(bl); } + void aggregate(const rgw_usage_data& usage) { + bytes_sent += usage.bytes_sent; + bytes_received += usage.bytes_received; + ops += usage.ops; + successful_ops += usage.successful_ops; + } +}; +WRITE_CLASS_ENCODER(rgw_usage_data) + + +struct rgw_usage_log_entry { + string owner; + string bucket; + uint64_t epoch; + map usage_map; + + rgw_usage_log_entry() {} + rgw_usage_log_entry(string& o, string& b) : owner(o), bucket(b) {} + + void encode(bufferlist& bl) const { + ENCODE_START(2, 2, bl); + ::encode(owner, bl); + ::encode(bucket, bl); + ::encode(epoch, bl); + ::encode(usage_map, bl); + ENCODE_FINISH(bl); + } + + + void decode(bufferlist::iterator& bl) { + DECODE_START(2, bl); + ::decode(owner, bl); + ::decode(bucket, bl); + ::decode(epoch, bl); + if (struct_v < 2) { + rgw_usage_data usage_data; + ::decode(usage_data.bytes_sent, bl); + ::decode(usage_data.bytes_received, bl); + ::decode(usage_data.ops, bl); + ::decode(usage_data.successful_ops, bl); + usage_map[""] = usage_data; + } else { + ::decode(usage_map, bl); + } + DECODE_FINISH(bl); + } + void aggregate(const rgw_usage_log_entry& e) { if (owner.empty()) { owner = e.owner; bucket = e.bucket; epoch = e.epoch; } - bytes_sent += e.bytes_sent; - bytes_received += e.bytes_received; - ops += e.ops; - successful_ops += e.successful_ops; + map::const_iterator iter; + for (iter = e.usage_map.begin(); iter != e.usage_map.end(); ++iter) { + add(iter->first, iter->second); + } + } + + void add(const string& category, const rgw_usage_data& data) { + usage_map[category].aggregate(data); } }; WRITE_CLASS_ENCODER(rgw_usage_log_entry) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index c347fd1846e2c..062ffade8681f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -646,6 +646,23 @@ static int remove_bucket(rgw_bucket& bucket, bool delete_children) return ret; } +void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry) +{ + formatter->open_array_section("categories"); + map::const_iterator uiter; + for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) { + const rgw_usage_data& usage = uiter->second; + formatter->open_object_section("entry"); + formatter->dump_string("category", uiter->first); + formatter->dump_int("bytes_sent", usage.bytes_sent); + formatter->dump_int("bytes_received", usage.bytes_received); + formatter->dump_int("ops", usage.ops); + formatter->dump_int("successful_ops", usage.successful_ops); + formatter->close_section(); // entry + } + formatter->close_section(); // categories +} + int main(int argc, char **argv) { vector args; @@ -1574,10 +1591,7 @@ next: utime_t ut(entry.epoch, 0); ut.gmtime(formatter->dump_stream("time")); formatter->dump_int("epoch", entry.epoch); - formatter->dump_int("bytes_sent", entry.bytes_sent); - formatter->dump_int("bytes_received", entry.bytes_received); - formatter->dump_int("ops", entry.ops); - formatter->dump_int("successful_ops", entry.successful_ops); + dump_usage_categories_info(formatter, entry); formatter->close_section(); // bucket formatter->flush(cout); } @@ -1600,11 +1614,8 @@ next: const rgw_usage_log_entry& entry = siter->second; formatter->open_object_section("user"); formatter->dump_string("user", siter->first); - formatter->dump_int("bytes_sent", entry.bytes_sent); - formatter->dump_int("bytes_received", entry.bytes_received); - formatter->dump_int("ops", entry.ops); - formatter->dump_int("successful_ops", entry.successful_ops); - formatter->close_section(); + dump_usage_categories_info(formatter, entry); + formatter->close_section(); // user formatter->flush(cout); } diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index 851427aeafe76..bae74a9d12730 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -162,7 +162,7 @@ void rgw_log_usage_finalize() usage_logger = NULL; } -static void log_usage(struct req_state *s) +static void log_usage(struct req_state *s, const string& op_name) { if (!usage_logger) return; @@ -174,24 +174,28 @@ static void log_usage(struct req_state *s) else user = s->user.user_id; - rgw_usage_log_entry entry(user, s->bucket.name, s->bytes_sent, s->bytes_received); + rgw_usage_log_entry entry(user, s->bucket.name); - entry.ops = 1; + rgw_usage_data data(s->bytes_sent, s->bytes_received); + + data.ops = 1; if (!s->err.is_err()) - entry.successful_ops = 1; + data.successful_ops = 1; + + entry.add(op_name, data); utime_t ts = ceph_clock_now(s->cct); usage_logger->insert(ts, entry); } -int rgw_log_op(struct req_state *s) +int rgw_log_op(struct req_state *s, const string& op_name) { struct rgw_log_entry entry; string bucket_id; if (s->enable_usage_log) - log_usage(s); + log_usage(s, op_name); if (!s->enable_ops_log) return 0; diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 0738809bce099..a653fbf1e3d84 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -117,7 +117,7 @@ struct rgw_intent_log_entry { }; WRITE_CLASS_ENCODER(rgw_intent_log_entry) -int rgw_log_op(struct req_state *s); +int rgw_log_op(struct req_state *s, const string& op_name); int rgw_log_intent(rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc); int rgw_log_intent(struct req_state *s, rgw_obj& obj, RGWIntentEvent intent); void rgw_log_usage_init(CephContext *cct); diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index fdcb381d77e16..93bae66fe2e96 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -318,7 +318,7 @@ void RGWProcess::handle_request(RGWRequest *req) req->log(s, "executing"); op->execute(); done: - rgw_log_op(s); + rgw_log_op(s, (op ? op->name() : "unknown")); int http_ret = s->err.http_ret; -- 2.39.5