rgw: usage now account categories

Instead of just keeping a flat usage info per bucket, we
now maintain a list of categories for which requests
usage is aggregated in. Ops are put in categories based
on their names.

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index f587bc95bc63a76617846fb47010fbaa6906944d..b6a8f399c7384283f4f82ac70b2f5d22f70421d8 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -198,24 +198,17 @@ struct rgw_bucket_dir {
 };
 WRITE_CLASS_ENCODER(rgw_bucket_dir)
 
-
-struct rgw_usage_log_entry {
-  string owner;
-  string bucket;
-  uint64_t epoch;
+struct rgw_usage_data {
   uint64_t bytes_sent;
   uint64_t bytes_received;
   uint64_t ops;
   uint64_t successful_ops;
 
-  rgw_usage_log_entry() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {}
-  rgw_usage_log_entry(string& o, string& b, uint64_t s, uint64_t r) : owner(o), bucket(b), bytes_sent(s), bytes_received(r), ops(0), successful_ops(0) {}
+  rgw_usage_data() : bytes_sent(0), bytes_received(0), ops(0), successful_ops(0) {}
+  rgw_usage_data(uint64_t sent, uint64_t received) : bytes_sent(sent), bytes_received(received), ops(0), successful_ops(0) {}
 
   void encode(bufferlist& bl) const {
     ENCODE_START(1, 1, bl);
-    ::encode(owner, bl);
-    ::encode(bucket, bl);
-    ::encode(epoch, bl);
     ::encode(bytes_sent, bl);
     ::encode(bytes_received, bl);
     ::encode(ops, bl);
@@ -223,12 +216,8 @@ struct rgw_usage_log_entry {
     ENCODE_FINISH(bl);
   }
 
-
-   void decode(bufferlist::iterator& bl) {
+  void decode(bufferlist::iterator& bl) {
     DECODE_START(1, bl);
-    ::decode(owner, bl);
-    ::decode(bucket, bl);
-    ::decode(epoch, bl);
     ::decode(bytes_sent, bl);
     ::decode(bytes_received, bl);
     ::decode(ops, bl);
@@ -236,16 +225,67 @@ struct rgw_usage_log_entry {
     DECODE_FINISH(bl);
   }
 
+  void aggregate(const rgw_usage_data& usage) {
+    bytes_sent += usage.bytes_sent;
+    bytes_received += usage.bytes_received;
+    ops += usage.ops;
+    successful_ops += usage.successful_ops;
+  }
+};
+WRITE_CLASS_ENCODER(rgw_usage_data)
+
+
+struct rgw_usage_log_entry {
+  string owner;
+  string bucket;
+  uint64_t epoch;
+  map<string, rgw_usage_data> usage_map;
+
+  rgw_usage_log_entry() {}
+  rgw_usage_log_entry(string& o, string& b) : owner(o), bucket(b) {}
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(2, 2, bl);
+    ::encode(owner, bl);
+    ::encode(bucket, bl);
+    ::encode(epoch, bl);
+    ::encode(usage_map, bl);
+    ENCODE_FINISH(bl);
+  }
+
+
+   void decode(bufferlist::iterator& bl) {
+    DECODE_START(2, bl);
+    ::decode(owner, bl);
+    ::decode(bucket, bl);
+    ::decode(epoch, bl);
+    if (struct_v < 2) {
+      rgw_usage_data usage_data;
+      ::decode(usage_data.bytes_sent, bl);
+      ::decode(usage_data.bytes_received, bl);
+      ::decode(usage_data.ops, bl);
+      ::decode(usage_data.successful_ops, bl);
+      usage_map[""] = usage_data;
+    } else {
+      ::decode(usage_map, bl);
+    }
+    DECODE_FINISH(bl);
+  }
+
   void aggregate(const rgw_usage_log_entry& e) {
     if (owner.empty()) {
       owner = e.owner;
       bucket = e.bucket;
       epoch = e.epoch;
     }
-    bytes_sent += e.bytes_sent;
-    bytes_received += e.bytes_received;
-    ops += e.ops;
-    successful_ops += e.successful_ops;
+    map<string, rgw_usage_data>::const_iterator iter;
+    for (iter = e.usage_map.begin(); iter != e.usage_map.end(); ++iter) {
+      add(iter->first, iter->second);
+    }
+  }
+
+  void add(const string& category, const rgw_usage_data& data) {
+    usage_map[category].aggregate(data);
   }
 };
 WRITE_CLASS_ENCODER(rgw_usage_log_entry)

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index c347fd1846e2ca2b731f9e9479a6abf33f308180..062ffade8681f1801a02e8765646fecf12ac6799 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -646,6 +646,23 @@ static int remove_bucket(rgw_bucket& bucket, bool delete_children)
   return ret;
 }
 
+void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log_entry& entry)
+{
+  formatter->open_array_section("categories");
+  map<string, rgw_usage_data>::const_iterator uiter;
+  for (uiter = entry.usage_map.begin(); uiter != entry.usage_map.end(); ++uiter) {
+    const rgw_usage_data& usage = uiter->second;
+    formatter->open_object_section("entry");
+    formatter->dump_string("category", uiter->first);
+    formatter->dump_int("bytes_sent", usage.bytes_sent);
+    formatter->dump_int("bytes_received", usage.bytes_received);
+    formatter->dump_int("ops", usage.ops);
+    formatter->dump_int("successful_ops", usage.successful_ops);
+    formatter->close_section(); // entry
+  }
+  formatter->close_section(); // categories
+}
+
 int main(int argc, char **argv) 
 {
   vector<const char*> args;
@@ -1574,10 +1591,7 @@ next:
           utime_t ut(entry.epoch, 0);
           ut.gmtime(formatter->dump_stream("time"));
           formatter->dump_int("epoch", entry.epoch);
-          formatter->dump_int("bytes_sent", entry.bytes_sent);
-          formatter->dump_int("bytes_received", entry.bytes_received);
-          formatter->dump_int("ops", entry.ops);
-          formatter->dump_int("successful_ops", entry.successful_ops);
+         dump_usage_categories_info(formatter, entry);
           formatter->close_section(); // bucket
           formatter->flush(cout);
         }
@@ -1600,11 +1614,8 @@ next:
         const rgw_usage_log_entry& entry = siter->second;
         formatter->open_object_section("user");
         formatter->dump_string("user", siter->first);
-        formatter->dump_int("bytes_sent", entry.bytes_sent);
-        formatter->dump_int("bytes_received", entry.bytes_received);
-        formatter->dump_int("ops", entry.ops);
-        formatter->dump_int("successful_ops", entry.successful_ops);
-        formatter->close_section();
+       dump_usage_categories_info(formatter, entry);
+        formatter->close_section(); // user
         formatter->flush(cout);
       }
 

diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc
index 851427aeafe76c5f3b214d9709774fc2a169206a..bae74a9d127305718456df6b035114471229abaa 100644 (file)
--- a/src/rgw/rgw_log.cc
+++ b/src/rgw/rgw_log.cc
@@ -162,7 +162,7 @@ void rgw_log_usage_finalize()
   usage_logger = NULL;
 }
 
-static void log_usage(struct req_state *s)
+static void log_usage(struct req_state *s, const string& op_name)
 {
   if (!usage_logger)
     return;
@@ -174,24 +174,28 @@ static void log_usage(struct req_state *s)
   else
     user = s->user.user_id;
 
-  rgw_usage_log_entry entry(user, s->bucket.name, s->bytes_sent, s->bytes_received);
+  rgw_usage_log_entry entry(user, s->bucket.name);
 
-  entry.ops = 1;
+  rgw_usage_data data(s->bytes_sent, s->bytes_received);
+
+  data.ops = 1;
   if (!s->err.is_err())
-    entry.successful_ops = 1;
+    data.successful_ops = 1;
+
+  entry.add(op_name, data);
 
   utime_t ts = ceph_clock_now(s->cct);
 
   usage_logger->insert(ts, entry);
 }
 
-int rgw_log_op(struct req_state *s)
+int rgw_log_op(struct req_state *s, const string& op_name)
 {
   struct rgw_log_entry entry;
   string bucket_id;
 
   if (s->enable_usage_log)
-    log_usage(s);
+    log_usage(s, op_name);
 
   if (!s->enable_ops_log)
     return 0;

diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h
index 0738809bce0994890810c560470a72d4370ff15e..a653fbf1e3d84cea16b3c03d08a3111b041cff3d 100644 (file)
--- a/src/rgw/rgw_log.h
+++ b/src/rgw/rgw_log.h
@@ -117,7 +117,7 @@ struct rgw_intent_log_entry {
 };
 WRITE_CLASS_ENCODER(rgw_intent_log_entry)
 
-int rgw_log_op(struct req_state *s);
+int rgw_log_op(struct req_state *s, const string& op_name);
 int rgw_log_intent(rgw_obj& obj, RGWIntentEvent intent, const utime_t& timestamp, bool utc);
 int rgw_log_intent(struct req_state *s, rgw_obj& obj, RGWIntentEvent intent);
 void rgw_log_usage_init(CephContext *cct);

diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc
index fdcb381d77e1617ad13cae00df291f1bca69b4c3..93bae66fe2e963821b2a7c1c598d7d48bdc28eaf 100644 (file)
--- a/src/rgw/rgw_main.cc
+++ b/src/rgw/rgw_main.cc
@@ -318,7 +318,7 @@ void RGWProcess::handle_request(RGWRequest *req)
   req->log(s, "executing");
   op->execute();
 done:
-  rgw_log_op(s);
+  rgw_log_op(s, (op ? op->name() : "unknown"));
 
   int http_ret = s->err.http_ret;