get usage now get capacity used by each bucket

Fixes:tracker.ceph.com/issues/16191
Signed-off-by: Ji Chen <insomnia@139.com>

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 4ec0fb7688118ac5c31db3274fbdaf5902751a30..2e4f8ad53f750537c89aef8344f8b2354726263c 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1674,14 +1674,20 @@ void RGWGetUsage::execute()
   op_ret = rgw_user_sync_all_stats(store, s->user->user_id);
   if (op_ret < 0) {
     ldout(store->ctx(), 0) << "ERROR: failed to sync user stats: " << dendl;
+    return;
+  }
+
+  op_ret = rgw_user_get_all_buckets_stats(store, s->user->user_id, buckets_usage);
+  if (op_ret < 0) {
+    cerr << "ERROR: failed to sync user stats: " << std::endl;
     return ;
   }
-  
+
   string user_str = s->user->user_id.to_str();
   op_ret = store->cls_user_get_header(user_str, &header);
   if (op_ret < 0) {
     ldout(store->ctx(), 0) << "ERROR: can't read user header: "  << dendl;
-    return ;
+    return;
   }
   
   return;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 11c88a37fa22e6f94da78b3e043bee36d0b2b16e..6462a84a97f5e9ec9e2078057e3c1acc2c8679d3 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -582,6 +582,7 @@ protected:
   map<string, bool> categories;
   map<rgw_user_bucket, rgw_usage_log_entry> usage;
   map<string, rgw_usage_log_entry> summary_map;
+  map<string, cls_user_bucket_entry> buckets_usage;
   cls_user_header header;
 public:
   RGWGetUsage() : sent_data(false), show_log_entries(true), show_log_sum(true){

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index b2d910e43a756b8042f18b853a4ce1f40ba1dd1b..ac35573a0dd5989f53b1703fed5bed27cdb3586c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -12466,6 +12466,38 @@ int RGWRados::cls_user_sync_bucket_stats(rgw_raw_obj& user_obj, const RGWBucketI
   return 0;
 }
 
+int RGWRados::cls_user_get_bucket_stats(rgw_bucket& bucket, cls_user_bucket_entry& entry)
+{
+  map<string, struct rgw_bucket_dir_header> headers;
+  RGWBucketInfo bucket_info;
+  RGWObjectCtx obj_ctx(this);
+  int ret = get_bucket_instance_info(obj_ctx, bucket, bucket_info, NULL, NULL);
+  if (ret < 0) {
+    return ret;
+  }
+
+  int r = cls_bucket_head(bucket_info, RGW_NO_SHARD, headers);
+  if (r < 0) {
+    ldout(cct, 20) << "cls_bucket_header() returned " << r << dendl;
+    return r;
+  }
+
+  bucket.convert(&entry.bucket);
+
+  map<string, struct rgw_bucket_dir_header>::iterator hiter = headers.begin();
+  for (; hiter != headers.end(); ++hiter) {
+    map<uint8_t, struct rgw_bucket_category_stats>::iterator iter = hiter->second.stats.begin();
+    for (; iter != hiter->second.stats.end(); ++iter) {
+      struct rgw_bucket_category_stats& header_stats = iter->second;
+      entry.size += header_stats.total_size;
+      entry.size_rounded += header_stats.total_size_rounded;
+      entry.count += header_stats.num_entries;
+    }
+  }
+
+  return 0;
+}
+
 int RGWRados::cls_user_list_buckets(rgw_raw_obj& obj,
                                     const string& in_marker,
                                     const string& end_marker,

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index c1ef8b80fa95ccea0683e1933aae1908aa8c26a1..ae20213439efdcdb418812078ead892fac70bd4e 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -3430,6 +3430,7 @@ public:
   int complete_sync_user_stats(const rgw_user& user_id);
   int cls_user_add_bucket(rgw_raw_obj& obj, list<cls_user_bucket_entry>& entries);
   int cls_user_remove_bucket(rgw_raw_obj& obj, const cls_user_bucket& bucket);
+  int cls_user_get_bucket_stats(rgw_bucket& bucket, cls_user_bucket_entry& entry);
 
   int check_quota(const rgw_user& bucket_owner, rgw_bucket& bucket,
                   RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size);

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 1c9e4d751b00e4e8f7990d762be1b1ae8f1b92db..d22d68c01abb0d7d2954a3adec83f05889ae03a0 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -404,6 +404,15 @@ static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log
   formatter->close_section(); // Category
 }
 
+static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry)
+{
+  formatter->open_object_section("Entry");
+  formatter->dump_string("Bucket", name);
+  formatter->dump_int("Bytes", entry.size);
+  formatter->dump_int("Bytes_Rounded", entry.size_rounded);
+  formatter->close_section(); // entry
+}
+
 void RGWGetUsage_ObjStore_S3::send_response()
 {
   if (op_ret < 0)
@@ -493,8 +502,21 @@ void RGWGetUsage_ObjStore_S3::send_response()
 
      formatter->close_section(); // summary
    }
-   formatter->close_section(); // usage
-   rgw_flush_formatter_and_reset(s, s->formatter);
+
+  formatter->open_array_section("CapacityUsed");
+  formatter->open_object_section("User");
+  formatter->open_array_section("Buckets");
+  map<string, cls_user_bucket_entry>::iterator biter;
+  for (biter = buckets_usage.begin(); biter != buckets_usage.end(); ++biter) {
+    const cls_user_bucket_entry& entry = biter->second;
+    dump_usage_bucket_info(formatter, biter->first, entry);
+  }
+  formatter->close_section(); // Buckets 
+  formatter->close_section(); // User 
+  formatter->close_section(); // CapacityUsed
+   
+  formatter->close_section(); // usage
+  rgw_flush_formatter_and_reset(s, s->formatter);
 }
 
 int RGWListBucket_ObjStore_S3::get_params()

diff --git a/src/rgw/rgw_usage.cc b/src/rgw/rgw_usage.cc
index 03a4ab4de0881f08da048d44e279e5458aa6400e..ef6365c5837f5dee8cdf40c1ca10a51679b61212 100644 (file)
--- a/src/rgw/rgw_usage.cc
+++ b/src/rgw/rgw_usage.cc
@@ -30,6 +30,15 @@ static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log
   formatter->close_section(); // categories
 }
 
+static void dump_usage_bucket_info(Formatter *formatter, string& name, cls_user_bucket_entry& entry)
+{
+  formatter->open_object_section("Entry");
+  formatter->dump_string("Bucket", name);
+  formatter->dump_int("Bytes", entry.size);
+  formatter->dump_int("Bytes_Rounded", entry.size_rounded);
+  formatter->close_section(); // entry
+}
+
 int RGWUsage::show(RGWRados *store, rgw_user& uid, uint64_t start_epoch,
                   uint64_t end_epoch, bool show_log_entries, bool show_log_sum,
                   map<string, bool> *categories,

diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc
index 2a2f3dd386abff1dc2bb4f1434c38893a5e85dcc..95bdab3a4a3e5fc52aeea58ee57e6bc069415158 100644 (file)
--- a/src/rgw/rgw_user.cc
+++ b/src/rgw/rgw_user.cc
@@ -89,6 +89,44 @@ int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id)
   return 0;
 }
 
+int rgw_user_get_all_buckets_stats(RGWRados *store, const rgw_user& user_id, map<string, cls_user_bucket_entry>&buckets_usage_map)
+{
+  CephContext *cct = store->ctx();
+  size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
+  bool done;
+  bool is_truncated;
+  string marker;
+  int ret;
+
+  do {
+    RGWUserBuckets user_buckets;
+    ret = rgw_read_user_buckets(store, user_id, user_buckets, marker,
+                               string(), max_entries, false, &is_truncated);
+    if (ret < 0) {
+      ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl;
+      return ret;
+    }
+    map<string, RGWBucketEnt>& buckets = user_buckets.get_buckets();
+    for (map<string, RGWBucketEnt>::iterator i = buckets.begin();
+         i != buckets.end();
+         ++i) {
+      marker = i->first;
+
+      RGWBucketEnt& bucket_ent = i->second;
+      cls_user_bucket_entry entry;
+      ret = store->cls_user_get_bucket_stats(bucket_ent.bucket, entry);
+      if (ret < 0) {
+        ldout(cct, 0) << "ERROR: could not get bucket stats: ret=" << ret << dendl;
+        return ret;
+      }
+      buckets_usage_map.insert(pair<string, cls_user_bucket_entry>(bucket_ent.bucket.name, entry));
+    }
+    done = (buckets.size() < max_entries);
+  } while (!done);
+
+  return 0;
+}
+
 /**
  * Save the given user information to storage.
  * Returns: 0 on success, -ERR# on failure.

diff --git a/src/rgw/rgw_user.h b/src/rgw/rgw_user.h
index 50cc1b62909f2d08f0ea9f5feff5a145f608678e..9c45148868964747ff8a3c3182b1406b2b9ddcad 100644 (file)
--- a/src/rgw/rgw_user.h
+++ b/src/rgw/rgw_user.h
@@ -53,6 +53,9 @@ extern int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id);
 /**
  * Get the anonymous (ie, unauthenticated) user info.
  */
+ 
+extern int rgw_user_get_all_buckets_stats(RGWRados *store, const rgw_user& user_id, map<string, cls_user_bucket_entry>&buckets_usage_map);
+
 extern void rgw_get_anon_user(RGWUserInfo& info);
 
 /**