rgw: bi log list/trim can get specific bucket shard

bucket shard can be specified on the bucket instance param. It can be
added like this: <bucket-instance>[:shard-id]

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 9cf4a734c33eeacfe9b76b021f9193af29da7a32..d0f9f5843dc7b34777e3a6d40dea667aabd60c5f 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2350,7 +2350,7 @@ next:
 
     do {
       list<rgw_bi_log_entry> entries;
-      ret = store->list_bi_log_entries(bucket, marker, max_entries - count, entries, &truncated);
+      ret = store->list_bi_log_entries(bucket, shard_id, marker, max_entries - count, entries, &truncated);
       if (ret < 0) {
         cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
         return -ret;
@@ -2382,7 +2382,7 @@ next:
       cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
       return -ret;
     }
-    ret = store->trim_bi_log_entries(bucket, start_marker, end_marker);
+    ret = store->trim_bi_log_entries(bucket, shard_id, start_marker, end_marker);
     if (ret < 0) {
       cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
       return -ret;

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 83e451378b4a7d12afe265fb76aa75d1c60035da..b8c2778b0ec5c0d6ddd0b86c680d4f184b3dfa16 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -233,6 +233,32 @@ int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersi
   return store->meta_mgr->remove_entry(bucket_instance_meta_handler, entry, objv_tracker);
 }
 
+int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *target_bucket_instance, int *shard_id)
+{
+  ssize_t pos = bucket_instance.rfind(':');
+  if (pos < 0) {
+    return -EINVAL;
+  }
+
+  string first = bucket_instance.substr(0, pos);
+  string second = bucket_instance.substr(pos + 1);
+
+  if (first.find(':') == string::npos) {
+    *shard_id = -1;
+    *target_bucket_instance = bucket_instance;
+    return 0;
+  }
+
+  *target_bucket_instance = first;
+  string err;
+  *shard_id = strict_strtol(second.c_str(), 10, &err);
+  if (!err.empty()) {
+    return -EINVAL;
+  }
+
+  return 0;
+}
+
 int rgw_bucket_set_attrs(RGWRados *store, RGWBucketInfo& bucket_info,
                          map<string, bufferlist>& attrs,
                          map<string, bufferlist>* rmattrs,

diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h
index 3bdd68c057d0d0256bbc361422d99d9a3041a4f0..d53da2f9ba63e485e1b14980726e3f997bfe253c 100644 (file)
--- a/src/rgw/rgw_bucket.h
+++ b/src/rgw/rgw_bucket.h
@@ -32,6 +32,8 @@ extern int rgw_bucket_instance_store_info(RGWRados *store, string& oid, bufferli
                                  map<string, bufferlist> *pattrs, RGWObjVersionTracker *objv_tracker,
                                  time_t mtime);
 
+extern int rgw_bucket_parse_bucket_instance(const string& bucket_instance, string *target_bucket_instance, int *shard_id);
+
 extern int rgw_bucket_instance_remove_entry(RGWRados *store, string& entry, RGWObjVersionTracker *objv_tracker);
 
 extern int rgw_bucket_delete_bucket_obj(RGWRados *store, string& bucket_name, RGWObjVersionTracker& objv_tracker);

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index a485954e2d53f9e56dd448860e955cb62de677c4..6cf4aaa9e85b252a6bc3085e7d002d6a3f497587 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3808,7 +3808,7 @@ int RGWRados::open_bucket_index_base(rgw_bucket& bucket, librados::IoCtx& index_
 }
 
 int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
-    vector<string>& bucket_objs) {
+    vector<string>& bucket_objs, int shard_id) {
   string bucket_oid_base;
   int ret = open_bucket_index_base(bucket, index_ctx, bucket_oid_base);
   if (ret < 0)
@@ -3820,16 +3820,16 @@ int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
   if (ret < 0)
     return ret;
 
-  get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs);
+  get_bucket_index_objects(bucket_oid_base, binfo.num_shards, bucket_objs, shard_id);
   return 0;
 }
 
 template<typename T>
 int RGWRados::open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
-    map<string, T>& bucket_objs)
+    map<string, T>& bucket_objs, int shard_id)
 {
   vector<string> oids;
-  int ret = open_bucket_index(bucket, index_ctx, oids);
+  int ret = open_bucket_index(bucket, index_ctx, oids, shard_id);
   if (ret < 0)
     return ret;
 
@@ -6101,7 +6101,7 @@ int RGWRados::list_raw_objects(rgw_bucket& pool, const string& prefix_filter,
   return oids.size();
 }
 
-int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max,
+int RGWRados::list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max,
                                   std::list<rgw_bi_log_entry>& result, bool *truncated)
 {
   ldout(cct, 20) << __func__ << bucket << " marker " << marker << " max " << max << dendl;
@@ -6109,12 +6109,12 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t m
 
   librados::IoCtx index_ctx;
   map<string, cls_rgw_bi_log_list_ret> bi_log_lists;
-  int r = open_bucket_index(bucket, index_ctx, bi_log_lists);
+  int r = open_bucket_index(bucket, index_ctx, bi_log_lists, shard_id);
   if (r < 0)
     return r;
 
   BucketIndexShardsManager marker_mgr;
-  bool has_shards = (bi_log_lists.size() > 1);
+  bool has_shards = (bi_log_lists.size() > 1 || shard_id >= 0);
   // If there are multiple shards for the bucket index object, the marker
   // should have the pattern '{shard_oid_1}#{shard_marker_1},{shard_oid_2}#
   // {shard_marker_2}...', if there is no sharding, the bi_log_list should
@@ -6183,15 +6183,15 @@ int RGWRados::list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t m
   return 0;
 }
 
-int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, string& start_marker, string& end_marker)
+int RGWRados::trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& start_marker, string& end_marker)
 {
   librados::IoCtx index_ctx;
   vector<string> bucket_objs;
-  int r = open_bucket_index(bucket, index_ctx, bucket_objs);
+  int r = open_bucket_index(bucket, index_ctx, bucket_objs, shard_id);
   if (r < 0)
     return r;
 
-  bool has_shards = bucket_objs.size() > 1;
+  bool has_shards = bucket_objs.size() > 1 || shard_id >= 0;
   BucketIndexShardsManager start_marker_mgr;
   r = start_marker_mgr.from_string(start_marker, has_shards, bucket_objs.front());
   if (r < 0)
@@ -6950,14 +6950,22 @@ int RGWRados::remove_temp_objects(string date, string time)
 }
 
 void RGWRados::get_bucket_index_objects(const string& bucket_oid_base,
-    const uint32_t num_shards, vector<string>& bucket_objects)
+    uint32_t num_shards, vector<string>& bucket_objects, int shard_id)
 {
   if (!num_shards) {
     bucket_objects.push_back(bucket_oid_base);
   } else {
     char buf[bucket_oid_base.size() + 32];
-    for (uint32_t i = 0; i < num_shards; ++i) {
-      snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i);
+    if (shard_id < 0) {
+      for (uint32_t i = 0; i < num_shards; ++i) {
+        snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i);
+        bucket_objects.push_back(string(buf));
+      }
+    } else {
+      if ((uint32_t)shard_id > num_shards) {
+        return;
+      }
+      snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id);
       bucket_objects.push_back(string(buf));
     }
   }

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 819f494925bd3e9b93ff9edb1be6d33949258059..d8f6c168a0f8bcb95f0cfe6bd0d25c51adc2fdf4 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1257,10 +1257,10 @@ class RGWRados
   int open_bucket_index_shard(rgw_bucket& bucket, librados::IoCtx& index_ctx,
       const string& obj_key, string *bucket_obj);
   int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
-      vector<string>& bucket_objs);
+      vector<string>& bucket_objs, int shard_id = -1);
   template<typename T>
   int open_bucket_index(rgw_bucket& bucket, librados::IoCtx& index_ctx,
-      map<string, T>& bucket_objs);
+      map<string, T>& bucket_objs, int shard_id = -1);
   void build_bucket_index_marker(const string& shard_name, const string& shard_marker,
       string *marker);
 
@@ -1874,8 +1874,8 @@ public:
 
     return cls_obj_complete_cancel(bucket, tag, oid.object, oid.get_hash_object());
   }
-  int list_bi_log_entries(rgw_bucket& bucket, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
-  int trim_bi_log_entries(rgw_bucket& bucket, string& marker, string& end_marker);
+  int list_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, uint32_t max, std::list<rgw_bi_log_entry>& result, bool *truncated);
+  int trim_bi_log_entries(rgw_bucket& bucket, int shard_id, string& marker, string& end_marker);
 
   int cls_obj_usage_log_add(const string& oid, rgw_usage_log_info& info);
   int cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
@@ -1960,8 +1960,8 @@ public:
    * num_shards [in] - number of bucket index object shards.
    * bucket_objs [out] - filled by this method, a list of bucket index objects.
    */
-  void get_bucket_index_objects(const string& bucket_oid_base, const uint32_t num_shards,
-      vector<string>& bucket_objs);
+  void get_bucket_index_objects(const string& bucket_oid_base, uint32_t num_shards,
+      vector<string>& bucket_objs, int shard_id = -1);
 
   /**
    * Get the bucket index object with the given base bucket index object and object key,

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 9f32fc9ebd3f96a0ded857ab4b14da3f47fdea54..1db6cadb0d8e0335288d75d4161d16cf4be41239 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -282,6 +282,12 @@ void RGWOp_BILog_List::execute() {
     return;
   }
 
+  int shard_id;
+  http_ret = rgw_bucket_parse_bucket_instance(bucket_instance, &bucket_instance, &shard_id);
+  if (http_ret < 0) {
+    return;
+  }
+
   if (!bucket_instance.empty()) {
     http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
     if (http_ret < 0) {
@@ -307,7 +313,7 @@ void RGWOp_BILog_List::execute() {
   send_response();
   do {
     list<rgw_bi_log_entry> entries;
-    int ret = store->list_bi_log_entries(bucket_info.bucket,
+    int ret = store->list_bi_log_entries(bucket_info.bucket, shard_id,
                                           marker, max_entries - count, 
                                           entries, &truncated);
     if (ret < 0) {
@@ -419,6 +425,13 @@ void RGWOp_BILog_Delete::execute() {
     http_ret = -EINVAL;
     return;
   }
+
+  int shard_id;
+  http_ret = rgw_bucket_parse_bucket_instance(bucket_instance, &bucket_instance, &shard_id);
+  if (http_ret < 0) {
+    return;
+  }
+
   if (!bucket_instance.empty()) {
     http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
     if (http_ret < 0) {
@@ -432,7 +445,7 @@ void RGWOp_BILog_Delete::execute() {
       return;
     }
   }
-  http_ret = store->trim_bi_log_entries(bucket_info.bucket, start_marker, end_marker);
+  http_ret = store->trim_bi_log_entries(bucket_info.bucket, shard_id, start_marker, end_marker);
   if (http_ret < 0) {
     dout(5) << "ERROR: trim_bi_log_entries() " << dendl;
   }