rgw: system user can list bucket shard

By setting the HTTP-Rgwx-Shard-Id header to the shard id.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 8b78c4d2cdfafde441e4b38fc8f9f912d5b381d6..545daf1926d594eb9d2fa10349b560a9b47cda38 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -3312,7 +3312,7 @@ next:
     formatter->open_array_section("objects");
     while (is_truncated) {
       map<string, RGWObjEnt> result;
-      int r = store->cls_bucket_list(bucket, marker, prefix, 1000, true,
+      int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
                                      result, &is_truncated, &marker,
                                      bucket_object_check_filter);
 

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 4247102872ba67bc99f1ff5ea09db27e88385feb..8959182365913a5a2db9201b54a696fc3cd42b47 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -849,7 +849,7 @@ int RGWBucket::check_object_index(RGWBucketAdminOpState& op_state,
   while (is_truncated) {
     map<string, RGWObjEnt> result;
 
-    int r = store->cls_bucket_list(bucket, marker, prefix, 1000, true,
+    int r = store->cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, 1000, true,
                                    result, &is_truncated, &marker,
                                    bucket_object_check_filter);
     if (r == -ENOENT) {

diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc
index 6c815e1c2cf55f6fa5c377fbc2d691896de645fa..bc6b8d6a6a315950a33ef69c8acb3c141ee2e11e 100644 (file)
--- a/src/rgw/rgw_op.cc
+++ b/src/rgw/rgw_op.cc
@@ -1611,6 +1611,9 @@ void RGWListBucket::execute()
   }
 
   RGWRados::Bucket target(store, s->bucket);
+  if (shard_id >= 0) {
+    target.set_shard_id(shard_id);
+  }
   RGWRados::Bucket::List list_op(&target);
 
   list_op.params.prefix = prefix;

diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h
index 254fbd4f5f70aa242b245ebca9d8cce3b3ad3598..3a60eabbc1c11d284ff17629f4b0ccd11be1b392 100644 (file)
--- a/src/rgw/rgw_op.h
+++ b/src/rgw/rgw_op.h
@@ -377,11 +377,13 @@ protected:
   int default_max;
   bool is_truncated;
 
+  int shard_id;
+
   int parse_max_keys();
 
 public:
   RGWListBucket() : list_versions(false), max(0),
-                    default_max(0), is_truncated(false) {}
+                    default_max(0), is_truncated(false), shard_id(-1) {}
   int verify_permission();
   void pre_exec();
   void execute();

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 4dab52a2ef687f3f95ab75a2e69b5f6146e73062..f2c190ff44c3d867fb424f7bf0b5755da020458c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3621,6 +3621,7 @@ int RGWRados::Bucket::List::list_objects(int max, vector<RGWObjEnt> *result,
   RGWRados *store = target->get_store();
   CephContext *cct = store->ctx();
   rgw_bucket& bucket = target->get_bucket();
+  int shard_id = target->get_shard_id();
 
   int count = 0;
   bool truncated = true;
@@ -3680,7 +3681,7 @@ int RGWRados::Bucket::List::list_objects(int max, vector<RGWObjEnt> *result,
       ldout(cct, 20) << "setting cur_marker=" << cur_marker.name << "[" << cur_marker.instance << "]" << dendl;
     }
     std::map<string, RGWObjEnt> ent_map;
-    int r = store->cls_bucket_list(bucket, cur_marker, cur_prefix, max + 1 - count, params.list_versions, ent_map,
+    int r = store->cls_bucket_list(bucket, shard_id, cur_marker, cur_prefix, max + 1 - count, params.list_versions, ent_map,
                             &truncated, &cur_marker);
     if (r < 0)
       return r;
@@ -5614,7 +5615,7 @@ int RGWRados::delete_bucket(rgw_bucket& bucket, RGWObjVersionTracker& objv_track
 
   do {
 #define NUM_ENTRIES 1000
-    r = cls_bucket_list(bucket, marker, prefix, NUM_ENTRIES, true, ent_map,
+    r = cls_bucket_list(bucket, RGW_NO_SHARD, marker, prefix, NUM_ENTRIES, true, ent_map,
                         &is_truncated, &marker);
     if (r < 0)
       return r;
@@ -9490,7 +9491,7 @@ int RGWRados::cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeou
   return CLSRGWIssueSetTagTimeout(index_ctx, bucket_objs, cct->_conf->rgw_bucket_index_max_aio, timeout)();
 }
 
-int RGWRados::cls_bucket_list(rgw_bucket& bucket, rgw_obj_key& start, const string& prefix,
+int RGWRados::cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& start, const string& prefix,
                              uint32_t num_entries, bool list_versions, map<string, RGWObjEnt>& m,
                              bool *is_truncated, rgw_obj_key *last_entry,
                              bool (*force_check_filter)(const string&  name))
@@ -9502,7 +9503,7 @@ int RGWRados::cls_bucket_list(rgw_bucket& bucket, rgw_obj_key& start, const stri
   // value - list result for the corresponding oid (shard), it is filled by the AIO callback
   map<int, string> oids;
   map<int, struct rgw_cls_list_ret> list_results;
-  int r = open_bucket_index(bucket, index_ctx, oids);
+  int r = open_bucket_index(bucket, index_ctx, oids, shard_id);
   if (r < 0)
     return r;
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 42aa0d47348945b3e6eefa14d7cc54fb6ef482b9..b600b980e4117bccb2a9b2875b8e0b6222409ef2 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2032,13 +2032,19 @@ public:
   class Bucket {
     RGWRados *store;
     rgw_bucket& bucket;
+    int shard_id;
 
   public:
-    Bucket(RGWRados *_store, rgw_bucket& _bucket) : store(_store), bucket(_bucket) {}
+    Bucket(RGWRados *_store, rgw_bucket& _bucket) : store(_store), bucket(_bucket), shard_id(RGW_NO_SHARD) {}
 
     RGWRados *get_store() { return store; }
     rgw_bucket& get_bucket() { return bucket; }
 
+    int get_shard_id() { return shard_id; }
+    void set_shard_id(int id) {
+      shard_id = id;
+    }
+
     class UpdateIndex {
       RGWRados::Bucket *target;
       string optag;
@@ -2446,7 +2452,7 @@ public:
                            list<rgw_obj_key> *remove_objs, uint16_t bilog_flags);
   int cls_obj_complete_cancel(BucketShard& bs, string& tag, rgw_obj& obj, uint16_t bilog_flags);
   int cls_obj_set_bucket_tag_timeout(rgw_bucket& bucket, uint64_t timeout);
-  int cls_bucket_list(rgw_bucket& bucket, rgw_obj_key& start, const string& prefix,
+  int cls_bucket_list(rgw_bucket& bucket, int shard_id, rgw_obj_key& start, const string& prefix,
                       uint32_t num_entries, bool list_versions, map<string, RGWObjEnt>& m,
                       bool *is_truncated, rgw_obj_key *last_entry,
                       bool (*force_check_filter)(const string&  name) = NULL);

diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc
index 12663e084783d211576f7f739455298145bc02a6..ef5b66779503ca469f1d281b94973e5ac93ae380 100644 (file)
--- a/src/rgw/rgw_rest_s3.cc
+++ b/src/rgw/rgw_rest_s3.cc
@@ -278,6 +278,17 @@ int RGWListBucket_ObjStore_S3::get_params()
   }
   delimiter = s->info.args.get("delimiter");
   encoding_type = s->info.args.get("encoding-type");
+  if (s->system_request) {
+    const char *shard_id_str = s->info.env->get("HTTP_RGWX_SHARD_ID");
+    if (shard_id_str) {
+      string err;
+      shard_id = strict_strtol(shard_id_str, 10, &err);
+      if (!err.empty()) {
+        ldout(s->cct, 5) << "bad shard id specified: " << shard_id_str << dendl;
+        return -EINVAL;
+      }
+    }
+  }
   return 0;
 }