rgw/dynamic-resharding: reshard bucket using the same instance id

Signed-off-by: Shilpa Jagannath <smanjara@redhat.com>

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 77b6305feebdd3581065656ad57e51ee5af46c62..8bcd76e3087524747b6ba7d019ba82ab17bf9336 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -2724,7 +2724,7 @@ int check_reshard_bucket_params(rgw::sal::RadosStore* store,
     return ret;
   }
 
-  if ((*bucket)->get_info().reshard_status != cls_rgw_reshard_status::NOT_RESHARDING) {
+  if ((*bucket)->get_info().layout.resharding != rgw::BucketReshardState::None) {
     // if in_progress or done then we have an old BucketInfo
     cerr << "ERROR: the bucket is currently undergoing resharding and "
       "cannot be added to the reshard list at this time" << std::endl;
@@ -6805,7 +6805,8 @@ next:
       max_entries = 1000;
     }
 
-    int max_shards = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0 ? bucket->get_info().layout.current_index.layout.normal.num_shards : 1);
+    const auto& index = bucket->get_info().layout.current_index;
+    int max_shards = index.layout.normal.num_shards;
 
     formatter->open_array_section("entries");
 
@@ -6813,8 +6814,7 @@ next:
     for (; i < max_shards; i++) {
       RGWRados::BucketShard bs(static_cast<rgw::sal::RadosStore*>(store)->getRados());
       int shard_id = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0  ? i : -1);
-
-      int ret = bs.init(bucket->get_key(), shard_id, bucket->get_info().layout.current_index, nullptr /* no RGWBucketInfo */, dpp());
+      int ret = bs.init(bucket->get_key(), shard_id, index, nullptr /* no RGWBucketInfo */, dpp());
       marker.clear();
 
       if (ret < 0) {
@@ -6872,9 +6872,23 @@ next:
       return EINVAL;
     }
 
-    ret = bucket->purge_instance(dpp());
-    if (ret < 0) {
-      return -ret;
+    const auto& index = bucket->get_info().layout.current_index;
+    int max_shards = index.layout.normal.num_shards;
+
+    for (int i = 0; i < max_shards; i++) {
+      RGWRados::BucketShard bs(static_cast<rgw::sal::RadosStore*>(store)->getRados());
+      int shard_id = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0  ? i : -1);
+      int ret = bs.init(bucket->get_key(), shard_id, index, nullptr /* no RGWBucketInfo */, dpp());
+      if (ret < 0) {
+        cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << i << "): " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
+
+      ret = static_cast<rgw::sal::RadosStore*>(store)->getRados()->bi_remove(dpp(), bs);
+      if (ret < 0) {
+        cerr << "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret) << std::endl;
+        return -ret;
+      }
     }
   }
 

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index 5de37cc1a418c09955a4faa0d2e66273aee13dcf..6813c3ffd07ad4b97c6c5cef426df2bc46a05554 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -1331,12 +1331,24 @@ int RGWBucketAdminOp::set_quota(rgw::sal::Store* store, RGWBucketAdminOpState& o
 
 static int purge_bucket_instance(rgw::sal::Store* store, const RGWBucketInfo& bucket_info, const DoutPrefixProvider *dpp)
 {
-  std::unique_ptr<rgw::sal::Bucket> bucket;
-  int ret = store->get_bucket(nullptr, bucket_info, &bucket);
-  if (ret < 0)
-    return ret;
-
-  return bucket->purge_instance(dpp);
+  const auto& index = bucket_info.layout.current_index;
+  int max_shards = index.layout.normal.num_shards;
+  for (int i = 0; i < max_shards; i++) {
+    RGWRados::BucketShard bs(static_cast<rgw::sal::RadosStore*>(store)->getRados());
+    int ret = bs.init(bucket_info.bucket, i, bucket_info.layout.current_index, nullptr, dpp);
+    if (ret < 0) {
+      cerr << "ERROR: bs.init(bucket=" << bucket_info.bucket << ", shard=" << i
+           << "): " << cpp_strerror(-ret) << std::endl;
+      return ret;
+    }
+    ret = static_cast<rgw::sal::RadosStore*>(store)->getRados()->bi_remove(dpp, bs);
+    if (ret < 0) {
+      cerr << "ERROR: failed to remove bucket index object: "
+           << cpp_strerror(-ret) << std::endl;
+      return ret;
+    }
+  }
+  return 0;
 }
 
 inline auto split_tenant(const std::string& bucket_name){

diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc
index 3d19f4366fb8310ee49688c55c81521350b944c6..4c514186815b6ba5afcbf821cc5a138db2dabc22 100644 (file)
--- a/src/rgw/rgw_orphan.cc
+++ b/src/rgw/rgw_orphan.cc
@@ -520,7 +520,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp,
     return 0;
   }
 
-  if (cur_bucket->get_info().reshard_status == cls_rgw_reshard_status::IN_PROGRESS) {
+  if (cur_bucket->get_info().layout.resharding != rgw::BucketReshardState::None) {
     ldpp_dout(dpp, 0) << __func__ << ": reshard in progress. Skipping "
                            << orphan_bucket.name << ": "
                            << orphan_bucket.bucket_id << dendl;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 30e338c8a396e9fe5065b7fada0b0ef6b3f7be09..8ca9b74daad0d247b9d88dba49f03e22ded5e20e 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -2747,12 +2747,12 @@ int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketIn
   return 0;
 }
 
-int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid)
+int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& target_layout, int sid)
 {
   bucket = bucket_info.bucket;
   shard_id = sid;
 
-  int ret = store->svc.bi_rados->open_bucket_index_shard(dpp, bucket_info, shard_id, idx_layout, &bucket_obj);
+  int ret = store->svc.bi_rados->open_bucket_index_shard(dpp, bucket_info, shard_id, target_layout, &bucket_obj);
   if (ret < 0) {
     ldpp_dout(dpp, 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl;
     return ret;
@@ -8397,13 +8397,12 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp,
    * few results, perhaps due to filtering or to a series of
    * namespaced entries */
 
-  ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": " <<
-    bucket_info.bucket <<
-    " start_after=\"" << start_after <<
-    "\", prefix=\"" << prefix <<
-    ", delimiter=\"" << delimiter <<
-    "\", shard_id=" << shard_id <<
-    "\", num_entries=" << num_entries <<
+  ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": " << bucket_info.bucket <<
+    " start_after=\"" << start_after.name <<
+    "[" << start_after.instance <<
+    "]\", prefix=\"" << prefix <<
+    "\" num_entries=" << num_entries <<
+    ", shard_id=" << shard_id <<
     ", list_versions=" << list_versions <<
     ", expansion_factor=" << expansion_factor <<
     ", force_check_filter is " <<
@@ -9078,7 +9077,7 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo
   RGWSI_RADOS::Pool index_pool;
   map<int, string> oids;
   map<int, struct rgw_cls_list_ret> list_results;
-  int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, -1, idx_layout, &index_pool, &oids, bucket_instance_ids);
+  int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, bucket_instance_ids);
   if (r < 0) {
     ldpp_dout(dpp, 20) << "cls_bucket_head: open_bucket_index() returned "
                    << r << dendl;
@@ -9086,6 +9085,7 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo
   }
 
   r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)();
+  ldout(cct, 20) << "CLSRGWIssueGetDirHeader issued" << dendl;
   if (r < 0) {
     ldpp_dout(dpp, 20) << "cls_bucket_head: CLSRGWIssueGetDirHeader() returned "
                    << r << dendl;

diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc
index 1c9bc453d6ab820dcb5c335546e3d0aae8720e29..715a37e7f78874d10fb3d44658b2f7f418389d3a 100644 (file)
--- a/src/rgw/rgw_reshard.cc
+++ b/src/rgw/rgw_reshard.cc
@@ -64,7 +64,7 @@ class BucketReshardShard {
   rgw::sal::RadosStore* store;
   const RGWBucketInfo& bucket_info;
   int num_shard;
-  const rgw::bucket_index_layout_generation& idx_layout;
+  const rgw::bucket_index_layout_generation& target_layout;
   RGWRados::BucketShard bs;
   vector<rgw_cls_bi_entry> entries;
   map<RGWObjCategory, rgw_bucket_category_stats> stats;
@@ -105,15 +105,15 @@ class BucketReshardShard {
 
 public:
   BucketReshardShard(const DoutPrefixProvider *dpp, 
-                     rgw::sal::RadosStore* _store, const RGWBucketInfo& _bucket_info,
-                     int _num_shard, const rgw::bucket_index_layout_generation& _idx_layout,
+                     rgw::sal::RadosStore *_store, const RGWBucketInfo& _bucket_info,
+                     int _num_shard, const rgw::bucket_index_layout_generation& _target_layout,
                      deque<librados::AioCompletion *>& _completions) :
-    store(_store), bucket_info(_bucket_info), idx_layout(_idx_layout), bs(store->getRados()),
+    store(_store), bucket_info(_bucket_info), target_layout(_target_layout), bs(store->getRados()),
     aio_completions(_completions)
   {
-    num_shard = (idx_layout.layout.normal.num_shards > 0 ? _num_shard : -1);
+    num_shard = (target_layout.layout.normal.num_shards > 0 ? _num_shard : -1);
 
-    bs.init(bucket_info.bucket, num_shard, idx_layout, nullptr /* no RGWBucketInfo */, dpp);
+    bs.init(bucket_info.bucket, num_shard, target_layout, nullptr /* no RGWBucketInfo */, dpp);
 
     max_aio_completions =
       store->ctx()->_conf.get_val<uint64_t>("rgw_reshard_max_aio");
@@ -199,10 +199,10 @@ public:
     store(_store), target_bucket_info(_target_bucket_info),
     num_target_shards(_num_target_shards)
   { 
-    const auto& idx_layout = target_bucket_info.layout.current_index;
+    const auto& target_layout = *target_bucket_info.layout.target_index;
     target_shards.resize(num_target_shards);
     for (int i = 0; i < num_target_shards; ++i) {
-      target_shards[i] = new BucketReshardShard(dpp, store, target_bucket_info, i, idx_layout, completions);
+      target_shards[i] = new BucketReshardShard(dpp, store, target_bucket_info, i, target_layout, completions);
     }
   }
 
@@ -327,43 +327,29 @@ int RGWBucketReshard::clear_index_shard_reshard_status(const DoutPrefixProvider
   return 0;
 }
 
-static int create_new_bucket_instance(rgw::sal::RadosStore* store,
+static int update_num_shards(rgw::sal::RadosStore *store,
                                      int new_num_shards,
                                      RGWBucketInfo& bucket_info,
                                      map<string, bufferlist>& attrs,
-                                     RGWBucketInfo& new_bucket_info,
                                       const DoutPrefixProvider *dpp)
 {
-  new_bucket_info = bucket_info;
 
-  store->getRados()->create_bucket_id(&new_bucket_info.bucket.bucket_id);
+  bucket_info.layout.target_index->layout.normal.num_shards = new_num_shards;
 
   bucket_info.layout.resharding = rgw::BucketReshardState::None;
 
-  new_bucket_info.new_bucket_instance_id.clear();
-  new_bucket_info.reshard_status = cls_rgw_reshard_status::NOT_RESHARDING;
-
-  int ret = store->svc()->bi->init_index(dpp, new_bucket_info);
+  int ret = static_cast<rgw::sal::RadosStore*>(store)->getRados()->put_bucket_instance_info(bucket_info, true, real_time(), &attrs, dpp);
   if (ret < 0) {
-    cerr << "ERROR: failed to init new bucket indexes: " << cpp_strerror(-ret) << std::endl;
-    return ret;
-  }
-
-  ret = store->getRados()->put_bucket_instance_info(new_bucket_info, true, real_time(), &attrs, dpp);
-  if (ret < 0) {
-    cerr << "ERROR: failed to store new bucket instance info: " << cpp_strerror(-ret) << std::endl;
+    cerr << "ERROR: failed to store updated bucket instance info: " << cpp_strerror(-ret) << std::endl;
     return ret;
   }
 
   return 0;
 }
 
-int RGWBucketReshard::create_new_bucket_instance(int new_num_shards,
-                                                 RGWBucketInfo& new_bucket_info,
-                                                 const DoutPrefixProvider *dpp)
+int RGWBucketReshard::update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp)
 {
-  return ::create_new_bucket_instance(store, new_num_shards,
-                                     bucket_info, bucket_attrs, new_bucket_info, dpp);
+  return ::update_num_shards(store, new_num_shards, bucket_info, bucket_attrs, dpp);
 }
 
 int RGWBucketReshard::cancel(const DoutPrefixProvider *dpp)
@@ -418,7 +404,6 @@ public:
        ldpp_dout(dpp, -1) << "Error: " << __func__ <<
          " clear_index_shard_status returned " << ret << dendl;
       }
-      bucket_info.new_bucket_instance_id.clear();
 
       // clears new_bucket_instance as well
       set_status(rgw::BucketReshardState::None, dpp);
@@ -563,6 +548,9 @@ int RGWBucketReshard::do_reshard(int num_shards,
     return ret;
   }
 
+  //increment generation number
+  bucket_info.layout.target_index->gen++;
+
   int num_target_shards = bucket_info.layout.target_index->layout.normal.num_shards;
 
   BucketReshardManager target_shards_mgr(dpp, store, bucket_info, num_target_shards);
@@ -579,8 +567,9 @@ int RGWBucketReshard::do_reshard(int num_shards,
     (*out) << "total entries:";
   }
 
+  auto current_shards = bucket_info.layout.current_index.layout.normal.num_shards;
   const int num_source_shards =
-    (bucket_info.layout.current_index.layout.normal.num_shards > 0 ? bucket_info.layout.current_index.layout.normal.num_shards : 1);
+    (current_shards > 0 ? current_shards : 1);
   string marker;
   for (int i = 0; i < num_source_shards; ++i) {
     bool is_truncated = true;
@@ -589,9 +578,13 @@ int RGWBucketReshard::do_reshard(int num_shards,
     while (is_truncated) {
       entries.clear();
       ret = store->getRados()->bi_list(dpp, bucket_info, i, null_object_filter, marker, max_entries, &entries, &is_truncated);
-      if (ret < 0 && ret != -ENOENT) {
-       derr << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl;
-       return ret;
+      if (ret < 0) {
+        if (ret == -ENOENT && i < (num_source_shards - 1)) {
+          continue;
+        } else {
+          derr << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl;
+          return ret;
+        }
       }
 
       for (auto iter = entries.begin(); iter != entries.end(); ++iter) {
@@ -701,8 +694,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries,
     return ret;
   }
 
-  RGWBucketInfo new_bucket_info;
-  ret = create_new_bucket_instance(num_shards, new_bucket_info, dpp);
+  ret = update_num_shards(num_shards, dpp);
   if (ret < 0) {
     return ret;
     goto error_out;

diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h
index 09600de81359958e38f0f80e7eeabf785d725ed4..a33af8f3783925edaa1e3f6af91a0a8729628d70 100644 (file)
--- a/src/rgw/rgw_reshard.h
+++ b/src/rgw/rgw_reshard.h
@@ -84,9 +84,7 @@ private:
   // allocated in at once
   static const std::initializer_list<uint16_t> reshard_primes;
 
-  int create_new_bucket_instance(int new_num_shards,
-                                RGWBucketInfo& new_bucket_info,
-                                 const DoutPrefixProvider *dpp);
+  int update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp);
   int do_reshard(int num_shards,
                 int max_entries,
                  bool verbose,
@@ -122,14 +120,15 @@ public:
   static int set_resharding_status(const DoutPrefixProvider *dpp,
                                    rgw::sal::RadosStore* store,
                                   const RGWBucketInfo& bucket_info,
-                                  const std::string& new_instance_id,
+                                  const std::string& instance_id,
                                   int32_t num_shards,
                                   cls_rgw_reshard_status status);
-  int set_resharding_status(const DoutPrefixProvider *dpp, const std::string& new_instance_id,
+  int set_resharding_status(const DoutPrefixProvider *dpp,
+                           const std::string& instance_id,
                            int32_t num_shards,
-                           cls_rgw_reshard_status status) {
+                            cls_rgw_reshard_status status) {
     return set_resharding_status(dpp, store, bucket_info,
-                                new_instance_id, num_shards, status);
+                                instance_id, num_shards, status);
   }
 
   static uint32_t get_max_prime_shards() {
@@ -222,8 +221,8 @@ protected:
     void stop();
 
     CephContext *get_cct() const override;
-    unsigned get_subsys() const;
-    std::ostream& gen_prefix(std::ostream& out) const;
+    unsigned get_subsys() const override;
+    std::ostream& gen_prefix(std::ostream& out) const override;
   };
 
   ReshardWorker *worker = nullptr;