rgw: fix update_bucket_id, reshard blocking logic

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index 163b1886391e1a1d21d056cba537531c58e12035..fb78c4a0c9a06025413633febac3741133e33c54 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -1028,6 +1028,11 @@ struct rgw_bucket {
     DECODE_FINISH(bl);
   }
 
+  void update_bucket_id(const string& new_bucket_id) {
+    bucket_id = new_bucket_id;
+    oid.clear();
+  }
+
   // format a key for the bucket/instance. pass delim=0 to skip a field
   std::string get_key(char tenant_delim = '/',
                       char id_delim = ':') const;

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 364da290efac552c3d96233d0c8445d7a6f28bbb..441fddd2946d248212dd5f413af6bb96e938132b 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -5182,6 +5182,21 @@ int rgw_policy_from_attrset(CephContext *cct, map<string, bufferlist>& attrset,
 }
 
 
+int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
+{
+  rgw_bucket bucket = bucket_info.bucket;
+  bucket.update_bucket_id(new_bucket_id);
+
+  RGWObjectCtx obj_ctx(store);
+
+  int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, nullptr, nullptr);
+  if (ret < 0) {
+    return ret;
+  }
+
+  return 0;
+}
+
 /** 
  * get listing of the objects in a bucket.
  *
@@ -8892,22 +8907,6 @@ int RGWRados::Object::get_manifest(RGWObjManifest **pmanifest)
   return 0;
 }
 
-int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id)
-{
-  rgw_bucket& bucket = bucket_info.bucket;
-  bucket.bucket_id = new_bucket_id;
-
-  int ret = get_bucket_instance_info(ctx, bucket, bucket_info, nullptr, nullptr);
-  if (ret < 0) {
-    return ret;
-  }
-  obj.bucket = bucket;
-  bucket_shard.bucket = bucket;
-  bs_initialized = false
-
-  return 0;
-}
-
 int RGWRados::Object::Read::get_attr(const char *name, bufferlist& dest)
 {
   RGWObjState *state;
@@ -9548,7 +9547,7 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t
 
   int r;
 
-#define NUM_RESHARD_RETRIES 3
+#define NUM_RESHARD_RETRIES 10
 
   for (int i = 0; i < NUM_RESHARD_RETRIES; ++i) {
     BucketShard *bs;
@@ -9565,13 +9564,20 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t
     RGWReshard reshard(store);
     string new_bucket_id;
     r = reshard.block_while_resharding(bs, &new_bucket_id);
-    if (r != -EAGAIN) {
-      if (r < 0) {
-        return r;
-      }
-      r = target->update_bucket_id(new_bucket_id);
-      break;
+    if (r == -ERR_BUSY_RESHARDING) {
+      continue;
+    }
+    if (r < 0) {
+      return r;
+    }
+    ldout(store->ctx(), 20) << "reshard completion identified, new_bucket_id=" << new_bucket_id << dendl;
+    i = 0; /* resharding is finished, make sure we can retry */
+    r = target->update_bucket_id(new_bucket_id);
+    if (r < 0) {
+      ldout(store->ctx(), 0) << "ERROR: update_bucket_id() new_bucket_id=" << new_bucket_id << " returned r=" << r << dendl;
+      return r;
     }
+    invalidate_bs();
   }
 
   if (r < 0) {

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index dedc439f4c6e994674f6a5232e3372e8826fc3c0..15b41c2c4f107d392ef27c3184fc31f14f185dbc 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -2675,8 +2675,6 @@ public:
     RGWBucketInfo& get_bucket_info() { return bucket_info; }
     int get_manifest(RGWObjManifest **pmanifest);
 
-    void update_bucket_id(const string& new_bucket_id);
-
     int get_bucket_shard(BucketShard **pbs) {
       if (!bs_initialized) {
         int r = bs.init(bucket_info.bucket, obj);
@@ -2853,6 +2851,8 @@ public:
     rgw_bucket& get_bucket() { return bucket; }
     RGWBucketInfo& get_bucket_info() { return bucket_info; }
 
+    int update_bucket_id(const string& new_bucket_id);
+
     int get_shard_id() { return shard_id; }
     void set_shard_id(int id) {
       shard_id = id;

diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc
index d656dae9633913b608bdbc8bee6f656b22a87977..7a631410fd79daf6149d856804f0669f50c35105 100644 (file)
--- a/src/rgw/rgw_reshard.cc
+++ b/src/rgw/rgw_reshard.cc
@@ -445,7 +445,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries,
     return ret;
   }
 
-  ret = set_resharding_status(bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_IN_PROGRESS);
+  ret = set_resharding_status(new_bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_IN_PROGRESS);
   if (ret < 0) {
     unlock_bucket();
     return ret;
@@ -464,7 +464,7 @@ sleep(10);
     return ret;
   }
 
-  ret = set_resharding_status(bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_DONE);
+  ret = set_resharding_status(new_bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_DONE);
   if (ret < 0) {
     unlock_bucket();
     return ret;
@@ -627,7 +627,7 @@ int RGWReshard::block_while_resharding(RGWRados::BucketShard *bs, string *new_bu
     sleep(default_reshard_sleep_duration);
   }
   ldout(store->ctx(), 0) << "RGWReshard::" << __func__ << " ERROR: bucket is still resharding, please retry" << dendl;
-  return -EAGAIN;
+  return -ERR_BUSY_RESHARDING;
 }
 
 BucketIndexLockGuard::BucketIndexLockGuard(CephContext *_cct, RGWRados* _store, const string& bucket_instance_id, const string& _oid, const librados::IoCtx& _io_ctx) :