cls/rgw: create an objclass method to guard in-progress resharding

This method will be called on the bucket index shard.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index f25bbe5fe1d081c683cb35bbb2122a08e3a90208..abae4bfbf17a1a77838082f444e22de826686bfa 100644 (file)
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -3665,6 +3665,32 @@ static int rgw_clear_bucket_resharding(cls_method_context_t hctx, bufferlist *in
   return write_bucket_header(hctx, &header);
 }
 
+static int rgw_guard_bucket_resharding(cls_method_context_t hctx, bufferlist *in,  bufferlist *out)
+{
+  cls_rgw_guard_bucket_resharding_op op;
+
+  bufferlist::iterator in_iter = in->begin();
+  try {
+    ::decode(op, in_iter);
+  } catch (buffer::error& err) {
+    CLS_LOG(1, "ERROR: cls_rgw_clear_bucket_resharding: failed to decode entry\n");
+    return -EINVAL;
+  }
+
+  struct rgw_bucket_dir_header header;
+  int rc = read_bucket_header(hctx, &header);
+  if (rc < 0) {
+    CLS_LOG(1, "ERROR: %s(): failed to read header\n", __func__);
+    return rc;
+  }
+
+  if (header.resharding()) {
+    return op.ret_err;
+  }
+
+  return 0;
+}
+
 static int rgw_get_bucket_resharding(cls_method_context_t hctx, bufferlist *in,  bufferlist *out)
 {
   cls_rgw_get_bucket_resharding_op op;
@@ -3737,6 +3763,7 @@ CLS_INIT(rgw)
   cls_method_handle_t h_rgw_reshard_remove;
   cls_method_handle_t h_rgw_set_bucket_resharding;
   cls_method_handle_t h_rgw_clear_bucket_resharding;
+  cls_method_handle_t h_rgw_guard_bucket_resharding;
   cls_method_handle_t h_rgw_get_bucket_resharding;
 
 
@@ -3796,6 +3823,8 @@ CLS_INIT(rgw)
                          rgw_set_bucket_resharding, &h_rgw_set_bucket_resharding);
   cls_register_cxx_method(h_class, "clear_bucket_resharding", CLS_METHOD_RD | CLS_METHOD_WR,
                          rgw_clear_bucket_resharding, &h_rgw_clear_bucket_resharding);
+  cls_register_cxx_method(h_class, "guard_bucket_resharding", CLS_METHOD_RD ,
+                         rgw_guard_bucket_resharding, &h_rgw_guard_bucket_resharding);
   cls_register_cxx_method(h_class, "get_bucket_resharding", CLS_METHOD_RD ,
                          rgw_get_bucket_resharding, &h_rgw_get_bucket_resharding);
 

diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc
index eed5d5b1d0a30c89d62f1255ff441bc9a70ea2f1..f67701ec27874126d1edf8f8b4e1c190f13a62d2 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.cc
+++ b/src/cls/rgw/cls_rgw_client.cc
@@ -885,3 +885,12 @@ int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
 
   return 0;
 }
+
+void cls_rgw_guard_bucket_resharding(librados::ObjectWriteOperation& op, int ret_err)
+{
+  bufferlist in, out;
+  struct cls_rgw_guard_bucket_resharding_op call;
+  call.ret_err = ret_err;
+  ::encode(call, in);
+  op.exec("rgw", "guard_bucket_resharding", in);
+}

diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h
index 2affa7dc20328d06daadb4a55dfa87d565c68512..8472cd0b59d1240c1766ceccf4dceec72c585c74 100644 (file)
--- a/src/cls/rgw/cls_rgw_client.h
+++ b/src/cls/rgw/cls_rgw_client.h
@@ -505,6 +505,7 @@ void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_re
 int cls_rgw_set_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
                                   const cls_rgw_bucket_instance_entry& entry);
 int cls_rgw_clear_bucket_resharding(librados::IoCtx& io_ctx, const string& oid);
+void cls_rgw_guard_bucket_resharding(librados::ObjectWriteOperation& op, int ret_err);
 int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid,
                                   cls_rgw_bucket_instance_entry *entry);
 

diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc
index 13121319963491ad12063ff8875bdb8ef05f13fc..82ad868bfa7fd4e8aa4bf35cee5eeb502f32b42b 100644 (file)
--- a/src/cls/rgw/cls_rgw_ops.cc
+++ b/src/cls/rgw/cls_rgw_ops.cc
@@ -516,6 +516,19 @@ void cls_rgw_clear_bucket_resharding_op::dump(Formatter *f) const
 {
 }
 
+void cls_rgw_guard_bucket_resharding_op::generate_test_instances(
+  list<cls_rgw_guard_bucket_resharding_op*>& ls)
+{
+  ls.push_back(new cls_rgw_guard_bucket_resharding_op);
+  ls.push_back(new cls_rgw_guard_bucket_resharding_op);
+}
+
+void cls_rgw_guard_bucket_resharding_op::dump(Formatter *f) const
+{
+  ::encode_json("ret_err", ret_err, f);
+}
+
+
 void cls_rgw_get_bucket_resharding_op::generate_test_instances(
   list<cls_rgw_get_bucket_resharding_op*>& ls)
 {

diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h
index 5d9a43f5e68df6a9db8c76ac940cb1d47d3a8be7..6f8107516bed84e871f5a6b614bedefa87da9d0b 100644 (file)
--- a/src/cls/rgw/cls_rgw_ops.h
+++ b/src/cls/rgw/cls_rgw_ops.h
@@ -1315,6 +1315,26 @@ struct cls_rgw_clear_bucket_resharding_op {
 };
 WRITE_CLASS_ENCODER(cls_rgw_clear_bucket_resharding_op)
 
+struct cls_rgw_guard_bucket_resharding_op  {
+  int ret_err{0};
+
+  void encode(bufferlist& bl) const {
+    ENCODE_START(1, 1, bl);
+    ::encode(ret_err, bl);
+    ENCODE_FINISH(bl);
+  }
+
+  void decode(bufferlist::iterator& bl) {
+    DECODE_START(1, bl);
+    ::decode(ret_err, bl);
+    DECODE_FINISH(bl);
+  }
+
+  static void generate_test_instances(list<cls_rgw_guard_bucket_resharding_op*>& o);
+  void dump(Formatter *f) const;
+};
+WRITE_CLASS_ENCODER(cls_rgw_guard_bucket_resharding_op)
+
 struct cls_rgw_get_bucket_resharding_op  {
 
   void encode(bufferlist& bl) const {

diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h
index f8d1c01183a9fd4894c3d77b966f5d6f78155cc5..163b1886391e1a1d21d056cba537531c58e12035 100644 (file)
--- a/src/rgw/rgw_common.h
+++ b/src/rgw/rgw_common.h
@@ -200,6 +200,7 @@ using ceph::crypto::MD5;
 #define ERR_MALFORMED_DOC        2204
 #define ERR_NO_ROLE_FOUND        2205
 #define ERR_DELETE_CONFLICT      2206
+#define ERR_BUSY_RESHARDING      2300
 
 #ifndef UINT32_MAX
 #define UINT32_MAX (0xffffffffu)

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index fde3a0a3149120179f749fad5f31991e16b5c82f..3e5838046dc3520413de453f3297da507010ef3c 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -9535,14 +9535,6 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t
   RGWRados *store = target->get_store();
   BucketShard *bs;
 
-  /* handle on going bucket resharding */
-  BucketIndexLockGuard guard(store->ctx(), store, target->get_bucket().bucket_id, target->get_bucket().oid,
-                            store->reshard_pool_ctx);
-  int ret = store->reshard->block_while_resharding(target->get_bucket().oid, guard);
-  if (ret < 0) {
-    return ret;
-  }
-
   ret = get_bucket_shard(&bs);
   if (ret < 0) {
     ldout(store->ctx(), 5) << "failed to get BucketShard object: ret=" << ret << dendl;
@@ -12157,6 +12149,7 @@ int RGWRados::cls_obj_prepare_op(BucketShard& bs, RGWModifyOp op, string& tag,
   
   ObjectWriteOperation o;
   cls_rgw_obj_key key(obj.key.get_index_key_name(), obj.key.instance);
+  cls_rgw_guard_resharding(o, -ERR_BUSY_RESHARDING);
   cls_rgw_bucket_prepare_op(o, op, tag, key, obj.key.get_loc(), get_zone().log_data, bilog_flags, zones_trace);
   return bs.index_ctx.operate(bs.bucket_obj, &o);
 }
@@ -12193,6 +12186,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag,
   ver.pool = pool;
   ver.epoch = epoch;
   cls_rgw_obj_key key(ent.key.name, ent.key.instance);
+  cls_rgw_guard_resharding(o, -ERR_BUSY_RESHARDING);
   cls_rgw_bucket_complete_op(o, op, tag, ver, key, dir_meta, pro,
                              get_zone().log_data, bilog_flags, zones_trace);
 

diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc
index 93fda7f88e370a8fdd5a4a059b315bec96eef150..389d1dcca940d69f403bd4240ea273d0506bc29c 100644 (file)
--- a/src/rgw/rgw_reshard.cc
+++ b/src/rgw/rgw_reshard.cc
@@ -177,7 +177,7 @@ const int num_retries = 10;
 const int default_reshard_sleep_duration = 30;
 
 int RGWReshard::block_while_resharding(const string& bucket_instance_oid,
-                                                                 BucketIndexLockGuard& guard)
+                                      BucketIndexLockGuard& guard)
 {
   int ret = 0;
   cls_rgw_bucket_instance_entry entry;
@@ -195,16 +195,15 @@ int RGWReshard::block_while_resharding(const string& bucket_instance_oid,
       return ret;
     }
 
-    if (entry.resharding) {
-      /* clear resharding uses the same lock */
-      ret = guard.unlock();
-      if (ret < 0) {
-       return ret;
-      }
-      sleep(default_reshard_sleep_duration);
-    } else {
+    ret = guard.unlock();
+    if (ret < 0) {
+      return ret;
+    }
+    if (!entry.resharding) {
       return 0;
     }
+    /* needed to unlock as clear resharding uses the same lock */
+    sleep(default_reshard_sleep_duration);
   }
   ldout(cct, 0) << "RGWReshard::" << __func__ << " ERROR: bucket is still resharding, please retry" << dendl;
   return -EAGAIN;