From: Yehuda Sadeh Date: Mon, 8 May 2017 21:43:37 +0000 (-0700) Subject: cls/rgw: create an objclass method to guard in-progress resharding X-Git-Tag: ses5-milestone6~8^2~7^2~60 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=7d44601e836d921027b07d297597039881dd4a21;p=ceph.git cls/rgw: create an objclass method to guard in-progress resharding This method will be called on the bucket index shard. Signed-off-by: Yehuda Sadeh --- diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index f25bbe5fe1d..abae4bfbf17 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -3665,6 +3665,32 @@ static int rgw_clear_bucket_resharding(cls_method_context_t hctx, bufferlist *in return write_bucket_header(hctx, &header); } +static int rgw_guard_bucket_resharding(cls_method_context_t hctx, bufferlist *in, bufferlist *out) +{ + cls_rgw_guard_bucket_resharding_op op; + + bufferlist::iterator in_iter = in->begin(); + try { + ::decode(op, in_iter); + } catch (buffer::error& err) { + CLS_LOG(1, "ERROR: cls_rgw_clear_bucket_resharding: failed to decode entry\n"); + return -EINVAL; + } + + struct rgw_bucket_dir_header header; + int rc = read_bucket_header(hctx, &header); + if (rc < 0) { + CLS_LOG(1, "ERROR: %s(): failed to read header\n", __func__); + return rc; + } + + if (header.resharding()) { + return op.ret_err; + } + + return 0; +} + static int rgw_get_bucket_resharding(cls_method_context_t hctx, bufferlist *in, bufferlist *out) { cls_rgw_get_bucket_resharding_op op; @@ -3737,6 +3763,7 @@ CLS_INIT(rgw) cls_method_handle_t h_rgw_reshard_remove; cls_method_handle_t h_rgw_set_bucket_resharding; cls_method_handle_t h_rgw_clear_bucket_resharding; + cls_method_handle_t h_rgw_guard_bucket_resharding; cls_method_handle_t h_rgw_get_bucket_resharding; @@ -3796,6 +3823,8 @@ CLS_INIT(rgw) rgw_set_bucket_resharding, &h_rgw_set_bucket_resharding); cls_register_cxx_method(h_class, "clear_bucket_resharding", CLS_METHOD_RD | CLS_METHOD_WR, rgw_clear_bucket_resharding, &h_rgw_clear_bucket_resharding); + cls_register_cxx_method(h_class, "guard_bucket_resharding", CLS_METHOD_RD , + rgw_guard_bucket_resharding, &h_rgw_guard_bucket_resharding); cls_register_cxx_method(h_class, "get_bucket_resharding", CLS_METHOD_RD , rgw_get_bucket_resharding, &h_rgw_get_bucket_resharding); diff --git a/src/cls/rgw/cls_rgw_client.cc b/src/cls/rgw/cls_rgw_client.cc index eed5d5b1d0a..f67701ec278 100644 --- a/src/cls/rgw/cls_rgw_client.cc +++ b/src/cls/rgw/cls_rgw_client.cc @@ -885,3 +885,12 @@ int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid, return 0; } + +void cls_rgw_guard_bucket_resharding(librados::ObjectWriteOperation& op, int ret_err) +{ + bufferlist in, out; + struct cls_rgw_guard_bucket_resharding_op call; + call.ret_err = ret_err; + ::encode(call, in); + op.exec("rgw", "guard_bucket_resharding", in); +} diff --git a/src/cls/rgw/cls_rgw_client.h b/src/cls/rgw/cls_rgw_client.h index 2affa7dc203..8472cd0b59d 100644 --- a/src/cls/rgw/cls_rgw_client.h +++ b/src/cls/rgw/cls_rgw_client.h @@ -505,6 +505,7 @@ void cls_rgw_reshard_remove(librados::ObjectWriteOperation& op, const cls_rgw_re int cls_rgw_set_bucket_resharding(librados::IoCtx& io_ctx, const string& oid, const cls_rgw_bucket_instance_entry& entry); int cls_rgw_clear_bucket_resharding(librados::IoCtx& io_ctx, const string& oid); +void cls_rgw_guard_bucket_resharding(librados::ObjectWriteOperation& op, int ret_err); int cls_rgw_get_bucket_resharding(librados::IoCtx& io_ctx, const string& oid, cls_rgw_bucket_instance_entry *entry); diff --git a/src/cls/rgw/cls_rgw_ops.cc b/src/cls/rgw/cls_rgw_ops.cc index 13121319963..82ad868bfa7 100644 --- a/src/cls/rgw/cls_rgw_ops.cc +++ b/src/cls/rgw/cls_rgw_ops.cc @@ -516,6 +516,19 @@ void cls_rgw_clear_bucket_resharding_op::dump(Formatter *f) const { } +void cls_rgw_guard_bucket_resharding_op::generate_test_instances( + list& ls) +{ + ls.push_back(new cls_rgw_guard_bucket_resharding_op); + ls.push_back(new cls_rgw_guard_bucket_resharding_op); +} + +void cls_rgw_guard_bucket_resharding_op::dump(Formatter *f) const +{ + ::encode_json("ret_err", ret_err, f); +} + + void cls_rgw_get_bucket_resharding_op::generate_test_instances( list& ls) { diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index 5d9a43f5e68..6f8107516be 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -1315,6 +1315,26 @@ struct cls_rgw_clear_bucket_resharding_op { }; WRITE_CLASS_ENCODER(cls_rgw_clear_bucket_resharding_op) +struct cls_rgw_guard_bucket_resharding_op { + int ret_err{0}; + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + ::encode(ret_err, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::iterator& bl) { + DECODE_START(1, bl); + ::decode(ret_err, bl); + DECODE_FINISH(bl); + } + + static void generate_test_instances(list& o); + void dump(Formatter *f) const; +}; +WRITE_CLASS_ENCODER(cls_rgw_guard_bucket_resharding_op) + struct cls_rgw_get_bucket_resharding_op { void encode(bufferlist& bl) const { diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index f8d1c01183a..163b1886391 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -200,6 +200,7 @@ using ceph::crypto::MD5; #define ERR_MALFORMED_DOC 2204 #define ERR_NO_ROLE_FOUND 2205 #define ERR_DELETE_CONFLICT 2206 +#define ERR_BUSY_RESHARDING 2300 #ifndef UINT32_MAX #define UINT32_MAX (0xffffffffu) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index fde3a0a3149..3e5838046dc 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -9535,14 +9535,6 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t RGWRados *store = target->get_store(); BucketShard *bs; - /* handle on going bucket resharding */ - BucketIndexLockGuard guard(store->ctx(), store, target->get_bucket().bucket_id, target->get_bucket().oid, - store->reshard_pool_ctx); - int ret = store->reshard->block_while_resharding(target->get_bucket().oid, guard); - if (ret < 0) { - return ret; - } - ret = get_bucket_shard(&bs); if (ret < 0) { ldout(store->ctx(), 5) << "failed to get BucketShard object: ret=" << ret << dendl; @@ -12157,6 +12149,7 @@ int RGWRados::cls_obj_prepare_op(BucketShard& bs, RGWModifyOp op, string& tag, ObjectWriteOperation o; cls_rgw_obj_key key(obj.key.get_index_key_name(), obj.key.instance); + cls_rgw_guard_resharding(o, -ERR_BUSY_RESHARDING); cls_rgw_bucket_prepare_op(o, op, tag, key, obj.key.get_loc(), get_zone().log_data, bilog_flags, zones_trace); return bs.index_ctx.operate(bs.bucket_obj, &o); } @@ -12193,6 +12186,7 @@ int RGWRados::cls_obj_complete_op(BucketShard& bs, RGWModifyOp op, string& tag, ver.pool = pool; ver.epoch = epoch; cls_rgw_obj_key key(ent.key.name, ent.key.instance); + cls_rgw_guard_resharding(o, -ERR_BUSY_RESHARDING); cls_rgw_bucket_complete_op(o, op, tag, ver, key, dir_meta, pro, get_zone().log_data, bilog_flags, zones_trace); diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 93fda7f88e3..389d1dcca94 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -177,7 +177,7 @@ const int num_retries = 10; const int default_reshard_sleep_duration = 30; int RGWReshard::block_while_resharding(const string& bucket_instance_oid, - BucketIndexLockGuard& guard) + BucketIndexLockGuard& guard) { int ret = 0; cls_rgw_bucket_instance_entry entry; @@ -195,16 +195,15 @@ int RGWReshard::block_while_resharding(const string& bucket_instance_oid, return ret; } - if (entry.resharding) { - /* clear resharding uses the same lock */ - ret = guard.unlock(); - if (ret < 0) { - return ret; - } - sleep(default_reshard_sleep_duration); - } else { + ret = guard.unlock(); + if (ret < 0) { + return ret; + } + if (!entry.resharding) { return 0; } + /* needed to unlock as clear resharding uses the same lock */ + sleep(default_reshard_sleep_duration); } ldout(cct, 0) << "RGWReshard::" << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; return -EAGAIN;