From d250fab244d4af2dfd9896edb10e78ba46491f80 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Thu, 11 May 2017 10:39:26 -0700 Subject: [PATCH] rgw: fix update_bucket_id, reshard blocking logic Signed-off-by: Yehuda Sadeh --- src/rgw/rgw_common.h | 5 ++++ src/rgw/rgw_rados.cc | 52 +++++++++++++++++++++++------------------- src/rgw/rgw_rados.h | 4 ++-- src/rgw/rgw_reshard.cc | 6 ++--- 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/rgw/rgw_common.h b/src/rgw/rgw_common.h index 163b1886391..fb78c4a0c9a 100644 --- a/src/rgw/rgw_common.h +++ b/src/rgw/rgw_common.h @@ -1028,6 +1028,11 @@ struct rgw_bucket { DECODE_FINISH(bl); } + void update_bucket_id(const string& new_bucket_id) { + bucket_id = new_bucket_id; + oid.clear(); + } + // format a key for the bucket/instance. pass delim=0 to skip a field std::string get_key(char tenant_delim = '/', char id_delim = ':') const; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 364da290efa..441fddd2946 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -5182,6 +5182,21 @@ int rgw_policy_from_attrset(CephContext *cct, map& attrset, } +int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id) +{ + rgw_bucket bucket = bucket_info.bucket; + bucket.update_bucket_id(new_bucket_id); + + RGWObjectCtx obj_ctx(store); + + int ret = store->get_bucket_instance_info(obj_ctx, bucket, bucket_info, nullptr, nullptr); + if (ret < 0) { + return ret; + } + + return 0; +} + /** * get listing of the objects in a bucket. * @@ -8892,22 +8907,6 @@ int RGWRados::Object::get_manifest(RGWObjManifest **pmanifest) return 0; } -int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id) -{ - rgw_bucket& bucket = bucket_info.bucket; - bucket.bucket_id = new_bucket_id; - - int ret = get_bucket_instance_info(ctx, bucket, bucket_info, nullptr, nullptr); - if (ret < 0) { - return ret; - } - obj.bucket = bucket; - bucket_shard.bucket = bucket; - bs_initialized = false - - return 0; -} - int RGWRados::Object::Read::get_attr(const char *name, bufferlist& dest) { RGWObjState *state; @@ -9548,7 +9547,7 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t int r; -#define NUM_RESHARD_RETRIES 3 +#define NUM_RESHARD_RETRIES 10 for (int i = 0; i < NUM_RESHARD_RETRIES; ++i) { BucketShard *bs; @@ -9565,13 +9564,20 @@ int RGWRados::Bucket::UpdateIndex::prepare(RGWModifyOp op, const string *write_t RGWReshard reshard(store); string new_bucket_id; r = reshard.block_while_resharding(bs, &new_bucket_id); - if (r != -EAGAIN) { - if (r < 0) { - return r; - } - r = target->update_bucket_id(new_bucket_id); - break; + if (r == -ERR_BUSY_RESHARDING) { + continue; + } + if (r < 0) { + return r; + } + ldout(store->ctx(), 20) << "reshard completion identified, new_bucket_id=" << new_bucket_id << dendl; + i = 0; /* resharding is finished, make sure we can retry */ + r = target->update_bucket_id(new_bucket_id); + if (r < 0) { + ldout(store->ctx(), 0) << "ERROR: update_bucket_id() new_bucket_id=" << new_bucket_id << " returned r=" << r << dendl; + return r; } + invalidate_bs(); } if (r < 0) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index dedc439f4c6..15b41c2c4f1 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -2675,8 +2675,6 @@ public: RGWBucketInfo& get_bucket_info() { return bucket_info; } int get_manifest(RGWObjManifest **pmanifest); - void update_bucket_id(const string& new_bucket_id); - int get_bucket_shard(BucketShard **pbs) { if (!bs_initialized) { int r = bs.init(bucket_info.bucket, obj); @@ -2853,6 +2851,8 @@ public: rgw_bucket& get_bucket() { return bucket; } RGWBucketInfo& get_bucket_info() { return bucket_info; } + int update_bucket_id(const string& new_bucket_id); + int get_shard_id() { return shard_id; } void set_shard_id(int id) { shard_id = id; diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index d656dae9633..7a631410fd7 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -445,7 +445,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries, return ret; } - ret = set_resharding_status(bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_IN_PROGRESS); + ret = set_resharding_status(new_bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_IN_PROGRESS); if (ret < 0) { unlock_bucket(); return ret; @@ -464,7 +464,7 @@ sleep(10); return ret; } - ret = set_resharding_status(bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_DONE); + ret = set_resharding_status(new_bucket_info.bucket.bucket_id, CLS_RGW_RESHARD_DONE); if (ret < 0) { unlock_bucket(); return ret; @@ -627,7 +627,7 @@ int RGWReshard::block_while_resharding(RGWRados::BucketShard *bs, string *new_bu sleep(default_reshard_sleep_duration); } ldout(store->ctx(), 0) << "RGWReshard::" << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; - return -EAGAIN; + return -ERR_BUSY_RESHARDING; } BucketIndexLockGuard::BucketIndexLockGuard(CephContext *_cct, RGWRados* _store, const string& bucket_instance_id, const string& _oid, const librados::IoCtx& _io_ctx) : -- 2.39.5