From: Casey Bodley Date: Sat, 24 Nov 2018 22:05:57 +0000 (-0500) Subject: rgw: move block_while_resharding() out of RGWReshardWait X-Git-Tag: v14.1.0~617^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=ab2f30842d7be579c925a43fd9c4d634ef7dd7ed;p=ceph.git rgw: move block_while_resharding() out of RGWReshardWait Signed-off-by: Casey Bodley --- diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index e0aca7084f0..7ba7dcaac65 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1631,7 +1631,7 @@ int RGWRados::init_complete() obj_tombstone_cache = new tombstone_cache_t(cct->_conf->rgw_obj_tombstone_cache_size); } - reshard_wait = std::make_shared(this); + reshard_wait = std::make_shared(); reshard = new RGWReshard(this); @@ -7119,9 +7119,72 @@ int RGWRados::block_while_resharding(RGWRados::BucketShard *bs, const RGWBucketInfo& bucket_info, optional_yield y) { - std::shared_ptr waiter = reshard_wait; + int ret = 0; + cls_rgw_bucket_instance_entry entry; + + const int num_retries = 10; + for (int i=0; i < num_retries; i++) { + ret = cls_rgw_get_bucket_resharding(bs->index_ctx, bs->bucket_obj, &entry); + if (ret < 0) { + ldout(cct, 0) << __func__ << " ERROR: failed to get bucket resharding :" << + cpp_strerror(-ret)<< dendl; + return ret; + } + if (!entry.resharding_in_progress()) { + *new_bucket_id = entry.new_bucket_instance_id; + return 0; + } + ldout(cct, 20) << "NOTICE: reshard still in progress; " << (i < num_retries - 1 ? "retrying" : "too many retries") << dendl; + + if (i == num_retries - 1) { + break; + } - return waiter->block_while_resharding(bs, new_bucket_id, bucket_info, y); + // If bucket is erroneously marked as resharding (e.g., crash or + // other error) then fix it. If we can take the bucket reshard + // lock then it means no other resharding should be taking place, + // and we're free to clear the flags. + { + // since we expect to do this rarely, we'll do our work in a + // block and erase our work after each try + + RGWObjectCtx obj_ctx(this); + const rgw_bucket& b = bs->bucket; + std::string bucket_id = b.get_key(); + RGWBucketReshardLock reshard_lock(this, bucket_info, true); + ret = reshard_lock.lock(); + if (ret < 0) { + ldout(cct, 20) << __func__ << + " INFO: failed to take reshard lock for bucket " << + bucket_id << "; expected if resharding underway" << dendl; + } else { + ldout(cct, 10) << __func__ << + " INFO: was able to take reshard lock for bucket " << + bucket_id << dendl; + ret = RGWBucketReshard::clear_resharding(this, bucket_info); + if (ret < 0) { + reshard_lock.unlock(); + ldout(cct, 0) << __func__ << + " ERROR: failed to clear resharding flags for bucket " << + bucket_id << dendl; + } else { + reshard_lock.unlock(); + ldout(cct, 5) << __func__ << + " INFO: apparently successfully cleared resharding flags for " + "bucket " << bucket_id << dendl; + continue; // if we apparently succeed immediately test again + } // if clear resharding succeeded + } // if taking of lock succeeded + } // block to encapsulate recovery from incomplete reshard + + ret = reshard_wait->wait(y); + if (ret < 0) { + ldout(cct, 0) << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; + return ret; + } + } + ldout(cct, 0) << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; + return -ERR_BUSY_RESHARDING; } int RGWRados::bucket_index_link_olh(const RGWBucketInfo& bucket_info, RGWObjState& olh_state, const rgw_obj& obj_instance, diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index ed6b935f7ec..7dc50ce0717 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -862,10 +862,9 @@ int RGWReshard::clear_bucket_resharding(const string& bucket_instance_oid, cls_r return 0; } -const int num_retries = 10; static const std::chrono::seconds default_reshard_sleep_duration(5); -int RGWReshardWait::do_wait(optional_yield y) +int RGWReshardWait::wait(optional_yield y) { std::unique_lock lock(mutex); @@ -913,78 +912,6 @@ void RGWReshardWait::stop() } } -int RGWReshardWait::block_while_resharding(RGWRados::BucketShard *bs, - string *new_bucket_id, - const RGWBucketInfo& bucket_info, - optional_yield y) -{ - int ret = 0; - cls_rgw_bucket_instance_entry entry; - - for (int i=0; i < num_retries; i++) { - ret = cls_rgw_get_bucket_resharding(bs->index_ctx, bs->bucket_obj, &entry); - if (ret < 0) { - ldout(store->ctx(), 0) << __func__ << " ERROR: failed to get bucket resharding :" << - cpp_strerror(-ret)<< dendl; - return ret; - } - if (!entry.resharding_in_progress()) { - *new_bucket_id = entry.new_bucket_instance_id; - return 0; - } - ldout(store->ctx(), 20) << "NOTICE: reshard still in progress; " << (i < num_retries - 1 ? "retrying" : "too many retries") << dendl; - - if (i == num_retries - 1) { - break; - } - - // If bucket is erroneously marked as resharding (e.g., crash or - // other error) then fix it. If we can take the bucket reshard - // lock then it means no other resharding should be taking place, - // and we're free to clear the flags. - { - // since we expect to do this rarely, we'll do our work in a - // block and erase our work after each try - - RGWObjectCtx obj_ctx(bs->store); - const rgw_bucket& b = bs->bucket; - std::string bucket_id = b.get_key(); - RGWBucketReshardLock reshard_lock(bs->store, bucket_info, true); - ret = reshard_lock.lock(); - if (ret < 0) { - ldout(store->ctx(), 20) << __func__ << - " INFO: failed to take reshard lock for bucket " << - bucket_id << "; expected if resharding underway" << dendl; - } else { - ldout(store->ctx(), 10) << __func__ << - " INFO: was able to take reshard lock for bucket " << - bucket_id << dendl; - ret = RGWBucketReshard::clear_resharding(bs->store, bucket_info); - if (ret < 0) { - reshard_lock.unlock(); - ldout(store->ctx(), 0) << __func__ << - " ERROR: failed to clear resharding flags for bucket " << - bucket_id << dendl; - } else { - reshard_lock.unlock(); - ldout(store->ctx(), 5) << __func__ << - " INFO: apparently successfully cleared resharding flags for " - "bucket " << bucket_id << dendl; - continue; // if we apparently succeed immediately test again - } // if clear resharding succeeded - } // if taking of lock succeeded - } // block to encapsulate recovery from incomplete reshard - - ret = do_wait(y); - if (ret < 0) { - ldout(store->ctx(), 0) << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; - return ret; - } - } - ldout(store->ctx(), 0) << __func__ << " ERROR: bucket is still resharding, please retry" << dendl; - return -ERR_BUSY_RESHARDING; -} - int RGWReshard::process_single_logshard(int logshard_num) { string marker; diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h index 74233db8e93..a715c5df1a6 100644 --- a/src/rgw/rgw_reshard.h +++ b/src/rgw/rgw_reshard.h @@ -173,7 +173,6 @@ public: }; class RGWReshardWait { - RGWRados *store; ceph::mutex mutex = ceph::make_mutex("RGWReshardWait::lock"); ceph::condition_variable cond; @@ -186,16 +185,12 @@ class RGWReshardWait { bool going_down{false}; - int do_wait(optional_yield y); public: - explicit RGWReshardWait(RGWRados *_store) : store(_store) {} + RGWReshardWait() = default; ~RGWReshardWait() { ceph_assert(going_down); } - int block_while_resharding(RGWRados::BucketShard *bs, - string *new_bucket_id, - const RGWBucketInfo& bucket_info, - optional_yield y); + int wait(optional_yield y); // unblock any threads waiting on reshard void stop(); };