From 5214187fe394458d82f5cd4db629dc545f4da35a Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Thu, 21 May 2020 18:24:29 +0530 Subject: [PATCH] rgw/dynamic-resharding: reshard bucket using the same instance id Signed-off-by: Shilpa Jagannath --- src/rgw/rgw_admin.cc | 28 ++++++++++++++----- src/rgw/rgw_bucket.cc | 24 ++++++++++++---- src/rgw/rgw_orphan.cc | 2 +- src/rgw/rgw_rados.cc | 20 +++++++------- src/rgw/rgw_reshard.cc | 62 ++++++++++++++++++------------------------ src/rgw/rgw_reshard.h | 17 ++++++------ 6 files changed, 85 insertions(+), 68 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 77b6305feebdd..8bcd76e308752 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -2724,7 +2724,7 @@ int check_reshard_bucket_params(rgw::sal::RadosStore* store, return ret; } - if ((*bucket)->get_info().reshard_status != cls_rgw_reshard_status::NOT_RESHARDING) { + if ((*bucket)->get_info().layout.resharding != rgw::BucketReshardState::None) { // if in_progress or done then we have an old BucketInfo cerr << "ERROR: the bucket is currently undergoing resharding and " "cannot be added to the reshard list at this time" << std::endl; @@ -6805,7 +6805,8 @@ next: max_entries = 1000; } - int max_shards = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0 ? bucket->get_info().layout.current_index.layout.normal.num_shards : 1); + const auto& index = bucket->get_info().layout.current_index; + int max_shards = index.layout.normal.num_shards; formatter->open_array_section("entries"); @@ -6813,8 +6814,7 @@ next: for (; i < max_shards; i++) { RGWRados::BucketShard bs(static_cast(store)->getRados()); int shard_id = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0 ? i : -1); - - int ret = bs.init(bucket->get_key(), shard_id, bucket->get_info().layout.current_index, nullptr /* no RGWBucketInfo */, dpp()); + int ret = bs.init(bucket->get_key(), shard_id, index, nullptr /* no RGWBucketInfo */, dpp()); marker.clear(); if (ret < 0) { @@ -6872,9 +6872,23 @@ next: return EINVAL; } - ret = bucket->purge_instance(dpp()); - if (ret < 0) { - return -ret; + const auto& index = bucket->get_info().layout.current_index; + int max_shards = index.layout.normal.num_shards; + + for (int i = 0; i < max_shards; i++) { + RGWRados::BucketShard bs(static_cast(store)->getRados()); + int shard_id = (bucket->get_info().layout.current_index.layout.normal.num_shards > 0 ? i : -1); + int ret = bs.init(bucket->get_key(), shard_id, index, nullptr /* no RGWBucketInfo */, dpp()); + if (ret < 0) { + cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << i << "): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + + ret = static_cast(store)->getRados()->bi_remove(dpp(), bs); + if (ret < 0) { + cerr << "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret) << std::endl; + return -ret; + } } } diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 5de37cc1a418c..6813c3ffd07ad 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1331,12 +1331,24 @@ int RGWBucketAdminOp::set_quota(rgw::sal::Store* store, RGWBucketAdminOpState& o static int purge_bucket_instance(rgw::sal::Store* store, const RGWBucketInfo& bucket_info, const DoutPrefixProvider *dpp) { - std::unique_ptr bucket; - int ret = store->get_bucket(nullptr, bucket_info, &bucket); - if (ret < 0) - return ret; - - return bucket->purge_instance(dpp); + const auto& index = bucket_info.layout.current_index; + int max_shards = index.layout.normal.num_shards; + for (int i = 0; i < max_shards; i++) { + RGWRados::BucketShard bs(static_cast(store)->getRados()); + int ret = bs.init(bucket_info.bucket, i, bucket_info.layout.current_index, nullptr, dpp); + if (ret < 0) { + cerr << "ERROR: bs.init(bucket=" << bucket_info.bucket << ", shard=" << i + << "): " << cpp_strerror(-ret) << std::endl; + return ret; + } + ret = static_cast(store)->getRados()->bi_remove(dpp, bs); + if (ret < 0) { + cerr << "ERROR: failed to remove bucket index object: " + << cpp_strerror(-ret) << std::endl; + return ret; + } + } + return 0; } inline auto split_tenant(const std::string& bucket_name){ diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 3d19f4366fb83..4c514186815b6 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -520,7 +520,7 @@ int RGWOrphanSearch::build_linked_oids_for_bucket(const DoutPrefixProvider *dpp, return 0; } - if (cur_bucket->get_info().reshard_status == cls_rgw_reshard_status::IN_PROGRESS) { + if (cur_bucket->get_info().layout.resharding != rgw::BucketReshardState::None) { ldpp_dout(dpp, 0) << __func__ << ": reshard in progress. Skipping " << orphan_bucket.name << ": " << orphan_bucket.bucket_id << dendl; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 30e338c8a396e..8ca9b74daad0d 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2747,12 +2747,12 @@ int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketIn return 0; } -int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid) +int RGWRados::BucketShard::init(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& target_layout, int sid) { bucket = bucket_info.bucket; shard_id = sid; - int ret = store->svc.bi_rados->open_bucket_index_shard(dpp, bucket_info, shard_id, idx_layout, &bucket_obj); + int ret = store->svc.bi_rados->open_bucket_index_shard(dpp, bucket_info, shard_id, target_layout, &bucket_obj); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl; return ret; @@ -8397,13 +8397,12 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, * few results, perhaps due to filtering or to a series of * namespaced entries */ - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": " << - bucket_info.bucket << - " start_after=\"" << start_after << - "\", prefix=\"" << prefix << - ", delimiter=\"" << delimiter << - "\", shard_id=" << shard_id << - "\", num_entries=" << num_entries << + ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": " << bucket_info.bucket << + " start_after=\"" << start_after.name << + "[" << start_after.instance << + "]\", prefix=\"" << prefix << + "\" num_entries=" << num_entries << + ", shard_id=" << shard_id << ", list_versions=" << list_versions << ", expansion_factor=" << expansion_factor << ", force_check_filter is " << @@ -9078,7 +9077,7 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo RGWSI_RADOS::Pool index_pool; map oids; map list_results; - int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, -1, idx_layout, &index_pool, &oids, bucket_instance_ids); + int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, idx_layout, &index_pool, &oids, bucket_instance_ids); if (r < 0) { ldpp_dout(dpp, 20) << "cls_bucket_head: open_bucket_index() returned " << r << dendl; @@ -9086,6 +9085,7 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo } r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)(); + ldout(cct, 20) << "CLSRGWIssueGetDirHeader issued" << dendl; if (r < 0) { ldpp_dout(dpp, 20) << "cls_bucket_head: CLSRGWIssueGetDirHeader() returned " << r << dendl; diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 1c9bc453d6ab8..715a37e7f7887 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -64,7 +64,7 @@ class BucketReshardShard { rgw::sal::RadosStore* store; const RGWBucketInfo& bucket_info; int num_shard; - const rgw::bucket_index_layout_generation& idx_layout; + const rgw::bucket_index_layout_generation& target_layout; RGWRados::BucketShard bs; vector entries; map stats; @@ -105,15 +105,15 @@ class BucketReshardShard { public: BucketReshardShard(const DoutPrefixProvider *dpp, - rgw::sal::RadosStore* _store, const RGWBucketInfo& _bucket_info, - int _num_shard, const rgw::bucket_index_layout_generation& _idx_layout, + rgw::sal::RadosStore *_store, const RGWBucketInfo& _bucket_info, + int _num_shard, const rgw::bucket_index_layout_generation& _target_layout, deque& _completions) : - store(_store), bucket_info(_bucket_info), idx_layout(_idx_layout), bs(store->getRados()), + store(_store), bucket_info(_bucket_info), target_layout(_target_layout), bs(store->getRados()), aio_completions(_completions) { - num_shard = (idx_layout.layout.normal.num_shards > 0 ? _num_shard : -1); + num_shard = (target_layout.layout.normal.num_shards > 0 ? _num_shard : -1); - bs.init(bucket_info.bucket, num_shard, idx_layout, nullptr /* no RGWBucketInfo */, dpp); + bs.init(bucket_info.bucket, num_shard, target_layout, nullptr /* no RGWBucketInfo */, dpp); max_aio_completions = store->ctx()->_conf.get_val("rgw_reshard_max_aio"); @@ -199,10 +199,10 @@ public: store(_store), target_bucket_info(_target_bucket_info), num_target_shards(_num_target_shards) { - const auto& idx_layout = target_bucket_info.layout.current_index; + const auto& target_layout = *target_bucket_info.layout.target_index; target_shards.resize(num_target_shards); for (int i = 0; i < num_target_shards; ++i) { - target_shards[i] = new BucketReshardShard(dpp, store, target_bucket_info, i, idx_layout, completions); + target_shards[i] = new BucketReshardShard(dpp, store, target_bucket_info, i, target_layout, completions); } } @@ -327,43 +327,29 @@ int RGWBucketReshard::clear_index_shard_reshard_status(const DoutPrefixProvider return 0; } -static int create_new_bucket_instance(rgw::sal::RadosStore* store, +static int update_num_shards(rgw::sal::RadosStore *store, int new_num_shards, RGWBucketInfo& bucket_info, map& attrs, - RGWBucketInfo& new_bucket_info, const DoutPrefixProvider *dpp) { - new_bucket_info = bucket_info; - store->getRados()->create_bucket_id(&new_bucket_info.bucket.bucket_id); + bucket_info.layout.target_index->layout.normal.num_shards = new_num_shards; bucket_info.layout.resharding = rgw::BucketReshardState::None; - new_bucket_info.new_bucket_instance_id.clear(); - new_bucket_info.reshard_status = cls_rgw_reshard_status::NOT_RESHARDING; - - int ret = store->svc()->bi->init_index(dpp, new_bucket_info); + int ret = static_cast(store)->getRados()->put_bucket_instance_info(bucket_info, true, real_time(), &attrs, dpp); if (ret < 0) { - cerr << "ERROR: failed to init new bucket indexes: " << cpp_strerror(-ret) << std::endl; - return ret; - } - - ret = store->getRados()->put_bucket_instance_info(new_bucket_info, true, real_time(), &attrs, dpp); - if (ret < 0) { - cerr << "ERROR: failed to store new bucket instance info: " << cpp_strerror(-ret) << std::endl; + cerr << "ERROR: failed to store updated bucket instance info: " << cpp_strerror(-ret) << std::endl; return ret; } return 0; } -int RGWBucketReshard::create_new_bucket_instance(int new_num_shards, - RGWBucketInfo& new_bucket_info, - const DoutPrefixProvider *dpp) +int RGWBucketReshard::update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp) { - return ::create_new_bucket_instance(store, new_num_shards, - bucket_info, bucket_attrs, new_bucket_info, dpp); + return ::update_num_shards(store, new_num_shards, bucket_info, bucket_attrs, dpp); } int RGWBucketReshard::cancel(const DoutPrefixProvider *dpp) @@ -418,7 +404,6 @@ public: ldpp_dout(dpp, -1) << "Error: " << __func__ << " clear_index_shard_status returned " << ret << dendl; } - bucket_info.new_bucket_instance_id.clear(); // clears new_bucket_instance as well set_status(rgw::BucketReshardState::None, dpp); @@ -563,6 +548,9 @@ int RGWBucketReshard::do_reshard(int num_shards, return ret; } + //increment generation number + bucket_info.layout.target_index->gen++; + int num_target_shards = bucket_info.layout.target_index->layout.normal.num_shards; BucketReshardManager target_shards_mgr(dpp, store, bucket_info, num_target_shards); @@ -579,8 +567,9 @@ int RGWBucketReshard::do_reshard(int num_shards, (*out) << "total entries:"; } + auto current_shards = bucket_info.layout.current_index.layout.normal.num_shards; const int num_source_shards = - (bucket_info.layout.current_index.layout.normal.num_shards > 0 ? bucket_info.layout.current_index.layout.normal.num_shards : 1); + (current_shards > 0 ? current_shards : 1); string marker; for (int i = 0; i < num_source_shards; ++i) { bool is_truncated = true; @@ -589,9 +578,13 @@ int RGWBucketReshard::do_reshard(int num_shards, while (is_truncated) { entries.clear(); ret = store->getRados()->bi_list(dpp, bucket_info, i, null_object_filter, marker, max_entries, &entries, &is_truncated); - if (ret < 0 && ret != -ENOENT) { - derr << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl; - return ret; + if (ret < 0) { + if (ret == -ENOENT && i < (num_source_shards - 1)) { + continue; + } else { + derr << "ERROR: bi_list(): " << cpp_strerror(-ret) << dendl; + return ret; + } } for (auto iter = entries.begin(); iter != entries.end(); ++iter) { @@ -701,8 +694,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries, return ret; } - RGWBucketInfo new_bucket_info; - ret = create_new_bucket_instance(num_shards, new_bucket_info, dpp); + ret = update_num_shards(num_shards, dpp); if (ret < 0) { return ret; goto error_out; diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h index 09600de813599..a33af8f378392 100644 --- a/src/rgw/rgw_reshard.h +++ b/src/rgw/rgw_reshard.h @@ -84,9 +84,7 @@ private: // allocated in at once static const std::initializer_list reshard_primes; - int create_new_bucket_instance(int new_num_shards, - RGWBucketInfo& new_bucket_info, - const DoutPrefixProvider *dpp); + int update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp); int do_reshard(int num_shards, int max_entries, bool verbose, @@ -122,14 +120,15 @@ public: static int set_resharding_status(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, const RGWBucketInfo& bucket_info, - const std::string& new_instance_id, + const std::string& instance_id, int32_t num_shards, cls_rgw_reshard_status status); - int set_resharding_status(const DoutPrefixProvider *dpp, const std::string& new_instance_id, + int set_resharding_status(const DoutPrefixProvider *dpp, + const std::string& instance_id, int32_t num_shards, - cls_rgw_reshard_status status) { + cls_rgw_reshard_status status) { return set_resharding_status(dpp, store, bucket_info, - new_instance_id, num_shards, status); + instance_id, num_shards, status); } static uint32_t get_max_prime_shards() { @@ -222,8 +221,8 @@ protected: void stop(); CephContext *get_cct() const override; - unsigned get_subsys() const; - std::ostream& gen_prefix(std::ostream& out) const; + unsigned get_subsys() const override; + std::ostream& gen_prefix(std::ostream& out) const override; }; ReshardWorker *worker = nullptr; -- 2.39.5