From 35a9ca54dd708e96d83ae9b7603fb9c63d47f9cf Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Thu, 21 May 2020 18:29:15 +0530 Subject: [PATCH] rgw/dynamic-resharding: Allow clean_index to take generation number Signed-off-by: Shilpa Jagannath --- src/rgw/rgw_rados.cc | 3 +- src/rgw/rgw_reshard.cc | 12 ++++-- src/rgw/services/svc_bi.h | 4 +- src/rgw/services/svc_bi_rados.cc | 63 +++++++++++++++++++++++++------- src/rgw/services/svc_bi_rados.h | 3 +- 5 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index c6e2d47952165..0ac39d42d7afa 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2399,7 +2399,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, /* only remove it if it's a different bucket instance */ if (orig_info.bucket.bucket_id != bucket.bucket_id) { - int r = svc.bi->clean_index(dpp, info); + int r = svc.bi->clean_index(dpp, info, std::nullopt); if (r < 0) { ldpp_dout(dpp, 0) << "WARNING: could not remove bucket index (r=" << r << ")" << dendl; } @@ -9253,7 +9253,6 @@ int RGWRados::cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo } r = CLSRGWIssueGetDirHeader(index_pool.ioctx(), oids, list_results, cct->_conf->rgw_bucket_index_max_aio)(); - ldout(cct, 20) << "CLSRGWIssueGetDirHeader issued" << dendl; if (r < 0) { ldpp_dout(dpp, 20) << "cls_bucket_head: CLSRGWIssueGetDirHeader() returned " << r << dendl; diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 5576276844237..9eb22ef746e99 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -696,14 +696,13 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries, ret = update_num_shards(num_shards, dpp); if (ret < 0) { - return ret; + // shard state is uncertain, but this will attempt to remove them anyway goto error_out; } if (reshard_log) { ret = reshard_log->update(dpp, bucket_info); if (ret < 0) { - return ret; goto error_out; } } @@ -734,7 +733,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries, // at this point since all we're using a best effort to remove old // shard objects - ret = store->svc()->bi->clean_index(dpp, bucket_info); + ret = store->svc()->bi->clean_index(dpp, bucket_info, std::nullopt); if (ret < 0) { lderr(store->ctx()) << "Error: " << __func__ << " failed to clean up old shards; " << @@ -755,7 +754,7 @@ error_out: // since the real problem is the issue that led to this error code // path, we won't touch ret and instead use another variable to // temporarily error codes - int ret2 = store->svc()->bi->clean_index(dpp, bucket_info); + int ret2 = store->svc()->bi->clean_index(dpp, bucket_info, std::nullopt); if (ret2 < 0) { ldpp_dout(dpp, -1) << "Error: " << __func__ << " failed to clean up shards from failed incomplete resharding; " << @@ -795,6 +794,11 @@ void RGWReshard::get_bucket_logshard_oid(const string& tenant, const string& buc int RGWReshard::add(const DoutPrefixProvider *dpp, cls_rgw_reshard_entry& entry) { + if (!store->svc()->zone->can_reshard()) { + ldpp_dout(dpp, 20) << __func__ << " Resharding is disabled" << dendl; + return 0; + } + string logshard_oid; get_bucket_logshard_oid(entry.tenant, entry.bucket_name, &logshard_oid); diff --git a/src/rgw/services/svc_bi.h b/src/rgw/services/svc_bi.h index abb68e3941840..e1d009ab3502e 100644 --- a/src/rgw/services/svc_bi.h +++ b/src/rgw/services/svc_bi.h @@ -30,14 +30,14 @@ public: virtual ~RGWSI_BucketIndex() {} virtual int init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info) = 0; - virtual int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info) = 0; + virtual int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, std::optional gen) = 0; virtual int read_stats(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWBucketEnt *stats, optional_yield y) = 0; - virtual int handle_overwrite(const DoutPrefixProvider *dpp, + virtual int handle_overwrite(const DoutPrefixProvider *dpp, const RGWBucketInfo& info, const RGWBucketInfo& orig_info) = 0; }; diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc index 0ae738767dde8..7d2cea00e64df 100644 --- a/src/rgw/services/svc_bi_rados.cc +++ b/src/rgw/services/svc_bi_rados.cc @@ -118,6 +118,45 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp, return 0; } +static void get_bucket_index_objects(const string& bucket_oid_base, + uint32_t num_shards, std::optional _gen_id, + map *_bucket_objects, + int shard_id = -1) +{ + auto& bucket_objects = *_bucket_objects; + auto gen_id = _gen_id.value_or(0); + if (!num_shards) { + bucket_objects[0] = bucket_oid_base; + } else { + char buf[bucket_oid_base.size() + 64]; + if (shard_id < 0) { + for (uint32_t i = 0; i < num_shards; ++i) { + if (gen_id) { + snprintf(buf, sizeof(buf), "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, i); + bucket_objects[i] = buf; + } else { + snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), i); + bucket_objects[i] = buf; + } + } + } else { + if ((uint32_t)shard_id > num_shards) { + return; + } else { + if (gen_id) { + snprintf(buf, sizeof(buf), "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, shard_id); + bucket_objects[shard_id] = buf; + } else { + // for backward compatibility, gen_id(0) will not be added in the object name + snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id); + bucket_objects[shard_id] = buf; + } + } + } + } +} + +/* static void get_bucket_index_objects(const string& bucket_oid_base, uint32_t num_shards, map *_bucket_objects, @@ -142,6 +181,7 @@ static void get_bucket_index_objects(const string& bucket_oid_base, } } } +*/ static void get_bucket_instance_ids(const RGWBucketInfo& bucket_info, int shard_id, @@ -186,10 +226,8 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index(const DoutPrefixProvider *dpp, return ret; } - //auto gen = bucket_info.layout.current_index.gen; - - // TODO: need reshard changes to add gen_id here - get_bucket_index_objects(bucket_oid_base, bucket_info.layout.current_index.layout.normal.num_shards, bucket_objs, shard_id); + get_bucket_index_objects(bucket_oid_base, idx_layout.layout.normal.num_shards, + idx_layout.gen, bucket_objs, shard_id); if (bucket_instance_ids) { // TODO: generation need to be passed here get_bucket_instance_ids(bucket_info, shard_id, bucket_instance_ids); @@ -209,8 +247,9 @@ void RGWSI_BucketIndex_RADOS::get_bucket_index_object(const string& bucket_oid_b } else { char buf[bucket_oid_base.size() + 64]; if (gen_id != 0) { - snprintf(buf, sizeof(buf), "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, shard_id); + snprintf(buf, sizeof(buf), "%s.%" PRIu64 ".%d", bucket_oid_base.c_str(), gen_id, shard_id); (*bucket_obj) = buf; + ldout(cct, 10) << "bucket_obj is " << (*bucket_obj) << dendl; } else { // for backward compatibility, gen_id(0) will not be added in the object name snprintf(buf, sizeof(buf), "%s.%d", bucket_oid_base.c_str(), shard_id); @@ -282,7 +321,7 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *d int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, - const rgw::bucket_index_layout_generation& idx_layout, + const rgw::bucket_index_layout_generation& target_layout, RGWSI_RADOS::Obj *bucket_obj) { RGWSI_RADOS::Pool index_pool; @@ -296,8 +335,8 @@ int RGWSI_BucketIndex_RADOS::open_bucket_index_shard(const DoutPrefixProvider *d string oid; - get_bucket_index_object(bucket_oid_base, idx_layout.layout.normal.num_shards, - shard_id, idx_layout.gen, &oid); + get_bucket_index_object(bucket_oid_base, target_layout.layout.normal.num_shards, + shard_id, target_layout.gen, &oid); *bucket_obj = svc.rados->obj(index_pool, oid); @@ -334,7 +373,6 @@ int RGWSI_BucketIndex_RADOS::cls_bucket_head(const DoutPrefixProvider *dpp, return 0; } - int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info) { RGWSI_RADOS::Pool index_pool; @@ -348,14 +386,14 @@ int RGWSI_BucketIndex_RADOS::init_index(const DoutPrefixProvider *dpp, RGWBucket dir_oid.append(bucket_info.bucket.bucket_id); map bucket_objs; - get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, &bucket_objs); + get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, std::nullopt, &bucket_objs); return CLSRGWIssueBucketIndexInit(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); } -int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info) +int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, std::optional gen) { RGWSI_RADOS::Pool index_pool; @@ -368,7 +406,7 @@ int RGWSI_BucketIndex_RADOS::clean_index(const DoutPrefixProvider *dpp, RGWBucke dir_oid.append(bucket_info.bucket.bucket_id); std::map bucket_objs; - get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, &bucket_objs); + get_bucket_index_objects(dir_oid, bucket_info.layout.current_index.layout.normal.num_shards, gen.value_or(0), &bucket_objs); return CLSRGWIssueBucketIndexClean(index_pool.ioctx(), bucket_objs, @@ -474,4 +512,3 @@ int RGWSI_BucketIndex_RADOS::handle_overwrite(const DoutPrefixProvider *dpp, return 0; } - diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h index 5912b380c8ad4..91ab953b43e1a 100644 --- a/src/rgw/services/svc_bi_rados.h +++ b/src/rgw/services/svc_bi_rados.h @@ -98,8 +98,7 @@ public: } int init_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info); - int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info); - + int clean_index(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, std::optional gen); /* RADOS specific */ -- 2.39.5