From 6d7f4ae52163f8e789433668b0335362b3519a3a Mon Sep 17 00:00:00 2001 From: Shilpa Manjarabad Jagannath Date: Thu, 9 Jul 2020 12:48:52 +0530 Subject: [PATCH] rgw: - make RGWBucketInfo a non-const reference in functions using guard_reshard() - remove unused 'instance_id' from cls_rgw_reshard_entry - other minor fixes Signed-off-by: Shilpa Manjarabad Jagannath --- src/rgw/rgw_admin.cc | 4 +-- src/rgw/rgw_bucket.cc | 2 +- src/rgw/rgw_rados.cc | 60 ++++++++++++++++++++++++------------------ src/rgw/rgw_rados.h | 59 ++++++++++++++++++++++++++--------------- src/rgw/rgw_reshard.cc | 30 +++++++++++---------- src/rgw/rgw_reshard.h | 2 +- 6 files changed, 92 insertions(+), 65 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index df670a2d8008f..05f4fd47ccd0b 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -6598,7 +6598,7 @@ next: int i = (specified_shard_id ? shard_id : 0); for (; i < max_shards; i++) { RGWRados::BucketShard bs(store->getRados()); - int ret = bs.init(bucket, i, index, std::nullopt, nullptr /* no RGWBucketInfo */, dpp()); + int ret = bs.init(bucket, i, index, nullptr /* no RGWBucketInfo */, dpp()); marker.clear(); if (ret < 0) { @@ -6662,7 +6662,7 @@ next: for (int i = 0; i < max_shards; i++) { RGWRados::BucketShard bs(store->getRados()); - int ret = bs.init(bucket, i, index, std::nullopt, nullptr /* no RGWBucketInfo */, dpp()); + int ret = bs.init(bucket, i, index, nullptr /* no RGWBucketInfo */, dpp()); if (ret < 0) { cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << i << "): " << cpp_strerror(-ret) << std::endl; return -ret; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index 178a402780d9f..0d6225c052a6c 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1629,7 +1629,7 @@ static int purge_bucket_instance(rgw::sal::RGWRadosStore *store, const RGWBucket for (int i = 0; i < max_shards; i++) { RGWRados::BucketShard bs(store->getRados()); int ret = bs.init(bucket_info.bucket, i, bucket_info.layout.current_index, - std::nullopt, nullptr, dpp); + nullptr, dpp); if (ret < 0) { cerr << "ERROR: bs.init(bucket=" << bucket_info.bucket << ", shard=" << i << "): " << cpp_strerror(-ret) << std::endl; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index ec44a4d836fd2..b0dcdb040e930 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -2556,7 +2556,9 @@ done_err: * fixes an issue where head objects were supposed to have a locator created, but ended * up without one */ -int RGWRados::fix_tail_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y) +int RGWRados::fix_tail_obj_locator(const DoutPrefixProvider *dpp, + RGWBucketInfo& bucket_info, rgw_obj_key& key, + bool fix, bool *need_fix, optional_yield y) { const rgw_bucket& bucket = bucket_info.bucket; rgw_obj obj(bucket, key); @@ -2665,8 +2667,7 @@ int RGWRados::BucketShard::init(const rgw_bucket& _bucket, } int RGWRados::BucketShard::init(const rgw_bucket& _bucket, - int sid, const rgw::bucket_index_layout_generation& current_layout, - std::optional target_layout, + int sid, std::optional idx_layout, RGWBucketInfo* bucket_info_out, const DoutPrefixProvider *dpp) { @@ -2686,13 +2687,9 @@ int RGWRados::BucketShard::init(const rgw_bucket& _bucket, string oid; - if (target_layout) { - ret = store->svc.bi_rados->open_bucket_index_shard(*bucket_info_p, shard_id, target_layout->layout.normal.num_shards, - target_layout->gen, &bucket_obj); - } else { - ret = store->svc.bi_rados->open_bucket_index_shard(*bucket_info_p, shard_id, current_layout.layout.normal.num_shards, - current_layout.gen, &bucket_obj); - } + ret = store->svc.bi_rados->open_bucket_index_shard(*bucket_info_p, shard_id, idx_layout->layout.normal.num_shards, + idx_layout->gen, &bucket_obj); + if (ret < 0) { ldout(store->ctx(), 0) << "ERROR: open_bucket_index_shard() returned ret=" << ret << dendl; return ret; @@ -4932,9 +4929,9 @@ int RGWRados::bucket_set_reshard(const RGWBucketInfo& bucket_info, const cls_rgw return CLSRGWIssueSetBucketResharding(index_pool.ioctx(), bucket_objs, entry, cct->_conf->rgw_bucket_index_max_aio)(); } -int RGWRados::defer_gc(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y) +int RGWRados::defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y) { - RGWObjectCtx *rctx = static_cast(ctx); std::string oid, key; get_obj_bucket_and_oid_loc(obj, oid, key); if (!rctx) @@ -5301,7 +5298,8 @@ static bool has_olh_tag(map& attrs) return (iter != attrs.end()); } -int RGWRados::get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, +int RGWRados::get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState *olh_state, RGWObjState **target_state, optional_yield y) { ceph_assert(olh_state->is_olh); @@ -5319,7 +5317,8 @@ int RGWRados::get_olh_target_state(const DoutPrefixProvider *dpp, RGWObjectCtx& return 0; } -int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, +int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, bool follow_olh, optional_yield y, bool assume_noent) { if (obj.empty()) { @@ -5497,7 +5496,8 @@ int RGWRados::get_obj_state_impl(const DoutPrefixProvider *dpp, RGWObjectCtx *rc return 0; } -int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, +int RGWRados::get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, bool follow_olh, optional_yield y, bool assume_noent) { int ret; @@ -5616,7 +5616,7 @@ int RGWRados::Object::Stat::finish() } int RGWRados::append_atomic_test(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, - const RGWBucketInfo& bucket_info, const rgw_obj& obj, + RGWBucketInfo& bucket_info, const rgw_obj& obj, ObjectOperation& op, RGWObjState **pstate, optional_yield y) { if (!rctx) @@ -5757,14 +5757,17 @@ int RGWRados::Object::prepare_atomic_modification(const DoutPrefixProvider *dpp, * bl: the contents of the attr * Returns: 0 on success, -ERR# otherwise. */ -int RGWRados::set_attr(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, const char *name, bufferlist& bl) +int RGWRados::set_attr(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, rgw_obj& obj, + const char *name, bufferlist& bl) { map attrs; attrs[name] = bl; - return set_attrs(dpp, ctx, bucket_info, obj, attrs, NULL, null_yield); + return set_attrs(dpp, rctx, bucket_info, obj, attrs, NULL, null_yield); } -int RGWRados::set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& src_obj, +int RGWRados::set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, rgw_obj& src_obj, map& attrs, map* rmattrs, optional_yield y) @@ -5779,7 +5782,6 @@ int RGWRados::set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucke if (r < 0) { return r; } - RGWObjectCtx *rctx = static_cast(ctx); ObjectWriteOperation op; RGWObjState *state = NULL; @@ -6450,7 +6452,7 @@ int RGWRados::Object::Read::iterate(const DoutPrefixProvider *dpp, int64_t ofs, } int RGWRados::iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, - const RGWBucketInfo& bucket_info, const rgw_obj& obj, + RGWBucketInfo& bucket_info, const rgw_obj& obj, off_t ofs, off_t end, uint64_t max_chunk_size, iterate_obj_cb cb, void *arg, optional_yield y) { @@ -7235,7 +7237,9 @@ int RGWRados::apply_olh_log(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx /* * read olh log and apply it */ -int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_zone_set *zones_trace) +int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, + RGWObjState *state, RGWBucketInfo& bucket_info, + const rgw_obj& obj, rgw_zone_set *zones_trace) { map > log; bool is_truncated; @@ -7255,7 +7259,9 @@ int RGWRados::update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, R return 0; } -int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& target_obj, bool delete_marker, rgw_bucket_dir_entry_meta *meta, +int RGWRados::set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, + RGWBucketInfo& bucket_info, const rgw_obj& target_obj, + bool delete_marker, rgw_bucket_dir_entry_meta *meta, uint64_t olh_epoch, real_time unmod_since, bool high_precision_time, optional_yield y, rgw_zone_set *zones_trace, bool log_data_change) { @@ -7485,7 +7491,9 @@ int RGWRados::remove_olh_pending_entries(const RGWBucketInfo& bucket_info, RGWOb return 0; } -int RGWRados::follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjectCtx& obj_ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target) +int RGWRados::follow_olh(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, + RGWObjectCtx& obj_ctx, RGWObjState *state, + const rgw_obj& olh_obj, rgw_obj *target) { map pending_entries; rgw_filter_attrset(state->attrset, RGW_ATTR_OLH_PENDING_PREFIX, &pending_entries); @@ -8139,7 +8147,7 @@ int RGWRados::bi_list(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket { BucketShard bs(this); int ret = bs.init(bucket_info.bucket, shard_id, bucket_info.layout.current_index, - std::nullopt, nullptr /* no RGWBucketInfo */, dpp); + nullptr /* no RGWBucketInfo */, dpp); if (ret < 0) { ldout(cct, 5) << "bs.init() returned ret=" << ret << dendl; return ret; @@ -8867,7 +8875,7 @@ int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list target_layout, + std::optional current_layout, RGWBucketInfo* out, const DoutPrefixProvider *dpp); int init(const RGWBucketInfo& bucket_info, const rgw_obj& obj); int init(const RGWBucketInfo& bucket_info, const rgw::bucket_index_layout_generation& idx_layout, int sid); @@ -1257,23 +1259,30 @@ public: * bl: the contents of the attr * Returns: 0 on success, -ERR# otherwise. */ - int set_attr(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, const char *name, bufferlist& bl); - - int set_attrs(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, rgw_obj& obj, - map& attrs, - map* rmattrs, - optional_yield y); - - int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, + int set_attr(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, rgw_obj& obj, + const char *name, bufferlist& bl); + + int set_attrs(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, rgw_obj& obj, + map& attrs, + map* rmattrs, + optional_yield y); + + int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, bool follow_olh, optional_yield y, bool assume_noent = false); - int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWObjState **state, optional_yield y) { + int get_obj_state(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, + RGWObjState **state, optional_yield y) { return get_obj_state(dpp, rctx, bucket_info, obj, state, true, y); } using iterate_obj_cb = int (*)(const rgw_raw_obj&, off_t, off_t, off_t, bool, RGWObjState*, void*); - int iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, const RGWBucketInfo& bucket_info, + int iterate_obj(const DoutPrefixProvider *dpp, RGWObjectCtx& ctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, off_t ofs, off_t end, uint64_t max_chunk_size, iterate_obj_cb cb, void *arg, optional_yield y); @@ -1336,8 +1345,12 @@ public: const rgw_obj& obj, bufferlist& obj_tag, map >& log, uint64_t *plast_ver, rgw_zone_set *zones_trace = nullptr); - int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, RGWObjState *state, const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_zone_set *zones_trace = nullptr); - int set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, const RGWBucketInfo& bucket_info, const rgw_obj& target_obj, bool delete_marker, rgw_bucket_dir_entry_meta *meta, + int update_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, + RGWObjState *state, RGWBucketInfo& bucket_info, + const rgw_obj& obj, rgw_zone_set *zones_trace = nullptr); + int set_olh(const DoutPrefixProvider *dpp, RGWObjectCtx& obj_ctx, + RGWBucketInfo& bucket_info, const rgw_obj& target_obj, + bool delete_marker, rgw_bucket_dir_entry_meta *meta, uint64_t olh_epoch, ceph::real_time unmod_since, bool high_precision_time, optional_yield y, rgw_zone_set *zones_trace = nullptr, bool log_data_change = false); int repair_olh(RGWObjState* state, const RGWBucketInfo& bucket_info, @@ -1347,7 +1360,9 @@ public: void check_pending_olh_entries(map& pending_entries, map *rm_pending_entries); int remove_olh_pending_entries(const RGWBucketInfo& bucket_info, RGWObjState& state, const rgw_obj& olh_obj, map& pending_attrs); - int follow_olh(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, RGWObjectCtx& ctx, RGWObjState *state, const rgw_obj& olh_obj, rgw_obj *target); + int follow_olh(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, + RGWObjectCtx& ctx, RGWObjState *state, + const rgw_obj& olh_obj, rgw_obj *target); int get_olh(const RGWBucketInfo& bucket_info, const rgw_obj& obj, RGWOLHInfo *olh); void gen_rand_obj_instance_name(rgw_obj_key *target_key); @@ -1479,7 +1494,8 @@ public: int list_gc_objs(int *index, string& marker, uint32_t max, bool expired_only, std::list& result, bool *truncated, bool& processing_queue); int process_gc(bool expired_only); bool process_expire_objects(const DoutPrefixProvider *dpp); - int defer_gc(const DoutPrefixProvider *dpp, void *ctx, const RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); + int defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, + RGWBucketInfo& bucket_info, const rgw_obj& obj, optional_yield y); int process_lc(); int list_lc_progress(string& marker, uint32_t max_entries, @@ -1496,7 +1512,8 @@ public: librados::IoCtx& dst_ioctx, const string& dst_oid, const string& dst_locator); int fix_head_obj_locator(const RGWBucketInfo& bucket_info, bool copy_obj, bool remove_bad, rgw_obj_key& key); - int fix_tail_obj_locator(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y); + int fix_tail_obj_locator(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, + rgw_obj_key& key, bool fix, bool *need_fix, optional_yield y); int check_quota(const rgw_user& bucket_owner, rgw_bucket& bucket, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, @@ -1533,7 +1550,7 @@ public: */ int check_disk_state(const DoutPrefixProvider *dpp, librados::IoCtx io_ctx, - const RGWBucketInfo& bucket_info, + RGWBucketInfo& bucket_info, rgw_bucket_dir_entry& list_state, rgw_bucket_dir_entry& object, bufferlist& suggested_updates, diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 1e99c067216e3..44d93ac781749 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -109,8 +109,8 @@ public: { num_shard = (bucket_info.layout.target_index->layout.normal.num_shards > 0 ? _num_shard : -1); - bs.init(bucket_info.bucket, num_shard, bucket_info.layout.current_index, - bucket_info.layout.target_index, nullptr /* no RGWBucketInfo */, dpp); + bs.init(bucket_info.bucket, num_shard, bucket_info.layout.target_index, + nullptr /* no RGWBucketInfo */, dpp); max_aio_completions = store->ctx()->_conf.get_val("rgw_reshard_max_aio"); @@ -317,18 +317,18 @@ int RGWBucketReshard::clear_index_shard_reshard_status(rgw::sal::RGWRadosStore* return 0; } -static int update_num_shards(rgw::sal::RGWRadosStore *store, - int new_num_shards, - RGWBucketInfo& bucket_info, - map& attrs, - const DoutPrefixProvider *dpp) +static int set_target_layout(rgw::sal::RGWRadosStore *store, + int new_num_shards, + RGWBucketInfo& bucket_info, + map& attrs, + const DoutPrefixProvider *dpp) { - if (!bucket_info.layout.target_index) { - bucket_info.layout.target_index.emplace(); - } + assert(!bucket_info.layout.target_index); + bucket_info.layout.target_index.emplace(); + bucket_info.layout.target_index->layout.normal.num_shards = new_num_shards; - bucket_info.layout.resharding = rgw::BucketReshardState::None; + bucket_info.layout.resharding = rgw::BucketReshardState::InProgress; int ret = store->getRados()->put_bucket_instance_info(bucket_info, true, real_time(), &attrs, dpp); if (ret < 0) { @@ -339,9 +339,9 @@ static int update_num_shards(rgw::sal::RGWRadosStore *store, return 0; } -int RGWBucketReshard::update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp) +int RGWBucketReshard::set_target_layout(int new_num_shards, const DoutPrefixProvider *dpp) { - return ::update_num_shards(store, new_num_shards, bucket_info, bucket_attrs, dpp); + return ::set_target_layout(store, new_num_shards, bucket_info, bucket_attrs, dpp); } int RGWBucketReshard::cancel() @@ -523,6 +523,7 @@ int RGWBucketReshard::do_reshard(int num_shards, // complete successfully BucketInfoReshardUpdate bucket_info_updater(dpp, store, bucket_info, bucket_attrs); + int ret = bucket_info_updater.start(); if (ret < 0) { ldout(store->ctx(), 0) << __func__ << ": failed to update bucket info ret=" << ret << dendl; @@ -643,6 +644,7 @@ int RGWBucketReshard::do_reshard(int num_shards, //overwrite current_index for the next reshard process bucket_info.layout.current_index = *bucket_info.layout.target_index; + bucket_info.layout.target_index = std::nullopt; // target_layout doesn't need to exist after reshard bucket_info.layout.resharding = rgw::BucketReshardState::None; ret = store->getRados()->put_bucket_instance_info(bucket_info, false, real_time(), &bucket_attrs, dpp); if (ret < 0) { @@ -670,7 +672,7 @@ int RGWBucketReshard::execute(int num_shards, int max_op_entries, return ret; } - ret = update_num_shards(num_shards, dpp); + ret = set_target_layout(num_shards, dpp); //modifies existing bucket if (ret < 0) { // shard state is uncertain, but this will attempt to remove them anyway goto error_out; diff --git a/src/rgw/rgw_reshard.h b/src/rgw/rgw_reshard.h index bec9cbda5169c..02a1e7d464fa2 100644 --- a/src/rgw/rgw_reshard.h +++ b/src/rgw/rgw_reshard.h @@ -84,7 +84,7 @@ private: // allocated in at once static const std::initializer_list reshard_primes; - int update_num_shards(int new_num_shards, const DoutPrefixProvider *dpp); + int set_target_layout(int new_num_shards, const DoutPrefixProvider *dpp); int do_reshard(int num_shards, int max_entries, bool verbose, -- 2.39.5