From dfae5bec7ac5639cf877f21c499b0e41311cdfb1 Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Wed, 16 Jun 2021 12:32:25 +0300 Subject: [PATCH] rgw/multisite: don't delete per shard status on init and pass correct generation and num shards when deleting per shard status objects when disabling during incremental sync Signed-off-by: Yuval Lifshitz --- src/rgw/rgw_data_sync.cc | 53 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc index 9bf8cf1fcfe2c..be716e9e61a3f 100644 --- a/src/rgw/rgw_data_sync.cc +++ b/src/rgw/rgw_data_sync.cc @@ -2761,13 +2761,13 @@ public: RGWInitBucketShardSyncStatusCoroutine(RGWDataSyncCtx *_sc, const rgw_bucket_sync_pair_info& _sync_pair, rgw_bucket_shard_sync_info& _status, - uint64_t latest_gen, + uint64_t gen, const BucketIndexShardsManager& _marker_mgr, RGWObjVersionTracker& objv_tracker, bool exclusive) : RGWCoroutine(_sc->cct), sc(_sc), sync_env(_sc->env), sync_pair(_sync_pair), - sync_status_oid(RGWBucketPipeSyncStatusManager::inc_status_oid(sc->source_zone, _sync_pair, latest_gen)), + sync_status_oid(RGWBucketPipeSyncStatusManager::inc_status_oid(sc->source_zone, _sync_pair, gen)), status(_status), objv_tracker(objv_tracker), marker_mgr(_marker_mgr), exclusive(exclusive) {} @@ -3000,23 +3000,22 @@ class InitBucketShardStatusCR : public RGWCoroutine { rgw_bucket_sync_pair_info pair; rgw_bucket_shard_sync_info status; RGWObjVersionTracker objv; - const uint64_t latest_gen; + const uint64_t gen; const BucketIndexShardsManager& marker_mgr; public: InitBucketShardStatusCR(RGWDataSyncCtx* sc, const rgw_bucket_sync_pair_info& pair, - uint64_t latest_gen, + uint64_t gen, const BucketIndexShardsManager& marker_mgr) - : RGWCoroutine(sc->cct), sc(sc), pair(pair), latest_gen(latest_gen), marker_mgr(marker_mgr) + : RGWCoroutine(sc->cct), sc(sc), pair(pair), gen(gen), marker_mgr(marker_mgr) {} int operate(const DoutPrefixProvider *dpp) { reenter(this) { // non exclusive create with empty status objv.generate_new_write_ver(cct); - yield call(new RGWInitBucketShardSyncStatusCoroutine(sc, pair, status, latest_gen, marker_mgr, objv, false)); + yield call(new RGWInitBucketShardSyncStatusCoroutine(sc, pair, status, gen, marker_mgr, objv, false)); if (retcode < 0) { - assert(retcode != -EEXIST && retcode != -ECANCELED); return set_cr_error(retcode); } return set_cr_done(); @@ -3029,7 +3028,7 @@ class InitBucketShardStatusCollectCR : public RGWShardCollectCR { static constexpr int max_concurrent_shards = 16; RGWDataSyncCtx* sc; rgw_bucket_sync_pair_info sync_pair; - const uint64_t latest_gen; + const uint64_t gen; const BucketIndexShardsManager& marker_mgr; const int num_shards; @@ -3045,11 +3044,11 @@ class InitBucketShardStatusCollectCR : public RGWShardCollectCR { public: InitBucketShardStatusCollectCR(RGWDataSyncCtx* sc, const rgw_bucket_sync_pair_info& sync_pair, - uint64_t latest_gen, + uint64_t gen, const BucketIndexShardsManager& marker_mgr, int num_shards) : RGWShardCollectCR(sc->cct, max_concurrent_shards), - sc(sc), sync_pair(sync_pair), latest_gen(latest_gen), marker_mgr(marker_mgr), num_shards(num_shards) + sc(sc), sync_pair(sync_pair), gen(gen), marker_mgr(marker_mgr), num_shards(num_shards) {} bool spawn_next() override { @@ -3057,7 +3056,7 @@ class InitBucketShardStatusCollectCR : public RGWShardCollectCR { return false; } sync_pair.source_bs.shard_id = shard++; - spawn(new InitBucketShardStatusCR(sc, sync_pair, latest_gen, marker_mgr), false); + spawn(new InitBucketShardStatusCR(sc, sync_pair, gen, marker_mgr), false); return true; } }; @@ -3072,11 +3071,11 @@ class RemoveBucketShardStatusCR : public RGWCoroutine { public: RemoveBucketShardStatusCR(RGWDataSyncCtx* sc, - const rgw_bucket_sync_pair_info& sync_pair, uint64_t latest_gen) + const rgw_bucket_sync_pair_info& sync_pair, uint64_t gen) : RGWCoroutine(sc->cct), sc(sc), sync_env(sc->env), sync_pair(sync_pair), obj(sync_env->svc->zone->get_zone_params().log_pool, - RGWBucketPipeSyncStatusManager::inc_status_oid(sc->source_zone, sync_pair, latest_gen)) + RGWBucketPipeSyncStatusManager::inc_status_oid(sc->source_zone, sync_pair, gen)) {} int operate(const DoutPrefixProvider *dpp) override { @@ -3099,7 +3098,7 @@ class RemoveBucketShardStatusCollectCR : public RGWShardCollectCR { RGWDataSyncCtx* const sc; RGWDataSyncEnv* const sync_env; rgw_bucket_sync_pair_info sync_pair; - const uint64_t latest_gen; + const uint64_t gen; const int num_shards; int shard = 0; @@ -3114,18 +3113,18 @@ class RemoveBucketShardStatusCollectCR : public RGWShardCollectCR { public: RemoveBucketShardStatusCollectCR(RGWDataSyncCtx* sc, const rgw_bucket_sync_pair_info& sync_pair, - uint64_t latest_gen, + uint64_t gen, int num_shards) : RGWShardCollectCR(sc->cct, max_concurrent_shards), - sc(sc), sync_env(sc->env), sync_pair(sync_pair), latest_gen(latest_gen), num_shards(num_shards) + sc(sc), sync_env(sc->env), sync_pair(sync_pair), gen(gen), num_shards(num_shards) {} bool spawn_next() override { - if (shard >= num_shards || status < 0) { // stop spawning on any errors + if (shard >= num_shards) { return false; } sync_pair.source_bs.shard_id = shard++; - spawn(new RemoveBucketShardStatusCR(sc, sync_pair, latest_gen), false); + spawn(new RemoveBucketShardStatusCR(sc, sync_pair, gen), false); return true; } }; @@ -3174,7 +3173,6 @@ public: if (check_compat) { // try to convert existing per-shard incremental status for backward compatibility yield call(new CheckAllBucketShardStatusIsIncremental(sc, sync_pair, num_shards, &all_incremental)); - ldout(cct, 20) << "check for 'all incremental' in compatibility mode" << dendl; if (retcode < 0) { return set_cr_error(retcode); } @@ -5259,7 +5257,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp) bucket_status.state == BucketSyncState::Stopped || bucket_stopped) { // if state is Init or Stopped, we query the remote RGW for ther state - yield call(new RGWReadRemoteBucketIndexLogInfoCR(sc, sync_pair.dest_bucket, &info)); + yield call(new RGWReadRemoteBucketIndexLogInfoCR(sc, sync_pair.dest_bucket, &info)); if (retcode < 0) { return set_cr_error(retcode); } @@ -5282,6 +5280,15 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp) } } + // if state was incremental, remove all per-shard status objects + if (bucket_status.state == BucketSyncState::Incremental) { + yield { + const auto num_shards = bucket_status.shards_done_with_gen.size(); + const auto gen = bucket_status.incremental_gen; + call(new RemoveBucketShardStatusCollectCR(sc, sync_pair, gen, num_shards)); + } + } + // check if local state is "stopped" yield call(new ReadCR(dpp, env->async_rados, env->svc->sysobj, status_obj, &bucket_status, false, &objv)); @@ -5301,13 +5308,9 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp) return set_cr_error(retcode); } } - yield { - const int num_shards = sync_pipe.dest_bucket_info.layout.current_index.layout.normal.num_shards; - call(new RemoveBucketShardStatusCollectCR(sc, sync_pair, info.latest_gen, num_shards)); - } RELEASE_LOCK(bucket_lease_cr); return set_cr_done(); - } + } if (bucket_stopped) { tn->log(20, SSTR("ERROR: switched from 'stop' to 'start' sync. while state is: " << bucket_status.state)); bucket_stopped = false; -- 2.39.5