From: Casey Bodley Date: Fri, 17 Mar 2017 13:55:47 +0000 (-0400) Subject: rgw: store realm epoch with sync status markers X-Git-Tag: v12.0.3~225^2~6 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=574ff5f5df62a0a1e9a8ff7e4a87cd65cc952d7e;p=ceph-ci.git rgw: store realm epoch with sync status markers sync status markers can't be compared between periods, so we need to record the current period's realm epoch with its markers. when the rgw_meta_sync_info.realm_epoch is more recent than the marker's realm_epoch, we must treat the marker as empty Signed-off-by: Casey Bodley --- diff --git a/src/rgw/rgw_json_enc.cc b/src/rgw/rgw_json_enc.cc index 067ad6fadbb..2a183b59195 100644 --- a/src/rgw/rgw_json_enc.cc +++ b/src/rgw/rgw_json_enc.cc @@ -1290,6 +1290,7 @@ void rgw_meta_sync_marker::decode_json(JSONObj *obj) utime_t ut; JSONDecoder::decode_json("timestamp", ut, obj); timestamp = ut.to_real_time(); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); } void rgw_meta_sync_marker::dump(Formatter *f) const @@ -1300,6 +1301,7 @@ void rgw_meta_sync_marker::dump(Formatter *f) const encode_json("total_entries", total_entries, f); encode_json("pos", pos, f); encode_json("timestamp", utime_t(timestamp), f); + encode_json("realm_epoch", realm_epoch, f); } void rgw_meta_sync_status::decode_json(JSONObj *obj) diff --git a/src/rgw/rgw_meta_sync_status.h b/src/rgw/rgw_meta_sync_status.h index e913e8ffb21..e34bb05be1d 100644 --- a/src/rgw/rgw_meta_sync_status.h +++ b/src/rgw/rgw_meta_sync_status.h @@ -55,28 +55,33 @@ struct rgw_meta_sync_marker { uint64_t total_entries; uint64_t pos; real_time timestamp; + epoch_t realm_epoch{0}; //< realm_epoch of period marker rgw_meta_sync_marker() : state(FullSync), total_entries(0), pos(0) {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(state, bl); ::encode(marker, bl); ::encode(next_step_marker, bl); ::encode(total_entries, bl); ::encode(pos, bl); ::encode(timestamp, bl); + ::encode(realm_epoch, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(state, bl); ::decode(marker, bl); ::decode(next_step_marker, bl); ::decode(total_entries, bl); ::decode(pos, bl); ::decode(timestamp, bl); + if (struct_v >= 2) { + ::decode(realm_epoch, bl); + } DECODE_FINISH(bl); } diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index d8de8e5d2cf..cac266f2278 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -1104,7 +1104,7 @@ public: sync_marker.timestamp = timestamp; } - ldout(sync_env->cct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << dendl; + ldout(sync_env->cct, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << " realm_epoch=" << sync_marker.realm_epoch << dendl; RGWRados *store = sync_env->store; return new RGWSimpleRadosWriteCR(sync_env->async_rados, store, @@ -1245,6 +1245,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine { const rgw_pool& pool; const std::string& period; //< currently syncing period id + const epoch_t realm_epoch; //< realm_epoch of period RGWMetadataLog* mdlog; //< log of syncing period uint32_t shard_id; rgw_meta_sync_marker& sync_marker; @@ -1292,11 +1293,13 @@ class RGWMetaSyncShardCR : public RGWCoroutine { public: RGWMetaSyncShardCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, RGWMetadataLog* mdlog, - uint32_t _shard_id, rgw_meta_sync_marker& _marker, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + rgw_meta_sync_marker& _marker, const std::string& period_marker, bool *_reset_backoff) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), pool(_pool), - period(period), mdlog(mdlog), shard_id(_shard_id), sync_marker(_marker), + period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), period_marker(period_marker), inc_lock("RGWMetaSyncShardCR::inc_lock"), reset_backoff(_reset_backoff) { *reset_backoff = false; @@ -1479,7 +1482,8 @@ public: temp_marker->state = rgw_meta_sync_marker::IncrementalSync; temp_marker->marker = std::move(temp_marker->next_step_marker); temp_marker->next_step_marker.clear(); - ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << dendl; + temp_marker->realm_epoch = realm_epoch; + ldout(sync_env->cct, 0) << *this << ": saving marker pos=" << temp_marker->marker << " realm_epoch=" << realm_epoch << dendl; using WriteMarkerCR = RGWSimpleRadosWriteCR; yield call(new WriteMarkerCR(sync_env->async_rados, sync_env->store, @@ -1550,6 +1554,14 @@ public: yield; } } + // if the period has advanced, we can't use the existing marker + if (sync_marker.realm_epoch < realm_epoch) { + ldout(sync_env->cct, 0) << "clearing marker=" << sync_marker.marker + << " from old realm_epoch=" << sync_marker.realm_epoch + << " (now " << realm_epoch << ')' << dendl; + sync_marker.realm_epoch = realm_epoch; + sync_marker.marker.clear(); + } mdlog_marker = sync_marker.marker; set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, sync_env->shard_obj_name(shard_id), @@ -1571,7 +1583,8 @@ public: } #define INCREMENTAL_MAX_ENTRIES 100 ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; - if (!period_marker.empty() && period_marker <= marker) { + if (!period_marker.empty() && period_marker <= mdlog_marker) { + ldout(cct, 10) << "mdlog_marker past period_marker=" << period_marker << dendl; done_with_period = true; break; } @@ -1638,6 +1651,7 @@ public: ldout(sync_env->cct, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " max_marker=" << max_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; if (done_with_period) { // return control to RGWMetaSyncCR and advance to the next period + ldout(sync_env->cct, 10) << *this << ": done with period" << dendl; break; } if (mdlog_marker == max_marker && can_adjust_marker) { @@ -1662,6 +1676,8 @@ public: if (!can_adjust_marker) { return -EAGAIN; } + + return set_cr_done(); } /* TODO */ return 0; @@ -1674,6 +1690,7 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR const rgw_pool& pool; const std::string& period; + epoch_t realm_epoch; RGWMetadataLog* mdlog; uint32_t shard_id; rgw_meta_sync_marker sync_marker; @@ -1682,16 +1699,18 @@ class RGWMetaSyncShardControlCR : public RGWBackoffControlCR static constexpr bool exit_on_error = false; // retry on all errors public: RGWMetaSyncShardControlCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, RGWMetadataLog* mdlog, - uint32_t _shard_id, const rgw_meta_sync_marker& _marker, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + const rgw_meta_sync_marker& _marker, std::string&& period_marker) : RGWBackoffControlCR(_sync_env->cct, exit_on_error), sync_env(_sync_env), - pool(_pool), period(period), mdlog(mdlog), shard_id(_shard_id), - sync_marker(_marker), period_marker(std::move(period_marker)) {} + pool(_pool), period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), + period_marker(std::move(period_marker)) {} RGWCoroutine *alloc_cr() override { - return new RGWMetaSyncShardCR(sync_env, pool, period, mdlog, shard_id, - sync_marker, period_marker, backoff_ptr()); + return new RGWMetaSyncShardCR(sync_env, pool, period, realm_epoch, mdlog, + shard_id, sync_marker, period_marker, backoff_ptr()); } RGWCoroutine *alloc_finisher_cr() override { @@ -1750,6 +1769,7 @@ public: yield { // get the mdlog for the current period (may be empty) auto& period_id = sync_status.sync_info.period; + auto realm_epoch = sync_status.sync_info.realm_epoch; auto mdlog = sync_env->store->meta_mgr->get_log(period_id); // prevent wakeup() from accessing shard_crs while we're spawning them @@ -1772,9 +1792,10 @@ public: } } - auto cr = new RGWMetaSyncShardControlCR(sync_env, pool, period_id, - mdlog, shard_id, marker, - std::move(period_marker)); + using ShardCR = RGWMetaSyncShardControlCR; + auto cr = new ShardCR(sync_env, pool, period_id, realm_epoch, + mdlog, shard_id, marker, + std::move(period_marker)); auto stack = spawn(cr, false); shard_crs[shard_id] = RefPair{cr, stack}; }