From: Shilpa Jagannath Date: Tue, 14 Mar 2023 02:14:24 +0000 (-0400) Subject: rgw/multisite: - add a lost_bid variable to handle is_highest_bidder() control flow X-Git-Tag: v19.0.0~1234^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=14c471382e2f9144c16100fc498a2db8e2da8f0f;p=ceph.git rgw/multisite: - add a lost_bid variable to handle is_highest_bidder() control flow - check for is_highest_bidder() before even attempting to take the lock - don't block on RGWMetaSyncShardNotifyCR() - other minor fixes Signed-off-by: Shilpa Jagannath --- diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 80a3cd488061..71def2e9de7f 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -2429,7 +2429,7 @@ options: - name: rgw_sync_lease_period type: int level: dev - default: 10_min + default: 2_min services: - rgw with_legacy: true @@ -2442,20 +2442,6 @@ options: services: - rgw with_legacy: true -- name: rgw_sync_work_period - type: uint - level: advanced - desc: Sync work period length - long_desc: Time in seconds a shard sync process will work before giving up lock. - default: 180 - min: 60 - services: - - rgw - with_legacy: true - - - - - name: rgw_sync_log_trim_max_buckets type: int level: advanced diff --git a/src/rgw/driver/rados/rgw_sync.cc b/src/rgw/driver/rados/rgw_sync.cc index e0743afc9f48..98ee31c98e12 100644 --- a/src/rgw/driver/rados/rgw_sync.cc +++ b/src/rgw/driver/rados/rgw_sync.cc @@ -1434,6 +1434,7 @@ class RGWMetaSyncShardCR : public RGWCoroutine { boost::intrusive_ptr lease_stack; bool lost_lock = false; + bool lost_bid = false; bool *reset_backoff; @@ -1554,6 +1555,12 @@ public: oid = full_sync_index_shard_oid(shard_id); can_adjust_marker = true; /* grab lock */ + + if (!sync_env->bid_manager->is_highest_bidder(shard_id)) { + tn->log(10, "not the highest bidder"); + return -EBUSY; + } + yield { uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; string lock_name = "sync_lock"; @@ -1590,61 +1597,15 @@ public: /* sync! */ do { if (!lease_cr->is_locked()) { - if (sync_env->bid_manager->is_highest_bidder(shard_id, ceph::coarse_mono_clock::now())) { - yield { - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - rgw::sal::RadosStore* store = sync_env->store; - - lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - lock_name, lock_duration, this, nullptr)); - lease_stack.reset(spawn(lease_cr.get(), false)); - lost_lock = false; - } - } else { - //lost the bid. exit gracefully. - lost_lock = true; - sync_marker.marker = marker; - ldpp_dout(sync_env->dpp, 5) << *this << ": saving marker " << marker << dendl; - - using WriteMarkerCR = RGWSimpleRadosWriteCR; - yield call(new WriteMarkerCR(sync_env->dpp, sync_env->store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - sync_marker)); - - if (retcode < 0) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to set sync marker: retcode=" << retcode << dendl; - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(retcode); - } - yield { - set_status("writing sync info"); - rgw::sal::RadosStore* store = sync_env->store; - call(new RGWSimpleRadosWriteCR(sync_env->dpp, store, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - sync_info)); - } - - if (retcode < 0) { - set_status("failed to write sync status"); - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to write sync status, retcode=" << retcode << dendl; - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(retcode); - } - - yield lease_cr->go_down(); - - lease_cr.reset(); - - drain_all(); - tn->log(5, "failed to take lease"); - return lease_cr->get_ret_status(); - set_sleeping(true); - yield; + tn->log(1, "lease is lost, abort"); + lost_lock = true; + break; } + + if (!sync_env->bid_manager->is_highest_bidder(shard_id)) { + tn->log(1, "lost bid"); + lost_bid = true; + break; } omapkeys = std::make_shared(); @@ -1694,7 +1655,9 @@ public: collect_children(); } - if (!lost_lock) { + if (lost_bid) { + yield call(marker_tracker->flush()); + } else if (!lost_lock) { /* update marker to reflect we're done with full sync */ if (can_adjust_marker) { // apply updates to a temporary marker, or operate() will send us @@ -1740,7 +1703,7 @@ public: return -EAGAIN; } - if (lost_lock) { + if (lost_lock || lost_bid) { return -EBUSY; } @@ -1762,6 +1725,12 @@ public: tn->log(10, "start incremental sync"); can_adjust_marker = true; /* grab lock */ + + if (!sync_env->bid_manager->is_highest_bidder(shard_id)) { + tn->log(10, "not the highest bidder"); + return -EBUSY; + } + if (!lease_cr) { /* could have had a lease_cr lock from previous state */ yield { uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; @@ -1808,62 +1777,17 @@ public: /* inc sync */ do { if (!lease_cr->is_locked()) { - if (sync_env->bid_manager->is_highest_bidder(shard_id, ceph::coarse_mono_clock::now())) { - yield { - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - rgw::sal::RadosStore* store = sync_env->store; - - lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - lock_name, lock_duration, this, nullptr)); - lease_stack.reset(spawn(lease_cr.get(), false)); - lost_lock = false; - } - } else { - //lost the bid. exit gracefully. - lost_lock = true; - sync_marker.marker = marker; - ldpp_dout(sync_env->dpp, 5) << *this << ": saving marker " << marker << dendl; - - using WriteMarkerCR = RGWSimpleRadosWriteCR; - yield call(new WriteMarkerCR(sync_env->dpp, sync_env->store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - sync_marker)); - - if (retcode < 0) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to set sync marker: retcode=" << retcode << dendl; - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(retcode); - } - yield { - set_status("writing sync info"); - rgw::sal::RadosStore* store = sync_env->store; - call(new RGWSimpleRadosWriteCR(sync_env->dpp, store, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - sync_info)); - } - - if (retcode < 0) { - set_status("failed to write sync status"); - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to write sync status, retcode=" << retcode << dendl; - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(retcode); - } - - yield lease_cr->go_down(); - - lease_cr.reset(); + lost_lock = true; + tn->log(1, "lease is lost, abort"); + break; + } - drain_all(); - tn->log(5, "failed to take lease"); - return lease_cr->get_ret_status(); - set_sleeping(true); - yield; - } + if (!sync_env->bid_manager->is_highest_bidder(shard_id)) { + tn->log(1, "lost bid"); + lost_bid = true; + break; } + #define INCREMENTAL_MAX_ENTRIES 100 ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << " truncated=" << truncated << dendl; if (!period_marker.empty() && period_marker <= mdlog_marker) { @@ -1961,7 +1885,7 @@ public: drain_all(); - if (lost_lock) { + if (lost_lock || lost_bid) { return -EBUSY; } @@ -2064,6 +1988,7 @@ class RGWMetaSyncCR : public RGWCoroutine { using ControlCRRef = boost::intrusive_ptr; using StackRef = boost::intrusive_ptr; using RefPair = std::pair; + RGWCoroutinesStack *notify_stack; map shard_crs; int ret{0}; @@ -2080,25 +2005,8 @@ public: int operate(const DoutPrefixProvider *dpp) override { reenter(this) { yield { - //create watch object for sync lock bids - const auto& control_pool = sync_env->store->svc()->zone->get_zone_params().control_pool; - auto control_obj = rgw_raw_obj{control_pool, meta_sync_bids_oid}; - - auto bid_manager = rgw::sync_fairness::create_rados_bid_manager( - sync_env->store, control_obj, sync_status.sync_info.num_shards); - ret = bid_manager->start(); - if (ret < 0) { - tn->log(0, SSTR("ERROR: failed to start bidding manager " << ret)); - return ret; - } - - //cr for broadcasting sync lock notifications in the background - sync_env->bid_manager = bid_manager.get(); - call(new RGWMetaSyncShardNotifyCR(sync_env, tn)); - if (retcode < 0) { - tn->log(5, SSTR("ERROR: failed to notify bidding information" << retcode)); - continue; - } + ldpp_dout(dpp, 10) << "broadcast sync lock notify" << dendl; + notify_stack = spawn(new RGWMetaSyncShardNotifyCR(sync_env, tn), false); } // loop through one period at a time @@ -2157,11 +2065,11 @@ public: } } // wait for each shard to complete - while (ret == 0 && num_spawned() > 0) { + while (ret == 0 && num_spawned() > 1) { yield wait_for_child(); collect(&ret, nullptr); } - drain_all(); + drain_all_but_stack(notify_stack); { // drop shard cr refs under lock std::lock_guard lock(mutex); @@ -2181,6 +2089,9 @@ public: rgw_raw_obj(pool, sync_env->status_oid()), sync_status.sync_info)); } + notify_stack->cancel(); + + drain_all(); } return 0; } @@ -2390,7 +2301,19 @@ int RGWRemoteMetaLog::run_sync(const DoutPrefixProvider *dpp, optional_yield y) if (num_shards != mdlog_info.num_shards) { ldpp_dout(dpp, -1) << "ERROR: can't sync, mismatch between num shards, master num_shards=" << mdlog_info.num_shards << " local num_shards=" << num_shards << dendl; return -EINVAL; - } + } + + // construct and start the bid manager for sync fairness + const auto& control_pool = store->svc()->zone->get_zone_params().control_pool; + auto control_obj = rgw_raw_obj{control_pool, meta_sync_bids_oid}; + + auto bid_manager = rgw::sync_fairness::create_rados_bid_manager( + store, control_obj, num_shards); + r = bid_manager->start(); + if (r < 0) { + return r; + } + sync_env.bid_manager = bid_manager.get(); RGWPeriodHistory::Cursor cursor; do { diff --git a/src/rgw/driver/rados/sync_fairness.cc b/src/rgw/driver/rados/sync_fairness.cc index bb2065f1d817..ded1cf56a719 100644 --- a/src/rgw/driver/rados/sync_fairness.cc +++ b/src/rgw/driver/rados/sync_fairness.cc @@ -32,7 +32,7 @@ using bid_value = uint16_t; using bid_vector = std::vector; // bid per replication log shard using notifier_id = uint64_t; -using bidder_map = std::unordered_map; +using bidder_map = boost::container::flat_map; struct BidRequest { bid_vector bids;