From: Radoslaw Zarzynski Date: Tue, 21 Apr 2020 18:38:31 +0000 (+0200) Subject: crimson/osd: rework handling of MOSDAlive. X-Git-Tag: wip-pdonnell-testing-20200918.022351~657^2~22 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=1f8ede3dcf3f1c2fb7673a2c94c2afc15c643558;p=ceph-ci.git crimson/osd: rework handling of MOSDAlive. 1. Migrate `MOSDAlive` crafting from `OSD` to `ShardServices`. 2. `send_alive()` takes the `want` epoch from callers instead of assuming it matches current OSDMap. Callers pass `pg_history_t::same_interval_since`. The passed epoch is handled similarly to `queue_want_up_thru()` in the classical. The message is sent once and when the desired epoch is greater than the one returned by `OSDMap::get_up_thru()`. 3. Send the message also when completing the `PeeringEvent`. Signed-off-by: Radoslaw Zarzynski --- diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 3d60c2a8c61..5725ad7b46d 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -16,7 +16,6 @@ #include "include/util.h" #include "messages/MCommand.h" -#include "messages/MOSDAlive.h" #include "messages/MOSDBeacon.h" #include "messages/MOSDBoot.h" #include "messages/MOSDMap.h" @@ -83,7 +82,7 @@ OSD::OSD(int id, uint32_t nonce, local_conf().get_val("osd_objectstore"), local_conf().get_val("osd_data"), local_conf().get_config_values())}, - shard_services{*this, *cluster_msgr, *public_msgr, *monc, *mgrc, *store}, + shard_services{*this, whoami, *cluster_msgr, *public_msgr, *monc, *mgrc, *store}, heartbeat{new Heartbeat{whoami, shard_services, *monc, hb_front_msgr, hb_back_msgr}}, // do this in background heartbeat_timer{[this] { update_heartbeat_peers(); }}, @@ -406,27 +405,6 @@ seastar::future<> OSD::_add_me_to_crush() }); } -seastar::future<> OSD::_send_alive() -{ - auto want = osdmap->get_epoch(); - logger().info( - "{} want {} up_thru_wanted {}", - __func__, - want, - up_thru_wanted); - if (!osdmap->exists(whoami)) { - logger().warn("{} DNE", __func__); - return seastar::now(); - } else if (want <= up_thru_wanted) { - logger().debug("{} {} <= {}", __func__, want, up_thru_wanted); - return seastar::now(); - } else { - up_thru_wanted = want; - auto m = make_message(osdmap->get_epoch(), want); - return monc->send_message(std::move(m)); - } -} - seastar::future<> OSD::handle_command(crimson::net::Connection* conn, Ref m) { diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index afee53d69de..93f58eb65cb 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -155,9 +155,6 @@ private: seastar::future> load_pg(spg_t pgid); seastar::future<> load_pgs(); - epoch_t up_thru_wanted = 0; - seastar::future<> _send_alive(); - // OSDMapService methods epoch_t get_up_epoch() const final { return up_epoch; diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc index 8c4cfbc89b6..b9eb6ec7a55 100644 --- a/src/crimson/osd/osd_operations/peering_event.cc +++ b/src/crimson/osd/osd_operations/peering_event.cc @@ -80,6 +80,9 @@ seastar::future<> PeeringEvent::start() pg->do_peering_event(evt, ctx); handle.exit(); return complete_rctx(pg); + }).then([this, pg] { + return pg->get_need_up_thru() ? shard_services.send_alive(pg->get_same_interval_since()) + : seastar::now(); }); } }).then([this, ref=std::move(ref)] { diff --git a/src/crimson/osd/osd_operations/pg_advance_map.cc b/src/crimson/osd/osd_operations/pg_advance_map.cc index 64eacea6806..a1092e9b375 100644 --- a/src/crimson/osd/osd_operations/pg_advance_map.cc +++ b/src/crimson/osd/osd_operations/pg_advance_map.cc @@ -80,7 +80,9 @@ seastar::future<> PGAdvanceMap::start() logger().info("PGAdvanceMap::start new pg {}", *pg); } return seastar::when_all_succeed( - pg->get_need_up_thru() ? osd._send_alive() : seastar::now(), + pg->get_need_up_thru() \ + ? osd.shard_services.send_alive(pg->get_same_interval_since()) + : seastar::now(), osd.shard_services.dispatch_context( pg->get_collection_ref(), std::move(rctx))); diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h index 9dc10cf3d6c..8c94590ec1b 100644 --- a/src/crimson/osd/pg.h +++ b/src/crimson/osd/pg.h @@ -449,6 +449,9 @@ public: bool get_need_up_thru() const { return peering_state.get_need_up_thru(); } + epoch_t get_same_interval_since() const { + return get_info().history.same_interval_since; + } const auto& get_pool() const { return peering_state.get_pool(); diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc index 7aa8ede6bed..b52cdfb7ecb 100644 --- a/src/crimson/osd/shard_services.cc +++ b/src/crimson/osd/shard_services.cc @@ -3,6 +3,8 @@ #include "crimson/osd/shard_services.h" +#include "messages/MOSDAlive.h" + #include "osd/osd_perf_counters.h" #include "osd/PeeringState.h" #include "crimson/common/config_proxy.h" @@ -28,12 +30,14 @@ namespace crimson::osd { ShardServices::ShardServices( OSDMapService &osdmap_service, + const int whoami, crimson::net::Messenger &cluster_msgr, crimson::net::Messenger &public_msgr, crimson::mon::Client &monc, crimson::mgr::Client &mgrc, crimson::os::FuturizedStore &store) : osdmap_service(osdmap_service), + whoami(whoami), cluster_msgr(cluster_msgr), public_msgr(public_msgr), monc(monc), @@ -283,4 +287,33 @@ HeartbeatStampsRef ShardServices::get_hb_stamps(int peer) return stamps->second; } +seastar::future<> ShardServices::send_alive(const epoch_t want) +{ + logger().info( + "{} want={} up_thru_wanted={}", + __func__, + want, + up_thru_wanted); + + if (want > up_thru_wanted) { + up_thru_wanted = want; + } else { + logger().debug("{} want={} <= up_thru_wanted={}; skipping", + __func__, want, up_thru_wanted); + return seastar::now(); + } + if (!osdmap->exists(whoami)) { + logger().warn("{} DNE", __func__); + return seastar::now(); + } if (const epoch_t up_thru = osdmap->get_up_thru(whoami); + up_thru_wanted > up_thru) { + logger().debug("{} up_thru_wanted={} up_thru={}", __func__, want, up_thru); + return monc.send_message( + make_message(osdmap->get_epoch(), want)); + } else { + logger().debug("{} {} <= {}", __func__, want, osdmap->get_up_thru(whoami)); + return seastar::now(); + } +} + }; diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index 38d5d192ebd..934e9143711 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -44,6 +44,7 @@ namespace crimson::osd { class ShardServices : public md_config_obs_t { using cached_map_t = boost::local_shared_ptr; OSDMapService &osdmap_service; + const int whoami; crimson::net::Messenger &cluster_msgr; crimson::net::Messenger &public_msgr; crimson::mon::Client &monc; @@ -61,6 +62,7 @@ class ShardServices : public md_config_obs_t { public: ShardServices( OSDMapService &osdmap_service, + const int whoami, crimson::net::Messenger &cluster_msgr, crimson::net::Messenger &public_msgr, crimson::mon::Client &monc, @@ -204,6 +206,11 @@ private: public: AsyncReserver local_reserver; AsyncReserver remote_reserver; + +private: + epoch_t up_thru_wanted = 0; +public: + seastar::future<> send_alive(epoch_t want); }; }