From: Samuel Just Date: Tue, 23 Jul 2019 03:07:57 +0000 (-0700) Subject: crimson/: move get_mnow() to ShardServices, pass to heartbeat X-Git-Tag: v15.1.0~1952^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=8f906e3cd75a7dceb1159e54ab2590b1a2a313a7;p=ceph.git crimson/: move get_mnow() to ShardServices, pass to heartbeat get_mnow isn't clearly at home in OSDMapService, and the other methods are needed from PeeringState, so let's consolidate on ShardServices for now. We probably ought OSDMapService state out of OSD into its own module at some point. Signed-off-by: Samuel Just --- diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index 808037c2d421..367c2ffce789 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -8,7 +8,7 @@ #include "crimson/common/config_proxy.h" #include "crimson/net/Connection.h" #include "crimson/net/Messenger.h" -#include "crimson/osd/osdmap_service.h" +#include "crimson/osd/shard_services.h" #include "crimson/mon/MonClient.h" #include "osd/OSDMap.h" @@ -21,7 +21,7 @@ namespace { } } -Heartbeat::Heartbeat(const OSDMapService& service, +Heartbeat::Heartbeat(const ceph::osd::ShardServices& service, ceph::mon::Client& monc, ceph::net::Messenger& front_msgr, ceph::net::Messenger& back_msgr) @@ -93,7 +93,7 @@ seastar::future<> Heartbeat::add_peer(osd_id_t peer, epoch_t epoch) auto found = peers.find(peer); if (found == peers.end()) { logger().info("add_peer({})", peer); - auto osdmap = service.get_map(); + auto osdmap = service.get_osdmap_service().get_map(); // TODO: use addrs return seastar::when_all_succeed( front_msgr.connect(osdmap->get_hb_front_addrs(peer).front(), @@ -122,7 +122,7 @@ seastar::future Heartbeat::remove_down_peers() } return seastar::map_reduce(std::move(osds), [this](auto& osd) { - auto osdmap = service.get_map(); + auto osdmap = service.get_osdmap_service().get_map(); if (!osdmap->is_up(osd)) { return remove_peer(osd).then([] { return seastar::make_ready_future(-1); @@ -143,7 +143,7 @@ seastar::future Heartbeat::remove_down_peers() void Heartbeat::add_reporter_peers(int whoami) { - auto osdmap = service.get_map(); + auto osdmap = service.get_osdmap_service().get_map(); // include next and previous up osds to ensure we have a fully-connected set set want; if (auto next = osdmap->get_next_up_osd_after(whoami); next >= 0) { @@ -185,7 +185,7 @@ seastar::future<> Heartbeat::update_peers(int whoami) }); }).then([=] { // or too few? - auto osdmap = service.get_map(); + auto osdmap = service.get_osdmap_service().get_map(); for (auto next = osdmap->get_next_up_osd_after(whoami); peers.size() < min_peers && next >= 0 && next != whoami; next = osdmap->get_next_up_osd_after(next)) { @@ -257,14 +257,15 @@ seastar::future<> Heartbeat::handle_ping(ceph::net::Connection* conn, auto min_message = static_cast( local_conf()->osd_heartbeat_min_size); auto reply = - make_message(m->fsid, - service.get_map()->get_epoch(), - MOSDPing::PING_REPLY, - m->ping_stamp, - m->mono_ping_stamp, - service.get_mnow(), - service.get_up_epoch(), - min_message); + make_message( + m->fsid, + service.get_osdmap_service().get_map()->get_epoch(), + MOSDPing::PING_REPLY, + m->ping_stamp, + m->mono_ping_stamp, + service.get_mnow(), + service.get_osdmap_service().get_up_epoch(), + min_message); return conn->send(reply); } @@ -336,14 +337,15 @@ seastar::future<> Heartbeat::send_heartbeats() if (con) { auto min_message = static_cast( local_conf()->osd_heartbeat_min_size); - auto ping = make_message(monc.get_fsid(), - service.get_map()->get_epoch(), - MOSDPing::PING, - sent_stamp, - mnow, - mnow, - service.get_up_epoch(), - min_message); + auto ping = make_message( + monc.get_fsid(), + service.get_osdmap_service().get_map()->get_epoch(), + MOSDPing::PING, + sent_stamp, + mnow, + mnow, + service.get_osdmap_service().get_up_epoch(), + min_message); return con->send(ping).then([&reply] { reply.unacknowledged++; return seastar::now(); @@ -366,7 +368,7 @@ seastar::future<> Heartbeat::send_failures() } auto failed_for = chrono::duration_cast( clock::now() - failed_since).count(); - auto osdmap = service.get_map(); + auto osdmap = service.get_osdmap_service().get_map(); auto failure_report = make_message(monc.get_fsid(), osd, @@ -385,12 +387,13 @@ seastar::future<> Heartbeat::send_failures() seastar::future<> Heartbeat::send_still_alive(osd_id_t osd, const entity_addrvec_t& addrs) { - auto still_alive = make_message(monc.get_fsid(), - osd, - addrs, - 0, - service.get_map()->get_epoch(), - MOSDFailure::FLAG_ALIVE); + auto still_alive = make_message( + monc.get_fsid(), + osd, + addrs, + 0, + service.get_osdmap_service().get_map()->get_epoch(), + MOSDFailure::FLAG_ALIVE); return monc.send_message(still_alive).then([=] { failure_pending.erase(osd); return seastar::now(); diff --git a/src/crimson/osd/heartbeat.h b/src/crimson/osd/heartbeat.h index a066f144066f..5c7cabc39a74 100644 --- a/src/crimson/osd/heartbeat.h +++ b/src/crimson/osd/heartbeat.h @@ -10,7 +10,10 @@ #include "crimson/net/Fwd.h" class MOSDPing; -class OSDMapService; + +namespace ceph::osd { + class ShardServices; +} namespace ceph::mon { class Client; @@ -22,7 +25,7 @@ class Heartbeat : public ceph::net::Dispatcher { public: using osd_id_t = int; - Heartbeat(const OSDMapService& service, + Heartbeat(const ceph::osd::ShardServices& service, ceph::mon::Client& monc, ceph::net::Messenger& front_msgr, ceph::net::Messenger& back_msgr); @@ -69,7 +72,7 @@ private: seastar::future<> start_messenger(ceph::net::Messenger& msgr, const entity_addrvec_t& addrs); private: - const OSDMapService& service; + const ceph::osd::ShardServices& service; ceph::mon::Client& monc; ceph::net::Messenger& front_msgr; ceph::net::Messenger& back_msgr; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 45921dcae3ff..044df1a80d9a 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -61,12 +61,12 @@ OSD::OSD(int id, uint32_t nonce, public_msgr{public_msgr}, monc{new ceph::mon::Client{public_msgr, *this}}, mgrc{new ceph::mgr::Client{public_msgr, *this}}, - heartbeat{new Heartbeat{*this, *monc, hb_front_msgr, hb_back_msgr}}, - heartbeat_timer{[this] { update_heartbeat_peers(); }}, store{ceph::os::FuturizedStore::create( local_conf().get_val("osd_objectstore"), local_conf().get_val("osd_data"))}, - shard_services{cluster_msgr, public_msgr, *monc, *mgrc, *store}, + shard_services{*this, cluster_msgr, public_msgr, *monc, *mgrc, *store}, + heartbeat{new Heartbeat{shard_services, *monc, hb_front_msgr, hb_back_msgr}}, + heartbeat_timer{[this] { update_heartbeat_peers(); }}, osdmap_gate("OSD::osdmap_gate", std::make_optional(std::ref(shard_services))) { osdmaps[0] = boost::make_local_shared(); @@ -528,11 +528,6 @@ MessageRef OSD::get_stats() return m; } -ceph::signedspan OSD::get_mnow() const -{ - return ceph::mono_clock::now() - startup_time; -} - OSD::cached_map_t OSD::get_map() const { return osdmap; diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index 97ebaf92af8f..6345b33dc7b3 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -71,9 +71,6 @@ class OSD final : public ceph::net::Dispatcher, std::unique_ptr monc; std::unique_ptr mgrc; - std::unique_ptr heartbeat; - seastar::timer heartbeat_timer; - SharedLRU osdmaps; SimpleLRU map_bl_cache; cached_map_t osdmap; @@ -112,6 +109,9 @@ class OSD final : public ceph::net::Dispatcher, ceph::osd::ShardServices shard_services; std::unordered_map> pgs; + std::unique_ptr heartbeat; + seastar::timer heartbeat_timer; + public: OSD(int id, uint32_t nonce, ceph::net::Messenger& cluster_msgr, @@ -139,7 +139,6 @@ private: seastar::future<> _send_alive(); // OSDMapService methods - ceph::signedspan get_mnow() const final; epoch_t get_up_epoch() const final { return up_epoch; } diff --git a/src/crimson/osd/osdmap_service.h b/src/crimson/osd/osdmap_service.h index f50b7ac6c225..effd45b79464 100644 --- a/src/crimson/osd/osdmap_service.h +++ b/src/crimson/osd/osdmap_service.h @@ -16,6 +16,5 @@ public: virtual seastar::future get_map(epoch_t e) = 0; /// get the latest map virtual cached_map_t get_map() const = 0; - virtual ceph::signedspan get_mnow() const = 0; virtual epoch_t get_up_epoch() const = 0; }; diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc index 686313dec706..5669285d0e1b 100644 --- a/src/crimson/osd/pg.cc +++ b/src/crimson/osd/pg.cc @@ -174,6 +174,16 @@ void PG::log_state_exit( events); } +ceph::signedspan PG::get_mnow() +{ + return shard_services.get_mnow(); +} + +HeartbeatStampsRef PG::get_hb_stamps(int peer) +{ + return shard_services.get_hb_stamps(peer); +} + void PG::init( ceph::os::CollectionRef coll, int role, diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h index 71ea028b0daa..37489c92f019 100644 --- a/src/crimson/osd/pg.h +++ b/src/crimson/osd/pg.h @@ -377,12 +377,8 @@ public: return OstreamTemp(CLOG_ERROR, nullptr); } - ceph::signedspan get_mnow() final { -#warning writeme - } - HeartbeatStampsRef get_hb_stamps(int peer) final { -#warning write me - } + ceph::signedspan get_mnow() final; + HeartbeatStampsRef get_hb_stamps(int peer) final; // Utility bool is_primary() const { diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc index c27d5bf25d6a..a1052f5a0282 100644 --- a/src/crimson/osd/shard_services.cc +++ b/src/crimson/osd/shard_services.cc @@ -27,12 +27,14 @@ namespace { namespace ceph::osd { ShardServices::ShardServices( + OSDMapService &osdmap_service, ceph::net::Messenger &cluster_msgr, ceph::net::Messenger &public_msgr, ceph::mon::Client &monc, ceph::mgr::Client &mgrc, ceph::os::FuturizedStore &store) - : cluster_msgr(cluster_msgr), + : osdmap_service(osdmap_service), + cluster_msgr(cluster_msgr), public_msgr(public_msgr), monc(monc), mgrc(mgrc), @@ -245,14 +247,10 @@ seastar::future<> ShardServices::osdmap_subscribe(version_t epoch, bool force_re } } -ceph::signedspan ShardServices::get_mnow() -{ -#warning write me -} - HeartbeatStampsRef ShardServices::get_hb_stamps(int peer) { -#warning write me +#warning writeme + return HeartbeatStampsRef(); } }; diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index 388335c7a0af..c89c4ddf6f6c 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -10,6 +10,7 @@ #include "msg/MessageRef.h" #include "crimson/os/cyan_collection.h" #include "osd/PeeringState.h" +#include "crimson/osd/osdmap_service.h" namespace ceph::net { class Messenger; @@ -39,6 +40,7 @@ namespace ceph::osd { */ class ShardServices { using cached_map_t = boost::local_shared_ptr; + OSDMapService &osdmap_service; ceph::net::Messenger &cluster_msgr; ceph::net::Messenger &public_msgr; ceph::mon::Client &monc; @@ -52,6 +54,7 @@ class ShardServices { public: ShardServices( + OSDMapService &osdmap_service, ceph::net::Messenger &cluster_msgr, ceph::net::Messenger &public_msgr, ceph::mon::Client &monc, @@ -71,6 +74,11 @@ public: return &cct; } + // OSDMapService + const OSDMapService &get_osdmap_service() const { + return osdmap_service; + } + // Op Tracking OperationRegistry registry; @@ -143,9 +151,12 @@ public: seastar::future<> osdmap_subscribe(version_t epoch, bool force_request); - ceph::signedspan get_mnow(); + // Time state + ceph::mono_time startup_time = ceph::mono_clock::now(); + ceph::signedspan get_mnow() const { + return ceph::mono_clock::now() - startup_time; + } HeartbeatStampsRef get_hb_stamps(int peer); - };