From 2b2f5b70092e0a1877e1a01d3847e2342e49731b Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Tue, 23 Aug 2022 21:49:16 +0000 Subject: [PATCH] crimson/osd: adjust get_pg_stats() and for_each_pg() to return futures Signed-off-by: Samuel Just --- src/crimson/admin/osd_admin.cc | 16 +++++++----- src/crimson/mgr/client.cc | 6 +++-- src/crimson/mgr/client.h | 2 +- src/crimson/osd/osd.cc | 38 ++++++++++++++++++----------- src/crimson/osd/osd.h | 6 ++--- src/crimson/osd/pg_shard_manager.cc | 6 +++-- src/crimson/osd/pg_shard_manager.h | 13 +++++++--- 7 files changed, 56 insertions(+), 31 deletions(-) diff --git a/src/crimson/admin/osd_admin.cc b/src/crimson/admin/osd_admin.cc index b5368738904..478256d3419 100644 --- a/src/crimson/admin/osd_admin.cc +++ b/src/crimson/admin/osd_admin.cc @@ -130,13 +130,17 @@ public: std::string_view format, ceph::bufferlist&& input) const final { - std::unique_ptr f{Formatter::create(format, - "json-pretty", - "json-pretty")}; + std::unique_ptr fref{Formatter::create(format, + "json-pretty", + "json-pretty")}; + Formatter *f = fref.get(); f->open_object_section("pgstate_history"); - osd.dump_pg_state_history(f.get()); - f->close_section(); - return seastar::make_ready_future(std::move(f)); + return osd.dump_pg_state_history( + f + ).then([fref=std::move(fref)]() mutable { + fref->close_section(); + return seastar::make_ready_future(std::move(fref)); + }); } private: const crimson::osd::OSD& osd; diff --git a/src/crimson/mgr/client.cc b/src/crimson/mgr/client.cc index 5d259c29e13..6e3d7cdd848 100644 --- a/src/crimson/mgr/client.cc +++ b/src/crimson/mgr/client.cc @@ -156,8 +156,10 @@ void Client::report() logger().warn("report: no conn available; raport skipped"); return seastar::now(); } - auto pg_stats = with_stats.get_stats(); - return conn->send(std::move(pg_stats)); + return with_stats.get_stats( + ).then([this](auto &&pg_stats) { + return conn->send(std::move(pg_stats)); + }); }); } diff --git a/src/crimson/mgr/client.h b/src/crimson/mgr/client.h index 17d62d03477..92f15767706 100644 --- a/src/crimson/mgr/client.h +++ b/src/crimson/mgr/client.h @@ -24,7 +24,7 @@ namespace crimson::mgr // implement WithStats if you want to report stats to mgr periodically class WithStats { public: - virtual MessageURef get_stats() const = 0; + virtual seastar::future get_stats() const = 0; virtual ~WithStats() {} }; diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index a9d3d1ea131..2db0ba82e7e 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -94,10 +94,14 @@ OSD::OSD(int id, uint32_t nonce, *public_msgr, *monc, *mgrc, store}, shard_services{pg_shard_manager.get_shard_services()}, heartbeat{new Heartbeat{whoami, shard_services, *monc, hb_front_msgr, hb_back_msgr}}, - // do this in background + // do this in background -- continuation rearms timer when complete tick_timer{[this] { - update_heartbeat_peers(); - update_stats(); + std::ignore = update_heartbeat_peers( + ).then([this] { + update_stats(); + tick_timer.arm( + std::chrono::seconds(TICK_INTERVAL)); + }); }}, asok{seastar::make_lw_shared()}, log_client(cluster_msgr.get(), LogClient::NO_FLAGS), @@ -656,18 +660,19 @@ void OSD::dump_status(Formatter* f) const f->dump_unsigned("num_pgs", pg_shard_manager.get_num_pgs()); } -void OSD::dump_pg_state_history(Formatter* f) const +seastar::future<> OSD::dump_pg_state_history(Formatter* f) const { f->open_array_section("pgs"); - pg_shard_manager.for_each_pg([f](auto &pgid, auto &pg) { + return pg_shard_manager.for_each_pg([f](auto &pgid, auto &pg) { f->open_object_section("pg"); f->dump_stream("pg") << pgid; const auto& peering_state = pg->get_peering_state(); f->dump_string("currently", peering_state.get_current_state()); peering_state.dump_history(f); f->close_section(); + }).then([f] { + f->close_section(); }); - f->close_section(); } void OSD::print(std::ostream& out) const @@ -809,13 +814,16 @@ void OSD::update_stats() }); } -MessageURef OSD::get_stats() const +seastar::future OSD::get_stats() const { // MPGStats::had_map_for is not used since PGMonitor was removed auto m = crimson::make_message(monc->get_fsid(), osdmap->get_epoch()); m->osd_stat = osd_stat; - m->pg_stat = pg_shard_manager.get_pg_stats(); - return m; + return pg_shard_manager.get_pg_stats( + ).then([m=std::move(m)](auto &&stats) mutable { + m->pg_stat = std::move(stats); + return seastar::make_ready_future(std::move(m)); + }); } uint64_t OSD::send_pg_stats() @@ -949,7 +957,8 @@ seastar::future<> OSD::committed_osd_maps(version_t first, pg_shard_manager.set_active(); beacon_timer.arm_periodic( std::chrono::seconds(local_conf()->osd_beacon_report_interval)); - tick_timer.arm_periodic( + // timer continuation rearms when complete + tick_timer.arm( std::chrono::seconds(TICK_INTERVAL)); } } else { @@ -1174,13 +1183,13 @@ seastar::future<> OSD::send_beacon() return monc->send_message(std::move(m)); } -void OSD::update_heartbeat_peers() +seastar::future<> OSD::update_heartbeat_peers() { if (!pg_shard_manager.is_active()) { - return; + return seastar::now();; } - pg_shard_manager.for_each_pg([this](auto &pgid, auto &pg) { + return pg_shard_manager.for_each_pg([this](auto &pgid, auto &pg) { vector up, acting; osdmap->pg_to_up_acting_osds(pgid.pgid, &up, nullptr, @@ -1192,8 +1201,9 @@ void OSD::update_heartbeat_peers() heartbeat->add_peer(osd, osdmap->get_epoch()); } } + }).then([this] { + heartbeat->update_peers(whoami); }); - heartbeat->update_peers(whoami); } seastar::future<> OSD::handle_peering_op( diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index 0db21eb84bb..1c6265f9083 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -95,7 +95,7 @@ class OSD final : public crimson::net::Dispatcher, osd_stat_t osd_stat; uint32_t osd_stat_seq = 0; void update_stats(); - MessageURef get_stats() const final; + seastar::future get_stats() const final; // AuthHandler methods void handle_authentication(const EntityName& name, @@ -134,7 +134,7 @@ public: seastar::future<> stop(); void dump_status(Formatter*) const; - void dump_pg_state_history(Formatter*) const; + seastar::future<> dump_pg_state_history(Formatter*) const; void print(std::ostream&) const; seastar::future<> send_incremental_map(crimson::net::ConnectionRef conn, @@ -213,7 +213,7 @@ private: bool should_restart() const; seastar::future<> restart(); seastar::future<> shutdown(); - void update_heartbeat_peers(); + seastar::future<> update_heartbeat_peers(); friend class PGAdvanceMap; public: diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc index 5571e2d0ee1..47757d816a8 100644 --- a/src/crimson/osd/pg_shard_manager.cc +++ b/src/crimson/osd/pg_shard_manager.cc @@ -69,9 +69,11 @@ seastar::future<> PGShardManager::stop_pgs() return local_state.stop_pgs(); } -std::map PGShardManager::get_pg_stats() const +seastar::future> +PGShardManager::get_pg_stats() const { - return local_state.get_pg_stats(); + return seastar::make_ready_future>( + local_state.get_pg_stats()); } seastar::future<> PGShardManager::broadcast_map_to_pgs(epoch_t epoch) diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h index f40ea25a6f2..24dbe95fb76 100644 --- a/src/crimson/osd/pg_shard_manager.h +++ b/src/crimson/osd/pg_shard_manager.h @@ -170,13 +170,20 @@ public: seastar::future<> load_pgs(); seastar::future<> stop_pgs(); - std::map get_pg_stats() const; - + seastar::future> get_pg_stats() const; + + /** + * for_each_pg + * + * Invokes f on each pg sequentially. Caller may rely on f not being + * invoked concurrently on multiple cores. + */ template - void for_each_pg(F &&f) const { + seastar::future<> for_each_pg(F &&f) const { for (auto &&pg: local_state.pg_map.get_pgs()) { std::apply(f, pg); } + return seastar::now(); } auto get_num_pgs() const { -- 2.39.5