crimson/osd: adjust get_pg_stats() and for_each_pg() to return futures

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/crimson/admin/osd_admin.cc b/src/crimson/admin/osd_admin.cc
index b5368738904054426608ff5948344ce6f2034be5..478256d341913d0a9e25551d146ad1acf9137340 100644 (file)
--- a/src/crimson/admin/osd_admin.cc
+++ b/src/crimson/admin/osd_admin.cc
@@ -130,13 +130,17 @@ public:
                                       std::string_view format,
                                       ceph::bufferlist&& input) const final
   {
-    std::unique_ptr<Formatter> f{Formatter::create(format,
-                                                   "json-pretty",
-                                                   "json-pretty")};
+    std::unique_ptr<Formatter> fref{Formatter::create(format,
+                                                     "json-pretty",
+                                                     "json-pretty")};
+    Formatter *f = fref.get();
     f->open_object_section("pgstate_history");
-    osd.dump_pg_state_history(f.get());
-    f->close_section();
-    return seastar::make_ready_future<tell_result_t>(std::move(f));
+    return osd.dump_pg_state_history(
+      f
+    ).then([fref=std::move(fref)]() mutable {
+      fref->close_section();
+      return seastar::make_ready_future<tell_result_t>(std::move(fref));
+    });
   }
 private:
   const crimson::osd::OSD& osd;

diff --git a/src/crimson/mgr/client.cc b/src/crimson/mgr/client.cc
index 5d259c29e13c01b83b3a3f532aef3bc377287a87..6e3d7cdd848c007970be328b8e2d5b6241499eb4 100644 (file)
--- a/src/crimson/mgr/client.cc
+++ b/src/crimson/mgr/client.cc
@@ -156,8 +156,10 @@ void Client::report()
       logger().warn("report: no conn available; raport skipped");
       return seastar::now();
     }
-    auto pg_stats = with_stats.get_stats();
-    return conn->send(std::move(pg_stats));
+    return with_stats.get_stats(
+    ).then([this](auto &&pg_stats) {
+      return conn->send(std::move(pg_stats));
+    });
   });
 }
 

diff --git a/src/crimson/mgr/client.h b/src/crimson/mgr/client.h
index 17d62d03477128d3fb3bc82956868eaeb17b4e0c..92f1576770643bb8f4ecbabda70ea221b6ae07a2 100644 (file)
--- a/src/crimson/mgr/client.h
+++ b/src/crimson/mgr/client.h
@@ -24,7 +24,7 @@ namespace crimson::mgr
 // implement WithStats if you want to report stats to mgr periodically
 class WithStats {
 public:
-  virtual MessageURef get_stats() const = 0;
+  virtual seastar::future<MessageURef> get_stats() const = 0;
   virtual ~WithStats() {}
 };
 

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index a9d3d1ea131538822468fa145d1e1ad22d4644cc..2db0ba82e7e095512ea69a55a371ff501a9c038a 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -94,10 +94,14 @@ OSD::OSD(int id, uint32_t nonce,
       *public_msgr, *monc, *mgrc, store},
     shard_services{pg_shard_manager.get_shard_services()},
     heartbeat{new Heartbeat{whoami, shard_services, *monc, hb_front_msgr, hb_back_msgr}},
-    // do this in background
+    // do this in background -- continuation rearms timer when complete
     tick_timer{[this] {
-      update_heartbeat_peers();
-      update_stats();
+      std::ignore = update_heartbeat_peers(
+      ).then([this] {
+       update_stats();
+       tick_timer.arm(
+         std::chrono::seconds(TICK_INTERVAL));
+      });
     }},
     asok{seastar::make_lw_shared<crimson::admin::AdminSocket>()},
     log_client(cluster_msgr.get(), LogClient::NO_FLAGS),
@@ -656,18 +660,19 @@ void OSD::dump_status(Formatter* f) const
   f->dump_unsigned("num_pgs", pg_shard_manager.get_num_pgs());
 }
 
-void OSD::dump_pg_state_history(Formatter* f) const
+seastar::future<> OSD::dump_pg_state_history(Formatter* f) const
 {
   f->open_array_section("pgs");
-  pg_shard_manager.for_each_pg([f](auto &pgid, auto &pg) {
+  return pg_shard_manager.for_each_pg([f](auto &pgid, auto &pg) {
     f->open_object_section("pg");
     f->dump_stream("pg") << pgid;
     const auto& peering_state = pg->get_peering_state();
     f->dump_string("currently", peering_state.get_current_state());
     peering_state.dump_history(f);
     f->close_section();
+  }).then([f] {
+    f->close_section();
   });
-  f->close_section();
 }
 
 void OSD::print(std::ostream& out) const
@@ -809,13 +814,16 @@ void OSD::update_stats()
   });
 }
 
-MessageURef OSD::get_stats() const
+seastar::future<MessageURef> OSD::get_stats() const
 {
   // MPGStats::had_map_for is not used since PGMonitor was removed
   auto m = crimson::make_message<MPGStats>(monc->get_fsid(), osdmap->get_epoch());
   m->osd_stat = osd_stat;
-  m->pg_stat = pg_shard_manager.get_pg_stats();
-  return m;
+  return pg_shard_manager.get_pg_stats(
+  ).then([m=std::move(m)](auto &&stats) mutable {
+    m->pg_stat = std::move(stats);
+    return seastar::make_ready_future<MessageURef>(std::move(m));
+  });
 }
 
 uint64_t OSD::send_pg_stats()
@@ -949,7 +957,8 @@ seastar::future<> OSD::committed_osd_maps(version_t first,
         pg_shard_manager.set_active();
         beacon_timer.arm_periodic(
           std::chrono::seconds(local_conf()->osd_beacon_report_interval));
-        tick_timer.arm_periodic(
+       // timer continuation rearms when complete
+        tick_timer.arm(
           std::chrono::seconds(TICK_INTERVAL));
       }
     } else {
@@ -1174,13 +1183,13 @@ seastar::future<> OSD::send_beacon()
   return monc->send_message(std::move(m));
 }
 
-void OSD::update_heartbeat_peers()
+seastar::future<> OSD::update_heartbeat_peers()
 {
   if (!pg_shard_manager.is_active()) {
-    return;
+    return seastar::now();;
   }
 
-  pg_shard_manager.for_each_pg([this](auto &pgid, auto &pg) {
+  return pg_shard_manager.for_each_pg([this](auto &pgid, auto &pg) {
     vector<int> up, acting;
     osdmap->pg_to_up_acting_osds(pgid.pgid,
                                  &up, nullptr,
@@ -1192,8 +1201,9 @@ void OSD::update_heartbeat_peers()
         heartbeat->add_peer(osd, osdmap->get_epoch());
       }
     }
+  }).then([this] {
+    heartbeat->update_peers(whoami);
   });
-  heartbeat->update_peers(whoami);
 }
 
 seastar::future<> OSD::handle_peering_op(

diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 0db21eb84bb71c84b25c0c9b1dfb75a2ece93083..1c6265f9083073269c3bd6d293cf4a1d43dd60ba 100644 (file)
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -95,7 +95,7 @@ class OSD final : public crimson::net::Dispatcher,
   osd_stat_t osd_stat;
   uint32_t osd_stat_seq = 0;
   void update_stats();
-  MessageURef get_stats() const final;
+  seastar::future<MessageURef> get_stats() const final;
 
   // AuthHandler methods
   void handle_authentication(const EntityName& name,
@@ -134,7 +134,7 @@ public:
   seastar::future<> stop();
 
   void dump_status(Formatter*) const;
-  void dump_pg_state_history(Formatter*) const;
+  seastar::future<> dump_pg_state_history(Formatter*) const;
   void print(std::ostream&) const;
 
   seastar::future<> send_incremental_map(crimson::net::ConnectionRef conn,
@@ -213,7 +213,7 @@ private:
   bool should_restart() const;
   seastar::future<> restart();
   seastar::future<> shutdown();
-  void update_heartbeat_peers();
+  seastar::future<> update_heartbeat_peers();
   friend class PGAdvanceMap;
 
 public:

diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc
index 5571e2d0ee12dbfb562653f03a2bb5b3df0140c9..47757d816a8d0a0766e4b733df41be7353edc243 100644 (file)
--- a/src/crimson/osd/pg_shard_manager.cc
+++ b/src/crimson/osd/pg_shard_manager.cc
@@ -69,9 +69,11 @@ seastar::future<> PGShardManager::stop_pgs()
   return local_state.stop_pgs();
 }
 
-std::map<pg_t, pg_stat_t> PGShardManager::get_pg_stats() const
+seastar::future<std::map<pg_t, pg_stat_t>>
+PGShardManager::get_pg_stats() const
 {
-  return local_state.get_pg_stats();
+  return seastar::make_ready_future<std::map<pg_t, pg_stat_t>>(
+    local_state.get_pg_stats());
 }
 
 seastar::future<> PGShardManager::broadcast_map_to_pgs(epoch_t epoch)

diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index f40ea25a6f24ae0f0c9056d9889ea08fcb6789c2..24dbe95fb766f6456371c302e468775c1e7260f7 100644 (file)
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -170,13 +170,20 @@ public:
   seastar::future<> load_pgs();
   seastar::future<> stop_pgs();
 
-  std::map<pg_t, pg_stat_t> get_pg_stats() const;
-
+  seastar::future<std::map<pg_t, pg_stat_t>> get_pg_stats() const;
+
+  /**
+   * for_each_pg
+   *
+   * Invokes f on each pg sequentially.  Caller may rely on f not being
+   * invoked concurrently on multiple cores.
+   */
   template <typename F>
-  void for_each_pg(F &&f) const {
+  seastar::future<> for_each_pg(F &&f) const {
     for (auto &&pg: local_state.pg_map.get_pgs()) {
       std::apply(f, pg);
     }
+    return seastar::now();
   }
 
   auto get_num_pgs() const {