crimson/: move get_mnow() to ShardServices, pass to heartbeat

get_mnow isn't clearly at home in OSDMapService, and the other methods
are needed from PeeringState, so let's consolidate on ShardServices
for now.  We probably ought OSDMapService state out of OSD into its
own module at some point.

Signed-off-by: Samuel Just <sjust@redhat.com>

diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc
index 808037c2d421376763d74d04c735afd76d7474ce..367c2ffce789d7090eb693acc04388d6cf563312 100644 (file)
--- a/src/crimson/osd/heartbeat.cc
+++ b/src/crimson/osd/heartbeat.cc
@@ -8,7 +8,7 @@
 #include "crimson/common/config_proxy.h"
 #include "crimson/net/Connection.h"
 #include "crimson/net/Messenger.h"
-#include "crimson/osd/osdmap_service.h"
+#include "crimson/osd/shard_services.h"
 #include "crimson/mon/MonClient.h"
 
 #include "osd/OSDMap.h"
@@ -21,7 +21,7 @@ namespace {
   }
 }
 
-Heartbeat::Heartbeat(const OSDMapService& service,
+Heartbeat::Heartbeat(const ceph::osd::ShardServices& service,
                      ceph::mon::Client& monc,
                      ceph::net::Messenger& front_msgr,
                      ceph::net::Messenger& back_msgr)
@@ -93,7 +93,7 @@ seastar::future<> Heartbeat::add_peer(osd_id_t peer, epoch_t epoch)
   auto found = peers.find(peer);
   if (found == peers.end()) {
     logger().info("add_peer({})", peer);
-    auto osdmap = service.get_map();
+    auto osdmap = service.get_osdmap_service().get_map();
     // TODO: use addrs
     return seastar::when_all_succeed(
         front_msgr.connect(osdmap->get_hb_front_addrs(peer).front(),
@@ -122,7 +122,7 @@ seastar::future<Heartbeat::osds_t> Heartbeat::remove_down_peers()
   }
   return seastar::map_reduce(std::move(osds),
     [this](auto& osd) {
-      auto osdmap = service.get_map();
+      auto osdmap = service.get_osdmap_service().get_map();
       if (!osdmap->is_up(osd)) {
         return remove_peer(osd).then([] {
           return seastar::make_ready_future<osd_id_t>(-1);
@@ -143,7 +143,7 @@ seastar::future<Heartbeat::osds_t> Heartbeat::remove_down_peers()
 
 void Heartbeat::add_reporter_peers(int whoami)
 {
-  auto osdmap = service.get_map();
+  auto osdmap = service.get_osdmap_service().get_map();
   // include next and previous up osds to ensure we have a fully-connected set
   set<int> want;
   if (auto next = osdmap->get_next_up_osd_after(whoami); next >= 0) {
@@ -185,7 +185,7 @@ seastar::future<> Heartbeat::update_peers(int whoami)
     });
   }).then([=] {
     // or too few?
-    auto osdmap = service.get_map();
+    auto osdmap = service.get_osdmap_service().get_map();
     for (auto next = osdmap->get_next_up_osd_after(whoami);
       peers.size() < min_peers && next >= 0 && next != whoami;
       next = osdmap->get_next_up_osd_after(next)) {
@@ -257,14 +257,15 @@ seastar::future<> Heartbeat::handle_ping(ceph::net::Connection* conn,
   auto min_message = static_cast<uint32_t>(
     local_conf()->osd_heartbeat_min_size);
   auto reply =
-    make_message<MOSDPing>(m->fsid,
-                           service.get_map()->get_epoch(),
-                           MOSDPing::PING_REPLY,
-                           m->ping_stamp,
-                          m->mono_ping_stamp,
-                          service.get_mnow(),
-                          service.get_up_epoch(),
-                           min_message);
+    make_message<MOSDPing>(
+      m->fsid,
+      service.get_osdmap_service().get_map()->get_epoch(),
+      MOSDPing::PING_REPLY,
+      m->ping_stamp,
+      m->mono_ping_stamp,
+      service.get_mnow(),
+      service.get_osdmap_service().get_up_epoch(),
+      min_message);
   return conn->send(reply);
 }
 
@@ -336,14 +337,15 @@ seastar::future<> Heartbeat::send_heartbeats()
           if (con) {
             auto min_message = static_cast<uint32_t>(
               local_conf()->osd_heartbeat_min_size);
-            auto ping = make_message<MOSDPing>(monc.get_fsid(),
-                                               service.get_map()->get_epoch(),
-                                               MOSDPing::PING,
-                                               sent_stamp,
-                                              mnow,
-                                              mnow,
-                                              service.get_up_epoch(),
-                                               min_message);
+            auto ping = make_message<MOSDPing>(
+             monc.get_fsid(),
+             service.get_osdmap_service().get_map()->get_epoch(),
+             MOSDPing::PING,
+             sent_stamp,
+             mnow,
+             mnow,
+             service.get_osdmap_service().get_up_epoch(),
+             min_message);
             return con->send(ping).then([&reply] {
               reply.unacknowledged++;
               return seastar::now();
@@ -366,7 +368,7 @@ seastar::future<> Heartbeat::send_failures()
       }
       auto failed_for = chrono::duration_cast<chrono::seconds>(
         clock::now() - failed_since).count();
-      auto osdmap = service.get_map();
+      auto osdmap = service.get_osdmap_service().get_map();
       auto failure_report =
         make_message<MOSDFailure>(monc.get_fsid(),
                                   osd,
@@ -385,12 +387,13 @@ seastar::future<> Heartbeat::send_failures()
 seastar::future<> Heartbeat::send_still_alive(osd_id_t osd,
                                               const entity_addrvec_t& addrs)
 {
-  auto still_alive = make_message<MOSDFailure>(monc.get_fsid(),
-                                               osd,
-                                               addrs,
-                                               0,
-                                               service.get_map()->get_epoch(),
-                                               MOSDFailure::FLAG_ALIVE);
+  auto still_alive = make_message<MOSDFailure>(
+    monc.get_fsid(),
+    osd,
+    addrs,
+    0,
+    service.get_osdmap_service().get_map()->get_epoch(),
+    MOSDFailure::FLAG_ALIVE);
   return monc.send_message(still_alive).then([=] {
     failure_pending.erase(osd);
     return seastar::now();

diff --git a/src/crimson/osd/heartbeat.h b/src/crimson/osd/heartbeat.h
index a066f144066f909843d260569872887879b0e53f..5c7cabc39a7421a224ae1b6e1f83e374a7a14839 100644 (file)
--- a/src/crimson/osd/heartbeat.h
+++ b/src/crimson/osd/heartbeat.h
@@ -10,7 +10,10 @@
 #include "crimson/net/Fwd.h"
 
 class MOSDPing;
-class OSDMapService;
+
+namespace ceph::osd {
+  class ShardServices;
+}
 
 namespace ceph::mon {
   class Client;
@@ -22,7 +25,7 @@ class Heartbeat : public ceph::net::Dispatcher {
 public:
   using osd_id_t = int;
 
-  Heartbeat(const OSDMapService& service,
+  Heartbeat(const ceph::osd::ShardServices& service,
            ceph::mon::Client& monc,
            ceph::net::Messenger& front_msgr,
            ceph::net::Messenger& back_msgr);
@@ -69,7 +72,7 @@ private:
   seastar::future<> start_messenger(ceph::net::Messenger& msgr,
                                    const entity_addrvec_t& addrs);
 private:
-  const OSDMapService& service;
+  const ceph::osd::ShardServices& service;
   ceph::mon::Client& monc;
   ceph::net::Messenger& front_msgr;
   ceph::net::Messenger& back_msgr;

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 45921dcae3ff0bf50310c474392023a93b36fa8b..044df1a80d9a89bf80193ede750d94beb96cac7e 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -61,12 +61,12 @@ OSD::OSD(int id, uint32_t nonce,
     public_msgr{public_msgr},
     monc{new ceph::mon::Client{public_msgr, *this}},
     mgrc{new ceph::mgr::Client{public_msgr, *this}},
-    heartbeat{new Heartbeat{*this, *monc, hb_front_msgr, hb_back_msgr}},
-    heartbeat_timer{[this] { update_heartbeat_peers(); }},
     store{ceph::os::FuturizedStore::create(
       local_conf().get_val<std::string>("osd_objectstore"),
       local_conf().get_val<std::string>("osd_data"))},
-    shard_services{cluster_msgr, public_msgr, *monc, *mgrc, *store},
+    shard_services{*this, cluster_msgr, public_msgr, *monc, *mgrc, *store},
+    heartbeat{new Heartbeat{shard_services, *monc, hb_front_msgr, hb_back_msgr}},
+    heartbeat_timer{[this] { update_heartbeat_peers(); }},
     osdmap_gate("OSD::osdmap_gate", std::make_optional(std::ref(shard_services)))
 {
   osdmaps[0] = boost::make_local_shared<OSDMap>();
@@ -528,11 +528,6 @@ MessageRef OSD::get_stats()
   return m;
 }
 
-ceph::signedspan OSD::get_mnow() const
-{
-  return ceph::mono_clock::now() - startup_time;
-}
-
 OSD::cached_map_t OSD::get_map() const
 {
   return osdmap;

diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h
index 97ebaf92af8fa5446880c617b2f76c54f61038a1..6345b33dc7b33692e007e3edcb284abea1c8adfd 100644 (file)
--- a/src/crimson/osd/osd.h
+++ b/src/crimson/osd/osd.h
@@ -71,9 +71,6 @@ class OSD final : public ceph::net::Dispatcher,
   std::unique_ptr<ceph::mon::Client> monc;
   std::unique_ptr<ceph::mgr::Client> mgrc;
 
-  std::unique_ptr<Heartbeat> heartbeat;
-  seastar::timer<seastar::lowres_clock> heartbeat_timer;
-
   SharedLRU<epoch_t, OSDMap> osdmaps;
   SimpleLRU<epoch_t, bufferlist, false> map_bl_cache;
   cached_map_t osdmap;
@@ -112,6 +109,9 @@ class OSD final : public ceph::net::Dispatcher,
   ceph::osd::ShardServices shard_services;
   std::unordered_map<spg_t, Ref<PG>> pgs;
 
+  std::unique_ptr<Heartbeat> heartbeat;
+  seastar::timer<seastar::lowres_clock> heartbeat_timer;
+
 public:
   OSD(int id, uint32_t nonce,
       ceph::net::Messenger& cluster_msgr,
@@ -139,7 +139,6 @@ private:
   seastar::future<> _send_alive();
 
   // OSDMapService methods
-  ceph::signedspan get_mnow() const final;
   epoch_t get_up_epoch() const final {
     return up_epoch;
   }

diff --git a/src/crimson/osd/osdmap_service.h b/src/crimson/osd/osdmap_service.h
index f50b7ac6c2257cc480ec718e9cbc57d2ec582fd8..effd45b7946489b01334020170a9ab96cb8899ee 100644 (file)
--- a/src/crimson/osd/osdmap_service.h
+++ b/src/crimson/osd/osdmap_service.h
@@ -16,6 +16,5 @@ public:
   virtual seastar::future<cached_map_t> get_map(epoch_t e) = 0;
   /// get the latest map
   virtual cached_map_t get_map() const = 0;
-  virtual ceph::signedspan get_mnow() const = 0;
   virtual epoch_t get_up_epoch() const = 0;
 };

diff --git a/src/crimson/osd/pg.cc b/src/crimson/osd/pg.cc
index 686313dec70639b7ad6b5b2838e27d30270943be..5669285d0e1b6139beb8510bead3e2dbc25f8851 100644 (file)
--- a/src/crimson/osd/pg.cc
+++ b/src/crimson/osd/pg.cc
@@ -174,6 +174,16 @@ void PG::log_state_exit(
     events);
 }
 
+ceph::signedspan PG::get_mnow()
+{
+  return shard_services.get_mnow();
+}
+
+HeartbeatStampsRef PG::get_hb_stamps(int peer)
+{
+  return shard_services.get_hb_stamps(peer);
+}
+
 void PG::init(
   ceph::os::CollectionRef coll,
   int role,

diff --git a/src/crimson/osd/pg.h b/src/crimson/osd/pg.h
index 71ea028b0daabee7ac97c0a903fcd2076005766e..37489c92f019802d301294a7aed8210d40dcd6e7 100644 (file)
--- a/src/crimson/osd/pg.h
+++ b/src/crimson/osd/pg.h
@@ -377,12 +377,8 @@ public:
     return OstreamTemp(CLOG_ERROR, nullptr);
   }
 
-  ceph::signedspan get_mnow() final {
-#warning writeme
-  }
-  HeartbeatStampsRef get_hb_stamps(int peer) final {
-#warning write me
-  }
+  ceph::signedspan get_mnow() final;
+  HeartbeatStampsRef get_hb_stamps(int peer) final;
 
   // Utility
   bool is_primary() const {

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index c27d5bf25d6a16c6d5b24f3bf361ac7200212db1..a1052f5a02822b5ef97a18bd359a7da4525834a2 100644 (file)
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -27,12 +27,14 @@ namespace {
 namespace ceph::osd {
 
 ShardServices::ShardServices(
+  OSDMapService &osdmap_service,
   ceph::net::Messenger &cluster_msgr,
   ceph::net::Messenger &public_msgr,
   ceph::mon::Client &monc,
   ceph::mgr::Client &mgrc,
   ceph::os::FuturizedStore &store)
-    : cluster_msgr(cluster_msgr),
+    : osdmap_service(osdmap_service),
+      cluster_msgr(cluster_msgr),
       public_msgr(public_msgr),
       monc(monc),
       mgrc(mgrc),
@@ -245,14 +247,10 @@ seastar::future<> ShardServices::osdmap_subscribe(version_t epoch, bool force_re
   }
 }
 
-ceph::signedspan ShardServices::get_mnow()
-{
-#warning write me
-}
-
 HeartbeatStampsRef ShardServices::get_hb_stamps(int peer)
 {
-#warning write me
+#warning writeme
+  return HeartbeatStampsRef();
 }
 
 };

diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 388335c7a0afc1bd9cdcde4fcdc4fda948f75f0e..c89c4ddf6f6cd2840f6fa20180abdfa73588c549 100644 (file)
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -10,6 +10,7 @@
 #include "msg/MessageRef.h"
 #include "crimson/os/cyan_collection.h"
 #include "osd/PeeringState.h"
+#include "crimson/osd/osdmap_service.h"
 
 namespace ceph::net {
   class Messenger;
@@ -39,6 +40,7 @@ namespace ceph::osd {
  */
 class ShardServices {
   using cached_map_t = boost::local_shared_ptr<const OSDMap>;
+  OSDMapService &osdmap_service;
   ceph::net::Messenger &cluster_msgr;
   ceph::net::Messenger &public_msgr;
   ceph::mon::Client &monc;
@@ -52,6 +54,7 @@ class ShardServices {
 
 public:
   ShardServices(
+    OSDMapService &osdmap_service,
     ceph::net::Messenger &cluster_msgr,
     ceph::net::Messenger &public_msgr,
     ceph::mon::Client &monc,
@@ -71,6 +74,11 @@ public:
     return &cct;
   }
 
+  // OSDMapService
+  const OSDMapService &get_osdmap_service() const {
+    return osdmap_service;
+  }
+
   // Op Tracking
   OperationRegistry registry;
 
@@ -143,9 +151,12 @@ public:
 
   seastar::future<> osdmap_subscribe(version_t epoch, bool force_request);
 
-  ceph::signedspan get_mnow();
+  // Time state
+  ceph::mono_time startup_time = ceph::mono_clock::now();
+  ceph::signedspan get_mnow() const {
+    return ceph::mono_clock::now() - startup_time;
+  }
   HeartbeatStampsRef get_hb_stamps(int peer);
-
 };