crimson/osd: update superblock managment

introduce PerShardState::superblock to be updated on
each shard once the OSDSingletonState instance is updated.

Note: PerShardState::superblock is a dependency for
      PG::cluster_osdmap_trim_lower_bound().

Signed-off-by: Matan Breizman <mbreizma@redhat.com>

diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc
index 53004f387b8862da6992524ea5302e61f821907f..cf1984d47d9a60731c9b5889815e1524fa48896f 100644 (file)
--- a/src/crimson/osd/osd.cc
+++ b/src/crimson/osd/osd.cc
@@ -403,7 +403,8 @@ seastar::future<> OSD::start()
     if (!superblock.cluster_osdmap_trim_lower_bound) {
       superblock.cluster_osdmap_trim_lower_bound = superblock.get_oldest_map();
     }
-    pg_shard_manager.set_superblock(superblock);
+    return pg_shard_manager.set_superblock(superblock);
+  }).then([this] {
     return pg_shard_manager.get_local_map(superblock.current_epoch);
   }).then([this](OSDMapService::local_cached_map_t&& map) {
     osdmap = make_local_shared_foreign(OSDMapService::local_cached_map_t(map));
@@ -989,11 +990,13 @@ seastar::future<> OSD::_handle_osd_map(Ref<MOSDMap> m)
         superblock.clean_thru = last;
       }
       pg_shard_manager.get_meta_coll().store_superblock(t, superblock);
-      pg_shard_manager.set_superblock(superblock);
-      logger().debug("OSD::handle_osd_map: do_transaction...");
-      return store.get_sharded_store().do_transaction(
-       pg_shard_manager.get_meta_coll().collection(),
-       std::move(t));
+      return pg_shard_manager.set_superblock(superblock).then(
+      [this, &t] {
+        logger().debug("OSD::handle_osd_map: do_transaction...");
+        return store.get_sharded_store().do_transaction(
+          pg_shard_manager.get_meta_coll().collection(),
+          std::move(t));
+      });
     });
   }).then([=, this] {
     // TODO: write to superblock and commit the transaction

diff --git a/src/crimson/osd/pg_shard_manager.cc b/src/crimson/osd/pg_shard_manager.cc
index 6061c856be26348435c5c74e878b378434ab8d32..1980b36271c2db4b89115a682b021f30dbb483be 100644 (file)
--- a/src/crimson/osd/pg_shard_manager.cc
+++ b/src/crimson/osd/pg_shard_manager.cc
@@ -105,4 +105,13 @@ seastar::future<> PGShardManager::set_up_epoch(epoch_t e) {
     });
 }
 
+seastar::future<> PGShardManager::set_superblock(OSDSuperblock superblock) {
+  ceph_assert(seastar::this_shard_id() == PRIMARY_CORE);
+  get_osd_singleton_state().set_singleton_superblock(superblock);
+  return shard_services.invoke_on_all(
+  [superblock = std::move(superblock)](auto &local_service) {
+    return local_service.local_state.update_shard_superblock(superblock);
+  });
+}
+
 }

diff --git a/src/crimson/osd/pg_shard_manager.h b/src/crimson/osd/pg_shard_manager.h
index bb74c18e37143c829bcb7e367548f79c48fc2968..1d101ce26dc23646224b3fc3883dce9ea7f7fdaa 100644 (file)
--- a/src/crimson/osd/pg_shard_manager.h
+++ b/src/crimson/osd/pg_shard_manager.h
@@ -129,8 +129,6 @@ public:
   FORWARD_TO_OSD_SINGLETON(init_meta_coll)
   FORWARD_TO_OSD_SINGLETON(get_meta_coll)
 
-  FORWARD_TO_OSD_SINGLETON(set_superblock)
-
   // Core OSDMap methods
   FORWARD_TO_OSD_SINGLETON(get_local_map)
   FORWARD_TO_OSD_SINGLETON(load_map_bl)
@@ -140,6 +138,8 @@ public:
 
   seastar::future<> set_up_epoch(epoch_t e);
 
+  seastar::future<> set_superblock(OSDSuperblock superblock);
+
   template <typename F>
   auto with_remote_shard_state(core_id_t core, F &&f) {
     return shard_services.invoke_on(

diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc
index faa224f99eaebd045d32abc1cecfdb208352cc42..208b7fc98df64cee1b8803ac40bcda9001e896ab 100644 (file)
--- a/src/crimson/osd/shard_services.cc
+++ b/src/crimson/osd/shard_services.cc
@@ -119,6 +119,13 @@ HeartbeatStampsRef PerShardState::get_hb_stamps(int peer)
   return stamps->second;
 }
 
+seastar::future<> PerShardState::update_shard_superblock(OSDSuperblock superblock)
+{
+  assert_core();
+  per_shard_superblock = std::move(superblock);
+  return seastar::now();
+}
+
 OSDSingletonState::OSDSingletonState(
   int whoami,
   crimson::net::Messenger &cluster_msgr,

diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h
index 50a4bc633135928dfb4cbee3fc4b2362c7971740..5df425a243cfe0a377b5dc249aa5e30ec7e9db28 100644 (file)
--- a/src/crimson/osd/shard_services.h
+++ b/src/crimson/osd/shard_services.h
@@ -181,6 +181,8 @@ class PerShardState {
   HeartbeatStampsRef get_hb_stamps(int peer);
   std::map<int, HeartbeatStampsRef> heartbeat_stamps;
 
+  seastar::future<> update_shard_superblock(OSDSuperblock superblock);
+
   // Time state
   const ceph::mono_time startup_time;
   ceph::signedspan get_mnow() const {
@@ -188,6 +190,8 @@ class PerShardState {
     return ceph::mono_clock::now() - startup_time;
   }
 
+  OSDSuperblock per_shard_superblock;
+
 public:
   PerShardState(
     int whoami,
@@ -256,7 +260,7 @@ private:
   }
 
   OSDSuperblock superblock;
-  void set_superblock(OSDSuperblock _superblock) {
+  void set_singleton_superblock(OSDSuperblock _superblock) {
     superblock = std::move(_superblock);
   }
 
@@ -509,6 +513,7 @@ public:
   FORWARD_TO_OSD_SINGLETON(send_pg_temp)
   FORWARD_TO_LOCAL_CONST(get_mnow)
   FORWARD_TO_LOCAL(get_hb_stamps)
+  FORWARD_TO_LOCAL(update_shard_superblock)
 
   FORWARD(pg_created, pg_created, local_state.pg_map)