From f72fe9c7ec94d04ca15a2361c7d0c4af666d43fa Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 23 Feb 2023 06:50:56 +0000 Subject: [PATCH] crimson/osd/heartbeat: share osdmap to peers when necessary Signed-off-by: Xuehan Xu Signed-off-by: Matan Breizman (cherry picked from commit 670bbaa4d858de3dd50d857370f76bbd1a13c702) --- src/crimson/osd/heartbeat.cc | 30 +++++++++++++++++++++++++++--- src/crimson/osd/heartbeat.h | 6 ++++-- src/crimson/osd/shard_services.cc | 14 +++++++++++++- src/crimson/osd/shard_services.h | 3 +++ 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index 94b3d688627..eb476f715da 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -30,7 +30,7 @@ namespace { } Heartbeat::Heartbeat(osd_id_t whoami, - const crimson::osd::ShardServices& service, + crimson::osd::ShardServices& service, crimson::mon::Client& monc, crimson::net::Messenger &front_msgr, crimson::net::Messenger &back_msgr) @@ -292,7 +292,28 @@ seastar::future<> Heartbeat::handle_ping(crimson::net::ConnectionRef conn, service.get_mnow(), service.get_up_epoch(), min_message); - return conn->send(std::move(reply)); + return conn->send(std::move(reply) + ).then([this, m, conn] { + return maybe_share_osdmap(conn, m); + }); +} + +seastar::future<> Heartbeat::maybe_share_osdmap( + crimson::net::ConnectionRef conn, + Ref m) +{ + const osd_id_t from = m->get_source().num(); + const epoch_t osdmap_epoch = service.get_map()->get_epoch(); + logger().info("{} peer id: {} epoch is {} while osdmap is {}", + __func__ , from, m->map_epoch, osdmap_epoch); + if (osdmap_epoch > m->map_epoch) { + logger().debug("{} sharing osdmap epoch of {} with peer id {}", + __func__, osdmap_epoch, from); + // Peer's newest map is m->map_epoch. Therfore it misses + // the osdmaps in the range of `m->map_epoch` to `osdmap_epoch`. + return service.send_incremental_map_to_osd(from, m->map_epoch); + } + return seastar::now(); } seastar::future<> Heartbeat::handle_reply(crimson::net::ConnectionRef conn, @@ -305,7 +326,10 @@ seastar::future<> Heartbeat::handle_reply(crimson::net::ConnectionRef conn, return seastar::now(); } auto& peer = found->second; - return peer.handle_reply(conn, m); + return peer.handle_reply(conn, m + ).then([this, conn, m] { + return maybe_share_osdmap(conn, m); + }); } seastar::future<> Heartbeat::handle_you_died() diff --git a/src/crimson/osd/heartbeat.h b/src/crimson/osd/heartbeat.h index 2d9cd8b2435..12c750b0762 100644 --- a/src/crimson/osd/heartbeat.h +++ b/src/crimson/osd/heartbeat.h @@ -27,7 +27,7 @@ public: using osd_id_t = int; Heartbeat(osd_id_t whoami, - const crimson::osd::ShardServices& service, + crimson::osd::ShardServices& service, crimson::mon::Client& monc, crimson::net::Messenger &front_msgr, crimson::net::Messenger &back_msgr); @@ -73,9 +73,11 @@ private: seastar::future<> start_messenger(crimson::net::Messenger& msgr, const entity_addrvec_t& addrs); + seastar::future<> maybe_share_osdmap(crimson::net::ConnectionRef, + Ref m); private: const osd_id_t whoami; - const crimson::osd::ShardServices& service; + crimson::osd::ShardServices& service; crimson::mon::Client& monc; crimson::net::Messenger &front_msgr; crimson::net::Messenger &back_msgr; diff --git a/src/crimson/osd/shard_services.cc b/src/crimson/osd/shard_services.cc index c8668ea8fe9..623570d0e23 100644 --- a/src/crimson/osd/shard_services.cc +++ b/src/crimson/osd/shard_services.cc @@ -730,6 +730,18 @@ seastar::future<> OSDSingletonState::send_incremental_map( } } - +seastar::future<> OSDSingletonState::send_incremental_map_to_osd( + int osd, + epoch_t first) +{ + if (osdmap->is_down(osd)) { + logger().info("{}: osd.{} is_down", __func__, osd); + return seastar::now(); + } else { + auto conn = cluster_msgr.connect( + osdmap->get_cluster_addrs(osd).front(), CEPH_ENTITY_TYPE_OSD); + return send_incremental_map(*conn, first); + } +} }; diff --git a/src/crimson/osd/shard_services.h b/src/crimson/osd/shard_services.h index 3cf807e8ca3..167f2fbbecb 100644 --- a/src/crimson/osd/shard_services.h +++ b/src/crimson/osd/shard_services.h @@ -235,6 +235,8 @@ private: crimson::net::Connection &conn, epoch_t first); + seastar::future<> send_incremental_map_to_osd(int osd, epoch_t first); + auto get_pool_info(int64_t poolid) { return get_meta_coll().load_final_pool_info(poolid); } @@ -451,6 +453,7 @@ public: FORWARD(with_throttle_while, with_throttle_while, local_state.throttler) FORWARD_TO_OSD_SINGLETON(send_incremental_map) + FORWARD_TO_OSD_SINGLETON(send_incremental_map_to_osd) FORWARD_TO_OSD_SINGLETON(osdmap_subscribe) FORWARD_TO_OSD_SINGLETON(queue_want_pg_temp) -- 2.39.5