From ce1ca97f840b1997575c5a910d41019a40b06d71 Mon Sep 17 00:00:00 2001 From: chunmei-liu Date: Tue, 7 Dec 2021 15:31:24 -0800 Subject: [PATCH] crimson/osd: fix heartbeat front and back blank ip when ceph.conf not set public ip & cluster ip, heartbeat will get blank ip address. when osd::_send_boot , classic osd will check if heartbeat front and back addrs are blank ip, if they are blank ip, will use public ip which is learned from mon to set into them. So implement them in crimson osd. Signed-off-by: chunmei-liu --- src/crimson/net/Messenger.h | 7 +++++-- src/crimson/net/SocketMessenger.cc | 27 +++++++++++++++++++++++++++ src/crimson/net/SocketMessenger.h | 1 + src/crimson/osd/heartbeat.cc | 10 ++++++++++ src/crimson/osd/heartbeat.h | 2 ++ src/crimson/osd/osd.cc | 26 ++++++++++++++++++++------ 6 files changed, 65 insertions(+), 8 deletions(-) diff --git a/src/crimson/net/Messenger.h b/src/crimson/net/Messenger.h index 5be2d689c3a1d..e7d57233a643c 100644 --- a/src/crimson/net/Messenger.h +++ b/src/crimson/net/Messenger.h @@ -36,13 +36,15 @@ using Throttle = crimson::common::Throttle; using SocketPolicy = ceph::net::Policy; class Messenger { - entity_name_t my_name; - entity_addrvec_t my_addrs; uint32_t crc_flags = 0; crimson::auth::AuthClient* auth_client = nullptr; crimson::auth::AuthServer* auth_server = nullptr; bool require_authorizer = true; +protected: + entity_name_t my_name; + entity_addrvec_t my_addrs; + public: Messenger(const entity_name_t& name) : my_name(name) @@ -61,6 +63,7 @@ public: my_addrs = addrs; return seastar::now(); } + virtual bool set_addr_unknowns(const entity_addrvec_t &addrs) = 0; using bind_ertr = crimson::errorator< crimson::ct_error::address_in_use, // The address (range) is already bound diff --git a/src/crimson/net/SocketMessenger.cc b/src/crimson/net/SocketMessenger.cc index 8f8ffd8f19ae0..2dd3bbf2dabfe 100644 --- a/src/crimson/net/SocketMessenger.cc +++ b/src/crimson/net/SocketMessenger.cc @@ -45,6 +45,33 @@ SocketMessenger::~SocketMessenger() ceph_assert(!listener); } +bool SocketMessenger::set_addr_unknowns(const entity_addrvec_t &addrs) +{ + bool ret = false; + + entity_addrvec_t newaddrs = my_addrs; + for (auto& a : newaddrs.v) { + if (a.is_blank_ip()) { + int type = a.get_type(); + int port = a.get_port(); + uint32_t nonce = a.get_nonce(); + for (auto& b : addrs.v) { + if (a.get_family() == b.get_family()) { + logger().debug(" assuming my addr {} matches provided addr {}", a, b); + a = b; + a.set_nonce(nonce); + a.set_type(type); + a.set_port(port); + ret = true; + break; + } + } + } + } + my_addrs = newaddrs; + return ret; +} + seastar::future<> SocketMessenger::set_myaddrs(const entity_addrvec_t& addrs) { assert(seastar::this_shard_id() == master_sid); diff --git a/src/crimson/net/SocketMessenger.h b/src/crimson/net/SocketMessenger.h index 32e5988aa2a13..cc01f4de0768a 100644 --- a/src/crimson/net/SocketMessenger.h +++ b/src/crimson/net/SocketMessenger.h @@ -63,6 +63,7 @@ class SocketMessenger final : public Messenger { seastar::future<> set_myaddrs(const entity_addrvec_t& addr) override; + bool set_addr_unknowns(const entity_addrvec_t &addr) override; // Messenger interfaces are assumed to be called from its own shard, but its // behavior should be symmetric when called from any shard. bind_ertr::future<> bind(const entity_addrvec_t& addr) override; diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index 02d6c0ea27321..4073f80ae3cf4 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -107,6 +107,16 @@ const entity_addrvec_t& Heartbeat::get_back_addrs() const return back_msgr->get_myaddrs(); } +crimson::net::MessengerRef Heartbeat::get_front_msgr() const +{ + return front_msgr; +} + +crimson::net::MessengerRef Heartbeat::get_back_msgr() const +{ + return back_msgr; +} + void Heartbeat::set_require_authorizer(bool require_authorizer) { if (front_msgr->get_require_authorizer() != require_authorizer) { diff --git a/src/crimson/osd/heartbeat.h b/src/crimson/osd/heartbeat.h index b4b295abc73bb..655581b997360 100644 --- a/src/crimson/osd/heartbeat.h +++ b/src/crimson/osd/heartbeat.h @@ -45,6 +45,8 @@ public: const entity_addrvec_t& get_front_addrs() const; const entity_addrvec_t& get_back_addrs() const; + crimson::net::MessengerRef get_front_msgr() const; + crimson::net::MessengerRef get_back_msgr() const; void set_require_authorizer(bool); // Dispatcher methods diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 7b27704f5f6ee..06f2209d3e964 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -446,15 +446,29 @@ seastar::future<> OSD::_send_boot() { state.set_booting(); - logger().info("hb_back_msgr: {}", heartbeat->get_back_addrs()); - logger().info("hb_front_msgr: {}", heartbeat->get_front_addrs()); - logger().info("cluster_msgr: {}", cluster_msgr->get_myaddr()); + entity_addrvec_t public_addrs = public_msgr->get_myaddrs(); + entity_addrvec_t cluster_addrs = cluster_msgr->get_myaddrs(); + entity_addrvec_t hb_back_addrs = heartbeat->get_back_addrs(); + entity_addrvec_t hb_front_addrs = heartbeat->get_front_addrs(); + if (cluster_msgr->set_addr_unknowns(public_addrs)) { + cluster_addrs = cluster_msgr->get_myaddrs(); + } + if (heartbeat->get_back_msgr()->set_addr_unknowns(cluster_addrs)) { + hb_back_addrs = heartbeat->get_back_addrs(); + } + if (heartbeat->get_front_msgr()->set_addr_unknowns(public_addrs)) { + hb_front_addrs = heartbeat->get_front_addrs(); + } + logger().info("hb_back_msgr: {}", hb_back_addrs); + logger().info("hb_front_msgr: {}", hb_front_addrs); + logger().info("cluster_msgr: {}", cluster_addrs); + auto m = crimson::make_message(superblock, osdmap->get_epoch(), boot_epoch, - heartbeat->get_back_addrs(), - heartbeat->get_front_addrs(), - cluster_msgr->get_myaddrs(), + hb_back_addrs, + hb_front_addrs, + cluster_addrs, CEPH_FEATURES_ALL); collect_sys_info(&m->metadata, NULL); return monc->send_message(std::move(m)); -- 2.39.5