From: chunmei-liu Date: Tue, 7 Dec 2021 23:31:24 +0000 (-0800) Subject: crimson/osd: fix heartbeat front and back blank ip X-Git-Tag: v17.1.0~273^2 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=ce1ca97f840b1997575c5a910d41019a40b06d71;p=ceph-ci.git crimson/osd: fix heartbeat front and back blank ip when ceph.conf not set public ip & cluster ip, heartbeat will get blank ip address. when osd::_send_boot , classic osd will check if heartbeat front and back addrs are blank ip, if they are blank ip, will use public ip which is learned from mon to set into them. So implement them in crimson osd. Signed-off-by: chunmei-liu --- diff --git a/src/crimson/net/Messenger.h b/src/crimson/net/Messenger.h index 5be2d689c3a..e7d57233a64 100644 --- a/src/crimson/net/Messenger.h +++ b/src/crimson/net/Messenger.h @@ -36,13 +36,15 @@ using Throttle = crimson::common::Throttle; using SocketPolicy = ceph::net::Policy; class Messenger { - entity_name_t my_name; - entity_addrvec_t my_addrs; uint32_t crc_flags = 0; crimson::auth::AuthClient* auth_client = nullptr; crimson::auth::AuthServer* auth_server = nullptr; bool require_authorizer = true; +protected: + entity_name_t my_name; + entity_addrvec_t my_addrs; + public: Messenger(const entity_name_t& name) : my_name(name) @@ -61,6 +63,7 @@ public: my_addrs = addrs; return seastar::now(); } + virtual bool set_addr_unknowns(const entity_addrvec_t &addrs) = 0; using bind_ertr = crimson::errorator< crimson::ct_error::address_in_use, // The address (range) is already bound diff --git a/src/crimson/net/SocketMessenger.cc b/src/crimson/net/SocketMessenger.cc index 8f8ffd8f19a..2dd3bbf2dab 100644 --- a/src/crimson/net/SocketMessenger.cc +++ b/src/crimson/net/SocketMessenger.cc @@ -45,6 +45,33 @@ SocketMessenger::~SocketMessenger() ceph_assert(!listener); } +bool SocketMessenger::set_addr_unknowns(const entity_addrvec_t &addrs) +{ + bool ret = false; + + entity_addrvec_t newaddrs = my_addrs; + for (auto& a : newaddrs.v) { + if (a.is_blank_ip()) { + int type = a.get_type(); + int port = a.get_port(); + uint32_t nonce = a.get_nonce(); + for (auto& b : addrs.v) { + if (a.get_family() == b.get_family()) { + logger().debug(" assuming my addr {} matches provided addr {}", a, b); + a = b; + a.set_nonce(nonce); + a.set_type(type); + a.set_port(port); + ret = true; + break; + } + } + } + } + my_addrs = newaddrs; + return ret; +} + seastar::future<> SocketMessenger::set_myaddrs(const entity_addrvec_t& addrs) { assert(seastar::this_shard_id() == master_sid); diff --git a/src/crimson/net/SocketMessenger.h b/src/crimson/net/SocketMessenger.h index 32e5988aa2a..cc01f4de076 100644 --- a/src/crimson/net/SocketMessenger.h +++ b/src/crimson/net/SocketMessenger.h @@ -63,6 +63,7 @@ class SocketMessenger final : public Messenger { seastar::future<> set_myaddrs(const entity_addrvec_t& addr) override; + bool set_addr_unknowns(const entity_addrvec_t &addr) override; // Messenger interfaces are assumed to be called from its own shard, but its // behavior should be symmetric when called from any shard. bind_ertr::future<> bind(const entity_addrvec_t& addr) override; diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index 02d6c0ea273..4073f80ae3c 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -107,6 +107,16 @@ const entity_addrvec_t& Heartbeat::get_back_addrs() const return back_msgr->get_myaddrs(); } +crimson::net::MessengerRef Heartbeat::get_front_msgr() const +{ + return front_msgr; +} + +crimson::net::MessengerRef Heartbeat::get_back_msgr() const +{ + return back_msgr; +} + void Heartbeat::set_require_authorizer(bool require_authorizer) { if (front_msgr->get_require_authorizer() != require_authorizer) { diff --git a/src/crimson/osd/heartbeat.h b/src/crimson/osd/heartbeat.h index b4b295abc73..655581b9973 100644 --- a/src/crimson/osd/heartbeat.h +++ b/src/crimson/osd/heartbeat.h @@ -45,6 +45,8 @@ public: const entity_addrvec_t& get_front_addrs() const; const entity_addrvec_t& get_back_addrs() const; + crimson::net::MessengerRef get_front_msgr() const; + crimson::net::MessengerRef get_back_msgr() const; void set_require_authorizer(bool); // Dispatcher methods diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 7b27704f5f6..06f2209d3e9 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -446,15 +446,29 @@ seastar::future<> OSD::_send_boot() { state.set_booting(); - logger().info("hb_back_msgr: {}", heartbeat->get_back_addrs()); - logger().info("hb_front_msgr: {}", heartbeat->get_front_addrs()); - logger().info("cluster_msgr: {}", cluster_msgr->get_myaddr()); + entity_addrvec_t public_addrs = public_msgr->get_myaddrs(); + entity_addrvec_t cluster_addrs = cluster_msgr->get_myaddrs(); + entity_addrvec_t hb_back_addrs = heartbeat->get_back_addrs(); + entity_addrvec_t hb_front_addrs = heartbeat->get_front_addrs(); + if (cluster_msgr->set_addr_unknowns(public_addrs)) { + cluster_addrs = cluster_msgr->get_myaddrs(); + } + if (heartbeat->get_back_msgr()->set_addr_unknowns(cluster_addrs)) { + hb_back_addrs = heartbeat->get_back_addrs(); + } + if (heartbeat->get_front_msgr()->set_addr_unknowns(public_addrs)) { + hb_front_addrs = heartbeat->get_front_addrs(); + } + logger().info("hb_back_msgr: {}", hb_back_addrs); + logger().info("hb_front_msgr: {}", hb_front_addrs); + logger().info("cluster_msgr: {}", cluster_addrs); + auto m = crimson::make_message(superblock, osdmap->get_epoch(), boot_epoch, - heartbeat->get_back_addrs(), - heartbeat->get_front_addrs(), - cluster_msgr->get_myaddrs(), + hb_back_addrs, + hb_front_addrs, + cluster_addrs, CEPH_FEATURES_ALL); collect_sys_info(&m->metadata, NULL); return monc->send_message(std::move(m));