From bf4938567943c80345966f9c5a3bdc75a913175b Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Thu, 13 Jul 2017 14:05:20 -0700 Subject: [PATCH] mon: add support public_bind_addr option To support running in dynamic enviornments (like Kubernetes) the mon needs to be able to advertise and ip address that is different from the ip address that it listens on locally. Added a new config option "public_bind_addr" which if set becomes the address that the mon will bind to locally. If empty (the default) the public_addr will be used to bind locally. added a new function on Messenger to set_addr which is called by ceph-mon to set the advertised address after doing the bind. also relaxed the "wrong node!" errors in AsyncMessenger and SimpleMessenger as its now valid to talk to a peer whose peer_addr_of_me is different from what we expect. Signed-off-by: Bassam Tabbara --- src/ceph_mon.cc | 27 ++++++++++++++++++--- src/common/config_opts.h | 1 + src/msg/Messenger.h | 8 ++++++ src/msg/async/AsyncConnection.cc | 6 ++--- src/msg/async/AsyncMessenger.cc | 9 +++++++ src/msg/async/AsyncMessenger.h | 1 + src/msg/simple/Pipe.cc | 6 ++--- src/msg/simple/SimpleMessenger.cc | 8 ++++++ src/msg/simple/SimpleMessenger.h | 1 + src/msg/xio/XioMessenger.h | 2 ++ src/test/direct_messenger/DirectMessenger.h | 1 + 11 files changed, 60 insertions(+), 10 deletions(-) diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc index ec60da2f7b8..400c8989545 100644 --- a/src/ceph_mon.cc +++ b/src/ceph_mon.cc @@ -707,18 +707,38 @@ int main(int argc, const char **argv) msgr->set_policy_throttlers(entity_name_t::TYPE_MDS, daemon_throttler, NULL); + entity_addr_t bind_addr = ipaddr; + entity_addr_t public_addr = ipaddr; + + // check if the public_bind_addr option is set + if (!g_conf->public_bind_addr.is_blank_ip()) { + bind_addr = g_conf->public_bind_addr; + + // set the default port if not already set + if (bind_addr.get_port() == 0) { + bind_addr.set_port(CEPH_MON_PORT); + } + } + dout(0) << "starting " << g_conf->name << " rank " << rank - << " at " << ipaddr + << " at public addr " << public_addr + << " at bind addr " << bind_addr << " mon_data " << g_conf->mon_data << " fsid " << monmap.get_fsid() << dendl; - err = msgr->bind(ipaddr); + err = msgr->bind(bind_addr); if (err < 0) { - derr << "unable to bind monitor to " << ipaddr << dendl; + derr << "unable to bind monitor to " << bind_addr << dendl; prefork.exit(1); } + // if the public and bind addr are different set the msgr addr + // to the public one, now that the bind is complete. + if (public_addr != bind_addr) { + msgr->set_addr(public_addr); + } + Messenger *mgr_msgr = Messenger::create(g_ceph_context, public_msgr_type, entity_name_t::MON(rank), "mon-mgrc", getpid(), 0); @@ -803,4 +823,3 @@ int main(int argc, const char **argv) prefork.signal_exit(0); return 0; } - diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 15dc65600e0..835a73d400a 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -16,6 +16,7 @@ OPTION(host, OPT_STR, "") // "" means that ceph will use short hostname OPTION(fsid, OPT_UUID, uuid_d()) OPTION(public_addr, OPT_ADDR, entity_addr_t()) +OPTION(public_bind_addr, OPT_ADDR, entity_addr_t()) OPTION(cluster_addr, OPT_ADDR, entity_addr_t()) OPTION(public_network, OPT_STR, "") OPTION(cluster_network, OPT_STR, "") diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h index a186ec3c875..7c1a0d1fad5 100644 --- a/src/msg/Messenger.h +++ b/src/msg/Messenger.h @@ -255,6 +255,14 @@ public: * @param addr The address to use as a template. */ virtual void set_addr_unknowns(const entity_addr_t &addr) = 0; + /** + * Set the address for this Messenger. This is useful if the Messenger + * binds to a specific address but advertises a different address on the + * the network. + * + * @param addr The address to use. + */ + virtual void set_addr(const entity_addr_t &addr) = 0; /// Get the default send priority. int get_default_send_priority() { return default_send_priority; } /** diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc index ef5a2c2ef91..3c535f76533 100644 --- a/src/msg/async/AsyncConnection.cc +++ b/src/msg/async/AsyncConnection.cc @@ -977,9 +977,9 @@ ssize_t AsyncConnection::_process_connection() << " not " << peer_addr << " - presumably this is the same node!" << dendl; } else { - ldout(async_msgr->cct, 0) << __func__ << " connect claims to be " - << paddr << " not " << peer_addr << " - wrong node!" << dendl; - goto fail; + ldout(async_msgr->cct, 10) << __func__ << " connect claims to be " + << paddr << " not " << peer_addr + << " (peer is possibly using public_bind_addr?) " << dendl; } } diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc index 4695844055f..1913e8f4c6e 100644 --- a/src/msg/async/AsyncMessenger.cc +++ b/src/msg/async/AsyncMessenger.cc @@ -634,6 +634,15 @@ void AsyncMessenger::set_addr_unknowns(const entity_addr_t &addr) } } +void AsyncMessenger::set_addr(const entity_addr_t &addr) +{ + Mutex::Locker l(lock); + entity_addr_t t = addr; + t.set_nonce(nonce); + set_myaddr(t); + _init_local_connection(); +} + void AsyncMessenger::shutdown_connections(bool queue_reset) { ldout(cct,1) << __func__ << " " << dendl; diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h index 0b1f2c8e930..7ebc7777c93 100644 --- a/src/msg/async/AsyncMessenger.h +++ b/src/msg/async/AsyncMessenger.h @@ -96,6 +96,7 @@ public: * @{ */ void set_addr_unknowns(const entity_addr_t &addr) override; + void set_addr(const entity_addr_t &addr) override; int get_dispatch_queue_len() override { return dispatch_queue.get_queue_len(); diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc index fc415df8a09..355c2528f93 100644 --- a/src/msg/simple/Pipe.cc +++ b/src/msg/simple/Pipe.cc @@ -1089,9 +1089,9 @@ int Pipe::connect() ldout(msgr->cct,0) << "connect claims to be " << paddr << " not " << peer_addr << " - presumably this is the same node!" << dendl; } else { - ldout(msgr->cct,0) << "connect claims to be " - << paddr << " not " << peer_addr << " - wrong node!" << dendl; - goto fail; + ldout(msgr->cct,10) << "connect claims to be " + << paddr << " not " << peer_addr + << " (peer is possibly using public_bind_addr?) " << dendl; } } diff --git a/src/msg/simple/SimpleMessenger.cc b/src/msg/simple/SimpleMessenger.cc index 84b6a253ec6..78e190d027e 100644 --- a/src/msg/simple/SimpleMessenger.cc +++ b/src/msg/simple/SimpleMessenger.cc @@ -159,6 +159,14 @@ void SimpleMessenger::set_addr_unknowns(const entity_addr_t &addr) } } +void SimpleMessenger::set_addr(const entity_addr_t &addr) +{ + entity_addr_t t = addr; + t.set_nonce(nonce); + set_myaddr(t); + init_local_connection(); +} + int SimpleMessenger::get_proto_version(int peer_type, bool connect) { int my_type = my_inst.name.type(); diff --git a/src/msg/simple/SimpleMessenger.h b/src/msg/simple/SimpleMessenger.h index 4ddc9767c4c..0a0512382eb 100644 --- a/src/msg/simple/SimpleMessenger.h +++ b/src/msg/simple/SimpleMessenger.h @@ -93,6 +93,7 @@ public: * @{ */ void set_addr_unknowns(const entity_addr_t& addr) override; + void set_addr(const entity_addr_t &addr) override; int get_dispatch_queue_len() override { return dispatch_queue.get_queue_len(); diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h index 9a81fb2473a..ea20d36bd8c 100644 --- a/src/msg/xio/XioMessenger.h +++ b/src/msg/xio/XioMessenger.h @@ -100,6 +100,8 @@ public: /* Messenger interface */ virtual void set_addr_unknowns(const entity_addr_t &addr) override { } /* XXX applicable? */ + virtual void set_addr(const entity_addr_t &addr) override + { } /* XXX applicable? */ virtual int get_dispatch_queue_len() { return 0; } /* XXX bogus? */ diff --git a/src/test/direct_messenger/DirectMessenger.h b/src/test/direct_messenger/DirectMessenger.h index dd9d39ed9d3..710fcfb3730 100644 --- a/src/test/direct_messenger/DirectMessenger.h +++ b/src/test/direct_messenger/DirectMessenger.h @@ -89,6 +89,7 @@ class DirectMessenger : public SimplePolicyMessenger { // unimplemented Messenger interface void set_addr_unknowns(const entity_addr_t &addr) override {} + void set_addr(const entity_addr_t &addr) override {} int get_dispatch_queue_len() override { return 0; } double get_dispatch_queue_max_age(utime_t now) override { return 0; } void set_cluster_protocol(int p) override {} -- 2.39.5