mon: add support public_bind_addr option

To support running in dynamic enviornments (like Kubernetes) the mon needs
to be able to advertise and ip address that is different from the ip address
that it listens on locally.

Added a new config option "public_bind_addr" which if set becomes the address
that the mon will bind to locally. If empty (the default) the public_addr
will be used to bind locally.

added a new function on Messenger to set_addr which is called by ceph-mon to set
the advertised address after doing the bind.

also relaxed the "wrong node!" errors in AsyncMessenger and SimpleMessenger as
its now valid to talk to a peer whose peer_addr_of_me is different from what
we expect.

Signed-off-by: Bassam Tabbara <bassam.tabbara@quantum.com>

diff --git a/src/ceph_mon.cc b/src/ceph_mon.cc
index ec60da2f7b8882d40b92a925b8a593504cc84815..400c8989545cf1994c9182a8f16cf90427f77d3d 100644 (file)
--- a/src/ceph_mon.cc
+++ b/src/ceph_mon.cc
@@ -707,18 +707,38 @@ int main(int argc, const char **argv)
   msgr->set_policy_throttlers(entity_name_t::TYPE_MDS, daemon_throttler,
                                     NULL);
 
+  entity_addr_t bind_addr = ipaddr;
+  entity_addr_t public_addr = ipaddr;
+
+  // check if the public_bind_addr option is set
+  if (!g_conf->public_bind_addr.is_blank_ip()) {
+    bind_addr = g_conf->public_bind_addr;
+
+    // set the default port if not already set
+    if (bind_addr.get_port() == 0) {
+      bind_addr.set_port(CEPH_MON_PORT);
+    }
+  }
+
   dout(0) << "starting " << g_conf->name << " rank " << rank
-       << " at " << ipaddr
+       << " at public addr " << public_addr
+       << " at bind addr " << bind_addr
        << " mon_data " << g_conf->mon_data
        << " fsid " << monmap.get_fsid()
        << dendl;
 
-  err = msgr->bind(ipaddr);
+  err = msgr->bind(bind_addr);
   if (err < 0) {
-    derr << "unable to bind monitor to " << ipaddr << dendl;
+    derr << "unable to bind monitor to " << bind_addr << dendl;
     prefork.exit(1);
   }
 
+  // if the public and bind addr are different set the msgr addr
+  // to the public one, now that the bind is complete.
+  if (public_addr != bind_addr) {
+    msgr->set_addr(public_addr);
+  }
+
   Messenger *mgr_msgr = Messenger::create(g_ceph_context, public_msgr_type,
                                          entity_name_t::MON(rank), "mon-mgrc",
                                          getpid(), 0);
@@ -803,4 +823,3 @@ int main(int argc, const char **argv)
   prefork.signal_exit(0);
   return 0;
 }
-

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 15dc65600e0437615f8b533f04e89b68bfd8c4c2..835a73d400a64e441af7fb63d786c5c4cf2e219f 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -16,6 +16,7 @@
 OPTION(host, OPT_STR, "") // "" means that ceph will use short hostname
 OPTION(fsid, OPT_UUID, uuid_d())
 OPTION(public_addr, OPT_ADDR, entity_addr_t())
+OPTION(public_bind_addr, OPT_ADDR, entity_addr_t())
 OPTION(cluster_addr, OPT_ADDR, entity_addr_t())
 OPTION(public_network, OPT_STR, "")
 OPTION(cluster_network, OPT_STR, "")

diff --git a/src/msg/Messenger.h b/src/msg/Messenger.h
index a186ec3c875a3a4ee30630fad2261b6896621e11..7c1a0d1fad5febe9ee507215aa37910508f4c9bc 100644 (file)
--- a/src/msg/Messenger.h
+++ b/src/msg/Messenger.h
@@ -255,6 +255,14 @@ public:
    * @param addr The address to use as a template.
    */
   virtual void set_addr_unknowns(const entity_addr_t &addr) = 0;
+  /**
+   * Set the address for this Messenger. This is useful if the Messenger
+   * binds to a specific address but advertises a different address on the
+   * the network.
+   *
+   * @param addr The address to use.
+   */
+  virtual void set_addr(const entity_addr_t &addr) = 0;
   /// Get the default send priority.
   int get_default_send_priority() { return default_send_priority; }
   /**

diff --git a/src/msg/async/AsyncConnection.cc b/src/msg/async/AsyncConnection.cc
index ef5a2c2ef9135bd6b66e1ade60691a9668ea4d98..3c535f765337b71689f16bbf4c000ffd7e43a440 100644 (file)
--- a/src/msg/async/AsyncConnection.cc
+++ b/src/msg/async/AsyncConnection.cc
@@ -977,9 +977,9 @@ ssize_t AsyncConnection::_process_connection()
                                 << " not " << peer_addr
                                 << " - presumably this is the same node!" << dendl;
           } else {
-            ldout(async_msgr->cct, 0) << __func__ << " connect claims to be "
-                                << paddr << " not " << peer_addr << " - wrong node!" << dendl;
-            goto fail;
+            ldout(async_msgr->cct, 10) << __func__ << " connect claims to be "
+                                << paddr << " not " << peer_addr
+                                << " (peer is possibly using public_bind_addr?) " << dendl;
           }
         }
 

diff --git a/src/msg/async/AsyncMessenger.cc b/src/msg/async/AsyncMessenger.cc
index 4695844055f122e965b1260308dc0477c46de9a7..1913e8f4c6eea8bb1f530a248094a21475e24925 100644 (file)
--- a/src/msg/async/AsyncMessenger.cc
+++ b/src/msg/async/AsyncMessenger.cc
@@ -634,6 +634,15 @@ void AsyncMessenger::set_addr_unknowns(const entity_addr_t &addr)
   }
 }
 
+void AsyncMessenger::set_addr(const entity_addr_t &addr)
+{
+  Mutex::Locker l(lock);
+  entity_addr_t t = addr;
+  t.set_nonce(nonce);
+  set_myaddr(t);
+  _init_local_connection();
+}
+
 void AsyncMessenger::shutdown_connections(bool queue_reset)
 {
   ldout(cct,1) << __func__ << " " << dendl;

diff --git a/src/msg/async/AsyncMessenger.h b/src/msg/async/AsyncMessenger.h
index 0b1f2c8e9308d793cc5831f46a1cfd43813d7288..7ebc7777c93e611f1ca49efa78f5edc4654a0b61 100644 (file)
--- a/src/msg/async/AsyncMessenger.h
+++ b/src/msg/async/AsyncMessenger.h
@@ -96,6 +96,7 @@ public:
    * @{
    */
   void set_addr_unknowns(const entity_addr_t &addr) override;
+  void set_addr(const entity_addr_t &addr) override;
 
   int get_dispatch_queue_len() override {
     return dispatch_queue.get_queue_len();

diff --git a/src/msg/simple/Pipe.cc b/src/msg/simple/Pipe.cc
index fc415df8a093978609f8b885426636703f8d4de3..355c2528f93801a4eea6957c443145293cab807b 100644 (file)
--- a/src/msg/simple/Pipe.cc
+++ b/src/msg/simple/Pipe.cc
@@ -1089,9 +1089,9 @@ int Pipe::connect()
       ldout(msgr->cct,0) << "connect claims to be " 
              << paddr << " not " << peer_addr << " - presumably this is the same node!" << dendl;
     } else {
-      ldout(msgr->cct,0) << "connect claims to be " 
-             << paddr << " not " << peer_addr << " - wrong node!" << dendl;
-      goto fail;
+      ldout(msgr->cct,10) << "connect claims to be "
+             << paddr << " not " << peer_addr
+              << " (peer is possibly using public_bind_addr?) " << dendl;
     }
   }
 

diff --git a/src/msg/simple/SimpleMessenger.cc b/src/msg/simple/SimpleMessenger.cc
index 84b6a253ec6e6ee72499e312d4d6737d09841f85..78e190d027e1c65c34792face5597938a285a41d 100644 (file)
--- a/src/msg/simple/SimpleMessenger.cc
+++ b/src/msg/simple/SimpleMessenger.cc
@@ -159,6 +159,14 @@ void SimpleMessenger::set_addr_unknowns(const entity_addr_t &addr)
   }
 }
 
+void SimpleMessenger::set_addr(const entity_addr_t &addr)
+{
+  entity_addr_t t = addr;
+  t.set_nonce(nonce);
+  set_myaddr(t);
+  init_local_connection();
+}
+
 int SimpleMessenger::get_proto_version(int peer_type, bool connect)
 {
   int my_type = my_inst.name.type();

diff --git a/src/msg/simple/SimpleMessenger.h b/src/msg/simple/SimpleMessenger.h
index 4ddc9767c4c70565e95b725f70d6f2424c3c3e1a..0a0512382eb3c337e2b8fb473fed3a6bde712466 100644 (file)
--- a/src/msg/simple/SimpleMessenger.h
+++ b/src/msg/simple/SimpleMessenger.h
@@ -93,6 +93,7 @@ public:
    * @{
    */
   void set_addr_unknowns(const entity_addr_t& addr) override;
+  void set_addr(const entity_addr_t &addr) override;
 
   int get_dispatch_queue_len() override {
     return dispatch_queue.get_queue_len();

diff --git a/src/msg/xio/XioMessenger.h b/src/msg/xio/XioMessenger.h
index 9a81fb2473a129f8d3fa49b47501eb0684a5f2d9..ea20d36bd8c5ee00060566718802abf060f8e3f1 100644 (file)
--- a/src/msg/xio/XioMessenger.h
+++ b/src/msg/xio/XioMessenger.h
@@ -100,6 +100,8 @@ public:
   /* Messenger interface */
   virtual void set_addr_unknowns(const entity_addr_t &addr) override
     { } /* XXX applicable? */
+  virtual void set_addr(const entity_addr_t &addr) override
+    { } /* XXX applicable? */
 
   virtual int get_dispatch_queue_len()
     { return 0; } /* XXX bogus? */

diff --git a/src/test/direct_messenger/DirectMessenger.h b/src/test/direct_messenger/DirectMessenger.h
index dd9d39ed9d32920d5e9ece621d13aab5f1ee456f..710fcfb373042fae7fa2ab1fa136456d29ffd031 100644 (file)
--- a/src/test/direct_messenger/DirectMessenger.h
+++ b/src/test/direct_messenger/DirectMessenger.h
@@ -89,6 +89,7 @@ class DirectMessenger : public SimplePolicyMessenger {
 
   // unimplemented Messenger interface
   void set_addr_unknowns(const entity_addr_t &addr) override {}
+  void set_addr(const entity_addr_t &addr) override {}
   int get_dispatch_queue_len() override { return 0; }
   double get_dispatch_queue_max_age(utime_t now) override { return 0; }
   void set_cluster_protocol(int p) override {}