msg/async/ProtocolV[12]: add ms_learn_addr_from_peer

Optionally learn our address based on what our peer (usually the mon) thinks we are
connecting from, as opposed to what our local socket used for the outbound connection.
This enables us to detect what our NATed address is (as a client) in those environments.

Default to true to match the v1 behavior.  This is a net change in
behavior for v2.

Signed-off-by: Sage Weil <sage@redhat.com>

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index f17b2f412c69a2acae6ba601dafcee7e7e0be4ea..c36bd614470230a9fa4416622dffcdb781b24626 100644 (file)
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -101,6 +101,7 @@ OPTION(perf, OPT_BOOL)       // enable internal perf counters
 SAFE_OPTION(ms_type, OPT_STR)   // messenger backend. It will be modified in runtime, so use SAFE_OPTION
 OPTION(ms_public_type, OPT_STR)   // messenger backend
 OPTION(ms_cluster_type, OPT_STR)   // messenger backend
+OPTION(ms_learn_addr_from_peer, OPT_BOOL)
 OPTION(ms_tcp_nodelay, OPT_BOOL)
 OPTION(ms_tcp_rcvbuf, OPT_INT)
 OPTION(ms_tcp_prefetch_max_size, OPT_U32) // max prefetch size, we limit this to avoid extra memcpy

diff --git a/src/common/options.cc b/src/common/options.cc
index 5181cf6c48d5f0165c4c5ab768a992da4f0acff7..cfc6a3c7530cef53f8c66406051183effde37d4a 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -841,6 +841,11 @@ std::vector<Option> get_global_options() {
     .add_see_also("ms_cluster_mode")
     .add_see_also("ms_service_mode"),
 
+    Option("ms_learn_addr_from_peer", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Learn address from what IP our first peer thinks we connect from")
+    .set_long_description("Use the IP address our first peer (usually a monitor) sees that we are connecting from.  This is useful if a client is behind some sort of NAT and we want to see it identified by its local (not NATed) address."),
+
     Option("ms_tcp_nodelay", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Disable Nagle's algorithm and send queued network traffic immediately"),

diff --git a/src/msg/async/ProtocolV1.cc b/src/msg/async/ProtocolV1.cc
index 1f56332f852d278fb94e5b679dd371db9cf43d8d..558e906c253d36a2c71e6f453752cfbc95aa866a 100644 (file)
--- a/src/msg/async/ProtocolV1.cc
+++ b/src/msg/async/ProtocolV1.cc
@@ -1375,25 +1375,44 @@ CtPtr ProtocolV1::handle_server_banner_and_identify(char *buffer, int r) {
 
   ldout(cct, 20) << __func__ << " connect peer addr for me is "
                  << peer_addr_for_me << dendl;
-  connection->lock.unlock();
-  messenger->learned_addr(peer_addr_for_me);
-  if (cct->_conf->ms_inject_internal_delays &&
-      cct->_conf->ms_inject_socket_failures) {
-    if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
-      ldout(cct, 10) << __func__ << " sleep for "
-                     << cct->_conf->ms_inject_internal_delays << dendl;
-      utime_t t;
-      t.set_from_double(cct->_conf->ms_inject_internal_delays);
-      t.sleep();
+  if (messenger->get_myaddrs().empty() ||
+      messenger->get_myaddrs().front().is_blank_ip()) {
+    sockaddr_storage ss;
+    socklen_t len = sizeof(ss);
+    getsockname(connection->cs.fd(), (sockaddr *)&ss, &len);
+    entity_addr_t a;
+    if (cct->_conf->ms_learn_addr_from_peer) {
+      ldout(cct, 1) << __func__ << " peer " << connection->target_addr
+                   << " says I am " << peer_addr_for_me << " (socket says "
+                   << (sockaddr*)&ss << ")" << dendl;
+      a = peer_addr_for_me;
+    } else {
+      ldout(cct, 1) << __func__ << " socket to  " << connection->target_addr
+                   << " says I am " << (sockaddr*)&ss
+                   << " (peer says " << peer_addr_for_me << ")" << dendl;
+      a.set_sockaddr((sockaddr *)&ss);
     }
-  }
-
-  connection->lock.lock();
-  if (state != CONNECTING_WAIT_BANNER_AND_IDENTIFY) {
-    ldout(cct, 1) << __func__
+    a.set_type(entity_addr_t::TYPE_LEGACY); // anything but NONE; learned_addr ignores this
+    a.set_port(0);
+    connection->lock.unlock();
+    messenger->learned_addr(a);
+    if (cct->_conf->ms_inject_internal_delays &&
+       cct->_conf->ms_inject_socket_failures) {
+      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
+       ldout(cct, 10) << __func__ << " sleep for "
+                      << cct->_conf->ms_inject_internal_delays << dendl;
+       utime_t t;
+       t.set_from_double(cct->_conf->ms_inject_internal_delays);
+       t.sleep();
+      }
+    }
+    connection->lock.lock();
+    if (state != CONNECTING_WAIT_BANNER_AND_IDENTIFY) {
+      ldout(cct, 1) << __func__
                   << " state changed while learned_addr, mark_down or "
-                  << " replacing must be happened just now" << dendl;
-    return nullptr;
+                   << " replacing must be happened just now" << dendl;
+      return nullptr;
+    }
   }
 
   bufferlist myaddrbl;

diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc
index 5098288946547936dbbc59be0ef858eb205519ca..377df6e632d924a3b22c4022adeee2326d1e3cf2 100644 (file)
--- a/src/msg/async/ProtocolV2.cc
+++ b/src/msg/async/ProtocolV2.cc
@@ -918,6 +918,12 @@ CtPtr ProtocolV2::handle_hello(ceph::bufferlist &payload)
                 << " peer_type=" << (int)hello.entity_type()
                 << " peer_addr_for_me=" << hello.peer_addr() << dendl;
 
+  sockaddr_storage ss;
+  socklen_t len = sizeof(ss);
+  getsockname(connection->cs.fd(), (sockaddr *)&ss, &len);
+  ldout(cct, 5) << __func__ << " getsockname says I am " << (sockaddr *)&ss
+               << " when talking to " << connection->target_addr << dendl;
+
   if (connection->get_peer_type() == -1) {
     connection->set_peer_type(hello.entity_type());
 
@@ -942,6 +948,45 @@ CtPtr ProtocolV2::handle_hello(ceph::bufferlist &payload)
     }
   }
 
+  if (messenger->get_myaddrs().empty() ||
+      messenger->get_myaddrs().front().is_blank_ip()) {
+    entity_addr_t a;
+    if (cct->_conf->ms_learn_addr_from_peer) {
+      ldout(cct, 1) << __func__ << " peer " << connection->target_addr
+                   << " says I am " << hello.peer_addr() << " (socket says "
+                   << (sockaddr*)&ss << ")" << dendl;
+      a = hello.peer_addr();
+    } else {
+      ldout(cct, 1) << __func__ << " socket to  " << connection->target_addr
+                   << " says I am " << (sockaddr*)&ss
+                   << " (peer says " << hello.peer_addr() << ")" << dendl;
+      a.set_sockaddr((sockaddr *)&ss);
+    }
+    a.set_type(entity_addr_t::TYPE_MSGR2); // anything but NONE; learned_addr ignores this
+    a.set_port(0);
+    connection->lock.unlock();
+    messenger->learned_addr(a);
+    if (cct->_conf->ms_inject_internal_delays &&
+        cct->_conf->ms_inject_socket_failures) {
+      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
+        ldout(cct, 10) << __func__ << " sleep for "
+                       << cct->_conf->ms_inject_internal_delays << dendl;
+        utime_t t;
+        t.set_from_double(cct->_conf->ms_inject_internal_delays);
+        t.sleep();
+      }
+    }
+    connection->lock.lock();
+    if (state != HELLO_CONNECTING) {
+      ldout(cct, 1) << __func__
+                    << " state changed while learned_addr, mark_down or "
+                    << " replacing must be happened just now" << dendl;
+      return nullptr;
+    }
+  }
+
+
+
   CtPtr callback;
   callback = bannerExchangeCallback;
   bannerExchangeCallback = nullptr;
@@ -1815,39 +1860,6 @@ CtPtr ProtocolV2::send_client_ident() {
     flags |= CEPH_MSG_CONNECT_LOSSY;
   }
 
-  if (messenger->get_myaddrs().empty() ||
-      messenger->get_myaddrs().front().is_blank_ip()) {
-    sockaddr_storage ss;
-    socklen_t len = sizeof(ss);
-    int r = getsockname(connection->cs.socket_fd(), (sockaddr *)&ss, &len);
-    ceph_assert(r == 0);
-    ldout(cct, 1) << __func__ << " getsockname reveals I am " << (sockaddr *)&ss
-                  << " when talking to " << connection->target_addr << dendl;
-    entity_addr_t a;
-    a.set_type(entity_addr_t::TYPE_MSGR2); // anything but NONE; learned_addr ignores this
-    a.set_sockaddr((sockaddr *)&ss);
-    a.set_port(0);
-    connection->lock.unlock();
-    messenger->learned_addr(a);
-    if (cct->_conf->ms_inject_internal_delays &&
-        cct->_conf->ms_inject_socket_failures) {
-      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
-        ldout(cct, 10) << __func__ << " sleep for "
-                       << cct->_conf->ms_inject_internal_delays << dendl;
-        utime_t t;
-        t.set_from_double(cct->_conf->ms_inject_internal_delays);
-        t.sleep();
-      }
-    }
-    connection->lock.lock();
-    if (state != SESSION_CONNECTING) {
-      ldout(cct, 1) << __func__
-                    << " state changed while learned_addr, mark_down or "
-                    << " replacing must be happened just now" << dendl;
-      return nullptr;
-    }
-  }
-
   auto client_ident = ClientIdentFrame::Encode(
       messenger->get_myaddrs(),
       connection->target_addr,