From: Sage Weil <sage@redhat.com>
Date: Thu, 21 Mar 2019 03:42:31 +0000 (-0500)
Subject: msg/async/ProtocolV[12]: add ms_learn_addr_from_peer
X-Git-Tag: v15.1.0~3007^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=05f56ab67fdd67a0fa4742a9578bacb2948bb8e9;p=ceph.git

msg/async/ProtocolV[12]: add ms_learn_addr_from_peer

Optionally learn our address based on what our peer (usually the mon) thinks we are
connecting from, as opposed to what our local socket used for the outbound connection.
This enables us to detect what our NATed address is (as a client) in those environments.

Default to true to match the v1 behavior.  This is a net change in
behavior for v2.

Signed-off-by: Sage Weil <sage@redhat.com>
---

diff --git a/src/common/legacy_config_opts.h b/src/common/legacy_config_opts.h
index f17b2f412c69..c36bd6144702 100644
--- a/src/common/legacy_config_opts.h
+++ b/src/common/legacy_config_opts.h
@@ -101,6 +101,7 @@ OPTION(perf, OPT_BOOL)       // enable internal perf counters
 SAFE_OPTION(ms_type, OPT_STR)   // messenger backend. It will be modified in runtime, so use SAFE_OPTION
 OPTION(ms_public_type, OPT_STR)   // messenger backend
 OPTION(ms_cluster_type, OPT_STR)   // messenger backend
+OPTION(ms_learn_addr_from_peer, OPT_BOOL)
 OPTION(ms_tcp_nodelay, OPT_BOOL)
 OPTION(ms_tcp_rcvbuf, OPT_INT)
 OPTION(ms_tcp_prefetch_max_size, OPT_U32) // max prefetch size, we limit this to avoid extra memcpy
diff --git a/src/common/options.cc b/src/common/options.cc
index 5181cf6c48d5..cfc6a3c7530c 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -841,6 +841,11 @@ std::vector<Option> get_global_options() {
     .add_see_also("ms_cluster_mode")
     .add_see_also("ms_service_mode"),
 
+    Option("ms_learn_addr_from_peer", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Learn address from what IP our first peer thinks we connect from")
+    .set_long_description("Use the IP address our first peer (usually a monitor) sees that we are connecting from.  This is useful if a client is behind some sort of NAT and we want to see it identified by its local (not NATed) address."),
+
     Option("ms_tcp_nodelay", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Disable Nagle's algorithm and send queued network traffic immediately"),
diff --git a/src/msg/async/ProtocolV1.cc b/src/msg/async/ProtocolV1.cc
index 1f56332f852d..558e906c253d 100644
--- a/src/msg/async/ProtocolV1.cc
+++ b/src/msg/async/ProtocolV1.cc
@@ -1375,25 +1375,44 @@ CtPtr ProtocolV1::handle_server_banner_and_identify(char *buffer, int r) {
 
   ldout(cct, 20) << __func__ << " connect peer addr for me is "
                  << peer_addr_for_me << dendl;
-  connection->lock.unlock();
-  messenger->learned_addr(peer_addr_for_me);
-  if (cct->_conf->ms_inject_internal_delays &&
-      cct->_conf->ms_inject_socket_failures) {
-    if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
-      ldout(cct, 10) << __func__ << " sleep for "
-                     << cct->_conf->ms_inject_internal_delays << dendl;
-      utime_t t;
-      t.set_from_double(cct->_conf->ms_inject_internal_delays);
-      t.sleep();
+  if (messenger->get_myaddrs().empty() ||
+      messenger->get_myaddrs().front().is_blank_ip()) {
+    sockaddr_storage ss;
+    socklen_t len = sizeof(ss);
+    getsockname(connection->cs.fd(), (sockaddr *)&ss, &len);
+    entity_addr_t a;
+    if (cct->_conf->ms_learn_addr_from_peer) {
+      ldout(cct, 1) << __func__ << " peer " << connection->target_addr
+		    << " says I am " << peer_addr_for_me << " (socket says "
+		    << (sockaddr*)&ss << ")" << dendl;
+      a = peer_addr_for_me;
+    } else {
+      ldout(cct, 1) << __func__ << " socket to  " << connection->target_addr
+		    << " says I am " << (sockaddr*)&ss
+		    << " (peer says " << peer_addr_for_me << ")" << dendl;
+      a.set_sockaddr((sockaddr *)&ss);
     }
-  }
-
-  connection->lock.lock();
-  if (state != CONNECTING_WAIT_BANNER_AND_IDENTIFY) {
-    ldout(cct, 1) << __func__
+    a.set_type(entity_addr_t::TYPE_LEGACY); // anything but NONE; learned_addr ignores this
+    a.set_port(0);
+    connection->lock.unlock();
+    messenger->learned_addr(a);
+    if (cct->_conf->ms_inject_internal_delays &&
+	cct->_conf->ms_inject_socket_failures) {
+      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
+	ldout(cct, 10) << __func__ << " sleep for "
+		       << cct->_conf->ms_inject_internal_delays << dendl;
+	utime_t t;
+	t.set_from_double(cct->_conf->ms_inject_internal_delays);
+	t.sleep();
+      }
+    }
+    connection->lock.lock();
+    if (state != CONNECTING_WAIT_BANNER_AND_IDENTIFY) {
+      ldout(cct, 1) << __func__
                   << " state changed while learned_addr, mark_down or "
-                  << " replacing must be happened just now" << dendl;
-    return nullptr;
+		    << " replacing must be happened just now" << dendl;
+      return nullptr;
+    }
   }
 
   bufferlist myaddrbl;
diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc
index 509828894654..377df6e632d9 100644
--- a/src/msg/async/ProtocolV2.cc
+++ b/src/msg/async/ProtocolV2.cc
@@ -918,6 +918,12 @@ CtPtr ProtocolV2::handle_hello(ceph::bufferlist &payload)
                 << " peer_type=" << (int)hello.entity_type()
                 << " peer_addr_for_me=" << hello.peer_addr() << dendl;
 
+  sockaddr_storage ss;
+  socklen_t len = sizeof(ss);
+  getsockname(connection->cs.fd(), (sockaddr *)&ss, &len);
+  ldout(cct, 5) << __func__ << " getsockname says I am " << (sockaddr *)&ss
+		<< " when talking to " << connection->target_addr << dendl;
+
   if (connection->get_peer_type() == -1) {
     connection->set_peer_type(hello.entity_type());
 
@@ -942,6 +948,45 @@ CtPtr ProtocolV2::handle_hello(ceph::bufferlist &payload)
     }
   }
 
+  if (messenger->get_myaddrs().empty() ||
+      messenger->get_myaddrs().front().is_blank_ip()) {
+    entity_addr_t a;
+    if (cct->_conf->ms_learn_addr_from_peer) {
+      ldout(cct, 1) << __func__ << " peer " << connection->target_addr
+		    << " says I am " << hello.peer_addr() << " (socket says "
+		    << (sockaddr*)&ss << ")" << dendl;
+      a = hello.peer_addr();
+    } else {
+      ldout(cct, 1) << __func__ << " socket to  " << connection->target_addr
+		    << " says I am " << (sockaddr*)&ss
+		    << " (peer says " << hello.peer_addr() << ")" << dendl;
+      a.set_sockaddr((sockaddr *)&ss);
+    }
+    a.set_type(entity_addr_t::TYPE_MSGR2); // anything but NONE; learned_addr ignores this
+    a.set_port(0);
+    connection->lock.unlock();
+    messenger->learned_addr(a);
+    if (cct->_conf->ms_inject_internal_delays &&
+        cct->_conf->ms_inject_socket_failures) {
+      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
+        ldout(cct, 10) << __func__ << " sleep for "
+                       << cct->_conf->ms_inject_internal_delays << dendl;
+        utime_t t;
+        t.set_from_double(cct->_conf->ms_inject_internal_delays);
+        t.sleep();
+      }
+    }
+    connection->lock.lock();
+    if (state != HELLO_CONNECTING) {
+      ldout(cct, 1) << __func__
+                    << " state changed while learned_addr, mark_down or "
+                    << " replacing must be happened just now" << dendl;
+      return nullptr;
+    }
+  }
+
+
+
   CtPtr callback;
   callback = bannerExchangeCallback;
   bannerExchangeCallback = nullptr;
@@ -1815,39 +1860,6 @@ CtPtr ProtocolV2::send_client_ident() {
     flags |= CEPH_MSG_CONNECT_LOSSY;
   }
 
-  if (messenger->get_myaddrs().empty() ||
-      messenger->get_myaddrs().front().is_blank_ip()) {
-    sockaddr_storage ss;
-    socklen_t len = sizeof(ss);
-    int r = getsockname(connection->cs.socket_fd(), (sockaddr *)&ss, &len);
-    ceph_assert(r == 0);
-    ldout(cct, 1) << __func__ << " getsockname reveals I am " << (sockaddr *)&ss
-                  << " when talking to " << connection->target_addr << dendl;
-    entity_addr_t a;
-    a.set_type(entity_addr_t::TYPE_MSGR2); // anything but NONE; learned_addr ignores this
-    a.set_sockaddr((sockaddr *)&ss);
-    a.set_port(0);
-    connection->lock.unlock();
-    messenger->learned_addr(a);
-    if (cct->_conf->ms_inject_internal_delays &&
-        cct->_conf->ms_inject_socket_failures) {
-      if (rand() % cct->_conf->ms_inject_socket_failures == 0) {
-        ldout(cct, 10) << __func__ << " sleep for "
-                       << cct->_conf->ms_inject_internal_delays << dendl;
-        utime_t t;
-        t.set_from_double(cct->_conf->ms_inject_internal_delays);
-        t.sleep();
-      }
-    }
-    connection->lock.lock();
-    if (state != SESSION_CONNECTING) {
-      ldout(cct, 1) << __func__
-                    << " state changed while learned_addr, mark_down or "
-                    << " replacing must be happened just now" << dendl;
-      return nullptr;
-    }
-  }
-
   auto client_ident = ClientIdentFrame::Encode(
       messenger->get_myaddrs(),
       connection->target_addr,