From 96c2879880889126b957b752a1c45da896cb1ae9 Mon Sep 17 00:00:00 2001
From: Kefu Chai <kchai@redhat.com>
Date: Fri, 26 Mar 2021 20:03:09 +0800
Subject: [PATCH] common/pick_address: Allow binding on loopback iface

in 6147c0917157efd2d35610e759685656a4989abb, "lo" is also skipped when
daemon is trying to find an address to bind. but that change reverts the
fix of 201b59204374ebdab91bb554b986577a97b19c36, to address the problem.

an option named "ms_bind_exclude_lo_iface" is added, it defaults to
"true". but it can be changed to false to allow daemon to bind on "lo".

Fixes: https://tracker.ceph.com/issues/50012
Signed-off-by: Kefu Chai <kchai@redhat.com>
(cherry picked from commit 7f01d36a2ca0576f1ff103ae3fa7c3662e93b722)
---
 src/common/options.cc      |  5 +++++
 src/common/pick_address.cc | 42 +++++++++++++++++++++++++----------
 src/common/pick_address.h  |  2 ++
 src/test/test_ipaddr.cc    | 45 ++++++++++++++++++++++++++++++--------
 4 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 96bbf56c1a023..0e76f628ed8b5 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1019,6 +1019,11 @@ std::vector<Option> get_global_options() {
     .set_default(100_M)
     .set_description("Limit messages that are read off the network but still being processed"),
 
+    Option("ms_bind_exclude_lo_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("Allow servers to bind loopback network interfaces (lo)"),
+
     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Bind servers to IPv4 address(es)")
diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index a4bdd68ae5203..ff7c32c4b7a4d 100644
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -45,9 +45,9 @@ bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
   return if_name.compare(ifa.ifa_name) == 0;
 }
 
-bool is_addr_allowed(const ifaddrs& ifa)
+bool is_addr_allowed(const ifaddrs& ifa, bool exclude_lo_iface)
 {
-  if (strcmp(ifa.ifa_name, "lo") == 0) {
+  if (exclude_lo_iface && strcmp(ifa.ifa_name, "lo") == 0) {
     return false;
   }
   if (boost::starts_with(ifa.ifa_name, "lo:")) {
@@ -113,6 +113,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node)
 {
   const auto ifs = get_str_list(interfaces);
@@ -123,7 +124,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   }
 
   for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
-    if (!is_addr_allowed(*addr)) {
+    if (!is_addr_allowed(*addr, exclude_lo_iface)) {
       continue;
     }
     if ((ifs.empty() ||
@@ -165,6 +166,7 @@ static void fill_in_one_address(CephContext *cct,
 				const struct ifaddrs *ifa,
 				const string &networks,
 				const string &interfaces,
+				bool exclude_lo_iface,
 				const char *conf_var,
 				int numa_node = -1)
 {
@@ -174,6 +176,7 @@ static void fill_in_one_address(CephContext *cct,
     CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6,
     networks,
     interfaces,
+    exclude_lo_iface,
     numa_node);
   if (!found) {
     lderr(cct) << "unable to find any IP address in networks '" << networks
@@ -227,22 +230,27 @@ void pick_addresses(CephContext *cct, int needs)
   }
   auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
 
+  const bool exclude_lo_iface =
+    cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
   if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
     public_addr.is_blank_ip() && !public_network.empty()) {
     fill_in_one_address(cct, ifa, public_network, public_network_interface,
-      "public_addr");
+			exclude_lo_iface,
+			"public_addr");
   }
 
   if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
     if (!cluster_network.empty()) {
       fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
-	"cluster_addr");
+			  exclude_lo_iface,
+			  "cluster_addr");
     } else {
       if (!public_network.empty()) {
         lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
         lderr(cct) << "    Using public network also for cluster network" << dendl;
         fill_in_one_address(cct, ifa, public_network, public_network_interface,
-          "cluster_addr");
+			    exclude_lo_iface,
+			    "cluster_addr");
       }
     }
   }
@@ -255,11 +263,15 @@ static int fill_in_one_address(
   unsigned ipv,
   const string &networks,
   const string &interfaces,
+  bool exclude_lo_iface,
   entity_addrvec_t *addrs,
   int numa_node = -1)
 {
-  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks,
-							interfaces, numa_node);
+  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
+							networks,
+							interfaces,
+							exclude_lo_iface,
+							numa_node);
   if (!found) {
     std::string ip_type = "";
     if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
@@ -373,24 +385,32 @@ int pick_addresses(
       !networks.empty()) {
     int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1;
     int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1;
+    const bool exclude_lo_iface =
+      cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
     // first try on preferred numa node (if >= 0), then anywhere.
     while (true) {
       // note: pass in ipv to filter the matching addresses
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
 	  (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
 	ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if (ipv & CEPH_PICK_ADDRESS_IPV6) {
 	ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
 	  !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
 	ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if (ipv4_r >= 0 && ipv6_r >= 0) {
diff --git a/src/common/pick_address.h b/src/common/pick_address.h
index 7d80d42fec640..2a41f5a8d03c9 100644
--- a/src/common/pick_address.h
+++ b/src/common/pick_address.h
@@ -80,6 +80,7 @@ bool have_local_addr(CephContext *cct, const std::list<entity_addr_t>& ls, entit
  *        are accepted if it is empty.
  * @param interfaces a comma separated list of interfaces for the allow list.
  *        all addresses are accepted if it is empty
+ * @param exclude_lo_iface filter out network interface named "lo"
  */
 const struct sockaddr *find_ip_in_subnet_list(
   CephContext *cct,
@@ -87,6 +88,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node=-1);
 
 int get_iface_numa_node(
diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc
index 3eccd5be42e6b..d2ab8c50eeed3 100644
--- a/src/test/test_ipaddr.cc
+++ b/src/test/test_ipaddr.cc
@@ -201,11 +201,22 @@ TEST(CommonIPAddr, TestV4_SkipLoopback)
   ipv4(&a_two, "127.0.0.1");
   ipv4(&a_three, "10.1.2.3");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "10.0.0.0/8", "");
+                           "10.0.0.0/8", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, TestV6_Simple)
@@ -326,11 +337,22 @@ TEST(CommonIPAddr, TestV6_SkipLoopback)
   ipv6(&a_three, "2001:1234:5678:90ab::beef");
   ipv6(&net, "ff00::1");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "2001:1234:5678:90ab::0/64", "");
+                           "2001:1234:5678:90ab::0/64", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, ParseNetwork_Empty)
@@ -687,7 +709,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.1.0.0/16",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -695,7 +718,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.2.0.0/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   // match by eth name
@@ -704,7 +728,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -712,7 +737,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   result = find_ip_in_subnet_list(
@@ -720,7 +746,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV6,
     "2001::/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
 }
 
-- 
2.39.5