From 7f01d36a2ca0576f1ff103ae3fa7c3662e93b722 Mon Sep 17 00:00:00 2001
From: Kefu Chai <kchai@redhat.com>
Date: Fri, 26 Mar 2021 20:03:09 +0800
Subject: [PATCH] common/pick_address: Allow binding on loopback iface

in 6147c0917157efd2d35610e759685656a4989abb, "lo" is also skipped when
daemon is trying to find an address to bind. but that change reverts the
fix of 201b59204374ebdab91bb554b986577a97b19c36, to address the problem.

an option named "ms_bind_exclude_lo_iface" is added, it defaults to
"true". but it can be changed to false to allow daemon to bind on "lo".

Fixes: https://tracker.ceph.com/issues/50012
Signed-off-by: Kefu Chai <kchai@redhat.com>
---
 src/common/options.cc      |  5 +++++
 src/common/pick_address.cc | 42 +++++++++++++++++++++++++----------
 src/common/pick_address.h  |  2 ++
 src/test/test_ipaddr.cc    | 45 ++++++++++++++++++++++++++++++--------
 4 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index 13d70e37db7f..3a9bf98c19cd 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -1020,6 +1020,11 @@ std::vector<Option> get_global_options() {
     .set_default(100_M)
     .set_description("Limit messages that are read off the network but still being processed"),
 
+    Option("ms_bind_exclude_lo_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("Allow servers to bind loopback network interfaces (lo)"),
+
     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Bind servers to IPv4 address(es)")
diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index a4bdd68ae520..ff7c32c4b7a4 100644
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -45,9 +45,9 @@ bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
   return if_name.compare(ifa.ifa_name) == 0;
 }
 
-bool is_addr_allowed(const ifaddrs& ifa)
+bool is_addr_allowed(const ifaddrs& ifa, bool exclude_lo_iface)
 {
-  if (strcmp(ifa.ifa_name, "lo") == 0) {
+  if (exclude_lo_iface && strcmp(ifa.ifa_name, "lo") == 0) {
     return false;
   }
   if (boost::starts_with(ifa.ifa_name, "lo:")) {
@@ -113,6 +113,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node)
 {
   const auto ifs = get_str_list(interfaces);
@@ -123,7 +124,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   }
 
   for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
-    if (!is_addr_allowed(*addr)) {
+    if (!is_addr_allowed(*addr, exclude_lo_iface)) {
       continue;
     }
     if ((ifs.empty() ||
@@ -165,6 +166,7 @@ static void fill_in_one_address(CephContext *cct,
 				const struct ifaddrs *ifa,
 				const string &networks,
 				const string &interfaces,
+				bool exclude_lo_iface,
 				const char *conf_var,
 				int numa_node = -1)
 {
@@ -174,6 +176,7 @@ static void fill_in_one_address(CephContext *cct,
     CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6,
     networks,
     interfaces,
+    exclude_lo_iface,
     numa_node);
   if (!found) {
     lderr(cct) << "unable to find any IP address in networks '" << networks
@@ -227,22 +230,27 @@ void pick_addresses(CephContext *cct, int needs)
   }
   auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
 
+  const bool exclude_lo_iface =
+    cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
   if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
     public_addr.is_blank_ip() && !public_network.empty()) {
     fill_in_one_address(cct, ifa, public_network, public_network_interface,
-      "public_addr");
+			exclude_lo_iface,
+			"public_addr");
   }
 
   if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
     if (!cluster_network.empty()) {
       fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
-	"cluster_addr");
+			  exclude_lo_iface,
+			  "cluster_addr");
     } else {
       if (!public_network.empty()) {
         lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
         lderr(cct) << "    Using public network also for cluster network" << dendl;
         fill_in_one_address(cct, ifa, public_network, public_network_interface,
-          "cluster_addr");
+			    exclude_lo_iface,
+			    "cluster_addr");
       }
     }
   }
@@ -255,11 +263,15 @@ static int fill_in_one_address(
   unsigned ipv,
   const string &networks,
   const string &interfaces,
+  bool exclude_lo_iface,
   entity_addrvec_t *addrs,
   int numa_node = -1)
 {
-  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks,
-							interfaces, numa_node);
+  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
+							networks,
+							interfaces,
+							exclude_lo_iface,
+							numa_node);
   if (!found) {
     std::string ip_type = "";
     if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
@@ -373,24 +385,32 @@ int pick_addresses(
       !networks.empty()) {
     int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1;
     int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1;
+    const bool exclude_lo_iface =
+      cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
     // first try on preferred numa node (if >= 0), then anywhere.
     while (true) {
       // note: pass in ipv to filter the matching addresses
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
 	  (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
 	ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if (ipv & CEPH_PICK_ADDRESS_IPV6) {
 	ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
 	  !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
 	ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+				     exclude_lo_iface,
+				     addrs,
                                      preferred_numa_node);
       }
       if (ipv4_r >= 0 && ipv6_r >= 0) {
diff --git a/src/common/pick_address.h b/src/common/pick_address.h
index 7d80d42fec64..2a41f5a8d03c 100644
--- a/src/common/pick_address.h
+++ b/src/common/pick_address.h
@@ -80,6 +80,7 @@ bool have_local_addr(CephContext *cct, const std::list<entity_addr_t>& ls, entit
  *        are accepted if it is empty.
  * @param interfaces a comma separated list of interfaces for the allow list.
  *        all addresses are accepted if it is empty
+ * @param exclude_lo_iface filter out network interface named "lo"
  */
 const struct sockaddr *find_ip_in_subnet_list(
   CephContext *cct,
@@ -87,6 +88,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node=-1);
 
 int get_iface_numa_node(
diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc
index 3eccd5be42e6..d2ab8c50eeed 100644
--- a/src/test/test_ipaddr.cc
+++ b/src/test/test_ipaddr.cc
@@ -201,11 +201,22 @@ TEST(CommonIPAddr, TestV4_SkipLoopback)
   ipv4(&a_two, "127.0.0.1");
   ipv4(&a_three, "10.1.2.3");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "10.0.0.0/8", "");
+                           "10.0.0.0/8", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, TestV6_Simple)
@@ -326,11 +337,22 @@ TEST(CommonIPAddr, TestV6_SkipLoopback)
   ipv6(&a_three, "2001:1234:5678:90ab::beef");
   ipv6(&net, "ff00::1");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "2001:1234:5678:90ab::0/64", "");
+                           "2001:1234:5678:90ab::0/64", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, ParseNetwork_Empty)
@@ -687,7 +709,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.1.0.0/16",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -695,7 +718,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.2.0.0/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   // match by eth name
@@ -704,7 +728,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -712,7 +737,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   result = find_ip_in_subnet_list(
@@ -720,7 +746,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV6,
     "2001::/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
 }
 
-- 
2.47.3