common/pick_address: Allow binding on loopback iface

in 6147c0917157efd2d35610e759685656a4989abb, "lo" is also skipped when
daemon is trying to find an address to bind. but that change reverts the
fix of 201b59204374ebdab91bb554b986577a97b19c36, to address the problem.

an option named "ms_bind_exclude_lo_iface" is added, it defaults to
"true". but it can be changed to false to allow daemon to bind on "lo".

Fixes: https://tracker.ceph.com/issues/50012
Signed-off-by: Kefu Chai <kchai@redhat.com>
(cherry picked from commit 7f01d36a2ca0576f1ff103ae3fa7c3662e93b722)

diff --git a/src/common/options.cc b/src/common/options.cc
index 85f4203be3686e466ca32ad73db66417bf47eb1a..a5dbd0b65cb7faea50069599249878705a6c1104 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -997,6 +997,11 @@ std::vector<Option> get_global_options() {
     .set_default(100_M)
     .set_description("Limit messages that are read off the network but still being processed"),
 
+    Option("ms_bind_exclude_lo_iface", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_flag(Option::FLAG_STARTUP)
+    .set_description("Allow servers to bind loopback network interfaces (lo)"),
+
     Option("ms_bind_ipv4", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
     .set_default(true)
     .set_description("Bind servers to IPv4 address(es)")

diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index 9117ebe8ca160c510bf1eee2378d04861f4f9924..11a67e0d700b75f06ecf2a0f6707980346fce205 100644 (file)
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -42,9 +42,9 @@ bool matches_with_name(const ifaddrs& ifa, const std::string& if_name)
   return if_name.compare(ifa.ifa_name) == 0;
 }
 
-bool is_addr_allowed(const ifaddrs& ifa)
+bool is_addr_allowed(const ifaddrs& ifa, bool exclude_lo_iface)
 {
-  if (strcmp(ifa.ifa_name, "lo") == 0) {
+  if (exclude_lo_iface && strcmp(ifa.ifa_name, "lo") == 0) {
     return false;
   }
   if (boost::starts_with(ifa.ifa_name, "lo:")) {
@@ -110,6 +110,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node)
 {
   const auto ifs = get_str_list(interfaces);
@@ -120,7 +121,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   }
 
   for (const auto* addr = ifa; addr != nullptr; addr = addr->ifa_next) {
-    if (!is_addr_allowed(*addr)) {
+    if (!is_addr_allowed(*addr, exclude_lo_iface)) {
       continue;
     }
     if ((ifs.empty() ||
@@ -162,6 +163,7 @@ static void fill_in_one_address(CephContext *cct,
                                const struct ifaddrs *ifa,
                                const string &networks,
                                const string &interfaces,
+                               bool exclude_lo_iface,
                                const char *conf_var,
                                int numa_node = -1)
 {
@@ -171,6 +173,7 @@ static void fill_in_one_address(CephContext *cct,
     CEPH_PICK_ADDRESS_IPV4|CEPH_PICK_ADDRESS_IPV6,
     networks,
     interfaces,
+    exclude_lo_iface,
     numa_node);
   if (!found) {
     lderr(cct) << "unable to find any IP address in networks '" << networks
@@ -224,22 +227,27 @@ void pick_addresses(CephContext *cct, int needs)
   }
   auto free_ifa = make_scope_guard([ifa] { freeifaddrs(ifa); });
 
+  const bool exclude_lo_iface =
+    cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
   if ((needs & CEPH_PICK_ADDRESS_PUBLIC) &&
     public_addr.is_blank_ip() && !public_network.empty()) {
     fill_in_one_address(cct, ifa, public_network, public_network_interface,
-      "public_addr");
+                       exclude_lo_iface,
+                       "public_addr");
   }
 
   if ((needs & CEPH_PICK_ADDRESS_CLUSTER) && cluster_addr.is_blank_ip()) {
     if (!cluster_network.empty()) {
       fill_in_one_address(cct, ifa, cluster_network, cluster_network_interface,
-       "cluster_addr");
+                         exclude_lo_iface,
+                         "cluster_addr");
     } else {
       if (!public_network.empty()) {
         lderr(cct) << "Public network was set, but cluster network was not set " << dendl;
         lderr(cct) << "    Using public network also for cluster network" << dendl;
         fill_in_one_address(cct, ifa, public_network, public_network_interface,
-          "cluster_addr");
+                           exclude_lo_iface,
+                           "cluster_addr");
       }
     }
   }
@@ -252,11 +260,15 @@ static int fill_in_one_address(
   unsigned ipv,
   const string &networks,
   const string &interfaces,
+  bool exclude_lo_iface,
   entity_addrvec_t *addrs,
   int numa_node = -1)
 {
-  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, networks,
-                                                       interfaces, numa_node);
+  const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv,
+                                                       networks,
+                                                       interfaces,
+                                                       exclude_lo_iface,
+                                                       numa_node);
   if (!found) {
     std::string ip_type = "";
     if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) {
@@ -370,24 +382,32 @@ int pick_addresses(
       !networks.empty()) {
     int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1;
     int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1;
+    const bool exclude_lo_iface =
+      cct->_conf.get_val<bool>("ms_bind_exclude_lo_iface");
     // first try on preferred numa node (if >= 0), then anywhere.
     while (true) {
       // note: pass in ipv to filter the matching addresses
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
          (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
        ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+                                    exclude_lo_iface,
+                                    addrs,
                                      preferred_numa_node);
       }
       if (ipv & CEPH_PICK_ADDRESS_IPV6) {
        ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+                                    exclude_lo_iface,
+                                    addrs,
                                      preferred_numa_node);
       }
       if ((ipv & CEPH_PICK_ADDRESS_IPV4) &&
          !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) {
        ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4,
-                                     networks, interfaces, addrs,
+                                     networks, interfaces,
+                                    exclude_lo_iface,
+                                    addrs,
                                      preferred_numa_node);
       }
       if (ipv4_r >= 0 && ipv6_r >= 0) {

diff --git a/src/common/pick_address.h b/src/common/pick_address.h
index 7d80d42fec640b63e78e2bde361ed047fb3d379b..2a41f5a8d03c96acb952daf511d74922c3f6d5ac 100644 (file)
--- a/src/common/pick_address.h
+++ b/src/common/pick_address.h
@@ -80,6 +80,7 @@ bool have_local_addr(CephContext *cct, const std::list<entity_addr_t>& ls, entit
  *        are accepted if it is empty.
  * @param interfaces a comma separated list of interfaces for the allow list.
  *        all addresses are accepted if it is empty
+ * @param exclude_lo_iface filter out network interface named "lo"
  */
 const struct sockaddr *find_ip_in_subnet_list(
   CephContext *cct,
@@ -87,6 +88,7 @@ const struct sockaddr *find_ip_in_subnet_list(
   unsigned ipv,
   const std::string &networks,
   const std::string &interfaces,
+  bool exclude_lo_iface,
   int numa_node=-1);
 
 int get_iface_numa_node(

diff --git a/src/test/test_ipaddr.cc b/src/test/test_ipaddr.cc
index efd736193389f3187b6d26821ee84b3f29f60280..1ba220f8964d54743735a42f5745ded919bc96cd 100644 (file)
--- a/src/test/test_ipaddr.cc
+++ b/src/test/test_ipaddr.cc
@@ -200,11 +200,22 @@ TEST(CommonIPAddr, TestV4_SkipLoopback)
   ipv4(&a_two, "127.0.0.1");
   ipv4(&a_three, "10.1.2.3");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "10.0.0.0/8", "");
+                           "10.0.0.0/8", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "127.0.0.0/8", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, TestV6_Simple)
@@ -325,11 +336,22 @@ TEST(CommonIPAddr, TestV6_SkipLoopback)
   ipv6(&a_three, "2001:1234:5678:90ab::beef");
   ipv6(&net, "ff00::1");
 
-  const struct sockaddr *result =
+  const struct sockaddr *result = nullptr;
+  result =
     find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
                            CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
-                           "2001:1234:5678:90ab::0/64", "");
+                           "2001:1234:5678:90ab::0/64", "", true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", true);
+  ASSERT_EQ(nullptr, result);
+  result =
+    find_ip_in_subnet_list(nullptr, (struct ifaddrs*)&one,
+                           CEPH_PICK_ADDRESS_IPV4 | CEPH_PICK_ADDRESS_IPV6,
+                           "::1/128", "", false);
+  ASSERT_EQ((struct sockaddr*)&a_one, result);
 }
 
 TEST(CommonIPAddr, ParseNetwork_Empty)
@@ -684,7 +706,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.1.0.0/16",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -692,7 +715,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.2.0.0/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   // match by eth name
@@ -701,7 +725,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth0");
+    "eth0",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_one, result);
 
   result = find_ip_in_subnet_list(
@@ -709,7 +734,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV4,
     "10.0.0.0/8",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_two, result);
 
   result = find_ip_in_subnet_list(
@@ -717,7 +743,8 @@ TEST(pick_address, find_ip_in_subnet_list)
     &one,
     CEPH_PICK_ADDRESS_IPV6,
     "2001::/16",
-    "eth1");
+    "eth1",
+    true);
   ASSERT_EQ((struct sockaddr*)&a_three, result);
 }