osd: Report health error if OSD public address is not within subnet

In a containerized environment after a OSD node reboot, due to
some race condition in systemd some OSDs registered their
v1/v2 public addresses on cluster network instead on
defined public_network. Report this inconsistency as a health
error as RADOS clients fail to connect to the cluster.

Fixes: https://tracker.ceph.com/issues/56057
Signed-off-by: Prashant D <pdhange@redhat.com>
(cherry picked from commit 1332cc428b3f73a0f7eb8ca715aa4bb5d34bbeb5)

diff --git a/doc/rados/operations/health-checks.rst b/doc/rados/operations/health-checks.rst
index d5246560212e1621adb9bd60f7f016937048f160..f667a3df5fad9e8b88cd64fbca584fdf8149d274 100644 (file)
--- a/doc/rados/operations/health-checks.rst
+++ b/doc/rados/operations/health-checks.rst
@@ -522,6 +522,16 @@ Since this migration can take a considerable amount of time to complete, we
 recommend that you begin the process well in advance of any update to Reef or
 to later releases.
 
+OSD_UNREACHABLE
+_______________
+
+Registered v1/v2 public address of one or more OSD(s) is/are out of the
+defined `public_network` subnet, which will prevent these unreachable OSDs
+from communicating with ceph clients properly.
+
+Even though these unreachable OSDs are in up state, rados clients
+will hang till TCP timeout before erroring out due to this inconsistency.
+
 POOL_FULL
 _________
 

diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc
index b1a36e0729128024c13b83adc2b4fe862c168c9e..2fd076808ac74b2b8737260c4b5c37cd3fa4bf7d 100644 (file)
--- a/src/common/pick_address.cc
+++ b/src/common/pick_address.cc
@@ -631,3 +631,22 @@ int get_iface_numa_node(
   return r;
 }
 
+bool is_addr_in_subnet(
+  CephContext *cct,
+  const std::string &networks,
+  const std::string &addr)
+{
+  const auto nets = get_str_list(networks);
+  ceph_assert(!nets.empty());
+  const auto &net = nets.front();
+  struct ifaddrs ifa;
+  unsigned ipv = CEPH_PICK_ADDRESS_IPV4;
+  struct sockaddr_in public_addr;
+
+  ifa.ifa_next = nullptr;
+  ifa.ifa_addr = (struct sockaddr*)&public_addr;
+  public_addr.sin_family = AF_INET;
+  inet_pton(AF_INET, addr.c_str(), &public_addr.sin_addr);
+
+  return matches_with_net(cct, ifa, net, ipv);
+}

diff --git a/src/common/pick_address.h b/src/common/pick_address.h
index 4fd77e546f1081051e808dcf80adf3dedced9cc7..40575d7d1551185389f1197deb7dd90b89525909 100644 (file)
--- a/src/common/pick_address.h
+++ b/src/common/pick_address.h
@@ -95,4 +95,9 @@ int get_iface_numa_node(
   const std::string& iface,
   int *node);
 
+bool is_addr_in_subnet(
+  CephContext *cct,
+  const std::string &networks,
+  const std::string &addr);
+
 #endif

diff --git a/src/osd/OSDMap.cc b/src/osd/OSDMap.cc
index 4d3160cdd56a3f468143e8d930c01db0b1c0e8bd..434146d9647edf942b43989b16ae11806207a092 100644 (file)
--- a/src/osd/OSDMap.cc
+++ b/src/osd/OSDMap.cc
@@ -38,6 +38,7 @@
 #include "crush/CrushTreeDumper.h"
 #include "common/Clock.h"
 #include "mon/PGMap.h"
+#include "common/pick_address.h"
 
 using std::list;
 using std::make_pair;
@@ -1633,6 +1634,27 @@ void OSDMap::get_full_osd_counts(set<int> *full, set<int> *backfill,
   }
 }
 
+void OSDMap::get_out_of_subnet_osd_counts(CephContext *cct,
+                                          std::string const &public_network,
+                                          set<int> *unreachable) const
+{
+  unreachable->clear();
+  for (int i = 0; i < max_osd; i++) {
+    if (exists(i) && is_up(i)) {
+      if (const auto& addrs = get_addrs(i).v; addrs.size() >= 2) {
+        auto v1_addr = addrs[0].ip_only_to_str();
+        if (!is_addr_in_subnet(cct, public_network, v1_addr)) {
+          unreachable->emplace(i);
+        }
+        auto v2_addr = addrs[1].ip_only_to_str();
+        if (!is_addr_in_subnet(cct, public_network, v2_addr)) {
+          unreachable->emplace(i);
+        }
+      }
+    }
+  }
+}
+
 void OSDMap::get_all_osds(set<int32_t>& ls) const
 {
   for (int i=0; i<max_osd; i++)
@@ -7241,6 +7263,29 @@ void OSDMap::check_health(CephContext *cct,
       checks->add("UNEVEN_WEIGHTS_STRETCH_MODE", HEALTH_WARN, ss.str(), 0);
     }
   }
+
+  // PUBLIC ADDRESS IS IN SUBNET MASK
+  {
+    auto public_network = cct->_conf.get_val<std::string>("public_network");
+
+    if (!public_network.empty()) {
+      set<int> unreachable;
+      get_out_of_subnet_osd_counts(cct, public_network, &unreachable);
+      if (unreachable.size()) {
+        ostringstream ss;
+        ss << unreachable.size()
+           << " osds(s) "
+           << (unreachable.size() == 1 ? "is" : "are")
+           << " not reachable";
+        auto& d = checks->add("OSD_UNREACHABLE", HEALTH_ERR, ss.str(), unreachable.size());
+        for (auto& i: unreachable) {
+          ostringstream ss;
+          ss << "osd." << i << "'s public address is not in '" << public_network << "' subnet";
+          d.detail.push_back(ss.str());
+        }
+      }
+    }
+  }
 }
 
 int OSDMap::parse_osd_id_list(const vector<string>& ls, set<int> *out,

diff --git a/src/osd/OSDMap.h b/src/osd/OSDMap.h
index 920bb8fdf50f2c40e5e054a9dc68df983c554ef9..963039d0213f330e7844309eced9ea71c7ffe98c 100644 (file)
--- a/src/osd/OSDMap.h
+++ b/src/osd/OSDMap.h
@@ -761,6 +761,9 @@ public:
   void get_full_osd_counts(std::set<int> *full, std::set<int> *backfill,
                           std::set<int> *nearfull) const;
 
+  void get_out_of_subnet_osd_counts(CephContext *cct,
+                                    std::string const &public_network,
+                                    std::set<int> *unreachable) const;
 
   /***** cluster state *****/
   /* osds */