mon: Verify version consistency across daemons

Add a function called check_daemon_versions()
Made minor changes based on pull request comments
Changed the way versions are checked along with some other minor modifications
Made minor modification to the error message
Modifications to include the hostname in the error message

Fixes: https://tracker.ceph.com/issues/46264
Signed-off-by: Tyler <tyler_sheehan@student.uml.edu>
Remove time since health errors aren't preserved if not set every time
Rename health error to DAEMON_OLD_VERSION
Reoganize code so that we can get versions from other daemon types
Change to map of version (key), a list of daemons (value)
Get actual daemon names from get_versions() requests
Fix message output based on how many daemons and how many versions
Add a config to be able to turn off this features
Give HEALTH_ERR if more than one older version exists
Fix minor indentation problem with health detail

Signed-off-by: David Zafman <dzafman@redhat.com>

diff --git a/src/common/options.cc b/src/common/options.cc
index 28ad9c79227898b9735f57aa15f3015852074827..8165692f95a8388ab0bccba59452c936daff61fc 100644 (file)
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2228,6 +2228,11 @@ std::vector<Option> get_global_options() {
                     "only be rejected later, when they are used.")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("mon_warn_on_older_version", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .add_service("mon")
+    .set_description("issue DAEMON_OLD_VERSION health warning if daemons are not all running the same version"),
+
     // PAXOS
 
     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)

diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 1a2da9de240c7a4b34f0127ff8cd3e52f200713a..582f99f9b5ffa0644110e33248126bb67e9f75b8 100644 (file)
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -16,6 +16,8 @@
 #include <limits.h>
 #include <sstream>
 #include <regex>
+#include <time.h>
+#include <iterator>
 
 #include "include/ceph_assert.h"
 #include "include/common_fwd.h"
@@ -682,6 +684,45 @@ bool HealthMonitor::check_leader_health()
 
   health_check_map_t next;
 
+ // DAEMON_OLD_VERSION
+  if (g_conf().get_val<bool>("mon_warn_on_older_version")) {
+  std::map<string, std::list<string> > all_versions;
+  mon->get_all_versions(all_versions);
+  if (all_versions.size() > 1) {
+    dout(20) << __func__ << " all_versions=" << all_versions << dendl;
+    // The last entry has the largest version
+    dout(20) << __func__ << " highest version daemon count "
+             << all_versions.rbegin()->second.size() << dendl;
+    // Erase last element (the highest version running)
+    all_versions.erase(all_versions.rbegin()->first);
+    ceph_assert(all_versions.size() > 0);
+    ostringstream ss;
+    unsigned daemon_count = 0;
+    for (auto& g:all_versions) {
+      daemon_count += g.second.size();
+    }
+    int ver_count = all_versions.size();
+    ceph_assert(!(daemon_count == 1 && ver_count != 1));
+    ss << "There " << (daemon_count == 1 ? "is a daemon" : "are daemons")
+       << " running " << (ver_count > 1 ? "multiple old versions" : "an older version")  << " of ceph";
+    health_status_t status;
+    if (ver_count > 1)
+      status = HEALTH_ERR;
+    else
+      status = HEALTH_WARN;
+    auto& d = next.add("DAEMON_OLD_VERSION", status, ss.str(), all_versions.size());
+    for (auto& g:all_versions) {
+      ostringstream ds;
+      for (auto& i : g.second) { // Daemon list
+        ds << i << " ";
+      }
+      ds << (g.second.size() == 1 ? "is" : "are")
+         << " running an older version of ceph: " << g.first;
+      d.detail.push_back(ds.str());
+    }
+  }
+  }
+
   // MON_DOWN
   {
     int max = mon->monmap->size();

diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 6ba0cbf0545fb4189d5f0a0a7e81e3c25018b3c0..7e1d30628d3f89c139c2804290d99737a3c1ea9d 100644 (file)
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -5359,6 +5359,23 @@ void Monitor::count_metadata(const string& field, Formatter *f)
   f->close_section();
 }
 
+void Monitor::get_all_versions(std::map<string, list<string> > &versions)
+{
+  // mon
+  get_versions(versions);
+  dout(20) << __func__ << " all versions=" << versions << dendl;
+}
+
+void Monitor::get_versions(std::map<string, list<string> > &versions)
+{
+  int i = 0;
+  for (auto& [rank, metadata] : mon_metadata) {
+    auto q = metadata.find("ceph_version_short");
+    versions[q->second].push_back(string("mon.") + monmap->get_name(rank));
+    i = i + 1;
+  }
+}
+
 int Monitor::print_nodes(Formatter *f, ostream& err)
 {
   map<string, list<string> > mons;     // hostname => mon

diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index c3a15a16abfff7590e136c1a43a2b1363315a12d..936af1ee91d4504d888ed71f34453b9b4bcfdec4 100644 (file)
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -965,6 +965,9 @@ private:
   int load_metadata();
   void count_metadata(const std::string& field, ceph::Formatter *f);
   void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  // get_all_versions() gathers version information from daemons for health check
+  void get_all_versions(std::map<string, list<string> > &versions);
+  void get_versions(std::map<string, list<string> > &versions);
 
   // features
   static CompatSet get_initial_supported_features();