From: Tyler <tyler_sheehan@student.uml.edu>
Date: Mon, 20 Jul 2020 17:18:29 +0000 (-0400)
Subject: mon: Verify version consistency across daemons
X-Git-Tag: v16.1.0~634^2~11
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5e463b84dcb34707ae1603ef4cf1d189062bf855;p=ceph.git

mon: Verify version consistency across daemons

Add a function called check_daemon_versions()
Made minor changes based on pull request comments
Changed the way versions are checked along with some other minor modifications
Made minor modification to the error message
Modifications to include the hostname in the error message

Fixes: https://tracker.ceph.com/issues/46264

Signed-off-by: Tyler <tyler_sheehan@student.uml.edu>

Remove time since health errors aren't preserved if not set every time
Rename health error to DAEMON_OLD_VERSION
Reoganize code so that we can get versions from other daemon types
Change to map of version (key), a list of daemons (value)
Get actual daemon names from get_versions() requests
Fix message output based on how many daemons and how many versions
Add a config to be able to turn off this features
Give HEALTH_ERR if more than one older version exists
Fix minor indentation problem with health detail

Signed-off-by: David Zafman <dzafman@redhat.com>
---

diff --git a/src/common/options.cc b/src/common/options.cc
index 28ad9c792278..8165692f95a8 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2228,6 +2228,11 @@ std::vector<Option> get_global_options() {
 		     "only be rejected later, when they are used.")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("mon_warn_on_older_version", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .add_service("mon")
+    .set_description("issue DAEMON_OLD_VERSION health warning if daemons are not all running the same version"),
+
     // PAXOS
 
     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 1a2da9de240c..582f99f9b5ff 100644
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -16,6 +16,8 @@
 #include <limits.h>
 #include <sstream>
 #include <regex>
+#include <time.h>
+#include <iterator>
 
 #include "include/ceph_assert.h"
 #include "include/common_fwd.h"
@@ -682,6 +684,45 @@ bool HealthMonitor::check_leader_health()
 
   health_check_map_t next;
 
+ // DAEMON_OLD_VERSION
+  if (g_conf().get_val<bool>("mon_warn_on_older_version")) {
+  std::map<string, std::list<string> > all_versions;
+  mon->get_all_versions(all_versions);
+  if (all_versions.size() > 1) {
+    dout(20) << __func__ << " all_versions=" << all_versions << dendl;
+    // The last entry has the largest version
+    dout(20) << __func__ << " highest version daemon count "
+             << all_versions.rbegin()->second.size() << dendl;
+    // Erase last element (the highest version running)
+    all_versions.erase(all_versions.rbegin()->first);
+    ceph_assert(all_versions.size() > 0);
+    ostringstream ss;
+    unsigned daemon_count = 0;
+    for (auto& g:all_versions) {
+      daemon_count += g.second.size();
+    }
+    int ver_count = all_versions.size();
+    ceph_assert(!(daemon_count == 1 && ver_count != 1));
+    ss << "There " << (daemon_count == 1 ? "is a daemon" : "are daemons")
+       << " running " << (ver_count > 1 ? "multiple old versions" : "an older version")  << " of ceph";
+    health_status_t status;
+    if (ver_count > 1)
+      status = HEALTH_ERR;
+    else
+      status = HEALTH_WARN;
+    auto& d = next.add("DAEMON_OLD_VERSION", status, ss.str(), all_versions.size());
+    for (auto& g:all_versions) {
+      ostringstream ds;
+      for (auto& i : g.second) { // Daemon list
+        ds << i << " ";
+      }
+      ds << (g.second.size() == 1 ? "is" : "are")
+         << " running an older version of ceph: " << g.first;
+      d.detail.push_back(ds.str());
+    }
+  }
+  }
+
   // MON_DOWN
   {
     int max = mon->monmap->size();
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 6ba0cbf0545f..7e1d30628d3f 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -5359,6 +5359,23 @@ void Monitor::count_metadata(const string& field, Formatter *f)
   f->close_section();
 }
 
+void Monitor::get_all_versions(std::map<string, list<string> > &versions)
+{
+  // mon
+  get_versions(versions);
+  dout(20) << __func__ << " all versions=" << versions << dendl;
+}
+
+void Monitor::get_versions(std::map<string, list<string> > &versions)
+{
+  int i = 0;
+  for (auto& [rank, metadata] : mon_metadata) {
+    auto q = metadata.find("ceph_version_short");
+    versions[q->second].push_back(string("mon.") + monmap->get_name(rank));
+    i = i + 1;
+  }
+}
+
 int Monitor::print_nodes(Formatter *f, ostream& err)
 {
   map<string, list<string> > mons;	// hostname => mon
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index c3a15a16abff..936af1ee91d4 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -965,6 +965,9 @@ private:
   int load_metadata();
   void count_metadata(const std::string& field, ceph::Formatter *f);
   void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  // get_all_versions() gathers version information from daemons for health check
+  void get_all_versions(std::map<string, list<string> > &versions);
+  void get_versions(std::map<string, list<string> > &versions);
 
   // features
   static CompatSet get_initial_supported_features();