From 5e463b84dcb34707ae1603ef4cf1d189062bf855 Mon Sep 17 00:00:00 2001
From: Tyler <tyler_sheehan@student.uml.edu>
Date: Mon, 20 Jul 2020 13:18:29 -0400
Subject: [PATCH] mon: Verify version consistency across daemons

Add a function called check_daemon_versions()
Made minor changes based on pull request comments
Changed the way versions are checked along with some other minor modifications
Made minor modification to the error message
Modifications to include the hostname in the error message

Fixes: https://tracker.ceph.com/issues/46264

Signed-off-by: Tyler <tyler_sheehan@student.uml.edu>

Remove time since health errors aren't preserved if not set every time
Rename health error to DAEMON_OLD_VERSION
Reoganize code so that we can get versions from other daemon types
Change to map of version (key), a list of daemons (value)
Get actual daemon names from get_versions() requests
Fix message output based on how many daemons and how many versions
Add a config to be able to turn off this features
Give HEALTH_ERR if more than one older version exists
Fix minor indentation problem with health detail

Signed-off-by: David Zafman <dzafman@redhat.com>
---
 src/common/options.cc    |  5 +++++
 src/mon/HealthMonitor.cc | 41 ++++++++++++++++++++++++++++++++++++++++
 src/mon/Monitor.cc       | 17 +++++++++++++++++
 src/mon/Monitor.h        |  3 +++
 4 files changed, 66 insertions(+)

diff --git a/src/common/options.cc b/src/common/options.cc
index 28ad9c7922789..8165692f95a83 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -2228,6 +2228,11 @@ std::vector<Option> get_global_options() {
 		     "only be rejected later, when they are used.")
     .set_flag(Option::FLAG_RUNTIME),
 
+    Option("mon_warn_on_older_version", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .add_service("mon")
+    .set_description("issue DAEMON_OLD_VERSION health warning if daemons are not all running the same version"),
+
     // PAXOS
 
     Option("paxos_stash_full_interval", Option::TYPE_INT, Option::LEVEL_ADVANCED)
diff --git a/src/mon/HealthMonitor.cc b/src/mon/HealthMonitor.cc
index 1a2da9de240c7..582f99f9b5ffa 100644
--- a/src/mon/HealthMonitor.cc
+++ b/src/mon/HealthMonitor.cc
@@ -16,6 +16,8 @@
 #include <limits.h>
 #include <sstream>
 #include <regex>
+#include <time.h>
+#include <iterator>
 
 #include "include/ceph_assert.h"
 #include "include/common_fwd.h"
@@ -682,6 +684,45 @@ bool HealthMonitor::check_leader_health()
 
   health_check_map_t next;
 
+ // DAEMON_OLD_VERSION
+  if (g_conf().get_val<bool>("mon_warn_on_older_version")) {
+  std::map<string, std::list<string> > all_versions;
+  mon->get_all_versions(all_versions);
+  if (all_versions.size() > 1) {
+    dout(20) << __func__ << " all_versions=" << all_versions << dendl;
+    // The last entry has the largest version
+    dout(20) << __func__ << " highest version daemon count "
+             << all_versions.rbegin()->second.size() << dendl;
+    // Erase last element (the highest version running)
+    all_versions.erase(all_versions.rbegin()->first);
+    ceph_assert(all_versions.size() > 0);
+    ostringstream ss;
+    unsigned daemon_count = 0;
+    for (auto& g:all_versions) {
+      daemon_count += g.second.size();
+    }
+    int ver_count = all_versions.size();
+    ceph_assert(!(daemon_count == 1 && ver_count != 1));
+    ss << "There " << (daemon_count == 1 ? "is a daemon" : "are daemons")
+       << " running " << (ver_count > 1 ? "multiple old versions" : "an older version")  << " of ceph";
+    health_status_t status;
+    if (ver_count > 1)
+      status = HEALTH_ERR;
+    else
+      status = HEALTH_WARN;
+    auto& d = next.add("DAEMON_OLD_VERSION", status, ss.str(), all_versions.size());
+    for (auto& g:all_versions) {
+      ostringstream ds;
+      for (auto& i : g.second) { // Daemon list
+        ds << i << " ";
+      }
+      ds << (g.second.size() == 1 ? "is" : "are")
+         << " running an older version of ceph: " << g.first;
+      d.detail.push_back(ds.str());
+    }
+  }
+  }
+
   // MON_DOWN
   {
     int max = mon->monmap->size();
diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc
index 6ba0cbf0545fb..7e1d30628d3f8 100644
--- a/src/mon/Monitor.cc
+++ b/src/mon/Monitor.cc
@@ -5359,6 +5359,23 @@ void Monitor::count_metadata(const string& field, Formatter *f)
   f->close_section();
 }
 
+void Monitor::get_all_versions(std::map<string, list<string> > &versions)
+{
+  // mon
+  get_versions(versions);
+  dout(20) << __func__ << " all versions=" << versions << dendl;
+}
+
+void Monitor::get_versions(std::map<string, list<string> > &versions)
+{
+  int i = 0;
+  for (auto& [rank, metadata] : mon_metadata) {
+    auto q = metadata.find("ceph_version_short");
+    versions[q->second].push_back(string("mon.") + monmap->get_name(rank));
+    i = i + 1;
+  }
+}
+
 int Monitor::print_nodes(Formatter *f, ostream& err)
 {
   map<string, list<string> > mons;	// hostname => mon
diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h
index c3a15a16abfff..936af1ee91d45 100644
--- a/src/mon/Monitor.h
+++ b/src/mon/Monitor.h
@@ -965,6 +965,9 @@ private:
   int load_metadata();
   void count_metadata(const std::string& field, ceph::Formatter *f);
   void count_metadata(const std::string& field, std::map<std::string,int> *out);
+  // get_all_versions() gathers version information from daemons for health check
+  void get_all_versions(std::map<string, list<string> > &versions);
+  void get_versions(std::map<string, list<string> > &versions);
 
   // features
   static CompatSet get_initial_supported_features();
-- 
2.39.5