mgr: create always_on class of modules

support an 'always on' class of ceph-mgr modules. the purpose of an
always-on module is to behave as built-in / non-optional functionality.
this is accomplished by treating always-on modules as enabled
irregardless of the enabled set produced by the manager monitor.
always-on modules that fail to load are reported as health errors.

always-on modules are intended to satisfy important dependencies in a
cluster. as such, the set of always-on modules is specified statically,
as opposed to a property on the Python class, to enable robust error
reporting of always on modules that fail to load.

Signed-off-by: Noah Watkins <nwatkins@redhat.com>

diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc
index 57ceaeeb8da3dc3fdb7fb49a7794b14d58104555..d2f4a9ceb6540d7094727b13c0dfc072a2c23c1c 100644 (file)
--- a/src/mgr/MgrStandby.cc
+++ b/src/mgr/MgrStandby.cc
@@ -188,6 +188,10 @@ void MgrStandby::send_beacon()
   // which we will transmit to the monitor.
   std::vector<MgrMap::ModuleInfo> module_info;
   for (const auto &module : modules) {
+    // do not announce always_on modules to the monitor
+    if (module->is_always_on()) {
+      continue;
+    }
     MgrMap::ModuleInfo info;
     info.name = module->get_name();
     info.error_string = module->get_error_string();

diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h
index 162d0c90cef05f40706de8c504782998a95b05ab..353e222d8f642dfae0938864a3c6e810f25b02c2 100644 (file)
--- a/src/mgr/PyModule.h
+++ b/src/mgr/PyModule.h
@@ -57,6 +57,7 @@ class PyModule
   mutable Mutex lock{"PyModule::lock"};
 private:
   const std::string module_name;
+  const bool always_on;
   std::string get_site_packages();
   int load_subclass_of(const char* class_name, PyObject** py_class);
 
@@ -96,8 +97,8 @@ public:
   PyObject *pClass = nullptr;
   PyObject *pStandbyClass = nullptr;
 
-  explicit PyModule(const std::string &module_name_)
-    : module_name(module_name_)
+  explicit PyModule(const std::string &module_name_, bool always_on)
+    : module_name(module_name_), always_on(always_on)
   {
   }
 
@@ -141,9 +142,14 @@ public:
     error_string = reason;
   }
 
-  bool is_enabled() const { Mutex::Locker l(lock) ; return enabled; }
+  bool is_enabled() const {
+    Mutex::Locker l(lock);
+    return enabled || always_on;
+  }
+
   bool is_failed() const { Mutex::Locker l(lock) ; return failed; }
   bool is_loaded() const { Mutex::Locker l(lock) ; return loaded; }
+  bool is_always_on() const { Mutex::Locker l(lock) ; return always_on; }
 
   const std::string &get_name() const {
     Mutex::Locker l(lock) ; return module_name;

diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
index ebb01a5c5b269522252d0b22349f3c2fe1cf3c16..d3ecdbb81c8c292560e585018d595cbc9dbde2f5 100644 (file)
--- a/src/mgr/PyModuleRegistry.cc
+++ b/src/mgr/PyModuleRegistry.cc
@@ -20,6 +20,7 @@
 #include "BaseMgrStandbyModule.h"
 #include "Gil.h"
 #include "MgrContext.h"
+#include "mgr/mgr_commands.h"
 
 #include "ActivePyModules.h"
 
@@ -70,9 +71,12 @@ void PyModuleRegistry::init()
   for (const auto& module_name : module_names) {
     dout(1) << "Loading python module '" << module_name << "'" << dendl;
 
+    const bool always_on = always_on_modules.find(module_name) !=
+      always_on_modules.end();
+
     // Everything starts disabled, set enabled flag on module
     // when we see first MgrMap
-    auto mod = std::make_shared<PyModule>(module_name);
+    auto mod = std::make_shared<PyModule>(module_name, always_on);
     int r = mod->load(pMainThreadState);
     if (r != 0) {
       // Don't use handle_pyerror() here; we don't have the GIL
@@ -142,6 +146,10 @@ void PyModuleRegistry::standby_start(MonClient &mc)
   std::set<std::string> failed_modules;
   for (const auto &i : modules) {
     if (!(i.second->is_enabled() && i.second->get_can_run())) {
+      // report always_on modules with a standby mode that won't run
+      if (i.second->is_always_on() && i.second->pStandbyClass) {
+        failed_modules.insert(i.second->get_name());
+      }
       continue;
     }
 
@@ -365,6 +373,16 @@ void PyModuleRegistry::get_health_checks(health_check_map_t *checks)
       }
     }
 
+    // report failed always_on modules as health errors
+    for (const auto& name : always_on_modules) {
+      if (!active_modules->module_exists(name)) {
+        if (failed_modules.find(name) == failed_modules.end() &&
+            dependency_modules.find(name) == dependency_modules.end()) {
+          failed_modules[name] = "Unknown error";
+        }
+      }
+    }
+
     if (!dependency_modules.empty()) {
       std::ostringstream ss;
       if (dependency_modules.size() == 1) {

diff --git a/src/mgr/mgr_commands.cc b/src/mgr/mgr_commands.cc
index aafee99140cf81d33e5b60de19338ea0278e3945..c509d68dd99e60353b483b94b6517eb61a030910 100644 (file)
--- a/src/mgr/mgr_commands.cc
+++ b/src/mgr/mgr_commands.cc
@@ -3,6 +3,19 @@
 
 #include "mgr_commands.h"
 
+// an always_on module behaves as if it provides built-in functionality. it's
+// always enabled, and always_on modules do not appear in the list of modules
+// that can be enabled or disabled by the user. a health warning error is
+// reported if a module in this set is cannot be loaded and started.
+//
+// NOTE: if a module is added to this set AND that module _may_ have been
+// enabled in some cluster (e.g. an existing module is being upgraded to
+// always-on status), then additional logic is needed in the monitor to trim the
+// enabled set (or upgrade script to run `disable <module`). At the time of
+// writing, the proposed always-on modules are being added within the same
+// release.
+const std::set<std::string> always_on_modules = {"crash"};
+
 /* The set of statically defined (C++-handled) commands.  This
  * does not include the Python-defined commands, which are loaded
  * in PyModules */

diff --git a/src/mgr/mgr_commands.h b/src/mgr/mgr_commands.h
index c6ed6c68d727fc4ff2a5d027a3cc508c976a81b0..e32aa54c939f4f65e34b1b37a45e2c4b15471951 100644 (file)
--- a/src/mgr/mgr_commands.h
+++ b/src/mgr/mgr_commands.h
@@ -4,6 +4,9 @@
 #pragma once
 
 #include "mon/MonCommand.h"
+#include <set>
+#include <string>
 #include <vector>
 
+extern const std::set<std::string> always_on_modules;
 extern const std::vector<MonCommand> mgr_commands;