From: Noah Watkins Date: Tue, 17 Jul 2018 05:24:48 +0000 (-0700) Subject: mgr: create always_on class of modules X-Git-Tag: v14.0.1~748^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cf292dfa8f9254e5095689c45a3d5e895e5df333;p=ceph.git mgr: create always_on class of modules support an 'always on' class of ceph-mgr modules. the purpose of an always-on module is to behave as built-in / non-optional functionality. this is accomplished by treating always-on modules as enabled irregardless of the enabled set produced by the manager monitor. always-on modules that fail to load are reported as health errors. always-on modules are intended to satisfy important dependencies in a cluster. as such, the set of always-on modules is specified statically, as opposed to a property on the Python class, to enable robust error reporting of always on modules that fail to load. Signed-off-by: Noah Watkins --- diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 57ceaeeb8da3..d2f4a9ceb654 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -188,6 +188,10 @@ void MgrStandby::send_beacon() // which we will transmit to the monitor. std::vector module_info; for (const auto &module : modules) { + // do not announce always_on modules to the monitor + if (module->is_always_on()) { + continue; + } MgrMap::ModuleInfo info; info.name = module->get_name(); info.error_string = module->get_error_string(); diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h index 162d0c90cef0..353e222d8f64 100644 --- a/src/mgr/PyModule.h +++ b/src/mgr/PyModule.h @@ -57,6 +57,7 @@ class PyModule mutable Mutex lock{"PyModule::lock"}; private: const std::string module_name; + const bool always_on; std::string get_site_packages(); int load_subclass_of(const char* class_name, PyObject** py_class); @@ -96,8 +97,8 @@ public: PyObject *pClass = nullptr; PyObject *pStandbyClass = nullptr; - explicit PyModule(const std::string &module_name_) - : module_name(module_name_) + explicit PyModule(const std::string &module_name_, bool always_on) + : module_name(module_name_), always_on(always_on) { } @@ -141,9 +142,14 @@ public: error_string = reason; } - bool is_enabled() const { Mutex::Locker l(lock) ; return enabled; } + bool is_enabled() const { + Mutex::Locker l(lock); + return enabled || always_on; + } + bool is_failed() const { Mutex::Locker l(lock) ; return failed; } bool is_loaded() const { Mutex::Locker l(lock) ; return loaded; } + bool is_always_on() const { Mutex::Locker l(lock) ; return always_on; } const std::string &get_name() const { Mutex::Locker l(lock) ; return module_name; diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc index ebb01a5c5b26..d3ecdbb81c8c 100644 --- a/src/mgr/PyModuleRegistry.cc +++ b/src/mgr/PyModuleRegistry.cc @@ -20,6 +20,7 @@ #include "BaseMgrStandbyModule.h" #include "Gil.h" #include "MgrContext.h" +#include "mgr/mgr_commands.h" #include "ActivePyModules.h" @@ -70,9 +71,12 @@ void PyModuleRegistry::init() for (const auto& module_name : module_names) { dout(1) << "Loading python module '" << module_name << "'" << dendl; + const bool always_on = always_on_modules.find(module_name) != + always_on_modules.end(); + // Everything starts disabled, set enabled flag on module // when we see first MgrMap - auto mod = std::make_shared(module_name); + auto mod = std::make_shared(module_name, always_on); int r = mod->load(pMainThreadState); if (r != 0) { // Don't use handle_pyerror() here; we don't have the GIL @@ -142,6 +146,10 @@ void PyModuleRegistry::standby_start(MonClient &mc) std::set failed_modules; for (const auto &i : modules) { if (!(i.second->is_enabled() && i.second->get_can_run())) { + // report always_on modules with a standby mode that won't run + if (i.second->is_always_on() && i.second->pStandbyClass) { + failed_modules.insert(i.second->get_name()); + } continue; } @@ -365,6 +373,16 @@ void PyModuleRegistry::get_health_checks(health_check_map_t *checks) } } + // report failed always_on modules as health errors + for (const auto& name : always_on_modules) { + if (!active_modules->module_exists(name)) { + if (failed_modules.find(name) == failed_modules.end() && + dependency_modules.find(name) == dependency_modules.end()) { + failed_modules[name] = "Unknown error"; + } + } + } + if (!dependency_modules.empty()) { std::ostringstream ss; if (dependency_modules.size() == 1) { diff --git a/src/mgr/mgr_commands.cc b/src/mgr/mgr_commands.cc index aafee99140cf..c509d68dd99e 100644 --- a/src/mgr/mgr_commands.cc +++ b/src/mgr/mgr_commands.cc @@ -3,6 +3,19 @@ #include "mgr_commands.h" +// an always_on module behaves as if it provides built-in functionality. it's +// always enabled, and always_on modules do not appear in the list of modules +// that can be enabled or disabled by the user. a health warning error is +// reported if a module in this set is cannot be loaded and started. +// +// NOTE: if a module is added to this set AND that module _may_ have been +// enabled in some cluster (e.g. an existing module is being upgraded to +// always-on status), then additional logic is needed in the monitor to trim the +// enabled set (or upgrade script to run `disable always_on_modules = {"crash"}; + /* The set of statically defined (C++-handled) commands. This * does not include the Python-defined commands, which are loaded * in PyModules */ diff --git a/src/mgr/mgr_commands.h b/src/mgr/mgr_commands.h index c6ed6c68d727..e32aa54c939f 100644 --- a/src/mgr/mgr_commands.h +++ b/src/mgr/mgr_commands.h @@ -4,6 +4,9 @@ #pragma once #include "mon/MonCommand.h" +#include +#include #include +extern const std::set always_on_modules; extern const std::vector mgr_commands;