From 37484af0b86083c02a7a506a0c9be02b3fd67b28 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 9 Apr 2018 15:25:45 -0400 Subject: [PATCH] mgr: rework kv store load path The locking and blocking around this was a bit tricky. Do the simple thing, and pull the load_store out to Mgr so that it can be safely done as part of the background_init process (just drop Mgr::lock across blocking actions). Signed-off-by: John Spray --- src/mgr/ActivePyModules.cc | 50 +++----------------- src/mgr/ActivePyModules.h | 1 + src/mgr/BaseMgrModule.cc | 47 +++++++++++++++++++ src/mgr/Mgr.cc | 93 +++++++++++++++++++------------------ src/mgr/Mgr.h | 2 +- src/mgr/MgrStandby.cc | 4 +- src/mgr/PyModule.cc | 48 +++++++++++++++++++ src/mgr/PyModule.h | 11 +++++ src/mgr/PyModuleRegistry.cc | 47 +++++++++++++------ src/mgr/PyModuleRegistry.h | 17 +++---- src/mgr/StandbyPyModules.cc | 7 ++- src/mgr/StandbyPyModules.h | 3 -- 12 files changed, 210 insertions(+), 120 deletions(-) diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 5bc1fbdcc46..ff4bd2d3ce0 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -36,6 +36,7 @@ #define dout_prefix *_dout << "mgr " << __func__ << " " ActivePyModules::ActivePyModules(PyModuleConfig &module_config_, + std::map store_data, DaemonStateIndex &ds, ClusterState &cs, MonClient &mc, LogChannelRef clog_, Objecter &objecter_, Client &client_, Finisher &f) @@ -43,6 +44,7 @@ ActivePyModules::ActivePyModules(PyModuleConfig &module_config_, monc(mc), clog(clog_), objecter(objecter_), client(client_), finisher(f), lock("ActivePyModules") { + store_cache = std::move(store_data); } ActivePyModules::~ActivePyModules() = default; @@ -426,7 +428,7 @@ bool ActivePyModules::get_store(const std::string &module_name, Mutex::Locker l(lock); PyEval_RestoreThread(tstate); - const std::string global_key = PyModuleRegistry::config_prefix + const std::string global_key = PyModule::config_prefix + module_name + "/" + key; dout(4) << __func__ << "key: " << global_key << dendl; @@ -446,7 +448,7 @@ bool ActivePyModules::get_config(const std::string &module_name, Mutex::Locker l(lock); PyEval_RestoreThread(tstate); - const std::string global_key = PyModuleRegistry::config_prefix + const std::string global_key = PyModule::config_prefix + module_name + "/" + key; dout(4) << __func__ << " key: " << global_key << dendl; @@ -469,7 +471,7 @@ PyObject *ActivePyModules::get_config_prefix(const std::string &module_name, Mutex::Locker l(lock); PyEval_RestoreThread(tstate); - const std::string base_prefix = PyModuleRegistry::config_prefix + const std::string base_prefix = PyModule::config_prefix + module_name + "/"; const std::string global_prefix = base_prefix + prefix; dout(4) << __func__ << " prefix: " << global_prefix << dendl; @@ -489,7 +491,7 @@ PyObject *ActivePyModules::get_config_prefix(const std::string &module_name, void ActivePyModules::set_store(const std::string &module_name, const std::string &key, const boost::optional& val) { - const std::string global_key = PyModuleRegistry::config_prefix + const std::string global_key = PyModule::config_prefix + module_name + "/" + key; Command set_cmd; @@ -534,45 +536,7 @@ void ActivePyModules::set_store(const std::string &module_name, void ActivePyModules::set_config(const std::string &module_name, const std::string &key, const boost::optional& val) { - const std::string global_key = PyModuleRegistry::config_prefix - + module_name + "/" + key; - - Command set_cmd; - { - PyThreadState *tstate = PyEval_SaveThread(); - Mutex::Locker l(lock); - PyEval_RestoreThread(tstate); - - Mutex::Locker lock(module_config.lock); - - if (val) { - module_config.config[global_key] = *val; - } else { - module_config.config.erase(global_key); - } - - std::ostringstream cmd_json; - JSONFormatter jf; - jf.open_object_section("cmd"); - if (val) { - jf.dump_string("prefix", "config set mgr"); - jf.dump_string("key", global_key); - jf.dump_string("val", *val); - } else { - jf.dump_string("prefix", "config rm"); - jf.dump_string("key", global_key); - } - jf.close_section(); - jf.flush(cmd_json); - set_cmd.run(&monc, cmd_json.str()); - } - set_cmd.wait(); - - if (set_cmd.r != 0) { - dout(0) << "`config set mgr" << global_key << " " << val << "` failed: " - << cpp_strerror(set_cmd.r) << dendl; - dout(0) << "mon returned " << set_cmd.r << ": " << set_cmd.outs << dendl; - } + module_config.set_config(&monc, module_name, key, val); } std::map ActivePyModules::get_services() const diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h index da0029f517f..69aaa5c9f07 100644 --- a/src/mgr/ActivePyModules.h +++ b/src/mgr/ActivePyModules.h @@ -47,6 +47,7 @@ class ActivePyModules public: ActivePyModules(PyModuleConfig &module_config, + std::map store_data, DaemonStateIndex &ds, ClusterState &cs, MonClient &mc, LogChannelRef clog_, Objecter &objecter_, Client &client_, Finisher &f); diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc index 6777a523888..ce42bf09a10 100644 --- a/src/mgr/BaseMgrModule.cc +++ b/src/mgr/BaseMgrModule.cc @@ -398,6 +398,47 @@ ceph_config_set(BaseMgrModule *self, PyObject *args) Py_RETURN_NONE; } + +static PyObject* +ceph_store_get(BaseMgrModule *self, PyObject *args) +{ + char *what = nullptr; + if (!PyArg_ParseTuple(args, "s:ceph_store_get", &what)) { + derr << "Invalid args!" << dendl; + return nullptr; + } + + std::string value; + bool found = self->py_modules->get_store(self->this_module->get_name(), + what, &value); + if (found) { + dout(10) << "ceph_store_get " << what << " found: " << value.c_str() << dendl; + return PyString_FromString(value.c_str()); + } else { + dout(4) << "ceph_store_get " << what << " not found " << dendl; + Py_RETURN_NONE; + } +} + + + +static PyObject* +ceph_store_set(BaseMgrModule *self, PyObject *args) +{ + char *key = nullptr; + char *value = nullptr; + if (!PyArg_ParseTuple(args, "sz:ceph_store_set", &key, &value)) { + return nullptr; + } + boost::optional val; + if (value) { + val = value; + } + self->py_modules->set_store(self->this_module->get_name(), key, val); + + Py_RETURN_NONE; +} + static PyObject* get_metadata(BaseMgrModule *self, PyObject *args) { @@ -544,6 +585,12 @@ PyMethodDef BaseMgrModule_methods[] = { {"_ceph_set_config", (PyCFunction)ceph_config_set, METH_VARARGS, "Set a configuration value"}, + {"_ceph_get_store", (PyCFunction)ceph_store_get, METH_VARARGS, + "Get a stored field"}, + + {"_ceph_set_store", (PyCFunction)ceph_store_set, METH_VARARGS, + "Set a stored field"}, + {"_ceph_get_counter", (PyCFunction)get_counter, METH_VARARGS, "Get a performance counter"}, diff --git a/src/mgr/Mgr.cc b/src/mgr/Mgr.cc index c20513e1e85..f78ec8e6417 100644 --- a/src/mgr/Mgr.cc +++ b/src/mgr/Mgr.cc @@ -150,6 +150,44 @@ void Mgr::background_init(Context *completion) })); } +std::map Mgr::load_store() +{ + assert(lock.is_locked_by_me()); + + dout(10) << "listing keys" << dendl; + JSONCommand cmd; + cmd.run(monc, "{\"prefix\": \"config-key ls\"}"); + lock.Unlock(); + cmd.wait(); + lock.Lock(); + assert(cmd.r == 0); + + std::map loaded; + + for (auto &key_str : cmd.json_result.get_array()) { + std::string const key = key_str.get_str(); + + dout(20) << "saw key '" << key << "'" << dendl; + + const std::string config_prefix = PyModule::config_prefix; + + if (key.substr(0, config_prefix.size()) == config_prefix) { + dout(20) << "fetching '" << key << "'" << dendl; + Command get_cmd; + std::ostringstream cmd_json; + cmd_json << "{\"prefix\": \"config-key get\", \"key\": \"" << key << "\"}"; + get_cmd.run(monc, cmd_json.str()); + lock.Unlock(); + get_cmd.wait(); + lock.Lock(); + assert(get_cmd.r == 0); + loaded[key] = get_cmd.outbl.to_str(); + } + } + + return loaded; +} + void Mgr::init() { Mutex::Locker l(lock); @@ -205,21 +243,25 @@ void Mgr::init() dout(4) << "waiting for config-keys..." << dendl; - // Preload config keys (`get` for plugins is to be a fast local - // operation, we we don't have to synchronize these later because - // all sets will come via mgr) - auto loaded_config = load_config(); - // Wait for MgrDigest... dout(4) << "waiting for MgrDigest..." << dendl; while (!digest_received) { digest_cond.Wait(lock); } + // Load module KV store + auto kv_store = load_store(); + + // Migrate config from KV store on luminous->mimic + // drop lock because we do blocking config sets to mon + lock.Unlock(); + py_module_registry->upgrade_config(monc, kv_store); + lock.Lock(); + // assume finisher already initialized in background_init dout(4) << "starting python modules..." << dendl; - py_module_registry->active_start(loaded_config, daemon_state, cluster_state, - *monc, clog, *objecter, *client, finisher); + py_module_registry->active_start(daemon_state, cluster_state, + kv_store, *monc, clog, *objecter, *client, finisher); dout(4) << "Complete." << dendl; initializing = false; @@ -315,43 +357,6 @@ void Mgr::load_all_metadata() } } -PyModuleConfig Mgr::load_config() -{ - assert(lock.is_locked_by_me()); - - dout(10) << "listing keys" << dendl; - JSONCommand cmd; - cmd.run(monc, "{\"prefix\": \"config-key ls\"}"); - lock.Unlock(); - cmd.wait(); - lock.Lock(); - assert(cmd.r == 0); - - PyModuleConfig loaded; - - for (auto &key_str : cmd.json_result.get_array()) { - std::string const key = key_str.get_str(); - - dout(20) << "saw key '" << key << "'" << dendl; - - const std::string config_prefix = PyModuleRegistry::config_prefix; - - if (key.substr(0, config_prefix.size()) == config_prefix) { - dout(20) << "fetching '" << key << "'" << dendl; - Command get_cmd; - std::ostringstream cmd_json; - cmd_json << "{\"prefix\": \"config-key get\", \"key\": \"" << key << "\"}"; - get_cmd.run(monc, cmd_json.str()); - lock.Unlock(); - get_cmd.wait(); - lock.Lock(); - assert(get_cmd.r == 0); - loaded.config[key] = get_cmd.outbl.to_str(); - } - } - - return loaded; -} void Mgr::shutdown() { diff --git a/src/mgr/Mgr.h b/src/mgr/Mgr.h index 173fe04a1d5..ee5f8c274c5 100644 --- a/src/mgr/Mgr.h +++ b/src/mgr/Mgr.h @@ -63,8 +63,8 @@ protected: LogChannelRef clog; LogChannelRef audit_clog; - PyModuleConfig load_config(); void load_all_metadata(); + std::map load_store(); void init(); bool initialized; diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index 46d809974ee..9de2e88abc7 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -131,7 +131,7 @@ int MgrStandby::init() monc.register_config_callback([this](const std::string &k, const std::string &v){ dout(10) << "config_callback: " << k << " : " << v << dendl; if (k.substr(0, 4) == "mgr/") { - const std::string global_key = PyModuleRegistry::config_prefix + k.substr(4); + const std::string global_key = PyModule::config_prefix + k.substr(4); py_module_registry.handle_config(global_key, v); return true; @@ -400,7 +400,7 @@ void MgrStandby::handle_mgr_map(MMgrMap* mmap) // I am the standby and someone else is active, start modules // in standby mode to do redirects if needed if (!py_module_registry.is_standby_running()) { - py_module_registry.standby_start(&monc); + py_module_registry.standby_start(); } } } diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc index da7b0adc403..89439860625 100644 --- a/src/mgr/PyModule.cc +++ b/src/mgr/PyModule.cc @@ -14,6 +14,7 @@ #include "BaseMgrModule.h" #include "BaseMgrStandbyModule.h" #include "PyOSDMap.h" +#include "MgrContext.h" #include "PyModule.h" @@ -24,6 +25,9 @@ #undef dout_prefix #define dout_prefix *_dout << "mgr[py] " +// definition for non-const static member +std::string PyModule::config_prefix = "mgr/"; + // Courtesy of http://stackoverflow.com/questions/1418015/how-to-get-python-exception-text #include #include @@ -131,6 +135,50 @@ PyModuleConfig::PyModuleConfig(PyModuleConfig &mconfig) PyModuleConfig::~PyModuleConfig() = default; + +void PyModuleConfig::set_config( + MonClient *monc, + const std::string &module_name, + const std::string &key, const boost::optional& val) +{ + const std::string global_key = PyModule::config_prefix + + module_name + "/" + key; + { + Mutex::Locker l(lock); + + if (val) { + config[global_key] = *val; + } else { + config.erase(global_key); + } + } + + Command set_cmd; + { + std::ostringstream cmd_json; + JSONFormatter jf; + jf.open_object_section("cmd"); + if (val) { + jf.dump_string("prefix", "config set mgr"); + jf.dump_string("key", global_key); + jf.dump_string("val", *val); + } else { + jf.dump_string("prefix", "config rm"); + jf.dump_string("key", global_key); + } + jf.close_section(); + jf.flush(cmd_json); + set_cmd.run(monc, cmd_json.str()); + } + set_cmd.wait(); + + if (set_cmd.r != 0) { + dout(0) << "`config set mgr" << global_key << " " << val << "` failed: " + << cpp_strerror(set_cmd.r) << dendl; + dout(0) << "mon returned " << set_cmd.r << ": " << set_cmd.outs << dendl; + } +} + std::string PyModule::get_site_packages() { std::stringstream site_packages; diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h index 7a314af9796..512313d60da 100644 --- a/src/mgr/PyModule.h +++ b/src/mgr/PyModule.h @@ -19,6 +19,9 @@ #include #include "common/Mutex.h" #include +#include + +class MonClient; std::string handle_pyerror(); @@ -68,6 +71,8 @@ private: std::vector commands; public: + static std::string config_prefix; + SafeThreadState pMyThreadState; PyObject *pClass = nullptr; PyObject *pStandbyClass = nullptr; @@ -142,4 +147,10 @@ public: PyModuleConfig(PyModuleConfig &mconfig); ~PyModuleConfig(); + + void set_config( + MonClient *monc, + const std::string &module_name, + const std::string &key, const boost::optional& val); + }; diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc index 686dbfc872c..0537a614a33 100644 --- a/src/mgr/PyModuleRegistry.cc +++ b/src/mgr/PyModuleRegistry.cc @@ -19,16 +19,12 @@ #include "PyOSDMap.h" #include "BaseMgrStandbyModule.h" #include "Gil.h" +#include "MgrContext.h" #include "ActivePyModules.h" #include "PyModuleRegistry.h" -// definition for non-const static member -std::string PyModuleRegistry::config_prefix; - - - #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mgr @@ -41,9 +37,6 @@ void PyModuleRegistry::init() { Mutex::Locker locker(lock); - // namespace in config-key prefixed by "mgr/" - config_prefix = std::string(g_conf->name.get_type_str()) + "/"; - // Set up global python interpreter #if PY_MAJOR_VERSION >= 3 #define WCHAR(s) L ## #s @@ -129,7 +122,9 @@ bool PyModuleRegistry::handle_mgr_map(const MgrMap &mgr_map_) } } -void PyModuleRegistry::standby_start(MonClient *monc) + + +void PyModuleRegistry::standby_start() { Mutex::Locker l(lock); assert(active_modules == nullptr); @@ -142,7 +137,7 @@ void PyModuleRegistry::standby_start(MonClient *monc) dout(4) << "Starting modules in standby mode" << dendl; standby_modules.reset(new StandbyPyModules( - monc, mgr_map, module_config, clog)); + mgr_map, module_config, clog)); std::set failed_modules; for (const auto &i : modules) { @@ -173,10 +168,10 @@ void PyModuleRegistry::standby_start(MonClient *monc) } void PyModuleRegistry::active_start( - PyModuleConfig &module_config, - DaemonStateIndex &ds, ClusterState &cs, MonClient &mc, - LogChannelRef clog_, Objecter &objecter_, Client &client_, - Finisher &f) + DaemonStateIndex &ds, ClusterState &cs, + const std::map &kv_store, + MonClient &mc, LogChannelRef clog_, Objecter &objecter_, + Client &client_, Finisher &f) { Mutex::Locker locker(lock); @@ -194,7 +189,8 @@ void PyModuleRegistry::active_start( } active_modules.reset(new ActivePyModules( - module_config, ds, cs, mc, clog_, objecter_, client_, f)); + module_config, kv_store, ds, cs, mc, + clog_, objecter_, client_, f)); for (const auto &i : modules) { // Anything we're skipping because of !can_run will be flagged @@ -399,3 +395,24 @@ void PyModuleRegistry::handle_config(const std::string &k, const std::string &v) } } +void PyModuleRegistry::upgrade_config( + MonClient *monc, + const std::map &old_config) +{ + // Only bother doing anything if we didn't already have + // some new-style config. + if (module_config.config.empty()) { + // Upgrade luminous->mimic: migrate config-key configuration + // into main configuration store + for(auto &i : old_config) { + auto last_slash = i.first.rfind('/'); + const std::string module_name = i.first.substr(4, i.first.substr(4).find('/')); + const std::string key = i.first.substr(last_slash + 1); + module_config.set_config(monc, module_name, key, i.second); + } + } else { + dout(10) << "Module configuration contains " + << module_config.config.size() << " keys" << dendl; + } +} + diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h index 8f64cf1d76f..c7563c29b94 100644 --- a/src/mgr/PyModuleRegistry.h +++ b/src/mgr/PyModuleRegistry.h @@ -59,8 +59,6 @@ private: PyModuleConfig module_config; public: - static std::string config_prefix; - void handle_config(const std::string &k, const std::string &v); /** @@ -89,13 +87,16 @@ public: void init(); + void upgrade_config( + MonClient *monc, + const std::map &old_config); + void active_start( - PyModuleConfig &module_config, - DaemonStateIndex &ds, ClusterState &cs, MonClient &mc, - LogChannelRef clog_, Objecter &objecter_, Client &client_, - Finisher &f); - void standby_start( - MonClient *monc); + DaemonStateIndex &ds, ClusterState &cs, + const std::map &kv_store, + MonClient &mc, LogChannelRef clog_, Objecter &objecter_, + Client &client_, Finisher &f); + void standby_start(); bool is_standby_running() const { diff --git a/src/mgr/StandbyPyModules.cc b/src/mgr/StandbyPyModules.cc index 3d7298206f7..560902eece3 100644 --- a/src/mgr/StandbyPyModules.cc +++ b/src/mgr/StandbyPyModules.cc @@ -32,10 +32,9 @@ StandbyPyModules::StandbyPyModules( - MonClient *monc_, const MgrMap &mgr_map_, + const MgrMap &mgr_map_, PyModuleConfig &module_config, LogChannelRef clog_) - : monc(monc_), - state(module_config), + : state(module_config), clog(clog_) { state.set_mgr_map(mgr_map_); @@ -127,7 +126,7 @@ bool StandbyPyModule::get_config(const std::string &key, PyThreadState *tstate = PyEval_SaveThread(); PyEval_RestoreThread(tstate); - const std::string global_key = PyModuleRegistry::config_prefix + const std::string global_key = PyModule::config_prefix + get_name() + "/" + key; dout(4) << __func__ << " key: " << global_key << dendl; diff --git a/src/mgr/StandbyPyModules.h b/src/mgr/StandbyPyModules.h index fe2de8f279a..2def578b385 100644 --- a/src/mgr/StandbyPyModules.h +++ b/src/mgr/StandbyPyModules.h @@ -95,8 +95,6 @@ private: mutable Mutex lock{"StandbyPyModules::lock"}; std::map> modules; - MonClient *monc; - StandbyPyModuleState state; LogChannelRef clog; @@ -104,7 +102,6 @@ private: public: StandbyPyModules( - MonClient *monc_, const MgrMap &mgr_map_, PyModuleConfig &module_config, LogChannelRef clog_); -- 2.39.5