From 6a8da7ca734726315c06929635c25dbb88599654 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 16 Nov 2017 06:45:45 -0500 Subject: [PATCH] mgr: load all modules (not just active ones) This is to enable us to learn more about the module before it is enabled, such as whether its can_run method return true. We can also use this to enable loading a module's commands before it is enabled, to give the user a better response when they try to use a command belong to a module that is not loaded. Signed-off-by: John Spray --- src/CMakeLists.txt | 1 + src/mgr/ActivePyModule.cc | 49 ++---- src/mgr/ActivePyModule.h | 6 +- src/mgr/ActivePyModules.cc | 18 +-- src/mgr/ActivePyModules.h | 4 +- src/mgr/MgrStandby.cc | 2 +- src/mgr/PyModule.cc | 306 ++++++++++++++++++++++++++++++++++++ src/mgr/PyModule.h | 68 ++++++++ src/mgr/PyModuleRegistry.cc | 267 ++----------------------------- src/mgr/PyModuleRegistry.h | 34 +--- src/mgr/PyModuleRunner.cc | 21 ++- src/mgr/PyModuleRunner.h | 28 ++-- src/mgr/StandbyPyModules.cc | 24 ++- src/mgr/StandbyPyModules.h | 10 +- 14 files changed, 451 insertions(+), 387 deletions(-) create mode 100644 src/mgr/PyModule.cc create mode 100644 src/mgr/PyModule.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8dbf6d17073..a672add1990 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -733,6 +733,7 @@ if (WITH_MGR) mgr/ActivePyModules.cc mgr/OSDHealthMetricCollector.cc mgr/StandbyPyModules.cc + mgr/PyModule.cc mgr/PyModuleRegistry.cc mgr/PyModuleRunner.cc mgr/PyFormatter.cc diff --git a/src/mgr/ActivePyModule.cc b/src/mgr/ActivePyModule.cc index 90040af930d..311144086ab 100644 --- a/src/mgr/ActivePyModule.cc +++ b/src/mgr/ActivePyModule.cc @@ -11,68 +11,39 @@ * Foundation. See file COPYING. */ -#include "BaseMgrModule.h" - #include "PyFormatter.h" #include "common/debug.h" #include "ActivePyModule.h" -//XXX courtesy of http://stackoverflow.com/questions/1418015/how-to-get-python-exception-text -#include -#include "include/assert.h" // boost clobbers this - #define dout_context g_ceph_context #define dout_subsys ceph_subsys_mgr #undef dout_prefix #define dout_prefix *_dout << "mgr " << __func__ << " " -// decode a Python exception into a string -std::string handle_pyerror() -{ - using namespace boost::python; - using namespace boost; - - PyObject *exc, *val, *tb; - object formatted_list, formatted; - PyErr_Fetch(&exc, &val, &tb); - handle<> hexc(exc), hval(allow_null(val)), htb(allow_null(tb)); - object traceback(import("traceback")); - if (!tb) { - object format_exception_only(traceback.attr("format_exception_only")); - formatted_list = format_exception_only(hexc, hval); - } else { - object format_exception(traceback.attr("format_exception")); - formatted_list = format_exception(hexc,hval, htb); - } - formatted = str("").join(formatted_list); - return extract(formatted); -} - int ActivePyModule::load(ActivePyModules *py_modules) { assert(py_modules); - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); // We tell the module how we name it, so that it can be consistent // with us in logging etc. auto pThisPtr = PyCapsule_New(this, nullptr, nullptr); auto pPyModules = PyCapsule_New(py_modules, nullptr, nullptr); - auto pModuleName = PyString_FromString(module_name.c_str()); + auto pModuleName = PyString_FromString(get_name().c_str()); auto pArgs = PyTuple_Pack(3, pModuleName, pPyModules, pThisPtr); - pClassInstance = PyObject_CallObject(pClass, pArgs); - Py_DECREF(pClass); + pClassInstance = PyObject_CallObject(py_module->pClass, pArgs); Py_DECREF(pModuleName); Py_DECREF(pArgs); if (pClassInstance == nullptr) { - derr << "Failed to construct class in '" << module_name << "'" << dendl; + derr << "Failed to construct class in '" << get_name() << "'" << dendl; derr << handle_pyerror() << dendl; return -EINVAL; } else { - dout(1) << "Constructed class from module: " << module_name << dendl; + dout(1) << "Constructed class from module: " << get_name() << dendl; } return load_commands(); @@ -82,7 +53,7 @@ void ActivePyModule::notify(const std::string ¬ify_type, const std::string &n { assert(pClassInstance != nullptr); - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); // Execute auto pValue = PyObject_CallMethod(pClassInstance, @@ -92,7 +63,7 @@ void ActivePyModule::notify(const std::string ¬ify_type, const std::string &n if (pValue != NULL) { Py_DECREF(pValue); } else { - derr << module_name << ".notify:" << dendl; + derr << get_name() << ".notify:" << dendl; derr << handle_pyerror() << dendl; // FIXME: callers can't be expected to handle a python module // that has spontaneously broken, but Mgr() should provide @@ -105,7 +76,7 @@ void ActivePyModule::notify_clog(const LogEntry &log_entry) { assert(pClassInstance != nullptr); - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); // Construct python-ized LogEntry PyFormatter f; @@ -120,7 +91,7 @@ void ActivePyModule::notify_clog(const LogEntry &log_entry) if (pValue != NULL) { Py_DECREF(pValue); } else { - derr << module_name << ".notify_clog:" << dendl; + derr << get_name() << ".notify_clog:" << dendl; derr << handle_pyerror() << dendl; // FIXME: callers can't be expected to handle a python module // that has spontaneously broken, but Mgr() should provide @@ -187,7 +158,7 @@ int ActivePyModule::handle_command( assert(ss != nullptr); assert(ds != nullptr); - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); PyFormatter f; cmdmap_dump(cmdmap, &f); diff --git a/src/mgr/ActivePyModule.h b/src/mgr/ActivePyModule.h index 9dbf92e2c33..b475610fbb1 100644 --- a/src/mgr/ActivePyModule.h +++ b/src/mgr/ActivePyModule.h @@ -58,11 +58,9 @@ private: std::string uri; public: - ActivePyModule(const std::string &module_name_, - PyObject *pClass_, - const SafeThreadState &my_ts_, + ActivePyModule(PyModuleRef py_module_, LogChannelRef clog_) - : PyModuleRunner(module_name_, pClass_, my_ts_, clog_) + : PyModuleRunner(py_module_, clog_) {} int load(ActivePyModules *py_modules); diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 47eb0a1325b..76b54203ded 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -12,7 +12,6 @@ */ // Include this first to get python headers earlier -#include "BaseMgrModule.h" #include "Gil.h" #include "common/errno.h" @@ -321,26 +320,23 @@ PyObject *ActivePyModules::get_python(const std::string &what) } } -int ActivePyModules::start_one(std::string const &module_name, - PyObject *pClass, const SafeThreadState &pMyThreadState) +int ActivePyModules::start_one(PyModuleRef py_module) { Mutex::Locker l(lock); - assert(modules.count(module_name) == 0); + assert(modules.count(py_module->get_name()) == 0); - modules[module_name].reset(new ActivePyModule( - module_name, pClass, - pMyThreadState, clog)); + modules[py_module->get_name()].reset(new ActivePyModule(py_module, clog)); - int r = modules[module_name]->load(this); + int r = modules[py_module->get_name()]->load(this); if (r != 0) { return r; } else { - dout(4) << "Starting thread for " << module_name << dendl; + dout(4) << "Starting thread for " << py_module->get_name() << dendl; // Giving Thread the module's module_name member as its // char* thread name: thread must not outlive module class lifetime. - modules[module_name]->thread.create( - modules[module_name]->get_name().c_str()); + modules[py_module->get_name()]->thread.create( + py_module->get_name().c_str()); return 0; } diff --git a/src/mgr/ActivePyModules.h b/src/mgr/ActivePyModules.h index 21e6529a933..6d0feedb417 100644 --- a/src/mgr/ActivePyModules.h +++ b/src/mgr/ActivePyModules.h @@ -108,9 +108,7 @@ public: int init(); void shutdown(); - int start_one(std::string const &module_name, - PyObject *pClass, - const SafeThreadState &pMyThreadState); + int start_one(PyModuleRef py_module); void dump_server(const std::string &hostname, const DaemonStateCollection &dmc, diff --git a/src/mgr/MgrStandby.cc b/src/mgr/MgrStandby.cc index cd8ec2ca36c..be0f6af1891 100644 --- a/src/mgr/MgrStandby.cc +++ b/src/mgr/MgrStandby.cc @@ -152,7 +152,7 @@ void MgrStandby::send_beacon() dout(1) << state_str() << dendl; set modules; - PyModuleRegistry::list_modules(&modules); + py_module_registry.list_modules(&modules); // Whether I think I am available (request MgrMonitor to set me // as available in the map) diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc new file mode 100644 index 00000000000..1e1d2659a46 --- /dev/null +++ b/src/mgr/PyModule.cc @@ -0,0 +1,306 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 John Spray + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#include "BaseMgrModule.h" +#include "BaseMgrStandbyModule.h" +#include "PyOSDMap.h" + +#include "PyModule.h" + +#include "common/debug.h" +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_mgr + +#undef dout_prefix +#define dout_prefix *_dout << "mgr[py] " + +// Courtesy of http://stackoverflow.com/questions/1418015/how-to-get-python-exception-text +#include +#include "include/assert.h" // boost clobbers this +// decode a Python exception into a string +std::string handle_pyerror() +{ + using namespace boost::python; + using namespace boost; + + PyObject *exc, *val, *tb; + object formatted_list, formatted; + PyErr_Fetch(&exc, &val, &tb); + handle<> hexc(exc), hval(allow_null(val)), htb(allow_null(tb)); + object traceback(import("traceback")); + if (!tb) { + object format_exception_only(traceback.attr("format_exception_only")); + formatted_list = format_exception_only(hexc, hval); + } else { + object format_exception(traceback.attr("format_exception")); + formatted_list = format_exception(hexc,hval, htb); + } + formatted = str("").join(formatted_list); + return extract(formatted); +} + + +namespace { + PyObject* log_write(PyObject*, PyObject* args) { + char* m = nullptr; + if (PyArg_ParseTuple(args, "s", &m)) { + auto len = strlen(m); + if (len && m[len-1] == '\n') { + m[len-1] = '\0'; + } + dout(4) << m << dendl; + } + Py_RETURN_NONE; + } + + PyObject* log_flush(PyObject*, PyObject*){ + Py_RETURN_NONE; + } + + static PyMethodDef log_methods[] = { + {"write", log_write, METH_VARARGS, "write stdout and stderr"}, + {"flush", log_flush, METH_VARARGS, "flush"}, + {nullptr, nullptr, 0, nullptr} + }; +} + + +std::string PyModule::get_site_packages() +{ + std::stringstream site_packages; + + // CPython doesn't auto-add site-packages dirs to sys.path for us, + // but it does provide a module that we can ask for them. + auto site_module = PyImport_ImportModule("site"); + assert(site_module); + + auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages"); + if (site_packages_fn != nullptr) { + auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr); + assert(site_packages_list); + + auto n = PyList_Size(site_packages_list); + for (Py_ssize_t i = 0; i < n; ++i) { + if (i != 0) { + site_packages << ":"; + } + site_packages << PyString_AsString(PyList_GetItem(site_packages_list, i)); + } + + Py_DECREF(site_packages_list); + Py_DECREF(site_packages_fn); + } else { + // Fall back to generating our own site-packages paths by imitating + // what the standard site.py does. This is annoying but it lets us + // run inside virtualenvs :-/ + + auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages"); + assert(site_packages_fn); + + auto known_paths = PySet_New(nullptr); + auto pArgs = PyTuple_Pack(1, known_paths); + PyObject_CallObject(site_packages_fn, pArgs); + Py_DECREF(pArgs); + Py_DECREF(known_paths); + Py_DECREF(site_packages_fn); + + auto sys_module = PyImport_ImportModule("sys"); + assert(sys_module); + auto sys_path = PyObject_GetAttrString(sys_module, "path"); + assert(sys_path); + + dout(1) << "sys.path:" << dendl; + auto n = PyList_Size(sys_path); + bool first = true; + for (Py_ssize_t i = 0; i < n; ++i) { + dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path, i)) << dendl; + if (first) { + first = false; + } else { + site_packages << ":"; + } + site_packages << PyString_AsString(PyList_GetItem(sys_path, i)); + } + + Py_DECREF(sys_path); + Py_DECREF(sys_module); + } + + Py_DECREF(site_module); + + return site_packages.str(); +} + + +int PyModule::load(PyThreadState *pMainThreadState) +{ + assert(pMainThreadState != nullptr); + + // Configure sub-interpreter and construct C++-generated python classes + { + SafeThreadState sts(pMainThreadState); + Gil gil(sts); + + auto thread_state = Py_NewInterpreter(); + if (thread_state == nullptr) { + derr << "Failed to create python sub-interpreter for '" << module_name << '"' << dendl; + return -EINVAL; + } else { + pMyThreadState.set(thread_state); + // Some python modules do not cope with an unpopulated argv, so lets + // fake one. This step also picks up site-packages into sys.path. + const char *argv[] = {"ceph-mgr"}; + PySys_SetArgv(1, (char**)argv); + + if (g_conf->daemonize) { + auto py_logger = Py_InitModule("ceph_logger", log_methods); +#if PY_MAJOR_VERSION >= 3 + PySys_SetObject("stderr", py_logger); + PySys_SetObject("stdout", py_logger); +#else + PySys_SetObject(const_cast("stderr"), py_logger); + PySys_SetObject(const_cast("stdout"), py_logger); +#endif + } + + // Configure sys.path to include mgr_module_path + std::string sys_path = std::string(Py_GetPath()) + ":" + get_site_packages() + + ":" + g_conf->get_val("mgr_module_path"); + dout(10) << "Computed sys.path '" << sys_path << "'" << dendl; + + PySys_SetPath(const_cast(sys_path.c_str())); + } + + PyMethodDef ModuleMethods[] = { + {nullptr} + }; + + // Initialize module + PyObject *ceph_module = Py_InitModule("ceph_module", ModuleMethods); + assert(ceph_module != nullptr); + + auto load_class = [ceph_module](const char *name, PyTypeObject *type) + { + type->tp_new = PyType_GenericNew; + if (PyType_Ready(type) < 0) { + assert(0); + } + Py_INCREF(type); + + PyModule_AddObject(ceph_module, name, (PyObject *)type); + }; + + load_class("BaseMgrModule", &BaseMgrModuleType); + load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType); + load_class("BasePyOSDMap", &BasePyOSDMapType); + load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType); + load_class("BasePyCRUSH", &BasePyCRUSHType); + } + + // Environment is all good, import the external module + { + Gil gil(pMyThreadState); + int r; + r = load_subclass_of("MgrModule", &pClass); + if (r) { + derr << "Class not found in module '" << module_name << "'" << dendl; + return r; + } + + // We've imported the module and found a MgrModule subclass, at this + // point the module is considered loaded. It might still not be + // runnable though, can_run populated later... + loaded = true; + + r = load_subclass_of("MgrStandbyModule", &pStandbyClass); + if (!r) { + dout(4) << "Standby mode available in module '" << module_name + << "'" << dendl; + } else { + dout(4) << "Standby mode not provided by module '" << module_name + << "'" << dendl; + } + + // Populate can_run by interrogating the module's callback that + // may check for dependencies etc + // TODO + + } + return 0; +} + +int PyModule::load_subclass_of(const char* base_class, PyObject** py_class) +{ + // load the base class + PyObject *mgr_module = PyImport_ImportModule("mgr_module"); + if (!mgr_module) { + derr << "Module not found: 'mgr_module'" << dendl; + derr << handle_pyerror() << dendl; + return -EINVAL; + } + auto mgr_module_type = PyObject_GetAttrString(mgr_module, base_class); + Py_DECREF(mgr_module); + if (!mgr_module_type) { + derr << "Unable to import MgrModule from mgr_module" << dendl; + derr << handle_pyerror() << dendl; + return -EINVAL; + } + + // find the sub class + PyObject *plugin_module = PyImport_ImportModule(module_name.c_str()); + if (!plugin_module) { + derr << "Module not found: '" << module_name << "'" << dendl; + derr << handle_pyerror() << dendl; + return -ENOENT; + } + auto locals = PyModule_GetDict(plugin_module); + Py_DECREF(plugin_module); + PyObject *key, *value; + Py_ssize_t pos = 0; + *py_class = nullptr; + while (PyDict_Next(locals, &pos, &key, &value)) { + if (!PyType_Check(value)) { + continue; + } + if (!PyObject_IsSubclass(value, mgr_module_type)) { + continue; + } + if (PyObject_RichCompareBool(value, mgr_module_type, Py_EQ)) { + continue; + } + auto class_name = PyString_AsString(key); + if (*py_class) { + derr << __func__ << ": ignoring '" + << module_name << "." << class_name << "'" + << ": only one '" << base_class + << "' class is loaded from each plugin" << dendl; + continue; + } + *py_class = value; + dout(4) << __func__ << ": found class: '" + << module_name << "." << class_name << "'" << dendl; + } + Py_DECREF(mgr_module_type); + + return *py_class ? 0 : -EINVAL; +} + +PyModule::~PyModule() +{ + if (pMyThreadState.ts != nullptr) { + Gil gil(pMyThreadState, true); + Py_XDECREF(pClass); + Py_XDECREF(pStandbyClass); + } +} + diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h new file mode 100644 index 00000000000..ae56e11167e --- /dev/null +++ b/src/mgr/PyModule.h @@ -0,0 +1,68 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2017 John Spray + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + */ + +#pragma once + +#include "Python.h" +#include "Gil.h" + +#include +#include "common/Mutex.h" +#include + +std::string handle_pyerror(); + +class PyModule +{ + mutable Mutex lock{"PyModule::lock"}; +private: + const std::string module_name; + std::string get_site_packages(); + int load_subclass_of(const char* class_name, PyObject** py_class); + + // Did the MgrMap identify this module as one that should run? + bool enabled = false; + + // Did we successfully import this python module and look up symbols? + // (i.e. is it possible to instantiate a MgrModule subclass instance?) + bool loaded = false; + + // Did the module identify itself as being able to run? + // (i.e. should we expect instantiating and calling serve() to work?) + bool can_run = false; + + // Did the module encounter an unexpected error while running? + // (e.g. throwing an exception from serve()) + bool failed = false; + +public: + SafeThreadState pMyThreadState; + PyObject *pClass = nullptr; + PyObject *pStandbyClass = nullptr; + + PyModule(const std::string &module_name_, bool const enabled_) + : module_name(module_name_), enabled(enabled_) + { + } + + ~PyModule(); + + int load(PyThreadState *pMainThreadState); + + bool is_enabled() const { Mutex::Locker l(lock) ; return enabled; } + const std::string &get_name() const + { Mutex::Locker l(lock) ; return module_name; } +}; + +typedef std::shared_ptr PyModuleRef; + diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc index a70e622be3e..dcda1e72f84 100644 --- a/src/mgr/PyModuleRegistry.cc +++ b/src/mgr/PyModuleRegistry.cc @@ -35,101 +35,11 @@ std::string PyModuleRegistry::config_prefix; #undef dout_prefix #define dout_prefix *_dout << "mgr[py] " -namespace { - PyObject* log_write(PyObject*, PyObject* args) { - char* m = nullptr; - if (PyArg_ParseTuple(args, "s", &m)) { - auto len = strlen(m); - if (len && m[len-1] == '\n') { - m[len-1] = '\0'; - } - dout(4) << m << dendl; - } - Py_RETURN_NONE; - } - - PyObject* log_flush(PyObject*, PyObject*){ - Py_RETURN_NONE; - } - static PyMethodDef log_methods[] = { - {"write", log_write, METH_VARARGS, "write stdout and stderr"}, - {"flush", log_flush, METH_VARARGS, "flush"}, - {nullptr, nullptr, 0, nullptr} - }; -} #undef dout_prefix #define dout_prefix *_dout << "mgr " << __func__ << " " - - -std::string PyModule::get_site_packages() -{ - std::stringstream site_packages; - - // CPython doesn't auto-add site-packages dirs to sys.path for us, - // but it does provide a module that we can ask for them. - auto site_module = PyImport_ImportModule("site"); - assert(site_module); - - auto site_packages_fn = PyObject_GetAttrString(site_module, "getsitepackages"); - if (site_packages_fn != nullptr) { - auto site_packages_list = PyObject_CallObject(site_packages_fn, nullptr); - assert(site_packages_list); - - auto n = PyList_Size(site_packages_list); - for (Py_ssize_t i = 0; i < n; ++i) { - if (i != 0) { - site_packages << ":"; - } - site_packages << PyString_AsString(PyList_GetItem(site_packages_list, i)); - } - - Py_DECREF(site_packages_list); - Py_DECREF(site_packages_fn); - } else { - // Fall back to generating our own site-packages paths by imitating - // what the standard site.py does. This is annoying but it lets us - // run inside virtualenvs :-/ - - auto site_packages_fn = PyObject_GetAttrString(site_module, "addsitepackages"); - assert(site_packages_fn); - - auto known_paths = PySet_New(nullptr); - auto pArgs = PyTuple_Pack(1, known_paths); - PyObject_CallObject(site_packages_fn, pArgs); - Py_DECREF(pArgs); - Py_DECREF(known_paths); - Py_DECREF(site_packages_fn); - - auto sys_module = PyImport_ImportModule("sys"); - assert(sys_module); - auto sys_path = PyObject_GetAttrString(sys_module, "path"); - assert(sys_path); - - dout(1) << "sys.path:" << dendl; - auto n = PyList_Size(sys_path); - bool first = true; - for (Py_ssize_t i = 0; i < n; ++i) { - dout(1) << " " << PyString_AsString(PyList_GetItem(sys_path, i)) << dendl; - if (first) { - first = false; - } else { - site_packages << ":"; - } - site_packages << PyString_AsString(PyList_GetItem(sys_path, i)); - } - - Py_DECREF(sys_path); - Py_DECREF(sys_module); - } - - Py_DECREF(site_module); - - return site_packages.str(); -} - int PyModuleRegistry::init(const MgrMap &map) { Mutex::Locker locker(lock); @@ -159,10 +69,15 @@ int PyModuleRegistry::init(const MgrMap &map) std::list failed_modules; + std::set module_names; + list_modules(&module_names); // Load python code - for (const auto& module_name : mgr_map.modules) { + for (const auto& module_name : module_names) { dout(1) << "Loading python module '" << module_name << "'" << dendl; - auto mod = std::make_unique(module_name); + + bool enabled = (mgr_map.modules.count(module_name) > 0); + + auto mod = std::make_shared(module_name, enabled); int r = mod->load(pMainThreadState); if (r != 0) { // Don't use handle_pyerror() here; we don't have the GIL @@ -185,158 +100,6 @@ int PyModuleRegistry::init(const MgrMap &map) return 0; } - -int PyModule::load(PyThreadState *pMainThreadState) -{ - assert(pMainThreadState != nullptr); - - // Configure sub-interpreter and construct C++-generated python classes - { - SafeThreadState sts(pMainThreadState); - Gil gil(sts); - - auto thread_state = Py_NewInterpreter(); - if (thread_state == nullptr) { - derr << "Failed to create python sub-interpreter for '" << module_name << '"' << dendl; - return -EINVAL; - } else { - pMyThreadState.set(thread_state); - // Some python modules do not cope with an unpopulated argv, so lets - // fake one. This step also picks up site-packages into sys.path. - const char *argv[] = {"ceph-mgr"}; - PySys_SetArgv(1, (char**)argv); - - if (g_conf->daemonize) { - auto py_logger = Py_InitModule("ceph_logger", log_methods); -#if PY_MAJOR_VERSION >= 3 - PySys_SetObject("stderr", py_logger); - PySys_SetObject("stdout", py_logger); -#else - PySys_SetObject(const_cast("stderr"), py_logger); - PySys_SetObject(const_cast("stdout"), py_logger); -#endif - } - - // Configure sys.path to include mgr_module_path - std::string sys_path = std::string(Py_GetPath()) + ":" + get_site_packages() - + ":" + g_conf->get_val("mgr_module_path"); - dout(10) << "Computed sys.path '" << sys_path << "'" << dendl; - - PySys_SetPath(const_cast(sys_path.c_str())); - } - - PyMethodDef ModuleMethods[] = { - {nullptr} - }; - - // Initialize module - PyObject *ceph_module = Py_InitModule("ceph_module", ModuleMethods); - assert(ceph_module != nullptr); - - auto load_class = [ceph_module](const char *name, PyTypeObject *type) - { - type->tp_new = PyType_GenericNew; - if (PyType_Ready(type) < 0) { - assert(0); - } - Py_INCREF(type); - - PyModule_AddObject(ceph_module, name, (PyObject *)type); - }; - - load_class("BaseMgrModule", &BaseMgrModuleType); - load_class("BaseMgrStandbyModule", &BaseMgrStandbyModuleType); - load_class("BasePyOSDMap", &BasePyOSDMapType); - load_class("BasePyOSDMapIncremental", &BasePyOSDMapIncrementalType); - load_class("BasePyCRUSH", &BasePyCRUSHType); - } - - // Environment is all good, import the external module - { - Gil gil(pMyThreadState); - int r; - r = load_subclass_of("MgrModule", &pClass); - if (r) { - derr << "Class not found in module '" << module_name << "'" << dendl; - return r; - } - r = load_subclass_of("MgrStandbyModule", &pStandbyClass); - if (!r) { - dout(4) << "Standby mode available in module '" << module_name - << "'" << dendl; - } else { - dout(4) << "Standby mode not provided by module '" << module_name - << "'" << dendl; - } - } - return 0; -} - -int PyModule::load_subclass_of(const char* base_class, PyObject** py_class) -{ - // load the base class - PyObject *mgr_module = PyImport_ImportModule("mgr_module"); - if (!mgr_module) { - derr << "Module not found: 'mgr_module'" << dendl; - derr << handle_pyerror() << dendl; - return -EINVAL; - } - auto mgr_module_type = PyObject_GetAttrString(mgr_module, base_class); - Py_DECREF(mgr_module); - if (!mgr_module_type) { - derr << "Unable to import MgrModule from mgr_module" << dendl; - derr << handle_pyerror() << dendl; - return -EINVAL; - } - - // find the sub class - PyObject *plugin_module = PyImport_ImportModule(module_name.c_str()); - if (!plugin_module) { - derr << "Module not found: '" << module_name << "'" << dendl; - derr << handle_pyerror() << dendl; - return -ENOENT; - } - auto locals = PyModule_GetDict(plugin_module); - Py_DECREF(plugin_module); - PyObject *key, *value; - Py_ssize_t pos = 0; - *py_class = nullptr; - while (PyDict_Next(locals, &pos, &key, &value)) { - if (!PyType_Check(value)) { - continue; - } - if (!PyObject_IsSubclass(value, mgr_module_type)) { - continue; - } - if (PyObject_RichCompareBool(value, mgr_module_type, Py_EQ)) { - continue; - } - auto class_name = PyString_AsString(key); - if (*py_class) { - derr << __func__ << ": ignoring '" - << module_name << "." << class_name << "'" - << ": only one '" << base_class - << "' class is loaded from each plugin" << dendl; - continue; - } - *py_class = value; - dout(4) << __func__ << ": found class: '" - << module_name << "." << class_name << "'" << dendl; - } - Py_DECREF(mgr_module_type); - - return *py_class ? 0 : -EINVAL; -} - -PyModule::~PyModule() -{ - if (pMyThreadState.ts != nullptr) { - Gil gil(pMyThreadState, true); - Py_XDECREF(pClass); - Py_XDECREF(pStandbyClass); - } -} - void PyModuleRegistry::standby_start(MonClient *monc) { Mutex::Locker l(lock); @@ -350,11 +113,13 @@ void PyModuleRegistry::standby_start(MonClient *monc) std::set failed_modules; for (const auto &i : modules) { + if (!i.second->is_enabled()) { + continue; + } + if (i.second->pStandbyClass) { dout(4) << "starting module " << i.second->get_name() << dendl; - int r = standby_modules->start_one(i.first, - i.second->pStandbyClass, - i.second->pMyThreadState); + int r = standby_modules->start_one(i.second); if (r != 0) { derr << "failed to start module '" << i.second->get_name() << "'" << dendl;; @@ -396,10 +161,12 @@ void PyModuleRegistry::active_start( config_, ds, cs, mc, clog_, objecter_, client_, f)); for (const auto &i : modules) { + if (!i.second->is_enabled()) { + continue; + } + dout(4) << "Starting " << i.first << dendl; - int r = active_modules->start_one(i.first, - i.second->pClass, - i.second->pMyThreadState); + int r = active_modules->start_one(i.second); if (r != 0) { derr << "Failed to run module in active mode ('" << i.first << "')" << dendl; diff --git a/src/mgr/PyModuleRegistry.h b/src/mgr/PyModuleRegistry.h index 4dec1008116..b821cb5c8ac 100644 --- a/src/mgr/PyModuleRegistry.h +++ b/src/mgr/PyModuleRegistry.h @@ -14,8 +14,8 @@ #pragma once -// Python.h comes first because otherwise it clobbers ceph's assert -#include "Python.h" +// First because it includes Python.h +#include "PyModule.h" #include #include @@ -26,37 +26,15 @@ #include "ActivePyModules.h" #include "StandbyPyModules.h" -class PyModule -{ -private: - const std::string module_name; - std::string get_site_packages(); - int load_subclass_of(const char* class_name, PyObject** py_class); -public: - SafeThreadState pMyThreadState; - PyObject *pClass = nullptr; - PyObject *pStandbyClass = nullptr; - - PyModule(const std::string &module_name_) - : module_name(module_name_) - { - } - - ~PyModule(); - - int load(PyThreadState *pMainThreadState); - std::string get_name() const { - return module_name; - } -}; /** * This class is responsible for setting up the python runtime environment * and importing the python modules. * * It is *not* responsible for constructing instances of their BaseMgrModule - * subclasses. + * subclasses: that is the job of ActiveMgrModule, which consumes the class + * references that we load here. */ class PyModuleRegistry { @@ -65,7 +43,7 @@ private: LogChannelRef clog; - std::map> modules; + std::map modules; std::unique_ptr active_modules; std::unique_ptr standby_modules; @@ -79,7 +57,7 @@ private: public: static std::string config_prefix; - static void list_modules(std::set *modules); + void list_modules(std::set *modules); PyModuleRegistry(LogChannelRef clog_) : clog(clog_) diff --git a/src/mgr/PyModuleRunner.cc b/src/mgr/PyModuleRunner.cc index 881ad16b33f..9e5a0da32dd 100644 --- a/src/mgr/PyModuleRunner.cc +++ b/src/mgr/PyModuleRunner.cc @@ -15,6 +15,8 @@ // Python.h comes first because otherwise it clobbers ceph's assert #include "Python.h" +#include "PyModule.h" + #include "common/debug.h" #include "mgr/Gil.h" @@ -24,19 +26,14 @@ #define dout_subsys ceph_subsys_mgr -std::string handle_pyerror(); - PyModuleRunner::~PyModuleRunner() { - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); if (pClassInstance) { Py_XDECREF(pClassInstance); pClassInstance = nullptr; } - - Py_DECREF(pClass); - pClass = nullptr; } int PyModuleRunner::serve() @@ -45,7 +42,7 @@ int PyModuleRunner::serve() // This method is called from a separate OS thread (i.e. a thread not // created by Python), so tell Gil to wrap this in a new thread state. - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); auto pValue = PyObject_CallMethod(pClassInstance, const_cast("serve"), nullptr); @@ -68,10 +65,10 @@ int PyModuleRunner::serve() Py_DECREF(pvalue_str); PyErr_Restore(ptype, pvalue, ptraceback); - clog->error() << "Unhandled exception from module '" << module_name + clog->error() << "Unhandled exception from module '" << get_name() << "' while running on mgr." << g_conf->name.get_id() << ": " << exc_msg; - derr << module_name << ".serve:" << dendl; + derr << get_name() << ".serve:" << dendl; derr << handle_pyerror() << dendl; return -EINVAL; } @@ -83,7 +80,7 @@ void PyModuleRunner::shutdown() { assert(pClassInstance != nullptr); - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); auto pValue = PyObject_CallMethod(pClassInstance, const_cast("shutdown"), nullptr); @@ -91,7 +88,7 @@ void PyModuleRunner::shutdown() if (pValue != NULL) { Py_DECREF(pValue); } else { - derr << "Failed to invoke shutdown() on " << module_name << dendl; + derr << "Failed to invoke shutdown() on " << get_name() << dendl; derr << handle_pyerror() << dendl; } } @@ -99,7 +96,7 @@ void PyModuleRunner::shutdown() void PyModuleRunner::log(int level, const std::string &record) { #undef dout_prefix -#define dout_prefix *_dout << "mgr[" << module_name << "] " +#define dout_prefix *_dout << "mgr[" << get_name() << "] " dout(level) << record << dendl; #undef dout_prefix #define dout_prefix *_dout << "mgr " << __func__ << " " diff --git a/src/mgr/PyModuleRunner.h b/src/mgr/PyModuleRunner.h index c4b861cf3b6..40b27d11f20 100644 --- a/src/mgr/PyModuleRunner.h +++ b/src/mgr/PyModuleRunner.h @@ -18,6 +18,8 @@ #include "common/LogClient.h" #include "mgr/Gil.h" +#include "PyModule.h" + /** * Implement the pattern of calling serve() on a module in a thread, * until shutdown() is called. @@ -25,16 +27,10 @@ class PyModuleRunner { protected: - const std::string module_name; - - // Passed in by whoever loaded our python module and looked up - // the symbols in it. - PyObject *pClass = nullptr; + // Info about the module we're going to run + PyModuleRef py_module; - // Passed in by whoever created our subinterpreter for us - SafeThreadState pMyThreadState = nullptr; - - // Populated when we construct our instance of pClass in load() + // Populated by descendent class PyObject *pClassInstance = nullptr; LogChannelRef clog; @@ -50,32 +46,28 @@ protected: void *entry() override; }; + public: int serve(); void shutdown(); void log(int level, const std::string &record); PyModuleRunner( - const std::string &module_name_, - PyObject *pClass_, - const SafeThreadState &pMyThreadState_, + PyModuleRef py_module_, LogChannelRef clog_) : - module_name(module_name_), - pClass(pClass_), pMyThreadState(pMyThreadState_), + py_module(py_module_), clog(clog_), thread(this) { - assert(pClass != nullptr); - assert(pMyThreadState.ts != nullptr); - assert(!module_name.empty()); + assert(py_module != nullptr); } ~PyModuleRunner(); PyModuleRunnerThread thread; - std::string const &get_name() const { return module_name; } + std::string const &get_name() const { return py_module->get_name(); } }; diff --git a/src/mgr/StandbyPyModules.cc b/src/mgr/StandbyPyModules.cc index 46c74e03e4d..5430658bf10 100644 --- a/src/mgr/StandbyPyModules.cc +++ b/src/mgr/StandbyPyModules.cc @@ -30,9 +30,6 @@ #undef dout_prefix #define dout_prefix *_dout << "mgr " << __func__ << " " -// Declaration fulfilled by ActivePyModules -std::string handle_pyerror(); - StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_, LogChannelRef clog_) @@ -77,17 +74,16 @@ void StandbyPyModules::shutdown() modules.clear(); } -int StandbyPyModules::start_one(std::string const &module_name, - PyObject *pClass, const SafeThreadState &pMyThreadState) +int StandbyPyModules::start_one(PyModuleRef py_module) { Mutex::Locker l(lock); + const std::string &module_name = py_module->get_name(); assert(modules.count(module_name) == 0); modules[module_name].reset(new StandbyPyModule( state, - module_name, pClass, - pMyThreadState, clog)); + py_module, clog)); if (modules.size() == 1) { load_config_thread.create("LoadConfig"); @@ -109,26 +105,26 @@ int StandbyPyModules::start_one(std::string const &module_name, int StandbyPyModule::load() { - Gil gil(pMyThreadState, true); + Gil gil(py_module->pMyThreadState, true); // We tell the module how we name it, so that it can be consistent // with us in logging etc. auto pThisPtr = PyCapsule_New(this, nullptr, nullptr); assert(pThisPtr != nullptr); - auto pModuleName = PyString_FromString(module_name.c_str()); + auto pModuleName = PyString_FromString(get_name().c_str()); assert(pModuleName != nullptr); auto pArgs = PyTuple_Pack(2, pModuleName, pThisPtr); Py_DECREF(pThisPtr); Py_DECREF(pModuleName); - pClassInstance = PyObject_CallObject(pClass, pArgs); + pClassInstance = PyObject_CallObject(py_module->pStandbyClass, pArgs); Py_DECREF(pArgs); if (pClassInstance == nullptr) { - derr << "Failed to construct class in '" << module_name << "'" << dendl; + derr << "Failed to construct class in '" << get_name() << "'" << dendl; derr << handle_pyerror() << dendl; return -EINVAL; } else { - dout(1) << "Constructed class from module: " << module_name << dendl; + dout(1) << "Constructed class from module: " << get_name() << dendl; return 0; } } @@ -172,7 +168,7 @@ bool StandbyPyModule::get_config(const std::string &key, PyEval_RestoreThread(tstate); const std::string global_key = PyModuleRegistry::config_prefix - + module_name + "/" + key; + + get_name() + "/" + key; dout(4) << __func__ << "key: " << global_key << dendl; @@ -190,7 +186,7 @@ std::string StandbyPyModule::get_active_uri() const { std::string result; state.with_mgr_map([&result, this](const MgrMap &mgr_map){ - auto iter = mgr_map.services.find(module_name); + auto iter = mgr_map.services.find(get_name()); if (iter != mgr_map.services.end()) { result = iter->second; } diff --git a/src/mgr/StandbyPyModules.h b/src/mgr/StandbyPyModules.h index 85edb6cc6e7..a22887bdeae 100644 --- a/src/mgr/StandbyPyModules.h +++ b/src/mgr/StandbyPyModules.h @@ -89,12 +89,10 @@ class StandbyPyModule : public PyModuleRunner StandbyPyModule( StandbyPyModuleState &state_, - const std::string &module_name_, - PyObject *pClass_, - const SafeThreadState &pMyThreadState_, + PyModuleRef py_module_, LogChannelRef clog_) : - PyModuleRunner(module_name_, pClass_, pMyThreadState_, clog_), + PyModuleRunner(py_module_, clog_), state(state_) { } @@ -139,9 +137,7 @@ public: const MgrMap &mgr_map_, LogChannelRef clog_); - int start_one(std::string const &module_name, - PyObject *pClass, - const SafeThreadState &pMyThreadState); + int start_one(PyModuleRef py_module); void shutdown(); -- 2.39.5