From f4763c32fa84f57c93c559d742203fb3fd2985a8 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 1 Nov 2017 12:08:03 -0400 Subject: [PATCH] mgr: emit cluster log message on serve() exception Fixes: http://tracker.ceph.com/issues/21999 Signed-off-by: John Spray --- src/mgr/ActivePyModule.h | 5 +++-- src/mgr/ActivePyModules.cc | 2 +- src/mgr/PyModuleRegistry.cc | 2 +- src/mgr/PyModuleRunner.cc | 17 +++++++++++++++++ src/mgr/PyModuleRunner.h | 7 ++++++- src/mgr/StandbyPyModules.cc | 7 ++++--- src/mgr/StandbyPyModules.h | 10 +++++++--- 7 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/mgr/ActivePyModule.h b/src/mgr/ActivePyModule.h index 0c2ee12f34e10..9dbf92e2c33ea 100644 --- a/src/mgr/ActivePyModule.h +++ b/src/mgr/ActivePyModule.h @@ -60,8 +60,9 @@ private: public: ActivePyModule(const std::string &module_name_, PyObject *pClass_, - const SafeThreadState &my_ts_) - : PyModuleRunner(module_name_, pClass_, my_ts_) + const SafeThreadState &my_ts_, + LogChannelRef clog_) + : PyModuleRunner(module_name_, pClass_, my_ts_, clog_) {} int load(ActivePyModules *py_modules); diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc index 83091d305e11c..47eb0a1325b89 100644 --- a/src/mgr/ActivePyModules.cc +++ b/src/mgr/ActivePyModules.cc @@ -330,7 +330,7 @@ int ActivePyModules::start_one(std::string const &module_name, modules[module_name].reset(new ActivePyModule( module_name, pClass, - pMyThreadState)); + pMyThreadState, clog)); int r = modules[module_name]->load(this); if (r != 0) { diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc index bab057c6c2846..938a9c5b9b869 100644 --- a/src/mgr/PyModuleRegistry.cc +++ b/src/mgr/PyModuleRegistry.cc @@ -309,7 +309,7 @@ void PyModuleRegistry::standby_start(MonClient *monc) dout(4) << "Starting modules in standby mode" << dendl; - standby_modules.reset(new StandbyPyModules(monc, mgr_map)); + standby_modules.reset(new StandbyPyModules(monc, mgr_map, clog)); std::set failed_modules; for (const auto &i : modules) { diff --git a/src/mgr/PyModuleRunner.cc b/src/mgr/PyModuleRunner.cc index 5e04e3da27c71..881ad16b33fc6 100644 --- a/src/mgr/PyModuleRunner.cc +++ b/src/mgr/PyModuleRunner.cc @@ -54,6 +54,23 @@ int PyModuleRunner::serve() if (pValue != NULL) { Py_DECREF(pValue); } else { + // This is not a very informative log message because it's an + // unknown/unexpected exception that we can't say much about. + + // Peek at the exception for the cluster log, before + // dumping the backtrace to log log next. + PyObject *ptype, *pvalue, *ptraceback; + PyErr_Fetch(&ptype, &pvalue, &ptraceback); + assert(ptype); + assert(pvalue); + PyObject *pvalue_str = PyObject_Str(pvalue); + std::string exc_msg = PyString_AsString(pvalue_str); + Py_DECREF(pvalue_str); + PyErr_Restore(ptype, pvalue, ptraceback); + + clog->error() << "Unhandled exception from module '" << module_name + << "' while running on mgr." << g_conf->name.get_id() + << ": " << exc_msg; derr << module_name << ".serve:" << dendl; derr << handle_pyerror() << dendl; return -EINVAL; diff --git a/src/mgr/PyModuleRunner.h b/src/mgr/PyModuleRunner.h index cb51df3eaaff1..c4b861cf3b6e7 100644 --- a/src/mgr/PyModuleRunner.h +++ b/src/mgr/PyModuleRunner.h @@ -15,6 +15,7 @@ #pragma once #include "common/Thread.h" +#include "common/LogClient.h" #include "mgr/Gil.h" /** @@ -36,6 +37,8 @@ protected: // Populated when we construct our instance of pClass in load() PyObject *pClassInstance = nullptr; + LogChannelRef clog; + class PyModuleRunnerThread : public Thread { PyModuleRunner *mod; @@ -55,10 +58,12 @@ public: PyModuleRunner( const std::string &module_name_, PyObject *pClass_, - const SafeThreadState &pMyThreadState_) + const SafeThreadState &pMyThreadState_, + LogChannelRef clog_) : module_name(module_name_), pClass(pClass_), pMyThreadState(pMyThreadState_), + clog(clog_), thread(this) { assert(pClass != nullptr); diff --git a/src/mgr/StandbyPyModules.cc b/src/mgr/StandbyPyModules.cc index e567269a3c75c..46c74e03e4dd4 100644 --- a/src/mgr/StandbyPyModules.cc +++ b/src/mgr/StandbyPyModules.cc @@ -34,8 +34,9 @@ std::string handle_pyerror(); -StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_) - : monc(monc_), load_config_thread(monc, &state) +StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_, + LogChannelRef clog_) + : monc(monc_), load_config_thread(monc, &state), clog(clog_) { state.set_mgr_map(mgr_map_); } @@ -86,7 +87,7 @@ int StandbyPyModules::start_one(std::string const &module_name, modules[module_name].reset(new StandbyPyModule( state, module_name, pClass, - pMyThreadState)); + pMyThreadState, clog)); if (modules.size() == 1) { load_config_thread.create("LoadConfig"); diff --git a/src/mgr/StandbyPyModules.h b/src/mgr/StandbyPyModules.h index 4f011464a1372..85edb6cc6e77d 100644 --- a/src/mgr/StandbyPyModules.h +++ b/src/mgr/StandbyPyModules.h @@ -91,9 +91,10 @@ class StandbyPyModule : public PyModuleRunner StandbyPyModuleState &state_, const std::string &module_name_, PyObject *pClass_, - const SafeThreadState &pMyThreadState_) + const SafeThreadState &pMyThreadState_, + LogChannelRef clog_) : - PyModuleRunner(module_name_, pClass_, pMyThreadState_), + PyModuleRunner(module_name_, pClass_, pMyThreadState_, clog_), state(state_) { } @@ -129,11 +130,14 @@ private: LoadConfigThread load_config_thread; + LogChannelRef clog; + public: StandbyPyModules( MonClient *monc_, - const MgrMap &mgr_map_); + const MgrMap &mgr_map_, + LogChannelRef clog_); int start_one(std::string const &module_name, PyObject *pClass, -- 2.39.5