mgr: emit cluster log message on serve() exception

Fixes: http://tracker.ceph.com/issues/21999
Signed-off-by: John Spray <john.spray@redhat.com>

diff --git a/src/mgr/ActivePyModule.h b/src/mgr/ActivePyModule.h
index 0c2ee12f34e109af1d0a356afcddbcab5f7e127f..9dbf92e2c33ea4a13eff80409d101e92b82380b6 100644 (file)
--- a/src/mgr/ActivePyModule.h
+++ b/src/mgr/ActivePyModule.h
@@ -60,8 +60,9 @@ private:
 public:
   ActivePyModule(const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &my_ts_)
-    : PyModuleRunner(module_name_, pClass_, my_ts_)
+      const SafeThreadState &my_ts_,
+      LogChannelRef clog_)
+    : PyModuleRunner(module_name_, pClass_, my_ts_, clog_)
   {}
 
   int load(ActivePyModules *py_modules);

diff --git a/src/mgr/ActivePyModules.cc b/src/mgr/ActivePyModules.cc
index 83091d305e11cf6c501d4670540f45ade55735e2..47eb0a1325b897e40552500c6afeed3dbd6b486d 100644 (file)
--- a/src/mgr/ActivePyModules.cc
+++ b/src/mgr/ActivePyModules.cc
@@ -330,7 +330,7 @@ int ActivePyModules::start_one(std::string const &module_name,
 
   modules[module_name].reset(new ActivePyModule(
       module_name, pClass,
-      pMyThreadState));
+      pMyThreadState, clog));
 
   int r = modules[module_name]->load(this);
   if (r != 0) {

diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
index bab057c6c2846a60586faec82a41c45290458aa4..938a9c5b9b869151ef78193367294219c7494bfb 100644 (file)
--- a/src/mgr/PyModuleRegistry.cc
+++ b/src/mgr/PyModuleRegistry.cc
@@ -309,7 +309,7 @@ void PyModuleRegistry::standby_start(MonClient *monc)
 
   dout(4) << "Starting modules in standby mode" << dendl;
 
-  standby_modules.reset(new StandbyPyModules(monc, mgr_map));
+  standby_modules.reset(new StandbyPyModules(monc, mgr_map, clog));
 
   std::set<std::string> failed_modules;
   for (const auto &i : modules) {

diff --git a/src/mgr/PyModuleRunner.cc b/src/mgr/PyModuleRunner.cc
index 5e04e3da27c71bb476614e45f2c0d7bffe1e20a9..881ad16b33fc6f38ec6cb8d43d2f75054e54793e 100644 (file)
--- a/src/mgr/PyModuleRunner.cc
+++ b/src/mgr/PyModuleRunner.cc
@@ -54,6 +54,23 @@ int PyModuleRunner::serve()
   if (pValue != NULL) {
     Py_DECREF(pValue);
   } else {
+    // This is not a very informative log message because it's an
+    // unknown/unexpected exception that we can't say much about.
+
+    // Peek at the exception for the cluster log, before
+    // dumping the backtrace to log log next.
+    PyObject *ptype, *pvalue, *ptraceback;
+    PyErr_Fetch(&ptype, &pvalue, &ptraceback);
+    assert(ptype);
+    assert(pvalue);
+    PyObject *pvalue_str = PyObject_Str(pvalue);
+    std::string exc_msg = PyString_AsString(pvalue_str);
+    Py_DECREF(pvalue_str);
+    PyErr_Restore(ptype, pvalue, ptraceback);
+    
+    clog->error() << "Unhandled exception from module '" << module_name
+                  << "' while running on mgr." << g_conf->name.get_id()
+                  << ": " << exc_msg;
     derr << module_name << ".serve:" << dendl;
     derr << handle_pyerror() << dendl;
     return -EINVAL;

diff --git a/src/mgr/PyModuleRunner.h b/src/mgr/PyModuleRunner.h
index cb51df3eaaff1132726aa0672612a916fa095ff0..c4b861cf3b6e7597afd4a361b2ed94fc99fd0139 100644 (file)
--- a/src/mgr/PyModuleRunner.h
+++ b/src/mgr/PyModuleRunner.h
@@ -15,6 +15,7 @@
 #pragma once
 
 #include "common/Thread.h"
+#include "common/LogClient.h"
 #include "mgr/Gil.h"
 
 /**
@@ -36,6 +37,8 @@ protected:
   // Populated when we construct our instance of pClass in load()
   PyObject *pClassInstance = nullptr;
 
+  LogChannelRef clog;
+
   class PyModuleRunnerThread : public Thread
   {
     PyModuleRunner *mod;
@@ -55,10 +58,12 @@ public:
   PyModuleRunner(
       const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &pMyThreadState_)
+      const SafeThreadState &pMyThreadState_,
+      LogChannelRef clog_)
     : 
       module_name(module_name_),
       pClass(pClass_), pMyThreadState(pMyThreadState_),
+      clog(clog_),
       thread(this)
   {
     assert(pClass != nullptr);

diff --git a/src/mgr/StandbyPyModules.cc b/src/mgr/StandbyPyModules.cc
index e567269a3c75c2dedc7d56a754c3db031544d12c..46c74e03e4dd4678caae42da0824660272e04a73 100644 (file)
--- a/src/mgr/StandbyPyModules.cc
+++ b/src/mgr/StandbyPyModules.cc
@@ -34,8 +34,9 @@
 std::string handle_pyerror();
 
 
-StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_)
-    : monc(monc_), load_config_thread(monc, &state)
+StandbyPyModules::StandbyPyModules(MonClient *monc_, const MgrMap &mgr_map_,
+    LogChannelRef clog_)
+    : monc(monc_), load_config_thread(monc, &state), clog(clog_)
 {
   state.set_mgr_map(mgr_map_);
 }
@@ -86,7 +87,7 @@ int StandbyPyModules::start_one(std::string const &module_name,
   modules[module_name].reset(new StandbyPyModule(
       state,
       module_name, pClass,
-      pMyThreadState));
+      pMyThreadState, clog));
 
   if (modules.size() == 1) {
     load_config_thread.create("LoadConfig");

diff --git a/src/mgr/StandbyPyModules.h b/src/mgr/StandbyPyModules.h
index 4f011464a137231336cf39a33b664785526eb9e0..85edb6cc6e77d08f7fc8572d1336eb5d6537d46c 100644 (file)
--- a/src/mgr/StandbyPyModules.h
+++ b/src/mgr/StandbyPyModules.h
@@ -91,9 +91,10 @@ class StandbyPyModule : public PyModuleRunner
       StandbyPyModuleState &state_,
       const std::string &module_name_,
       PyObject *pClass_,
-      const SafeThreadState &pMyThreadState_)
+      const SafeThreadState &pMyThreadState_,
+      LogChannelRef clog_)
     :
-      PyModuleRunner(module_name_, pClass_, pMyThreadState_),
+      PyModuleRunner(module_name_, pClass_, pMyThreadState_, clog_),
       state(state_)
   {
   }
@@ -129,11 +130,14 @@ private:
 
   LoadConfigThread load_config_thread;
 
+  LogChannelRef clog;
+
 public:
 
   StandbyPyModules(
       MonClient *monc_,
-      const MgrMap &mgr_map_);
+      const MgrMap &mgr_map_,
+      LogChannelRef clog_);
 
   int start_one(std::string const &module_name,
                 PyObject *pClass,