mgr: add mgr_subinterpreter_modules config

This commit adds a mgr_subinterpreter_modules config to cause specified
modules (or all if * is specified) to be loaded in individual
subinterpreters.

This changes the default behavior of ceph-mgr from running each module
in a distinct subinterpreter to running them all in the same main
interpreter.  We can reintroduce subinterpreter support over time by
adding modules to the list as we test them.

Fixes: https://tracker.ceph.com/issues/73857
Fixes: https://tracker.ceph.com/issues/73859
Signed-off-by: Samuel Just <sjust@redhat.com>
(cherry picked from commit 239b0dc8a9c42449ee1faa1bf78bdcc380345ae2)

Conflicts:
  - src/mgr/PyModule.cc
     #include "common/JSONFormatter.h" - removed (missing commit 3ab70dd in tentacle), not in tentacle
     dtor - still missing commit 3366ef5 on tentacle causing conflicts,
      taking tentacle changes with use_main_interpreter and end the interpater pMyThreadState.ts

diff --git a/src/common/options/CMakeLists.txt b/src/common/options/CMakeLists.txt
index 60cdbc3f224b8bc2795c73ed1e621f4df17dd0e8..916f55dcfbd3a2f567b406b80b67c021cf6673d2 100644 (file)
--- a/src/common/options/CMakeLists.txt
+++ b/src/common/options/CMakeLists.txt
@@ -82,6 +82,8 @@ if(WITH_MGR)
   endif()
 endif()
 
+set(mgr_subinterpreter_modules "")
+
 add_options(global)
 add_options(cephfs-mirror)
 add_options(crimson)

diff --git a/src/common/options/mgr.yaml.in b/src/common/options/mgr.yaml.in
index 5c9fb5ddbd3dffa8c7d853bdb3e541d846428fd8..491c05115240c6862872dab55ea79fc6be6eb6ee 100644 (file)
--- a/src/common/options/mgr.yaml.in
+++ b/src/common/options/mgr.yaml.in
@@ -145,6 +145,19 @@ options:
   - mgr_module_path
   flags:
   - startup
+- name: mgr_subinterpreter_modules
+  type: str
+  level: advanced
+  desc: List of manager modules to load in independent subinterpreters
+  long_desc: A comma delimited list of module names. This list is read by manager
+    when it starts. By default, manager loads each module into the main interpreter.
+    Modules in this list will instead be loaded into independent subinterpreters.
+    Specifying '*' will cause all modules to be run in independent subinterpreters.
+  default: @mgr_subinterpreter_modules@
+  services:
+  - mgr
+  flags:
+  - startup
 - name: mgr_initial_modules
   type: str
   level: basic

diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc
index 44be043aeb1d5083682010b6d7861bb126896441..a25c45c2645aa3c17ec3331bc2813971c16bad3d 100644 (file)
--- a/src/mgr/PyModule.cc
+++ b/src/mgr/PyModule.cc
@@ -21,6 +21,7 @@
 
 #include "include/stringify.h"
 #include "common/BackTrace.h"
+#include "common/split.h"
 #include "global/signal_handler.h"
 
 #include "common/debug.h"
@@ -333,8 +334,23 @@ int PyModule::load(PyThreadState *pMainThreadState)
 {
   ceph_assert(pMainThreadState != nullptr);
 
-  // Configure sub-interpreter
-  {
+  const auto subinterpreter_modules_opt = g_conf().get_val<std::string>(
+    "mgr_subinterpreter_modules"
+  );
+  auto subinterpreter_modules = ceph::split(subinterpreter_modules_opt);
+  use_main_interpreter = std::count(
+    subinterpreter_modules.begin(), subinterpreter_modules.end(), "*"
+  ) == 0 && std::count(
+    subinterpreter_modules.begin(), subinterpreter_modules.end(), module_name
+  ) == 0;
+
+  if (use_main_interpreter) {
+    // Use main interpreter
+    derr << "Loading module " << module_name << " in main interpreter" << dendl;
+    pMyThreadState.set(pMainThreadState);
+  } else {
+    // Configure sub-interpreter
+    derr << "Loading module " << module_name << " in sub-interpreter" << dendl;
     SafeThreadState sts(pMainThreadState);
     Gil gil(sts);
 
@@ -346,6 +362,7 @@ int PyModule::load(PyThreadState *pMainThreadState)
       pMyThreadState.set(thread_state);
     }
   }
+
   // Environment is all good, import the external module
   {
     Gil gil(pMyThreadState);
@@ -757,6 +774,9 @@ PyModule::~PyModule()
     Py_XDECREF(pClass);
     Py_XDECREF(pStandbyClass);
     Py_XDECREF(pPickleModule);
+    if (use_main_interpreter) {
+      Py_EndInterpreter(pMyThreadState.ts);
+    }
   }
 }
 

diff --git a/src/mgr/PyModule.h b/src/mgr/PyModule.h
index 4cdb5e0b8e7f77531d20b00014c0d15238573ec0..f65e8c0dbc5982e06b294dd4d0e57be47f6aeb02 100644 (file)
--- a/src/mgr/PyModule.h
+++ b/src/mgr/PyModule.h
@@ -104,6 +104,9 @@ public:
   PyObject *pStandbyClass = nullptr;
   PyObject *pPickleModule = nullptr;
 
+  // true unless module in mgr_subinterpreter_modules
+  bool use_main_interpreter = true;
+
   explicit PyModule(const std::string &module_name_)
     : module_name(module_name_)
   {