prometheus: Fix prometheus shutdown/restart

The prometheus serve method uses block which it should not as it can
cause several problems when shutting down (restarting) ceph-mgr. As a
result the restart of the ceph-mgr module will block until time-out if
the prometheus module is enabled. This fixes the problem by using a
simple simple shutdown_event instead of block() and stopping the server
afterwards.

Signed-off-by: Boris Ranto <branto@redhat.com>

diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
index c7daa128dd8af16fee1ae408895653f2684d8d18..858adf6e143303c82291e673a7956a0e1bece232 100644 (file)
--- a/src/pybind/mgr/prometheus/module.py
+++ b/src/pybind/mgr/prometheus/module.py
@@ -4,6 +4,7 @@ import errno
 import math
 import os
 import socket
+import threading
 from collections import OrderedDict
 from mgr_module import MgrModule, MgrStandbyModule
 
@@ -341,6 +342,7 @@ class Module(MgrModule):
         super(Module, self).__init__(*args, **kwargs)
         self.metrics = Metrics()
         self.schema = OrderedDict()
+        self.shutdown_event = threading.Event()
         _global_instance['plugin'] = self
 
     def get_health(self):
@@ -627,16 +629,22 @@ class Module(MgrModule):
         self.log.info('Starting engine...')
         cherrypy.engine.start()
         self.log.info('Engine started.')
-        cherrypy.engine.block()
+        # wait for the shutdown event
+        self.shutdown_event.wait()
+        self.shutdown_event.clear()
+        cherrypy.engine.stop()
+        self.log.info('Engine stopped.')
 
     def shutdown(self):
         self.log.info('Stopping engine...')
-        cherrypy.engine.wait(state=cherrypy.engine.states.STARTED)
-        cherrypy.engine.exit()
-        self.log.info('Stopped engine')
+        self.shutdown_event.set()
 
 
 class StandbyModule(MgrStandbyModule):
+    def __init__(self, *args, **kwargs):
+        super(StandbyModule, self).__init__(*args, **kwargs)
+        self.shutdown_event = threading.Event()
+
     def serve(self):
         server_addr = self.get_localized_config('server_addr', '::')
         server_port = self.get_localized_config('server_port', DEFAULT_PORT)
@@ -671,12 +679,14 @@ class StandbyModule(MgrStandbyModule):
         cherrypy.tree.mount(Root(), '/', {})
         self.log.info('Starting engine...')
         cherrypy.engine.start()
-        self.log.info("Waiting for engine...")
-        cherrypy.engine.wait(state=cherrypy.engine.states.STOPPED)
         self.log.info('Engine started.')
+        # Wait for shutdown event
+        self.shutdown_event.wait()
+        self.shutdown_event.clear()
+        cherrypy.engine.stop()
+        self.log.info('Engine stopped.')
 
     def shutdown(self):
         self.log.info("Stopping engine...")
-        cherrypy.engine.wait(state=cherrypy.engine.states.STARTED)
-        cherrypy.engine.stop()
+        self.shutdown_event.set()
         self.log.info("Stopped engine")