From a662caa79b8fc30e36df0da93f368b470d26564e Mon Sep 17 00:00:00 2001 From: Boris Ranto Date: Tue, 1 May 2018 01:06:50 +0200 Subject: [PATCH] prometheus: Fix prometheus shutdown/restart The prometheus serve method uses block which it should not as it can cause several problems when shutting down (restarting) ceph-mgr. As a result the restart of the ceph-mgr module will block until time-out if the prometheus module is enabled. This fixes the problem by using a simple simple shutdown_event instead of block() and stopping the server afterwards. Signed-off-by: Boris Ranto --- src/pybind/mgr/prometheus/module.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py index c7daa128dd8af..858adf6e14330 100644 --- a/src/pybind/mgr/prometheus/module.py +++ b/src/pybind/mgr/prometheus/module.py @@ -4,6 +4,7 @@ import errno import math import os import socket +import threading from collections import OrderedDict from mgr_module import MgrModule, MgrStandbyModule @@ -341,6 +342,7 @@ class Module(MgrModule): super(Module, self).__init__(*args, **kwargs) self.metrics = Metrics() self.schema = OrderedDict() + self.shutdown_event = threading.Event() _global_instance['plugin'] = self def get_health(self): @@ -627,16 +629,22 @@ class Module(MgrModule): self.log.info('Starting engine...') cherrypy.engine.start() self.log.info('Engine started.') - cherrypy.engine.block() + # wait for the shutdown event + self.shutdown_event.wait() + self.shutdown_event.clear() + cherrypy.engine.stop() + self.log.info('Engine stopped.') def shutdown(self): self.log.info('Stopping engine...') - cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) - cherrypy.engine.exit() - self.log.info('Stopped engine') + self.shutdown_event.set() class StandbyModule(MgrStandbyModule): + def __init__(self, *args, **kwargs): + super(StandbyModule, self).__init__(*args, **kwargs) + self.shutdown_event = threading.Event() + def serve(self): server_addr = self.get_localized_config('server_addr', '::') server_port = self.get_localized_config('server_port', DEFAULT_PORT) @@ -671,12 +679,14 @@ class StandbyModule(MgrStandbyModule): cherrypy.tree.mount(Root(), '/', {}) self.log.info('Starting engine...') cherrypy.engine.start() - self.log.info("Waiting for engine...") - cherrypy.engine.wait(state=cherrypy.engine.states.STOPPED) self.log.info('Engine started.') + # Wait for shutdown event + self.shutdown_event.wait() + self.shutdown_event.clear() + cherrypy.engine.stop() + self.log.info('Engine stopped.') def shutdown(self): self.log.info("Stopping engine...") - cherrypy.engine.wait(state=cherrypy.engine.states.STARTED) - cherrypy.engine.stop() + self.shutdown_event.set() self.log.info("Stopped engine") -- 2.39.5