Ensure console loggers die when the job does

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/task/console_log.py b/teuthology/task/console_log.py
index 97cf83450df2b15556caa23aaba3215526372b76..f9450b2a81793432a5e39c8e19ec2b2b2aa2bf18 100644 (file)
--- a/teuthology/task/console_log.py
+++ b/teuthology/task/console_log.py
@@ -2,6 +2,7 @@ import logging
 import os
 
 from teuthology.orchestra.cluster import Cluster
+from teuthology.exit import exiter
 
 from . import Task
 
@@ -42,7 +43,8 @@ class ConsoleLog(Task):
         if not self.enabled:
             return
         super(ConsoleLog, self).setup()
-        self.processes = list()
+        self.processes = dict()
+        self.signal_handlers = list()
         self.setup_archive()
 
     def setup_archive(self):
@@ -65,7 +67,15 @@ class ConsoleLog(Task):
                 "%s.log" % remote.shortname,
             )
             proc = remote.console.spawn_sol_log(log_path)
-            self.processes.append(proc)
+            self.processes[remote.shortname] = proc
+
+        # Install a signal handler to make sure the console-logging
+        # processes are terminated if the job is killed
+        def kill_console_loggers(signal_, frame):
+            for (name, proc) in self.processes.items():
+                log.debug("Killing console logger for %s", name)
+                proc.terminate()
+        exiter.add_handler(15, kill_console_loggers)
 
     def end(self):
         if not self.enabled:
@@ -74,7 +84,7 @@ class ConsoleLog(Task):
         self.stop_logging()
 
     def stop_logging(self, force=False):
-        for proc in self.processes:
+        for proc in self.processes.values():
             if proc.poll() is not None:
                 continue
             if force:
@@ -82,6 +92,10 @@ class ConsoleLog(Task):
             else:
                 proc.terminate()
 
+        # Remove any signal handlers
+        for handler in self.signal_handlers:
+            handler.remove()
+
     def teardown(self):
         if not self.enabled:
             return

diff --git a/teuthology/test/task/test_console_log.py b/teuthology/test/task/test_console_log.py
index b9731d47e3e33ffa21cdd92243d43d17e027b7dd..c55e9ac63db1a3f6515668e5bc5d56a73ee2b218 100644 (file)
--- a/teuthology/test/task/test_console_log.py
+++ b/teuthology/test/task/test_console_log.py
@@ -74,6 +74,6 @@ class TestConsoleLog(TestTask):
     def test_end(self, m_pconsole):
         with self.klass(self.ctx, self.task_config) as task:
             pass
-        for proc in task.processes:
+        for proc in task.processes.values():
             assert proc.terminate.called_once_with()
             assert proc.kill.called_once_with()