dispatcher: Refuse to start extra dispatchers

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/dispatcher/__init__.py b/teuthology/dispatcher/__init__.py
index 14218835b18ec52aea266c0403aa9ee7ec16cf89..273f747e278ed30a3a44e7514e90a40501a6e15f 100644 (file)
--- a/teuthology/dispatcher/__init__.py
+++ b/teuthology/dispatcher/__init__.py
@@ -1,5 +1,7 @@
+import getpass
 import logging
 import os
+import psutil
 import subprocess
 import sys
 import yaml
@@ -69,6 +71,14 @@ def main(args):
     if archive_dir is None:
         archive_dir = teuth_config.archive_base
 
+    # Refuse to start more than one dispatcher per machine type
+    procs = find_dispatcher_processes(tube)
+    if procs:
+        raise RuntimeError(
+            "There is already a teuthology-dispatcher process running:"
+            f" {procs}"
+        )
+
     # setup logging for disoatcher in {log_dir}
     loglevel = logging.INFO
     if verbose:
@@ -184,6 +194,29 @@ def main(args):
     return max(returncodes)
 
 
+def find_dispatcher_processes(machine_type):
+    user = getpass.getuser()
+    def match(proc):
+        if proc.username() != user:
+            return False
+        cmdline = proc.cmdline()
+        if len(cmdline) < 3:
+            return False
+        if not cmdline[1].endswith("/teuthology-dispatcher"):
+            return False
+        if cmdline[2] == "--supervisor":
+            return False
+        if machine_type not in cmdline:
+            return False
+        if proc.pid == os.getpid():
+            return False
+        return True
+
+    attrs = ["pid", "username", "cmdline"]
+    procs = list(filter(match, psutil.process_iter(attrs=attrs)))
+    return procs
+
+
 def lock_machines(job_config):
     report.try_push_job_info(job_config, dict(status='running'))
     fake_ctx = supervisor.create_fake_context(job_config, block=True)