run_tasks: Add instrumentation for task time

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/exporter.py b/teuthology/exporter.py
index b3ee9dcf2fb1b8ce06cac398c3d088bab1d64dc2..776cc873b793d402b8c25c9fda80b753b08344f0 100644 (file)
--- a/teuthology/exporter.py
+++ b/teuthology/exporter.py
@@ -194,6 +194,12 @@ JobTime = Summary(
     ["suite"],
 )
 
+TaskTime = Summary(
+    "teuthology_task_duration_seconds",
+    "Time spent executing a task",
+    ["name", "phase"],
+)
+
 
 def main(args):
     exporter = TeuthologyExporter(interval=int(args["--interval"]))

diff --git a/teuthology/run_tasks.py b/teuthology/run_tasks.py
index 598947c80761bb17a3e62ec3649227b6740cc0db..e40830ca64f75e316e3e9cbd3da6144369c00e1d 100644 (file)
--- a/teuthology/run_tasks.py
+++ b/teuthology/run_tasks.py
@@ -11,6 +11,8 @@ from copy import deepcopy
 from humanfriendly import format_timespan
 import sentry_sdk
 
+import teuthology.exporter as exporter
+
 from teuthology.config import config as teuth_config
 from teuthology.exceptions import ConnectionLostError
 from teuthology.job_status import set_status, get_status
@@ -103,7 +105,8 @@ def run_tasks(tasks, ctx):
             manager = run_one_task(taskname, ctx=ctx, config=config)
             if hasattr(manager, '__enter__'):
                 stack.append((taskname, manager))
-                manager.__enter__()
+                with exporter.TaskTime.labels(taskname, "enter").time():
+                    manager.__enter__()
     except BaseException as e:
         if isinstance(e, ConnectionLostError):
             # Prevent connection issues being flagged as failures
@@ -185,7 +188,8 @@ def run_tasks(tasks, ctx):
                 log.debug('Unwinding manager %s', taskname)
                 timer.mark('%s exit' % taskname)
                 try:
-                    suppress = manager.__exit__(*exc_info)
+                    with exporter.TaskTime.labels(taskname, "exit").time():
+                        suppress = manager.__exit__(*exc_info)
                 except Exception as e:
                     if isinstance(e, ConnectionLostError):
                         # Prevent connection issues being flagged as failures