add review changes

Signed-off-by: Shraddha Agrawal <shraddha.agrawal000@gmail.com>

diff --git a/teuthology/dispatcher/__init__.py b/teuthology/dispatcher/__init__.py
index 3e03a2b8ae5ed4d8834e9d0fea018b85cac2190f..b460d624dfb823041c73b84bc5a6ddfb13e25e60 100644 (file)
--- a/teuthology/dispatcher/__init__.py
+++ b/teuthology/dispatcher/__init__.py
@@ -26,10 +26,7 @@ def sentinel(path):
     if not os.path.exists(path):
         return False
     file_mtime = datetime.utcfromtimestamp(os.path.getmtime(path))
-    if file_mtime > start_time:
-        return True
-    else:
-        return False
+    return file_mtime > start_time
 
 
 def restart():
@@ -71,8 +68,7 @@ def main(args):
     if verbose:
         loglevel = logging.DEBUG
     log.setLevel(loglevel)
-    log_file_path = os.path.join(log_dir, 'dispatcher.{tube}.{pid}'.format(
-        pid=os.getpid(), tube=tube))
+    log_file_path = os.path.join(log_dir, f"dispatcher.{tube}.{os.getpid()}")
     setup_log_file(log_file_path)
     install_except_hook()
 

diff --git a/teuthology/dispatcher/supervisor.py b/teuthology/dispatcher/supervisor.py
index 4c71eeada42e3fe783bfc95c297c17ab752eb66a..65b1c1b30eebbf1039ab78ce564bfa4bab929dac 100644 (file)
--- a/teuthology/dispatcher/supervisor.py
+++ b/teuthology/dispatcher/supervisor.py
@@ -5,7 +5,6 @@ import time
 import yaml
 
 from datetime import datetime
-from tarfile import ReadError
 
 import teuthology
 from teuthology import report
@@ -14,14 +13,12 @@ from teuthology.config import config as teuth_config
 from teuthology.exceptions import SkipJob
 from teuthology import setup_log_file, install_except_hook
 from teuthology.lock.ops import reimage_many
-from teuthology.misc import get_user
+from teuthology.misc import get_user, archive_logs, compress_logs
 from teuthology.config import FakeNamespace
 from teuthology.job_status import get_status
 from teuthology.nuke import nuke
 from teuthology.kill import kill_job
-from teuthology.misc import pull_directory
 from teuthology.task.internal import add_remotes
-from teuthology.orchestra import run
 from teuthology.misc import decanonicalize_hostname as shortname
 from teuthology.lock import query
 
@@ -43,8 +40,8 @@ def main(args):
         loglevel = logging.DEBUG
     log.setLevel(loglevel)
 
-    log_file_path = os.path.join(job_config['archive_path'], 'supervisor.{job_id}'.format(
-                                 job_id=job_config['job_id']))
+    log_file_path = os.path.join(job_config['archive_path'],
+                                 f"supervisor.{job_config['job_id']}")
     setup_log_file(log_file_path)
     install_except_hook()
 
@@ -190,7 +187,8 @@ def unlock_targets(job_config):
         log.info('Unlocking machines...')
         fake_ctx = create_fake_context(job_config)
         for machine in locked:
-            teuthology.lock.ops.unlock_one(fake_ctx, machine, job_info['owner'],
+            teuthology.lock.ops.unlock_one(fake_ctx,
+                                           machine, job_info['owner'],
                                            job_info['archive_path'])
     if job_status != 'pass' and job_config.get('nuke-on-error', False):
         log.info('Nuking machines...')
@@ -238,18 +236,13 @@ def run_with_watchdog(process, job_config):
 
 
 def create_fake_context(job_config, block=False):
-    if job_config['owner'] is None:
-        job_config['owner'] = get_user()
-
-    if 'os_version' in job_config:
-        os_version = job_config['os_version']
-    else:
-        os_version = None
+    owner = job_config.get('owner', get_user())
+    os_version = job_config.get('os_version', None)
 
     ctx_args = {
         'config': job_config,
         'block': block,
-        'owner': job_config['owner'],
+        'owner': owner,
         'archive': job_config['archive_path'],
         'machine_type': job_config['machine_type'],
         'os_type': job_config['os_type'],
@@ -257,8 +250,7 @@ def create_fake_context(job_config, block=False):
         'name': job_config['name'],
     }
 
-    fake_ctx = FakeNamespace(ctx_args)
-    return fake_ctx
+    return FakeNamespace(ctx_args)
 
 
 def transfer_archives(run_name, job_id, archive_base, job_config):
@@ -276,49 +268,3 @@ def transfer_archives(run_name, job_id, archive_base, job_config):
             archive_logs(ctx, log_path, log_type)
     else:
         log.info('No archives to transfer.')
-
-
-def archive_logs(ctx, remote_path, log_path):
-    """
-    Archive directories from all nodes in a cliuster. It pulls all files in
-    remote_path dir to job's archive dir under log_path dir.
-    """
-    path = os.path.join(ctx.archive, 'remote')
-    if (not os.path.exists(path)):
-        os.mkdir(path)
-    for remote in ctx.cluster.remotes.keys():
-        sub = os.path.join(path, remote.shortname)
-        if (not os.path.exists(sub)):
-            os.makedirs(sub)
-        try:
-            pull_directory(remote, remote_path, os.path.join(sub, log_path))
-        except ReadError:
-            pass
-
-
-def compress_logs(ctx, remote_dir):
-    """
-    Compress all files in remote_dir from all nodes in a cluster.
-    """
-    log.info('Compressing logs...')
-    run.wait(
-        ctx.cluster.run(
-            args=[
-                'sudo',
-                'find',
-                remote_dir,
-                '-name',
-                '*.log',
-                '-print0',
-                run.Raw('|'),
-                'sudo',
-                'xargs',
-                '-0',
-                '--no-run-if-empty',
-                '--',
-                'gzip',
-                '--',
-            ],
-            wait=False,
-        ),
-    )

diff --git a/teuthology/misc.py b/teuthology/misc.py
index e444a9a9180d25dc7e8e5d63a65c2e6757a1dd40..8cd7070653d0f71cd734fe3afb77585931a3be26 100644 (file)
--- a/teuthology/misc.py
+++ b/teuthology/misc.py
@@ -15,6 +15,9 @@ import yaml
 import json
 import re
 import pprint
+import datetime
+
+from tarfile import ReadError
 
 from teuthology.util.compat import urljoin, urlopen, HTTPError
 
@@ -31,7 +34,6 @@ from teuthology.orchestra.opsys import DEFAULT_OS_VERSION
 
 log = logging.getLogger(__name__)
 
-import datetime
 stamp = datetime.datetime.now().strftime("%y%m%d%H%M")
 
 is_arm = lambda x: x.startswith('tala') or x.startswith(
@@ -1355,8 +1357,52 @@ def add_remote_path(ctx, local_dir, remote_dir):
     with open(os.path.join(ctx.archive, 'info.yaml'), 'r+') as info_file:
         info_yaml = yaml.safe_load(info_file)
         info_file.seek(0)
-        if 'archive' not in info_yaml:
-            info_yaml['archive'] = {local_dir: remote_dir}
-        else:
+        if 'archive' in info_yaml:
             info_yaml['archive'][local_dir] = remote_dir
-        yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
\ No newline at end of file
+        else:
+            info_yaml['archive'] = {local_dir: remote_dir}
+        yaml.safe_dump(info_yaml, info_file, default_flow_style=False)
+
+
+def archive_logs(ctx, remote_path, log_path):
+    """
+    Archive directories from all nodes in a cliuster. It pulls all files in
+    remote_path dir to job's archive dir under log_path dir.
+    """
+    path = os.path.join(ctx.archive, 'remote')
+    os.mkdirs(path, exist_ok=True)
+    for remote in ctx.cluster.remotes.keys():
+        sub = os.path.join(path, remote.shortname)
+        os.mkdirs(sub, exist_ok=True)
+        try:
+            pull_directory(remote, remote_path, os.path.join(sub, log_path))
+        except ReadError:
+            pass
+
+
+def compress_logs(ctx, remote_dir):
+    """
+    Compress all files in remote_dir from all nodes in a cluster.
+    """
+    log.info('Compressing logs...')
+    run.wait(
+        ctx.cluster.run(
+            args=[
+                'sudo',
+                'find',
+                remote_dir,
+                '-name',
+                '*.log',
+                '-print0',
+                run.Raw('|'),
+                'sudo',
+                'xargs',
+                '-0',
+                '--no-run-if-empty',
+                '--',
+                'gzip',
+                '--',
+            ],
+            wait=False,
+        ),
+    )