From e2563b5559a785b68c0b164e804aa7a7c1e8a16f Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 14 Nov 2015 10:00:03 +0100 Subject: [PATCH] internal: the buildpackages task must come first Otherwise the machines will be locked and remain idle while the packages are building. When there are 20 jobs requiring a package, it means an average of 40 machines locked idle while the packages are building. Signed-off-by: Loic Dachary --- teuthology/run.py | 10 +++++----- teuthology/task/internal.py | 23 ++++++++++++++--------- teuthology/test/task/test_internal.py | 18 +++++++++++------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/teuthology/run.py b/teuthology/run.py index 5776496ad0..641bd9a6bd 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -173,7 +173,10 @@ def validate_tasks(config): def get_initial_tasks(lock, config, machine_type): - init_tasks = [{'internal.check_packages': None}] + init_tasks = [ + {'internal.check_packages': None}, + {'internal.buildpackages_prep': None}, + ] if 'roles' in config and lock: msg = ('You cannot specify targets in a config file when using the ' + '--lock option') @@ -214,10 +217,7 @@ def get_initial_tasks(lock, config, machine_type): {'internal.sudo': None}, {'internal.syslog': None}, ]) - init_tasks.extend([ - {'internal.timer': None}, - {'internal.buildpackages_prep': None}, - ]) + init_tasks.append({'internal.timer': None}) if 'roles' in config: init_tasks.extend([ diff --git a/teuthology/task/internal.py b/teuthology/task/internal.py index 982ec60c0a..8be59843e9 100644 --- a/teuthology/task/internal.py +++ b/teuthology/task/internal.py @@ -348,7 +348,7 @@ def push_inventory(ctx, config): except Exception: log.exception("Error pushing inventory") -BUILDPACKAGES_SWAPPED = 0 +BUILDPACKAGES_FIRST = 0 BUILDPACKAGES_OK = 1 BUILDPACKAGES_REMOVED = 2 BUILDPACKAGES_NOTHING = 3 @@ -362,26 +362,31 @@ def buildpackages_prep(ctx, config): BUILDPACKAGES_NOTHING if there is no buildpackages task BUILDPACKAGES_REMOVED if there is a buildpackages task but no install task - BUILDPACKAGES_SWAPPED if a buildpackages task was moved before an install task - BUILDPACKAGES_OK if a buildpackages task already is before the install task + BUILDPACKAGES_FIRST if a buildpackages task was moved at the beginning + BUILDPACKAGES_OK if a buildpackages task already at the beginning """ index = 0 install_index = None buildpackages_index = None + buildpackages_prep_index = None for task in ctx.config['tasks']: if task.keys()[0] == 'install': install_index = index if task.keys()[0] == 'buildpackages': buildpackages_index = index + if task.keys()[0] == 'internal.buildpackages_prep': + buildpackages_prep_index = index index += 1 - if buildpackages_index is not None and install_index is not None: - if buildpackages_index > install_index: - log.info('buildpackages moved before the install task') + if (buildpackages_index is not None and + install_index is not None): + if buildpackages_index > buildpackages_prep_index + 1: + log.info('buildpackages moved to be the first task') buildpackages = ctx.config['tasks'].pop(buildpackages_index) - ctx.config['tasks'].insert(install_index, buildpackages) - return BUILDPACKAGES_SWAPPED + ctx.config['tasks'].insert(buildpackages_prep_index + 1, + buildpackages) + return BUILDPACKAGES_FIRST else: - log.info('buildpackages is before the install task') + log.info('buildpackages is already the first task') return BUILDPACKAGES_OK elif buildpackages_index is not None and install_index is None: ctx.config['tasks'].pop(buildpackages_index) diff --git a/teuthology/test/task/test_internal.py b/teuthology/test/task/test_internal.py index 2cb0bf12a2..4d125f5036 100644 --- a/teuthology/test/task/test_internal.py +++ b/teuthology/test/task/test_internal.py @@ -13,27 +13,31 @@ class TestInternal(object): # self.ctx.config = { 'tasks': [] } assert internal.buildpackages_prep(self.ctx, - self.ctx.config) == internal.BUILDPACKAGES_NOTHING + self.ctx.config) == internal.BUILDPACKAGES_NOTHING # - # move the buildpackages tasks before the install task + # make the buildpackages tasks the first to run # self.ctx.config = { 'tasks': [ { 'atask': None }, + { 'internal.buildpackages_prep': None }, + { 'btask': None }, { 'install': None }, { 'buildpackages': None } ], } assert internal.buildpackages_prep(self.ctx, - self.ctx.config) == internal.BUILDPACKAGES_SWAPPED + self.ctx.config) == internal.BUILDPACKAGES_FIRST assert self.ctx.config == { 'tasks': [ { 'atask': None }, + { 'internal.buildpackages_prep': None }, { 'buildpackages': None }, + { 'btask': None }, { 'install': None } ], } # - # the buildpackages task already is before the install task + # the buildpackages task already the first task to run # assert internal.buildpackages_prep(self.ctx, - self.ctx.config) == internal.BUILDPACKAGES_OK + self.ctx.config) == internal.BUILDPACKAGES_OK # # no buildpackages task # @@ -41,7 +45,7 @@ class TestInternal(object): 'tasks': [ { 'install': None } ], } assert internal.buildpackages_prep(self.ctx, - self.ctx.config) == internal.BUILDPACKAGES_NOTHING + self.ctx.config) == internal.BUILDPACKAGES_NOTHING # # no install task: the buildpackages task must be removed # @@ -49,5 +53,5 @@ class TestInternal(object): 'tasks': [ { 'buildpackages': None } ], } assert internal.buildpackages_prep(self.ctx, - self.ctx.config) == internal.BUILDPACKAGES_REMOVED + self.ctx.config) == internal.BUILDPACKAGES_REMOVED assert self.ctx.config == {'tasks': []} -- 2.39.5