From 16f6c4f4fc18b612307dd43debda9fc0cb6a6dd5 Mon Sep 17 00:00:00 2001 From: Kyr Shatskyy Date: Tue, 7 Apr 2020 23:43:49 +0200 Subject: [PATCH] suite/run: add more interaction for sleep-before-teardown Signed-off-by: Kyr Shatskyy (cherry picked from commit c43303e8892671cec389e68f7643b10624c4b7fc) --- requirements2.txt | 3 ++- requirements3.txt | 3 ++- scripts/openstack.py | 2 +- scripts/suite.py | 11 ++++++++--- setup.py | 1 + teuthology/suite/run.py | 41 ++++++++++++++++++++++++++++++++++++++--- 6 files changed, 52 insertions(+), 9 deletions(-) diff --git a/requirements2.txt b/requirements2.txt index ff3c1dda6..128aaab4d 100644 --- a/requirements2.txt +++ b/requirements2.txt @@ -38,7 +38,9 @@ futures==3.2.0 # via s3transfer gevent==1.4.0 greenlet==0.4.15 # via gevent httplib2==0.10.3 +humanfriendly==8.1 idna==2.5 # via requests +ipy==1.0 ipaddress==1.0.18 # via cryptography iso8601==0.1.11 # via keystoneauth1, oslo.utils, python-neutronclient, python-novaclient jinja2==2.10.1 # via ansible @@ -107,7 +109,6 @@ virtualenv==15.1.0 # via tox warlock==1.2.0 # via python-glanceclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient xmltodict==0.12.0 -ipy==1.0 # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/requirements3.txt b/requirements3.txt index f1fcedd5e..b86dd1624 100644 --- a/requirements3.txt +++ b/requirements3.txt @@ -34,7 +34,9 @@ docutils==0.14 # via botocore gevent==1.4.0 greenlet==0.4.15 # via gevent httplib2==0.10.3 +humanfriendly==8.1 idna==2.5 # via requests +ipy==1.0 iso8601==0.1.11 # via keystoneauth1, oslo.utils, python-neutronclient, python-novaclient jinja2==2.10.1 # via ansible jmespath==0.9.4 # via boto3, botocore @@ -99,7 +101,6 @@ virtualenv==15.1.0 # via tox warlock==1.2.0 # via python-glanceclient wrapt==1.10.10 # via debtcollector, positional, python-glanceclient xmltodict==0.12.0 -ipy==1.0 # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/scripts/openstack.py b/scripts/openstack.py index c95f54bc5..bfdcbfa8e 100644 --- a/scripts/openstack.py +++ b/scripts/openstack.py @@ -189,7 +189,7 @@ def get_suite_parser(): ) parser.add_argument( '--sleep-before-teardown', - help='Number of seconds to sleep before tearing down the target VMs', + help='Number of seconds to sleep before the teardown', default=0 ) return parser diff --git a/scripts/suite.py b/scripts/suite.py index 4a244678a..c8b921f01 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -80,9 +80,14 @@ Standard arguments: Validate that git SHA1s passed to -S exist. [default: true] --sleep-before-teardown - Number of seconds to sleep before tearing down - the test cluster(s). Use with care, as this - applies to all jobs in the run. + Number of seconds to sleep before teardown. + Use with care, as this applies to all jobs in the + run. This option is used along with --limit one. + If the --limit ommitted then it's forced to 1. + If the --limit is greater than 4, then user must + confirm it interactively to avoid massive lock + of resources, however --non-interactive option + can be used to skip user input. [default: 0] Scheduler arguments: diff --git a/setup.py b/setup.py index 3e7b9984e..4d562b3fc 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ setup( 'ansible>=2.0', 'prettytable', 'manhole', + 'humanfriendly', ], extras_require = { 'coverage': [ 'mysqlclient == 1.4.2'], diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index ab7d6f421..7b2173523 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -6,6 +6,9 @@ import re import time import yaml +from six.moves import input +from humanfriendly import format_timespan + from datetime import datetime from tempfile import NamedTemporaryFile @@ -356,14 +359,13 @@ class Run(object): if num_jobs: self.write_result() - def collect_jobs(self, arch, configs, newest=False): + def collect_jobs(self, arch, configs, newest=False, limit=0): jobs_to_schedule = [] jobs_missing_packages = [] for description, fragment_paths in configs: base_frag_paths = [ util.strip_fragment_path(x) for x in fragment_paths ] - limit = self.args.limit or 0 if limit > 0 and len(jobs_to_schedule) >= limit: log.info( 'Stopped after {limit} jobs due to --limit={limit}'.format( @@ -518,13 +520,46 @@ class Run(object): ).name self.base_yaml_paths.insert(0, base_yaml_path) + # compute job limit in respect of --sleep-before-teardown + job_limit = self.args.limit or 0 + sleep_before_teardown = int(self.args.sleep_before_teardown or 0) + if sleep_before_teardown: + if job_limit == 0: + log.warning('The --sleep-before-teardown option was provided: ' + 'only 1 job will be scheduled. ' + 'Use --limit to run more jobs') + # give user a moment to read this warning + time.sleep(5) + job_limit = 1 + elif self.args.non_interactive: + log.warning( + 'The --sleep-before-teardown option is active. ' + 'There will be a maximum {} jobs running ' + 'which will fall asleep for {}' + .format(job_limit, format_timespan(sleep_before_teardown))) + elif job_limit > 4: + are_you_insane=( + 'There are {total} configs and {maximum} job limit is used. ' + 'Do you really want to lock all machines needed for ' + 'this run for {that_long}? (y/N):' + .format( + that_long=format_timespan(sleep_before_teardown), + total=len(configs), + maximum=job_limit)) + while True: + insane=(input(are_you_insane) or 'n').lower() + if insane == 'y': + break + elif insane == 'n': + exit(0) + # if newest, do this until there are no missing packages # if not, do it once backtrack = 0 limit = self.args.newest while backtrack <= limit: jobs_missing_packages, jobs_to_schedule = \ - self.collect_jobs(arch, configs, self.args.newest) + self.collect_jobs(arch, configs, self.args.newest, job_limit) if jobs_missing_packages and self.args.newest: new_sha1 = \ util.find_git_parent('ceph', self.base_config.sha1) -- 2.47.3