From db118a749015af9fbf1c1b569710c6f29acd218c Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 14 Jun 2016 21:41:16 -0700 Subject: [PATCH] teuthology-suite: Implement --newest If --newest is supplied, missing packages for one of the required distro/versions are not fatal; instead, use a web service to backtrack to older sha1s in an attempt to find the newest commonly-built version. Limit backtracking to '--newest' commits (default 10). See http://github.com/ceph/gitserver for the web service source. Signed-off-by: Dan Mick --- docs/siteconfig.rst | 4 ++ scripts/suite.py | 5 +++ teuthology/config.py | 1 + teuthology/suite/__init__.py | 2 +- teuthology/suite/run.py | 81 ++++++++++++++++++++++-------------- teuthology/suite/util.py | 34 +++++++++++++++ 6 files changed, 94 insertions(+), 33 deletions(-) diff --git a/docs/siteconfig.rst b/docs/siteconfig.rst index 2d41fda468..5706150b10 100644 --- a/docs/siteconfig.rst +++ b/docs/siteconfig.rst @@ -51,6 +51,10 @@ Here is a sample configuration with many of the options set and documented:: # Gitbuilder archive that stores e.g. ceph packages gitbuilder_host: gitbuilder.example.com + # URL for 'gitserver' helper web application + # see http://github.com/ceph/gitserver + githelper_base_url: http://git.ceph.com:8080 + # Verify the packages signatures check_package_signatures: true diff --git a/scripts/suite.py b/scripts/suite.py index 1faba07e43..b37433d1d7 100644 --- a/scripts/suite.py +++ b/scripts/suite.py @@ -35,6 +35,11 @@ Standard arguments: If both -S and -c are supplied, -S wins, and there is no validation that sha1 is contained in branch + -n , --newest + Search for the newest revision built on all + required distro/versions, starting from + either --ceph or --sha1, backtracking + up to commits [default: 10] -k , --kernel The kernel branch to run against; if not supplied, the installed kernel is unchanged diff --git a/teuthology/config.py b/teuthology/config.py index 0b16f5b2f1..acd450d945 100644 --- a/teuthology/config.py +++ b/teuthology/config.py @@ -138,6 +138,7 @@ class TeuthologyConfig(YamlConfig): 'ceph_git_url': None, 'ceph_qa_suite_git_url': None, 'gitbuilder_host': 'gitbuilder.ceph.com', + 'githelper_base_url': 'http://git.ceph.com:8080', 'check_package_signatures': True, 'lab_domain': 'front.sepia.ceph.com', 'lock_server': 'http://paddles.front.sepia.ceph.com/', diff --git a/teuthology/suite/__init__.py b/teuthology/suite/__init__.py index d350e83ab3..7a7f308f51 100644 --- a/teuthology/suite/__init__.py +++ b/teuthology/suite/__init__.py @@ -36,7 +36,7 @@ def process_args(args): key = rename_args.get(key) or key if key == 'suite': value = value.replace('/', ':') - elif key in ('limit', 'priority', 'num'): + elif key in ('limit', 'priority', 'num', 'newest'): value = int(value) elif key == 'subset' and value is not None: # take input string '2/3' and turn into (2, 3) diff --git a/teuthology/suite/run.py b/teuthology/suite/run.py index 38c1c19418..23175b9deb 100644 --- a/teuthology/suite/run.py +++ b/teuthology/suite/run.py @@ -8,7 +8,9 @@ import yaml from datetime import datetime from ..config import config, JobConfig -from ..exceptions import (BranchNotFoundError, CommitNotFoundError,) +from ..exceptions import ( + BranchNotFoundError, CommitNotFoundError, VersionNotFoundError +) from ..misc import deep_merge, get_results_url from . import util @@ -23,7 +25,7 @@ class Run(object): WAIT_PAUSE = 5 * 60 __slots__ = ( 'args', 'name', 'base_config', 'suite_repo_path', 'base_yaml_paths', - 'base_args', + 'base_args', 'package_versions', ) def __init__(self, args): @@ -33,6 +35,8 @@ class Run(object): self.args = args self.name = self.make_run_name() self.base_config = self.create_initial_config() + # caches package versions to minimize requests to gbs + self.package_versions = dict() if self.args.suite_dir: self.suite_repo_path = self.args.suite_dir @@ -148,7 +152,8 @@ class Run(object): return ceph_hash def choose_ceph_version(self, ceph_hash): - if config.suite_verify_ceph_hash: + if config.suite_verify_ceph_hash and not self.args.newest: + # don't bother if newest; we'll search for an older one # Get the ceph package version ceph_version = util.package_version_for_hash( ceph_hash, self.args.kernel_flavor, self.args.distro, @@ -255,27 +260,8 @@ class Run(object): if results_url: log.info("Test results viewable at %s", results_url) - def schedule_suite(self): - """ - Schedule the suite run. Returns the number of jobs scheduled. - """ - name = self.name - arch = util.get_arch(self.base_config.machine_type) - suite_name = self.base_config.suite - suite_path = os.path.join( - self.suite_repo_path, 'suites', - self.base_config.suite.replace(':', '/')) - log.debug('Suite %s in %s' % (suite_name, suite_path)) - configs = [ - (combine_path(suite_name, item[0]), item[1]) for item in - build_matrix(suite_path, subset=self.args.subset) - ] - log.info('Suite %s in %s generated %d jobs (not yet filtered)' % ( - suite_name, suite_path, len(configs))) - def collect_jobs(self, arch, configs): - # used as a local cache for package versions from gitbuilder - package_versions = dict() + def collect_jobs(self, arch, configs, newest=False): jobs_to_schedule = [] jobs_missing_packages = [] for description, fragment_paths in configs: @@ -346,28 +332,35 @@ class Run(object): args=arg ) + sha1 = self.base_config.sha1 if config.suite_verify_ceph_hash: full_job_config = dict() deep_merge(full_job_config, self.base_config.to_dict()) deep_merge(full_job_config, parsed_yaml) flavor = util.get_install_task_flavor(full_job_config) - sha1 = self.base_config.sha1 # Get package versions for this sha1, os_type and flavor. If # we've already retrieved them in a previous loop, they'll be # present in package_versions and gitbuilder will not be asked # again for them. - package_versions = util.get_package_versions( - sha1, - os_type, - flavor, - package_versions - ) + try: + self.package_versions = util.get_package_versions( + sha1, + os_type, + flavor, + self.package_versions + ) + except VersionNotFoundError: + pass if not util.has_packages_for_distro(sha1, os_type, flavor, - package_versions): + self.package_versions): m = "Packages for os_type '{os}', flavor {flavor} and " + \ "ceph hash '{ver}' not found" log.error(m.format(os=os_type, flavor=flavor, ver=sha1)) jobs_missing_packages.append(job) + # optimization: one missing package causes backtrack in newest mode; + # no point in continuing the search + if newest: + return jobs_missing_packages, None jobs_to_schedule.append(job) return jobs_missing_packages, jobs_to_schedule @@ -419,7 +412,31 @@ class Run(object): log.info('Suite %s in %s generated %d jobs (not yet filtered)' % ( suite_name, suite_path, len(configs))) - jobs_missing_packages, jobs_to_schedule = self.collect_jobs(arch, configs) + # if newest, do this until there are no missing packages + # if not, do it once + backtrack = 0 + limit = self.args.newest + while backtrack < limit: + jobs_missing_packages, jobs_to_schedule = \ + self.collect_jobs(arch, configs, self.args.newest) + if jobs_missing_packages and self.args.newest: + self.base_config.sha1 = \ + util.find_git_parent('ceph', self.base_config.sha1) + if self.base_config.sha1 is None: + util.schedule_fail( + name, message='Backtrack for --newest failed' + ) + backtrack += 1 + continue + if backtrack: + log.info("--newest supplied, backtracked %d commits to %s" % + (backtrack, self.base_config.sha1)) + break + else: + util.schedule_fail( + name, + message='Exceeded %d backtracks; raise --newest value' % limit + ) self.schedule_jobs(jobs_missing_packages, jobs_to_schedule, name) diff --git a/teuthology/suite/util.py b/teuthology/suite/util.py index ef771671f1..145281f9eb 100644 --- a/teuthology/suite/util.py +++ b/teuthology/suite/util.py @@ -453,3 +453,37 @@ def teuthology_schedule(args, verbose, dry_run, log_prefix=''): )) if not dry_run or (dry_run and verbose > 1): subprocess.check_call(args=args) + + +def find_git_parent(project, sha1): + + base_url = config.githelper_base_url + if not base_url: + log.warning('githelper_base_url not set, --newest disabled') + return None + + def refresh(project): + url = '%s/%s.git/refresh' % (base_url, project) + resp = requests.get(url) + if not resp.ok: + log.error('git refresh failed for %s: %s', project, resp.content) + + def get_sha1s(project, commitish, count): + url = '/'.join((base_url, '%s.git' % project, + 'history/?commitish=%s&count=%d' % (commitish, count))) + resp = requests.get(url) + resp.raise_for_status() + sha1s = resp.json()['sha1s'] + if len(sha1s) != count: + log.error('can''t find %d parents of %s in %s: %s', + int(count), sha1, project, resp.json()['error']) + return sha1s + + # XXX don't do this every time?.. + refresh(project) + # we want the one just before sha1; list two, return the second + sha1s = get_sha1s(project, sha1, 2) + if len(sha1s) == 2: + return sha1s[1] + else: + return None -- 2.39.5