From: Zack Cerza Date: Thu, 10 Jul 2014 01:02:47 +0000 (-0600) Subject: Use the new JobConfig object X-Git-Tag: 1.1.0~1334^2~8 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=fb97e85aec60f331bf0af37e9b5a2810ca7c97f6;p=teuthology.git Use the new JobConfig object Signed-off-by: Zack Cerza --- diff --git a/teuthology/suite.py b/teuthology/suite.py index db79b17771..ea6b540bd4 100644 --- a/teuthology/suite.py +++ b/teuthology/suite.py @@ -18,7 +18,7 @@ from tempfile import NamedTemporaryFile import teuthology from . import lock -from .config import config +from .config import config, JobConfig from .repo_utils import enforce_repo_state, BranchNotFoundError log = logging.getLogger(__name__) @@ -31,8 +31,7 @@ def main(args): dry_run = args['--dry-run'] base_yaml_paths = args[''] - suite = args['--suite'] - nice_suite = suite.replace('/', ':') + suite = args['--suite'].replace('/', ':') ceph_branch = args['--ceph'] kernel_branch = args['--kernel'] kernel_flavor = args['--flavor'] @@ -51,7 +50,7 @@ def main(args): config.results_email = email timeout = args['--timeout'] - name = make_run_name(nice_suite, ceph_branch, kernel_branch, kernel_flavor, + name = make_run_name(suite, ceph_branch, kernel_branch, kernel_flavor, machine_type) if suite_dir: @@ -59,22 +58,25 @@ def main(args): else: suite_repo_path = fetch_suite_repo(suite_branch, test_name=name) - job_config = create_initial_config(nice_suite, ceph_branch, + job_config = create_initial_config(suite, ceph_branch, teuthology_branch, kernel_branch, kernel_flavor, distro, machine_type) + job_config.name = name + job_config.priority = priority + if email: + job_config.email = email + if owner: + job_config.owner = owner + with NamedTemporaryFile(prefix='schedule_suite_', delete=False) as base_yaml: - base_yaml.write(yaml.safe_dump(job_config)) + base_yaml.write(str(job_config)) base_yaml_path = base_yaml.name base_yaml_paths.insert(0, base_yaml_path) - prepare_and_schedule(owner=owner, - name=name, - suite=suite, - machine_type=machine_type, + prepare_and_schedule(job_config=job_config, suite_repo_path=suite_repo_path, base_yaml_paths=base_yaml_paths, - priority=priority, limit=limit, num=num, timeout=timeout, @@ -143,7 +145,7 @@ def fetch_suite_repo(branch, test_name): return suite_repo_path -def create_initial_config(nice_suite, ceph_branch, teuthology_branch, +def create_initial_config(suite, ceph_branch, teuthology_branch, kernel_branch, kernel_flavor, distro, machine_type): """ Put together the config file used as the basis for each job in the run. @@ -206,7 +208,7 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch, log.info("teuthology branch: %s", teuthology_branch) config_input = dict( - nice_suite=nice_suite, + suite=suite, ceph_branch=ceph_branch, ceph_hash=ceph_hash, teuthology_branch=teuthology_branch, @@ -214,53 +216,52 @@ def create_initial_config(nice_suite, ceph_branch, teuthology_branch, distro=distro, s3_branch=s3_branch, ) - job_config = substitute_placeholders(dict_templ, config_input) - job_config.update(kernel_dict) + conf_dict = substitute_placeholders(dict_templ, config_input) + conf_dict.update(kernel_dict) + job_config = JobConfig.from_dict(conf_dict) return job_config -def prepare_and_schedule(owner, name, suite, machine_type, suite_repo_path, - base_yaml_paths, priority, limit, num, timeout, - dry_run, verbose): +def prepare_and_schedule(job_config, suite_repo_path, base_yaml_paths, limit, + num, timeout, dry_run, verbose): """ Puts together some "base arguments" with which to execute teuthology-schedule for each job, then passes them and other parameters to schedule_suite(). Finally, schedules a "last-in-suite" job that sends an email to the specified address (if one is configured). """ - email = config.results_email - arch = get_arch(machine_type) + arch = get_arch(job_config.machine_type) base_args = [ os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-schedule'), - '--name', name, + '--name', job_config.name, '--num', str(num), - '--worker', get_worker(machine_type), + '--worker', get_worker(job_config.machine_type), ] - if priority: - base_args.extend(['--priority', str(priority)]) + if job_config.priority: + base_args.extend(['--priority', str(job_config.priority)]) if verbose: base_args.append('-v') - if owner: - base_args.extend(['--owner', owner]) + if job_config.owner: + base_args.extend(['--owner', job_config.owner]) - suite_path = os.path.join(suite_repo_path, 'suites', suite) + suite_path = os.path.join(suite_repo_path, 'suites', + job_config.suite.replace(':', '/')) num_jobs = schedule_suite( - name=suite, + job_config=job_config, path=suite_path, base_yamls=base_yaml_paths, base_args=base_args, arch=arch, - machine_type=machine_type, limit=limit, dry_run=dry_run, ) - if email and num_jobs: + if job_config.email and num_jobs: arg = copy.deepcopy(base_args) arg.append('--last-in-suite') - arg.extend(['--email', email]) + arg.extend(['--email', job_config.email]) if timeout: arg.extend(['--timeout', timeout]) if dry_run: @@ -419,12 +420,11 @@ def get_branch_info(project, branch, project_owner='ceph'): return resp.json() -def schedule_suite(name, +def schedule_suite(job_config, path, base_yamls, base_args, arch, - machine_type, limit=0, dry_run=True, ): @@ -432,13 +432,15 @@ def schedule_suite(name, schedule one suite. returns number of jobs scheduled """ + machine_type = job_config.machine_type + suite_name = job_config.suite count = 0 - log.debug('Suite %s in %s' % (name, path)) - configs = [(combine_path(name, item[0]), item[1]) for item in + log.debug('Suite %s in %s' % (suite_name, path)) + configs = [(combine_path(suite_name, item[0]), item[1]) for item in build_matrix(path)] job_count = len(configs) log.info('Suite %s in %s generated %d jobs' % ( - name, path, len(configs))) + suite_name, path, len(configs))) for description, fragment_paths in configs: if limit > 0 and count >= limit: @@ -626,6 +628,7 @@ def substitute_placeholders(input_dict, values_dict): # Template for the config that becomes the base for each generated job config dict_templ = { 'branch': Placeholder('ceph_branch'), + 'teuthology_branch': Placeholder('teuthology_branch'), 'machine_type': Placeholder('machine_type'), 'nuke-on-error': True, 'os_type': {'distro': None}, @@ -677,10 +680,9 @@ dict_templ = { 'sha1': Placeholder('ceph_hash'), } }, - 'suite': Placeholder('nice_suite'), + 'suite': Placeholder('suite'), 'tasks': [ {'chef': None}, {'clock.check': None} ], - 'teuthology_branch': Placeholder('teuthology_branch'), }