From: Zack Cerza Date: Tue, 1 Oct 2013 23:41:46 +0000 (-0500) Subject: A first pass at integration with paddles X-Git-Tag: 1.1.0~1839^2~16 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e3e0775c0dc65a00512ceb2c999ab269865bc165;p=teuthology.git A first pass at integration with paddles Signed-off-by: Zack Cerza --- diff --git a/teuthology/results.py b/teuthology/results.py new file mode 100755 index 000000000..342a07aef --- /dev/null +++ b/teuthology/results.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +import sys +import os +import yaml +import json +import re +import httplib2 +import logging + +from teuthology.config import config + + +log = logging.getLogger(__name__) + + +class RequestFailedError(RuntimeError): + def __init__(self, resp, content): + self.status = resp.status + self.reason = resp.reason + self.content = content + try: + self.content_obj = json.loads(content) + self.message = self.content_obj['message'] + except ValueError: + #self.message = '' + self.message = self.content + + def __str__(self): + templ = "Request failed with status {status}: {reason}: {message}" + + return templ.format( + status=self.status, + reason=self.reason, + message=self.message, + ) + + +class ResultsSerializer(object): + yamls = ('orig.config.yaml', 'config.yaml', 'info.yaml', 'summary.yaml') + + def __init__(self, archive_base): + self.archive_base = archive_base + + def json_for_job(self, run_name, job_id, pretty=False): + job_archive_dir = os.path.join(self.archive_base, + run_name, + job_id) + job_info = {} + for yaml_name in self.yamls: + yaml_path = os.path.join(job_archive_dir, yaml_name) + if not os.path.exists(yaml_path): + continue + with file(yaml_path) as yaml_file: + partial_info = yaml.safe_load(yaml_file) + if partial_info is not None: + job_info.update(partial_info) + + if 'job_id' not in job_info: + job_info['job_id'] = job_id + + if pretty: + job_json = json.dumps(job_info, sort_keys=True, indent=4) + else: + job_json = json.dumps(job_info) + + return job_json + + def print_pretty_json(self, json_obj): + log.info('\n'.join([l.rstrip() for l in json_obj.splitlines()])) + + def jobs_for_run(self, run_name): + archive_dir = os.path.join(self.archive_base, run_name) + if not os.path.isdir(archive_dir): + return {} + jobs = {} + for item in os.listdir(archive_dir): + if not re.match('\d+$', item): + continue + job_id = item + job_dir = os.path.join(archive_dir, job_id) + if os.path.isdir(job_dir): + jobs[job_id] = job_dir + return jobs + + @property + def all_runs(self): + archive_base = self.archive_base + if not os.path.isdir(archive_base): + return [] + runs = [] + for run_name in os.listdir(archive_base): + if not os.path.isdir(os.path.join(archive_base, run_name)): + continue + runs.append(run_name) + return runs + + +class ResultsPoster(object): + def __init__(self, archive_base, base_uri=None): + self.archive_base = archive_base + self.base_uri = base_uri or config.results_server + self.base_uri = self.base_uri.rstrip('/') + self.serializer = ResultsSerializer(archive_base) + + def post_all_runs(self): + all_runs = self.serializer.all_runs + last_run = self.last_run + if last_run and last_run in all_runs: + next_index = all_runs.index(last_run) + 1 + runs = all_runs[next_index:] + else: + runs = all_runs + num_runs = len(runs) + num_jobs = 0 + log.info("Posting %s runs", num_runs) + for run in runs: + job_count = self.post_run(run) + num_jobs += job_count + self.last_run = run + log.info("Total: %s jobs in %s runs", num_jobs, num_runs) + + def post_run(self, run_name): + jobs = self.serializer.jobs_for_run(run_name) + log.info("{name} {jobs} jobs".format( + name=run_name, + jobs=len(jobs), + )) + if jobs: + h = httplib2.Http() + run_json = json.dumps({'name': run_name}) + resp, content = h.request( + "{base}/runs/".format(base=self.base_uri, name=run_name), + 'POST', + run_json, + headers={'content-type': 'application/json'}, + ) + if resp.status == 200: + for job_id in jobs.keys(): + self.post_job(run_name, job_id) + elif resp.status != 200: + message = json.loads(content).get('message', '') + if message.endswith('already exists'): + log.info(" already present; skipped") + else: + raise RequestFailedError(resp, content) + return len(jobs) + + def post_job(self, run_name, job_id): + job_json = self.serializer.json_for_job(run_name, job_id) + h = httplib2.Http() + resp, content = h.request( + "{base}/runs/{name}/".format(base=self.base_uri, name=run_name,), + 'POST', + job_json, + headers={'content-type': 'application/json'}, + ) + try: + message = json.loads(content).get('message', '') + except ValueError: + message = '' + + if message.endswith('already exists'): + resp, content = h.request( + "{base}/runs/{name}/".format( + base=self.base_uri, + name=run_name,), + 'PUT', + job_json, + headers={'content-type': 'application/json'}, + ) + if resp.status != 200: + raise RequestFailedError(resp, content) + return job_id + + @property + def last_run(self): + if hasattr(self, '__last_run'): + return self.__last_run + elif os.path.exists('last_successful_run'): + with file('last_successful_run') as f: + self.__last_run = f.read().strip() + return self.__last_run + + @last_run.setter + def last_run(self, run_name): + self.__last_run = run_name + with file('last_successful_run', 'w') as f: + f.write(run_name) + + +def main(argv): + archive_base = os.path.abspath(os.path.expanduser(argv[1])) + poster = ResultsPoster(archive_base) + poster.post_all_runs() + + +if __name__ == "__main__": + main(sys.argv)