From ebb4b49d4baa9fd8318c0da1bda1a0b3d97fa8ea Mon Sep 17 00:00:00 2001 From: Shylesh Kumar Date: Thu, 14 Jun 2018 21:55:37 +0530 Subject: [PATCH] exec failure fix Signed-off-by: Shylesh Kumar --- scripts/gencov.py | 16 +++++ setup.py | 1 + teuthology/coverage_report.py | 50 ++++++++++++++++ teuthology/gencov.py | 77 ++++++++++++++++++++++++ teuthology/misc.py | 4 +- teuthology/orchestra/remote.py | 4 +- teuthology/results.py | 88 ++++++++++++++++++++++------ teuthology/run.py | 2 +- teuthology/task/exec.py | 64 ++++++++++---------- teuthology/task/internal/__init__.py | 4 +- 10 files changed, 255 insertions(+), 55 deletions(-) create mode 100644 scripts/gencov.py create mode 100644 teuthology/coverage_report.py create mode 100644 teuthology/gencov.py diff --git a/scripts/gencov.py b/scripts/gencov.py new file mode 100644 index 0000000000..95d3096bdd --- /dev/null +++ b/scripts/gencov.py @@ -0,0 +1,16 @@ +import docopt + +import teuthology.coverage_report +import teuthology.gencov +import sys + +doc = """ +usage: teuthology-gencov -h + teuthology-gencov + + """ + +def main(argv=sys.argv[1:]): + args = docopt.docopt(doc, argv=argv) + teuthology.gencov.main(args) + diff --git a/setup.py b/setup.py index c6736844dc..2553028b5d 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,7 @@ setup( 'teuthology-queue = scripts.queue:main', 'teuthology-prune-logs = scripts.prune_logs:main', 'teuthology-describe-tests = scripts.describe_tests:main', + 'teuthology-gencov = scripts.gencov:main', ], }, diff --git a/teuthology/coverage_report.py b/teuthology/coverage_report.py new file mode 100644 index 0000000000..89e61a930a --- /dev/null +++ b/teuthology/coverage_report.py @@ -0,0 +1,50 @@ +import os + +N_ENTRIES=10 +genpath="/a/code_coverage_logs/" + + +def gen_html(genpath, N_ENTRIES): + if not os.path.exists(genpath): + print 'path doesnt exists' + files = sorted([genpath+ent for ent in os.listdir(genpath)],key=os.path.getctime, reverse=True) + for ent in files: + pass + #print os.path.basename(ent) + body="" + for ent in files: + if not os.path.isdir(ent): + pass + else: + body=body+\ + "\n"+""+\ + "".format(os.path.basename(ent)+"/index.html")+os.path.basename(ent)+""+\ + ""+\ + "".format(os.path.basename(ent))+"Run"+""+\ + "\n" + + page=" \n \ + Code Coverage\n \ + \n \ +

Downstream: CEPH Code Coverage

\n\ + \n \ + \n\ + \n\ + \n\ + \n\ + \n\ + \n\ + \n\ + {} \ + \n \ + \n ".format(body) + fd=open(genpath+"index.html", "w+") + fd.writelines(page) + fd.close() + + + +if __name__ =="__main__": + gen_html(genpath, N_ENTRIES) diff --git a/teuthology/gencov.py b/teuthology/gencov.py new file mode 100644 index 0000000000..09e81a3a58 --- /dev/null +++ b/teuthology/gencov.py @@ -0,0 +1,77 @@ +import os +import sys +import logging +import subprocess +import shlex + +import coverage_report + +log=logging.getLogger() +hdlr=logging.FileHandler('/a/code_coverage_logs/coverage.log') +formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') +hdlr.setFormatter(formatter) +log.addHandler(hdlr) +log.setLevel(logging.INFO) + +def generate_coverage_compile(cover_dir): + log.info("Inside generate_coverage_compile") + merged_total=cover_dir+"/total.info" + filtered_total=cover_dir+"/filtered_total.info" +# cmd="lcov --rc lcov_branch_coverage=1 " + cmd="lcov " + ilist=os.listdir(cover_dir) + for ent in ilist: + if '*.info' in ent: + subprocess.Popen( + args=[ + 'sed', '-i','s/\/sourcebuild/\/tmp\/build\/{}/'.format(\ + os.path.basename(cover_dir)), \ + cover_dir+"/"+ent] + ) + + for ent in ilist: + if 'info' in ent: + addstr= " -a " + cover_dir+"/"+ent + tstr=" -t "+ ent.split("_")[0] + cmd = cmd + addstr+tstr + log.info(cmd) + cmd=cmd + " -o "+ merged_total + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + assert(os.path.exists(merged_total)) + + proc=subprocess.Popen( + args=[ + 'sed', '-i','s/\/sourcebuild/\/tmp\/build\/{}/'.format(\ + os.path.basename(cover_dir)), \ + merged_total] + ) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + + cmd="lcov --remove "+merged_total+" '/usr/include/*' '/usr/lib/*' " +\ + " -o "+ filtered_total + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + assert(os.path.exists(filtered_total)) + + cmd="genhtml " + " -o "+cover_dir+" {}".format(filtered_total) + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + assert(os.path.exists(cover_dir+"/index.html")) + + + +def main(args): + log = logging.getLogger(__name__) + basedir="/a/code_coverage_logs/" + coverdir=basedir+args[''] + generate_coverage_compile(coverdir) + gen_path="/a/code_coverage_logs/" + coverage_report.gen_html(gen_path, 10) diff --git a/teuthology/misc.py b/teuthology/misc.py index 0a7226a1b9..53d453f1f6 100644 --- a/teuthology/misc.py +++ b/teuthology/misc.py @@ -789,7 +789,7 @@ def pull_directory(remote, remotedir, localdir): if not os.path.exists(localdir): os.mkdir(localdir) r = remote.get_tar_stream(remotedir, sudo=True) - tar = tarfile.open(mode='r|gz', fileobj=r.stdout) + tar = tarfile.open(mode='r|gz', fileobj=r.stdout, dereference=True) while True: ti = tar.next() if ti is None: @@ -799,7 +799,7 @@ def pull_directory(remote, remotedir, localdir): # ignore silently; easier to just create leading dirs below # XXX this mean empty dirs are not transferred pass - elif ti.isfile(): + elif ti.isfile() or ti.issym(): sub = safepath.munge(ti.name) safepath.makedirs(root=localdir, path=os.path.dirname(sub)) tar.makefile(ti, targetpath=os.path.join(localdir, sub)) diff --git a/teuthology/orchestra/remote.py b/teuthology/orchestra/remote.py index 0dde83f974..785adf8021 100644 --- a/teuthology/orchestra/remote.py +++ b/teuthology/orchestra/remote.py @@ -399,7 +399,7 @@ class Remote(object): args.append('sudo') args.extend([ 'tar', - 'cz', + 'czh', '-f', remote_temp_path, '-C', path, '--', @@ -421,7 +421,7 @@ class Remote(object): args.append('sudo') args.extend([ 'tar', - 'cz', + 'czh', '-f', '-', '-C', path, '--', diff --git a/teuthology/results.py b/teuthology/results.py index c5baa91b27..9a7ee7e713 100644 --- a/teuthology/results.py +++ b/teuthology/results.py @@ -3,6 +3,7 @@ import sys import time import logging import subprocess +import shlex from collections import OrderedDict from textwrap import dedent from textwrap import fill @@ -14,12 +15,14 @@ from .report import ResultsReporter log = logging.getLogger(__name__) + UNFINISHED_STATUSES = ('queued', 'running', 'waiting') def main(args): log = logging.getLogger(__name__) + if args['--verbose']: teuthology.log.setLevel(logging.DEBUG) @@ -46,6 +49,7 @@ def note_rerun_params(subset, seed): def results(archive_dir, name, email, timeout, dry_run): + log.info('Inside results.....') starttime = time.time() if timeout: @@ -81,31 +85,81 @@ def results(archive_dir, name, email, timeout, dry_run): body=body, ) finally: + log.info("Inside FINALLY") generate_coverage(archive_dir, name) +def generate_coverage_compile(cover_dir): + log.info("Inside generate_coverage_compile") + merged_total=cover_dir+"/total.info" + filtered_total=cover_dir+"/filtered_total.info" + cmd="lcov --rc lcov_branch_coverage=1 " + ilist=os.listdir(cover_dir) + for ent in ilist: + subprocess.Popen( + args=[ + 'sed', '-i','s/\/sourcebuild/\/tmp\/build\/{}/'.format(\ + os.path.basename(cover_dir)), \ + cover_dir+"/"+ent] + ) + + for ent in ilist: + addstr= " -a " + cover_dir+"/"+ent + tstr=" -t "+ ent.split("_")[0] + cmd = cmd + addstr+tstr + cmd=cmd + " -o "+ merged_total + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + assert(os.path.exists(merged_total)) + + proc=subprocess.Popen( + args=[ + 'sed', '-i','s/\/sourcebuild/\/tmp\/build\/{}/'.format(\ + os.path.basename(cover_dir)), \ + merged_total] + ) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + + cmd="lcov --remove "+merged_total+" '/usr/include/*' '/usr/lib/*' " +\ + " -o "+ filtered_total + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0] + log.info(proc_stdout) + assert(os.path.exists(filtered_total)) + + cmd="genhtml " + " -o "+cover_dir+" {}".format(filtered_total) + log.info(cmd) + proc=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + proc_stdout=proc.communicate()[0].strip() + log.info(proc_stdout) + assert(os.path.exists(cover_dir+"/index.html")) + def generate_coverage(archive_dir, name): coverage_config_keys = ('coverage_output_dir', 'coverage_html_dir', - 'coverage_tools_dir') + 'coverage_tools_dir') for key in coverage_config_keys: - if key not in config.to_dict(): - log.warn( - "'%s' not in teuthology config; skipping coverage report", - key) - return + if key not in config.to_dict(): + log.warn( + "'%s' not in teuthology config; skipping coverage report", + key) + return log.info('starting coverage generation') subprocess.Popen( - args=[ - os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'), - '-v', - '-o', - os.path.join(config.coverage_output_dir, name), - '--html-output', - os.path.join(config.coverage_html_dir, name), - '--cov-tools-dir', - config.coverage_tools_dir, - archive_dir, - ], + args=[ + os.path.join(os.path.dirname(sys.argv[0]), 'teuthology-coverage'), + '-v', + '-o', + os.path.join(config.coverage_output_dir, name), + '--html-output', + os.path.join(config.coverage_html_dir, name), + '--cov-tools-dir', + config.coverage_tools_dir, + archive_dir, + ], ) diff --git a/teuthology/run.py b/teuthology/run.py index 236cc0eb3a..2e65fbc4b4 100644 --- a/teuthology/run.py +++ b/teuthology/run.py @@ -362,7 +362,7 @@ def main(args): if suite_repo: teuth_config.ceph_qa_suite_git_url = suite_repo - # overwrite the config values of os_{type,version} if corresponding + # overwrite the config values of os_{type,version} if corresponding # command-line arguments are provided if os_type: config["os_type"] = os_type diff --git a/teuthology/task/exec.py b/teuthology/task/exec.py index 8d1540a6ad..d57df2ef8d 100644 --- a/teuthology/task/exec.py +++ b/teuthology/task/exec.py @@ -50,34 +50,36 @@ def task(ctx, config): config = dict((id_, a) for id_ in roles) for role, ls in config.iteritems(): - (remote,) = ctx.cluster.only(role).remotes.iterkeys() - log.info('Running commands on role %s host %s', role, remote.name) - for c in ls: - c.replace('$TESTDIR', testdir) - if retry: - with safe_while(sleep=sleep_for_retry, tries=retry, - action="exec_with_retry") as proceed: - while proceed(): - proc = remote.run( - args=[ - 'sudo', - 'TESTDIR={tdir}'.format(tdir=testdir), - 'bash', - '-c', - c], - timeout=timeout, - check_status=False, - wait=True, - ) - if proc.exitstatus == 0: - break - else: - remote.run( - args=[ - 'sudo', - 'TESTDIR={tdir}'.format(tdir=testdir), - 'bash', - '-c', - c], - timeout=timeout - ) + if 'mon' in role or 'osd' in role \ + or 'client' in role: + (remote,) = ctx.cluster.only(role).remotes.iterkeys() + log.info('Running commands on role %s host %s', role, remote.name) + for c in ls: + c.replace('$TESTDIR', testdir) + if retry: + with safe_while(sleep=sleep_for_retry, tries=retry, + action="exec_with_retry") as proceed: + while proceed(): + proc = remote.run( + args=[ + 'sudo', + 'TESTDIR={tdir}'.format(tdir=testdir), + 'bash', + '-c', + c], + timeout=timeout, + check_status=False, + wait=True, + ) + if proc.exitstatus == 0: + break + else: + remote.run( + args=[ + 'sudo', + 'TESTDIR={tdir}'.format(tdir=testdir), + 'bash', + '-c', + c], + timeout=timeout + ) diff --git a/teuthology/task/internal/__init__.py b/teuthology/task/internal/__init__.py index fd5e7f3f85..74582a6258 100644 --- a/teuthology/task/internal/__init__.py +++ b/teuthology/task/internal/__init__.py @@ -364,10 +364,10 @@ def archive(ctx, config): # Check for coredumps and pull binaries fetch_binaries_for_coredumps(path, rem) - log.info('Removing archive directory...') + log.info('Removing archive directory as sudo...') run.wait( ctx.cluster.run( - args=['rm', '-rf', '--', archive_dir], + args=['sudo', 'rm', '-rf', '--', archive_dir], wait=False, ), ) -- 2.39.5

\n\ + CEPH COVERAGE RUNS \n\ +
Coverage results	Teuthology results

Downstream: CEPH Code Coverage

CEPH COVERAGE RUNS