--- /dev/null
+'''
+Compiling ceph from source on each node
+for code coverage runs
+'''
+import contextlib
+import logging
+import os
+import re
+import shlex
+import subprocess
+
+from subprocess import Popen
+from cStringIO import StringIO
+from teuthology import misc as teuthology
+from teuthology.parallel import parallel
+from teuthology.orchestra import run, remote
+from teuthology.config import config as teuth_config
+
+log = logging.getLogger(__name__)
+COVERAGE_LOGS="/a/code_coverage_logs/"
+run_dir=""
+
+def aggregate_info(ctx, tdir, curator, ilist, run_dir):
+ '''
+ Aggregate the results from all the nodes to generate
+ a single .info file
+ '''
+
+ coverdir="/tmp/coverage"
+ JID_PREFIX=os.path.basename(ctx.archive)
+ cmd = "mkdir -p {}".format(coverdir)
+ r=curator.run(args=cmd)
+ dst=""
+
+ mergefile=coverdir+"/coverage_merged.info"
+
+ cmd="lcov --rc lcov_branch_coverage=1 "
+ for ent in ilist:
+ addstr=" -a "+tdir+ent
+ tstr=" -t "+ent.split(".")[0]
+ cmd = cmd + addstr+tstr
+ cmd = cmd +" -o "+mergefile
+ log.info(cmd)
+
+ r=curator.run(args=cmd)
+
+ '''
+ rename file with JID_PREFIX
+ '''
+ new_mergefile=coverdir+"/"+JID_PREFIX+"_coverage_merged.info"
+ cmd = "mv "+mergefile+" "+new_mergefile
+ log.info("renaming {src} to {dst}".format(src=mergefile,\
+ dst=new_mergefile))
+ log.info(cmd)
+ r=curator.run(args=cmd)
+
+ '''
+ transfer file to teuthology master node
+ '''
+ assert(os.path.exists(run_dir))
+ dst=curator.get_file(new_mergefile, False, run_dir)
+ assert(os.path.exists(dst))
+ log.info("xfrd file {} to {}".format(new_mergefile, dst))
+
+
+ '''
+ cmd="genhtml -o {archive_dir} {coverfile}".format( \
+ archive_dir=coverdir, coverfile=mergefile)
+ log.info(cmd)
+ r=curator.run(args=cmd)
+ teuthology.pull_directory(curator, coverdir, ctx.archive)
+ '''
+
+def _gather_data(ctx, remote, rpath, curator):
+ tdir="/tmp/cov/"+ctx.archive
+ lpath=""
+ if remote != curator:
+ if not os.path.exists(tdir):
+ os.makedirs(tdir)
+ lpath=remote.get_file(rpath, False, tdir)
+ assert(os.path.exists(lpath))
+ log.info("Transfered file {} to {} ".format(rpath, lpath))
+ #pass the file to curator
+ # direct copying from remote->remote not working
+ fn=os.path.basename(rpath)
+ src=tdir+"/{fn}".format(fn=fn)
+
+ with file(src, 'rb') as f:
+ teuthology.sudo_write_file(
+ remote=curator,
+ path=rpath,
+ data=f,
+ )
+ remote.run(
+ args=[
+ 'sudo',
+ 'chmod',
+ 'a=rx',
+ '--',
+ rpath,
+ ]
+ )
+
+
+
+def gather_data(ctx, config, remote, srcdir, curator):
+ this_infofile="{hostname}.info".format(hostname=remote.shortname)
+ ''' Generate .info file on respective nodes and copy it to
+ local node
+ '''
+ cmd="sudo bash -c '(cd /sourcebuild/ceph && \
+ exec lcov --rc lcov_branch_coverage=1 \
+ --capture -d . --output-file {nodename})'"\
+ .format(nodename=this_infofile)
+ log.info(cmd)
+ r=remote.run(args=cmd)
+
+ _gather_data(ctx, remote, srcdir+this_infofile, curator)
+
+@contextlib.contextmanager
+def task(ctx, config):
+ """
+ generate reports on respective nodes but fetch all of them to local
+ so that we can merge the reports
+ """
+ global run_dir
+ srcdir="/sourcebuild/ceph/"
+ dstdir="{ctxdir}/".format(ctxdir=ctx.archive)
+ run_dir=COVERAGE_LOGS+os.path.basename(os.path.dirname(ctx.archive))+"/"
+
+
+ if os.path.exists(dstdir):
+ print 'coverage directory present'
+ else:
+ os.makedirs(dstdir)
+ curator=ctx.cluster.remotes.keys()[0]
+
+ with parallel() as ptask:
+ for remote, roles in ctx.cluster.remotes.iteritems():
+ ptask.spawn(gather_data, ctx, config, remote, srcdir, curator)
+ ilist=[]
+ for remote, roles in ctx.cluster.remotes.iteritems():
+ node=remote.shortname+".info"
+ ilist.append(node)
+
+ aggregate_info(ctx, srcdir, curator, ilist, run_dir)
+ yield
+
+ log.info("DONE WITH COLLECTION")
+
+
--- /dev/null
+'''
+Compiling ceph from source on each node
+for code coverage runs
+'''
+import contextlib
+import logging
+import os
+import re
+import shlex
+import subprocess
+
+from cStringIO import StringIO
+from teuthology import misc as teuthology
+from teuthology.parallel import parallel
+from teuthology.orchestra import run
+from teuthology.config import config as teuth_config
+
+log = logging.getLogger(__name__)
+COVERAGE_LOGS = "/a/code_coverage_logs/"
+
+
+def get_sourcedir(ctx, config, remote, rdir):
+ '''
+ get source code directory along with gcno files
+ on the machine where we want to generate report
+ '''
+ ldir = "/tmp/build/{}/".format(os.path.basename(os.path.dirname(ctx.archive)))
+ if not os.path.exists(ldir):
+ os.makedirs(ldir)
+#o teuthology.pull_directory(remote, rdir, ldir)
+ cmd="scp -r {rhost}:{rpath} {lpath}".format(\
+ rhost=remote.name, rpath=rdir, \
+ lpath=ldir)
+ log.info(cmd)
+
+ p=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
+ p_stdout=p.communicate()[0]
+ log.info(p_stdout)
+
+
+def copy_compile_utils(ctx, config, remote):
+ FILES=['update-cmakelists', 'update-docmake']
+ destdir = '/usr/bin'
+ for filename in FILES:
+ log.info('Shipping %r...', filename)
+ src = os.path.join(os.path.dirname(__file__), "util/"+filename)
+ dst = os.path.join(destdir, filename)
+ with file(src, 'rb') as f:
+ teuthology.sudo_write_file(
+ remote=remote,
+ path=dst,
+ data=f,
+ )
+ remote.run(
+ args=[
+ 'sudo',
+ 'chmod',
+ 'a=rx',
+ '--',
+ dst,
+ ]
+ )
+
+
+def copy_utils(ctx, config, remote):
+ FILES = ['daemon-helper', 'adjust-ulimits']
+ destdir = '/usr/bin'
+ for filename in FILES:
+ log.info('Shipping %r...', filename)
+ src = os.path.join(os.path.dirname(teuthology.__file__), "task/install/"+filename)
+ dst = os.path.join(destdir, filename)
+ with file(src, 'rb') as f:
+ teuthology.sudo_write_file(
+ remote=remote,
+ path=dst,
+ data=f,
+ )
+ f.seek(0)
+ remote.run(
+ args=[
+ 'sudo',
+ 'chmod',
+ 'a=rx',
+ '--',
+ dst,
+ ]
+ )
+
+
+def get_dependencies(ctx, config, remote):
+ #1. enable repos
+ #2. get dependencies
+ cmd='sudo subscription-manager --force register \
+ --serverurl=subscription.rhsm.stage.redhat.com:443/subscription \
+ --baseurl=https://cdn.stage.redhat.com --username=qa@redhat.com \
+ --password=redhatqa --auto-attach'
+ r=remote.run(args=cmd)
+ cmd='sudo subscription-manager repos --enable=rhel-7-server-extras-rpms'
+ r=remote.run(args=cmd)
+ cmd='sudo subscription-manager repos --enable=rhel-7-server-optional-rpms'
+ r=remote.run(args=cmd)
+ cmd='sudo sed -i -- \'s/enabled=0/enabled=1/g\' /etc/yum.repos.d/epel.repo'
+ r=remote.run(args=cmd)
+ cmd='sudo yum repolist'
+ r=remote.run(args=cmd)
+
+ cmd = "sudo bash -c '(cd /sourcebuild/ceph && exec git checkout luminous)'"
+ r=remote.run(args=cmd)
+ cmd="sudo bash -c '(cd /sourcebuild/ceph && exec git submodule update --init --recursive)'"
+ r=remote.run(args=cmd)
+ cmd='sudo sed -i -- \'s/yum install subscription-manager/yum install \
+ -y subscription-manager/g\' /sourcebuild/ceph/install-deps.sh'
+ r=remote.run(args=cmd)
+ cmd="sudo bash -c '(cd /sourcebuild/ceph && exec ./install-deps.sh)'"
+ r=remote.run(args=cmd)
+ cmd="sudo bash -c '(sudo yum -y install lcov)'"
+ r=remote.run(args=cmd)
+
+ copy_compile_utils(ctx, config, remote)
+
+ #edit CMakefiles.txt and do_cmake.sh according to gcov need
+ cmd="sudo update-cmakelists " + "/sourcebuild/ceph/CMakeLists.txt"
+ r=remote.run(args=cmd)
+ #1. set INSTALL_PREFIX=/usr in do_cmake
+ cmd="sudo update-docmake " + "/sourcebuild/ceph/do_cmake.sh"
+ r=remote.run(args=cmd)
+
+ #cmd="sudo bash -c '(cd /sourcebuild/ceph && source scl_source enable devtoolset-7 && exec ./do_cmake.sh)'"
+ cmd="sudo bash -c '(cd /sourcebuild/ceph && exec ./do_cmake.sh)'"
+ r=remote.run(args=cmd)
+ cmd="sudo bash -c '(cd /sourcebuild/ceph/build && exec make -j4)'"
+ r=remote.run(args=cmd)
+ cmd="sudo bash -c '(cd /sourcebuild/ceph/build && exec make install)'"
+ r=remote.run(args=cmd)
+ #cmd="sudo bash -c '(cp -ar /usr/local/lib64/python2.7/site-packages/* /usr/lib64/python2.7/site-packages/)'"
+ #r=remote.run(args=cmd)
+ cmd="sudo mkdir /etc/ceph"
+ r=remote.run(args=cmd)
+ cmd="sudo mkdir /var/log/ceph"
+ r=remote.run(args=cmd)
+ cmd="sudo mkdir /var/lib/ceph"
+ r=remote.run(args=cmd)
+
+ ''' copy adjust-ulimits, daemon-helper scripts '''
+ copy_utils(ctx, config, remote)
+
+ #update permissions for testdir so that
+ # teuthology can delete archive/coverage/gcda files
+ test_dir=teuthology.get_testdir(ctx)
+ cmd="sudo chmod -R 0777 {tdir}".format(tdir=test_dir)
+ r=remote.run(args=cmd)
+
+ #before CEPH task
+ #- ceph:
+ #conf:
+ # osd:
+ # osd max object name len : 400
+ # osd max object namespace len : 64
+ #TODO
+ #gather gcda files
+ #aggreagate
+ #publish report
+
+
+def start_compile(ctx, config, remote, builddir):
+ #1. edit cmakefile for gcov entry
+ #2.compile
+ pass
+
+
+def compile_with_gcov(ctx, config, grepo, remote, builddir):
+ ''' please use variables once task working'''
+ try:
+ r = remote.run(args=['sudo', 'mkdir', '/sourcebuild'], \
+ )
+ except:
+ pass
+
+ r = remote.run (args=['sudo', 'git','config',\
+ '--global','http.sslVerify', 'false']
+ )
+# r = remote.run(args=['sudo', 'git', 'clone',\
+# 'https://gitlab.cee.redhat.com/ceph/ceph.git','/sourcebuild/ceph'], timeout=3600,\
+# check_status=False, wait=True)
+
+ r = remote.run(args=['sudo', 'git', 'clone',\
+ 'https://github.com/ceph/ceph.git','/sourcebuild/ceph'], timeout=3600,\
+ check_status=False, wait=True)
+
+ get_dependencies(ctx, config, remote)
+ #start_compile()
+
+
+
+
+@contextlib.contextmanager
+def task(ctx, config):
+ """
+ This will replace Install task for the tests.
+ fetch source from a git repo on all the nodes and compile
+ them locally. this is because we need to preserve .gcno files
+ for gcov report generation purpose.
+
+ ex:
+ - tasks:
+ make-coverage:
+ ceph:
+ radosbench:
+ """
+ ''' Hard coding for POC purpose'''
+
+ grepo="https://gitlab.cee.redhat.com/ceph/ceph.git"
+ builddir="/sourcebuild/ceph/"
+
+ with parallel() as ptask:
+ for remote, roles in ctx.cluster.remotes.iteritems():
+ ptask.spawn(compile_with_gcov, ctx, config, grepo, remote, builddir)
+
+
+ '''
+ Transfer source dir which has gcno files and source
+ only from one node
+ '''
+
+ log.info("Transferring the source directory")
+ remote=next(iter(ctx.cluster.remotes))
+ get_sourcedir(ctx, config, remote, builddir)
+
+ '''create dir with run name in COVERAGE_LOGS directory'''
+ rundir=COVERAGE_LOGS+os.path.basename(os.path.dirname(ctx.archive))
+ log.info("rundir = {}".format(rundir))
+ try:
+ os.mkdir(rundir)
+ except:
+ log.error("Failed to create rundir")
+
+ yield
+ #TODO: cleanup still pending
+
+
projects / ceph.git / commitdiff