import copy
import logging
import os
-import re
import types
-from subprocess import check_output, check_call
+from subprocess import check_call, check_output
from teuthology import misc as teuthology
+from teuthology import packaging
+from teuthology import misc
from teuthology.config import config as teuth_config
-from teuthology.task import install
from teuthology.openstack import OpenStack
-import urlparse
log = logging.getLogger(__name__)
-def get_tag_branch_sha1(gitbuilder):
- """The install config may have contradicting tag/branch and sha1.
- When suite.py prepares the jobs, it always overrides the sha1 with
- whatever default is provided on the command line with --distro
- and what is found in the gitbuilder. If it turns out that the
- tag or the branch in the install config task is about another sha1,
- it will override anyway. For instance:
+class LocalGitbuilderProject(packaging.GitbuilderProject):
- install:
- tag: v0.94.1
+ def __init__(self):
+ pass
- will be changed into
- install:
- tag: v0.94.1
- sha1: 12345
-
- even though v0.94.1 is not sha1 12345. This is does not cause
- problem with the install task because
- GitbuilderProject._get_uri_reference is used to figure out what to
- install from the gitbuilder and this function gives priority to
- the tag, if not found the branch, if not found the sha1.
-
- It is however confusing and this function returns a sha1 that is
- consistent with the tag or the branch being used.
-
- """
-
- uri_reference = gitbuilder.uri_reference
- url = gitbuilder.base_url
- assert '/' + uri_reference in url, \
- (url + ' (from template ' + teuth_config.baseurl_template +
- ') does not contain /' + uri_reference)
- log.info('uri_reference ' + uri_reference)
- if uri_reference.startswith('ref/'):
- ref = re.sub('^ref/', '', uri_reference) # do not use basename because the ref may contain a /
- ceph_git_url = teuth_config.get_ceph_git_url()
- cmd = "git ls-remote " + ceph_git_url + " " + ref
- output = check_output(cmd, shell=True)
- if not output:
- raise Exception(cmd + " returns nothing")
- lines = output.splitlines()
- if len(lines) != 1:
- raise Exception(
- cmd + " returns " + output +
- " which contains " + str(len(lines)) +
- " lines instead of exactly one")
- log.info(cmd + " returns " + lines[0])
- (sha1, ref) = lines[0].split()
- if ref.startswith('refs/heads/'):
- tag = None
- branch = re.sub('^refs/heads/', '', ref)
- elif ref.startswith('refs/tags/'):
- tag = re.sub('^refs/tags/', '', ref)
- branch = None
- else:
- sha1 = os.path.basename(uri_reference)
- tag = None
- branch = None
- return (tag, branch, sha1)
-
-def get_config_install(ctx, config):
+def apply_overrides(ctx, config):
if config is None:
config = {}
else:
if overrides:
install_overrides = overrides.get('install', {})
teuthology.deep_merge(config, install_overrides.get(project, {}))
+ return config
+
+def get_config_install(ctx, config):
+ config = apply_overrides(ctx, config)
log.debug('install config %s' % config)
- return [config]
+ return [(config.get('flavor', 'basic'),
+ config.get('tag', ''),
+ config.get('branch', ''),
+ config.get('sha1'))]
def get_config_install_upgrade(ctx, config):
- config = copy.deepcopy(config)
- r = install.upgrade_remote_to_config(ctx, config).values()
- log.info("get_config_install_upgrade " + str(r))
- return r
+ log.debug('install.upgrade config before override %s' % config)
+ configs = []
+ for (role, role_config) in config.iteritems():
+ if role_config is None:
+ role_config = {}
+ o = apply_overrides(ctx, role_config)
+
+ log.debug('install.upgrade config ' + str(role_config) +
+ ' and with overrides ' + str(o))
+ # for install.upgrade overrides are actually defaults
+ configs.append((o.get('flavor', 'basic'),
+ role_config.get('tag', o.get('tag', '')),
+ role_config.get('branch', o.get('branch', '')),
+ role_config.get('sha1', o.get('sha1'))))
+ return configs
GET_CONFIG_FUNCTIONS = {
'install': get_config_install,
configs.extend(lookup_configs(ctx, value))
return configs
+def get_sha1(ref):
+ url = teuth_config.get_ceph_git_url()
+ ls_remote = check_output("git ls-remote " + url + " " + ref, shell=True)
+ return ls_remote.split()[0]
+
def task(ctx, config):
"""
Build Ceph packages. This task will automagically be run
assert isinstance(config, dict), \
'task only accepts a dict for config not ' + str(config)
d = os.path.join(os.path.dirname(__file__), 'buildpackages')
- for remote in ctx.cluster.remotes.iterkeys():
- for install_config in lookup_configs(ctx, ctx.config):
- gitbuilder = install._get_gitbuilder_project(
- ctx, remote, install_config)
- (tag, branch, sha1) = get_tag_branch_sha1(gitbuilder)
- check_call(
- "flock --close /tmp/buildpackages " +
- "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent",
- shell=True)
- url = gitbuilder.base_url
- target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
- target = os.path.dirname(target) + '-' + sha1
- openstack = OpenStack()
- if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']:
- select = '^(vps|eg)-'
- else:
- select = ''
- build_flavor = openstack.flavor(config['machine'], select)
- http_flavor = openstack.flavor({
- 'disk': 40, # GB
- 'ram': 1024, # MB
- 'cpus': 1,
- }, select)
- cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
- " flock --close /tmp/buildpackages-" + sha1 +
- " make -C " + d +
- " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
- " CEPH_PKG_TYPE=" + gitbuilder.pkg_type +
- " CEPH_OS_TYPE=" + gitbuilder.os_type +
- # os_version is from the remote and will be 7.1.23 for CentOS 7
- # instead of the expected 7.0 for all 7.* CentOS
- " CEPH_OS_VERSION=" + gitbuilder._get_version() +
- " CEPH_DIST=" + gitbuilder.distro +
- " CEPH_ARCH=" + gitbuilder.arch +
- " CEPH_SHA1=" + sha1 +
- " CEPH_TAG=" + (tag or '') +
- " CEPH_BRANCH=" + (branch or '') +
- " CEPH_FLAVOR=" + gitbuilder.flavor +
- " GITBUILDER_URL=" + url +
- " BUILD_FLAVOR=" + build_flavor +
- " HTTP_FLAVOR=" + http_flavor +
- " " + target +
- " ")
- log.info("buildpackages: " + cmd)
- check_call(cmd, shell=True)
- teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
- log.info('Finished buildpackages')
+ os_type = misc.get_distro(ctx)
+ os_version = misc.get_distro_version(ctx)
+ arch = ctx.config.get('arch', 'x86_64')
+ dist = LocalGitbuilderProject()._get_distro(distro=os_type,
+ version=os_version)
+ pkg_type = misc.get_pkg_type(os_type)
+ check_call(
+ "flock --close /tmp/buildpackages " +
+ "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent",
+ shell=True)
+ for (flavor, tag, branch, sha1) in lookup_configs(ctx, ctx.config):
+ if tag:
+ sha1 = get_sha1(tag)
+ elif branch:
+ sha1 = get_sha1(branch)
+ log.info("building flavor = " + flavor + "," +
+ " tag = " + tag + "," +
+ " branch = " + branch + "," +
+ " sha1 = " + sha1)
+ target = ('ceph-' +
+ pkg_type + '-' +
+ dist + '-' +
+ arch + '-' +
+ flavor + '-' +
+ sha1)
+ openstack = OpenStack()
+ openstack.set_provider()
+ if openstack.provider == 'ovh':
+ select = '^(vps|eg)-'
+ else:
+ select = ''
+ build_flavor = openstack.flavor(config['machine'], select)
+ http_flavor = openstack.flavor({
+ 'disk': 40, # GB
+ 'ram': 1024, # MB
+ 'cpus': 1,
+ }, select)
+ cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " +
+ " flock --close /tmp/buildpackages-" + sha1 +
+ " make -C " + d +
+ " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() +
+ " CEPH_PKG_TYPE=" + pkg_type +
+ " CEPH_OS_TYPE=" + os_type +
+ " CEPH_OS_VERSION=" + os_version +
+ " CEPH_DIST=" + dist +
+ " CEPH_ARCH=" + arch +
+ " CEPH_SHA1=" + sha1 +
+ " CEPH_TAG=" + tag +
+ " CEPH_BRANCH=" + branch +
+ " CEPH_FLAVOR=" + flavor +
+ " BUILD_FLAVOR=" + build_flavor +
+ " HTTP_FLAVOR=" + http_flavor +
+ " " + target +
+ " ")
+ log.info("buildpackages: " + cmd)
+ check_call(cmd, shell=True)
+ teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
+ log.info('Finished buildpackages')
echo --without-tcmalloc --without-cryptopp
fi
}
+
+#
+# for a given $sha1 in the $ceph_dir repository, lookup all references
+# from the remote origin and tags matching the sha1. Add a symbolic
+# link in $ref_dir to the $sha1 for each reference found. If the
+# reference is a tag, also add a symbolic link to the commit to which
+# the tag points, if it is an annotated tag.
+#
+function link_same() {
+ local ref_dir=$1
+ local ceph_dir=$2
+ local sha1=$3
+
+ mkdir -p $ref_dir
+ (
+ cd ${ceph_dir}
+ git for-each-ref refs/tags/** refs/remotes/origin/** | grep $sha1 | \
+ while read sha1 type ref ; do
+ if test $type = 'tag' ; then
+ commit_sha1=$(git rev-parse $ref^{commit})
+ if test $commit_sha1 != $sha1 ; then
+ echo ../sha1/$sha1 ../sha1/$commit_sha1
+ fi
+ fi
+ echo ../sha1/$sha1 $(basename $ref)
+ done
+ ) | while read from to ; do
+ ( cd $ref_dir ; ln -sf $from $to )
+ done
+}
+
+function test_link_same() {
+ local d=/tmp/link_same$$
+ mkdir -p $d/primary
+ cd $d/primary
+ git init
+ touch a ; git add a ; git commit -m 'm' a
+ git tag tag1
+ tag1=$(git rev-parse HEAD)
+ git branch branch1
+ touch b ; git add b ; git commit -m 'm' b
+ git tag --annotate -m 'a' tag2
+ tag2=$(git rev-parse tag2)
+ sha1_tag2=$(git rev-parse tag2^{commit})
+ git branch branch2
+ touch c ; git add c ; git commit -m 'm' c
+ git branch branch3
+ sha1_branch3=$(git rev-parse branch3)
+
+ git clone $d/primary $d/secondary
+ cd $d/secondary
+ mkdir $d/ref $d/sha1
+
+ touch $d/sha1/$sha1_branch3
+ link_same $d/ref $d/secondary $sha1_branch3
+ test $(readlink --canonicalize $d/ref/branch3) = $d/sha1/$sha1_branch3 || return 1
+ test $(readlink --canonicalize $d/ref/master) = $d/sha1/$sha1_branch3 || return 1
+
+ touch $d/sha1/$tag2
+ link_same $d/ref $d/secondary $tag2
+ test $(readlink --canonicalize $d/ref/tag2) = $d/sha1/$tag2 || return 1
+ test $(readlink --canonicalize $d/sha1/$sha1_tag2) = $d/sha1/$tag2 || return 1
+
+ touch $d/sha1/$tag1
+ link_same $d/ref $d/secondary $tag1
+ test $(readlink --canonicalize $d/ref/tag1) = $d/sha1/$tag1 || return 1
+ test $(readlink --canonicalize $d/ref/branch1) = $d/sha1/$tag1 || return 1
+
+ rm -fr $d
+}
+
+if test "$1" = "TEST" ; then
+ shopt -s -o xtrace
+ PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
+ test_link_same
+fi
projects / ceph.git / commitdiff