From be2077676fffe3d6e6bd8e5bde6bd0d5b7293a8f Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Fri, 9 Oct 2015 10:05:58 +0200 Subject: [PATCH] buildpackages: walk the whole config tree to find sha1 to build It is not enough to look for the first install task. In upgrade tests, the install.upgrade task requires more packages to be built. In more complicated tests using sequential and parallel tasks, the actual install or install.upgrade task may be deeper in the config tree. Signed-off-by: Loic Dachary (cherry picked from commit 4d89c9e4dee3ededc403509010093a55be54a95a) --- .../buildpackages/tasks/branch.yaml | 10 ++ .../buildpackages/tasks/builpackages.yaml | 4 - .../buildpackages/tasks/default.yaml | 14 ++ .../teuthology/buildpackages/tasks/tag.yaml | 11 ++ tasks/buildpackages.py | 147 ++++++++------- tasks/tests/test_buildpackages.py | 170 ++++++++++++++++++ 6 files changed, 289 insertions(+), 67 deletions(-) create mode 100644 suites/teuthology/buildpackages/tasks/branch.yaml delete mode 100644 suites/teuthology/buildpackages/tasks/builpackages.yaml create mode 100644 suites/teuthology/buildpackages/tasks/default.yaml create mode 100644 suites/teuthology/buildpackages/tasks/tag.yaml create mode 100644 tasks/tests/test_buildpackages.py diff --git a/suites/teuthology/buildpackages/tasks/branch.yaml b/suites/teuthology/buildpackages/tasks/branch.yaml new file mode 100644 index 0000000000000..b93c5d07c4a32 --- /dev/null +++ b/suites/teuthology/buildpackages/tasks/branch.yaml @@ -0,0 +1,10 @@ +roles: + - [mon.0, client.0] +tasks: + - install: + # branch has precedence over sha1 + branch: hammer + sha1: e5b6eea91cc37434f78a987d2dd1d3edd4a23f3f # dumpling + - exec: + client.0: + - ceph --version | grep 'version 0.94' diff --git a/suites/teuthology/buildpackages/tasks/builpackages.yaml b/suites/teuthology/buildpackages/tasks/builpackages.yaml deleted file mode 100644 index a77186be73bb3..0000000000000 --- a/suites/teuthology/buildpackages/tasks/builpackages.yaml +++ /dev/null @@ -1,4 +0,0 @@ -roles: - - [mon.0, client.0] -tasks: - - install: diff --git a/suites/teuthology/buildpackages/tasks/default.yaml b/suites/teuthology/buildpackages/tasks/default.yaml new file mode 100644 index 0000000000000..cb583c7634ab4 --- /dev/null +++ b/suites/teuthology/buildpackages/tasks/default.yaml @@ -0,0 +1,14 @@ +roles: + - [client.0] +tasks: + - install: + tag: v0.94.1 + - exec: + client.0: + - ceph --version | grep 'version 0.94.1' + - install.upgrade: + client.0: + tag: v0.94.3 + - exec: + client.0: + - ceph --version | grep 'version 0.94.3' diff --git a/suites/teuthology/buildpackages/tasks/tag.yaml b/suites/teuthology/buildpackages/tasks/tag.yaml new file mode 100644 index 0000000000000..126749c9bb0be --- /dev/null +++ b/suites/teuthology/buildpackages/tasks/tag.yaml @@ -0,0 +1,11 @@ +roles: + - [mon.0, client.0] +tasks: + - install: + # tag has precedence over branch and sha1 + tag: v0.94.1 + branch: firefly + sha1: e5b6eea91cc37434f78a987d2dd1d3edd4a23f3f # dumpling + - exec: + client.0: + - ceph --version | grep 'version 0.94.1' diff --git a/tasks/buildpackages.py b/tasks/buildpackages.py index b25b0c8c488d9..27e5d4b5e090e 100644 --- a/tasks/buildpackages.py +++ b/tasks/buildpackages.py @@ -1,15 +1,21 @@ """ Build ceph packages +Unit tests: + +py.test -v -s tests/test_buildpackages.py + Integration tests: -teuthology-openstack --verbose --key-name myself --key-filename ~/Downloads/myself --ceph hammer --suite teuthology/buildpackages +teuthology-openstack --verbose --key-name myself --key-filename ~/Downloads/myself --ceph infernalis --suite teuthology/buildpackages """ +import copy import logging import os -import subprocess -from teuthology import packaging +import re +import types +from subprocess import check_output, check_call from teuthology import misc as teuthology from teuthology.config import config as teuth_config from teuthology.task import install @@ -18,10 +24,6 @@ import urlparse log = logging.getLogger(__name__) -def get_install_config(ctx): - for task in ctx.config['tasks']: - if task.keys()[0] == 'install': - config = task['install'] def get_tag_branch_sha1(gitbuilder): """The install config may have contradicting tag/branch and sha1. When suite.py prepares the jobs, it always overrides the sha1 with @@ -82,8 +84,13 @@ def get_tag_branch_sha1(gitbuilder): tag = None branch = None return (tag, branch, sha1) + +def get_config_install(ctx, config): if config is None: config = {} + else: + config = copy.deepcopy(config) + assert isinstance(config, dict), \ "task install only supports a dictionary for configuration" @@ -94,7 +101,33 @@ def get_tag_branch_sha1(gitbuilder): install_overrides = overrides.get('install', {}) teuthology.deep_merge(config, install_overrides.get(project, {})) log.debug('install config %s' % config) - return config + return [config] + +def get_config_install_upgrade(ctx, config): + config = copy.deepcopy(config) + r = install.upgrade_remote_to_config(ctx, config).values() + log.info("get_config_install_upgrade " + str(r)) + return r + +GET_CONFIG_FUNCTIONS = { + 'install': get_config_install, + 'install.upgrade': get_config_install_upgrade, +} + +def lookup_configs(ctx, node): + configs = [] + if type(node) is types.ListType: + for leaf in node: + configs.extend(lookup_configs(ctx, leaf)) + elif type(node) is types.DictType: + for (key, value) in node.iteritems(): + if key in ('install', 'install.upgrade'): + configs.extend(GET_CONFIG_FUNCTIONS[key](ctx, value)) + elif key in ('overrides',): + pass + else: + configs.extend(lookup_configs(ctx, value)) + return configs def task(ctx, config): """ @@ -126,59 +159,47 @@ def task(ctx, config): assert isinstance(config, dict), \ 'task only accepts a dict for config not ' + str(config) d = os.path.join(os.path.dirname(__file__), 'buildpackages') - install_config = get_install_config(ctx) - log.info('install_config ' + str(install_config)) for remote in ctx.cluster.remotes.iterkeys(): - gitbuilder = install._get_gitbuilder_project( - ctx, remote, install_config) - tag = packaging._get_config_value_for_remote( - ctx, remote, install_config, 'tag') - branch = packaging._get_config_value_for_remote( - ctx, remote, install_config, 'branch') - sha1 = packaging._get_config_value_for_remote( - ctx, remote, install_config, 'sha1') - uri_reference = gitbuilder.uri_reference - url = gitbuilder.base_url - assert '/' + uri_reference in url, \ - (url + ' (from template ' + teuth_config.baseurl_template + - ') does not contain /' + uri_reference) - if 'ref/' in uri_reference: - ref = os.path.basename(uri_reference) - else: - ref = '' - subprocess.check_call( - "flock --close /tmp/buildpackages " + - "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent", - shell=True) - target = os.path.dirname(urlparse.urlparse(url).path.strip('/')) - target = os.path.dirname(target) + '-' + sha1 - openstack = OpenStack() - select = '^(vps|eg)-' - build_flavor = openstack.flavor(config['machine'], select) - http_flavor = openstack.flavor({ - 'disk': 10, # GB - 'ram': 1024, # MB - 'cpus': 1, - }, select) - cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " + - " flock --close /tmp/buildpackages-" + sha1 + - " make -C " + d + - " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() + - " CEPH_PKG_TYPE=" + gitbuilder.pkg_type + - " CEPH_OS_TYPE=" + gitbuilder.os_type + - " CEPH_OS_VERSION=" + gitbuilder.os_version + - " CEPH_DIST=" + gitbuilder.distro + - " CEPH_ARCH=" + gitbuilder.arch + - " CEPH_SHA1=" + sha1 + - " CEPH_TAG=" + (tag or '') + - " CEPH_BRANCH=" + (branch or '') + - " CEPH_REF=" + ref + - " GITBUILDER_URL=" + url + - " BUILD_FLAVOR=" + build_flavor + - " HTTP_FLAVOR=" + http_flavor + - " " + target + - " ") - log.info("buildpackages: " + cmd) - subprocess.check_call(cmd, shell=True) - teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '') - log.info('Finished buildpackages') + for install_config in lookup_configs(ctx, ctx.config): + gitbuilder = install._get_gitbuilder_project( + ctx, remote, install_config) + (tag, branch, sha1) = get_tag_branch_sha1(gitbuilder) + check_call( + "flock --close /tmp/buildpackages " + + "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent", + shell=True) + url = gitbuilder.base_url + target = os.path.dirname(urlparse.urlparse(url).path.strip('/')) + target = os.path.dirname(target) + '-' + sha1 + openstack = OpenStack() + if 'cloud.ovh.net' in os.environ['OS_AUTH_URL']: + select = '^(vps|eg)-' + else: + select = '' + build_flavor = openstack.flavor(config['machine'], select) + http_flavor = openstack.flavor({ + 'disk': 10, # GB + 'ram': 1024, # MB + 'cpus': 1, + }, select) + cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; " + + " flock --close /tmp/buildpackages-" + sha1 + + " make -C " + d + + " CEPH_GIT_URL=" + teuth_config.get_ceph_git_url() + + " CEPH_PKG_TYPE=" + gitbuilder.pkg_type + + " CEPH_OS_TYPE=" + gitbuilder.os_type + + " CEPH_OS_VERSION=" + gitbuilder.os_version + + " CEPH_DIST=" + gitbuilder.distro + + " CEPH_ARCH=" + gitbuilder.arch + + " CEPH_SHA1=" + sha1 + + " CEPH_TAG=" + (tag or '') + + " CEPH_BRANCH=" + (branch or '') + + " GITBUILDER_URL=" + url + + " BUILD_FLAVOR=" + build_flavor + + " HTTP_FLAVOR=" + http_flavor + + " " + target + + " ") + log.info("buildpackages: " + cmd) + check_call(cmd, shell=True) + teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '') + log.info('Finished buildpackages') diff --git a/tasks/tests/test_buildpackages.py b/tasks/tests/test_buildpackages.py new file mode 100644 index 0000000000000..fed5aa02b9180 --- /dev/null +++ b/tasks/tests/test_buildpackages.py @@ -0,0 +1,170 @@ +# py.test -v -s tests/test_buildpackages.py + +from mock import patch, Mock + +from .. import buildpackages +from teuthology import packaging + +def test_get_tag_branch_sha1(): + gitbuilder = packaging.GitbuilderProject( + 'ceph', + { + 'os_type': 'centos', + 'os_version': '7.0', + }) + (tag, branch, sha1) = buildpackages.get_tag_branch_sha1(gitbuilder) + assert tag == None + assert branch == None + assert sha1 is not None + + gitbuilder = packaging.GitbuilderProject( + 'ceph', + { + 'os_type': 'centos', + 'os_version': '7.0', + 'sha1': 'asha1', + }) + (tag, branch, sha1) = buildpackages.get_tag_branch_sha1(gitbuilder) + assert tag == None + assert branch == None + assert sha1 == 'asha1' + + remote = Mock + remote.arch = 'x86_64' + remote.os = Mock + remote.os.name = 'ubuntu' + remote.os.version = '14.04' + remote.os.codename = 'trusty' + remote.system_type = 'deb' + ctx = Mock + ctx.cluster = Mock + ctx.cluster.remotes = {remote: ['client.0']} + + expected_tag = 'v0.94.1' + expected_sha1 = 'expectedsha1' + def check_output(cmd, shell): + assert shell == True + return expected_sha1 + " refs/tags/" + expected_tag + with patch.multiple( + buildpackages, + check_output=check_output, + ): + gitbuilder = packaging.GitbuilderProject( + 'ceph', + { + 'os_type': 'centos', + 'os_version': '7.0', + 'sha1': 'asha1', + 'all': { + 'tag': tag, + }, + }, + ctx = ctx, + remote = remote) + (tag, branch, sha1) = buildpackages.get_tag_branch_sha1(gitbuilder) + assert tag == expected_tag + assert branch == None + assert sha1 == expected_sha1 + + expected_branch = 'hammer' + expected_sha1 = 'otherexpectedsha1' + def check_output(cmd, shell): + assert shell == True + return expected_sha1 + " refs/heads/" + expected_branch + with patch.multiple( + buildpackages, + check_output=check_output, + ): + gitbuilder = packaging.GitbuilderProject( + 'ceph', + { + 'os_type': 'centos', + 'os_version': '7.0', + 'sha1': 'asha1', + 'all': { + 'branch': branch, + }, + }, + ctx = ctx, + remote = remote) + (tag, branch, sha1) = buildpackages.get_tag_branch_sha1(gitbuilder) + assert tag == None + assert branch == expected_branch + assert sha1 == expected_sha1 + +def test_lookup_configs(): + expected_system_type = 'deb' + def make_remote(): + remote = Mock() + remote.arch = 'x86_64' + remote.os = Mock() + remote.os.name = 'ubuntu' + remote.os.version = '14.04' + remote.os.codename = 'trusty' + remote.system_type = expected_system_type + return remote + ctx = Mock() + class cluster: + remote1 = make_remote() + remote2 = make_remote() + remotes = { + remote1: ['client.0'], + remote2: ['mon.a','osd.0'], + } + def only(self, role): + result = Mock() + if role in ('client.0',): + result.remotes = { cluster.remote1: None } + elif role in ('osd.0', 'mon.a'): + result.remotes = { cluster.remote2: None } + else: + result.remotes = None + return result + ctx.cluster = cluster() + ctx.config = { + 'roles': [ ['client.0'], ['mon.a','osd.0'] ], + } + + # nothing -> nothing + assert buildpackages.lookup_configs(ctx, {}) == [] + assert buildpackages.lookup_configs(ctx, {1:[1,2,3]}) == [] + assert buildpackages.lookup_configs(ctx, [[1,2,3]]) == [] + assert buildpackages.lookup_configs(ctx, None) == [] + + # + # the overrides applies to install and to install.upgrade + # that have no tag, branch or sha1 + # + config = { + 'overrides': { + 'install': { + 'ceph': { + 'sha1': 'overridesha1', + 'tag': 'overridetag', + 'branch': 'overridebranch', + }, + }, + }, + 'tasks': [ + { + 'install': { + 'sha1': 'installsha1', + }, + }, + { + 'install.upgrade': { + 'osd.0': { + }, + 'client.0': { + 'sha1': 'client0sha1', + }, + }, + } + ], + } + ctx.config = config + expected_configs = [{'branch': 'overridebranch', 'sha1': 'overridesha1', 'tag': 'overridetag'}, + {'project': 'ceph', 'branch': 'overridebranch', 'sha1': 'overridesha1', 'tag': 'overridetag'}, + {'project': 'ceph', 'sha1': 'client0sha1'}] + + assert buildpackages.lookup_configs(ctx, config) == expected_configs -- 2.39.5