From: Nathan Cutler Date: Sun, 19 Feb 2017 22:13:44 +0000 (+0100) Subject: buildpackages: conditionally fetch tags from ceph/ceph.git X-Git-Tag: 1.1.0~211^2~80 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1403e29db490dd42553e181f3400dfe822952bab;p=teuthology.git buildpackages: conditionally fetch tags from ceph/ceph.git A new parameter, canonical_tags, is added to .teuthology.yaml defaulting to true. A new command-line option "--no-canonical-tags" is added to teuthology-openstack (when given, canonical_tags is set to false in .teuthology.yaml; otherwise it defaults to true). The buildpackages task is modified to take the appropriate action based on the value of canonical_tags taken from .teuthology.yaml. Also, an info message is displayed reminding the user that the command-line parameter only takes place when the teuthology VM is created. Like --simultaneous-jobs, the parameter has no effect in subsequent runs against an existing teuthology environment. Signed-off-by: Nathan Cutler Conflicts: scripts/openstack.py --- diff --git a/scripts/openstack.py b/scripts/openstack.py index 68e91adbf..299b371cb 100644 --- a/scripts/openstack.py +++ b/scripts/openstack.py @@ -248,6 +248,11 @@ def get_openstack_parser(): 'as NAME:URL, NAME!PRIORITY:URL or @FILENAME, for details see below.'), default=None, ) + parser.add_argument( + '--no-canonical-tags', + action='store_true', default=False, + help='configure remote teuthology to not fetch tags from http://github.com/ceph/ceph.git in buildpackages task', + ) return parser def get_parser(): diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py index 1d62f2cb6..f2aced1b1 100644 --- a/teuthology/openstack/__init__.py +++ b/teuthology/openstack/__init__.py @@ -738,7 +738,8 @@ class TeuthologyOpenStack(OpenStack): '--simultaneous-jobs'): del original_argv[0:2] elif original_argv[0] in ('--teardown', - '--upload'): + '--upload', + '--no-canonical-tags'): del original_argv[0] elif os.path.isabs(original_argv[0]): remote_path = self._upload_yaml_file(original_argv[0]) @@ -944,19 +945,23 @@ ssh access : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/ self.args.ceph_workbench_branch) ceph_workbench += (" --ceph-workbench-git-url " + self.args.ceph_workbench_git_url) + canonical_tags = "--no-canonical-tags" if self.args.no_canonical_tags else "" log.debug("OPENRC = " + openrc + " " + "TEUTHOLOGY_USERNAME = " + self.username + " " + "CLONE_OPENSTACK = " + clone + " " + "UPLOAD = " + upload + " " + "CEPH_WORKBENCH = " + ceph_workbench + " " + - "NWORKERS = " + str(self.args.simultaneous_jobs)) + "NWORKERS = " + str(self.args.simultaneous_jobs) + + "CANONICAL_TAGS = " + + ("(empty string)" if canonical_tags == "" else canonical_tags)) content = (template. replace('OPENRC', openrc). replace('TEUTHOLOGY_USERNAME', self.username). replace('CLONE_OPENSTACK', clone). replace('UPLOAD', upload). replace('CEPH_WORKBENCH', ceph_workbench). - replace('NWORKERS', str(self.args.simultaneous_jobs))) + replace('NWORKERS', str(self.args.simultaneous_jobs)). + replace('CANONICAL_TAGS', canonical_tags)) open(path, 'w').write(content) log.debug("get_user_data: " + content + " written to " + path) return path diff --git a/teuthology/openstack/openstack-user-data.txt b/teuthology/openstack/openstack-user-data.txt index f537e86b9..1a6b72b8c 100644 --- a/teuthology/openstack/openstack-user-data.txt +++ b/teuthology/openstack/openstack-user-data.txt @@ -12,6 +12,6 @@ packages: runcmd: - su - -c '(set -x ; CLONE_OPENSTACK && cd teuthology && ./bootstrap install)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - echo 'export OPENRC' | tee /home/TEUTHOLOGY_USERNAME/openrc.sh - - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 + - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH CANONICAL_TAGS --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1 - /etc/init.d/teuthology restart final_message: "teuthology is up and running after $UPTIME seconds" diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh index 9904dcd8f..3036e39e9 100755 --- a/teuthology/openstack/setup-openstack.sh +++ b/teuthology/openstack/setup-openstack.sh @@ -35,6 +35,7 @@ function create_config() { local ip="$5" local flavor_select="$6" local archive_upload="$7" + local canonical_tags="$8" if test "$flavor_select" ; then flavor_select="flavor-select-regexp: $flavor_select" @@ -62,6 +63,7 @@ queue_host: localhost lab_domain: $labdomain max_job_time: 32400 # 9 hours teuthology_path: . +canonical_tags: $canonical_tags openstack: clone: git clone http://github.com/ceph/teuthology user-data: teuthology/openstack/openstack-{os_type}-{os_version}-user-data.txt @@ -520,6 +522,7 @@ function main() { local do_populate_paddles=false local do_setup_pulpito=false local do_clobber=false + local canonical_tags=true export LC_ALL=C @@ -598,6 +601,9 @@ function main() { --clobber) do_clobber=true ;; + --no-canonical-tags) + canonical_tags=false + ;; *) echo $1 is not a known option return 1 @@ -639,7 +645,7 @@ function main() { : ${nameserver:=$ip} if $do_create_config ; then - create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" "$flavor_select" "$archive_upload" || return 1 + create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" "$flavor_select" "$archive_upload" "$canonical_tags" || return 1 setup_ansible "$subnets" $labdomain || return 1 setup_ssh_config || return 1 setup_authorized_keys || return 1 diff --git a/teuthology/task/buildpackages.py b/teuthology/task/buildpackages.py index c7a515227..ac7a17b44 100644 --- a/teuthology/task/buildpackages.py +++ b/teuthology/task/buildpackages.py @@ -215,9 +215,18 @@ def task(ctx, config): " BUILD_FLAVOR=" + build_flavor + " HTTP_FLAVOR=" + http_flavor + " HTTP_ARCH=" + default_arch + + " BUILDPACKAGES_CANONICAL_TAGS=" + + ("true" if teuth_config.canonical_tags else "false") + " " + target + " ") - log.info("buildpackages: " + cmd) + log.info("Executing the following make command to build {} packages. " \ + "Note that some values in the command, like CEPH_GIT_URL " \ + "and BUILDPACKAGES_CANONICAL_TAGS, may differ from similar " \ + "command-line parameter values. This is because " \ + "the values used by this task are taken from the teuthology " \ + "configuration file. If in doubt, tear down your teuthology " \ + "instance and start again from scratch.".format(pkg_type)) + log.info("buildpackages make command: " + cmd) misc.sh(cmd) teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '') log.info('Finished buildpackages') diff --git a/teuthology/task/buildpackages/Makefile b/teuthology/task/buildpackages/Makefile index de20fbb75..d0d4da1a2 100644 --- a/teuthology/task/buildpackages/Makefile +++ b/teuthology/task/buildpackages/Makefile @@ -72,7 +72,7 @@ ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${P for delay in 1 2 4 8 8 8 8 8 8 8 8 8 16 16 16 16 16 32 32 32 64 128 256 512 ; do if ssh -o 'ConnectTimeout=3' $$ip bash -c '"grep -q READYTORUN /var/log/cloud-init*.log"' ; then break ; else sleep $$delay ; fi ; done ; \ scp make-${CEPH_PKG_TYPE}.sh common.sh ubuntu@$$ip: ; \ packages_repository=$(call get_ip,${