buildpackages: conditionally fetch tags from ceph/ceph.git

A new parameter, canonical_tags, is added to .teuthology.yaml defaulting to
true.

A new command-line option "--no-canonical-tags" is added to teuthology-openstack
(when given, canonical_tags is set to false in .teuthology.yaml; otherwise it
defaults to true).

The buildpackages task is modified to take the appropriate action based on the
value of canonical_tags taken from .teuthology.yaml. Also, an info message is
displayed reminding the user that the command-line parameter only takes place
when the teuthology VM is created. Like --simultaneous-jobs, the parameter has
no effect in subsequent runs against an existing teuthology environment.

Signed-off-by: Nathan Cutler <ncutler@suse.com>
Conflicts:
	scripts/openstack.py

diff --git a/scripts/openstack.py b/scripts/openstack.py
index 68e91adbf1ac28910e522bbc3a1f42ea45e04586..299b371cb3130f771c6e5a010e48185c9cd07f9d 100644 (file)
--- a/scripts/openstack.py
+++ b/scripts/openstack.py
@@ -248,6 +248,11 @@ def get_openstack_parser():
               'as NAME:URL, NAME!PRIORITY:URL or @FILENAME, for details see below.'),
         default=None,
     )
+    parser.add_argument(
+        '--no-canonical-tags',
+        action='store_true', default=False,
+        help='configure remote teuthology to not fetch tags from http://github.com/ceph/ceph.git in buildpackages task',
+    )
     return parser
 
 def get_parser():

diff --git a/teuthology/openstack/__init__.py b/teuthology/openstack/__init__.py
index 1d62f2cb6b7961000786ed3e1177154793de5a1c..f2aced1b1cab98957b968196552be9aee80bef75 100644 (file)
--- a/teuthology/openstack/__init__.py
+++ b/teuthology/openstack/__init__.py
@@ -738,7 +738,8 @@ class TeuthologyOpenStack(OpenStack):
                                     '--simultaneous-jobs'):
                 del original_argv[0:2]
             elif original_argv[0] in ('--teardown',
-                                      '--upload'):
+                                      '--upload',
+                                      '--no-canonical-tags'):
                 del original_argv[0]
             elif os.path.isabs(original_argv[0]):
                 remote_path = self._upload_yaml_file(original_argv[0])
@@ -944,19 +945,23 @@ ssh access           : ssh {identity}{username}@{ip} # logs in /usr/share/nginx/
                                self.args.ceph_workbench_branch)
             ceph_workbench += (" --ceph-workbench-git-url " +
                                self.args.ceph_workbench_git_url)
+        canonical_tags = "--no-canonical-tags" if self.args.no_canonical_tags else ""
         log.debug("OPENRC = " + openrc + " " +
                   "TEUTHOLOGY_USERNAME = " + self.username + " " +
                   "CLONE_OPENSTACK = " + clone + " " +
                   "UPLOAD = " + upload + " " +
                   "CEPH_WORKBENCH = " + ceph_workbench + " " +
-                  "NWORKERS = " + str(self.args.simultaneous_jobs))
+                  "NWORKERS = " + str(self.args.simultaneous_jobs) +
+                  "CANONICAL_TAGS = " +
+                  ("(empty string)" if canonical_tags == "" else canonical_tags))
         content = (template.
                    replace('OPENRC', openrc).
                    replace('TEUTHOLOGY_USERNAME', self.username).
                    replace('CLONE_OPENSTACK', clone).
                    replace('UPLOAD', upload).
                    replace('CEPH_WORKBENCH', ceph_workbench).
-                   replace('NWORKERS', str(self.args.simultaneous_jobs)))
+                   replace('NWORKERS', str(self.args.simultaneous_jobs)).
+                   replace('CANONICAL_TAGS', canonical_tags))
         open(path, 'w').write(content)
         log.debug("get_user_data: " + content + " written to " + path)
         return path

diff --git a/teuthology/openstack/openstack-user-data.txt b/teuthology/openstack/openstack-user-data.txt
index f537e86b9ff282efb7ddff3e168d13af95217582..1a6b72b8c838bf03c9f8e191b2c6aa440bc58990 100644 (file)
--- a/teuthology/openstack/openstack-user-data.txt
+++ b/teuthology/openstack/openstack-user-data.txt
@@ -12,6 +12,6 @@ packages:
 runcmd:
  - su - -c '(set -x ; CLONE_OPENSTACK && cd teuthology && ./bootstrap install)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1
  - echo 'export OPENRC' | tee /home/TEUTHOLOGY_USERNAME/openrc.sh
- - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1
+ - su - -c '(set -x ; source openrc.sh ; cd teuthology ; source virtualenv/bin/activate ; openstack keypair delete teuthology || true ; teuthology/openstack/setup-openstack.sh --nworkers NWORKERS UPLOAD CEPH_WORKBENCH CANONICAL_TAGS --setup-all)' TEUTHOLOGY_USERNAME >> /tmp/init.out 2>&1
  - /etc/init.d/teuthology restart
 final_message: "teuthology is up and running after $UPTIME seconds"

diff --git a/teuthology/openstack/setup-openstack.sh b/teuthology/openstack/setup-openstack.sh
index 9904dcd8f49a3424d165072dc29de597aa956efe..3036e39e98e645a66bd67811c275c0ca90dd8ac7 100755 (executable)
--- a/teuthology/openstack/setup-openstack.sh
+++ b/teuthology/openstack/setup-openstack.sh
@@ -35,6 +35,7 @@ function create_config() {
     local ip="$5"
     local flavor_select="$6"
     local archive_upload="$7"
+    local canonical_tags="$8"
 
     if test "$flavor_select" ; then
         flavor_select="flavor-select-regexp: $flavor_select"
@@ -62,6 +63,7 @@ queue_host: localhost
 lab_domain: $labdomain
 max_job_time: 32400 # 9 hours
 teuthology_path: .
+canonical_tags: $canonical_tags
 openstack:
   clone: git clone http://github.com/ceph/teuthology
   user-data: teuthology/openstack/openstack-{os_type}-{os_version}-user-data.txt
@@ -520,6 +522,7 @@ function main() {
     local do_populate_paddles=false
     local do_setup_pulpito=false
     local do_clobber=false
+    local canonical_tags=true
 
     export LC_ALL=C
 
@@ -598,6 +601,9 @@ function main() {
             --clobber)
                 do_clobber=true
                 ;;
+            --no-canonical-tags)
+                canonical_tags=false
+                ;;
             *)
                 echo $1 is not a known option
                 return 1
@@ -639,7 +645,7 @@ function main() {
     : ${nameserver:=$ip}
 
     if $do_create_config ; then
-        create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" "$flavor_select" "$archive_upload" || return 1
+        create_config "$network" "$subnets" "$nameserver" "$labdomain" "$ip" "$flavor_select" "$archive_upload" "$canonical_tags" || return 1
         setup_ansible "$subnets" $labdomain || return 1
         setup_ssh_config || return 1
         setup_authorized_keys || return 1

diff --git a/teuthology/task/buildpackages.py b/teuthology/task/buildpackages.py
index c7a51522733a36a322e15bcbdb3081ca340b9540..ac7a17b44caa5f04173c81b6a8398d9e3afd02b1 100644 (file)
--- a/teuthology/task/buildpackages.py
+++ b/teuthology/task/buildpackages.py
@@ -215,9 +215,18 @@ def task(ctx, config):
                " BUILD_FLAVOR=" + build_flavor +
                " HTTP_FLAVOR=" + http_flavor +
                " HTTP_ARCH=" + default_arch +
+               " BUILDPACKAGES_CANONICAL_TAGS=" + 
+               ("true" if teuth_config.canonical_tags else "false") +
                " " + target +
                " ")
-        log.info("buildpackages: " + cmd)
+        log.info("Executing the following make command to build {} packages. " \
+                 "Note that some values in the command, like CEPH_GIT_URL " \
+                 "and BUILDPACKAGES_CANONICAL_TAGS, may differ from similar " \
+                 "command-line parameter values. This is because " \
+                 "the values used by this task are taken from the teuthology " \
+                 "configuration file. If in doubt, tear down your teuthology " \
+                 "instance and start again from scratch.".format(pkg_type))
+        log.info("buildpackages make command: " + cmd)
         misc.sh(cmd)
     teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
     log.info('Finished buildpackages')

diff --git a/teuthology/task/buildpackages/Makefile b/teuthology/task/buildpackages/Makefile
index de20fbb755117a208c4735da5c5f6ca025286685..d0d4da1a26b304718287d3292b234950ffeb95fb 100644 (file)
--- a/teuthology/task/buildpackages/Makefile
+++ b/teuthology/task/buildpackages/Makefile
@@ -72,7 +72,7 @@ ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${P
        for delay in 1 2 4 8 8 8 8 8 8 8 8 8 16 16 16 16 16 32 32 32 64 128 256 512 ; do if ssh -o 'ConnectTimeout=3' $$ip bash -c '"grep -q READYTORUN /var/log/cloud-init*.log"' ; then break ; else sleep $$delay ; fi ; done ; \
        scp make-${CEPH_PKG_TYPE}.sh common.sh ubuntu@$$ip: ; \
        packages_repository=$(call get_ip,${<F}) ; \
-       timeout $(TIMEOUT_BUILD) ssh -tt -A ubuntu@$$ip bash ./make-${CEPH_PKG_TYPE}.sh $$packages_repository ${CEPH_DIST} ${CEPH_GIT_URL} ${CEPH_SHA1} ${CEPH_FLAVOR} ${CEPH_ARCH}
+       timeout $(TIMEOUT_BUILD) ssh -tt -A ubuntu@$$ip bash ./make-${CEPH_PKG_TYPE}.sh $$packages_repository ${CEPH_DIST} ${CEPH_GIT_URL} ${CEPH_SHA1} ${CEPH_FLAVOR} ${CEPH_ARCH} ${BUILDPACKAGES_CANONICAL_TAGS}
        mkdir -p ${D}/${@D} ; touch ${D}/$@
 
 clobber:

diff --git a/teuthology/task/buildpackages/common.sh b/teuthology/task/buildpackages/common.sh
index eb9bc646678535121c46d999a54e50f1329ecd79..6c219fcf76dc4b98ff37a8838ba87aeb3c6abc95 100644 (file)
--- a/teuthology/task/buildpackages/common.sh
+++ b/teuthology/task/buildpackages/common.sh
@@ -42,7 +42,11 @@ function get_ceph() {
     test -d ceph || git clone ${git_ceph_url} ceph
     cd ceph
     if test -d src ; then # so we don't try to fetch when using a fixture
-       git fetch --tags http://github.com/ceph/ceph
+       if test "x$BUILDPACKAGES_CANONICAL_TAGS" != "xfalse" ; then
+          echo "Fetching canonical tags from http://github.com/ceph/ceph (to disable, " \
+               "set BUILDPACKAGES_CANONICAL_TAGS=false in the environment)"
+          git fetch --tags http://github.com/ceph/ceph
+       fi
     fi
     git fetch --tags ${git_ceph_url}
     git checkout ${sha1}

diff --git a/teuthology/task/buildpackages/make-deb.sh b/teuthology/task/buildpackages/make-deb.sh
index 1e5600e7cfe0976d79565163272949fc69f5461b..df93e162f046f5219a13ca89ee3c391213d4ad10 100755 (executable)
--- a/teuthology/task/buildpackages/make-deb.sh
+++ b/teuthology/task/buildpackages/make-deb.sh
@@ -28,10 +28,12 @@ git_ceph_url=$3
 sha1=$4
 flavor=$5
 arch=$6
+canonical_tags=$7
 
 sudo apt-get update
 sudo apt-get install -y git
 
+export BUILDPACKAGES_CANONICAL_TAGS=$canonical_tags
 source $(dirname $0)/common.sh
 
 init_ceph $git_ceph_url $sha1

diff --git a/teuthology/task/buildpackages/make-rpm.sh b/teuthology/task/buildpackages/make-rpm.sh
index d6e09285201ee68f8fd6e58ac9a9f4247c9189dd..32db263fee5c9a67e2e057b582107e3dee133e42 100755 (executable)
--- a/teuthology/task/buildpackages/make-rpm.sh
+++ b/teuthology/task/buildpackages/make-rpm.sh
@@ -29,6 +29,7 @@ git_ceph_url=$3
 sha1=$4
 flavor=$5
 arch=$6
+canonical_tags=$7
 
 suse=false
 [[ $codename =~ suse ]] && suse=true
@@ -40,6 +41,7 @@ else
     sudo yum install -y git
 fi
 
+export BUILDPACKAGES_CANONICAL_TAGS=$canonical_tags
 source $(dirname $0)/common.sh
 
 init_ceph $git_ceph_url $sha1