buildpackages: build the repository and the packages

http://tracker.ceph.com/issues/13031 Fixes: #13031

Signed-off-by: Loic Dachary <loic@dachary.org>
(cherry picked from commit e38a6d69dfb25fae11902f537b6f2b752689236f)

diff --git a/suites/teuthology/buildpackages/buildpackages.yaml b/suites/teuthology/buildpackages/buildpackages.yaml
index 83bd704d12e9911a9d742d4ae75ae85b1f1a1d5e..53c02bf83fa89d36621f1845d5348de8ca2ff1a3 100644 (file)
--- a/suites/teuthology/buildpackages/buildpackages.yaml
+++ b/suites/teuthology/buildpackages/buildpackages.yaml
@@ -3,3 +3,7 @@ roles:
 tasks:
     - buildpackages:
     - install:
+        machine:
+          disk: 40 # GB
+          ram: 15000 # MB
+          cpus: 16

diff --git a/tasks/buildpackages.py b/tasks/buildpackages.py
index a1ecb1c490ec5c317e644159426aa18574877525..a2bdb669ac2ee74bad6b2d411f7e4840cbaf1b7d 100644 (file)
--- a/tasks/buildpackages.py
+++ b/tasks/buildpackages.py
@@ -1,14 +1,14 @@
 """
 Build ceph packages
 """
-import contextlib
 import logging
 import os
 import subprocess
-from teuthology import contextutil, packaging
+from teuthology import packaging
 from teuthology import misc as teuthology
 from teuthology.config import config as teuth_config
 from teuthology.task import install
+from teuthology.openstack import OpenStack
 import urlparse
 
 log = logging.getLogger(__name__)
@@ -31,7 +31,6 @@ def get_install_config(ctx):
     log.debug('install config %s' % config)
     return config
 
-@contextlib.contextmanager
 def task(ctx, config):
     """
     Build Ceph packages. This task will automagically be run
@@ -63,6 +62,7 @@ def task(ctx, config):
         'task only accepts a dict for config not ' + str(config)
     d = os.path.join(os.path.dirname(__file__), 'buildpackages')
     install_config = get_install_config(ctx)
+    log.info('install_config ' + str(install_config))
     for remote in ctx.cluster.remotes.iterkeys():
         gitbuilder = install._get_gitbuilder_project(
             ctx, remote, install_config)
@@ -81,20 +81,29 @@ def task(ctx, config):
             ref = os.path.basename(uri_reference)
         else:
             ref = ''
-        url = urlparse.urlparse(url)
         subprocess.check_call(
-            "make -C " + d + " " + os.environ['HOME'] + "/.ssh-agent")
-        subprocess.check_call(
-            "source ~/.ssh-agent ; make -C " + d +
-            " CEPH_OS_TYPE=" + gitbuilder.os_type +
-            " CEPH_OS_VERSION=" + gitbuilder.os_version +
-            " CEPH_DIST=" + gitbuilder.distro +
-            " CEPH_ARCH=" + gitbuilder.arch +
-            " CEPH_SHA1=" + (sha1 or '') +
-            " CEPH_TAG=" + (tag or '') +
-            " CEPH_BRANCH=" + (branch or '') +
-            " CEPH_REF=" + ref +
-            " GITBUILDER_URL=" + url +
-            " packages-" + url.path.strip('/') +
-            " ", shell=True)
+            "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent",
+            shell=True)
+        target = os.path.dirname(urlparse.urlparse(url).path.strip('/'))
+        target = os.path.dirname(target) + '-' + sha1
+        openstack = OpenStack()
+        select = '^(vps|eg)-'
+        flavor = openstack.flavor(config['machine'], select)
+        cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; make -C " + d +
+               " CEPH_GIT_BASE_URL=" + teuth_config.ceph_git_base_url + 
+               " CEPH_OS_TYPE=" + gitbuilder.os_type +
+               " CEPH_OS_VERSION=" + gitbuilder.os_version +
+               " CEPH_DIST=" + gitbuilder.distro +
+               " CEPH_ARCH=" + gitbuilder.arch +
+               " CEPH_SHA1=" + (sha1 or '')  +
+               " CEPH_TAG=" + (tag or '') +
+               " CEPH_BRANCH=" + (branch or '') +
+               " CEPH_REF=" + ref +
+               " GITBUILDER_URL=" + url +
+               " FLAVOR=" + flavor +
+               " " + target +
+               " ")
+        log.info("buildpackages: " + cmd)
+        subprocess.check_call(cmd, shell=True)
+    teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '')
     log.info('Finished buildpackages')

diff --git a/tasks/buildpackages/Makefile b/tasks/buildpackages/Makefile
index 339f4a2c111cf901ac77fe13133ba1f6be145a6b..354caf9b9094e60cde277629f905c2b3246dc1b1 100644 (file)
--- a/tasks/buildpackages/Makefile
+++ b/tasks/buildpackages/Makefile
@@ -1,31 +1,40 @@
+SHELL=/bin/bash
 D=/tmp/stampsdir
+VPATH=${D}
 
 define get_ip
-  $(shell openstack server show -f json ${1} | jq '.[] | select(.Field == "addresses") | .Value' | perl -pe 's/.*?=([\d\.]+).*/$$1/')
+$$(openstack server show -f json $(1) | jq '.[] | select(.Field == "addresses") | .Value' | perl -pe 's/.*?=([\d\.]+).*/$$1/')
 endef
 
 MY_IP=$(shell hostname -I | cut -f1 -d' ')
 
-$HOME/.ssh_agent:
-       ssh-agent -s > $HOME/.ssh_agent
-       source $HOME/.ssh_agent ; ssh-add
-       grep -q ssh_agent ~/.bashrc_teuthology || echo 'source $HOME/.ssh_agent' >> ~/.bashrc_teuthology
+${HOME}/.ssh_agent:
+       ssh-agent -s > ${HOME}/.ssh_agent
+       source ${HOME}/.ssh_agent ; ssh-add ; ssh-add -l
+       grep -q ssh_agent ~/.bashrc_teuthology || echo 'source ${HOME}/.ssh_agent' >> ~/.bashrc_teuthology
 
-${D}/packages-repository:
-        openstack server create --image 'teuthology-ubuntu-14.04' --flavor 'vps-ssd-1' --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30
-        ip=$(call get_ip,$@)
-        ssh $$ip sudo apt-get install -y nginx
-        ssh $$ip sudo chown -R ubuntu /usr/share/nginx/html
-        perl -pi -e "s/gitbuilder_host.*/gitbuilder_host: $$ip/" ~/.teuthology.yaml
-        mkdir -p ${@D} ; $touch $@
+packages-repository:
+       openstack server create --image 'teuthology-ubuntu-14.04' --flavor 'vps-ssd-1' --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30
+       ip=$(call get_ip,$@) ; \
+       ssh $$ip sudo apt-get install -y nginx ; \
+       ssh $$ip sudo chown -R ubuntu /usr/share/nginx/html ; \
+       perl -pi -e "s/^gitbuilder_host:.*/gitbuilder_host: $$ip/" ~/.teuthology.yaml
+       mkdir -p ${D}/${@D} ; touch ${D}/$@
 
 FLAVOR=vps-ssd-1
 
-${D}/packages-ceph-deb-${CEPH_DIST}-${CEPH_ARCH}-basic/${CEPH_REF}: ${D}/packages-repository
-        openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}' --flavor ${FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30
-        ip=$(call get_ip,$@)
-        ssh $$ip sudo apt-get install -y git
-        scp make-deb.sh $$ip:
-        ssh -A $$ip make-debs.sh /opt/release $(call get_ip,$<)
+ceph-deb-${CEPH_DIST}-${CEPH_ARCH}-basic-${CEPH_SHA1}: packages-repository
+       openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}' --flavor ${FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30
+       set -ex ; \
+       ip=$(call get_ip,$@) ; \
+       echo 'sudo apt-get update ; sudo apt-get install -y git ; test -d ceph || git clone ${CEPH_GIT_BASE_URL}/ceph ; cd ceph ; git checkout ${CEPH_SHA1}' | ssh $$ip bash ; \
+       scp make-debs.sh $$ip: ; \
+       packages_repository=$(call get_ip,${<F}) ; \
+       echo 'cd ceph ; sudo chown $${USER} /opt ; sudo apt-get install -y reprepro ; git checkout origin/master install-deps.sh ; ./install-deps.sh ; ../make-debs.sh /opt/release '$$packages_repository | ssh -A $$ip bash
        openstack server delete $@
-        mkdir -p ${@D} ; $touch $@
+       mkdir -p ${D}/${@D} ; touch ${D}/$@
+
+clobber:
+       pkill ssh-agent || true
+       rm -f ${HOME}/.ssh_agent
+       rm -fr ${D}

diff --git a/tasks/buildpackages/make-debs.sh b/tasks/buildpackages/make-debs.sh
index 46e703ef2cd6b6e283678fd3fea92f2cf20d3294..7a35ab6047a7736fc68c15b03340c8a3fcb56b90 100755 (executable)
--- a/tasks/buildpackages/make-debs.sh
+++ b/tasks/buildpackages/make-debs.sh
@@ -47,7 +47,7 @@ function build_package() {
     # remove all files not under git so they are not
     # included in the distribution.
     #
-    git clean -dxf
+    git clean -qdxff
     #
     # creating the distribution tarbal requires some configure
     # options (otherwise parts of the source tree will be left out).