From 39a0de3cb2b414fc5dbae31b89c9e1b6a17df99d Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Tue, 15 Sep 2015 18:43:06 +0200 Subject: [PATCH] buildpackages: build the repository and the packages http://tracker.ceph.com/issues/13031 Fixes: #13031 Signed-off-by: Loic Dachary (cherry picked from commit e38a6d69dfb25fae11902f537b6f2b752689236f) --- .../buildpackages/buildpackages.yaml | 4 ++ tasks/buildpackages.py | 45 +++++++++++------- tasks/buildpackages/Makefile | 47 +++++++++++-------- tasks/buildpackages/make-debs.sh | 2 +- 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/suites/teuthology/buildpackages/buildpackages.yaml b/suites/teuthology/buildpackages/buildpackages.yaml index 83bd704d12e99..53c02bf83fa89 100644 --- a/suites/teuthology/buildpackages/buildpackages.yaml +++ b/suites/teuthology/buildpackages/buildpackages.yaml @@ -3,3 +3,7 @@ roles: tasks: - buildpackages: - install: + machine: + disk: 40 # GB + ram: 15000 # MB + cpus: 16 diff --git a/tasks/buildpackages.py b/tasks/buildpackages.py index a1ecb1c490ec5..a2bdb669ac2ee 100644 --- a/tasks/buildpackages.py +++ b/tasks/buildpackages.py @@ -1,14 +1,14 @@ """ Build ceph packages """ -import contextlib import logging import os import subprocess -from teuthology import contextutil, packaging +from teuthology import packaging from teuthology import misc as teuthology from teuthology.config import config as teuth_config from teuthology.task import install +from teuthology.openstack import OpenStack import urlparse log = logging.getLogger(__name__) @@ -31,7 +31,6 @@ def get_install_config(ctx): log.debug('install config %s' % config) return config -@contextlib.contextmanager def task(ctx, config): """ Build Ceph packages. This task will automagically be run @@ -63,6 +62,7 @@ def task(ctx, config): 'task only accepts a dict for config not ' + str(config) d = os.path.join(os.path.dirname(__file__), 'buildpackages') install_config = get_install_config(ctx) + log.info('install_config ' + str(install_config)) for remote in ctx.cluster.remotes.iterkeys(): gitbuilder = install._get_gitbuilder_project( ctx, remote, install_config) @@ -81,20 +81,29 @@ def task(ctx, config): ref = os.path.basename(uri_reference) else: ref = '' - url = urlparse.urlparse(url) subprocess.check_call( - "make -C " + d + " " + os.environ['HOME'] + "/.ssh-agent") - subprocess.check_call( - "source ~/.ssh-agent ; make -C " + d + - " CEPH_OS_TYPE=" + gitbuilder.os_type + - " CEPH_OS_VERSION=" + gitbuilder.os_version + - " CEPH_DIST=" + gitbuilder.distro + - " CEPH_ARCH=" + gitbuilder.arch + - " CEPH_SHA1=" + (sha1 or '') + - " CEPH_TAG=" + (tag or '') + - " CEPH_BRANCH=" + (branch or '') + - " CEPH_REF=" + ref + - " GITBUILDER_URL=" + url + - " packages-" + url.path.strip('/') + - " ", shell=True) + "make -C " + d + " " + os.environ['HOME'] + "/.ssh_agent", + shell=True) + target = os.path.dirname(urlparse.urlparse(url).path.strip('/')) + target = os.path.dirname(target) + '-' + sha1 + openstack = OpenStack() + select = '^(vps|eg)-' + flavor = openstack.flavor(config['machine'], select) + cmd = (". " + os.environ['HOME'] + "/.ssh_agent ; make -C " + d + + " CEPH_GIT_BASE_URL=" + teuth_config.ceph_git_base_url + + " CEPH_OS_TYPE=" + gitbuilder.os_type + + " CEPH_OS_VERSION=" + gitbuilder.os_version + + " CEPH_DIST=" + gitbuilder.distro + + " CEPH_ARCH=" + gitbuilder.arch + + " CEPH_SHA1=" + (sha1 or '') + + " CEPH_TAG=" + (tag or '') + + " CEPH_BRANCH=" + (branch or '') + + " CEPH_REF=" + ref + + " GITBUILDER_URL=" + url + + " FLAVOR=" + flavor + + " " + target + + " ") + log.info("buildpackages: " + cmd) + subprocess.check_call(cmd, shell=True) + teuth_config.gitbuilder_host = openstack.get_ip('packages-repository', '') log.info('Finished buildpackages') diff --git a/tasks/buildpackages/Makefile b/tasks/buildpackages/Makefile index 339f4a2c111cf..354caf9b9094e 100644 --- a/tasks/buildpackages/Makefile +++ b/tasks/buildpackages/Makefile @@ -1,31 +1,40 @@ +SHELL=/bin/bash D=/tmp/stampsdir +VPATH=${D} define get_ip - $(shell openstack server show -f json ${1} | jq '.[] | select(.Field == "addresses") | .Value' | perl -pe 's/.*?=([\d\.]+).*/$$1/') +$$(openstack server show -f json $(1) | jq '.[] | select(.Field == "addresses") | .Value' | perl -pe 's/.*?=([\d\.]+).*/$$1/') endef MY_IP=$(shell hostname -I | cut -f1 -d' ') -$HOME/.ssh_agent: - ssh-agent -s > $HOME/.ssh_agent - source $HOME/.ssh_agent ; ssh-add - grep -q ssh_agent ~/.bashrc_teuthology || echo 'source $HOME/.ssh_agent' >> ~/.bashrc_teuthology +${HOME}/.ssh_agent: + ssh-agent -s > ${HOME}/.ssh_agent + source ${HOME}/.ssh_agent ; ssh-add ; ssh-add -l + grep -q ssh_agent ~/.bashrc_teuthology || echo 'source ${HOME}/.ssh_agent' >> ~/.bashrc_teuthology -${D}/packages-repository: - openstack server create --image 'teuthology-ubuntu-14.04' --flavor 'vps-ssd-1' --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 - ip=$(call get_ip,$@) - ssh $$ip sudo apt-get install -y nginx - ssh $$ip sudo chown -R ubuntu /usr/share/nginx/html - perl -pi -e "s/gitbuilder_host.*/gitbuilder_host: $$ip/" ~/.teuthology.yaml - mkdir -p ${@D} ; $touch $@ +packages-repository: + openstack server create --image 'teuthology-ubuntu-14.04' --flavor 'vps-ssd-1' --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 + ip=$(call get_ip,$@) ; \ + ssh $$ip sudo apt-get install -y nginx ; \ + ssh $$ip sudo chown -R ubuntu /usr/share/nginx/html ; \ + perl -pi -e "s/^gitbuilder_host:.*/gitbuilder_host: $$ip/" ~/.teuthology.yaml + mkdir -p ${D}/${@D} ; touch ${D}/$@ FLAVOR=vps-ssd-1 -${D}/packages-ceph-deb-${CEPH_DIST}-${CEPH_ARCH}-basic/${CEPH_REF}: ${D}/packages-repository - openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}' --flavor ${FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 - ip=$(call get_ip,$@) - ssh $$ip sudo apt-get install -y git - scp make-deb.sh $$ip: - ssh -A $$ip make-debs.sh /opt/release $(call get_ip,$<) +ceph-deb-${CEPH_DIST}-${CEPH_ARCH}-basic-${CEPH_SHA1}: packages-repository + openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}' --flavor ${FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 + set -ex ; \ + ip=$(call get_ip,$@) ; \ + echo 'sudo apt-get update ; sudo apt-get install -y git ; test -d ceph || git clone ${CEPH_GIT_BASE_URL}/ceph ; cd ceph ; git checkout ${CEPH_SHA1}' | ssh $$ip bash ; \ + scp make-debs.sh $$ip: ; \ + packages_repository=$(call get_ip,${