From 2b25080d4faf0ae9ddb0000808830cd4d5c7bea1 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Mon, 14 Sep 2015 13:35:41 +0200 Subject: [PATCH] buildpackages: build ceph packages and upload them Build Ceph packages from source for the required revision, os_type, os_version and architecture and upload them to the gitbuilder repository. http://tracker.ceph.com/issues/13031 Fixes: #13031 Signed-off-by: Loic Dachary --- suites/teuthology/buildpackages/+ | 0 .../buildpackages/buildpackages.yaml | 5 + .../buildpackages/ubuntu_14.04.yaml | 1 + tasks/buildpackages.py | 73 ++++++++++ tasks/buildpackages/Makefile | 31 ++++ tasks/buildpackages/make-debs.sh | 134 ++++++++++++++++++ 6 files changed, 244 insertions(+) create mode 100644 suites/teuthology/buildpackages/+ create mode 100644 suites/teuthology/buildpackages/buildpackages.yaml create mode 120000 suites/teuthology/buildpackages/ubuntu_14.04.yaml create mode 100644 tasks/buildpackages.py create mode 100644 tasks/buildpackages/Makefile create mode 100755 tasks/buildpackages/make-debs.sh diff --git a/suites/teuthology/buildpackages/+ b/suites/teuthology/buildpackages/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/suites/teuthology/buildpackages/buildpackages.yaml b/suites/teuthology/buildpackages/buildpackages.yaml new file mode 100644 index 0000000000000..83bd704d12e99 --- /dev/null +++ b/suites/teuthology/buildpackages/buildpackages.yaml @@ -0,0 +1,5 @@ +roles: + - [mon.0, client.0] +tasks: + - buildpackages: + - install: diff --git a/suites/teuthology/buildpackages/ubuntu_14.04.yaml b/suites/teuthology/buildpackages/ubuntu_14.04.yaml new file mode 120000 index 0000000000000..b6def031cc5a0 --- /dev/null +++ b/suites/teuthology/buildpackages/ubuntu_14.04.yaml @@ -0,0 +1 @@ +../../../distros/supported/ubuntu_14.04.yaml \ No newline at end of file diff --git a/tasks/buildpackages.py b/tasks/buildpackages.py new file mode 100644 index 0000000000000..98031bbcb8032 --- /dev/null +++ b/tasks/buildpackages.py @@ -0,0 +1,73 @@ +""" +Build ceph packages +""" +import contextlib +import logging +import os +import subprocess +from teuthology import install +from teuthology import misc as teuthology +from teuthology import contextutil, packaging +import urlparse + +log = logging.getLogger(__name__) + +@contextlib.contextmanager +def task(ctx, config): + """ + Build Ceph packages. This task will automagically be run + before the task that need to install packages (this is taken + care of by the internal teuthology task). + + The config should be as follows: + + buildpackages: + machine: + disk: 40 # GB + ram: 15000 # MB + cpus: 16 + + example: + + tasks: + - buildpackages: + machine: + disk: 40 # GB + ram: 15000 # MB + cpus: 16 + - install: + """ + log.info('Beginning buildpackages...') + assert isinstance(config, dict), \ + 'task only accepts a dict for config not ' + str(config) + d = os.path.join(os.path.dirname(__file__), 'buildpackages') + for remote in ctx.cluster.remotes.iterkeys(): + gitbuilder = install._get_gitbuilder_project(ctx, remote, config) + tag = _get_config_value_for_remote(ctx, remote, config, 'tag') + branch = _get_config_value_for_remote(ctx, remote, config, 'branch') + sha1 = _get_config_value_for_remote(ctx, remote, config, 'sha1') + uri_reference = gitbuilder.uri_reference + url = gitbuilder.base_url + assert '/' + uri_reference in url, \ + url + ' (from template ' + config.baseurl_template + ') does not contain /' + uri_reference + if 'ref/' in uri_reference: + ref = os.path.basename(uri_reference) + else: + ref = '' + url = urlparse.urlparse(url) + subprocess.check_call( + "make -C " + d + " " + os.environ['HOME'] + "/.ssh-agent") + subprocess.check_call( + "source ~/.ssh-agent ; make -C " + d + + " CEPH_OS_TYPE=" + gitbuilder.os_type + + " CEPH_OS_VERSION=" + gitbuilder.os_version + + " CEPH_DIST=" + gitbuilder.distro + + " CEPH_ARCH=" + gitbuilder.arch + + " CEPH_SHA1=" + (sha1 or '') + + " CEPH_TAG=" + (tag or '') + + " CEPH_BRANCH=" + (branch or '') + + " CEPH_REF=" + ref + + " GITBUILDER_URL=" + url + + " packages-" + url.path.strip('/') + + " ", shell=True) + log.info('Finished buildpackages') diff --git a/tasks/buildpackages/Makefile b/tasks/buildpackages/Makefile new file mode 100644 index 0000000000000..339f4a2c111cf --- /dev/null +++ b/tasks/buildpackages/Makefile @@ -0,0 +1,31 @@ +D=/tmp/stampsdir + +define get_ip + $(shell openstack server show -f json ${1} | jq '.[] | select(.Field == "addresses") | .Value' | perl -pe 's/.*?=([\d\.]+).*/$$1/') +endef + +MY_IP=$(shell hostname -I | cut -f1 -d' ') + +$HOME/.ssh_agent: + ssh-agent -s > $HOME/.ssh_agent + source $HOME/.ssh_agent ; ssh-add + grep -q ssh_agent ~/.bashrc_teuthology || echo 'source $HOME/.ssh_agent' >> ~/.bashrc_teuthology + +${D}/packages-repository: + openstack server create --image 'teuthology-ubuntu-14.04' --flavor 'vps-ssd-1' --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 + ip=$(call get_ip,$@) + ssh $$ip sudo apt-get install -y nginx + ssh $$ip sudo chown -R ubuntu /usr/share/nginx/html + perl -pi -e "s/gitbuilder_host.*/gitbuilder_host: $$ip/" ~/.teuthology.yaml + mkdir -p ${@D} ; $touch $@ + +FLAVOR=vps-ssd-1 + +${D}/packages-ceph-deb-${CEPH_DIST}-${CEPH_ARCH}-basic/${CEPH_REF}: ${D}/packages-repository + openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}' --flavor ${FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --wait $@ ; sleep 30 + ip=$(call get_ip,$@) + ssh $$ip sudo apt-get install -y git + scp make-deb.sh $$ip: + ssh -A $$ip make-debs.sh /opt/release $(call get_ip,$<) + openstack server delete $@ + mkdir -p ${@D} ; $touch $@ diff --git a/tasks/buildpackages/make-debs.sh b/tasks/buildpackages/make-debs.sh new file mode 100755 index 0000000000000..46e703ef2cd6b --- /dev/null +++ b/tasks/buildpackages/make-debs.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# Copyright (C) 2015 Red Hat +# +# Author: Loic Dachary +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Library Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library Public License for more details. +# +set -xe + +base=${1:-/tmp/release} +gitbuilder_host=$2 + +codename=$(lsb_release -sc) +releasedir=$base/$(lsb_release -si)/WORKDIR +# +# git describe provides a version that is +# a) human readable +# b) is unique for each commit +# c) compares higher than any previous commit +# d) contains the short hash of the commit +# +vers=$(git describe --match "v*" | sed s/^v//) +# +# always set the debian version to 1 which is ok because the debian +# directory is included in the sources and the upstream version will +# change each time it is modified. +# +dvers="$vers-1" +sha1=$(git rev-parse HEAD) +: ${NPROC:=$(nproc)} +ceph_dir=$(pwd) + +function build_package() { + + rm -fr $releasedir + mkdir -p $releasedir + # + # remove all files not under git so they are not + # included in the distribution. + # + git clean -dxf + # + # creating the distribution tarbal requires some configure + # options (otherwise parts of the source tree will be left out). + # + ./autogen.sh + ./configure --with-rocksdb --with-ocf \ + --with-nss --with-debug --enable-cephfs-java \ + --with-lttng --with-babeltrace + # + # use distdir= to set the name of the top level directory of the + # tarbal to match the desired version + # + make distdir=ceph-$vers dist + # + # rename the tarbal to match debian conventions and extract it + # + mv ceph-$vers.tar.gz $releasedir/ceph_$vers.orig.tar.gz + tar -C $releasedir -zxf $releasedir/ceph_$vers.orig.tar.gz + # + # copy the debian directory over + # + cp -a debian $releasedir/ceph-$vers/debian + cd $releasedir + # + # uncomment to remove -dbg packages + # because they are large and take time to build + # + #perl -ni -e 'print if(!(/^Package: .*-dbg$/../^$/))' ceph-$vers/debian/control + #perl -pi -e 's/--dbg-package.*//' ceph-$vers/debian/rules + # + # update the changelog to match the desired version + # + cd ceph-$vers + local chvers=$(head -1 debian/changelog | perl -ne 's/.*\(//; s/\).*//; print') + if [ "$chvers" != "$dvers" ]; then + DEBEMAIL="contact@ceph.com" dch -D $codename --force-distribution -b -v "$dvers" "new version" + fi + # + # create the packages + # a) with ccache to speed things up when building repeatedly + # b) do not sign the packages + # c) use half of the available processors + # + if test $NPROC -gt 1 ; then + j=-j${NPROC} + fi + PATH=/usr/lib/ccache:$PATH dpkg-buildpackage $j -uc -us +} + +function build_repo() { + local gitbuilder_host=$1 + + cd ${releasedir}/.. + # + # Create a repository in a directory with a name structured + # as + # + arch=x86_64 + base=ceph-deb-$codename-$arch-basic + sha1_dir=$codename/$base/sha1/$sha1 + mkdir -p $sha1_dir/conf + cat > $sha1_dir/conf/distributions < $sha1_dir/version + echo $sha1 > $sha1_dir/sha1 + ref_dir=$codename/$base/ref + mkdir -p $ref_dir + ( cd ${ceph_dir} ; git for-each-ref refs/tags/** refs/heads/** ) | grep $sha1 | while read sha1 type ref ; do + base_ref=$(basename $ref) + ( cd $ref_dir ; ln -sf ../sha1/$sha1 $base_ref ) + done + if test "$gitbuilder_host" ; then + cd $codename + RSYNC_RSH='ssh -o StrictHostKeyChecking=false' rsync -av $base/ $gitbuilder_host:/usr/share/nginx/html/$base/ + fi +} + +build_package +build_repo $gitbuilder_host -- 2.39.5