From ae4791b41b9917f1e1823789b1d65a9d3525204c Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Mon, 21 Mar 2016 20:38:49 -0700 Subject: [PATCH] ceph-build-arm64: Test for arm64 build, separate from ceph-build Some things want to be different in this build; let's keep them away from the main ceph build until we're sure they can coexist peacefully. Signed-off-by: Dan Mick --- ceph-build-arm64/build/build_deb | 173 ++++++++++++++++++ ceph-build-arm64/build/build_rpm | 119 ++++++++++++ ceph-build-arm64/build/setup | 73 ++++++++ ceph-build-arm64/build/setup_pbuilder | 80 ++++++++ .../config/definitions/ceph-build.yml | 56 ++++++ 5 files changed, 501 insertions(+) create mode 100644 ceph-build-arm64/build/build_deb create mode 100644 ceph-build-arm64/build/build_rpm create mode 100644 ceph-build-arm64/build/setup create mode 100755 ceph-build-arm64/build/setup_pbuilder create mode 100644 ceph-build-arm64/config/definitions/ceph-build.yml diff --git a/ceph-build-arm64/build/build_deb b/ceph-build-arm64/build/build_deb new file mode 100644 index 00000000..4974da63 --- /dev/null +++ b/ceph-build-arm64/build/build_deb @@ -0,0 +1,173 @@ +#!/bin/bash +set -ex + +# Only do actual work when we are a DEB distro +if test -f /etc/redhat-release ; then + exit 0 +fi + +VENV="$WORKSPACE/venv/bin" + +get_bptag() { + dist=$1 + + [ "$dist" = "sid" ] && dver="" + [ "$dist" = "jessie" ] && dver="~bpo80+1" + [ "$dist" = "wheezy" ] && dver="~bpo70+1" + [ "$dist" = "squeeze" ] && dver="~bpo60+1" + [ "$dist" = "lenny" ] && dver="~bpo50+1" + [ "$dist" = "trusty" ] && dver="$dist" + [ "$dist" = "saucy" ] && dver="$dist" + [ "$dist" = "precise" ] && dver="$dist" + [ "$dist" = "oneiric" ] && dver="$dist" + [ "$dist" = "natty" ] && dver="$dist" + [ "$dist" = "maverick" ] && dver="$dist" + [ "$dist" = "lucid" ] && dver="$dist" + [ "$dist" = "karmic" ] && dver="$dist" + + echo $dver +} + +BPTAG=`get_bptag $DIST` + +# slap -rc to the ref if we are doing a release-candidate build +[ "$RC" = true ] && chacra_ref="$BRANCH-rc" || chacra_ref="$BRANCH" +[ "$TEST" = true ] && chacra_ref="test" +vers=`cat ./dist/version` +distro=`python -c "exec 'import platform; print platform.linux_distribution()[0].lower()'"` +chacra_endpoint="ceph/${chacra_ref}/${distro}/${DIST}/${ARCH}" + +# this exists in scripts/build_utils.sh +check_binary_existence $chacra_endpoint + +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + +# create a release directory for ceph-build tools +mkdir -p release +cp -a dist release/${vers} +echo $DIST > release/${vers}/debian_dists +echo "${vers}-1" > release/${vers}/debian_version + +cd release/$vers + + +# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK +# FIXME: I don't think we need this 'hack' anymore +# Dirty Hack: +baddist=$(echo $DIST | grep -ic -e squeeze -e wheezy || true) +if [ $baddist -eq 1 ] +then + sed -i 's/ libbabeltrace-ctf-dev, libbabeltrace-dev,//g' ceph_${vers}-1.dsc || true + sed -i 's/ liblttng-ust-dev//g' ceph_${vers}-1.dsc || true + +fi +# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK + + +# unpack sources +dpkg-source -x ceph_${vers}-1.dsc + + +# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK +if [ $baddist -eq 1 ] +then + rm -vf *.orig.tar.gz || true + grep -v babeltrace ceph-${vers}/debian/control | grep -v liblttng > ceph-${vers}/debian/control.new + mv -v ceph-${vers}/debian/control.new ceph-${vers}/debian/control +fi +# HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK HACK + + +( cd ceph-${vers} + DEB_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1, p') + BP_VERSION=${DEB_VERSION}${BPTAG} + dch -D $DIST --force-distribution -b -v "$BP_VERSION" "$comment" +) +dpkg-source -b ceph-${vers} + +echo "Building Debian" +cd "$WORKSPACE" +# Before, at this point, this script called the below contents that +# was part of /srv/ceph-buid/build_debs.sh. Now everything is in here, in one +# place, no need to checkout/clone anything. WYSIWYG:: +# +# sudo $bindir/build_debs.sh ./release /srv/debian-base $vers + + +releasedir="./release" +pbuilddir="/srv/debian-base" +cephver=$vers + +echo version $cephver + +# This used to live in a *file* on /src/ceph-build. Now it lives here because +# it doesn't make sense to have a file that lives in /srv/ that we then +# concatenate to get its contents. + +dvers=`cat $releasedir/$cephver/debian_version` +echo deb vers $dvers + +# This function used to live as /srv/gen_debian_version.sh +gen_debian_version() { + raw=$1 + dist=$2 + + [ "$dist" = "sid" ] && dver="$raw" + [ "$dist" = "jessie" ] && dver="$raw~bpo80+1" + [ "$dist" = "wheezy" ] && dver="$raw~bpo70+1" + [ "$dist" = "squeeze" ] && dver="$raw~bpo60+1" + [ "$dist" = "lenny" ] && dver="$raw~bpo50+1" + [ "$dist" = "precise" ] && dver="$raw$dist" + [ "$dist" = "saucy" ] && dver="$raw$dist" + [ "$dist" = "trusty" ] && dver="$raw$dist" + [ "$dist" = "xenial" ] && dver="$raw$dist" + + echo $dver +} + +# FIXME this looks exactly like `setup_pbuilder`, we probably don't need this +# or we need to refactor. +sudo pbuilder --clean + +bpvers=`gen_debian_version $dvers $DIST` +echo deb vers $bpvers + + +echo building debs for $DIST +if [ `dpkg-architecture -qDEB_BUILD_ARCH` = "i386" ] ; then + # Architecture dependent, independent and source + sudo pbuilder build \ + --distribution $DIST \ + --basetgz $pbuilddir/$DIST.tgz \ + --buildresult $releasedir/$cephver \ + --debbuildopts "-j`grep -c processor /proc/cpuinfo`" \ + $releasedir/$cephver/ceph_$bpvers.dsc +else + # Binary only architecture dependent + sudo pbuilder build \ + --binary-arch \ + --distribution $DIST \ + --basetgz $pbuilddir/$DIST.tgz \ + --buildresult $releasedir/$cephver \ + --debbuildopts "-j`grep -c processor /proc/cpuinfo`" \ + $releasedir/$cephver/ceph_$bpvers.dsc +fi + +# do lintian checks +bpvers=`gen_debian_version $dvers $DIST` +echo lintian checks for $bpvers +echo lintian --allow-root $releasedir/$cephver/*$bpvers*.deb + +[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + +# push binaries to chacra +# find release/$vers/ | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} diff --git a/ceph-build-arm64/build/build_rpm b/ceph-build-arm64/build/build_rpm new file mode 100644 index 00000000..928ed051 --- /dev/null +++ b/ceph-build-arm64/build/build_rpm @@ -0,0 +1,119 @@ +#!/bin/bash +set -ex + +if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then + exit 0 +fi + +VENV="$WORKSPACE/venv/bin" + +get_rpm_dist() { + LSB_RELEASE=/usr/bin/lsb_release + [ ! -x $LSB_RELEASE ] && echo unknown && exit + + ID=`$LSB_RELEASE --short --id` + + case $ID in + RedHatEnterpriseServer) + RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` + DIST=rhel$RELEASE + DISTRO=rhel + ;; + CentOS) + RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` + DIST=el$RELEASE + DISTRO=centos + ;; + Fedora) + RELEASE=`$LSB_RELEASE --short --release` + DIST=fc$RELEASE + DISTRO=fedora + ;; + SUSE\ LINUX) + DESC=`$LSB_RELEASE --short --description` + RELEASE=`$LSB_RELEASE --short --release` + case $DESC in + *openSUSE*) + DIST=opensuse$RELEASE + DISTRO=opensuse + ;; + *Enterprise*) + DIST=sles$RELEASE + DISTRO=sles + ;; + esac + ;; + *) + DIST=unknown + DISTRO=unknown + ;; + esac + + echo $DIST +} + +get_rpm_dist +dist=$DIST +[ -z "$dist" ] && echo no dist && exit 1 +echo dist $dist + +vers=`cat ./dist/version` +# slap -rc to the ref if we are doing a release-candidate build +[ "$RC" = true ] && chacra_ref="$BRANCH-rc" || chacra_ref="$BRANCH" +[ "$TEST" = true ] && chacra_ref="test" +chacra_baseurl="ceph/${chacra_ref}/${DISTRO}/${RELEASE}" + +# this exists in scripts/build_utils.sh +check_binary_existence $chacra_baseurl/$ARCH + +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + + +# create a release directory for ceph-build tools +mkdir -p release +cp -a dist release/${vers} +echo $DIST > release/${vers}/debian_dists +echo "${vers}-1" > release/${vers}/debian_version + +echo "Building RPMs" + +# The below contents ported from /srv/ceph-build/build_rpms.sh :: +# $bindir/build_rpms.sh ./release $vers +# + +releasedir="./release" +cephver=$vers + +cd $releasedir/$cephver || exit 1 + +# Set up build area +BUILDAREA=./rpm/$dist +mkdir -p ${BUILDAREA}/{SOURCES,SRPMS,SPECS,RPMS,BUILD} +cp -a ceph-*.tar.bz2 ${BUILDAREA}/SOURCES/. +cp -a ceph.spec ${BUILDAREA}/SPECS/. +cp -a rpm/*.patch ${BUILDAREA}/SOURCES/. || true + +# Build RPMs +BUILDAREA=`readlink -fn ${BUILDAREA}` ### rpm wants absolute path +cd ${BUILDAREA}/SPECS +rpmbuild -ba --define "_topdir ${BUILDAREA}" ceph.spec + +echo done + +# Make sure we execute at the top level directory +cd "$WORKSPACE" + +[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + +# push binaries to chacra +find release/${vers}/rpm/*/SRPMS | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_baseurl}/source +find release/${vers}/rpm/*/RPMS/* | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_baseurl}/${ARCH} diff --git a/ceph-build-arm64/build/setup b/ceph-build-arm64/build/setup new file mode 100644 index 00000000..388252bd --- /dev/null +++ b/ceph-build-arm64/build/setup @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Ceph distributed storage system +# +# Copyright (C) 2014 Red Hat +# +# Author: Loic Dachary +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +set -ex +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " KEYID=${KEYID}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + +DIR=/tmp/install-deps.$$ +trap "rm -fr $DIR" EXIT +mkdir -p $DIR +if test $(id -u) != 0 ; then + SUDO=sudo +fi +export LC_ALL=C # the following is vulnerable to i18n + +if test -f /etc/redhat-release ; then + $SUDO yum install -y redhat-lsb-core +fi + +if which apt-get > /dev/null ; then + $SUDO apt-get install -y lsb-release +fi + +# unpack the tar.gz that contains the debian dir +cd dist +tar xzf *.orig.tar.gz +cd ceph-* +pwd + + + +case $(lsb_release -si) in +CentOS|Fedora|SUSE*|RedHatEnterpriseServer) + case $(lsb_release -si) in + SUSE*) + $SUDO zypper -y yum-utils + ;; + *) + $SUDO yum install -y yum-utils + ;; + esac + sed -e 's/@//g' < ceph.spec.in > $DIR/ceph.spec + $SUDO yum-builddep -y $DIR/ceph.spec + ;; +*) + echo "$(lsb_release -si) is unknown, dependencies will have to be installed manually." + ;; +esac + +pkgs=( "chacractl>=0.0.4" ) +install_python_packages "pkgs[@]" + +# create the .chacractl config file using global variables +make_chacractl_config diff --git a/ceph-build-arm64/build/setup_pbuilder b/ceph-build-arm64/build/setup_pbuilder new file mode 100755 index 00000000..d9985755 --- /dev/null +++ b/ceph-build-arm64/build/setup_pbuilder @@ -0,0 +1,80 @@ +#!/bin/sh -x +# This file will set the tgz images needed for pbuilder on a given host. It has +# some hard-coded values like `/srv/debian-base` because it gets built every +# time this file is executed - completely ephemeral. If a Debian host will use +# pbuilder, then it will need this. Since it is not idempotent it makes +# everything a bit slower. ## FIXME ## + +set -e + +# Only run when we are a Debian or Debian-based distro +if test -f /etc/redhat-release ; then + exit 0 +fi + +basedir="/srv/debian-base" + +# Ensure that the basedir directory exists +sudo mkdir -p "$basedir" + +# This used to live in a *file* on /srv/ceph-build as +# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make +# sense to have a file that lives in /srv/ that we then concatenate to get its +# contents. what. +# By using $DIST we are narrowing down to updating only the distro image we +# need, unlike before where we updated everything on every server on every +# build. + +os="debian" +[ "$DIST" = "precise" ] && os="ubuntu" +[ "$DIST" = "saucy" ] && os="ubuntu" +[ "$DIST" = "trusty" ] && os="ubuntu" +[ "$DIST" = "xenial" ] && os="ubuntu" + +if [ $os = "debian" ]; then + mirror="http://www.gtlib.gatech.edu/pub/debian" +elif [ "$ARCH" = "arm64" ]; then + mirror="http://ports.ubuntu.com/ubuntu-ports" +else + mirror="http://us.archive.ubuntu.com/ubuntu" +fi + +# ensure that the tgz is valid, otherwise remove it so that it can be recreated +# again +pbuild_tar="$basedir/$DIST.tgz" +is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"` +file_size_kb=`du -k "$pbuild_tar" | cut -f1` + +if $is_not_tar; then + sudo rm -f "$pbuild_tar" +fi + +if [ $file_size_kb -lt 1 ]; then + sudo rm -f "$pbuild_tar" +fi + +# Ordinarily pbuilder only pulls packages from "main". ceph depends on +# packages like python-virtualenv which are in "universe". We have to configure +# pbuilder to look in "universe". Otherwise the build would fail with a message similar +# to: +# The following packages have unmet dependencies: +# pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package. +# Depends: xmlstarlet which is a virtual package. +# Unable to resolve dependencies! Giving up... +echo 'COMPONENTS="main universe"' > ~/.pbuilderrc + +sudo pbuilder --clean + +if [ -e $basedir/$DIST.tgz ]; then + echo updating $DIST base.tgz + sudo pbuilder update \ + --basetgz $basedir/$DIST.tgz \ + --distribution $DIST \ + --mirror "$mirror" +else + echo building $DIST base.tgz + sudo pbuilder create \ + --basetgz $basedir/$DIST.tgz \ + --distribution $DIST \ + --mirror "$mirror" +fi diff --git a/ceph-build-arm64/config/definitions/ceph-build.yml b/ceph-build-arm64/config/definitions/ceph-build.yml new file mode 100644 index 00000000..127d1593 --- /dev/null +++ b/ceph-build-arm64/config/definitions/ceph-build.yml @@ -0,0 +1,56 @@ +- job: + name: ceph-build-arm64 + project-type: matrix + disabled: true + defaults: global + display-name: 'ceph-build-arm64' + block-downstream: false + block-upstream: false + properties: + - github: + url: https://github.com/ceph/ceph + execution-strategy: + combination-filter: (DIST == "xenial" && ARCH == "arm64") + axes: + - axis: + type: label-expression + name: ARCH + values: + - arm64 + - axis: + type: label-expression + name: DIST + values: + - xenial + + builders: + - shell: | + echo "Cleaning up top-level workarea (shared among workspaces)" + rm -rf dist + rm -rf venv + rm -rf release + - copyartifact: + project: ceph-setup + filter: 'dist/**' + which-build: last-successful + # general setup + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + # debian build scripts + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup_pbuilder + - ../../build/build_deb + # rpm build scripts + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/build_rpm + + wrappers: + - inject-passwords: + global: true + mask-password-params: true -- 2.39.5