From: Boris Ranto <branto@redhat.com>
Date: Wed, 28 Sep 2016 12:30:15 +0000 (+0200)
Subject: diamond: Add debian support
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bff35839c6a052d0f0dab9f0c734356af83b360f;p=ceph-build.git

diamond: Add debian support

Signed-off-by: Boris Ranto <branto@redhat.com>
---

diff --git a/diamond-build/build/build_deb b/diamond-build/build/build_deb
new file mode 100644
index 00000000..ac7d70df
--- /dev/null
+++ b/diamond-build/build/build_deb
@@ -0,0 +1,118 @@
+#!/bin/bash
+set -ex
+
+# Only do actual work when we are a DEB distro
+if test -f /etc/redhat-release ; then
+    exit 0
+fi
+
+cd $WORKSPACE
+
+get_bptag() {
+    dist=$1
+
+    [ "$dist" = "sid" ] && dver=""
+    [ "$dist" = "jessie" ] && dver="~bpo80+1"
+    [ "$dist" = "wheezy" ] && dver="~bpo70+1"
+    [ "$dist" = "squeeze" ] && dver="~bpo60+1"
+    [ "$dist" = "lenny" ] && dver="~bpo50+1"
+    [ "$dist" = "xenial" ] && dver="$dist"
+    [ "$dist" = "trusty" ] && dver="$dist"
+    [ "$dist" = "saucy" ] && dver="$dist"
+    [ "$dist" = "precise" ] && dver="$dist"
+    [ "$dist" = "oneiric" ] && dver="$dist"
+    [ "$dist" = "natty" ] && dver="$dist"
+    [ "$dist" = "maverick" ] && dver="$dist"
+    [ "$dist" = "lucid" ] && dver="$dist"
+    [ "$dist" = "karmic" ] && dver="$dist"
+
+    echo $dver
+}
+
+
+# slap -rc to the ref if we are doing a release-candidate build
+chacra_ref="$BRANCH"
+[ "$RC" = true ] && chacra_ref="$BRANCH-rc"
+[ "$TEST" = true ] && chacra_ref="test"
+
+ARCH=$(dpkg-architecture -qDEB_BUILD_ARCH)
+DISTRO=""
+case $DIST in
+    jessie|wheezy)
+        DISTRO="debian"
+        ;;
+    *)
+        DISTRO="ubuntu"
+        ;;
+esac
+
+debian_version=${VERSION}-1
+
+gen_debian_version() {
+    raw=$1
+    dist=$2
+
+    [ "$dist" = "sid" ] && dver="$raw"
+    [ "$dist" = "jessie" ] && dver="$raw~bpo80+1"
+    [ "$dist" = "wheezy" ] && dver="$raw~bpo70+1"
+    [ "$dist" = "squeeze" ] && dver="$raw~bpo60+1"
+    [ "$dist" = "lenny" ] && dver="$raw~bpo50+1"
+    [ "$dist" = "precise" ] && dver="$raw$dist"
+    [ "$dist" = "saucy" ] && dver="$raw$dist"
+    [ "$dist" = "trusty" ] && dver="$raw$dist"
+    [ "$dist" = "xenial" ] && dver="$raw$dist"
+
+    echo $dver
+}
+
+BPVER=`gen_debian_version $debian_version $DIST`
+
+chacra_endpoint="diamond/${BRANCH}/${SHA1}/${DISTRO}/${DIST}"
+chacra_check_url="${chacra_endpoint}/diamond_${BPVER}_${ARCH}.deb"
+
+if [ "$THROWAWAY" = false ] ; then
+    # this exists in scripts/build_utils.sh
+    check_binary_existence $chacra_check_url
+fi
+
+HOST=$(hostname --short)
+echo "Building on $(hostname)"
+echo "  DIST=${DIST}"
+echo "  ARCH=${ARCH}"
+echo "  WS=$WORKSPACE"
+echo "  PWD=$(pwd)"
+echo "*****"
+env
+echo "*****"
+
+# Use pbuilder
+echo "Building debs"
+
+pbuilddir="/srv/debian-base"
+
+sudo pbuilder --clean
+
+mkdir -p dist/deb
+
+echo "Building debs for $DIST"
+sudo pbuilder build \
+    --distribution $DIST \
+    --basetgz $pbuilddir/$DIST.tgz \
+    --buildresult dist/deb/ \
+    --debbuildopts "-j`grep -c processor /proc/cpuinfo`" \
+    dist/diamond_$VERSION.dsc
+
+# Make sure we execute at the top level directory
+cd "$WORKSPACE"
+
+[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags=""
+
+if [ "$THROWAWAY" = false ] ; then
+    # push binaries to chacra
+    find dist/deb/ | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH}/
+
+    # start repo creation
+    $VENV/chacractl repo update ${chacra_endpoint}
+
+    echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}
+fi
diff --git a/diamond-build/build/build_rpm b/diamond-build/build/build_rpm
index 423c9073..c63a62a8 100644
--- a/diamond-build/build/build_rpm
+++ b/diamond-build/build/build_rpm
@@ -1,6 +1,7 @@
 #!/bin/bash
 set -ex
 
+# Only do actual work when we are an RPM distro
 if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then
     exit 0
 fi
@@ -71,7 +72,6 @@ echo "  DIST=${DIST}"
 echo "  ARCH=${ARCH}"
 echo "  WS=$WORKSPACE"
 echo "  PWD=$(pwd)"
-echo "  BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP"
 echo "*****"
 env
 echo "*****"
@@ -104,7 +104,6 @@ cd "$WORKSPACE"
 
 if [ "$THROWAWAY" = false ] ; then
     # push binaries to chacra
-    echo "$SRPM" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source/
     find dist/rpm/$DIST/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/
 
     # start repo creation
diff --git a/diamond-build/build/setup_pbuilder b/diamond-build/build/setup_pbuilder
new file mode 100755
index 00000000..0e352c74
--- /dev/null
+++ b/diamond-build/build/setup_pbuilder
@@ -0,0 +1,91 @@
+#!/bin/sh -x
+# This file will set the tgz images needed for pbuilder on a given host. It has
+# some hard-coded values like `/srv/debian-base` because it gets built every
+# time this file is executed - completely ephemeral.  If a Debian host will use
+# pbuilder, then it will need this. Since it is not idempotent it makes
+# everything a bit slower. ## FIXME ##
+
+set -e
+
+# Only run when we are a Debian or Debian-based distro
+if test -f /etc/redhat-release ; then
+    exit 0
+fi
+
+basedir="/srv/debian-base"
+
+# Ensure that the basedir directory exists
+sudo mkdir -p "$basedir"
+
+# This used to live in a *file* on /srv/ceph-build as
+# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make
+# sense to have a file that lives in /srv/ that we then concatenate to get its
+# contents.  what.
+# By using $DIST we are narrowing down to updating only the distro image we
+# need, unlike before where we updated everything on every server on every
+# build.
+
+os="debian"
+[ "$DIST" = "precise" ] && os="ubuntu"
+[ "$DIST" = "saucy" ] && os="ubuntu"
+[ "$DIST" = "trusty" ] && os="ubuntu"
+[ "$DIST" = "xenial" ] && os="ubuntu"
+
+if [ $os = "debian" ]; then
+    mirror="http://www.gtlib.gatech.edu/pub/debian"
+    # this assumes that newer Debian releases are being added to
+    # /etc/apt/trusted.gpg that is also the default location for Ubuntu trusted
+    # keys. The slave should ensure that the needed keys are added accordingly
+    # to this location.
+    debootstrapopts='DEBOOTSTRAPOPTS=( "--keyring" "/etc/apt/trusted.gpg" )'
+    components='COMPONENTS="main contrib"'
+elif [ "$ARCH" = "arm64" ]; then
+    mirror="http://ports.ubuntu.com/ubuntu-ports"
+    debootstrapopts=""
+    components='COMPONENTS="main universe"'
+else
+    mirror="http://us.archive.ubuntu.com/ubuntu"
+    debootstrapopts=""
+    components='COMPONENTS="main universe"'
+fi
+
+# ensure that the tgz is valid, otherwise remove it so that it can be recreated
+# again
+pbuild_tar="$basedir/$DIST.tgz"
+is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"`
+file_size_kb=`du -k "$pbuild_tar" | cut -f1`
+
+if $is_not_tar; then
+    sudo rm -f "$pbuild_tar"
+fi
+
+if [ $file_size_kb -lt 1 ]; then
+    sudo rm -f "$pbuild_tar"
+fi
+
+# Ordinarily pbuilder only pulls packages from "main".  ceph depends on
+# packages like python-virtualenv which are in "universe". We have to configure
+# pbuilder to look in "universe". Otherwise the build would fail with a message similar
+# to:
+#    The following packages have unmet dependencies:
+#      pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package.
+#                                      Depends: xmlstarlet which is a virtual package.
+#     Unable to resolve dependencies!  Giving up...
+echo "$components" > ~/.pbuilderrc
+echo "$debootstrapopts" >> ~/.pbuilderrc
+
+sudo pbuilder --clean
+
+if [ -e $basedir/$DIST.tgz ]; then
+    echo updating $DIST base.tgz
+    sudo pbuilder update \
+    --basetgz $basedir/$DIST.tgz \
+    --distribution $DIST \
+    --mirror "$mirror"
+else
+    echo building $DIST base.tgz
+    sudo pbuilder create \
+    --basetgz $basedir/$DIST.tgz \
+    --distribution $DIST \
+    --mirror "$mirror"
+fi
diff --git a/diamond-build/build/validate_deb b/diamond-build/build/validate_deb
new file mode 100644
index 00000000..0322b47a
--- /dev/null
+++ b/diamond-build/build/validate_deb
@@ -0,0 +1,7 @@
+#!/bin/bash
+set -ex
+
+# Only do actual work when we are a DEB distro
+if test -f /etc/redhat-release ; then
+    exit 0
+fi
diff --git a/diamond-build/config/definitions/diamond-build.yml b/diamond-build/config/definitions/diamond-build.yml
index 05eed809..5ffb067f 100644
--- a/diamond-build/config/definitions/diamond-build.yml
+++ b/diamond-build/config/definitions/diamond-build.yml
@@ -29,6 +29,14 @@
           values:
             - centos6
             - centos7
+            - trusty
+            - xenial
+            - jessie
+            - precise
+            - wheezy
+      - axis:
+          type: dynamic
+          name: DIST
       - axis:
           type: dynamic
           name: DIST
@@ -58,6 +66,14 @@
           properties-file: ${WORKSPACE}/dist/branch
       - inject:
           properties-file: ${WORKSPACE}/dist/version
+      # debian build scripts
+      - shell:
+          !include-raw:
+            - ../../build/validate_deb
+            - ../../../scripts/build_utils.sh
+            - ../../build/setup
+            - ../../build/setup_pbuilder
+            - ../../build/build_deb
       # rpm build scripts
       - shell:
           !include-raw:
diff --git a/diamond-setup/build/build b/diamond-setup/build/build
index 603310bb..40eb6741 100644
--- a/diamond-setup/build/build
+++ b/diamond-setup/build/build
@@ -1,5 +1,6 @@
 #!/bin/bash -ex
 
+## Get the basic setup/info
 HOST=$(hostname --short)
 echo "Building on ${HOST}"
 echo "  DIST=${DIST}"
@@ -16,30 +17,47 @@ fi
 
 echo "Building version $(git describe) Branch $BRANCH"
 
+## Make sure the repo is clean
+# Remove all untracked files
+echo "Cleaning up the repo"
+git clean -fxd
+
 # Make sure the dist directory is clean
 rm -rf dist
 mkdir -p dist
 
-# Install any setup-time deps
+## Install any setup-time deps
+# We need this for mk-build-deps
+sudo apt-get install equivs
+# Run the install-deps.sh upstream script if it exists
 if [ -x install-deps.sh ]; then
   echo "Ensuring dependencies are installed"
   ./install-deps.sh
 fi
 
-# Get the version
+## Get the version
 VERSION=$(./version.sh)
 
-# Make sure the repo is clean
-echo "Cleaning up the repo"
-git clean -fxd
-
+## Build the source tarball
 echo "Building source distribution"
 python setup.py sdist
 
-# Prepare the spec file for build
+## Prepare the spec file for build
 sed -e "s/@VERSION@/${VERSION}/g" < diamond.spec.in > dist/diamond.spec
 
-# Save these so that we can later inject them into the build script
+## Prepare the debian files
+# Bump the changelog
+dch -v "$VERSION" "New release ($VERSION)"
+
+# Install build-time dependencies
+yes | sudo mk-build-deps --install debian/control
+
+# Create .dsc and source tarball
+sudo dpkg-buildpackage -S -us -uc
+
+cp ../diamond_$VERSION* dist/deb/
+
+## Save these so that we can later inject them into the build script
 cat > dist/sha1 << EOF
 SHA1=${GIT_COMMIT}
 EOF
diff --git a/diamond/config/definitions/diamond.yml b/diamond/config/definitions/diamond.yml
index f334d4e9..3cb36791 100644
--- a/diamond/config/definitions/diamond.yml
+++ b/diamond/config/definitions/diamond.yml
@@ -25,7 +25,7 @@
       - string:
           name: DISTROS
           description: "A list of distros to build for. Available options are: xenial, centos7, centos6, trusty, precise, wheezy, and jessie"
-          default: "centos6 centos7"
+          default: "centos7 trusty"
 
       - string:
           name: ARCHS