From: Boris Ranto Date: Mon, 31 Oct 2016 21:16:19 +0000 (+0100) Subject: calamari: Simplify to add debian build X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F530%2Fhead;p=ceph-build.git calamari: Simplify to add debian build The debian build does not currently support creating source packages so we need to merge the build and setup jobs into a single job. This patch also simplifies various other bits and adds the support for the debian build. Signed-off-by: Boris Ranto --- diff --git a/calamari-build/build/build_rpm b/calamari-build/build/build_rpm deleted file mode 100644 index 098775ab..00000000 --- a/calamari-build/build/build_rpm +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -ex - -if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then - exit 0 -fi - -cd $WORKSPACE - -get_rpm_dist() { - LSB_RELEASE=/usr/bin/lsb_release - [ ! -x $LSB_RELEASE ] && echo unknown && exit - - ID=`$LSB_RELEASE --short --id` - - case $ID in - RedHatEnterpriseServer) - RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` - DIST=rhel$RELEASE - DISTRO=rhel - ;; - CentOS) - RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` - DIST=el$RELEASE - DISTRO=centos - ;; - Fedora) - RELEASE=`$LSB_RELEASE --short --release` - DIST=fc$RELEASE - DISTRO=fedora - ;; - SUSE\ LINUX) - DESC=`$LSB_RELEASE --short --description` - RELEASE=`$LSB_RELEASE --short --release` - case $DESC in - *openSUSE*) - DIST=opensuse$RELEASE - DISTRO=opensuse - ;; - *Enterprise*) - DIST=sles$RELEASE - DISTRO=sles - ;; - esac - ;; - *) - DIST=unknown - DISTRO=unknown - ;; - esac - - echo $DIST -} - -get_rpm_dist -dist=$DIST -[ -z "$dist" ] && echo no dist && exit 1 -echo dist $dist - -chacra_endpoint="calamari/${BRANCH}/${SHA1}/${DISTRO}/${RELEASE}" -chacra_check_url="${chacra_endpoint}/${ARCH}/calamari-server-${VERSION}-${RPM_RELEASE}.${DIST}.${ARCH}.rpm" - -if [ "$THROWAWAY" = false ] ; then - # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url -fi - -HOST=$(hostname --short) -echo "Building on $(hostname)" -echo " DIST=${DIST}" -echo " ARCH=${ARCH}" -echo " WS=$WORKSPACE" -echo " PWD=$(pwd)" -echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" -echo "*****" -env -echo "*****" - -# Install the dependencies -sudo yum-builddep -y dist/calamari-server.spec - -# Create the source rpm -echo "Building SRPM" -rpmbuild \ - --define "_sourcedir ./dist" \ - --define "_specdir ." \ - --define "_builddir ." \ - --define "_srcrpmdir ." \ - --define "_rpmdir ." \ - --define "dist .any" \ - --define "fedora 21" \ - --define "rhel 7" \ - --nodeps -bs dist/calamari-server.spec -SRPM=$(readlink -f *.src.rpm) - -# Build the binaries -echo "Building RPMs" -sudo mock -r epel-${RELEASE}-${ARCH} --resultdir=./dist/rpm/"%(dist)s"/"%(target_arch)s"/ ${SRPM} - -# Make sure we execute at the top level directory -cd "$WORKSPACE" - -[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" - -if [ "$THROWAWAY" = false ] ; then - # push binaries to chacra - echo "$SRPM" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source/ - find dist/rpm/$DIST/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/ - - # start repo creation - $VENV/chacractl repo update ${chacra_endpoint} - - echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} -fi diff --git a/calamari-build/build/setup b/calamari-build/build/setup deleted file mode 100644 index bccd321c..00000000 --- a/calamari-build/build/setup +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# -# Ceph distributed storage system -# -# Copyright (C) 2016 Red Hat -# -# Author: Boris Ranto -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -set -ex -HOST=$(hostname --short) -echo "Building on $(hostname)" -echo " DIST=${DIST}" -echo " BPTAG=${BPTAG}" -echo " KEYID=${KEYID}" -echo " WS=$WORKSPACE" -echo " PWD=$(pwd)" -echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" -echo "*****" -env -echo "*****" - -if test $(id -u) != 0 ; then - SUDO=sudo -fi -export LC_ALL=C # the following is vulnerable to i18n - -if test -f /etc/redhat-release ; then - $SUDO yum install -y redhat-lsb-core -fi - -if which apt-get > /dev/null ; then - $SUDO apt-get install -y lsb-release -fi - -case $(lsb_release -si) in -CentOS|Fedora|SUSE*|RedHatEnterpriseServer) - case $(lsb_release -si) in - SUSE*) - $SUDO zypper -y yum-utils - ;; - *) - $SUDO yum install -y yum-utils mock - ;; - esac - ;; -*) - echo "$(lsb_release -si) is unknown, dependencies will have to be installed manually." - ;; -esac - -pkgs=( "chacractl>=0.0.4" ) -install_python_packages "pkgs[@]" - -# ask shaman which chacra instance to use -chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` -# create the .chacractl config file using global variables -make_chacractl_config $chacra_url diff --git a/calamari-build/build/validate_rpm b/calamari-build/build/validate_rpm deleted file mode 100644 index 17b41482..00000000 --- a/calamari-build/build/validate_rpm +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -ex - -# only do work if we are a RPM distro -if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then - exit 0 -fi diff --git a/calamari-build/config/definitions/calamari-build.yml b/calamari-build/config/definitions/calamari-build.yml deleted file mode 100644 index 44bf074a..00000000 --- a/calamari-build/config/definitions/calamari-build.yml +++ /dev/null @@ -1,72 +0,0 @@ -- job: - name: calamari-build - project-type: matrix - defaults: global - display-name: 'calamari-build' - block-downstream: false - block-upstream: false - concurrent: true - properties: - - github: - url: https://github.com/ceph/calamari - execution-strategy: - combination-filter: DIST==AVAILABLE_DIST && ARCH==AVAILABLE_ARCH && (ARCH=="x86_64" || (ARCH == "arm64" && (DIST == "xenial" || DIST == "centos7"))) - axes: - - axis: - type: label-expression - name: MACHINE_SIZE - values: - - small - - axis: - type: label-expression - name: AVAILABLE_ARCH - values: - - x86_64 - - arm64 - - axis: - type: label-expression - name: AVAILABLE_DIST - values: - - centos6 - - centos7 - - axis: - type: dynamic - name: DIST - values: - - DISTROS - - axis: - type: dynamic - name: ARCH - values: - - ARCHS - - - - builders: - - shell: | - echo "Cleaning up top-level workarea (shared among workspaces)" - rm -rf dist - rm -rf venv - rm -rf release - - copyartifact: - project: calamari-setup - filter: 'dist/**' - which-build: last-successful - - inject: - properties-file: ${WORKSPACE}/dist/sha1 - - inject: - properties-file: ${WORKSPACE}/dist/branch - - inject: - properties-file: ${WORKSPACE}/dist/version - # rpm build scripts - - shell: - !include-raw: - - ../../build/validate_rpm - - ../../../scripts/build_utils.sh - - ../../build/setup - - ../../build/build_rpm - - wrappers: - - inject-passwords: - global: true - mask-password-params: true diff --git a/calamari-setup/build/build b/calamari-setup/build/build deleted file mode 100644 index 5dfc4ffe..00000000 --- a/calamari-setup/build/build +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -ex - -HOST=$(hostname --short) -echo "Building on ${HOST}" -echo " DIST=${DIST}" -echo " BPTAG=${BPTAG}" -echo " WS=$WORKSPACE" -echo " PWD=$(pwd)" -echo " BRANCH=$BRANCH" -echo " SHA1=$GIT_COMMIT" - -if [ -x "$BRANCH" ] ; then - echo "No git branch was supplied" - exit 1 -fi - -# Make sure the repo is clean -echo "Cleaning up the repo" -git clean -fxd - -# Make sure the dist directory is clean -rm -rf dist -mkdir -p dist - -echo "Building version $(git describe) Branch $BRANCH" - -# Install any setup-time deps -if [ -x install-deps.sh ]; then - echo "Ensuring dependencies are installed" - ./install-deps.sh -fi - -# Get the version -eval $(./get-versions.sh) -RPM_RELEASE=$(echo $REVISION | tr '-' '_') - -# Create the source tarball and move it to dist -echo "Building source distribution" -make dist -mv ../calamari-server_*.tar.gz dist/ - -# Prepare the spec file for build -sed -e "s/@VERSION@/${VERSION}/g" -e "s/@RELEASE@/${RPM_RELEASE}/g" < calamari-server.spec.in > dist/calamari-server.spec - -# Save these so that we can later inject them into the build script -cat > dist/sha1 << EOF -SHA1=${GIT_COMMIT} -EOF - -cat > dist/branch << EOF -BRANCH=${BRANCH} -EOF - -cat > dist/version << EOF -VERSION=${VERSION} -RPM_RELEASE=${RPM_RELEASE} -EOF diff --git a/calamari-setup/config/definitions/calamari-setup.yml b/calamari-setup/config/definitions/calamari-setup.yml deleted file mode 100644 index d9b48c33..00000000 --- a/calamari-setup/config/definitions/calamari-setup.yml +++ /dev/null @@ -1,48 +0,0 @@ -- job: - name: calamari-setup - description: "This job step checks out the branch and builds the tarballs, diffs, and dsc that are passed to the calamari-build step.\r\n\r\nNotes:\r\nJob needs to run on a releatively recent debian system. The Restrict where run feature is used to specifiy an appropriate label.\r\nThe clear workspace before checkout box for the git plugin is used." - # we do not need to pin this to trusty anymore for the new jenkins instance - # FIXME: unpin when this gets ported over - node: small && trusty - display-name: 'calamari-setup' - logrotate: - daysToKeep: -1 - numToKeep: 25 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - block-downstream: false - block-upstream: false - concurrent: true - properties: - - github: - url: https://github.com/ceph/calamari - - parameters: - - string: - name: BRANCH - description: "The git branch (or tag) to build" - - scm: - - git: - url: git@github.com:ceph/calamari.git - # Use the SSH key attached to the ceph-jenkins GitHub account. - credentials-id: '39fa150b-b2a1-416e-b334-29a9a2c0b32d' - branches: - - $BRANCH - skip-tag: true - wipe-workspace: true - - builders: - - shell: - !include-raw ../../build/build - - publishers: - - archive: - artifacts: 'dist/**' - allow-empty: false - latest-only: false - - wrappers: - - inject-passwords: - global: true - mask-password-params: true diff --git a/calamari/build/build_deb b/calamari/build/build_deb new file mode 100644 index 00000000..9f5d2e57 --- /dev/null +++ b/calamari/build/build_deb @@ -0,0 +1,46 @@ +#! /usr/bin/bash +set -ex + +# Only do actual work when we are a DEB distro +if test "$DISTRO" != "debian" -a "$DISTRO" != "ubuntu"; then + exit 0 +fi + + +## Install any setup-time deps +# We need this for mk-build-deps +sudo apt-get install equivs + +# Run the install-deps.sh upstream script if it exists +if [ -x install-deps.sh ]; then + echo "Ensuring dependencies are installed" + ./install-deps.sh +fi + + +## Get some basic information about the system and the repository +DEB_ARCH=$(dpkg-architecture -qDEB_BUILD_ARCH) +# Get VERSION, REVISION +eval $(./get-versions.sh) + + +## Install debian build-time dependencies +yes | sudo mk-build-deps --install debian/control + +## Clean any old and build the new packages +rm -f ../calamari-server*.deb ../calamari-server*.changes +make dpkg + + +## Upload the created DEBs to chacra +chacra_endpoint="calamari/${BRANCH}/${GIT_COMMIT}/${DISTRO}/${DIST}" + +[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + +# push binaries to chacra +find ../calamari-server*.deb ../calamari-server*.changes | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${DEB_ARCH}/ + +# start repo creation +$VENV/chacractl repo update ${chacra_endpoint} + +echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} diff --git a/calamari/build/build_rpm b/calamari/build/build_rpm new file mode 100644 index 00000000..df2c5739 --- /dev/null +++ b/calamari/build/build_rpm @@ -0,0 +1,73 @@ +#! /usr/bin/bash +set -ex + +# Only do actual work when we are an RPM distro +if test "$DISTRO" != "fedora" -a "$DISTRO" != "centos" -a "$DISTRO" != "rhel"; then + exit 0 +fi + + +## Install any setup-time deps (to make dist package) +# We need this to get the major version from lsb_release +yum install -y redhat-lsb-core + +# Run the install-deps.sh upstream script if it exists +if [ -x install-deps.sh ]; then + echo "Ensuring dependencies are installed" + ./install-deps.sh +fi + + +## Get some basic information about the system and the repository +RELEASE="$(lsb_release --short -r | cut -d '.' -f 1)" # sytem release +# Get VERSION, REVISION (and RPM_RELEASE) +eval $(./get-versions.sh) +RPM_RELEASE=$(echo $REVISION | tr '-' '_') # the '-' has a special meaning + + +## Build the source tarball +echo "Building source distribution" +make dist +mv ../calamari-server_*.tar.gz dist/ + + +## Prepare the spec file for build +sed -e "s/@VERSION@/${VERSION}/g" -e "s/@RELEASE@/${RPM_RELEASE}/g" < calamari-server.spec.in > dist/calamari-server.spec + + +## Install the build-time dependencies (for rpmbuild -bs) +sudo yum-builddep -y dist/calamari.spec + + +## Create the source rpm +echo "Building SRPM" +rpmbuild \ + --define "_sourcedir ./dist" \ + --define "_specdir ." \ + --define "_builddir ." \ + --define "_srcrpmdir ." \ + --define "_rpmdir ." \ + --define "dist .any" \ + --define "fedora 21" \ + --define "rhel 7" \ + --nodeps -bs dist/calamari-server.spec +SRPM=$(readlink -f *.src.rpm) + + +## Build the binaries with mock +echo "Building RPMs" +sudo mock -r ${MOCK_TARGET}-${RELEASE}-${ARCH} --resultdir=./dist/rpm/"%(dist)s"/"%(target_arch)s"/ ${SRPM} + + +## Upload the created RPMs to chacra +chacra_endpoint="calamari/${BRANCH}/${GIT_COMMIT}/${DISTRO}/${RELEASE}" + +[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + +# push binaries to chacra +find ~/rpmbuild/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/ + +# start repo creation +$VENV/chacractl repo update ${chacra_endpoint} + +echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} diff --git a/calamari/build/setup b/calamari/build/setup new file mode 100644 index 00000000..41939805 --- /dev/null +++ b/calamari/build/setup @@ -0,0 +1,44 @@ +#! /usr/bin/bash +# +# Ceph distributed storage system +# +# Copyright (C) 2016 Red Hat +# +# Author: Boris Ranto +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +set -ex + +# Make sure we execute at the top level directory before we do anything +cd $WORKSPACE + +# This will set the DISTRO and MOCK_TARGET variables +get_distro_and_target + +# Perform a clean-up +git clean -fxd + +# Make sure the dist directory is clean +rm -rf dist +mkdir -p dist + +# Print some basic system info +HOST=$(hostname --short) +echo "Building on $(hostname) with the following env" +echo "*****" +env +echo "*****" + +export LC_ALL=C # the following is vulnerable to i18n + +pkgs=( "chacractl>=0.0.4" ) +install_python_packages "pkgs[@]" + +# ask shaman which chacra instance to use +chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +# create the .chacractl config file using global variables +make_chacractl_config $chacra_url diff --git a/calamari/config/definitions/calamari.yml b/calamari/config/definitions/calamari.yml index adc2dbca..68dda871 100644 --- a/calamari/config/definitions/calamari.yml +++ b/calamari/config/definitions/calamari.yml @@ -1,43 +1,26 @@ - job: name: calamari - description: 'This is the main calamari build task which builds for testing purposes.' - project-type: multijob + project-type: matrix defaults: global - concurrent: true display-name: 'calamari' - logrotate: - daysToKeep: -1 - numToKeep: 25 - artifactDaysToKeep: 25 - artifactNumToKeep: 25 block-downstream: false block-upstream: false - properties: - - github: - url: https://github.com/ceph/calamari - + concurrent: true parameters: - string: name: BRANCH description: "The git branch (or tag) to build" - default: master - string: name: DISTROS description: "A list of distros to build for. Available options are: xenial, centos7, centos6, trusty, precise, wheezy, and jessie" - default: "centos6 centos7" + default: "centos7 precise xenial" - string: name: ARCHS description: "A list of architectures to build for. Available options are: x86_64, and arm64" default: "x86_64" - - bool: - name: THROWAWAY - description: " -Default: False. When True it will not POST binaries to chacra. Artifacts will not be around for long. Useful to test builds." - default: false - - bool: name: FORCE description: " @@ -46,25 +29,77 @@ If this is unchecked, then nothing is built or pushed if they already exist in c If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra." - string: - name: CALAMARI_BUILD_VIRTUALENV + name: BUILD_VIRTUALENV description: "Base parent path for virtualenv locations, set to avoid issues with extremely long paths that are incompatible with tools like pip. Defaults to '/tmp/' (note the trailing slash, which is required)." default: "/tmp/" + execution-strategy: + combination-filter: DIST==AVAILABLE_DIST && ARCH==AVAILABLE_ARCH && (ARCH=="x86_64" || (ARCH == "arm64" && (DIST == "xenial" || DIST == "centos7"))) + axes: + - axis: + type: label-expression + name: MACHINE_SIZE + values: + - huge + - axis: + type: label-expression + name: AVAILABLE_ARCH + values: + - x86_64 + - arm64 + - axis: + type: label-expression + name: AVAILABLE_DIST + values: + - centos6 + - centos7 + - trusty + - xenial + - jessie + - precise + - wheezy + - axis: + type: dynamic + name: DIST + - axis: + type: dynamic + name: DIST + values: + - DISTROS + - axis: + type: dynamic + name: ARCH + values: + - ARCHS + + scm: + - git: + url: git@github.com:ceph/calamari.git + # Use the SSH key attached to the ceph-jenkins GitHub account. + credentials-id: '39fa150b-b2a1-416e-b334-29a9a2c0b32d' + branches: + - $BRANCH + skip-tag: true + wipe-workspace: true + builders: - - multijob: - name: 'calamari setup phase' - condition: SUCCESSFUL - projects: - - name: calamari-setup - current-parameters: true - exposed-scm: false - - multijob: - name: 'calamari build phase' - condition: SUCCESSFUL - projects: - - name: calamari-build - current-parameters: true - exposed-scm: false + - shell: | + echo "Cleaning up top-level workarea (shared among workspaces)" + rm -rf dist + rm -rf venv + rm -rf release + # debian build scripts + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/build_deb + # rpm build scripts + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/build_rpm wrappers: - inject-passwords: