From 139f91396a9dd25441c4f4823e76f68702095966 Mon Sep 17 00:00:00 2001 From: Boris Ranto Date: Mon, 5 Sep 2016 16:05:15 +0200 Subject: [PATCH] calamari: Add initial files for jenkins build Signed-off-by: Boris Ranto --- calamari-build/build/build_rpm | 114 ++++++++++++++++++ calamari-build/build/setup | 62 ++++++++++ calamari-build/build/validate_rpm | 7 ++ .../config/definitions/calamari-build.yml | 72 +++++++++++ calamari-setup/build/build | 57 +++++++++ .../config/definitions/calamari-setup.yml | 48 ++++++++ calamari/config/definitions/calamari.yml | 72 +++++++++++ 7 files changed, 432 insertions(+) create mode 100644 calamari-build/build/build_rpm create mode 100644 calamari-build/build/setup create mode 100644 calamari-build/build/validate_rpm create mode 100644 calamari-build/config/definitions/calamari-build.yml create mode 100644 calamari-setup/build/build create mode 100644 calamari-setup/config/definitions/calamari-setup.yml create mode 100644 calamari/config/definitions/calamari.yml diff --git a/calamari-build/build/build_rpm b/calamari-build/build/build_rpm new file mode 100644 index 00000000..098775ab --- /dev/null +++ b/calamari-build/build/build_rpm @@ -0,0 +1,114 @@ +#!/bin/bash +set -ex + +if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then + exit 0 +fi + +cd $WORKSPACE + +get_rpm_dist() { + LSB_RELEASE=/usr/bin/lsb_release + [ ! -x $LSB_RELEASE ] && echo unknown && exit + + ID=`$LSB_RELEASE --short --id` + + case $ID in + RedHatEnterpriseServer) + RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` + DIST=rhel$RELEASE + DISTRO=rhel + ;; + CentOS) + RELEASE=`$LSB_RELEASE --short --release | cut -d. -f1` + DIST=el$RELEASE + DISTRO=centos + ;; + Fedora) + RELEASE=`$LSB_RELEASE --short --release` + DIST=fc$RELEASE + DISTRO=fedora + ;; + SUSE\ LINUX) + DESC=`$LSB_RELEASE --short --description` + RELEASE=`$LSB_RELEASE --short --release` + case $DESC in + *openSUSE*) + DIST=opensuse$RELEASE + DISTRO=opensuse + ;; + *Enterprise*) + DIST=sles$RELEASE + DISTRO=sles + ;; + esac + ;; + *) + DIST=unknown + DISTRO=unknown + ;; + esac + + echo $DIST +} + +get_rpm_dist +dist=$DIST +[ -z "$dist" ] && echo no dist && exit 1 +echo dist $dist + +chacra_endpoint="calamari/${BRANCH}/${SHA1}/${DISTRO}/${RELEASE}" +chacra_check_url="${chacra_endpoint}/${ARCH}/calamari-server-${VERSION}-${RPM_RELEASE}.${DIST}.${ARCH}.rpm" + +if [ "$THROWAWAY" = false ] ; then + # this exists in scripts/build_utils.sh + check_binary_existence $chacra_check_url +fi + +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " ARCH=${ARCH}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + +# Install the dependencies +sudo yum-builddep -y dist/calamari-server.spec + +# Create the source rpm +echo "Building SRPM" +rpmbuild \ + --define "_sourcedir ./dist" \ + --define "_specdir ." \ + --define "_builddir ." \ + --define "_srcrpmdir ." \ + --define "_rpmdir ." \ + --define "dist .any" \ + --define "fedora 21" \ + --define "rhel 7" \ + --nodeps -bs dist/calamari-server.spec +SRPM=$(readlink -f *.src.rpm) + +# Build the binaries +echo "Building RPMs" +sudo mock -r epel-${RELEASE}-${ARCH} --resultdir=./dist/rpm/"%(dist)s"/"%(target_arch)s"/ ${SRPM} + +# Make sure we execute at the top level directory +cd "$WORKSPACE" + +[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + +if [ "$THROWAWAY" = false ] ; then + # push binaries to chacra + echo "$SRPM" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source/ + find dist/rpm/$DIST/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/ + + # start repo creation + $VENV/chacractl repo update ${chacra_endpoint} + + echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} +fi diff --git a/calamari-build/build/setup b/calamari-build/build/setup new file mode 100644 index 00000000..bccd321c --- /dev/null +++ b/calamari-build/build/setup @@ -0,0 +1,62 @@ +#!/bin/bash +# +# Ceph distributed storage system +# +# Copyright (C) 2016 Red Hat +# +# Author: Boris Ranto +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +set -ex +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " KEYID=${KEYID}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + +if test $(id -u) != 0 ; then + SUDO=sudo +fi +export LC_ALL=C # the following is vulnerable to i18n + +if test -f /etc/redhat-release ; then + $SUDO yum install -y redhat-lsb-core +fi + +if which apt-get > /dev/null ; then + $SUDO apt-get install -y lsb-release +fi + +case $(lsb_release -si) in +CentOS|Fedora|SUSE*|RedHatEnterpriseServer) + case $(lsb_release -si) in + SUSE*) + $SUDO zypper -y yum-utils + ;; + *) + $SUDO yum install -y yum-utils mock + ;; + esac + ;; +*) + echo "$(lsb_release -si) is unknown, dependencies will have to be installed manually." + ;; +esac + +pkgs=( "chacractl>=0.0.4" ) +install_python_packages "pkgs[@]" + +# ask shaman which chacra instance to use +chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +# create the .chacractl config file using global variables +make_chacractl_config $chacra_url diff --git a/calamari-build/build/validate_rpm b/calamari-build/build/validate_rpm new file mode 100644 index 00000000..17b41482 --- /dev/null +++ b/calamari-build/build/validate_rpm @@ -0,0 +1,7 @@ +#!/bin/bash +set -ex + +# only do work if we are a RPM distro +if [[ ! -f /etc/redhat-release && ! -f /usr/bin/zypper ]] ; then + exit 0 +fi diff --git a/calamari-build/config/definitions/calamari-build.yml b/calamari-build/config/definitions/calamari-build.yml new file mode 100644 index 00000000..44bf074a --- /dev/null +++ b/calamari-build/config/definitions/calamari-build.yml @@ -0,0 +1,72 @@ +- job: + name: calamari-build + project-type: matrix + defaults: global + display-name: 'calamari-build' + block-downstream: false + block-upstream: false + concurrent: true + properties: + - github: + url: https://github.com/ceph/calamari + execution-strategy: + combination-filter: DIST==AVAILABLE_DIST && ARCH==AVAILABLE_ARCH && (ARCH=="x86_64" || (ARCH == "arm64" && (DIST == "xenial" || DIST == "centos7"))) + axes: + - axis: + type: label-expression + name: MACHINE_SIZE + values: + - small + - axis: + type: label-expression + name: AVAILABLE_ARCH + values: + - x86_64 + - arm64 + - axis: + type: label-expression + name: AVAILABLE_DIST + values: + - centos6 + - centos7 + - axis: + type: dynamic + name: DIST + values: + - DISTROS + - axis: + type: dynamic + name: ARCH + values: + - ARCHS + + + + builders: + - shell: | + echo "Cleaning up top-level workarea (shared among workspaces)" + rm -rf dist + rm -rf venv + rm -rf release + - copyartifact: + project: calamari-setup + filter: 'dist/**' + which-build: last-successful + - inject: + properties-file: ${WORKSPACE}/dist/sha1 + - inject: + properties-file: ${WORKSPACE}/dist/branch + - inject: + properties-file: ${WORKSPACE}/dist/version + # rpm build scripts + - shell: + !include-raw: + - ../../build/validate_rpm + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/build_rpm + + wrappers: + - inject-passwords: + global: true + mask-password-params: true diff --git a/calamari-setup/build/build b/calamari-setup/build/build new file mode 100644 index 00000000..5dfc4ffe --- /dev/null +++ b/calamari-setup/build/build @@ -0,0 +1,57 @@ +#!/bin/bash -ex + +HOST=$(hostname --short) +echo "Building on ${HOST}" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BRANCH=$BRANCH" +echo " SHA1=$GIT_COMMIT" + +if [ -x "$BRANCH" ] ; then + echo "No git branch was supplied" + exit 1 +fi + +# Make sure the repo is clean +echo "Cleaning up the repo" +git clean -fxd + +# Make sure the dist directory is clean +rm -rf dist +mkdir -p dist + +echo "Building version $(git describe) Branch $BRANCH" + +# Install any setup-time deps +if [ -x install-deps.sh ]; then + echo "Ensuring dependencies are installed" + ./install-deps.sh +fi + +# Get the version +eval $(./get-versions.sh) +RPM_RELEASE=$(echo $REVISION | tr '-' '_') + +# Create the source tarball and move it to dist +echo "Building source distribution" +make dist +mv ../calamari-server_*.tar.gz dist/ + +# Prepare the spec file for build +sed -e "s/@VERSION@/${VERSION}/g" -e "s/@RELEASE@/${RPM_RELEASE}/g" < calamari-server.spec.in > dist/calamari-server.spec + +# Save these so that we can later inject them into the build script +cat > dist/sha1 << EOF +SHA1=${GIT_COMMIT} +EOF + +cat > dist/branch << EOF +BRANCH=${BRANCH} +EOF + +cat > dist/version << EOF +VERSION=${VERSION} +RPM_RELEASE=${RPM_RELEASE} +EOF diff --git a/calamari-setup/config/definitions/calamari-setup.yml b/calamari-setup/config/definitions/calamari-setup.yml new file mode 100644 index 00000000..8aab7e2e --- /dev/null +++ b/calamari-setup/config/definitions/calamari-setup.yml @@ -0,0 +1,48 @@ +- job: + name: calamari-setup + description: "This job step checks out the branch and builds the tarballs, diffs, and dsc that are passed to the calamari-build step.\r\n\r\nNotes:\r\nJob needs to run on a releatively recent debian system. The Restrict where run feature is used to specifiy an appropriate label.\r\nThe clear workspace before checkout box for the git plugin is used." + # we do not need to pin this to trusty anymore for the new jenkins instance + # FIXME: unpin when this gets ported over + node: huge && trusty + display-name: 'calamari-setup' + logrotate: + daysToKeep: -1 + numToKeep: 25 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + block-downstream: false + block-upstream: false + concurrent: true + properties: + - github: + url: https://github.com/ceph/calamari + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + + scm: + - git: + url: git@github.com:ceph/calamari.git + # Use the SSH key attached to the ceph-jenkins GitHub account. + credentials-id: '39fa150b-b2a1-416e-b334-29a9a2c0b32d' + branches: + - $BRANCH + skip-tag: true + wipe-workspace: true + + builders: + - shell: + !include-raw ../../build/build + + publishers: + - archive: + artifacts: 'dist/**' + allow-empty: false + latest-only: false + + wrappers: + - inject-passwords: + global: true + mask-password-params: true diff --git a/calamari/config/definitions/calamari.yml b/calamari/config/definitions/calamari.yml new file mode 100644 index 00000000..adc2dbca --- /dev/null +++ b/calamari/config/definitions/calamari.yml @@ -0,0 +1,72 @@ +- job: + name: calamari + description: 'This is the main calamari build task which builds for testing purposes.' + project-type: multijob + defaults: global + concurrent: true + display-name: 'calamari' + logrotate: + daysToKeep: -1 + numToKeep: 25 + artifactDaysToKeep: 25 + artifactNumToKeep: 25 + block-downstream: false + block-upstream: false + properties: + - github: + url: https://github.com/ceph/calamari + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + default: master + + - string: + name: DISTROS + description: "A list of distros to build for. Available options are: xenial, centos7, centos6, trusty, precise, wheezy, and jessie" + default: "centos6 centos7" + + - string: + name: ARCHS + description: "A list of architectures to build for. Available options are: x86_64, and arm64" + default: "x86_64" + + - bool: + name: THROWAWAY + description: " +Default: False. When True it will not POST binaries to chacra. Artifacts will not be around for long. Useful to test builds." + default: false + + - bool: + name: FORCE + description: " +If this is unchecked, then nothing is built or pushed if they already exist in chacra. This is the default. + +If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra." + + - string: + name: CALAMARI_BUILD_VIRTUALENV + description: "Base parent path for virtualenv locations, set to avoid issues with extremely long paths that are incompatible with tools like pip. Defaults to '/tmp/' (note the trailing slash, which is required)." + default: "/tmp/" + + builders: + - multijob: + name: 'calamari setup phase' + condition: SUCCESSFUL + projects: + - name: calamari-setup + current-parameters: true + exposed-scm: false + - multijob: + name: 'calamari build phase' + condition: SUCCESSFUL + projects: + - name: calamari-build + current-parameters: true + exposed-scm: false + + wrappers: + - inject-passwords: + global: true + mask-password-params: true -- 2.39.5