From a6472e61ecc0a93553366dcdd22ce47c16cd00c2 Mon Sep 17 00:00:00 2001 From: Gregory Meno Date: Wed, 2 Mar 2016 11:36:07 -0800 Subject: [PATCH] adds jobs to build diamond Signed-off-by: Gregory Meno --- diamond-build/build/build | 131 ++++++++++++++++++ diamond-build/build/setup | 24 ++++ diamond-build/config/definitions/diamond | 48 +++++++ diamond-docs/build/build | 25 ++++ diamond-docs/config/definitions/diamond | 34 +++++ diamond-pull-requests/build/setup | 11 ++ .../config/definitions/diamond | 84 +++++++++++ .../setup/playbooks/setup.yml | 11 ++ .../setup/playbooks/tasks/ubuntu.yml | 18 +++ diamond-tag/build/build | 17 +++ diamond-tag/config/definitions/ceph-tag.yml | 51 +++++++ diamond/config/definitions/diamond | 62 +++++++++ 12 files changed, 516 insertions(+) create mode 100644 diamond-build/build/build create mode 100644 diamond-build/build/setup create mode 100644 diamond-build/config/definitions/diamond create mode 100644 diamond-docs/build/build create mode 100644 diamond-docs/config/definitions/diamond create mode 100644 diamond-pull-requests/build/setup create mode 100644 diamond-pull-requests/config/definitions/diamond create mode 100644 diamond-pull-requests/setup/playbooks/setup.yml create mode 100644 diamond-pull-requests/setup/playbooks/tasks/ubuntu.yml create mode 100644 diamond-tag/build/build create mode 100644 diamond-tag/config/definitions/ceph-tag.yml create mode 100644 diamond/config/definitions/diamond diff --git a/diamond-build/build/build b/diamond-build/build/build new file mode 100644 index 00000000..28d60638 --- /dev/null +++ b/diamond-build/build/build @@ -0,0 +1,131 @@ +#!/bin/bash + +# This is the script that runs inside Jenkins. +# http://jenkins.ceph.com/job/diamond/ + +set -x +set -e + +HOST=$(hostname --short) +echo "Building on ${HOST}" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BRANCH=$BRANCH" + +# FIXME A very naive way to just list the RPM $DIST that we currently support. +# We should be a bit more lenient to allow any rhel/centos/sles/suse +rpm_dists="rhel centos6 centos7 centos" +deb_dists="precise wheezy squeeze trusty jessie" + +VENV="$WORKSPACE/venv/bin" + +# A helper to match an item in a list of items, like python's `if item in list` +listcontains() { + for word in $2; do + [[ $word = $1 ]] && return 0 + done + return 1 +} + +if listcontains $DIST "$rpm_dists" +then + # Tag tree and update version number in change log and + # in setup.py before building. + + # this exists in scripts/build_utils.sh + get_rpm_dist + + REPO=rpm-repo + BUILDAREA=./rpmbuild + DIST=el6 + RPM_BUILD=$(lsb_release -s -c) + + [ "$TEST" = true ] && chacra_ref="test" || chacra_ref="$BRANCH" + chacra_endpoint="diamond/${chacra_ref}/${DISTRO}/${DISTRO_VERSION}" + + # this exists in scripts/build_utils.sh for diamond, binaries have + # no architecture so we POST them to 'noarch' for rpms + check_binary_existence $chacra_endpoint/noarch + + if [ ! -e setup.py ] ; then + echo "Are we in the right directory" + exit 1 + fi + + # Create Tarball + python setup.py sdist --formats=bztar + + # Build RPM + mkdir -p rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + BUILDAREA=`readlink -fn ${BUILDAREA}` ### rpm wants absolute path + cp diamond.spec ${BUILDAREA}/SPECS + cp dist/*.tar.bz2 ${BUILDAREA}/SOURCES + echo "buildarea is: ${BUILDAREA}" + rpmbuild -ba --define "_topdir ${BUILDAREA}" --define "_unpackaged_files_terminate_build 0" ${BUILDAREA}/SPECS/diamond.spec + + [ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + + find ${BUILDAREA}/SRPMS | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source + find ${BUILDAREA}/RPMS/* | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/noarch + + exit 0 + +elif listcontains $DIST "$deb_dists" +then + + # Tag tree and update version number in change log and + # in setup.py before building. + + REPO=debian-repo + COMPONENT=main + DEB_DIST="sid wheezy squeeze jessie precise raring trusty" + DEB_BUILD=$(lsb_release -s -c) + #XXX only releases until we fix this + RELEASE=1 + DISTRO=`python -c "exec 'import platform; print platform.linux_distribution()[0].lower()'"` + + [ "$TEST" = true ] && chacra_ref="test" || chacra_ref="$BRANCH" + # diamond isn't architecture dependant, so we use 'all' as architecture and + # uses 'universal' to signal this is not specific to any distro version + chacra_endpoint="diamond/${chacra_ref}/${DISTRO}/universal/all" + + # this exists in scripts/build_utils.sh + check_binary_existence $chacra_endpoint + + if [ ! -d debian ] ; then + echo "Are we in the right directory" + exit 1 + fi + + # Apply backport tag if release build + if [ $RELEASE -eq 1 ] ; then + DEB_VERSION=$(dpkg-parsechangelog | sed -rne 's,^Version: (.*),\1, p') + BP_VERSION=${DEB_VERSION}${BPTAG} + dch -D $DIST --force-distribution -b -v "$BP_VERSION" "$comment" + dpkg-source -b . + fi + + # Build Package + echo "Building for dist: $DEB_BUILD" + # we no longer sign the .dsc or .changes files (done by default with + # the `-k$KEYID` flag), so explicitly tell the tool not to sign them + dpkg-buildpackage -uc -us + if [ $? -ne 0 ] ; then + echo "Build failed" + exit 2 + fi + + [ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" + + # push binaries to chacra + # the binaries are created in one directory up from $WORKSPACE + find ../ | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} + + echo "Done" + +else + echo "Can't determine build host type, I suck. Sorry." + exit 4 +fi diff --git a/diamond-build/build/setup b/diamond-build/build/setup new file mode 100644 index 00000000..e58822b8 --- /dev/null +++ b/diamond-build/build/setup @@ -0,0 +1,24 @@ +#!/bin/bash + +# This is the script that runs inside Jenkins. +# http://jenkins.ceph.com/job/diamond/ + +set -x +set -e + +# ensure dependencies are installed for RPM hosts, because they are required and +# we are not building with a contained environment like we do on DEB builds + +# TODO: use Mock to build diamond rpm's and avoid this + +rpm_deps="python-devel python-virtualenv python-mock python-tox pytest" + +if test -f /etc/redhat-release ; then + sudo yum install -y $rpm_deps +fi + +pkgs=( "chacractl>=0.0.4" ) +install_python_packages "pkgs[@]" + +# create the .chacractl config file using global variables +make_chacractl_config diff --git a/diamond-build/config/definitions/diamond b/diamond-build/config/definitions/diamond new file mode 100644 index 00000000..bdcf84e6 --- /dev/null +++ b/diamond-build/config/definitions/diamond @@ -0,0 +1,48 @@ +- job: + name: diamond-build + node: small && trusty + project-type: matrix + defaults: global + display-name: 'diamond-build' + concurrent: true + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + + scm: + - git: + url: https://github.com/ceph/diamond.git + branches: + - $BRANCH + browser: auto + skip-tag: true + timeout: 20 + wipe-workspace: true + + axes: + - axis: + type: label-expression + name: ARCH + values: + - x86_64 + - axis: + type: label-expression + name: DIST + values: + - trusty + - jessie + - centos6 + - centos7 + + builders: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../build/build + + wrappers: + - inject-passwords: + global: true + mask-password-params: true diff --git a/diamond-docs/build/build b/diamond-docs/build/build new file mode 100644 index 00000000..6e5960ea --- /dev/null +++ b/diamond-docs/build/build @@ -0,0 +1,25 @@ +#!/bin/bash + +set -ex + +# Create the virtualenv +virtualenv venv +. venv/bin/activate + +# Define and ensure the PIP cache +PIP_SDIST_INDEX="$HOME/.cache/pip" +mkdir -p $PIP_SDIST_INDEX + +# Install the package by trying with the cache first, otherwise doing a download only, and then +# trying to install from the cache again. +if ! venv/bin/pip install --find-links="file://$PIP_SDIST_INDEX" --no-index tox; then + venv/bin/pip install --download-directory="$PIP_SDIST_INDEX" tox + venv/bin/pip install --find-links="file://$PIP_SDIST_INDEX" --no-index tox +fi + + +# create the docs build with tox +tox -rv -e docs + +# publish docs to http://docs.ceph.com/docs/diamond +rsync -auv --delete .tox/docs/tmp/html/* /var/diamond/docs/ diff --git a/diamond-docs/config/definitions/diamond b/diamond-docs/config/definitions/diamond new file mode 100644 index 00000000..3507bdf1 --- /dev/null +++ b/diamond-docs/config/definitions/diamond @@ -0,0 +1,34 @@ +- job: + name: diamond-docs + node: docs + project-type: freestyle + defaults: global + display-name: 'diamond: docs build' + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + properties: + - github: + url: https://github.com/ceph/diamond + logrotate: + daysToKeep: -1 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + triggers: + - github + + scm: + - git: + url: https://github.com/ceph/diamond + branches: + - master + browser: auto + skip-tag: true + timeout: 20 + + builders: + - shell: + !include-raw ../../build/build diff --git a/diamond-pull-requests/build/setup b/diamond-pull-requests/build/setup new file mode 100644 index 00000000..a13ecbe7 --- /dev/null +++ b/diamond-pull-requests/build/setup @@ -0,0 +1,11 @@ +#!/bin/bash + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "ansible" "tox" ) +install_python_packages "pkgs[@]" + + +# run ansible to get this current host to meet our requirements, specifying +# a local connection and 'localhost' as the host where to execute +cd "$WORKSPACE/ceph-build/diamond-pull-requests/setup/playbooks" +$VENV/ansible-playbook -i "localhost," -c local setup.yml diff --git a/diamond-pull-requests/config/definitions/diamond b/diamond-pull-requests/config/definitions/diamond new file mode 100644 index 00000000..e3a38dee --- /dev/null +++ b/diamond-pull-requests/config/definitions/diamond @@ -0,0 +1,84 @@ +# multiple scm requires definition of each scm with `name` so that they can be +# referenced later in `job` +# reference: http://docs.openstack.org/infra/jenkins-job-builder/scm.html +- scm: + name: diamond + scm: + - git: + url: https://github.com/ceph/diamond.git + branches: + - ${sha1} + refspec: +refs/pull/*:refs/remotes/origin/pr/* + browser: auto + timeout: 20 + skip-tag: true + wipe-workspace: false + basedir: "diamond" + +- scm: + name: ceph-build + scm: + - git: + url: https://github.com/ceph/ceph-build.git + browser-url: https://github.com/ceph/ceph-build + timeout: 20 + skip-tag: true + wipe-workspace: false + basedir: "ceph-build" + + +- job: + name: diamond-pull-requests + node: trusty + project-type: freestyle + defaults: global + concurrent: true + display-name: 'diamond: Pull Requests' + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + properties: + - github: + url: https://github.com/ceph/diamond/ + logrotate: + daysToKeep: 15 + numToKeep: 30 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + parameters: + - string: + name: sha1 + description: "A pull request ID, like 'origin/pr/72/head'" + + triggers: + - github-pull-request: + admin-list: + - alfredodeza + - ktdreyer + - trhoden + white-list: + - xarses + - angdraug + - mbroz + - ddiss + - osynge + org-list: + - ceph + trigger-phrase: '' + only-trigger-phrase: false + github-hooks: true + permit-all: false + auto-close-on-fail: false + + scm: + - diamond + - ceph-build + + builders: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - shell: "cd $WORKSPACE/diamond && $WORKSPACE/venv/bin/tox -rv" diff --git a/diamond-pull-requests/setup/playbooks/setup.yml b/diamond-pull-requests/setup/playbooks/setup.yml new file mode 100644 index 00000000..f45d5f90 --- /dev/null +++ b/diamond-pull-requests/setup/playbooks/setup.yml @@ -0,0 +1,11 @@ +--- + +- hosts: localhost + user: jenkins-build + sudo: True + + tasks: + - include: tasks/ubuntu.yml + when: ansible_distribution == "Ubuntu" + + # TODO: maybe add RPM or Debian handling? diff --git a/diamond-pull-requests/setup/playbooks/tasks/ubuntu.yml b/diamond-pull-requests/setup/playbooks/tasks/ubuntu.yml new file mode 100644 index 00000000..23bbff8f --- /dev/null +++ b/diamond-pull-requests/setup/playbooks/tasks/ubuntu.yml @@ -0,0 +1,18 @@ +--- + - name: add deadsnakes ppa + action: apt_repository repo=ppa:fkrull/deadsnakes state=present + + - name: "update apt repo" + action: apt update_cache=yes + + - name: install python requirements + action: apt pkg={{ item }} state=installed + with_items: + - python-software-properties + - python-dev + - python-setuptools + - python2.6 + - python2.7 + + - name: install pip + action: easy_install name=pip diff --git a/diamond-tag/build/build b/diamond-tag/build/build new file mode 100644 index 00000000..49f860c4 --- /dev/null +++ b/diamond-tag/build/build @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ex + +if [ "$TAG" = false ] ; then + echo "Assuming tagging process has succeeded before because TAG was set to false" + exit 0 +fi + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "ansible" ) +install_python_packages "pkgs[@]" + +# run ansible to do all the tagging and release specifying +# a local connection and 'localhost' as the host where to execute +cd "$WORKSPACE/ceph-build/ansible/" +$VENV/ansible-playbook -i "localhost," -c local release.yml --extra-vars="version=$VERSION branch=$BRANCH release=stable clean=true project=diamond" diff --git a/diamond-tag/config/definitions/ceph-tag.yml b/diamond-tag/config/definitions/ceph-tag.yml new file mode 100644 index 00000000..cc6df2e9 --- /dev/null +++ b/diamond-tag/config/definitions/ceph-tag.yml @@ -0,0 +1,51 @@ +- scm: + name: ceph-build + scm: + - git: + url: https://github.com/ceph/ceph-build.git + browser: auto + timeout: 20 + skip-tag: true + wipe-workspace: true + basedir: "ceph-build" + +- job: + name: diamond-tag + description: "This job clones diamond and sets the right version from the tag, pushing back to diamond.git" + display-name: 'diamond-tag' + logrotate: + daysToKeep: -1 + numToKeep: 25 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + block-downstream: false + block-upstream: false + properties: + - github: + url: https://github.com/ceph/diamond + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + default: "master" + - string: + name: VERSION + description: "The version for release, e.g. 1.5.30" + scm: + - ceph-build + + builders: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/build + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - ssh-agent-credentials: + # "jenkins-build" SSH key, needed so we can push to + # diamond.git + user: '39fa150b-b2a1-416e-b334-29a9a2c0b32d' diff --git a/diamond/config/definitions/diamond b/diamond/config/definitions/diamond new file mode 100644 index 00000000..1599abaa --- /dev/null +++ b/diamond/config/definitions/diamond @@ -0,0 +1,62 @@ +- job: + name: diamond + project-type: multijob + defaults: global + display-name: 'diamond' + concurrent: true + quiet-period: 5 + block-downstream: false + block-upstream: false + retry-count: 3 + + parameters: + - string: + name: BRANCH + description: "The git branch or tag to build. Defaults to master" + default: "master" + + - bool: + name: TEST + description: " +If this is unchecked, then the builds will be pushed to chacra with the correct ref. This is the default. + +If this is checked, then the builds will be pushed to chacra under the 'test' ref." + + - bool: + name: TAG + description: "When this is checked, Jenkins will remove the previous tag and recreate it again, changing the control files and committing again. When this is unchecked, Jenkins will not do any commit or tag operations. If you've already created the private tag separately, then leave this unchecked. +Defaults to checked." + default: true + + - bool: + name: FORCE + description: " +If this is unchecked, then then nothing is built or pushed if they already exist in chacra. This is the default. + +If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra." + + - string: + name: VERSION + description: "The version for release, e.g. 0.94.4" + + builders: + - multijob: + name: 'diamond tag phase' + condition: SUCCESSFUL + projects: + - name: diamond-tag + current-parameters: true + exposed-scm: false + + - multijob: + name: 'diamond build phase' + condition: SUCCESSFUL + projects: + - name: diamond-build + current-parameters: true + exposed-scm: false + + wrappers: + - inject-passwords: + global: true + mask-password-params: true -- 2.39.5