From 92a8d2eb145cd112d199862beb63dc8ad9829f1b Mon Sep 17 00:00:00 2001 From: David Galloway Date: Tue, 1 Feb 2022 14:46:50 -0500 Subject: [PATCH] ceph-dev-container-only: Create job This job will allow users to build a container for https://quay.ceph.io/repository/ceph-ci/ceph?tab=tags given a branch name. Should work for ceph.git and ceph-ci.git. This is mostly useful for when packages and a repo successfully get built for a ceph-dev*build job but the container build fails. Signed-off-by: David Galloway --- ceph-dev-container-only/build/build_rpm | 57 +++++++ ceph-dev-container-only/build/failure | 22 +++ .../definitions/ceph-dev-container-only.yml | 141 ++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100644 ceph-dev-container-only/build/build_rpm create mode 100644 ceph-dev-container-only/build/failure create mode 100644 ceph-dev-container-only/config/definitions/ceph-dev-container-only.yml diff --git a/ceph-dev-container-only/build/build_rpm b/ceph-dev-container-only/build/build_rpm new file mode 100644 index 00000000..a7cb5d79 --- /dev/null +++ b/ceph-dev-container-only/build/build_rpm @@ -0,0 +1,57 @@ +#!/bin/bash +# vim: ts=4 sw=4 expandtab +set -ex +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " KEYID=${KEYID}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo "*****" +env +echo "*****" + +if test $(id -u) != 0 ; then + SUDO=sudo +fi + +get_rpm_dist + +BRANCH=`branch_slash_filter $BRANCH` + +# Normalize variables across rpm/deb builds +NORMAL_DISTRO=$DISTRO +NORMAL_DISTRO_VERSION=$RELEASE +NORMAL_ARCH=$ARCH + +chacra_endpoint="ceph/${BRANCH}/${SHA1}/${DISTRO}/${RELEASE}" + +SHAMAN_URL="https://shaman.ceph.com/api/search/?project=ceph&distros=centos/${RELEASE}/${ARCH}&sha1=${SHA1}&ref=${BRANCH}&flavor=${FLAVOR}" + +loop=0 +ready=false +while ((loop < 15)); do + if [[ $(curl -s "$SHAMAN_URL" | jq -r '.[0].status') == 'ready' ]] ; then ready=true; break; fi + ((loop = loop + 1)) + sleep 60 +done + +if [[ "$ready" == "false" ]] ; then + echo "FAIL: timed out waiting for shaman repo to be built: https://shaman.ceph.com/api/repos/${chacra_endpoint}/flavors/${FLAVOR}/" + echo + echo "NOTE: You should only use this job if there was already a successful ceph-dev*build job!" +fi + +SHA1=$(curl -s "$SHAMAN_URL" | jq -r '.[0].sha1') + +pushd $WORKSPACE/ceph-container +$SUDO -E CI_CONTAINER=true BASEOS_REGISTRY="quay.io/centos" SHA1=${SHA1} OSD_FLAVOR=${FLAVOR} CONTAINER_FLAVOR=${BRANCH},${DISTRO},${RELEASE} \ + /bin/bash ./contrib/build-push-ceph-container-imgs.sh +popd +$SUDO rm -rf $WORKSPACE/ceph-container + +# update shaman with the completed build status +if $NOTIFY_SHAMAN; then + update_build_status "completed" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH +fi diff --git a/ceph-dev-container-only/build/failure b/ceph-dev-container-only/build/failure new file mode 100644 index 00000000..063fea54 --- /dev/null +++ b/ceph-dev-container-only/build/failure @@ -0,0 +1,22 @@ +#!/bin/bash -ex + +# The ceph-container dir is supposed to get deleted in the build_rpm script. +# We used to add '|| true' to the container build so the dir would still get +# deleted even if it failed. This changed in https://github.com/ceph/ceph-build/pull/1603 +# So now we need to delete the directory or the Wipe Workspace plugin will fail on the next build. +cd $WORKSPACE +sudo rm -rf ceph-container + +get_rpm_dist +# note: the failed_build_status call relies on normalized variable names that +# are infered by the builds themselves. If the build fails before these are +# set, they will be posted with empty values +BRANCH=`branch_slash_filter $BRANCH` + +# Normalize variables across rpm/deb builds +NORMAL_DISTRO=$DISTRO +NORMAL_DISTRO_VERSION=$RELEASE +NORMAL_ARCH=$ARCH + +# update shaman with the failed build status +failed_build_status "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH diff --git a/ceph-dev-container-only/config/definitions/ceph-dev-container-only.yml b/ceph-dev-container-only/config/definitions/ceph-dev-container-only.yml new file mode 100644 index 00000000..596e58f4 --- /dev/null +++ b/ceph-dev-container-only/config/definitions/ceph-dev-container-only.yml @@ -0,0 +1,141 @@ +- job: + name: ceph-dev-container-only + node: 'gigantic && centos8' + project-type: matrix + defaults: global + display-name: 'ceph-dev-container-only: Builds a quay.ceph.io/ceph-ci container given a BRANCH' + block-downstream: false + block-upstream: false + concurrent: true + properties: + - build-discarder: + days-to-keep: 30 + artifact-days-to-keep: 30 + + scm: + - git: + url: git@github.com:ceph/ceph-container.git + basedir: ceph-container + credentials-id: 'jenkins-build' + branches: + - $CONTAINER_BRANCH + skip-tag: true + wipe-workspace: true + + execution-strategy: + combination-filter: | + DIST == AVAILABLE_DIST && ARCH == AVAILABLE_ARCH && + (ARCH == "x86_64" || (ARCH == "arm64" && ["xenial", "bionic", "centos7", "centos8"].contains(DIST))) + axes: + - axis: + type: label-expression + name: MACHINE_SIZE + values: + - gigantic + - axis: + type: label-expression + name: AVAILABLE_ARCH + values: + - x86_64 + - arm64 + - axis: + type: label-expression + name: AVAILABLE_DIST + values: + - centos8 + - centos9 + - axis: + type: dynamic + name: DIST + values: + - DISTROS + - axis: + type: dynamic + name: ARCH + values: + - ARCHS + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build. NOTE: This branch must already be built and packages pushed to a chacra node!" + default: master + + - string: + name: SHA1 + description: "Change to a specific SHA1 if desired." + default: "latest" + + - string: + name: DISTROS + description: "A list of distros to build for. Available options are: centos8 or centos9" + default: "centos8" + + - string: + name: ARCHS + description: "A list of architectures to build for. Available options are: x86_64, and arm64" + default: "x86_64 arm64" + + - choice: + name: FLAVOR + choices: + - default + - crimson + - jaeger + default: "default" + description: "Type of Ceph build, choices are: crimson, jaeger, default. Defaults to: 'default'" + + - string: + name: CONTAINER_BRANCH + description: "For CI_CONTAINER: Branch of ceph-container to use" + default: master + + - string: + name: CONTAINER_REPO_HOSTNAME + description: "For CI_CONTAINER: Name of container repo server (i.e. 'quay.io')" + default: "quay.ceph.io" + + - string: + name: CONTAINER_REPO_ORGANIZATION + description: "For CI_CONTAINER: Name of container repo organization (i.e. 'ceph-ci')" + default: "ceph-ci" + + - bool: + name: NOTIFY_SHAMAN + description: "Should we tell shaman this container built and change the corresponding build to READY?" + default: true + + + builders: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/build_rpm + + publishers: + - postbuildscript: + builders: + - role: SLAVE + build-on: + - FAILURE + - ABORTED + build-steps: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/failure + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - credentials-binding: + - text: + credential-id: shaman-api-key + variable: SHAMAN_API_KEY + - username-password-separated: + credential-id: dmick-quay + username: CONTAINER_REPO_USERNAME + password: CONTAINER_REPO_PASSWORD + - build-name: + name: "#${BUILD_NUMBER} ${BRANCH}, ${DISTROS}, ${ARCH}, ${FLAVOR}" -- 2.39.5