From 6004a859f184590792738264d84e443cf2407f91 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Wed, 13 Nov 2024 16:10:20 -0700 Subject: [PATCH] Add ceph-dev-pipeline This may ultimately replace the existing ceph-*build jobs; for now, it is a work in-progress. Signed-off-by: Zack Cerza --- ceph-dev-new-build/build/setup_pbuilder | 5 + ceph-dev-pipeline/build/Jenkinsfile | 170 ++++++++++++++++++ .../config/definitions/ceph-dev-pipeline.yml | 92 ++++++++++ scripts/setup_sccache.sh | 42 ++++- 4 files changed, 306 insertions(+), 3 deletions(-) create mode 100644 ceph-dev-pipeline/build/Jenkinsfile create mode 100644 ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml diff --git a/ceph-dev-new-build/build/setup_pbuilder b/ceph-dev-new-build/build/setup_pbuilder index be1d9f03..bc1c5cbb 100755 --- a/ceph-dev-new-build/build/setup_pbuilder +++ b/ceph-dev-new-build/build/setup_pbuilder @@ -13,3 +13,8 @@ if test -f /etc/redhat-release ; then fi setup_pbuilder use_gcc + +if [ "$SCCACHE" = true ] ; then + setup_pbuilderrc + setup_sccache_pbuilder_hook +fi diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile new file mode 100644 index 00000000..c462a4ad --- /dev/null +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -0,0 +1,170 @@ +ceph_build_repo = "https://github.com/ceph/ceph-build" +ceph_build_branch = "jenkinsfile" +base_node_label = "gigantic" +ubuntu_releases = [ + "noble", // 24.04 + "jammy", // 22.04 + "focal", // 20.04 +] +debian_releases = [ + "bookworm", // v12 (2023) + "bullseye", // v11 (2021) +] + +pipeline { + agent any + stages { + stage("source distribution") { + steps { + script { + if ( ! env.SETUP_BUILD_ID ) { + def setup_build = build( + job: "ceph-dev-new-setup", + parameters: [ + string(name: "BRANCH", value: env.BRANCH), + string(name: "FLAVOR", value: env.FLAVOR), + string(name: "SCCACHE", value: env.SCCACHE), + string(name: "DWZ", value: env.DWZ), + ] + ) + env.SETUP_BUILD_ID = setup_build.getNumber() + } + println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}" + def setup_build_url = new URI([env.JENKINS_URL, "job", "ceph-dev-new-setup", env.SETUP_BUILD_ID].join("/")).normalize() + println "${setup_build_url}" + } + } + } + stage("parallel build") { + matrix { + agent { + label "${DIST}&&${ARCH}&&${base_node_label}" + } + when { + beforeAgent true + allOf { + expression { env.DISTROS.contains(env.DIST) } + expression { env.ARCHS.contains(env.ARCH) } + } + } + axes { + axis { + name 'DIST' + values 'centos9', 'jammy' + } + axis { + name 'ARCH' + values 'x86_64', 'arm64' + } + } + stages { + stage("node") { + steps { + script { + def node_shortname = env.NODE_NAME.split('\\+')[-1] + def node_url = new URI([env.JENKINS_URL, "computer", env.NODE_NAME].join("/")).normalize() + echo "DIST=${env.DIST} ARCH=${env.ARCH} FLAVOR=${env.FLAVOR}\n${node_shortname}\n${node_url}" + } + sh "cat /etc/os-release" + } + } + stage("checkout ceph-build") { + steps { + checkout scmGit( + branches: [[name: ceph_build_branch]], + userRemoteConfigs: [[url: ceph_build_repo]], + extensions: [ + [$class: 'CleanBeforeCheckout'] + ], + ) + } + } + stage("copy artifacts") { + steps { + copyArtifacts( + projectName: "ceph-dev-new-setup", + selector: specific(buildNumber: env.SETUP_BUILD_ID), + filter: "dist/sha1,dist/version,dist/other_envvars,dist/ceph_*.orig.tar.gz", + ) + script { + def sha1_props = readProperties file: "${WORKSPACE}/dist/sha1" + env.SHA1 = sha1_props.SHA1 + println "SHA1=${env.SHA1}" + env.VERSION = readFile file: "${WORKSPACE}/dist/version" + def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" + def build_description = """SHA1=${env.SHA1}
+ VERSION=${env.VERSION}
+ shaman builds for this branch+commit + """ + buildDescription build_description + } + sh "sha256sum dist/*" + sh "cat dist/sha1 dist/version" + } + } + stage("build") { + environment { + CHACRACTL_KEY = credentials('chacractl-key') + SHAMAN_API_KEY = credentials('shaman-api-key') + CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci') + SCCACHE_BUCKET_CREDS = credentials('ibm-cloud-sccache-bucket') + } + steps { + script { + env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR + env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW + env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR + env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW + def props = readProperties file: "${WORKSPACE}/dist/other_envvars" + for (p in props) { + env."${p.key}" = p.value + } + if ( (debian_releases + ubuntu_releases).contains(env.DIST) ) { + def sccache_flag = "-DWITH_SCCACHE=ON" + if ( env.SCCACHE == "true" && ! env.CEPH_EXTRA_CMAKE_ARGS.contains(sccache_flag) ) { + env.CEPH_EXTRA_CMAKE_ARGS = "${env.CEPH_EXTRA_CMAKE_ARGS} ${sccache_flag}" + } + sh """#!/bin/bash + . ./ceph-dev-new-build/build/validate_deb + . ./scripts/build_utils.sh + . ./ceph-dev-new-build/build/setup_deb + . ./scripts/setup_sccache.sh + . ./ceph-dev-new-build/build/setup_pbuilder + . ./ceph-dev-new-build/build/build_deb + """ + } else if ( env.DIST =~ /^(centos|rhel|fedora).*/ ) { + sh """#!/bin/bash + . ./ceph-dev-new-build/build/validate_rpm + . ./scripts/build_utils.sh + . ./scripts/setup_sccache.sh + . ./ceph-dev-new-build/build/setup_rpm + [ "$CI_BUILD" = "true" ] || exit 0 + reset_sccache + . ./ceph-dev-new-build/build/build_rpm + """ + } else if ( env.DIST =~ /suse|sles/ ) { + sh """#!/bin/bash + . ./ceph-dev-new-build/build/validate_osc + . ./scripts/build_utils.sh + . ./ceph-dev-new-build/build/setup_osc + . ./ceph-dev-new-build/build/build_osc + """ + } else if ( env.DIST =~ /windows/ ) { + sh """#!/bin/bash + . ./ceph-dev-new-build/build/validate_mingw + . ./scripts/build_utils.sh + . ./ceph-dev-new-build/build/setup_mingw + . ./ceph-dev-new-build/build/build_mingw + """ + } else { + throw new Exception("DIST '${env.DIST}' is invalid!") + } + } + sh "command -v sccache && sccache --show-adv-stats || true" + } + } + } + } + } + } +} diff --git a/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml b/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml new file mode 100644 index 00000000..a0b51718 --- /dev/null +++ b/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml @@ -0,0 +1,92 @@ +- job: + name: ceph-dev-pipeline + description: ceph-dev-pipeline + project-type: pipeline + concurrent: true + pipeline-scm: + scm: + - git: + url: https://github.com/ceph/ceph-build + branches: + - jenkinsfile + shallow-clone: true + submodule: + disable: true + wipe-workspace: true + script-path: ceph-dev-pipeline/build/Jenkinsfile + lightweight-checkout: true + do-not-fetch-tags: true + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + default: sccache_main + + - string: + name: DISTROS + description: "A list of distros to build for. Available options are: centos9, centos8, noble, jammy, focal, and windows" + default: "centos9" + + - string: + name: ARCHS + description: "A list of architectures to build for. Available options are: x86_64 and arm64" + default: "x86_64" + + - bool: + name: THROWAWAY + description: "Whether to push any binaries to Chacra" + default: true + + - bool: + name: FORCE + description: "Whether to push new binaries to Chacra if some are already present" + default: true + + - choice: + name: FLAVOR + choices: + - default + - crimson + - jaeger + default: "default" + description: "Type of Ceph build, choices are: crimson, jaeger, default. Defaults to: 'default'" + + - bool: + name: CI_CONTAINER + description: "Whether to build and push container images" + default: false + + - string: + name: CONTAINER_REPO_HOSTNAME + description: "FQDN of container repo server (e.g. 'quay.io')" + default: "quay-quay-quay.apps.os.sepia.ceph.com" + + - string: + name: CONTAINER_REPO_ORGANIZATION + description: "Name of container repo organization (e.g. 'ceph-ci')" + default: "ceph-ci" + + - bool: + name: DWZ + description: "Use dwz to make debuginfo packages smaller" + default: false + + - bool: + name: SCCACHE + description: "Use sccache to speed up compilation" + default: true + + - string: + name: SETUP_BUILD_ID + description: "Use the source distribution from this ceph-dev-new-setup build instead of creating a new one" + default: "" + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - credentials-binding: + - text: + credential-id: shaman-api-key + variable: SHAMAN_API_KEY diff --git a/scripts/setup_sccache.sh b/scripts/setup_sccache.sh index 2b9b8ccf..08138c27 100644 --- a/scripts/setup_sccache.sh +++ b/scripts/setup_sccache.sh @@ -3,8 +3,10 @@ set -ex +SCCACHE_URL="https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-$(uname -m)-unknown-linux-musl.tar.gz" + function write_sccache_conf() { - export SCCACHE_CONF=$WORKSPACE/sccache.conf + export SCCACHE_CONF=${SCCACHE_CONF:-$WORKSPACE/sccache.conf} cat << EOF > $SCCACHE_CONF [cache.s3] bucket = "ceph-sccache" @@ -28,6 +30,40 @@ EOF } function install_sccache () { - SCCACHE_URL="https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-$(uname -m)-unknown-linux-musl.tar.gz" - curl -L $SCCACHE_URL | sudo tar --no-anchored --strip-components=1 -C /usr/local/bin/ -xzf - sccache + local sudo + if [ "$(id -u)" != "0" ]; then + sudo="sudo" + fi + curl -L $SCCACHE_URL | $sudo tar --no-anchored --strip-components=1 -C /usr/local/bin/ -xzf - sccache +} + +function setup_pbuilderrc () { + cat >> ~/.pbuilderrc << EOF +export SCCACHE="${SCCACHE}" +export SCCACHE_CONF=/etc/sccache.conf +export DWZ="${DWZ}" +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" +EOF + sudo cp ~/.pbuilderrc /root/.pbuilderrc +} + +function setup_sccache_pbuilder_hook () { + for hook_dir in $(ls -d ~/.pbuilder/hook*.d); do + hook=$hook_dir/D09-setup-sccache + cp $BASH_SOURCE $hook + cat >> $hook << EOF +if [ "$SCCACHE" = true ] ; then + write_sccache_conf + write_aws_credentials + install_sccache +fi +EOF + chmod +x $hook + done +} + +function reset_sccache () { + sccache --zero-stats + sccache --stop-server } -- 2.39.5