From f9ef5775022aea501a3cd84b3bd6397153216be2 Mon Sep 17 00:00:00 2001 From: Adarsha Dinda Date: Wed, 18 Jun 2025 19:05:06 +0530 Subject: [PATCH] Added Jenkins job for ceph build --- ceph-build-utils/build/Jenkinsfile | 146 ++++++++++++++++++ .../config/ceph-build-pipeline.yaml | 92 +++++++++++ 2 files changed, 238 insertions(+) create mode 100644 ceph-build-utils/build/Jenkinsfile create mode 100644 ceph-build-utils/config/ceph-build-pipeline.yaml diff --git a/ceph-build-utils/build/Jenkinsfile b/ceph-build-utils/build/Jenkinsfile new file mode 100644 index 000000000..7a0cfe2f5 --- /dev/null +++ b/ceph-build-utils/build/Jenkinsfile @@ -0,0 +1,146 @@ +ceph_build_repo = "https://github.com/ceph/ceph-build" +ceph_build_branch = "build_utils_refactor" +base_node_label = "gigantic" +ubuntu_releases = [ + "noble", // 24.04 + "jammy", // 22.04 + "focal", // 20.04 +] +debian_releases = [ + "bookworm", // v12 (2023) + "bullseye", // v11 (2021) +] + +pipeline { + agent any + stages { + stage("source distribution") { + steps { + script { + if ( ! env.SETUP_BUILD_ID ) { + def setup_build = build( + job: env.SETUP_JOB, + parameters: [ + string(name: "BRANCH", value: env.BRANCH), + string(name: "FLAVOR", value: env.FLAVOR), + string(name: "SCCACHE", value: env.SCCACHE), + string(name: "DWZ", value: env.DWZ), + ] + ) + env.SETUP_BUILD_ID = setup_build.getNumber() + } + println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}" + def setup_build_url = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize() + println "${setup_build_url}" + } + } + } + stage("parallel build") { + matrix { + agent { + label "${DIST}&&${ARCH}&&${base_node_label}" + } + when { + beforeAgent true + allOf { + expression { env.DISTROS.contains(env.DIST) } + expression { env.ARCHS.contains(env.ARCH) } + } + } + axes { + axis { + name 'DIST' + values 'centos9', 'jammy' + } + axis { + name 'ARCH' + values 'x86_64', 'arm64' + } + } + stages { + stage("node") { + steps { + script { + def node_shortname = env.NODE_NAME.split('\\+')[-1] + def node_url = new URI([env.JENKINS_URL, "computer", env.NODE_NAME].join("/")).normalize() + echo "DIST=${env.DIST} ARCH=${env.ARCH} FLAVOR=${env.FLAVOR}\n${node_shortname}\n${node_url}" + } + sh "cat /etc/os-release" + } + } + stage("checkout ceph-build") { + steps { + checkout scmGit( + branches: [[name: ceph_build_branch]], + userRemoteConfigs: [[url: ceph_build_repo]], + extensions: [ + [$class: 'CleanBeforeCheckout'] + ], + ) + } + } + stage("copy artifacts") { + steps { + copyArtifacts( + projectName: env.SETUP_JOB, + selector: specific(buildNumber: env.SETUP_BUILD_ID), + filter: "dist/sha1,dist/version,dist/other_envvars,dist/ceph_*.orig.tar.gz,dist/ceph-*.tar.bz2", + ) + script { + def sha1_props = readProperties file: "${WORKSPACE}/dist/sha1" + env.SHA1 = sha1_props.SHA1 + println "SHA1=${env.SHA1}" + env.VERSION = readFile file: "${WORKSPACE}/dist/version" + def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" + def build_description = """SHA1=${env.SHA1}
+ VERSION=${env.VERSION}
+ shaman builds for this branch+commit + """ + buildDescription build_description + } + sh "sha256sum dist/*" + sh "cat dist/sha1 dist/version" + } + } + stage("build") { + environment { + CHACRACTL_KEY = credentials('chacractl-key') + SHAMAN_API_KEY = credentials('shaman-api-key') + CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci') + SCCACHE_BUCKET_CREDS = credentials('ibm-cloud-sccache-bucket') + } + steps { + script { + if ( (debian_releases + ubuntu_releases).contains(env.DIST) ) { + def sccache_flag = "-DWITH_SCCACHE=ON" + if ( env.SCCACHE == "true" && ! env.CEPH_EXTRA_CMAKE_ARGS.contains(sccache_flag) ) { + env.CEPH_EXTRA_CMAKE_ARGS = "${env.CEPH_EXTRA_CMAKE_ARGS} ${sccache_flag}" + } + sh """#!/bin/bash + . ./ceph-build/build/validate_deb + . ./scripts/build_utils.sh + . ./ceph-build/build/setup_deb + . ./scripts/setup_sccache.sh + . ./ceph-build/build/setup_pbuilder + . ./ceph-build/build/build_deb + """ + } else if ( env.DIST =~ /^(centos|rhel|fedora).*/ ) { + sh """#!/bin/bash + . ./ceph-build/build/validate_rpm + . ./scripts/build_utils.sh + . ./scripts/setup_sccache.sh + . ./ceph-build/build/setup_rpm + . ./ceph-build/build/build_rpm + """ + } else { + throw new Exception("DIST '${env.DIST}' is invalid!") + } + } + sh "command -v sccache && sccache --show-adv-stats || true" + } + } + } + } + } + } +} diff --git a/ceph-build-utils/config/ceph-build-pipeline.yaml b/ceph-build-utils/config/ceph-build-pipeline.yaml new file mode 100644 index 000000000..a25795bc7 --- /dev/null +++ b/ceph-build-utils/config/ceph-build-pipeline.yaml @@ -0,0 +1,92 @@ +- job: + name: preserve-adarsha-test + description: ceph-build-utils-pipeline + project-type: pipeline + quiet-period: 1 + concurrent: true + pipeline-scm: + scm: + - git: + url: https://github.com/ceph/ceph-build + branches: + - ${{CEPH_BUILD_BRANCH}} + shallow-clone: true + submodule: + disable: true + wipe-workspace: true + script-path: ceph-build-utils/build/Jenkinsfile + lightweight-checkout: true + do-not-fetch-tags: true + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + default: main + + - string: + name: DISTROS + description: "A list of distros to build for. Available options are: centos9, centos8, noble, jammy, focal, and windows" + default: "centos9 jammy" + + - string: + name: ARCHS + description: "A list of architectures to build for. Available options are: x86_64 and arm64" + default: "x86_64 arm64" + + - bool: + name: THROWAWAY + description: "Whether to push any binaries to Chacra" + default: false + + - bool: + name: FORCE + description: "Whether to push new binaries to Chacra if some are already present" + default: false + + - choice: + name: FLAVOR + choices: + - default + - crimson + - jaeger + default: "default" + description: "Type of Ceph build, choices are: crimson, jaeger, default. Defaults to: 'default'" + + - bool: + name: DWZ + description: "Use dwz to make debuginfo packages smaller" + default: false + + - bool: + name: SCCACHE + description: "Use sccache to speed up compilation" + default: true + + - string: + name: SETUP_BUILD_ID + description: "Use the source distribution from this ceph-dev-new-setup build instead of creating a new one" + default: "" + + - string: + name: CEPH_BUILD_BRANCH + description: "Use the Jenkinsfile from this ceph-build branch" + default: build_utils_refactor + + - choice: + name: SETUP_JOB + choices: + - ceph-dev-setup + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - credentials-binding: + - text: + credential-id: shaman-api-key + variable: SHAMAN_API_KEY + - username-password-separated: + credential-id: quay.ceph.io-ceph-prerelease + username: CONTAINER_REPO_USERNAME + password: CONTAINER_REPO_PASSWORD \ No newline at end of file -- 2.47.3