From: David Galloway Date: Wed, 13 Aug 2025 23:01:14 +0000 (-0400) Subject: Foo X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=438fbacda383d82fe449c0fdf83af80fd502dbb6;p=ceph-build.git Foo Signed-off-by: David Galloway --- diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile index 2c614a5d..edf53e54 100644 --- a/ceph-dev-pipeline/build/Jenkinsfile +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -136,7 +136,37 @@ def get_ceph_release_repo_text(base_url) { pipeline { agent any + // environment { + // release builds must use ceph-releases; ignore any user-provided value + // CEPH_REPO = "${ params.RELEASE_BUILD?.trim() ? 'git@github.com:ceph/ceph-releases.git' : (params.CEPH_REPO ?: 'https://github.com/ceph/ceph') }" + // CEPH_REPO_LOCKED = "${ params.RELEASE_BUILD?.trim() ? 'true' : 'false' }" + // } stages { + stage('create ceph release tag') { + when { + expression { return params.RELEASE_BUILD?.trim() } // only run if defined & true + } + steps { + script { + def createTagJob = build( + job: 'preserve-ceph-tag-dgalloway', + parameters: [ + string(name: 'VERSION', value: env.VERSION ?: ''), + string(name: 'BRANCH', value: env.BRANCH ?: ''), + string(name: 'FORCE_VERSION', value: env.FORCE_VERSION ?: ''), + string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''), + string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''), + string(name: 'TAG', value: env.TAG ?: ''), + string(name: 'TAG_PHASE', value: 'create') + ], + ) + env.CEPH_TAG_CREATE_JOB_ID = createTagJob.getNumber() + println "CEPH_TAG_CREATE_JOB_ID=${env.CEPH_TAG_CREATE_JOB_ID}" + env.CEPH_TAG_CREATE_JOB_URL = new URI([env.JENKINS_URL, "job", "preserve-ceph-tag-dgalloway", env.CEPH_TAG_CREATE_JOB_ID].join("/")).normalize() + println "${env.SETUP_BUILD_URL}" + } + } + } stage("source distribution") { steps { script { @@ -148,6 +178,10 @@ pipeline { // Below are just for ceph-source-dist string(name: "CEPH_REPO", value: env.CEPH_REPO), string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH), + // Below are only for actual releases + string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''), + string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''), + string(name: 'VERSION', value: env.VERSION ?: '') ] ) env.SETUP_BUILD_ID = setup_build.getNumber() @@ -219,6 +253,7 @@ pipeline { stage("node") { steps { script { + sh "env" build_matrix["${DIST}_${ARCH}"] = env.CI_COMPILE.toBoolean() sh "hostname -f" def node_shortname = env.NODE_NAME.split('\\+')[-1] @@ -263,6 +298,7 @@ pipeline { ) } script { + sh "env" def sha1_props = readProperties file: "${WORKSPACE}/dist/sha1" env.SHA1 = sha1_props.SHA1 println "SHA1=${env.SHA1}" @@ -276,6 +312,12 @@ pipeline { sha1_ui_value = "${env.SHA1}" } def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" + script { + def props = readProperties file: "${WORKSPACE}/dist/other_envvars" + for (p in props) { + env."${p.key}" = p.value + } + } def build_description = """\ BRANCH=${branch_ui_value}
SHA1=${sha1_ui_value}
@@ -290,12 +332,7 @@ pipeline { } sh "sha256sum dist/*" sh "cat dist/sha1 dist/version" - script { - def props = readProperties file: "${WORKSPACE}/dist/other_envvars" - for (p in props) { - env."${p.key}" = p.value - } - } + sh '''#!/bin/bash set -ex cd dist @@ -315,6 +352,9 @@ pipeline { } steps { script { + if (env.RELEASE_BUILD?.trim()) { + def chacra_url = "https://chacra.ceph.com/" + } sh './scripts/setup_chacractl.sh' def chacra_url = sh( script: '''grep url ~/.chacractl | cut -d'"' -f2''', diff --git a/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml b/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml index 48b041e3..84a3b12f 100644 --- a/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml +++ b/ceph-dev-pipeline/config/definitions/ceph-dev-pipeline.yml @@ -1,6 +1,6 @@ - job: - name: ceph-dev-pipeline - description: ceph-dev-pipeline + name: preserve-ceph-dev-pipeline-dgalloway + description: preserve-ceph-dev-pipeline-dgalloway project-type: pipeline quiet-period: 1 concurrent: true @@ -102,13 +102,13 @@ - choice: name: SETUP_JOB choices: - - ceph-source-dist + - preserve-ceph-source-dist-dgalloway - ceph-dev-new-setup - string: name: CEPH_BUILD_BRANCH description: "Use the Jenkinsfile from this ceph-build branch" - default: main + default: release-with-pipeline wrappers: - inject-passwords: diff --git a/ceph-source-dist/build/Jenkinsfile b/ceph-source-dist/build/Jenkinsfile index 7a5724f6..84be83cb 100644 --- a/ceph-source-dist/build/Jenkinsfile +++ b/ceph-source-dist/build/Jenkinsfile @@ -1,29 +1,41 @@ pipeline { agent { - label "gigantic" + label "gigantic&&!arm64" } stages { stage("repository") { steps { dir("ceph") { - checkout scmGit( - branches: [[name: env.BRANCH]], - userRemoteConfigs: [[ - url: env.CEPH_REPO, - credentialsId: 'jenkins-build' - ]], - extensions: [ - [$class: 'CleanBeforeCheckout'], - [ - $class: 'CloneOption', - shallow: true, - depth: 100, - timeout: 90 + script { + // Choose repo + ref based on RELEASE_BUILD bool + // These are intentionally not definable by parent jobs for security + def repoUrl = params.RELEASE_BUILD ? 'git@github.com:ceph/ceph-releases.git' : env.CEPH_REPO + def refToCheckout = params.RELEASE_BUILD ? "v${params.VERSION}" : env.BRANCH + + // Make the chosen repo visible to later stages if needed + env.CEPH_REPO = repoUrl + + checkout scmGit( + branches: [[name: refToCheckout]], + userRemoteConfigs: [[ + url: repoUrl, + credentialsId: 'jenkins-build' + ]], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [ + $class: 'CloneOption', + shallow: true, + depth: 100, + timeout: 90 + ], ], - ], - ) + ) + } script { - sh 'git fetch --tags https://github.com/ceph/ceph.git' + if (!params.RELEASE_BUILD?.toBoolean()) { + sh 'git fetch --tags https://github.com/ceph/ceph.git' + } } } } @@ -55,12 +67,20 @@ pipeline { if [ "$ceph_version_tarball" != "" ]; then break; fi done echo tarball vers $ceph_version_tarball - + ## TODO: CMAKE ARGS FROM ceph-setup? ln ceph.spec dist/ ln ceph-$ceph_version_tarball.$extension dist/ echo "SHA1=$(git rev-parse HEAD)" > dist/sha1 - echo "BRANCH=${BRANCH}" > dist/branch + + if [ "${RELEASE_BUILD:-}" = "true" ]; then + echo "CEPH_REPO=https://github.com/ceph/ceph-releases" > dist/other_envvars + echo "RELEASE_BUILD=true" >> dist/other_envvars + echo "chacra_url=https://chacra.ceph.com/" >> dist/other_envvars + echo "BRANCH=${BRANCH}-release" > dist/branch + else + echo "BRANCH=${BRANCH}" > dist/branch + fi mv dist .. ''' diff --git a/ceph-source-dist/config/definitions/ceph-source-dist.yml b/ceph-source-dist/config/definitions/ceph-source-dist.yml index 758122b3..f92ce5c6 100644 --- a/ceph-source-dist/config/definitions/ceph-source-dist.yml +++ b/ceph-source-dist/config/definitions/ceph-source-dist.yml @@ -1,5 +1,5 @@ - job: - name: ceph-source-dist + name: preserve-ceph-source-dist-dgalloway project-type: pipeline concurrent: true pipeline-scm: @@ -22,7 +22,7 @@ artifact-days-to-keep: -1 artifact-num-to-keep: 50 - copyartifact: - projects: ceph-dev-pipeline,ceph-dev,ceph-dev-build,ceph-dev-new,ceph-dev-new-build + projects: ceph-dev-pipeline,ceph-dev,ceph-dev-build,ceph-dev-new,ceph-dev-new-build,preserve-ceph-dev-pipeline-dgalloway parameters: - choice: diff --git a/ceph-tag/build/build b/ceph-tag/build/build index 1fbde3d1..86ce70d2 100644 --- a/ceph-tag/build/build +++ b/ceph-tag/build/build @@ -17,5 +17,6 @@ else # run ansible to do all the tagging and release specifying # a local connection and 'localhost' as the host where to execute cd "$WORKSPACE/ceph-build/ansible/" - $VENV/ansible-playbook -i "localhost," -c local release.yml --extra-vars="stage=push version=$VERSION branch=$BRANCH force_version=$FORCE_VERSION release=$RELEASE_TYPE tag=$TAG project=ceph token=$GITHUB_TOKEN" + + $VENV/ansible-playbook -i "localhost," -c local release.yml --extra-vars="stage=$TAG_PHASE version=$VERSION branch=$BRANCH force_version=$FORCE_VERSION release=$RELEASE_TYPE tag=$TAG project=ceph token=$GITHUB_TOKEN" fi diff --git a/ceph-tag/config/definitions/ceph-tag.yml b/ceph-tag/config/definitions/ceph-tag.yml index ed39b1b5..d15b7652 100644 --- a/ceph-tag/config/definitions/ceph-tag.yml +++ b/ceph-tag/config/definitions/ceph-tag.yml @@ -1,6 +1,6 @@ - job: - name: ceph-tag - node: bionic + name: preserve-ceph-tag-dgalloway + node: 'bionic&&!arm64' description: "This job checks out the version commit previously pushed to ceph-releases.git and pushes it to ceph.git." display-name: 'ceph-tag' block-downstream: false diff --git a/ceph/build/create_tag b/ceph/build/create_tag new file mode 100644 index 00000000..5319c4d1 --- /dev/null +++ b/ceph/build/create_tag @@ -0,0 +1,17 @@ +#!/bin/bash + +set -ex + +# the following two methods exist in scripts/build_utils.sh +pkgs=( "ansible" ) +TEMPVENV=$(create_venv_dir) +VENV=${TEMPVENV}/bin +install_python_packages $TEMPVENV "pkgs[@]" + +# remove "-release" from $BRANCH variable in case it was accidentally passed in the Jenkins UI +BRANCH=${BRANCH//-release/} + +# run ansible to do all the tagging and release specifying +# a local connection and 'localhost' as the host where to execute +cd "$WORKSPACE/ceph-build/ansible/" +$VENV/ansible-playbook -i "localhost," -c local release.yml -vvv --extra-vars="stage=create version=$VERSION branch=$BRANCH force_version=$FORCE_VERSION release=$RELEASE_TYPE tag=$TAG throwaway=$THROWAWAY project=ceph" diff --git a/ceph/config/definitions/ceph.yml b/ceph/config/definitions/ceph.yml index a28412bf..74ec799b 100644 --- a/ceph/config/definitions/ceph.yml +++ b/ceph/config/definitions/ceph.yml @@ -1,9 +1,9 @@ - job: - name: ceph + name: preserve-ceph-release-dgalloway description: 'This is the main ceph build task which uses chacra.ceph.com.' project-type: multijob defaults: global - display-name: 'ceph' + display-name: 'preserve-ceph-release-dgalloway' block-downstream: false block-upstream: false concurrent: true @@ -22,6 +22,10 @@ description: "The git branch (or tag) to build (e.g., pacific) DO NOT INCLUDE '-release'" default: main + - string: + name: VERSION + description: "The version for release, e.g. 0.94.4" + - bool: name: TEST description: " @@ -54,10 +58,6 @@ If this is unchecked, then then nothing is built or pushed if they already exist If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra." - - string: - name: VERSION - description: "The version for release, e.g. 0.94.4" - - choice: name: RELEASE_TYPE description: " @@ -71,63 +71,32 @@ SECURITY: Builds from BRANCH-release branch in ceph-private.git (private repo)." - HOTFIX - SECURITY - - string: - name: CEPH_BUILD_VIRTUALENV - description: "Base parent path for virtualenv locations, set to avoid issues with extremely long paths that are incompatible with tools like pip. Defaults to '/tmp/' (note the trailing slash, which is required)." - default: "/tmp/" - - string: name: DISTROS description: "A list of distros to build for. Available options are: centos9, centos8, centos7, centos6, noble, jammy, focal, bionic, xenial, trusty, precise, wheezy, jessie, buster, bullseye, bookworm" - default: "noble jammy focal centos8 centos9 bookworm" + default: "noble jammy centos8 centos9 bookworm" - string: name: ARCHS description: "A list of architectures to build for. Available options are: x86_64, and arm64" default: "x86_64 arm64" - - string: - name: CONTAINER_REPO_HOSTNAME - description: "Name of (prerelease) container repo server (i.e. 'quay.ceph.io')" - default: "quay.ceph.io" - - - string: - name: CONTAINER_REPO_ORGANIZATION - description: "Name of (prerelease) container repo organization (i.e. 'ceph'). Container build script will add prerelease-" - default: "ceph" - - builders: - - multijob: - name: 'ceph setup phase' - condition: SUCCESSFUL - projects: - - name: ceph-setup - current-parameters: true - exposed-scm: false - - copyartifact: - project: ceph-setup - filter: ceph-build/ansible/ceph/dist/sha1 - which-build: multijob-build - - inject: - properties-file: ${{WORKSPACE}}/ceph-build/ansible/ceph/dist/sha1 - - multijob: - name: 'ceph build phase' - condition: SUCCESSFUL - projects: - - name: ceph-build - current-parameters: true - exposed-scm: false - - multijob: - name: 'ceph tag phase' - condition: SUCCESSFUL - projects: - - name: ceph-tag - current-parameters: true - exposed-scm: false + publishers: + - trigger-parameterized-builds: + - project: "preserve-ceph-dev-pipeline-dgalloway" + condition: SUCCESS + trigger-with-no-params: false + current-parameters: true + predefined-parameters: | + FLAVOR=default + CI_COMPILE=true + CI_CONTAINER=false + DWZ=false + SCCACHE=false + SETUP_JOB=preserve-ceph-source-dist-dgalloway + RELEASE_BUILD=true + block: true wrappers: - - inject-passwords: - global: true - mask-password-params: true - build-name: - name: "#${{BUILD_NUMBER}} ${{BRANCH}}, ${{SHA1}}" + name: "#${{BUILD_NUMBER}} ${{BRANCH}}, ${{VERSION}}" diff --git a/scripts/setup_chacractl.sh b/scripts/setup_chacractl.sh index 608a930d..7c65ddef 100755 --- a/scripts/setup_chacractl.sh +++ b/scripts/setup_chacractl.sh @@ -8,7 +8,9 @@ pipx ensurepath pipx install uv ~/.local/bin/uv tool install chacractl -chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +if [ -n "$chacra_url" ]; then + chacra_url=$(curl -u "$SHAMAN_API_USER:$SHAMAN_API_KEY" https://shaman.ceph.com/api/nodes/next/) +fi cat > $HOME/.chacractl << EOF url = "$chacra_url" user = "$CHACRACTL_USER"