return text.toString()
}
+// Stage 1 (source distribution): trigger the ceph-source-dist job and record its number/URL OR reuse SETUP_BUILD_ID.
+def doSourceDistribution() {
+ if ( ! env.SETUP_BUILD_ID ) {
+ def setup_build = build(
+ job: env.SETUP_JOB,
+ parameters: [
+ string(name: "BRANCH", value: env.BRANCH),
+ // Below are just for ceph-source-dist
+ string(name: "SHA1", value: env.SHA1),
+ string(name: "CEPH_REPO", value: env.CEPH_REPO),
+ string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH),
+ // Below are only for actual releases
+ string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''),
+ string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''),
+ string(name: 'VERSION', value: env.VERSION ?: '')
+ ]
+ )
+ env.SETUP_BUILD_ID = setup_build.getNumber()
+ }
+ println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}"
+ env.SETUP_BUILD_URL = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize()
+ println "${env.SETUP_BUILD_URL}"
+}
+
+
+// Stage 2 (node): seed build_matrix for this cell, log the assigned node, and install the container runtime. Kept inline (no helper) so it can read the matrix-axis bindings DIST/ARCH/FLAVOR directly.
+// Stage 3 (checkout ceph-build): clone the ceph-build repo into the workspace. Kept inline because it's a single declarative `checkout` step with no script-block logic.
+
+// Stage 4 (copy artifacts): copy artifacts from the setup job, verify SHA1, set VERSION/build description, and extract the source tarball.
+def doCopyArtifactsStage() {
+ def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2"
+ def os = get_os_info(env.DIST)
+ if ( env.CI_COMPILE && os.pkg_type == "deb" ) {
+ artifact_filter += ",dist/ceph_*.diff.gz,dist/ceph_*.dsc"
+ }
+ println artifact_filter
+ copyArtifacts(
+ projectName: env.SETUP_JOB,
+ selector: specific(buildNumber: env.SETUP_BUILD_ID),
+ filter: artifact_filter,
+ )
+ sh 'sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d\" \" -f3 > $WORKSPACE/cursor'
+ def sha1_trimmed = env.SHA1.trim().toLowerCase()
+ def sha1_props = readProperties file: "${env.WORKSPACE}/dist/sha1"
+ def sha1_from_artifact = sha1_props.SHA1.trim().toLowerCase()
+ if ( env.SHA1 && sha1_from_artifact != sha1_trimmed ) {
+ error message: "SHA1 from artifact (${sha1_from_artifact}) does not match parameter value (${sha1_trimmed})"
+ } else if ( ! env.SHA1 ) {
+ env.SHA1 = sha1_from_artifact
+ }
+ println "SHA1=${sha1_trimmed}"
+ env.VERSION = readFile(file: "${env.WORKSPACE}/dist/version").trim()
+ // In a release build, dist/other_envvars contains CEPH_REPO, and chacra_url
+ // as written during ceph-source-dist but we don't to be able to define
+ // ceph-releases.git or chacra.ceph.com as parameters for ceph-dev-pipeline.
+ def props = readProperties file: "${env.WORKSPACE}/dist/other_envvars"
+ for (def key in props.keySet()) {
+ env."${key}" = props[key]
+ }
+ def branch_ui_value = env.BRANCH
+ def sha1_ui_value = env.SHA1
+ if (env.CEPH_REPO?.find(/https?:\/\/github.com\//)) {
+ // If this is a release build, link to ceph-release.git's $BRANCH-release branch
+ def suffix = (env.RELEASE_BUILD?.trim() == "true") ? "-release" : ""
+
+ def branch_url = "${env.CEPH_REPO}/tree/${env.BRANCH}${suffix}"
+ branch_ui_value = "<a href=\"${branch_url}\">${env.BRANCH}${suffix}</a>"
+ def commit_url = "${env.CEPH_REPO}/commit/${env.SHA1}"
+ sha1_ui_value = "<a href=\"${commit_url}\">${env.SHA1}</a>"
+ }
+ def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}"
+ def build_description = """\
+ BRANCH=${branch_ui_value}<br />
+ SHA1=${sha1_ui_value}<br />
+ VERSION=${env.VERSION}<br />
+ DISTROS=${env.DISTROS}<br />
+ ARCHS=${env.ARCHS}<br />
+ FLAVORS=${env.FLAVORS}<br />
+ <a href="${env.SETUP_BUILD_URL}">SETUP_BUILD_ID=${env.SETUP_BUILD_ID}</a><br />
+ <a href="${shaman_url}">shaman builds for this branch+commit</a>
+ """.stripIndent()
+ buildDescription build_description
+ sh "sha256sum dist/*"
+ sh "cat dist/sha1 dist/version"
+ sh '''#!/bin/bash
+ set -ex
+ cd dist
+ mkdir ceph
+ tar --strip-components=1 -C ceph -xjf ceph-${VERSION}.tar.bz2 ceph-${VERSION}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script}
+ '''
+}
+
+// Stage 5 (check for built packages): check chacra for an existing build (and skip compilation if found, unless FORCE) and set up the Pulp client if needed.
+def doCheckForBuiltPackagesStage() {
+ sh './scripts/setup_chacractl.sh'
+ def chacra_url = sh(
+ script: '''grep url ~/.chacractl | cut -d'"' -f2''',
+ returnStdout: true,
+ ).trim()
+ def os = get_os_info(env.DIST)
+ def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/"
+ def chacractl_rc = sh(
+ script: "${env.HOME}/.local/bin/chacractl exists binaries/${chacra_endpoint}",
+ returnStatus: true,
+ )
+ if ( chacractl_rc == 0 && env.FORCE != "true" ) {
+ println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).")
+ build_matrix["${env.DIST}_${env.ARCH}"] = false
+ }
+
+ // Set up Pulp client
+ if ( env.PULP_UPLOAD == "true" ) {
+ withCredentials([
+ usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD')
+ ]) {
+ sh './scripts/setup_pulp.sh'
+ }
+ }
+}
+
+// Stage 6 (builder container): log into container registries and pull/build/push the ceph-build container image for this matrix cell.
+def doBuilderContainerStage() {
+ env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build"
+ sh '''#!/bin/bash
+ set -ex
+ podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
+ podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io
+ '''
+ def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}"
+ def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}"
+ sh """#!/bin/bash -ex
+ podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \
+ podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \
+ true
+ """
+ def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef'])
+ sh """#!/bin/bash
+ set -ex
+ echo > .env
+ echo "WITH_CRIMSON=${withCrimson}" >> .env
+ cd dist/ceph
+ python3 src/script/build-with-container.py --env-file=${env.WORKSPACE}/.env --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} --image-variant=packages -d ${env.DIST} -e build-container
+ podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
+ """
+ sh """#!/bin/bash -ex
+ podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
+ podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag}
+ """
+}
+
+// Stage 7 (build): run the actual package build via build-with-container.py: assemble cmake/sccache/dwz flags per distro+flavor, then extract cephadm.
+def doBuildStage() {
+ def os = get_os_info(env.DIST)
+ sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} ${env.ARCH}"
+ env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR
+ env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW
+ def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}"
+ def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}"
+ def bwc_command = bwc_command_base
+ def bwc_cmd_sccache_flags = ""
+ if ( env.DWZ == "false" ) {
+ sh '''#!/bin/bash
+ echo "DWZ=$DWZ" >> .env
+ '''
+ bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
+ }
+ if ( env.SCCACHE == "true" ) {
+ sh '''#!/bin/bash
+ echo "SCCACHE=$SCCACHE" >> .env
+ echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env
+ echo "SCCACHE_ERROR_LOG=/ceph/sccache_log.txt" >> .env
+ echo "SCCACHE_LOG=debug" >> .env
+ echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env
+ echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env
+ echo "CEPH_BUILD_NORMALIZE_PATHS=true" >> .env
+ '''
+ // TODO: un-hardcode this
+ writeFile(
+ file: "dist/ceph/sccache.conf",
+ text: """\
+ [cache.s3]
+ bucket = "ceph-sccache"
+ endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud"
+ use_ssl = true
+ key_prefix = ""
+ server_side_encryption = false
+ no_credentials = false
+ region = "auto"
+ """
+ )
+ bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
+ }
+ def ceph_extra_cmake_args = "";
+ def deb_build_profiles = "";
+ switch (env.FLAVOR) {
+ case "default":
+ ceph_extra_cmake_args += " -DALLOCATOR=tcmalloc"
+ ceph_extra_cmake_args += " -DWITH_SYSTEM_BOOST=OFF -DWITH_BOOST_VALGRIND=ON"
+ if (os.version_name == "focal") {
+ ceph_extra_cmake_args += " -DWITH_STATIC_LIBSTDCXX=ON"
+ }
+ break
+ case "debug":
+ ceph_extra_cmake_args += " -DCMAKE_BUILD_TYPE=Debug"
+ break
+ default:
+ println "FLAVOR=${env.FLAVOR} is invalid"
+ assert false
+ }
+ bwc_command = "${bwc_command} ${bwc_cmd_sccache_flags}"
+ if ( os.pkg_type == "deb" ) {
+ def sccache_flag = "-DWITH_SCCACHE=ON"
+ if ( env.SCCACHE == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) {
+ ceph_extra_cmake_args += " ${sccache_flag}"
+ }
+ if ( deb_build_profiles ) {
+ sh """#!/bin/bash
+ echo "DEB_BUILD_PROFILES=${deb_build_profiles}" >> .env
+ """
+ }
+ bwc_command = "${bwc_command} -e debs"
+ } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) {
+ def rpmbuild_args = ""
+ if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache"
+ if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz"
+ if ( env.FLAVOR == "default" ) rpmbuild_args += " -R--with=tcmalloc"
+ bwc_command = "${bwc_command}${rpmbuild_args} -e rpm"
+ } else if ( env.DIST =~ /suse|sles/ ) {
+ throw new Exception("bwc not implemented for ${env.DIST}")
+ } else if ( env.DIST =~ /windows/ ) {
+ throw new Exception("bwc not implemented for ${env.DIST}")
+ } else {
+ throw new Exception("DIST '${env.DIST}' is invalid!")
+ }
+ sh """#!/bin/bash
+ echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env
+ """
+ sh """#!/bin/bash -ex
+ cd dist/ceph
+ ln ../ceph-${env.VERSION}.tar.bz2 .
+ ${bwc_command}
+ """
+ if ( os.pkg_type == "deb" ) {
+ sh """#!/bin/bash -ex
+ cd dist/ceph
+ ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${env.VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm"
+ ln ./cephadm ../../
+ """
+ } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) {
+ sh """#!/bin/bash -ex
+ cd dist/ceph
+ ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm"
+ ln ./cephadm ../../
+ """
+ }
+}
+
+// Stage 7 (build) post-always: archive the sccache log and remove .qa symlinks Jenkins gets stuck recursing into.
+def doBuildPostAlways() {
+ if (fileExists('dist/ceph/sccache_log.txt')) {
+ sh """
+ if [ -f "${env.WORKSPACE}/dist/ceph/sccache_log.txt" ]; then
+ ln dist/ceph/sccache_log.txt sccache_log_${env.DIST}_${env.ARCH}_${env.FLAVOR}.txt
+ fi
+ """
+ // The below is to work around an issue where Jenkins would recurse into
+ // dist/ceph/qa and get stuck trying to follow the .qa symlinks. This was
+ // discovered by seeing many messages about "too many levels of symbolic links"
+ // in the system journal.
+ sh "find ${env.WORKSPACE}/dist/ceph/ -name .qa -exec rm {} \\;"
+ archiveArtifacts(
+ artifacts: 'sccache_log*.txt',
+ allowEmptyArchive: true,
+ fingerprint: true,
+ )
+ }
+}
+
+// Stage 8 (upload packages): build the ceph-release rpm (for rpm distros) and upload the packages to chacra and optionally Pulp.
+def doUploadPackagesStage() {
+ def chacra_url = sh(
+ script: '''grep url ~/.chacractl | cut -d'"' -f2''',
+ returnStdout: true,
+ ).trim()
+ def os = get_os_info(env.DIST)
+ if ( os.pkg_type == "rpm" ) {
+ sh """#!/bin/bash
+ set -ex
+ cd ./dist/ceph
+ mkdir -p ./rpmbuild/SRPMS/
+ ln ceph-*.src.rpm ./rpmbuild/SRPMS/
+ """
+ // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true
+ env.SHA1 = env.TEST?.toBoolean() ? 'test' : env.SHA1
+ def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}/")
+ writeFile(
+ file: "dist/ceph/rpmbuild/SPECS/ceph-release.spec",
+ text: spec_text,
+ )
+ def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}")
+ writeFile(
+ file: "dist/ceph/rpmbuild/SOURCES/ceph.repo",
+ text: repo_text,
+ )
+ def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}"
+ def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}"
+ def bwc_command = "${bwc_command_base} -e custom -- rpmbuild -bb --define \\'_topdir /ceph/rpmbuild\\' /ceph/rpmbuild/SPECS/ceph-release.spec"
+ sh """#!/bin/bash
+ set -ex
+ cd ${env.WORKSPACE}/dist/ceph
+ ${bwc_command}
+ """
+ }
+ sh """#!/bin/bash
+ export CHACRA_URL="${chacra_url}"
+ export OS_NAME="${os.name}"
+ export OS_VERSION="${os.version}"
+ export OS_VERSION_NAME="${os.version_name}"
+ export OS_PKG_TYPE="${os.pkg_type}"
+ if [ "${env.THROWAWAY}" != "true" ]; then ./scripts/chacra_upload.sh; fi
+ """
+
+ sh """#!/bin/bash
+ if [ "${env.THROWAWAY}" != "true" ] && [ "${env.PULP_UPLOAD}" == "true" ]; then
+ export CEPH_VERSION="${env.VERSION}"
+ export OS_NAME="${os.name}"
+ export OS_VERSION="${os.version}"
+ export OS_VERSION_NAME="${os.version_name}"
+ export OS_PKG_TYPE="${os.pkg_type}"
+ export SHA1="${env.SHA1}"
+ export FLAVOR="${env.FLAVOR}"
+
+ ./scripts/pulp_upload.sh
+ else
+ echo "Skipping pulp upload because PULP_UPLOAD=${env.PULP_UPLOAD}"
+ fi
+ """
+}
+
+// Stage 9 (container): build the Ceph container image via scripts/build_container.
+def doContainerStage() {
+ env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR
+ env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW
+ def os = get_os_info(env.DIST)
+ def cephver = env.VERSION.trim()
+ sh """#!/bin/bash
+ export DISTRO=${os.name}
+ export RELEASE=${os.version}
+ export cephver=${cephver}
+ if [ "\$PULP_REGISTRY_UPLOAD" == "true" ]; then
+ export REMOVE_LOCAL_IMAGES=false
+ fi
+ ./scripts/build_container
+ """
+}
+
+// Stage 10 (registry upload): mirror the just-built dev container image from Quay to the Pulp container registry.
+def doRegistryUploadStage() {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'pulp-auth',
+ usernameVariable: 'PULP_USERNAME',
+ passwordVariable: 'PULP_PASSWORD'
+ )
+ ]) {
+ sh """#!/bin/bash
+ set -ex
+ ./scripts/pulp_container_push.sh
+ """
+ }
+}
+
pipeline {
agent any
stages {
stage("source distribution") {
steps {
- script {
- if ( ! env.SETUP_BUILD_ID ) {
- def setup_build = build(
- job: env.SETUP_JOB,
- parameters: [
- string(name: "BRANCH", value: env.BRANCH),
- // Below are just for ceph-source-dist
- string(name: "SHA1", value: env.SHA1),
- string(name: "CEPH_REPO", value: env.CEPH_REPO),
- string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH),
- // Below are only for actual releases
- string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''),
- string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''),
- string(name: 'VERSION', value: env.VERSION ?: '')
- ]
- )
- env.SETUP_BUILD_ID = setup_build.getNumber()
- }
- println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}"
- env.SETUP_BUILD_URL = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize()
- println "${env.SETUP_BUILD_URL}"
- }
+ script { doSourceDistribution() }
}
}
stage("parallel build") {
}
}
stages {
+ // Stage 2 (node): seed build_matrix for this cell, log the assigned node, and install the container runtime. Kept inline (no helper) so it can read the matrix-axis bindings DIST/ARCH/FLAVOR directly.
stage("node") {
steps {
script {
sh "cat /etc/os-release"
}
}
+ // Stage 3 (checkout ceph-build): clone the ceph-build repo into the workspace. Kept inline because it's a single declarative `checkout` step with no script-block logic.
stage("checkout ceph-build") {
steps {
checkout scmGit(
}
stage("copy artifacts") {
steps {
- script {
- def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2"
- def os = get_os_info(env.DIST)
- if ( env.CI_COMPILE && os.pkg_type == "deb" ) {
- artifact_filter += ",dist/ceph_*.diff.gz,dist/ceph_*.dsc"
- }
- println artifact_filter
- copyArtifacts(
- projectName: env.SETUP_JOB,
- selector: specific(buildNumber: env.SETUP_BUILD_ID),
- filter: artifact_filter,
- )
- }
- sh 'sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d\" \" -f3 > $WORKSPACE/cursor'
- script {
- def sha1_trimmed = env.SHA1.trim().toLowerCase()
- def sha1_props = readProperties file: "${WORKSPACE}/dist/sha1"
- sha1_from_artifact = sha1_props.SHA1.trim().toLowerCase()
- if ( env.SHA1 && sha1_from_artifact != sha1_trimmed ) {
- error message: "SHA1 from artifact (${sha1_from_artifact}) does not match parameter value (${sha1_trimmed})"
- } else if ( ! env.SHA1 ) {
- env.SHA1 = sha1_from_artifact
- }
- println "SHA1=${sha1_trimmed}"
- env.VERSION = readFile(file: "${WORKSPACE}/dist/version").trim()
- script {
- // In a release build, dist/other_envvars contains CEPH_REPO, and chacra_url
- // as written during ceph-source-dist but we don't to be able to define
- // ceph-releases.git or chacra.ceph.com as parameters for ceph-dev-pipeline.
- def props = readProperties file: "${WORKSPACE}/dist/other_envvars"
- for (def key in props.keySet()) {
- env."${key}" = props[key]
- }
- }
- def branch_ui_value = env.BRANCH
- def sha1_ui_value = env.SHA1
- if (env.CEPH_REPO?.find(/https?:\/\/github.com\//)) {
- // If this is a release build, link to ceph-release.git's $BRANCH-release branch
- def suffix = (env.RELEASE_BUILD?.trim() == "true") ? "-release" : ""
-
- def branch_url = "${env.CEPH_REPO}/tree/${env.BRANCH}${suffix}"
- branch_ui_value = "<a href=\"${branch_url}\">${env.BRANCH}${suffix}</a>"
- def commit_url = "${env.CEPH_REPO}/commit/${env.SHA1}"
- sha1_ui_value = "<a href=\"${commit_url}\">${env.SHA1}</a>"
- }
- def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}"
- def build_description = """\
- BRANCH=${branch_ui_value}<br />
- SHA1=${sha1_ui_value}<br />
- VERSION=${env.VERSION}<br />
- DISTROS=${env.DISTROS}<br />
- ARCHS=${env.ARCHS}<br />
- FLAVORS=${env.FLAVORS}<br />
- <a href="${env.SETUP_BUILD_URL}">SETUP_BUILD_ID=${env.SETUP_BUILD_ID}</a><br />
- <a href="${shaman_url}">shaman builds for this branch+commit</a>
- """.stripIndent()
- buildDescription build_description
- }
- sh "sha256sum dist/*"
- sh "cat dist/sha1 dist/version"
- sh '''#!/bin/bash
- set -ex
- cd dist
- mkdir ceph
- tar --strip-components=1 -C ceph -xjf ceph-${VERSION}.tar.bz2 ceph-${VERSION}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script}
- '''
+ script { doCopyArtifactsStage() }
}
}
stage("check for built packages") {
SHAMAN_API_KEY = credentials('shaman-api-key')
}
steps {
- script {
- sh './scripts/setup_chacractl.sh'
- def chacra_url = sh(
- script: '''grep url ~/.chacractl | cut -d'"' -f2''',
- returnStdout: true,
- ).trim()
- def os = get_os_info(env.DIST)
- def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/"
- def chacractl_rc = sh(
- script: "$HOME/.local/bin/chacractl exists binaries/${chacra_endpoint}",
- returnStatus: true,
- )
- if ( chacractl_rc == 0 && env.FORCE != "true" ) {
- println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).")
- build_matrix["${DIST}_${ARCH}"] = false
- }
-
- // Set up Pulp client
- if ( env.PULP_UPLOAD == "true" ) {
- withCredentials([
- usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD')
- ]) {
- sh './scripts/setup_pulp.sh'
- }
- }
- }
+ script { doCheckForBuiltPackagesStage() }
}
}
stage("builder container") {
expression { return build_matrix["${DIST}_${ARCH}"] == true }
}
steps {
- script {
- env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build"
- sh '''#!/bin/bash
- set -ex
- podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
- podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io
- '''
- def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}"
- def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}"
- sh """#!/bin/bash -ex
- podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \
- podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \
- true
- """
- def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef'])
- sh """#!/bin/bash
- set -ex
- echo > .env
- echo "WITH_CRIMSON=${withCrimson}" >> .env
- cd dist/ceph
- python3 src/script/build-with-container.py --env-file=${env.WORKSPACE}/.env --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} --image-variant=packages -d ${DIST} -e build-container
- podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
- """
- sh """#!/bin/bash -ex
- podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short}
- podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag}
- """
- }
+ script { doBuilderContainerStage() }
}
}
stage("build") {
expression { return build_matrix["${DIST}_${ARCH}"] == true }
}
steps {
- script {
- def os = get_os_info(env.DIST)
- sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} $ARCH"
- env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR
- env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW
- def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}"
- def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} --image-variant=packages --ceph-version ${env.VERSION}"
- def bwc_command = bwc_command_base
- def bwc_cmd_sccache_flags = ""
- if ( env.DWZ == "false" ) {
- sh '''#!/bin/bash
- echo "DWZ=$DWZ" >> .env
- '''
- bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
- }
- if ( env.SCCACHE == "true" ) {
- sh '''#!/bin/bash
- echo "SCCACHE=$SCCACHE" >> .env
- echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env
- echo "SCCACHE_ERROR_LOG=/ceph/sccache_log.txt" >> .env
- echo "SCCACHE_LOG=debug" >> .env
- echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env
- echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env
- echo "CEPH_BUILD_NORMALIZE_PATHS=true" >> .env
- '''
- // TODO: un-hardcode this
- writeFile(
- file: "dist/ceph/sccache.conf",
- text: """\
- [cache.s3]
- bucket = "ceph-sccache"
- endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud"
- use_ssl = true
- key_prefix = ""
- server_side_encryption = false
- no_credentials = false
- region = "auto"
- """
- )
- bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
- }
- def ceph_extra_cmake_args = "";
- def deb_build_profiles = "";
- switch (env.FLAVOR) {
- case "default":
- ceph_extra_cmake_args += " -DALLOCATOR=tcmalloc"
- ceph_extra_cmake_args += " -DWITH_SYSTEM_BOOST=OFF -DWITH_BOOST_VALGRIND=ON"
- if (os.version_name == "focal") {
- ceph_extra_cmake_args += " -DWITH_STATIC_LIBSTDCXX=ON"
- }
- break
- case "debug":
- ceph_extra_cmake_args += " -DCMAKE_BUILD_TYPE=Debug"
- break
- default:
- println "FLAVOR=${env.FLAVOR} is invalid"
- assert false
- }
- bwc_command = "${bwc_command} ${bwc_cmd_sccache_flags}"
- if ( os.pkg_type == "deb" ) {
- def sccache_flag = "-DWITH_SCCACHE=ON"
- if ( env.SCCACHE == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) {
- ceph_extra_cmake_args += " ${sccache_flag}"
- }
- if ( deb_build_profiles ) {
- sh """#!/bin/bash
- echo "DEB_BUILD_PROFILES=${deb_build_profiles}" >> .env
- """
- }
- bwc_command = "${bwc_command} -e debs"
- } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) {
- def rpmbuild_args = ""
- if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache"
- if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz"
- if ( env.FLAVOR == "default" ) rpmbuild_args += " -R--with=tcmalloc"
- bwc_command = "${bwc_command}${rpmbuild_args} -e rpm"
- } else if ( env.DIST =~ /suse|sles/ ) {
- throw new Exception("bwc not implemented for ${env.DIST}")
- } else if ( env.DIST =~ /windows/ ) {
- throw new Exception("bwc not implemented for ${env.DIST}")
- } else {
- throw new Exception("DIST '${env.DIST}' is invalid!")
- }
- sh """#!/bin/bash
- echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env
- """
- sh """#!/bin/bash -ex
- cd dist/ceph
- ln ../ceph-${env.VERSION}.tar.bz2 .
- ${bwc_command}
- """
- if ( os.pkg_type == "deb" ) {
- sh """#!/bin/bash -ex
- cd dist/ceph
- ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm"
- ln ./cephadm ../../
- """
- } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) {
- sh """#!/bin/bash -ex
- cd dist/ceph
- ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm"
- ln ./cephadm ../../
- """
- }
- }
+ script { doBuildStage() }
}
post {
always {
- script {
- if (fileExists('dist/ceph/sccache_log.txt')) {
- sh """
- if [ -f "${env.WORKSPACE}/dist/ceph/sccache_log.txt" ]; then
- ln dist/ceph/sccache_log.txt sccache_log_${env.DIST}_${env.ARCH}_${env.FLAVOR}.txt
- fi
- """
- // The below is to work around an issue where Jenkins would recurse into
- // dist/ceph/qa and get stuck trying to follow the .qa symlinks. This was
- // discovered by seeing many messages about "too many levels of symbolic links"
- // in the system journal.
- sh "find ${env.WORKSPACE}/dist/ceph/ -name .qa -exec rm {} \\;"
- archiveArtifacts(
- artifacts: 'sccache_log*.txt',
- allowEmptyArchive: true,
- fingerprint: true,
- )
- }
- }
+ script { doBuildPostAlways() }
}
unsuccessful {
script {
def os = get_os_info(env.DIST)
- sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} $ARCH"
+ sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} ${env.ARCH}"
}
}
}
expression { return build_matrix["${DIST}_${ARCH}"] == true }
}
steps {
- script {
- def chacra_url = sh(
- script: '''grep url ~/.chacractl | cut -d'"' -f2''',
- returnStdout: true,
- ).trim()
- def os = get_os_info(env.DIST)
- if ( os.pkg_type == "rpm" ) {
- sh """#!/bin/bash
- set -ex
- cd ./dist/ceph
- mkdir -p ./rpmbuild/SRPMS/
- ln ceph-*.src.rpm ./rpmbuild/SRPMS/
- """
- // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true
- env.SHA1 = env.TEST?.toBoolean() ? 'test' : env.SHA1
- def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}/")
- writeFile(
- file: "dist/ceph/rpmbuild/SPECS/ceph-release.spec",
- text: spec_text,
- )
- def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}")
- writeFile(
- file: "dist/ceph/rpmbuild/SOURCES/ceph.repo",
- text: repo_text,
- )
- def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${ARCH}.${FLAVOR}"
- def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} --image-variant=packages --ceph-version ${env.VERSION}"
- bwc_command = "${bwc_command_base} -e custom -- rpmbuild -bb --define \\'_topdir /ceph/rpmbuild\\' /ceph/rpmbuild/SPECS/ceph-release.spec"
- sh """#!/bin/bash
- set -ex
- cd $WORKSPACE/dist/ceph
- ${bwc_command}
- """
- }
- sh """#!/bin/bash
- export CHACRA_URL="${chacra_url}"
- export OS_NAME="${os.name}"
- export OS_VERSION="${os.version}"
- export OS_VERSION_NAME="${os.version_name}"
- export OS_PKG_TYPE="${os.pkg_type}"
- if [ "$THROWAWAY" != "true" ]; then ./scripts/chacra_upload.sh; fi
- """
-
- sh """#!/bin/bash
- if [ "$THROWAWAY" != "true" ] && [ "$PULP_UPLOAD" == "true" ]; then
- export CEPH_VERSION="${env.VERSION}"
- export OS_NAME="${os.name}"
- export OS_VERSION="${os.version}"
- export OS_VERSION_NAME="${os.version_name}"
- export OS_PKG_TYPE="${os.pkg_type}"
- export SHA1="${env.SHA1}"
- export FLAVOR="${env.FLAVOR}"
-
- ./scripts/pulp_upload.sh
- else
- echo "Skipping pulp upload because PULP_UPLOAD=$PULP_UPLOAD"
- fi
- """
- }
+ script { doUploadPackagesStage() }
}
post {
success {
script {
def os = get_os_info(env.DIST)
- sh "./scripts/update_shaman.sh completed ceph ${os.name} ${os.version_name} $ARCH"
+ sh "./scripts/update_shaman.sh completed ceph ${os.name} ${os.version_name} ${env.ARCH}"
}
}
unsuccessful {
script {
def os = get_os_info(env.DIST)
- sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} $ARCH"
+ sh "./scripts/update_shaman.sh failed ceph ${os.name} ${os.version_name} ${env.ARCH}"
}
}
}
CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci')
}
steps {
- script {
- env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR
- env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW
- def os = get_os_info(env.DIST)
- cephver = env.VERSION.trim()
- sh """#!/bin/bash
- export DISTRO=${os.name}
- export RELEASE=${os.version}
- export cephver=${cephver}
- if [ "\$PULP_REGISTRY_UPLOAD" == "true" ]; then
- export REMOVE_LOCAL_IMAGES=false
- fi
- ./scripts/build_container
- """
- }
+ script { doContainerStage() }
}
}
stage("registry upload") {
}
}
steps {
- script {
- withCredentials([
- usernamePassword(
- credentialsId: 'pulp-auth',
- usernameVariable: 'PULP_USERNAME',
- passwordVariable: 'PULP_PASSWORD'
- )
- ]) {
- sh """#!/bin/bash
- set -ex
- ./scripts/pulp_container_push.sh
- """
- }
- }
+ script { doRegistryUploadStage() }
}
}
}
projects / ceph-build.git / commitdiff