From: Zack Cerza <zack@cerza.org>
Date: Mon, 10 Feb 2025 19:58:34 +0000 (-0700)
Subject: ceph-dev-pipeline: Build in containers
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=9fa5ed9a75cac78f2f5aa9c4506b8ccb13a45855;p=ceph-build.git

ceph-dev-pipeline: Build in containers

Signed-off-by: Zack Cerza <zack@cerza.org>
---

diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile
index 298d7a89..1e126be5 100644
--- a/ceph-dev-pipeline/build/Jenkinsfile
+++ b/ceph-dev-pipeline/build/Jenkinsfile
@@ -2,15 +2,41 @@ ceph_build_repo = "https://github.com/ceph/ceph-build"
 ceph_build_branch = "main"
 base_node_label = "gigantic"
 ubuntu_releases = [
-  "noble", // 24.04
-  "jammy", // 22.04
-  "focal", // 20.04
+  "noble": "24.04",
+  "jammy": "22.04",
+  "focal": "20.04",
 ]
 debian_releases = [
-  "bookworm", // v12 (2023)
-  "bullseye", // v11 (2021)
+  "bookworm": "12",
+  "bullseye": "11",
 ]
 
+def get_os_info(dist) {
+  def os = [
+    "name": dist,
+    "version": dist,
+    "version_name": dist,
+    "pkg_type": "NONE",
+  ]
+  def matcher = dist =~ /^(centos|rhel|fedora)(\d+)/
+  if ( matcher.find() ) {
+    os.name = matcher.group(1)
+    os.version = os.version_name = matcher.group(2)
+    os.pkg_type = "rpm"
+  } else if ( debian_releases.keySet().contains(dist) ) {
+    os.name = "debian"
+    os.version = debian_releases[dist]
+    os.pkg_type = "deb"
+  } else if ( ubuntu_releases.keySet().contains(dist) ) {
+    os.name = "ubuntu"
+    os.version = ubuntu_releases[env.DIST]
+    os.pkg_type = "deb"
+  }
+  // We need to set matcher to null right after using it to avoid a java.io.NotSerializableException
+  matcher = null
+  return os
+}
+
 pipeline {
   agent any
   stages {
@@ -68,9 +94,17 @@ pipeline {
           stage("node") {
             steps {
               script {
+                sh "hostname -f"
                 def node_shortname = env.NODE_NAME.split('\\+')[-1]
                 def node_url = new URI([env.JENKINS_URL, "computer", env.NODE_NAME].join("/")).normalize()
-                echo "DIST=${env.DIST} ARCH=${env.ARCH} FLAVOR=${env.FLAVOR}\n${node_shortname}\n${node_url}"
+                println("DIST=${env.DIST} ARCH=${env.ARCH} FLAVOR=${env.FLAVOR}")
+                println("${node_shortname}")
+                println("${node_url}")
+                def os = get_os_info(env.DIST)
+                println("OS_NAME=${os.name}")
+                println("OS_PKG_TYPE=${os.pkg_type}")
+                println("OS_VERSION=${os.version}")
+                println("OS_VERSION_NAME=${os.version_name}")
               }
               sh "cat /etc/os-release"
             }
@@ -120,6 +154,72 @@ pipeline {
               }
               sh "sha256sum dist/*"
               sh "cat dist/sha1 dist/version"
+              script {
+                def props = readProperties file: "${WORKSPACE}/dist/other_envvars"
+                for (p in props) {
+                  env."${p.key}" = p.value
+                }
+              }
+              sh '''#!/bin/bash
+                set -ex
+                cd dist
+                mkdir ceph
+                tar --strip-components=1 -C ceph -xjf ceph-$VERSION.tar.bz2
+              '''
+            }
+          }
+          stage("check for built packages") {
+            when {
+              environment name: 'THROWAWAY', value: 'false'
+              environment name: 'CI_COMPILE', value: 'true'
+            }
+            environment {
+              CHACRACTL_KEY = credentials('chacractl-key')
+              SHAMAN_API_KEY = credentials('shaman-api-key')
+            }
+            steps {
+              script {
+                sh './scripts/setup_chacractl.sh'
+                def chacra_url = sh(
+                  script: '''grep url ~/.chacractl | cut -d'"' -f2''',
+                  returnStdout: true,
+                ).trim()
+                def os = get_os_info(env.DIST)
+                def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/"
+                def chacractl_rc = sh(
+                  script: "$WORKSPACE/.venv/bin/chacractl exists binaries/${chacra_endpoint}",
+                  returnStatus: true,
+                )
+                if ( chacractl_rc == 0 && env.FORCE != "true" ) {
+                  println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).")
+                  env.CI_COMPILE = "false"
+                }
+              }
+            }
+          }
+          stage("builder container") {
+            environment {
+              CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci')
+            }
+            when {
+              environment name: 'CI_COMPILE', value: 'true'
+            }
+            steps {
+              script {
+                env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build"
+                def ceph_builder_tag = "${env.BRANCH}.${env.DIST}.${ARCH}"
+                sh '''#!/bin/bash
+                  set -ex
+                  podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION}
+                '''
+                def os = get_os_info(env.DIST)
+                sh """#!/bin/bash
+                  set -ex
+                  cd dist/ceph
+                  python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} -e build-container
+                  podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag}
+                """
+              }
             }
           }
           stage("build") {
@@ -128,70 +228,131 @@ pipeline {
               SHAMAN_API_KEY = credentials('shaman-api-key')
               SCCACHE_BUCKET_CREDS = credentials('ibm-cloud-sccache-bucket')
             }
+            when {
+              environment name: 'CI_COMPILE', value: 'true'
+            }
             steps {
               script {
+                def os = get_os_info(env.DIST)
+                sh """#!/bin/bash
+                  . ./scripts/build_utils.sh
+                  update_build_status "started" "ceph" ${os.name} ${os.version_name} $ARCH
+                """
                 env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR
                 env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW
-                def props = readProperties file: "${WORKSPACE}/dist/other_envvars"
-                for (p in props) {
-                  env."${p.key}" = p.value
+                sh 'echo > .env'
+                def bwc_cmd_sccache_flags = ""
+                if ( env.DWZ == "false" ) {
+                  sh '''#!/bin/bash
+                    echo "DWZ=$DWZ" >> .env
+                  '''
+                  bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
                 }
-                if ( (debian_releases + ubuntu_releases).contains(env.DIST) ) {
+                if ( env.SCCACHE == "true" ) {
+                  sh '''#!/bin/bash
+                    echo "SCCACHE=$SCCACHE" >> .env
+                    echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env
+                    echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env
+                    echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env
+                  '''
+                  // TODO: un-hardcode this
+                  writeFile(
+                    file: "dist/ceph/sccache.conf",
+                    text: """\
+                      [cache.s3]
+                      bucket = "ceph-sccache"
+                      endpoint = "s3.us-south.cloud-object-storage.appdomain.cloud"
+                      use_ssl = true
+                      key_prefix = ""
+                      server_side_encryption = false
+                      no_credentials = false
+                      region = "auto"
+                    """
+                  )
+                  bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env";
+                }
+                def ceph_builder_tag = "${env.BRANCH}.${env.DIST}.${ARCH}"
+                def bwc_command = "python3 src/script/build-with-container.py ${bwc_cmd_sccache_flags} --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${DIST} --ceph-version ${env.VERSION}"
+                if ( os.pkg_type == "deb" ) {
                   def sccache_flag = "-DWITH_SCCACHE=ON"
                   if ( env.SCCACHE == "true" && ! env.CEPH_EXTRA_CMAKE_ARGS.contains(sccache_flag) ) {
                     env.CEPH_EXTRA_CMAKE_ARGS = "${env.CEPH_EXTRA_CMAKE_ARGS} ${sccache_flag}"
                   }
-                  sh """#!/bin/bash
-                    . ./ceph-dev-new-build/build/validate_deb
-                    . ./scripts/build_utils.sh
-                    . ./ceph-dev-new-build/build/setup_deb
-                    . ./scripts/setup_sccache.sh
-                    . ./ceph-dev-new-build/build/setup_pbuilder
-                    . ./ceph-dev-new-build/build/build_deb
-                  """
+                  bwc_command = "${bwc_command} -e debs"
                 } else if ( env.DIST =~ /^(centos|rhel|fedora).*/ ) {
-                  sh """#!/bin/bash
-                    . ./ceph-dev-new-build/build/validate_rpm
-                    . ./scripts/build_utils.sh
-                    . ./scripts/setup_sccache.sh
-                    . ./ceph-dev-new-build/build/setup_rpm
-                    [ "$CI_COMPILE" = "true" ] || exit 0
-                    reset_sccache
-                    . ./ceph-dev-new-build/build/build_rpm
-                  """
+                  def rpmbuild_args = ""
+                  if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache"
+                  if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz"
+                  bwc_command = "${bwc_command}${rpmbuild_args} -e rpm"
                 } else if ( env.DIST =~ /suse|sles/ ) {
+                  throw new Exception("bwc not implemented for ${env.DIST}")
+                } else if ( env.DIST =~ /windows/ ) {
+                  throw new Exception("bwc not implemented for ${env.DIST}")
+                } else {
+                  throw new Exception("DIST '${env.DIST}' is invalid!")
+                }
+                sh """#!/bin/bash
+                    set -ex
+                    cd dist/ceph
+                    ln ../ceph-${env.VERSION}.tar.bz2 .
+                    ${bwc_command}
+                """
+                if ( os.pkg_type == "rpm" ) {
                   sh """#!/bin/bash
-                    . ./ceph-dev-new-build/build/validate_osc
-                    . ./scripts/build_utils.sh
-                    . ./ceph-dev-new-build/build/setup_osc
-                    . ./ceph-dev-new-build/build/build_osc
+                      set -ex
+                      cd ./dist/ceph
+                      mkdir -p ./rpmbuild/SRPMS/
+                      ln ceph-*.src.rpm ./rpmbuild/SRPMS/
                   """
-                } else if ( env.DIST =~ /windows/ ) {
+                  def chacra_url = sh(
+                    script: '''grep url ~/.chacractl | cut -d'"' -f2''',
+                    returnStdout: true,
+                  ).trim()
                   sh """#!/bin/bash
-                    . ./ceph-dev-new-build/build/validate_mingw
-                    . ./scripts/build_utils.sh
-                    . ./ceph-dev-new-build/build/setup_mingw
-                    . ./ceph-dev-new-build/build/build_mingw
+                      . ./scripts/build_utils.sh
+                      export chacra_url=${chacra_url}
+                      export chacra_ref=${BRANCH}
+                      export DISTRO=${os.name}
+                      export RELEASE=${os.version_name}
+                      build_ceph_release_rpm ${WORKSPACE}/dist/ceph/rpmbuild true
                   """
-                } else {
-                  throw new Exception("DIST '${env.DIST}' is invalid!")
                 }
               }
             }
             post {
+              always {
+                script {
+                  // Jenkins does not use sudo to wipe the workspace, so fix the owner before we finish
+                  sh 'sudo chown -R $USER $WORKSPACE/dist/ceph'
+                }
+              }
               success {
-                sh '''#!/bin/bash
-                . /etc/os-release
-                . ./scripts/build_utils.sh
-                update_build_status "completed" "ceph" $ID $VERSION_ID $ARCH
-                '''
+                script {
+                  def chacra_url = sh(
+                    script: '''grep url ~/.chacractl | cut -d'"' -f2''',
+                    returnStdout: true,
+                  ).trim()
+                  def os = get_os_info(env.DIST)
+                  sh """#!/bin/bash
+                    export CHACRA_URL="${chacra_url}"
+                    export OS_NAME="${os.name}"
+                    export OS_VERSION="${os.version}"
+                    export OS_VERSION_NAME="${os.version_name}"
+                    export OS_PKG_TYPE="${os.pkg_type}"
+                    if [ "$THROWAWAY" != "true" ]; then ./scripts/chacra_upload.sh; fi
+                  """
+                  sh """#!/bin/bash
+                  . ./scripts/build_utils.sh
+                  update_build_status "completed" "ceph" ${os.name} ${os.version_name} $ARCH
+                  """
+                }
               }
               unsuccessful {
-                sh '''#!/bin/bash
-                . /etc/os-release
+                def os = get_os_info(env.DIST)
+                sh """#!/bin/bash
                 . ./scripts/build_utils.sh
-                update_build_status "failed" "ceph" $ID $VERSION_ID $ARCH
-                '''
+                update_build_status "failed" "ceph" ${os.name} ${os.version_name} $ARCH
+                """
               }
             }
           }
diff --git a/scripts/chacra_upload.sh b/scripts/chacra_upload.sh
new file mode 100755
index 00000000..1fb9d060
--- /dev/null
+++ b/scripts/chacra_upload.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+# vim: ts=4 sw=4 expandtab
+set -ex
+
+cd "$WORKSPACE"
+VENV="${WORKSPACE}/.venv"
+chacra_endpoint="ceph/${BRANCH}/${SHA1}/${OS_NAME}/${OS_VERSION_NAME}"
+[ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags=""
+if [ "$OS_PKG_TYPE" = "rpm" ]; then
+  RPM_RELEASE=`grep Release dist/ceph/ceph.spec | sed 's/Release:[ \t]*//g' | cut -d '%' -f 1`
+  RPM_VERSION=`grep Version dist/ceph/ceph.spec | sed 's/Version:[ \t]*//g'`
+  PACKAGE_MANAGER_VERSION="$RPM_VERSION-$RPM_RELEASE"
+  BUILDAREA="${WORKSPACE}/dist/ceph/rpmbuild"
+  find dist/ceph/rpmbuild/SRPMS | grep rpm | $VENV/bin/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source/flavors/${FLAVOR}
+  find dist/ceph/rpmbuild/RPMS/* | grep rpm | $VENV/bin/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}
+  if [ -f ${BUILDAREA}/RPMS/noarch/cephadm-*.rpm ] ; then
+      rpm2cpio ${BUILDAREA}/RPMS/noarch/cephadm-*.rpm  | cpio -i --to-stdout *sbin/cephadm > cephadm
+      echo cephadm | $VENV/bin/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}
+  fi
+elif [ "$OS_PKG_TYPE" = "deb" ]; then
+  PACKAGE_MANAGER_VERSION="${VERSION}-1${OS_VERSION_NAME}"
+  find ${WORKSPACE}/dist/ceph/ | \
+    egrep "*(\.changes|\.deb|\.ddeb|\.dsc|ceph[^/]*\.gz)$" | \
+    egrep -v "(Packages|Sources|Contents)" | \
+    $VENV/bin/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}
+  # FIXME need the real path here
+  if [ -f release/${VERSION}/cephadm_${VERSION}*.deb ] ; then
+    dpkg-deb --fsys-tarfile release/${VERSION}/cephadm_${VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm
+    echo cephadm | $VENV/bin/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}
+  fi
+fi
+# write json file with build info
+  cat > $WORKSPACE/repo-extra.json << EOF
+{
+    "version":"$VERSION",
+    "package_manager_version":"$PACKAGE_MANAGER_VERSION",
+    "build_url":"$BUILD_URL",
+    "root_build_cause":"$ROOT_BUILD_CAUSE",
+    "node_name":"$NODE_NAME",
+    "job_name":"$JOB_NAME"
+}
+EOF
+chacra_repo_endpoint="${chacra_endpoint}/flavors/${FLAVOR}"
+# post the json to repo-extra json to chacra
+curl -X POST -H "Content-Type:application/json" --data "@$WORKSPACE/repo-extra.json" -u $CHACRACTL_USER:$CHACRACTL_KEY ${CHACRA_URL}repos/${chacra_repo_endpoint}/extra/
+# start repo creation
+$VENV/bin/chacractl repo update ${chacra_repo_endpoint}
+
+echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}/flavors/${FLAVOR}/
diff --git a/scripts/setup_chacractl.sh b/scripts/setup_chacractl.sh
new file mode 100755
index 00000000..dc83b178
--- /dev/null
+++ b/scripts/setup_chacractl.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# vim: ts=4 sw=4 expandtab
+set -ex
+. $(dirname ${0})/build_utils.sh
+
+cd "$WORKSPACE"
+VENV="${WORKSPACE}/.venv"
+python3 -m venv $VENV
+pkgs=( "chacractl>=0.0.21" )
+install_python_packages $VENV "pkgs[@]"
+
+chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/`
+make_chacractl_config $chacra_url
+echo $chacra_url