From: Zack Cerza Date: Thu, 29 May 2025 00:11:35 +0000 (-0600) Subject: Add ceph-trigger-build job X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5dc08ae433b0dc06a25918afbe8c050299a2b11a;p=ceph-build.git Add ceph-trigger-build job Signed-off-by: Zack Cerza --- diff --git a/ceph-trigger-build/README.md b/ceph-trigger-build/README.md new file mode 100644 index 00000000..57c6defa --- /dev/null +++ b/ceph-trigger-build/README.md @@ -0,0 +1,25 @@ +# ceph-trigger-build + +This pipeline's role is to: + 1. Be triggered by a git push event + 2. Determine which job or pipeline to use for the actual compile and + packaging + 3. Build one or more sets of parameters + 4. For each set, trigger a second job or pipeline with those parameters + +Parameter sets are constructed in this way: + 1. A set of default values is constructed based on the branch name + 2. The head commit is examined for git trailers + a. See https://git-scm.com/docs/git-interpret-trailers + 3. If a "CEPH-BUILD-JOB: ceph-dev-pipeline" trailer is found, other parameter + values are overridden by any trailers matching them + +An example set of trailers: + + CEPH-BUILD-JOB: ceph-dev-pipeline + DISTROS: jammy centos9 + ARCHS: x86_64 arm64 + +NOTE: During a transitional period, before this can fully replace the legacy +trigger jobs, it will *not* trigger the legacy build jobs if the legacy triggers +are enabled. diff --git a/ceph-trigger-build/build/Jenkinsfile b/ceph-trigger-build/build/Jenkinsfile new file mode 100644 index 00000000..44451d97 --- /dev/null +++ b/ceph-trigger-build/build/Jenkinsfile @@ -0,0 +1,190 @@ +import groovy.json.JsonBuilder + +def pretty(obj) { + return new JsonBuilder(obj).toPrettyString() +} + +// These parameters are able to be parsed from git trailers +def gitTrailerParameterNames = [ + "ARCHS", + "CEPH_BUILD_BRANCH", + "CEPH_BUILD_JOB", + "CI_COMPILE", + "CI_CONTAINER", + "DISTROS", + "DWZ", + "FLAVORS", + "SCCACHE", +] +// These are the default parameter values for the pipeline +def defaults = [ + 'CEPH_BUILD_JOB': 'ceph-dev-new', + 'DISTROS': 'centos9 jammy windows', + 'ARCHS': 'x86_64 arm64', + 'FLAVOR': 'default', +] +// This will later hold the initial set of parameters, before any branch-based +// values are inserted. +def initialParams = [:] +// this will later hold parameters parsed from git trailers +def trailerParams = [:] +// This will later hold one or more parameter sets. Each parameter set will +// result in a triggered job. +def paramMaps = [] +// This will later hold the build's description; we need to store it so that +// we can append to it later, as there is no way to read it. +def description = ""; + +// This encodes the same logic as the ceph-dev-new-trigger job. +// It returns a list of one or more parameter sets. +// For ceph-dev-pipeline, only one set is returned. +def params_from_branch(initialParams) { + def singleSet = ( initialParams['CEPH_BUILD_JOB'].contains('ceph-dev-pipeline') ) + def params = [initialParams.clone()] + switch (initialParams.BRANCH) { + case ~/.*reef.*/: + break + case ~/.*squid.*/: + break + case ~/.*tentacle.*/: + if ( !singleSet ) { + params << params[0].clone() + params[-1]['ARCHS'] = 'x86_64' + params[-1]['DISTROS'] = 'centos9' + params[-1]['FLAVOR'] = 'crimson' + } else { + params[0]['FLAVOR'] += ' crimson' + } + break + case ~/.*centos9-only.*/: + params[0]['DISTROS'] = 'centos9' + break + case ~/.*crimson-only.*/: + params[0]['ARCHS'] = 'x86_64' + params[0]['DISTROS'] = 'centos9' + params[0]['FLAVOR'] = 'crimson' + break + default: + if ( !singleSet ) { + params << params[0].clone() + params[-1]['ARCHS'] = 'x86_64' + params[-1]['DISTROS'] = 'centos9' + params[-1]['FLAVOR'] = 'crimson' + } else { + params[0]['FLAVOR'] += ' crimson' + } + } + if ( singleSet ) { + params[0]['FLAVORS'] = params[0]['FLAVOR'] + params[0].remove('FLAVOR') + } + return params +} + +pipeline { + agent any + stages { + stage("Prepare parameters") { + steps { + script { + initialParams.BRANCH = env.ref.replace("refs/heads/", "") + initialParams.putAll(defaults) + println("BRANCH=${initialParams.BRANCH}") + } + script { + println("SHA1=${env.head_commit_id}") + } + script { + println("pusher=${env.pusher}") + } + script { + println("Looking for git trailer parameters: ${gitTrailerParameterNames}") + writeFile( + file: "head_commit_message.txt", + text: env.head_commit_message, + ) + def trailer = sh( + script: "git interpret-trailers --parse head_commit_message.txt", + returnStdout: true, + ) + println("trailer: ${trailer}") + for (item in trailer.split("\n")) { + def matcher = item =~ /(.+): (.+)/ + if (matcher.matches()) { + key = matcher[0][1].replace("-", "_").toUpperCase() + value = matcher[0][2] + if ( key in gitTrailerParameterNames && value ) { + trailerParams[key] = value + } + } + } + } + script { + if ( trailerParams.containsKey('CEPH_BUILD_JOB') ) { + initialParams['CEPH_BUILD_JOB'] = trailerParams['CEPH_BUILD_JOB'] + } + paramMaps = params_from_branch(initialParams) + if ( initialParams['CEPH_BUILD_JOB'].contains('ceph-dev-pipeline') ) { + paramMaps[0].putAll(trailerParams) + } + } + script { + println("Final parameters: ${pretty(paramMaps)}") + } + script { + paramMaps.each { paramMap -> + paramMap.each { key, value -> description += "${key}=${value}
" } + description += "---\n" + } + buildDescription description.trim() + } + } + } + stage("Trigger job") { + steps { + script { + for (paramsMap in paramMaps) { + // Before we trigger, we need to transform the parameter sets from + // the base Groovy types into the types expected by Jenkins + def paramsList = [] + paramsMap.each { + entry -> paramsList.push(string(name: entry.key, value: entry.value)) + } + def job = paramsMap.CEPH_BUILD_JOB + def buildId = "_ID_" + if ( job.contains("ceph-dev-pipeline") ) { + triggeredBuild = build( + job: job, + parameters: paramsList, + wait: false, + waitForStart: true, + ) + buildId = triggeredBuild.getId() + println("triggered pipeline: ${pretty(paramsMap)}") + } else { + legacy_trigger_enabled = Jenkins.instance.getItem("ceph-dev-new-trigger").isBuildable(); + if ( legacy_trigger_enabled ) { + println("skipped triggering since legacy trigger is enabled: ${pretty(paramsMap)}") + } else { + triggeredBuild = build( + job: job, + parameters: paramsList, + wait: false, + waitForStart: true, + ) + buildId = triggeredBuild.getId() + println("triggered legacy: ${pretty(paramsMap)}") + } + } + def buildUrl = new URI([env.JENKINS_URL, "job", job, buildId].join("/")).normalize() + description = """\ + ${description}
+ ${job} ${buildId} + """.trim() + buildDescription(description) + } + } + } + } + } +} diff --git a/ceph-trigger-build/config/definitions/ceph-trigger-build.yml b/ceph-trigger-build/config/definitions/ceph-trigger-build.yml new file mode 100644 index 00000000..c0f7f8d1 --- /dev/null +++ b/ceph-trigger-build/config/definitions/ceph-trigger-build.yml @@ -0,0 +1,62 @@ +- job: + name: preserve-ceph-trigger-build + description: "this is a proof-of-concept and will not actually trigger builds." + node: built-in + project-type: pipeline + defaults: global + concurrent: true + quiet-period: 0 + block-downstream: false + block-upstream: false + pipeline-scm: + scm: + - git: + url: https://github.com/ceph/ceph-build + branches: + - ceph-trigger-build + shallow-clone: true + submodule: + disable: true + wipe-workspace: true + script-path: ceph-trigger-build/build/Jenkinsfile + lightweight-checkout: true + do-not-fetch-tags: true + properties: + - build-discarder: + num-to-keep: 500 + artifact-days-to-keep: -1 + artifact-num-to-keep: -1 + - github: + url: https://github.com/ceph/ceph-ci + + triggers: + - generic-webhook-trigger: + token: ceph-trigger-build + token-credential-id: ceph-trigger-build-token + print-contrib-var: true + header-params: + - key: X-GitHub-Event + - key: X-GitHub-Hook-ID + - key: X-GitHub-Delivery + post-content-params: + - type: JSONPath + key: head_commit_message + value: $.head_commit.message + - type: JSONPath + key: head_commit_id + value: $.head_commit.id + - type: JSONPath + key: ref + value: $.ref + - type: JSONPath + key: pusher + value: $.pusher.name + # github sends push events for branch deletion, and those events + # are missing commit-related fields, so we must special-case + # them to prevent failures + - type: JSONPath + key: is_delete + value: $.deleted + regex-filter-text: $is_delete + regex-filter-expression: "(?i)false" + cause: "Push to $ref by $pusher"