Add ceph-trigger-build job

Signed-off-by: Zack Cerza <zack@cerza.org>

diff --git a/ceph-trigger-build/README.md b/ceph-trigger-build/README.md
new file mode 100644 (file)
index 0000000..57c6def
--- /dev/null
+++ b/ceph-trigger-build/README.md
@@ -0,0 +1,25 @@
+# ceph-trigger-build
+
+This pipeline's role is to:
+  1. Be triggered by a git push event
+  2. Determine which job or pipeline to use for the actual compile and
+     packaging
+  3. Build one or more sets of parameters
+  4. For each set, trigger a second job or pipeline with those parameters
+
+Parameter sets are constructed in this way:
+  1. A set of default values is constructed based on the branch name
+  2. The head commit is examined for git trailers
+    a. See https://git-scm.com/docs/git-interpret-trailers
+  3. If a "CEPH-BUILD-JOB: ceph-dev-pipeline" trailer is found, other parameter
+     values are overridden by any trailers matching them
+
+An example set of trailers:
+
+    CEPH-BUILD-JOB: ceph-dev-pipeline
+    DISTROS: jammy centos9
+    ARCHS: x86_64 arm64
+
+NOTE: During a transitional period, before this can fully replace the legacy
+trigger jobs, it will *not* trigger the legacy build jobs if the legacy triggers
+are enabled.

diff --git a/ceph-trigger-build/build/Jenkinsfile b/ceph-trigger-build/build/Jenkinsfile
new file mode 100644 (file)
index 0000000..44451d9
--- /dev/null
+++ b/ceph-trigger-build/build/Jenkinsfile
@@ -0,0 +1,190 @@
+import groovy.json.JsonBuilder
+
+def pretty(obj) {
+  return new JsonBuilder(obj).toPrettyString()
+}
+
+// These parameters are able to be parsed from git trailers
+def gitTrailerParameterNames = [
+  "ARCHS",
+  "CEPH_BUILD_BRANCH",
+  "CEPH_BUILD_JOB",
+  "CI_COMPILE",
+  "CI_CONTAINER",
+  "DISTROS",
+  "DWZ",
+  "FLAVORS",
+  "SCCACHE",
+]
+// These are the default parameter values for the pipeline
+def defaults = [
+  'CEPH_BUILD_JOB': 'ceph-dev-new',
+  'DISTROS': 'centos9 jammy windows',
+  'ARCHS': 'x86_64 arm64',
+  'FLAVOR': 'default',
+]
+// This will later hold the initial set of parameters, before any branch-based
+// values are inserted.
+def initialParams = [:]
+// this will later hold parameters parsed from git trailers
+def trailerParams = [:]
+// This will later hold one or more parameter sets. Each parameter set will
+// result in a triggered job.
+def paramMaps = []
+// This will later hold the build's description; we need to store it so that
+// we can append to it later, as there is no way to read it.
+def description = "";
+
+// This encodes the same logic as the ceph-dev-new-trigger job.
+// It returns a list of one or more parameter sets.
+// For ceph-dev-pipeline, only one set is returned.
+def params_from_branch(initialParams) {
+  def singleSet = ( initialParams['CEPH_BUILD_JOB'].contains('ceph-dev-pipeline') )
+  def params = [initialParams.clone()]
+  switch (initialParams.BRANCH) {
+    case ~/.*reef.*/:
+      break
+    case ~/.*squid.*/:
+      break
+    case ~/.*tentacle.*/:
+      if ( !singleSet ) {
+        params << params[0].clone()
+        params[-1]['ARCHS'] = 'x86_64'
+        params[-1]['DISTROS'] = 'centos9'
+        params[-1]['FLAVOR'] = 'crimson'
+      } else {
+        params[0]['FLAVOR'] += ' crimson'
+      }
+      break
+    case ~/.*centos9-only.*/:
+      params[0]['DISTROS'] = 'centos9'
+      break
+    case ~/.*crimson-only.*/:
+      params[0]['ARCHS'] = 'x86_64'
+      params[0]['DISTROS'] = 'centos9'
+      params[0]['FLAVOR'] = 'crimson'
+      break
+    default:
+      if ( !singleSet ) {
+        params << params[0].clone()
+        params[-1]['ARCHS'] = 'x86_64'
+        params[-1]['DISTROS'] = 'centos9'
+        params[-1]['FLAVOR'] = 'crimson'
+      } else {
+        params[0]['FLAVOR'] += ' crimson'
+      }
+  }
+  if ( singleSet ) {
+    params[0]['FLAVORS'] = params[0]['FLAVOR']
+    params[0].remove('FLAVOR')
+  }
+  return params
+}
+
+pipeline {
+  agent any
+  stages {
+    stage("Prepare parameters") {
+      steps {
+        script {
+          initialParams.BRANCH = env.ref.replace("refs/heads/", "")
+          initialParams.putAll(defaults)
+          println("BRANCH=${initialParams.BRANCH}")
+        }
+        script {
+          println("SHA1=${env.head_commit_id}")
+        }
+        script {
+          println("pusher=${env.pusher}")
+        }
+        script {
+          println("Looking for git trailer parameters: ${gitTrailerParameterNames}")
+          writeFile(
+            file: "head_commit_message.txt",
+            text: env.head_commit_message,
+          )
+          def trailer = sh(
+            script: "git interpret-trailers --parse head_commit_message.txt",
+            returnStdout: true,
+          )
+          println("trailer: ${trailer}")
+          for (item in trailer.split("\n")) {
+            def matcher = item =~ /(.+): (.+)/
+            if (matcher.matches()) {
+              key = matcher[0][1].replace("-", "_").toUpperCase()
+              value = matcher[0][2]
+              if ( key in gitTrailerParameterNames && value ) {
+                trailerParams[key] = value
+              }
+            }
+          }
+        }
+        script {
+          if ( trailerParams.containsKey('CEPH_BUILD_JOB') ) {
+            initialParams['CEPH_BUILD_JOB'] = trailerParams['CEPH_BUILD_JOB']
+          }
+          paramMaps = params_from_branch(initialParams)
+          if ( initialParams['CEPH_BUILD_JOB'].contains('ceph-dev-pipeline') ) {
+            paramMaps[0].putAll(trailerParams)
+          }
+        }
+        script {
+          println("Final parameters: ${pretty(paramMaps)}")
+        }
+        script {
+          paramMaps.each { paramMap ->
+            paramMap.each { key, value -> description += "${key}=${value}<br />" }
+            description += "---\n"
+          }
+          buildDescription description.trim()
+        }
+      }
+    }
+    stage("Trigger job") {
+      steps {
+        script {
+          for (paramsMap in paramMaps) {
+            // Before we trigger, we need to transform the parameter sets from
+            // the base Groovy types into the types expected by Jenkins
+            def paramsList = []
+            paramsMap.each {
+              entry -> paramsList.push(string(name: entry.key, value: entry.value))
+            }
+            def job = paramsMap.CEPH_BUILD_JOB
+            def buildId = "_ID_"
+            if ( job.contains("ceph-dev-pipeline") ) {
+              triggeredBuild = build(
+                job: job,
+                parameters: paramsList,
+                wait: false,
+                waitForStart: true,
+              )
+              buildId = triggeredBuild.getId()
+              println("triggered pipeline: ${pretty(paramsMap)}")
+            } else {
+              legacy_trigger_enabled = Jenkins.instance.getItem("ceph-dev-new-trigger").isBuildable();
+              if ( legacy_trigger_enabled ) {
+                println("skipped triggering since legacy trigger is enabled: ${pretty(paramsMap)}")
+              } else {
+                triggeredBuild = build(
+                  job: job,
+                  parameters: paramsList,
+                  wait: false,
+                  waitForStart: true,
+                )
+                buildId = triggeredBuild.getId()
+                println("triggered legacy: ${pretty(paramsMap)}")
+              }
+            }
+            def buildUrl = new URI([env.JENKINS_URL, "job", job, buildId].join("/")).normalize()
+            description = """\
+              ${description}<br />
+              <a href="${buildUrl}">${job} ${buildId}</a>
+            """.trim()
+            buildDescription(description)
+          }
+        }
+      }
+    }
+  }
+}

diff --git a/ceph-trigger-build/config/definitions/ceph-trigger-build.yml b/ceph-trigger-build/config/definitions/ceph-trigger-build.yml
new file mode 100644 (file)
index 0000000..c0f7f8d
--- /dev/null
+++ b/ceph-trigger-build/config/definitions/ceph-trigger-build.yml
@@ -0,0 +1,62 @@
+- job:
+    name: preserve-ceph-trigger-build
+    description: "this is a proof-of-concept and will not actually trigger builds."
+    node: built-in
+    project-type: pipeline
+    defaults: global
+    concurrent: true
+    quiet-period: 0
+    block-downstream: false
+    block-upstream: false
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ceph-trigger-build
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: ceph-trigger-build/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+    properties:
+      - build-discarder:
+          num-to-keep: 500
+          artifact-days-to-keep: -1
+          artifact-num-to-keep: -1
+      - github:
+          url: https://github.com/ceph/ceph-ci
+
+    triggers:
+      - generic-webhook-trigger:
+          token: ceph-trigger-build
+          token-credential-id: ceph-trigger-build-token
+          print-contrib-var: true
+          header-params:
+            - key: X-GitHub-Event
+            - key: X-GitHub-Hook-ID
+            - key: X-GitHub-Delivery
+          post-content-params:
+            - type: JSONPath
+              key: head_commit_message
+              value: $.head_commit.message
+            - type: JSONPath
+              key: head_commit_id
+              value: $.head_commit.id
+            - type: JSONPath
+              key: ref
+              value: $.ref
+            - type: JSONPath
+              key: pusher
+              value: $.pusher.name
+            # github sends push events for branch deletion, and those events
+            # are missing commit-related fields, so we must special-case
+            # them to prevent failures
+            - type: JSONPath
+              key: is_delete
+              value: $.deleted
+          regex-filter-text: $is_delete
+          regex-filter-expression: "(?i)false"
+          cause: "Push to $ref by $pusher"