cve-pipeline: Creation

This enables us to build packages and containers from ceph-private.git privately.
This includes Jenkins build logs, artifacts, packages, and containers.

Packages will be pushed to the internal-only pulp instance and containers
to the internal-only quay instance.

To avoid job duplication, each job pulls in the respective Jenkinsfiles e.g.,
  - cve-pipeline -> ceph-dev-pipeline
  - cve-source-dist -> ceph-source-dist

The only difference is the parameters we pass to the job.  The key differences
there are CEPH_REPO points to ceph-private.git whereas ceph-dev-pipeline is
not capable of doing so.  We also tell the pipeline not to use a public
chacra instance.

Signed-off-by: David Galloway <david.galloway@ibm.com>

diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile
index d51ce8cb4cd53b862726bbff399abe5630a0fcbe..4ee709091568ccbe11b53adf75c7dd5805eb5196 100644 (file)
--- a/ceph-dev-pipeline/build/Jenkinsfile
+++ b/ceph-dev-pipeline/build/Jenkinsfile
@@ -624,7 +624,8 @@ pipeline {
           }
           stage("builder container") {
             environment {
-              CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci')
+              // Use quay-int.front cred if cve-pipeline, otherwise default to quay.ceph.io cred
+              CONTAINER_REPO_CREDS = credentials("${env.JOB_NAME == 'cve-pipeline' ? 'quay-int-login' : 'quay-ceph-io-ceph-ci'}")
               DOCKER_HUB_CREDS = credentials('dgalloway-docker-hub')
             }
             when {
@@ -688,7 +689,7 @@ pipeline {
               expression { env.CI_CONTAINER == 'true' && container_distros.contains(env.DIST) }
             }
             environment {
-              CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci')
+              CONTAINER_REPO_CREDS = credentials("${env.JOB_NAME == 'cve-pipeline' ? 'quay-int-login' : 'quay-ceph-io-ceph-ci'}")
             }
             steps {
               script { doContainerStage() }

diff --git a/cve-pipeline/config/definitions/cve-pipeline.yml b/cve-pipeline/config/definitions/cve-pipeline.yml
new file mode 100644 (file)
index 0000000..6f04915
--- /dev/null
+++ b/cve-pipeline/config/definitions/cve-pipeline.yml
@@ -0,0 +1,128 @@
+- job:
+    name: cve-pipeline
+    properties:
+      - authorization:
+          inheritance-strategy: none
+          GROUP:ceph*security:
+            - job-read
+            - job-discover
+            - job-build
+            - job-cancel
+            - job-configure
+            - job-workspace
+            - run-replay
+            - run-update
+            - run-delete
+            - scm-tag
+    description: |
+      This Jenkins pipeline is only used to develop CVE fixes.  Its differences are:
+        - The job itself is not public-facing.  It does not inherit the RBAC under https://jenkins.ceph.com/manage/configureSecurity/
+        - ceph-dev-pipeline pushes to the internal/private pulp instance instead of a public chacra instance
+    project-type: pipeline
+    quiet-period: 1
+    concurrent: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: ceph-dev-pipeline/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+
+    parameters:
+      - string:
+          name: BRANCH
+          description: "The branch from ceph-private.git to build"
+          default: main
+
+      - choice:
+          name: SHA1
+          description: "Intentionally blank.  ceph-dev-pipeline requires this"
+          choices:
+            - ''
+
+      - choice:
+          name: CEPH_REPO
+          choices:
+            - git@github.com:ceph/ceph-private.git
+
+      - string:
+          name: DISTROS
+          description: "A list of distros to build for. Available options are: centos9, noble, jammy, bookworm, trixie"
+          default: "noble centos9 bookworm trixie"
+
+      - string:
+          name: ARCHS
+          description: "A list of architectures to build for. Available options are: x86_64, and arm64"
+          default: "x86_64 arm64"
+
+      - bool:
+          name: CI_COMPILE
+          description: "Whether to compile and build packages"
+          default: true
+
+      - bool:
+          name: THROWAWAY
+          description: "DO NOT UNCHECK.  This will push to a chacra node publicly!"
+          default: true
+
+      - bool:
+          name: PULP_UPLOAD
+          description: "If you want packages, this must be true.  We do not push to chacra for CVE builds."
+          default: true
+
+      - choice:
+          name: FLAVORS
+          choices:
+            - default
+
+      - bool:
+          name: CI_CONTAINER
+          description: "Whether to build and push container images"
+          default: true
+
+      - string:
+          name: CONTAINER_REPO_HOSTNAME
+          description: "FQDN of container repo server (e.g. 'quay.io')"
+          default: "quay-int.front.sepia.ceph.com"
+
+      - string:
+          name: CONTAINER_REPO_ORGANIZATION
+          description: "Name of container repo organization (e.g. 'ceph-ci')"
+          default: "ceph-ci"
+
+      - bool:
+          name: DWZ
+          description: "Use dwz to make debuginfo packages smaller"
+          default: false
+
+      - bool:
+          name: SCCACHE
+          description: "Use sccache to speed up compilation"
+          default: true
+
+      - string:
+          name: SETUP_BUILD_ID
+          description: "Reuse the source distribution from this cve-source-dist build instead of creating a new one"
+          default: ""
+
+      - choice:
+          name: SETUP_JOB
+          choices:
+            - cve-source-dist
+
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: "Use the Jenkinsfile from this ceph-build branch"
+          default: main
+
+    wrappers:
+      - build-name:
+          name: "#${{BUILD_NUMBER}} ${{BRANCH}}, ${{VERSION}}"
+

diff --git a/cve-source-dist/config/definitions/cve-source-dist.yml b/cve-source-dist/config/definitions/cve-source-dist.yml
new file mode 100644 (file)
index 0000000..3e96d85
--- /dev/null
+++ b/cve-source-dist/config/definitions/cve-source-dist.yml
@@ -0,0 +1,78 @@
+- job:
+    name: cve-source-dist
+    description: This job is only called from https://jenkins.ceph.com/job/cve-pipeline.  The entire job and its artifacts are only visible to the ceph/security Github Team.
+    project-type: pipeline
+    concurrent: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: ceph-source-dist/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+    properties:
+      - build-discarder:
+          days-to-keep: -1
+          num-to-keep: 100
+          artifact-days-to-keep: -1
+          artifact-num-to-keep: 50
+      - copyartifact:
+          projects: cve-pipeline
+      - authorization:
+          inheritance-strategy: none
+          GROUP:ceph*security:
+            - job-read
+            - job-discover
+            - job-build
+            - job-cancel
+            - job-configure
+            - job-workspace
+            - run-replay
+            - run-update
+            - run-delete
+            - scm-tag
+
+    parameters:
+      - choice:
+          name: CEPH_REPO
+          choices:
+            - git@github.com:ceph/ceph-private.git
+
+      - string:
+          name: BRANCH
+          description: "The Ceph branch to build"
+
+      - string:
+          name: SHA1
+          description: "The specific commit to build"
+
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: "Use the Jenkinsfile from this ceph-build branch"
+          default: main
+
+    scm:
+      - git:
+          url: ${{CEPH_REPO}}
+          # Use the SSH key attached to the ceph-jenkins GitHub account.
+          credentials-id: "jenkins-build"
+          branches:
+            - $BRANCH
+          timeout: 20
+          skip-tag: true
+          wipe-workspace: true
+
+    wrappers:
+      - inject-passwords:
+          global: true
+          mask-password-params: true
+      - credentials-binding:
+          - text:
+              credential-id: shaman-api-key
+              variable: SHAMAN_API_KEY