From: deepssin <deepssin@redhat.com>
Date: Tue, 21 Apr 2026 12:02:38 +0000 (+0000)
Subject: Add teuthology-nightly-cadence (and trigger) plus teuthology-runner
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=75804403fff5bc6f7d30eca6ced9a6dc52361acf;p=ceph-build.git

Add teuthology-nightly-cadence (and trigger) plus teuthology-runner

Signed-off-by: deepssin <deepssin@redhat.com>
---

diff --git a/ceph-release-containers/build/Jenkinsfile b/ceph-release-containers/build/Jenkinsfile
index 7c1c68d5..b60f0624 100644
--- a/ceph-release-containers/build/Jenkinsfile
+++ b/ceph-release-containers/build/Jenkinsfile
@@ -102,6 +102,7 @@ pipeline {
                 ./make-manifest-list.py --version ${VERSION}
               else
                 ./make-manifest-list.py
+              fi
               '''
           }
         }
diff --git a/teuthology-nightly-cadence/README.rst b/teuthology-nightly-cadence/README.rst
new file mode 100644
index 00000000..23640bbc
--- /dev/null
+++ b/teuthology-nightly-cadence/README.rst
@@ -0,0 +1,14 @@
+teuthology-nightly-cadence
+===========================
+
+Thin Jenkins jobs that expand **daily/weekly cadence** locally into **SUITE_RUNS_JSON** (partition /
+subset per suite) and invoke **teuthology-runner**.
+
+- ``teuthology-nightly-cadence``: builds JSON from ``cadenceSteps`` / day-of-year in this folder's Jenkinsfile, then triggers the runner job (default ``teuthology-runner``; set **TEUTHOLOGY_RUNNER_JOB_NAME** for a test copy).
+- ``teuthology-nightly-cadence-trigger``: JJB ``timed`` block with ``TZ=Etc/UTC`` (same pattern as
+  ``ceph-dev-cron``). Schedules echo ``qa/crontab/teuthology-cronjobs``: weekday **06:00** UTC → **daily**
+  cadence; **Sunday 20:00** UTC → **weekly** cadence (cf. ``00 20 * * 0`` weekly main runs there).
+  ``wait: false``. Edit ``triggers`` in the YAML to tune times.
+
+Scripts and Shaman/suite logic live under ``teuthology-runner/``. Ensure the **teuthology-runner**
+job exists in Jenkins (see ``teuthology-runner/config/definitions/teuthology-runner.yml``).
diff --git a/teuthology-nightly-cadence/build/Jenkinsfile b/teuthology-nightly-cadence/build/Jenkinsfile
new file mode 100644
index 00000000..fe8c7ec0
--- /dev/null
+++ b/teuthology-nightly-cadence/build/Jenkinsfile
@@ -0,0 +1,254 @@
+/**
+ * teuthology-nightly-cadence: expands daily/weekly cadence into SUITE_RUNS_JSON and invokes teuthology-runner.
+ * Cadence definitions live here (partition/subset).
+ */
+import java.util.Calendar
+import java.util.TimeZone
+
+pipeline {
+    agent { label 'built-in' }
+    options {
+        timestamps()
+        timeout(time: 8, unit: 'HOURS')
+        buildDiscarder(logRotator(numToKeepStr: '25'))
+    }
+    parameters {
+        string(name: 'CEPH_BUILD_BRANCH', defaultValue: 'main', description: 'ceph-build branch for teuthology-runner SCM.')
+        string(name: 'TEUTHOLOGY_RUNNER_JOB_NAME', defaultValue: 'teuthology-runner', description: 'Jenkins job name for the runner pipeline.')
+        string(name: 'AGENT_LABEL', defaultValue: 'teuthology', description: 'Agent label for teuthology-runner.')
+        choice(name: 'CADENCE', choices: ['daily', 'weekly'], description: 'daily ≈ nop+smoke; weekly ≈ ceph.git teuthology-cronjobs style.')
+        choice(name: 'CEPH_BRANCH', choices: ['main', 'tentacle', 'squid', 'reef'], description: 'Ceph branch.')
+        string(name: 'CEPH_REPO', defaultValue: 'https://github.com/ceph/ceph.git', description: 'Ceph git URL.')
+        string(name: 'CEPH_SHA1', defaultValue: '', description: 'Optional SHA1. Empty = resolve from Shaman for SHAMAN_WAIT_PLATFORMS, then pass explicitly to teuthology-runner.')
+        booleanParam(name: 'USE_WORKSPACE_TEUTHOLOGY', defaultValue: true, description: 'Clone teuthology in runner workspace.')
+        string(name: 'TEUTHOLOGY_REPO_URL', defaultValue: 'https://github.com/ceph/teuthology.git', description: 'Teuthology repo.')
+        string(name: 'TEUTHOLOGY_BRANCH', defaultValue: 'main', description: 'Teuthology branch.')
+        string(name: 'TEUTHOLOGY_SCRIPT_DIR', defaultValue: '', description: 'When USE_WORKSPACE_TEUTHOLOGY false.')
+        string(name: 'TEUTHOLOGY_VIRTUALENV_PATH', defaultValue: '', description: 'When USE_WORKSPACE_TEUTHOLOGY false.')
+        string(name: 'SUITE_REPO', defaultValue: 'https://github.com/ceph/ceph.git', description: 'teuthology-suite --suite-repo.')
+        string(name: 'SUITE_MACHINE_TYPE', defaultValue: 'smithi', description: 'teuthology-suite --machine-type.')
+        string(name: 'TEUTH_CONFIG_OVERRIDE_YAML', defaultValue: '', description: 'Optional teuthology-suite argv.')
+        string(name: 'SHAMAN_WAIT_TIMEOUT', defaultValue: '7200', description: 'Shaman wait timeout (seconds).')
+        string(name: 'SHAMAN_WAIT_INTERVAL', defaultValue: '120', description: 'Shaman poll interval.')
+        string(name: 'SHAMAN_WAIT_PLATFORMS', defaultValue: 'rocky-10-default,ubuntu-jammy-default,centos-9-default', description: 'Shaman platforms.')
+        booleanParam(name: 'SKIP_SHAMAN_WAIT', defaultValue: false, description: 'Skip Shaman wait.')
+        booleanParam(name: 'WAIT_FOR_RUNS', defaultValue: false, description: 'teuthology-wait in runner.')
+        string(name: 'SUITE_WAIT_SLEEP', defaultValue: '15', description: 'Sleep before teuthology-wait.')
+        booleanParam(name: 'RUN_AGGREGATE', defaultValue: false, description: 'Aggregate in runner.')
+        string(name: 'PADDLES_URL', defaultValue: '', description: 'Optional Paddles override.')
+        string(name: 'PULPITO_BASE', defaultValue: 'https://pulpito.ceph.com', description: 'Pulpito base URL for teuthology-runner.')
+    }
+    stages {
+        stage('teuthology-runner') {
+            steps {
+                script {
+                    def runnerJob = (params.TEUTHOLOGY_RUNNER_JOB_NAME ?: 'teuthology-runner').trim()
+                    if (!runnerJob) {
+                        runnerJob = 'teuthology-runner'
+                    }
+                    def resolvedSha = params.CEPH_SHA1?.trim()
+                    if (!resolvedSha) {
+                        def branch = params.CEPH_BRANCH?.trim() ?: ''
+                        if (!(branch ==~ '^[a-zA-Z0-9/._-]+$')) {
+                            error("CEPH_BRANCH contains unsupported characters: ${branch}")
+                        }
+                        def timeout = params.SHAMAN_WAIT_TIMEOUT?.trim() ?: '7200'
+                        def interval = params.SHAMAN_WAIT_INTERVAL?.trim() ?: '120'
+                        if (!(timeout ==~ '^[0-9]+$') || timeout == '0') {
+                            error("SHAMAN_WAIT_TIMEOUT must be a positive integer: ${timeout}")
+                        }
+                        if (!(interval ==~ '^[0-9]+$') || interval == '0') {
+                            error("SHAMAN_WAIT_INTERVAL must be a positive integer: ${interval}")
+                        }
+                        def platforms = params.SHAMAN_WAIT_PLATFORMS?.trim() ?: 'rocky-10-default,ubuntu-jammy-default,centos-9-default'
+                        def shamanScript = "${env.WORKSPACE}/teuthology-runner/scripts/wait_for_shaman_sha1.py"
+                        if (!fileExists(shamanScript)) {
+                            error("Missing ${shamanScript}")
+                        }
+                        resolvedSha = sh(
+                            script: "python3 \"${shamanScript}\" --branch \"${branch}\" --timeout \"${timeout}\" --interval \"${interval}\" --platform \"${platforms}\" --use-available-sha",
+                            returnStdout: true,
+                        ).trim()
+                        echo "Resolved CEPH_SHA1 from Shaman for branch=${branch} platforms=${platforms}: ${resolvedSha}"
+                    } else {
+                        echo "Using explicit CEPH_SHA1 from parameters: ${resolvedSha}"
+                    }
+                    if (!resolvedSha || !(resolvedSha ==~ /^[a-fA-F0-9]{7,40}$/)) {
+                        error("Resolved CEPH_SHA1 is invalid: ${resolvedSha}")
+                    }
+                    def rows = expandCadenceToRows(params.CADENCE, params.CEPH_BRANCH)
+                    if (!rows) {
+                        error("No suites for CADENCE=${params.CADENCE} CEPH_BRANCH=${params.CEPH_BRANCH}")
+                    }
+                    def json = suiteRunsRowsToJson(rows)
+                    def tp = []
+                    tp << string(name: 'AGENT_LABEL', value: params.AGENT_LABEL.trim())
+                    tp << string(name: 'CEPH_BUILD_BRANCH', value: params.CEPH_BUILD_BRANCH.trim())
+                    tp << string(name: 'CEPH_BRANCH', value: params.CEPH_BRANCH)
+                    tp << string(name: 'CEPH_REPO', value: params.CEPH_REPO.trim())
+                    tp << string(name: 'CEPH_SHA1', value: resolvedSha)
+                    tp << booleanParam(name: 'USE_WORKSPACE_TEUTHOLOGY', value: params.USE_WORKSPACE_TEUTHOLOGY)
+                    tp << string(name: 'TEUTHOLOGY_REPO_URL', value: params.TEUTHOLOGY_REPO_URL.trim())
+                    tp << string(name: 'TEUTHOLOGY_BRANCH', value: params.TEUTHOLOGY_BRANCH.trim())
+                    if (!params.USE_WORKSPACE_TEUTHOLOGY) {
+                        tp << string(name: 'TEUTHOLOGY_SCRIPT_DIR', value: params.TEUTHOLOGY_SCRIPT_DIR?.trim() ?: '')
+                        tp << string(name: 'TEUTHOLOGY_VIRTUALENV_PATH', value: params.TEUTHOLOGY_VIRTUALENV_PATH?.trim() ?: '')
+                    }
+                    if (params.TEUTH_CONFIG_OVERRIDE_YAML?.trim()) {
+                        tp << string(name: 'TEUTH_CONFIG_OVERRIDE_YAML', value: params.TEUTH_CONFIG_OVERRIDE_YAML.trim())
+                    }
+                    tp << string(name: 'SUITE_REPO', value: params.SUITE_REPO.trim())
+                    tp << string(name: 'SUITE_MACHINE_TYPE', value: params.SUITE_MACHINE_TYPE.trim())
+                    if (params.PADDLES_URL?.trim()) {
+                        tp << string(name: 'PADDLES_URL', value: params.PADDLES_URL.trim())
+                    }
+                    tp << string(name: 'SHAMAN_WAIT_TIMEOUT', value: params.SHAMAN_WAIT_TIMEOUT.trim())
+                    tp << string(name: 'SHAMAN_WAIT_INTERVAL', value: params.SHAMAN_WAIT_INTERVAL.trim())
+                    tp << string(name: 'SHAMAN_WAIT_PLATFORMS', value: params.SHAMAN_WAIT_PLATFORMS.trim())
+                    tp << string(name: 'PULPITO_BASE', value: (params.PULPITO_BASE ?: 'https://pulpito.ceph.com').trim())
+                    tp << booleanParam(name: 'SKIP_SHAMAN_WAIT', value: params.SKIP_SHAMAN_WAIT)
+                    tp << booleanParam(name: 'WAIT_FOR_RUNS', value: params.WAIT_FOR_RUNS)
+                    tp << string(name: 'SUITE_WAIT_SLEEP', value: params.SUITE_WAIT_SLEEP.trim())
+                    tp << booleanParam(name: 'RUN_AGGREGATE', value: params.RUN_AGGREGATE)
+                    tp << text(name: 'SUITE_RUNS_JSON', value: json)
+                    build job: runnerJob, wait: true, propagate: true, parameters: tp
+                }
+            }
+        }
+    }
+}
+
+List expandCadenceToRows(String cadence, String branch) {
+    def steps = cadenceSteps(cadence, branch.toLowerCase())
+    if (!steps) {
+        return []
+    }
+    def utcCal = Calendar.getInstance(TimeZone.getTimeZone('UTC'))
+    def doy = utcCal.get(Calendar.DAY_OF_YEAR)
+    def rows = []
+    for (def st in steps) {
+        def part = st.partitions as int
+        def idx = doy % part
+        def subset = "${idx}/${part}"
+        def lim = st.limit != null ? st.limit as String : "${part}"
+        def thr = st.threshold != null ? st.threshold as String : "${part}"
+        def pr = st.priority ?: ''
+        def m = [
+            suite: st.suite,
+            limit: lim,
+            threshold: thr,
+            subset: subset,
+            priority: pr,
+            forcePriority: st.forcePriority ? true : false,
+        ]
+        if (st.flavor) {
+            m.flavor = st.flavor
+        }
+        if (st.kernel) {
+            m.kernel = st.kernel
+        }
+        if (st.filter) {
+            m.filter = st.filter
+        }
+        rows << m
+    }
+    return rows
+}
+
+def cadenceSteps(String cadence, String branch) {
+    def b = branch.toLowerCase()
+    if (cadence?.toLowerCase() == 'daily') {
+        return [
+            [suite: 'teuthology/nop', partitions: 1, priority: '1', forcePriority: true],
+            [suite: 'smoke', partitions: 1, priority: '100', forcePriority: true],
+        ]
+    }
+    if (cadence?.toLowerCase() != 'weekly') {
+        return []
+    }
+    switch (b) {
+        case 'main':
+            return [
+                [suite: 'rados', partitions: 100000, priority: '101', forcePriority: true],
+                [suite: 'rados', partitions: 100000, priority: '101', forcePriority: true, flavor: 'debug'],
+                [suite: 'orch', partitions: 64, priority: '950', forcePriority: false],
+                [suite: 'rbd', partitions: 128, priority: '950', forcePriority: false],
+                [suite: 'fs', partitions: 512, priority: '700', forcePriority: false],
+                [suite: 'powercycle', partitions: 4, priority: '950', forcePriority: false],
+                [suite: 'rgw', partitions: 30000, limit: '1', threshold: '1', priority: '150', forcePriority: false],
+                [suite: 'krbd', partitions: 4, priority: '950', forcePriority: false, kernel: 'testing'],
+                [suite: 'crimson-rados', partitions: 1, priority: '101', forcePriority: true, flavor: 'debug'],
+                [suite: 'crimson-rados', partitions: 1, priority: '101', forcePriority: true],
+            ]
+        case 'tentacle':
+            return [
+                [suite: 'rados', partitions: 100000, priority: '831', forcePriority: false],
+                [suite: 'orch', partitions: 64, priority: '830', forcePriority: false],
+                [suite: 'rbd', partitions: 128, priority: '830', forcePriority: false],
+                [suite: 'fs', partitions: 512, priority: '830', forcePriority: false],
+                [suite: 'powercycle', partitions: 4, priority: '830', forcePriority: false],
+                [suite: 'rgw', partitions: 30000, limit: '1', threshold: '1', priority: '830', forcePriority: false],
+                [suite: 'krbd', partitions: 4, priority: '830', forcePriority: false, kernel: 'testing'],
+            ]
+        case 'squid':
+            return [
+                [suite: 'rados', partitions: 100000, priority: '921', forcePriority: false],
+                [suite: 'orch', partitions: 64, priority: '920', forcePriority: false],
+                [suite: 'rbd', partitions: 128, priority: '920', forcePriority: false],
+                [suite: 'fs', partitions: 512, priority: '920', forcePriority: false],
+                [suite: 'powercycle', partitions: 4, priority: '920', forcePriority: false],
+                [suite: 'rgw', partitions: 30000, limit: '1', threshold: '1', priority: '920', forcePriority: false],
+                [suite: 'krbd', partitions: 4, priority: '920', forcePriority: false, kernel: 'testing'],
+            ]
+        case 'reef':
+            return [
+                [suite: 'rados', partitions: 100000, priority: '920', forcePriority: false],
+                [suite: 'orch', partitions: 64, priority: '920', forcePriority: false],
+                [suite: 'rbd', partitions: 128, priority: '920', forcePriority: false],
+                [suite: 'fs', partitions: 512, priority: '920', forcePriority: false],
+                [suite: 'powercycle', partitions: 4, priority: '920', forcePriority: false],
+                [suite: 'rgw', partitions: 30000, limit: '1', threshold: '1', priority: '920', forcePriority: false],
+                [suite: 'krbd', partitions: 4, priority: '920', forcePriority: false, kernel: 'testing'],
+            ]
+        default:
+            return []
+    }
+}
+
+String suiteRunsRowsToJson(List rows) {
+    def sb = new StringBuilder()
+    sb.append('[')
+    def first = true
+    for (def row in rows) {
+        if (!first) {
+            sb.append(',')
+        }
+        first = false
+        sb.append('{')
+        def f2 = true
+        row.each { k, v ->
+            if (!f2) {
+                sb.append(',')
+            }
+            f2 = false
+            sb.append('"').append(k).append('":')
+            if (v instanceof Boolean) {
+                sb.append(v ? 'true' : 'false')
+            } else if (v instanceof Number) {
+                sb.append(v.toString())
+            } else {
+                sb.append('"').append(jsonStringEscape(v != null ? v.toString() : '')).append('"')
+            }
+        }
+        sb.append('}')
+    }
+    sb.append(']')
+    return sb.toString()
+}
+
+String jsonStringEscape(String s) {
+    if (s == null) {
+        return ''
+    }
+    return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n').replace('\r', '\\r').replace('\t', '\\t')
+}
diff --git a/teuthology-nightly-cadence/build/trigger/Jenkinsfile b/teuthology-nightly-cadence/build/trigger/Jenkinsfile
new file mode 100644
index 00000000..5686e52e
--- /dev/null
+++ b/teuthology-nightly-cadence/build/trigger/Jenkinsfile
@@ -0,0 +1,67 @@
+/**
+ * Thin trigger job that fires teuthology-nightly-cadence with wait:false.
+ * builds: Sunday UTC → weekly cadence (Sunday 20:00 trigger matches qa/crontab/teuthology-cronjobs
+ * weekly block); any other day → daily. Manual builds use CADENCE / CEPH_BRANCH parameters.
+ */
+import java.util.Calendar
+import java.util.TimeZone
+
+pipeline {
+    agent any
+    options {
+        buildDiscarder(logRotator(numToKeepStr: '40'))
+        skipDefaultCheckout()
+        timestamps()
+        ansiColor('xterm')
+    }
+    parameters {
+        choice(name: 'CADENCE', choices: ['daily', 'weekly'], description: 'Cadence tier.')
+        choice(name: 'CEPH_BRANCH', choices: ['main', 'tentacle', 'squid', 'reef'], description: 'Ceph branch to test.')
+        string(
+            name: 'CEPH_BUILD_BRANCH',
+            defaultValue: 'main',
+            description: 'ceph-build branch for the triggered teuthology-nightly-cadence build (SCM / Jenkinsfile).',
+        )
+        string(
+            name: 'TEUTHOLOGY_RUNNER_JOB_NAME',
+            defaultValue: 'teuthology-runner',
+            description: 'Forwarded to teuthology-nightly-cadence: Jenkins job name for the runner pipeline.',
+        )
+        string(
+            name: 'TEUTHOLOGY_NIGHTLY_CADENCE_JOB',
+            defaultValue: 'teuthology-nightly-cadence',
+            description: 'Jenkins job name of the cadence pipeline to trigger.',
+        )
+    }
+    stages {
+        stage('Trigger teuthology-nightly-cadence') {
+            steps {
+                script {
+                    def timerCause = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')
+                    def cadence = params.CADENCE
+                    if (timerCause && !timerCause.isEmpty()) {
+                        def utcCal = Calendar.getInstance(TimeZone.getTimeZone('UTC'))
+                        def dow = utcCal.get(Calendar.DAY_OF_WEEK)
+                        // Calendar: Sunday=1 … Saturday=7 (avoid java.time / static enum fields in sandbox)
+                        cadence = (dow == 1) ? 'weekly' : 'daily'
+                        echo "Timer-triggered build: CADENCE=${cadence} (Sunday=weekly per weekly cron line; else daily)."
+                    }
+                    def cephBranch = params.CEPH_BRANCH
+                    def cadenceJob = (params.TEUTHOLOGY_NIGHTLY_CADENCE_JOB ?: 'teuthology-nightly-cadence').trim() ?: 'teuthology-nightly-cadence'
+                    echo "Triggering ${cadenceJob} CADENCE=${cadence} CEPH_BRANCH=${cephBranch}"
+                    def rj = (params.TEUTHOLOGY_RUNNER_JOB_NAME ?: 'teuthology-runner').trim() ?: 'teuthology-runner'
+                    build(
+                        wait: false,
+                        job: cadenceJob,
+                        parameters: [
+                            string(name: 'CADENCE', value: cadence),
+                            string(name: 'CEPH_BRANCH', value: cephBranch),
+                            string(name: 'CEPH_BUILD_BRANCH', value: params.CEPH_BUILD_BRANCH.trim()),
+                            string(name: 'TEUTHOLOGY_RUNNER_JOB_NAME', value: rj),
+                        ],
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/teuthology-nightly-cadence/config/definitions/teuthology-nightly-cadence.yml b/teuthology-nightly-cadence/config/definitions/teuthology-nightly-cadence.yml
new file mode 100644
index 00000000..d997593a
--- /dev/null
+++ b/teuthology-nightly-cadence/config/definitions/teuthology-nightly-cadence.yml
@@ -0,0 +1,76 @@
+# Teuthology nightly cadence: runner + thin trigger.
+# Reference: ceph.git qa/crontab/teuthology-cronjobs and schedule_subset.sh.
+
+- job:
+    name: preserve-teuthology-nightly-cadence
+    description: |
+      Invokes job teuthology-runner in cadence mode (Shaman wait, schedule_subset-style
+      partitions/subsets). Optional WAIT_FOR_RUNS on teuthology-runner. Requires teuthology-runner
+      job in Jenkins. Ceph SHA from CEPH_SHA1 or branch tip via git ls-remote in teuthology-runner.
+    project-type: pipeline
+    quiet-period: 2
+    concurrent: false
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: teuthology-nightly-cadence/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+    parameters:
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: "ceph-build branch for Jenkinsfile and scripts"
+          default: "main"
+      - string:
+          name: TEUTHOLOGY_RUNNER_JOB_NAME
+          description: "Jenkins job name for the runner pipeline (default teuthology-runner)"
+          default: "preserve-teuthology-runner"
+
+- job:
+    name: preserve-teuthology-nightly-cadence-trigger
+    description: |
+      Triggers teuthology-nightly-cadence with wait:false. Timed triggers follow the same schedule as
+      ceph.git qa/crontab/teuthology-cronjobs.
+    project-type: pipeline
+    quiet-period: 0
+    concurrent: true
+    triggers:
+      - timed: |
+          TZ=Etc/UTC
+          # Daily tier: weekday morning.
+          0 6 * * 1-5
+          # Weekly tier: Sunday 20:00 UTC.
+          0 20 * * 0
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: teuthology-nightly-cadence/build/trigger/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+    parameters:
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: "ceph-build branch for Jenkinsfile"
+          default: "main"
+      - string:
+          name: TEUTHOLOGY_RUNNER_JOB_NAME
+          description: "Forwarded to teuthology-nightly-cadence (runner job Jenkins name)"
+          default: "preserve-teuthology-runner"
+      - string:
+          name: TEUTHOLOGY_NIGHTLY_CADENCE_JOB
+          description: "Jenkins job name of the cadence pipeline to trigger"
+          default: "preserve-teuthology-nightly-cadence"
diff --git a/teuthology-runner/README.rst b/teuthology-runner/README.rst
new file mode 100644
index 00000000..a56361f8
--- /dev/null
+++ b/teuthology-runner/README.rst
@@ -0,0 +1,16 @@
+teuthology-runner
+=================
+
+Schedules Teuthology suites from **caller-provided data** only.
+When **CEPH_SHA1** is empty, the branch tip is resolved with **``git ls-remote``** (no ceph repo clone).
+
+**SUITE_RUNS_JSON** (text parameter): JSON array of objects. Each object must include ``suite``
+(optional alias ``name``). Optional per-row fields: ``limit``, ``threshold``, ``subset``, ``priority``,
+``flavor``, ``kernel``, ``filter``, ``forcePriority``, ``suiteSha``. Missing fields fall back to job
+parameters ``SUITE_LIMIT``, ``SUITE_JOB_THRESHOLD``, ``SUITE_SUBSET``, ``SUITE_SHA``.
+
+**SUITE_LIST** (comma-separated names): used when ``SUITE_RUNS_JSON`` is empty; each run uses the
+global ``SUITE_LIMIT`` / ``SUITE_JOB_THRESHOLD`` / ``SUITE_SUBSET`` / ``SUITE_SHA``.
+
+Callers: **teuthology-nightly-cadence** builds JSON from its own cadence tables; **release-tracker-workflow**
+resolves a suite list and passes ``SUITE_LIST`` plus only the optional parameters that are set.
diff --git a/teuthology-runner/build/Jenkinsfile b/teuthology-runner/build/Jenkinsfile
new file mode 100644
index 00000000..e253eb25
--- /dev/null
+++ b/teuthology-runner/build/Jenkinsfile
@@ -0,0 +1,391 @@
+/**
+ * teuthology-runner: schedule Teuthology suite(s) from caller-supplied data; optional Shaman wait,
+ * teuthology-wait, Paddles aggregate. Pass SUITE_RUNS_JSON or SUITE_LIST.
+ */
+import com.cloudbees.groovy.cps.NonCPS
+
+pipeline {
+    agent { label "${params.AGENT_LABEL}" }
+    options {
+        timestamps()
+        timeout(time: 48, unit: 'HOURS')
+        buildDiscarder(logRotator(numToKeepStr: '40'))
+    }
+    environment {
+        PIPELINE_DIR = "${WORKSPACE}/teuthology-runner"
+        PULPITO_BASE = "${params.PULPITO_BASE ?: 'https://pulpito.ceph.com'}"
+    }
+    parameters {
+        string(name: 'PULPITO_BASE', defaultValue: 'https://pulpito.ceph.com', description: 'Pulpito base URL.')
+        string(name: 'AGENT_LABEL', defaultValue: 'teuthology', description: 'Jenkins agent label.')
+        string(name: 'CEPH_BUILD_BRANCH', defaultValue: 'main', description: 'ceph-build branch for this job SCM.')
+        string(name: 'CEPH_BRANCH', defaultValue: 'main', description: 'Ceph branch.')
+        string(name: 'CEPH_REPO', defaultValue: 'https://github.com/ceph/ceph.git', description: 'Ceph git URL.')
+        string(name: 'CEPH_SHA1', defaultValue: '', description: 'Optional SHA1. Empty = branch tip via git ls-remote.')
+        booleanParam(name: 'USE_WORKSPACE_TEUTHOLOGY', defaultValue: true, description: 'Clone teuthology into workspace.')
+        string(name: 'TEUTHOLOGY_REPO_URL', defaultValue: 'https://github.com/ceph/teuthology.git', description: 'Teuthology repo URL.')
+        string(name: 'TEUTHOLOGY_BRANCH', defaultValue: 'main', description: 'Teuthology branch.')
+        string(name: 'TEUTHOLOGY_SCRIPT_DIR', defaultValue: '', description: 'When USE_WORKSPACE_TEUTHOLOGY false: teuthology path.')
+        string(name: 'TEUTHOLOGY_VIRTUALENV_PATH', defaultValue: '', description: 'When USE_WORKSPACE_TEUTHOLOGY false: venv path.')
+        string(name: 'TEUTH_CONFIG_OVERRIDE_YAML', defaultValue: '', description: 'Optional extra argv for teuthology-suite (appended by run_teuthology_suite.sh).')
+        string(name: 'SUITE_REPO', defaultValue: 'https://github.com/ceph/ceph.git', description: 'teuthology-suite --suite-repo.')
+        string(name: 'SUITE_SHA', defaultValue: '', description: 'Optional --suite-sha1.')
+        string(name: 'SUITE_MACHINE_TYPE', defaultValue: 'trial', description: 'teuthology-suite --machine-type.')
+        string(name: 'SUITE_LIMIT', defaultValue: '1', description: 'Default --limit')
+        string(name: 'SUITE_JOB_THRESHOLD', defaultValue: '', description: 'Default --job-threshold for SUITE_LIST mode / JSON row fallback.')
+        string(name: 'SUITE_SUBSET', defaultValue: '', description: 'Default --subset for SUITE_LIST mode / JSON row fallback.')
+        string(name: 'SUITE_LIST', defaultValue: '', description: 'Comma-separated suite names when SUITE_RUNS_JSON empty.')
+        text(name: 'SUITE_RUNS_JSON', defaultValue: '', description: 'JSON array of objects: suite (required), optional limit, threshold, subset, priority, flavor, kernel, filter, forcePriority, suiteSha.')
+        string(name: 'PADDLES_URL', defaultValue: '', description: 'Override Paddles for aggregate; empty = /etc/teuthology.yaml results_server.')
+        string(name: 'SHAMAN_WAIT_TIMEOUT', defaultValue: '7200', description: 'Shaman wait timeout (seconds).')
+        string(name: 'SHAMAN_WAIT_INTERVAL', defaultValue: '120', description: 'Shaman poll interval (seconds).')
+        string(name: 'SHAMAN_WAIT_PLATFORMS', defaultValue: 'rocky-10-default,ubuntu-jammy-default,centos-9-default', description: 'Shaman --platform list.')
+        booleanParam(name: 'SKIP_SHAMAN_WAIT', defaultValue: false, description: 'Skip Shaman wait.')
+        booleanParam(name: 'WAIT_FOR_RUNS', defaultValue: false, description: 'Run teuthology-wait after scheduling.')
+        string(name: 'SUITE_WAIT_SLEEP', defaultValue: '15', description: 'Sleep before teuthology-wait.')
+        booleanParam(name: 'RUN_AGGREGATE', defaultValue: false, description: 'Write aggregate_table.txt.')
+    }
+    stages {
+        stage('Checkout ceph-build') {
+            steps {
+                checkout scm
+            }
+        }
+        stage('Resolve Ceph SHA') {
+            steps {
+                script {
+                    if (params.CEPH_SHA1?.trim()) {
+                        env.SHA1 = params.CEPH_SHA1.trim()
+                        echo "Using CEPH_SHA1 parameter: ${env.SHA1}"
+                    } else {
+                        def repo = params.CEPH_REPO.trim()
+                        def branch = params.CEPH_BRANCH.trim()
+                        if (!repo || !branch) {
+                            error('CEPH_REPO and CEPH_BRANCH are required when CEPH_SHA1 is empty')
+                        }
+                        if (!(branch ==~ '^[a-zA-Z0-9/._-]+$')) {
+                            error("CEPH_BRANCH contains unsupported characters: ${branch}")
+                        }
+                        if (!(repo ==~ '^[a-zA-Z0-9@.:/_-]+$')) {
+                            error('CEPH_REPO format rejected (use a plain https git URL)')
+                        }
+                        withEnv([
+                            "LS_REMOTE_REPO=${repo}",
+                            "LS_REMOTE_BRANCH=${branch}",
+                        ]) {
+                            env.SHA1 = sh(
+                                script: 'git ls-remote "$LS_REMOTE_REPO" "refs/heads/$LS_REMOTE_BRANCH" | head -1 | awk \'{print $1}\'',
+                                returnStdout: true,
+                            ).trim()
+                        }
+                        echo "CEPH_BRANCH=${branch} SHA1=${env.SHA1} (git ls-remote)"
+                    }
+                    if (!env.SHA1 || env.SHA1 == 'unknown') {
+                        error('Could not resolve SHA1 (ls-remote empty: check CEPH_REPO and branch ref)')
+                    }
+                    if (!(env.SHA1 ==~ /^[a-fA-F0-9]{7,40}$/)) {
+                        error("Resolved SHA1 does not look like a git commit: ${env.SHA1}")
+                    }
+                }
+            }
+        }
+        stage('Setup teuthology') {
+            steps {
+                script {
+                    if (params.USE_WORKSPACE_TEUTHOLOGY) {
+                        def teuthDir = "${env.WORKSPACE}/teuthology"
+                        dir(teuthDir) {
+                            checkout([
+                                $class: 'GitSCM',
+                                branches: [[name: "*/${params.TEUTHOLOGY_BRANCH ?: 'main'}"]],
+                                extensions: [[$class: 'CloneOption', depth: 1, shallow: true]],
+                                userRemoteConfigs: [[url: params.TEUTHOLOGY_REPO_URL.trim()]],
+                            ])
+                        }
+                        dir(teuthDir) {
+                            if (fileExists('bootstrap')) {
+                                sh './bootstrap'
+                            } else {
+                                sh 'python3 -m venv virtualenv'
+                                sh 'virtualenv/bin/pip install --upgrade pip'
+                                sh 'virtualenv/bin/pip install -r requirements.txt 2>/dev/null || true'
+                                sh 'virtualenv/bin/pip install -e .'
+                            }
+                        }
+                        env.SCRIPT_DIR = teuthDir
+                        env.VIRTUALENV_PATH = "${teuthDir}/virtualenv"
+                    } else {
+                        def sd = params.TEUTHOLOGY_SCRIPT_DIR?.trim()
+                        def ve = params.TEUTHOLOGY_VIRTUALENV_PATH?.trim()
+                        if (!sd || !ve) {
+                            error('USE_WORKSPACE_TEUTHOLOGY is false: set TEUTHOLOGY_SCRIPT_DIR and TEUTHOLOGY_VIRTUALENV_PATH')
+                        }
+                        env.SCRIPT_DIR = sd
+                        env.VIRTUALENV_PATH = ve
+                    }
+                    env.TEUTH_CONFIG_OVERRIDE_YAML = params.TEUTH_CONFIG_OVERRIDE_YAML?.trim() ?: ''
+                    echo "Teuthology SCRIPT_DIR=${env.SCRIPT_DIR} VIRTUALENV_PATH=${env.VIRTUALENV_PATH}"
+                }
+            }
+        }
+        stage('Wait for Shaman') {
+            when { expression { return !params.SKIP_SHAMAN_WAIT } }
+            steps {
+                script {
+                    def w = "${env.PIPELINE_DIR}/scripts/wait_for_shaman_sha1.py"
+                    if (!fileExists(w)) {
+                        error("Missing ${w}")
+                    }
+                    sh "python3 ${w} --branch ${params.CEPH_BRANCH} --sha1 ${env.SHA1} --timeout ${params.SHAMAN_WAIT_TIMEOUT} --interval ${params.SHAMAN_WAIT_INTERVAL} --platform ${params.SHAMAN_WAIT_PLATFORMS}"
+                }
+            }
+        }
+        stage('Schedule suites') {
+            steps {
+                script {
+                    def teuthologyConfigPath = '/etc/teuthology.yaml'
+                    if (!fileExists(teuthologyConfigPath)) {
+                        error("Required site config not found: ${teuthologyConfigPath}")
+                    }
+                    def cfgText = readFile(teuthologyConfigPath)
+                    def aggUrl = parseAggPaddlesUrlFromTeuthYaml(cfgText)
+                    if (!aggUrl) {
+                        error("Could not parse results_server from ${teuthologyConfigPath}")
+                    }
+                    env.AGG_PADDLES_URL = params.PADDLES_URL?.trim() ?: (aggUrl.endsWith('/') ? aggUrl : (aggUrl + '/'))
+
+                    def runEnvBase = [
+                        "HOME=${env.WORKSPACE}",
+                    ] + paddlesTlsSslEnv() + [
+                        "TEUTHOLOGY_CONFIG=${teuthologyConfigPath}",
+                        "SCRIPT_DIR=${env.SCRIPT_DIR}",
+                        "VIRTUALENV_PATH=${env.VIRTUALENV_PATH}",
+                    ]
+
+                    def runInfos = []
+                    def jsonText = params.SUITE_RUNS_JSON?.trim()
+
+                    if (jsonText) {
+                        def items = parseSuiteRunsJson(jsonText)
+                        if (!items) {
+                            error('SUITE_RUNS_JSON must be a JSON array of objects with a "suite" field')
+                        }
+                        withEnv(runEnvBase) {
+                            for (def item in items) {
+                                if (!(item instanceof Map)) {
+                                    error('SUITE_RUNS_JSON: each element must be an object')
+                                }
+                                def row = item as Map
+                                def runName = scheduleFromRow(row)
+                                def sn = row.suite?.toString()?.trim() ?: row.name?.toString()?.trim()
+                                runInfos << [sn, runName]
+                                echo "Scheduled run: ${runName}"
+                            }
+                        }
+                    } else {
+                        def suites = params.SUITE_LIST?.trim()?.split(',')?.collect { it.trim() }?.findAll { it }
+                        if (!suites) {
+                            error('Set SUITE_RUNS_JSON (JSON array) or non-empty SUITE_LIST')
+                        }
+                        def suiteNamePattern = ~'^[a-zA-Z0-9_/:-]+$'
+                        for (suite in suites) {
+                            def s = suite?.toString()?.trim()
+                            if (!s || !(s ==~ suiteNamePattern)) {
+                                error("Invalid suite name: ${suite}")
+                            }
+                        }
+                        withEnv(runEnvBase) {
+                            for (suite in suites) {
+                                def runName = scheduleFromRow([
+                                    suite: suite,
+                                    limit: params.SUITE_LIMIT?.trim() ?: '1',
+                                    threshold: params.SUITE_JOB_THRESHOLD?.trim() ?: '',
+                                    subset: params.SUITE_SUBSET?.trim() ?: '',
+                                ])
+                                runInfos << [suite as String, runName]
+                                echo "Scheduled run: ${runName}"
+                            }
+                        }
+                    }
+
+                    env.TEUTHOLOGY_RUN_NAMES = runInfos.collect { it[1] }.join(',')
+                    env.TEUTHOLOGY_RUN_NAME = runInfos ? runInfos[0][1] : ''
+                    writeFile file: "${WORKSPACE}/teuthology_run_infos.txt", text: runInfos.collect { "${it[0]}|${it[1]}" }.join('\n')
+                }
+            }
+        }
+        stage('Wait for suite runs') {
+            when { expression { return params.WAIT_FOR_RUNS } }
+            steps {
+                script {
+                    def names = env.TEUTHOLOGY_RUN_NAMES?.trim()
+                    if (!names) {
+                        echo 'No teuthology runs to wait for.'
+                        return
+                    }
+                    def runNameList = names.split(',').collect { it.trim() }.findAll { it }
+                    sleep(time: params.SUITE_WAIT_SLEEP.toInteger(), unit: 'SECONDS')
+                    def waitEnv = ["HOME=${env.WORKSPACE}"] + paddlesTlsSslEnv()
+                    if (fileExists('/etc/teuthology.yaml')) {
+                        waitEnv << 'TEUTHOLOGY_CONFIG=/etc/teuthology.yaml'
+                    }
+                    withEnv(waitEnv) {
+                        dir(env.SCRIPT_DIR) {
+                            for (runName in runNameList) {
+                                try {
+                                    sh "${env.VIRTUALENV_PATH}/bin/teuthology-wait --run ${runName}"
+                                } catch (Exception e) {
+                                    echo "teuthology-wait failed for ${runName}: ${e.message}"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        stage('Aggregate results') {
+            when { expression { return params.RUN_AGGREGATE && env.TEUTHOLOGY_RUN_NAMES?.trim() } }
+            steps {
+                script {
+                    def runNames = env.TEUTHOLOGY_RUN_NAMES.split(',').collect { it.trim() }.findAll { it }
+                    def aggScript = "${env.PIPELINE_DIR}/scripts/aggregate_suite_results.py"
+                    if (!fileExists(aggScript)) {
+                        writeFile file: "${WORKSPACE}/aggregate_table.txt", text: "Runs: ${env.TEUTHOLOGY_RUN_NAMES}"
+                    } else {
+                        def runFlags = runNames.collect { "--run ${it}" }.join(' ')
+                        withEnv(paddlesTlsSslEnv()) {
+                            sh "python3 ${aggScript} ${runFlags} --paddles-url ${env.AGG_PADDLES_URL} --out ${WORKSPACE}/aggregate_table.txt"
+                        }
+                    }
+                }
+            }
+        }
+        stage('Archive artifacts') {
+            steps {
+                archiveArtifacts artifacts: 'teuthology_run_infos.txt,aggregate_table.txt', allowEmptyArchive: true
+            }
+        }
+    }
+}
+
+@NonCPS
+List parseSuiteRunsJson(String json) {
+    def raw = new groovy.json.JsonSlurper().parseText(json)
+    if (!(raw instanceof List)) {
+        return null
+    }
+    def out = []
+    for (def o in (raw as List)) {
+        if (o instanceof Map) {
+            def copy = [:]
+            o.each { k, v -> copy[k] = v }
+            out << copy
+        }
+    }
+    return out
+}
+
+@NonCPS
+def parseAggPaddlesUrlFromTeuthYaml(String cfgText) {
+    def m = (cfgText =~ ~'(?m)^\\s*results_server\\s*:\\s*(\\S+)\\s*$')
+    if (m.find()) {
+        return m.group(1).trim()
+    }
+    m = (cfgText =~ ~'(?m)^\\s*lock_server\\s*:\\s*(\\S+)\\s*$')
+    if (m.find()) {
+        return m.group(1).trim()
+    }
+    return null
+}
+
+def paddlesTlsSslEnv() {
+    return [
+        'REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt',
+        'SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt',
+    ]
+}
+
+String coalesceRow(Map row, String key, String paramFallback) {
+    if (row.containsKey(key) && row[key] != null && row[key].toString().trim()) {
+        return row[key].toString().trim()
+    }
+    return paramFallback ?: ''
+}
+
+boolean rowForcePriority(Map row) {
+    def v = row.forcePriority
+    if (v == null) {
+        return false
+    }
+    if (v instanceof Boolean) {
+        return v.booleanValue()
+    }
+    return 'true'.equalsIgnoreCase(v.toString().trim())
+}
+
+String scheduleFromRow(Map row) {
+    def suite = row.suite?.toString()?.trim() ?: row.name?.toString()?.trim()
+    if (!suite) {
+        error('Each schedule row must include "suite" (or "name")')
+    }
+    def lim = coalesceRow(row, 'limit', params.SUITE_LIMIT?.trim() ?: '1')
+    def thr = coalesceRow(row, 'threshold', params.SUITE_JOB_THRESHOLD?.trim() ?: '')
+    def subset = coalesceRow(row, 'subset', params.SUITE_SUBSET?.trim() ?: '')
+    def priority = coalesceRow(row, 'priority', '')
+    def flavor = coalesceRow(row, 'flavor', '')
+    def kernel = coalesceRow(row, 'kernel', '')
+    def filter = coalesceRow(row, 'filter', '')
+    def fp = rowForcePriority(row) ? 'true' : ''
+    def suiteSha = coalesceRow(row, 'suiteSha', params.SUITE_SHA?.trim() ?: '')
+
+    def safeName = suite.replaceAll('/', '_')
+    def runName = null
+    dir(env.SCRIPT_DIR) {
+        def outFile = "${env.WORKSPACE}/suite_out_${safeName}_${System.currentTimeMillis()}.txt"
+        def runner = "${env.PIPELINE_DIR}/scripts/run_teuthology_suite.sh"
+        if (!fileExists(runner)) {
+            error("run_teuthology_suite.sh not found: ${runner}")
+        }
+        def machineType = params.SUITE_MACHINE_TYPE?.trim() ?: 'smithi'
+        def suiteRepo = params.SUITE_REPO?.trim() ?: ''
+        def overrideYaml = env.TEUTH_CONFIG_OVERRIDE_YAML?.trim() ?: ''
+        def envList = [
+            "SUITE_NAME=${suite}",
+            "CEPH_BRANCH=${params.CEPH_BRANCH}",
+            "CEPH_SHA1=${env.SHA1}",
+            "TEUTH_CONFIG_OVERRIDE_YAML=${overrideYaml}",
+            "CEPH_REPO=${params.CEPH_REPO}",
+            "SUITE_LIMIT=${lim}",
+            "MACHINE_TYPE=${machineType}",
+            "SUITE_REPO=${suiteRepo}",
+            "SUITE_JOB_THRESHOLD=${thr}",
+            "SUITE_SUBSET=${subset}",
+            "SUITE_PRIORITY=${priority}",
+            "SUITE_FLAVOR=${flavor}",
+            "SUITE_KERNEL=${kernel}",
+            "SUITE_FILTER=${filter}",
+            "SUITE_FORCE_PRIORITY=${fp}",
+            "SUITE_SHA=${suiteSha}",
+        ]
+        withEnv(envList) {
+            sh(script: "bash \"${runner}\" > \"${outFile}\" 2>&1; true", returnStatus: true)
+        }
+        def out = readFile(outFile)
+        echo "teuthology-suite output (tail):\n${out.take(50000)}"
+        def prefix = 'Job scheduled with name '
+        def i = out.indexOf(prefix)
+        if (i >= 0) {
+            def start = i + prefix.length()
+            def lineEnd = out.indexOf('\n', start)
+            if (lineEnd < 0) {
+                lineEnd = out.length()
+            }
+            def restOfLine = out.substring(start, lineEnd).trim()
+            runName = restOfLine.split()[0]
+        }
+        if (!runName) {
+            error("teuthology-suite did not schedule '${suite}'. Expected 'Job scheduled with name' in output.")
+        }
+    }
+    return runName
+}
diff --git a/teuthology-runner/config/definitions/teuthology-runner.yml b/teuthology-runner/config/definitions/teuthology-runner.yml
new file mode 100644
index 00000000..ade4f671
--- /dev/null
+++ b/teuthology-runner/config/definitions/teuthology-runner.yml
@@ -0,0 +1,28 @@
+# Shared Teuthology job: Shaman (optional), schedule suites, optional teuthology-wait, optional aggregate.
+# Used by teuthology-nightly-cadence and release-tracker-workflow .
+- job:
+    name: preserve-teuthology-runner
+    description: |
+      Resolve Ceph SHA, optional Shaman wait, clone/bootstrap teuthology, schedule suite(s) from
+      SUITE_RUNS_JSON or SUITE_LIST. Optional teuthology-wait and Paddles aggregate.
+    project-type: pipeline
+    quiet-period: 1
+    concurrent: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/ceph/ceph-build
+            branches:
+              - ${{CEPH_BUILD_BRANCH}}
+            shallow-clone: true
+            submodule:
+              disable: true
+            wipe-workspace: true
+      script-path: teuthology-runner/build/Jenkinsfile
+      lightweight-checkout: true
+      do-not-fetch-tags: true
+    parameters:
+      - string:
+          name: CEPH_BUILD_BRANCH
+          description: "ceph-build branch for Jenkinsfile and scripts"
+          default: "main"
diff --git a/teuthology-runner/scripts/aggregate_suite_results.py b/teuthology-runner/scripts/aggregate_suite_results.py
new file mode 100644
index 00000000..6bb0636a
--- /dev/null
+++ b/teuthology-runner/scripts/aggregate_suite_results.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Fetch job results from Paddles for one or more runs and output aggregated pass/fail table.
+
+Usage:
+  aggregate_suite_results.py --run RUN [--run RUN ...] [--paddles-url URL] [--out FILE]
+"""
+import argparse
+import sys
+try:
+    import requests
+except ImportError:
+    print("pip install requests", file=sys.stderr)
+    sys.exit(2)
+
+DEFAULT_PADDLES = "http://paddles.front.sepia.ceph.com/"
+
+
+def get_jobs(paddles_url, run_name, fields=None):
+    fields = fields or ["job_id", "status", "description"]
+    if "job_id" not in fields:
+        fields = list(fields) + ["job_id"]
+    uri = f"{paddles_url.rstrip('/')}/runs/{run_name}/jobs/?fields={','.join(fields)}"
+    try:
+        r = requests.get(uri, timeout=60)
+        r.raise_for_status()
+        return r.json()
+    except Exception as e:
+        print(f"Failed to get jobs: {e}", file=sys.stderr)
+        return None
+
+
+def suite_from_desc(desc):
+    return (desc or "unknown").split("/")[0] if "/" in (desc or "") else (desc or "unknown").split()[0] if desc else "unknown"
+
+
+def aggregate(jobs):
+    by_suite = {}
+    for j in jobs:
+        desc = j.get("description") or ""
+        status = (j.get("status") or "unknown").lower()
+        suite = suite_from_desc(desc)
+        if suite not in by_suite:
+            by_suite[suite] = {"pass": 0, "fail": 0}
+        if status == "pass":
+            by_suite[suite]["pass"] += 1
+        else:
+            by_suite[suite]["fail"] += 1
+    return {s: "PASS" if c["fail"] == 0 and c["pass"] > 0 else "FAIL" for s, c in by_suite.items()}
+
+
+def merged_table(paddles_url, run_names):
+    combined = {}
+    any_jobs = False
+    for run_name in run_names:
+        jobs = get_jobs(paddles_url, run_name)
+        if jobs is None:
+            return None
+        if not jobs:
+            continue
+        any_jobs = True
+        combined.update(aggregate(jobs))
+    if not any_jobs:
+        return "No jobs found"
+    return "\n".join(
+        ["Suite | Status", "------|------"]
+        + [f"{k} | {v}" for k, v in sorted(combined.items())]
+    )
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--run",
+        action="append",
+        dest="runs",
+        required=True,
+        metavar="RUN",
+        help="Teuthology run name (repeat for multiple runs; merged into one table).",
+    )
+    ap.add_argument("--paddles-url", default=DEFAULT_PADDLES)
+    ap.add_argument("--out", default=None)
+    args = ap.parse_args()
+    table = merged_table(args.paddles_url, args.runs)
+    if table is None:
+        sys.exit(1)
+    print(table)
+    if args.out:
+        with open(args.out, "w") as f:
+            f.write(table)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/teuthology-runner/scripts/run_teuthology_suite.sh b/teuthology-runner/scripts/run_teuthology_suite.sh
new file mode 100644
index 00000000..9e59a9f2
--- /dev/null
+++ b/teuthology-runner/scripts/run_teuthology_suite.sh
@@ -0,0 +1,133 @@
+#!/usr/bin/env bash
+# Run teuthology-suite with fixed argv, intended for Jenkins (nightly cadence).
+# Required env: VIRTUALENV_PATH, SUITE_NAME, CEPH_BRANCH, CEPH_SHA1
+# Optional: TEUTH_CONFIG_OVERRIDE_YAML (legacy: OVERRIDE_YAML), MACHINE_TYPE, CEPH_REPO, SUITE_LIMIT, SUITE_JOB_THRESHOLD, SUITE_SUBSET,
+# SUITE_REPO, SUITE_SHA, SUITE_PRIORITY (-p), SUITE_KERNEL, SUITE_FILTER, SUITE_FLAVOR, SUITE_FORCE_PRIORITY
+set -euo pipefail
+
+: "${VIRTUALENV_PATH:?VIRTUALENV_PATH must be set}"
+: "${SUITE_NAME:?SUITE_NAME must be set}"
+: "${CEPH_BRANCH:?CEPH_BRANCH must be set}"
+: "${CEPH_SHA1:?CEPH_SHA1 must be set}"
+
+if [[ ! "${SUITE_NAME}" =~ ^[a-zA-Z0-9_/:-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_NAME: ${SUITE_NAME}" >&2
+  exit 1
+fi
+if [[ ! "${CEPH_BRANCH}" =~ ^[a-zA-Z0-9/._-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid CEPH_BRANCH: ${CEPH_BRANCH}" >&2
+  exit 1
+fi
+if [[ "${CEPH_SHA1}" == "unknown" ]]; then
+  :
+elif [[ ! "${CEPH_SHA1}" =~ ^[a-fA-F0-9]+$ ]] || [[ ${#CEPH_SHA1} -lt 7 ]]; then
+  echo "run_teuthology_suite.sh: invalid CEPH_SHA1: ${CEPH_SHA1}" >&2
+  exit 1
+fi
+
+TEUTHOLOGY_SUITE="${VIRTUALENV_PATH}/bin/teuthology-suite"
+if [[ ! -f "${TEUTHOLOGY_SUITE}" ]]; then
+  echo "run_teuthology_suite.sh: teuthology-suite not found: ${TEUTHOLOGY_SUITE}" >&2
+  exit 1
+fi
+
+MACHINE_TYPE="${MACHINE_TYPE:-trial}"
+CEPH_REPO="${CEPH_REPO:-https://github.com/ceph/ceph.git}"
+SUITE_LIMIT="${SUITE_LIMIT:-1}"
+
+if [[ ! "${MACHINE_TYPE}" =~ ^[a-zA-Z0-9_.-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid MACHINE_TYPE: ${MACHINE_TYPE}" >&2
+  exit 1
+fi
+if [[ ! "${SUITE_LIMIT}" =~ ^[0-9]+$ ]] || [[ "${SUITE_LIMIT}" == "0" ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_LIMIT (positive integer): ${SUITE_LIMIT}" >&2
+  exit 1
+fi
+
+if [[ -n "${SUITE_REPO:-}" ]] && [[ ! "${SUITE_REPO}" =~ ^[a-zA-Z0-9@.:/_-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_REPO: ${SUITE_REPO}" >&2
+  exit 1
+fi
+if [[ -n "${SUITE_SHA:-}" ]]; then
+  if [[ ! "${SUITE_SHA}" =~ ^[a-fA-F0-9]+$ ]] || [[ ${#SUITE_SHA} -lt 7 ]]; then
+    echo "run_teuthology_suite.sh: invalid SUITE_SHA: ${SUITE_SHA}" >&2
+    exit 1
+  fi
+fi
+
+if [[ -n "${SUITE_JOB_THRESHOLD:-}" ]]; then
+  if [[ ! "${SUITE_JOB_THRESHOLD}" =~ ^[0-9]+$ ]]; then
+    echo "run_teuthology_suite.sh: invalid SUITE_JOB_THRESHOLD (non-negative integer): ${SUITE_JOB_THRESHOLD}" >&2
+    exit 1
+  fi
+fi
+if [[ -n "${SUITE_SUBSET:-}" ]]; then
+  if [[ ! "${SUITE_SUBSET}" =~ ^[0-9]+/[0-9]+$ ]]; then
+    echo "run_teuthology_suite.sh: invalid SUITE_SUBSET (expected N/M, e.g. 1/10000): ${SUITE_SUBSET}" >&2
+    exit 1
+  fi
+fi
+
+if [[ -n "${SUITE_PRIORITY:-}" ]]; then
+  if [[ ! "${SUITE_PRIORITY}" =~ ^[0-9]+$ ]]; then
+    echo "run_teuthology_suite.sh: invalid SUITE_PRIORITY: ${SUITE_PRIORITY}" >&2
+    exit 1
+  fi
+fi
+if [[ -n "${SUITE_KERNEL:-}" ]] && [[ ! "${SUITE_KERNEL}" =~ ^[a-zA-Z0-9._-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_KERNEL: ${SUITE_KERNEL}" >&2
+  exit 1
+fi
+if [[ -n "${SUITE_FILTER:-}" ]] && [[ ! "${SUITE_FILTER}" =~ ^[a-zA-Z0-9/_.-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_FILTER: ${SUITE_FILTER}" >&2
+  exit 1
+fi
+if [[ -n "${SUITE_FLAVOR:-}" ]] && [[ ! "${SUITE_FLAVOR}" =~ ^[a-zA-Z0-9._-]+$ ]]; then
+  echo "run_teuthology_suite.sh: invalid SUITE_FLAVOR: ${SUITE_FLAVOR}" >&2
+  exit 1
+fi
+
+set -- \
+  "${TEUTHOLOGY_SUITE}" \
+  --suite "${SUITE_NAME}" \
+  --machine-type "${MACHINE_TYPE}" \
+  --ceph "${CEPH_BRANCH}" \
+  --ceph-repo "${CEPH_REPO}" \
+  --limit "${SUITE_LIMIT}"
+
+if [[ -n "${SUITE_PRIORITY:-}" ]]; then
+  set -- "$@" -p "${SUITE_PRIORITY}"
+fi
+if [[ -n "${SUITE_JOB_THRESHOLD:-}" ]]; then
+  set -- "$@" --job-threshold "${SUITE_JOB_THRESHOLD}"
+fi
+if [[ -n "${SUITE_SUBSET:-}" ]]; then
+  set -- "$@" --subset "${SUITE_SUBSET}"
+fi
+
+set -- "$@" --sha1 "${CEPH_SHA1}"
+
+if [[ -n "${SUITE_REPO:-}" ]]; then
+  set -- "$@" --suite-repo "${SUITE_REPO}"
+fi
+if [[ -n "${SUITE_SHA:-}" ]]; then
+  set -- "$@" --suite-sha1 "${SUITE_SHA}"
+fi
+if [[ -n "${SUITE_FILTER:-}" ]]; then
+  set -- "$@" --filter "${SUITE_FILTER}"
+fi
+if [[ -n "${SUITE_FLAVOR:-}" ]]; then
+  set -- "$@" --flavor "${SUITE_FLAVOR}"
+fi
+if [[ -n "${SUITE_KERNEL:-}" ]]; then
+  set -- "$@" --kernel "${SUITE_KERNEL}"
+fi
+if [[ "${SUITE_FORCE_PRIORITY:-}" == "true" ]]; then
+  set -- "$@" --force-priority
+fi
+TEUTH_CFG_OVERRIDE="${TEUTH_CONFIG_OVERRIDE_YAML:-${OVERRIDE_YAML:-}}"
+if [[ -n "${TEUTH_CFG_OVERRIDE}" ]]; then
+  set -- "$@" "${TEUTH_CFG_OVERRIDE}"
+fi
+
+exec "$@"
diff --git a/teuthology-runner/scripts/wait_for_shaman_sha1.py b/teuthology-runner/scripts/wait_for_shaman_sha1.py
new file mode 100644
index 00000000..f0a213d1
--- /dev/null
+++ b/teuthology-runner/scripts/wait_for_shaman_sha1.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""Wait for Shaman availability for branch/platform(s).
+
+Modes:
+  - With --sha1: wait until that exact SHA exists on all platforms.
+  - Without --sha1 and without --use-available-sha: resolve branch tip via
+    git ls-remote and wait for that exact SHA on all platforms.
+  - Without --sha1 and with --use-available-sha: pick the newest SHA that is
+    already available on Shaman, without waiting.
+Exits 0 when ready, 1 on timeout. Prints SHA1 on success.
+"""
+import argparse
+import subprocess
+import sys
+import time
+try:
+    import requests
+except ImportError:
+    print("pip install requests", file=sys.stderr)
+    sys.exit(2)
+
+SHAMAN_BUILD_URL = "https://shaman.ceph.com/api/repos/ceph/{branch}/{sha1}/{os_type}/{os_version}/flavors/{flavor}/"
+SHAMAN_LATEST_URL = "https://shaman.ceph.com/api/repos/ceph/{branch}/latest/{os_type}/{os_version}/flavors/{flavor}/"
+
+
+def _fetch_builds(url):
+    r = requests.get(url, timeout=30)
+    if not r.ok:
+        return []
+    payload = r.json()
+    return payload if isinstance(payload, list) else [payload]
+
+
+def sha1_on_platform(branch, platform, sha1, arch="x86_64"):
+    parts = platform.split("-", 2)
+    if len(parts) < 3:
+        return False
+    os_type, os_version, flavor = parts[0], parts[1], parts[2]
+    url = SHAMAN_BUILD_URL.format(
+        branch=branch,
+        sha1=sha1,
+        os_type=os_type,
+        os_version=os_version,
+        flavor=flavor,
+    )
+    try:
+        builds = _fetch_builds(url)
+        for b in builds:
+            if arch in b.get("archs", []) and b.get("sha1") == sha1:
+                return True
+        return False
+    except Exception:
+        return False
+
+
+def latest_sha_on_platform(branch, platform, arch="x86_64"):
+    parts = platform.split("-", 2)
+    if len(parts) < 3:
+        return None
+    os_type, os_version, flavor = parts[0], parts[1], parts[2]
+    url = SHAMAN_LATEST_URL.format(branch=branch, os_type=os_type, os_version=os_version, flavor=flavor)
+    try:
+        builds = _fetch_builds(url)
+        for b in builds:
+            if arch in b.get("archs", []) and b.get("sha1"):
+                return b.get("sha1")
+        return None
+    except Exception:
+        return None
+
+
+def resolve_branch_tip_sha(repo, branch):
+    try:
+        out = subprocess.check_output(
+            ["git", "ls-remote", repo, f"refs/heads/{branch}"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+            timeout=30,
+        ).strip()
+    except Exception:
+        return None
+    if not out:
+        return None
+    sha = out.split()[0].strip().lower()
+    if len(sha) < 7:
+        return None
+    return sha
+
+
+def latest_shas_on_platform(branch, platform, arch="x86_64"):
+    parts = platform.split("-", 2)
+    if len(parts) < 3:
+        return []
+    os_type, os_version, flavor = parts[0], parts[1], parts[2]
+    url = SHAMAN_LATEST_URL.format(branch=branch, os_type=os_type, os_version=os_version, flavor=flavor)
+    try:
+        builds = _fetch_builds(url)
+        out = []
+        for b in builds:
+            if arch in b.get("archs", []) and b.get("sha1"):
+                out.append(b.get("sha1"))
+        return out
+    except Exception:
+        return []
+
+
+def newest_common_latest_sha(branch, platforms, arch="x86_64"):
+    platform_shas = [latest_shas_on_platform(branch, p, arch) for p in platforms]
+    if not all(platform_shas):
+        return None
+    common = set(platform_shas[0])
+    for shas in platform_shas[1:]:
+        common &= set(shas)
+    if not common:
+        return None
+    # Keep platform order from the first platform list (newest first).
+    for sha in platform_shas[0]:
+        if sha in common:
+            return sha
+    return None
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--branch", required=True)
+    ap.add_argument("--sha1", default="")
+    ap.add_argument(
+        "--platform",
+        default="rocky-10-default,ubuntu-jammy-default,centos-9-default",
+        help="Comma-separated Shaman platform keys (os-osver-flavor), e.g. ubuntu-noble-default.",
+    )
+    ap.add_argument("--timeout", type=int, default=3600)
+    ap.add_argument("--interval", type=int, default=60)
+    ap.add_argument("--arch", default="x86_64")
+    ap.add_argument("--repo", default="https://github.com/ceph/ceph.git")
+    ap.add_argument(
+        "--use-available-sha",
+        action="store_true",
+        help="Without --sha1, pick newest SHA common across platforms without waiting.",
+    )
+    args = ap.parse_args()
+    branch = args.branch.strip().lower()
+    sha1 = args.sha1.strip().lower() if args.sha1 else ""
+    platforms = [p.strip() for p in args.platform.split(",") if p.strip()] or ["rocky-10-default", "ubuntu-jammy-default", "centos-9-default"]
+
+    # Fast path: no polling. Return newest currently available common SHA.
+    if not sha1 and args.use_available_sha:
+        chosen = newest_common_latest_sha(branch, platforms, args.arch)
+        if chosen:
+            print(chosen)
+            return 0
+        print(f"No common available SHA on Shaman for {branch}", file=sys.stderr)
+        return 1
+
+    if not sha1:
+        sha1 = resolve_branch_tip_sha(args.repo.strip(), branch)
+        if not sha1:
+            print(
+                f"Could not resolve branch tip via git ls-remote for repo={args.repo} branch={branch}",
+                file=sys.stderr,
+            )
+            return 1
+
+    start = time.monotonic()
+    while True:
+        if sha1:
+            if all(sha1_on_platform(branch, p, sha1, args.arch) for p in platforms):
+                print(sha1)
+                return 0
+        if time.monotonic() - start >= args.timeout:
+            if sha1:
+                print(f"Timeout: SHA1 {sha1} not on Shaman for {branch}", file=sys.stderr)
+            return 1
+        time.sleep(args.interval)
+
+
+if __name__ == "__main__":
+    sys.exit(main())