From: Kautilya Tripathi Date: Thu, 9 Jul 2026 02:56:30 +0000 (+0530) Subject: ceph-perf: bound CBT hangs and make comment triggers reliable X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=bc886cbb304c43acbdd9d3f90e8d8c239bca1eca;p=ceph-build.git ceph-perf: bound CBT hangs and make comment triggers reliable radosbench prefill has wedged bath01 for days with no job timeout. Cap each run-cbt.sh at 1h and the job at 8h, kill leftover Ceph processes around CBT, and require an explicit "jenkins test {osd-flavor} perf" comment so label-only auto-builds do not queue forever on a stuck performance node. Signed-off-by: Kautilya Tripathi --- diff --git a/ceph-perf-pull-requests/README.rst b/ceph-perf-pull-requests/README.rst index 49849c92f..2c35d2766 100644 --- a/ceph-perf-pull-requests/README.rst +++ b/ceph-perf-pull-requests/README.rst @@ -24,12 +24,16 @@ updated with the comparison report. Triggering ---------- -On a ``ceph/ceph`` pull request, comment:: +On a ``ceph/ceph`` pull request, comment (required; case-insensitive):: jenkins test classic perf jenkins test crimson perf -Pull requests with the ``performance`` label may also be built automatically. +Builds are comment-only (``only-trigger-phrase: true``). The ``performance`` +label is not used as an auto-trigger. See an explicit comment for each flavor. +Each CBT run is bounded to 1 hour and the whole job to 8 hours so a hung +``radosbench`` cannot hold a performance node indefinitely. + See ``doc/dev/continuous-integration.rst`` in the Ceph tree for an overview of how this job fits into CI. diff --git a/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml b/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml index 99df4d628..69b5539d0 100644 --- a/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml +++ b/ceph-perf-pull-requests/config/definitions/ceph-perf-pull-requests.yml @@ -69,15 +69,42 @@ timeout 7200 src/script/run-make.sh \ --cmake-args "$cmake_args" \ vstart-base crimson-osd + # Leftover daemons from a previous hung run can wedge radosbench forever. + if test -x build/../src/stop.sh || test -x src/stop.sh ; then + src/stop.sh --crimson || true + src/stop.sh || true + fi + sudo pkill -9 -f '(crimson-osd|ceph-osd|ceph-mon|ceph-mgr|rados bench)' || true + sleep 2 . ${{WORKSPACE}}/gh-venv/bin/activate + # CBT itself has no set -e / timeout; radosbench prefill has hung for days on + # bath01. Bound the whole run-cbt.sh (vstart + bench + stop) to 1h. + set +e if test {osd-flavor} = "classic" ; then - env PATH="${{WORKSPACE}}/gh-venv/bin:$PATH" \ + timeout --foreground --signal=KILL 3600 \ + env PATH="${{WORKSPACE}}/gh-venv/bin:$PATH" \ src/script/run-cbt.sh --build-dir $PWD/build --source-dir $PWD --cbt $WORKSPACE/cbt -a $archive_dir {benchmark-yaml} --classical + cbt_rc=$? else - env PATH="${{WORKSPACE}}/gh-venv/bin:$PATH" \ + timeout --foreground --signal=KILL 3600 \ + env PATH="${{WORKSPACE}}/gh-venv/bin:$PATH" \ src/script/run-cbt.sh --build-dir $PWD/build --source-dir $PWD --cbt $WORKSPACE/cbt -a $archive_dir {benchmark-yaml} + cbt_rc=$? + fi + set -e + # Always try to tear down; ignore failures so we still report the CBT status. + src/stop.sh --crimson || true + src/stop.sh || true + sudo pkill -9 -f '(crimson-osd|ceph-osd|ceph-mon|ceph-mgr|rados bench)' || true + if test "$cbt_rc" -eq 124 -o "$cbt_rc" -eq 137 ; then + echo "ERROR: run-cbt.sh timed out after 3600s (rc=$cbt_rc); likely hung radosbench/Crimson on this node" >&2 + exit 1 + fi + if test "$cbt_rc" -ne 0 ; then + echo "ERROR: run-cbt.sh failed with rc=$cbt_rc" >&2 + exit "$cbt_rc" fi - # run-cbt.sh is /bin/sh without set -e; a failed cbt.py still exits 0. + # run-cbt.sh is /bin/sh without set -e; a failed cbt.py can still exit 0. test -n "$(find "$archive_dir" -mindepth 1 -print -quit 2>/dev/null)" - builder: @@ -126,7 +153,7 @@ concurrent: true node: performance display-name: 'ceph: {osd-flavor} perf test' - description: 'CBT performance regression check ({osd-flavor} flavor): builds ceph-pr and ceph-main, runs the checked-in radosbench 4K read workload, and compares results via cbt/compare.py. The GitHub check fails when compare.py reports a statistically worse result, too much noise, or errors.' + description: 'CBT performance regression check ({osd-flavor} flavor): builds ceph-pr and ceph-main, runs the checked-in radosbench 4K read workload, and compares results via cbt/compare.py. The GitHub check fails when compare.py reports a statistically worse result, too much noise, or errors. Trigger with comment: jenkins test {osd-flavor} perf' quiet-period: 5 block-downstream: false block-upstream: false @@ -163,11 +190,14 @@ allow-whitelist-orgs-as-admins: true org-list: - ceph - trigger-phrase: 'jenkins test {osd-flavor} perf' - skip-build-phrase: '^jenkins do not test.*' - only-trigger-phrase: false - white-list-labels: - - performance + # Comment-only: reliable path. Auto-build on every push with the + # performance label was confusing when the sole performance node was + # wedged (builds queued forever and looked like "trigger never works"). + # Use an explicit comment for each flavor. + # ghprb matches trigger-phrase as a Java regex (case-insensitive). + trigger-phrase: '(?i)jenkins\s+test\s+{osd-flavor}\s+perf\b' + skip-build-phrase: '(?i)^jenkins do not test.*' + only-trigger-phrase: true github-hooks: true permit-all: false auto-close-on-fail: false @@ -214,6 +244,9 @@ exit 1 ;; esac + # Clear leftover cluster processes from a previous hung/aborted run on + # this performance node before we spend hours rebuilding. + sudo pkill -9 -f '(crimson-osd|ceph-osd|ceph-mon|ceph-mgr|rados bench)' || true virtualenv -q --python python3 ${{WORKSPACE}}/gh-venv . ${{WORKSPACE}}/gh-venv/bin/activate pip install setuptools @@ -271,9 +304,21 @@ - FAILURE - ABORTED build-steps: - - shell: "sudo reboot" + - shell: | + #!/bin/bash + # Kill hung CBT/Ceph first so reboot is not the only recovery path + # when the agent is still responsive enough to run this step. + sudo pkill -9 -f '(crimson-osd|ceph-osd|ceph-mon|ceph-mgr|rados bench|cbt\.py|run-cbt\.sh)' || true + sudo reboot wrappers: + - timeout: + # Two Crimson builds (≤2h each) + two CBT runs (≤1h each) + setup/compare. + # Absolute ceiling so a wedged radosbench cannot hold the node for days. + timeout: 480 + timeout-var: 'BUILD_TIMEOUT' + fail: true + type: absolute - credentials-binding: - file: credential-id: cephacheck.2020-04-29.private-key.pem