From: jitendrasahu1803 Date: Fri, 12 Jun 2026 05:13:36 +0000 (+0530) Subject: builder-reimage: add OS label validation, MAAS state check, and abort handling X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=e5ef6ade93537e3e6ffda766a75f912c524f106c;p=ceph-build.git builder-reimage: add OS label validation, MAAS state check, and abort handling Signed-off-by: jitendrasahu1803 --- diff --git a/builder-reimage/build/Jenkins_builder-reimage.py b/builder-reimage/build/Jenkins_builder-reimage.py index a0f2a60ac..bc7d627f4 100755 --- a/builder-reimage/build/Jenkins_builder-reimage.py +++ b/builder-reimage/build/Jenkins_builder-reimage.py @@ -87,12 +87,16 @@ async def get_cached_machines(client): # Wait for status # ------------------------------------------------------------------------- async def wait_for_status(client, system_id, expected): - expected = expected.lower() + if isinstance(expected, str): + expected_states = {expected.lower()} + else: + expected_states = {x.lower() for x in expected} + start = time.time() while time.time() - start < STATUS_WAIT_TIMEOUT: m = await client.machines.get(system_id=system_id) - if get_normalized_status(m) == expected: + if get_normalized_status(m) in expected_states: return True await asyncio.sleep(5) @@ -141,9 +145,17 @@ async def release_machine(machine): if get_normalized_status(machine) == "deployed": await machine.release() +# ------------------------------------------------------------------------- +# Commission +# ------------------------------------------------------------------------- +async def commission_machine(machine): + if get_normalized_status(machine) == "new": + await machine.commission() + # ------------------------------------------------------------------------- # Reimage # ------------------------------------------------------------------------- + async def reimage_machine(client, hostname, os_release, log_file): machines = await get_cached_machines(client) m = next((x for x in machines if x.hostname == hostname), None) @@ -155,6 +167,7 @@ async def reimage_machine(client, hostname, os_release, log_file): m = await client.machines.get(system_id=m.system_id) status = get_normalized_status(m) + log(f"[INFO] Current state for {hostname}: {status}", log_file) if status == "deployed": log(f"[ACTION] Releasing {hostname}", log_file) @@ -162,8 +175,41 @@ async def reimage_machine(client, hostname, os_release, log_file): if not await wait_for_status(client, m.system_id, "ready"): return False + if status == "new": + log(f"[ACTION] Commissioning {hostname}", log_file) + await commission_machine(m) + if not await wait_for_status(client, m.system_id, "ready"): + return False + return await deploy_machine(client, hostname, os_release, log_file) +# ------------------------------------------------------------------------- +# Abort Deploy +# ------------------------------------------------------------------------- +async def abort_deploy_machine(client, hostname, log_file): + machines = await get_cached_machines(client) + m = next((x for x in machines if x.hostname == hostname), None) + + if not m: + log(f"[WARN] {hostname} not found", log_file) + return False + + m = await client.machines.get(system_id=m.system_id) + status = get_normalized_status(m) + + if status == "deploying": + log(f"[ACTION] Aborting deployment for {hostname}", log_file) + try: + await m.abort() + log(f"[SUCCESS] Deployment aborted for {hostname}", log_file) + return True + except Exception as e: + log(f"[ERROR] Failed to abort deployment for {hostname}: {e}", log_file) + return False + + log(f"[INFO] No active deployment to abort for {hostname} (state: {status})", log_file) + return True + # ------------------------------------------------------------------------- # Main # ------------------------------------------------------------------------- @@ -180,7 +226,6 @@ async def main(): client = await connect_maas() client._machines_cache = await client.machines.list() - # UPDATED: multi-machine support if args.action == "reimage": if not args.machine: print("Missing --machine") @@ -208,6 +253,43 @@ async def main(): log(f"Success: {success}", log_file) log(f"Failed: {failed}", log_file) + if failed > 0: + sys.exit(1) + + sys.exit(0) + + if args.action == "abort_deploy": + if not args.machine: + print("Missing --machine") + return + + machine_list = parse_machine_list(args.machine, args.os) + + success = 0 + failed = 0 + + for hostname, _ in machine_list: + log(f"\n[ABORT CHECK] {hostname}", log_file) + try: + result = await abort_deploy_machine(client, hostname, log_file) + if result: + success += 1 + else: + failed += 1 + except Exception as e: + log(f"[ERROR] {hostname}: {e}", log_file) + failed += 1 + + log("\n===== ABORT SUMMARY =====", log_file) + log(f"Total: {len(machine_list)}", log_file) + log(f"Success: {success}", log_file) + log(f"Failed: {failed}", log_file) + + if failed > 0: + sys.exit(1) + + sys.exit(0) + # ------------------------------------------------------------------------- if __name__ == "__main__": diff --git a/builder-reimage/build/Jenkinsfile b/builder-reimage/build/Jenkinsfile index 9adfa97d0..d00cde1ef 100644 --- a/builder-reimage/build/Jenkinsfile +++ b/builder-reimage/build/Jenkinsfile @@ -34,169 +34,251 @@ pipeline { branches["Node-${machine}"] = { node('teuthology') { - stage("Checkout-${machine}") { - checkout scm - } - def WORK_DIR = "${env.WORKSPACE}/ci-work-${machine}" def TARGET_FQDN = "${machine}.${env.DOMAIN}" def INVENTORY_FILE = "${WORK_DIR}/repos/secret-repo/ansible/inventory/group_vars/jenkins_builders.yml" + def JENKINS_NODE_STATE_FILE = "${WORK_DIR}/jenkins-node-state.json" + def JENKINS_GIT_KEY_FILE = "${WORK_DIR}/jenkins_git_key" - stage("Prepare-${machine}") { - withCredentials([ - sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY') - ]) { - sh """#!/bin/bash - set -euo pipefail - - mkdir -p "${WORK_DIR}" - - cp "${SSH_KEY}" /tmp/jenkins_git_key - chmod 600 /tmp/jenkins_git_key + try { - bash builder-reimage/build/prepare_env.sh \ - "${TARGET_FQDN}" \ - "${WORK_DIR}" \ - true \ - git@github.com:ceph/ceph-build.git \ - git@github.com:ceph/ceph-cm-ansible.git \ - git@github.com:ceph/ceph-sepia-secrets.git - """ + stage("Checkout-${machine}") { + checkout scm } - } - - stage("Sync Labels-${machine}") { - withCredentials([ - usernamePassword( - credentialsId: 'jenkins-api-token', - usernameVariable: 'JOB_BUILDER_USER', - passwordVariable: 'JOB_BUILDER_PASS' - ) - ]) { - withEnv([ - "SYNC_MACHINE=${machine}", - "SYNC_TARGET_FQDN=${TARGET_FQDN}", - "SYNC_INVENTORY_FILE=${INVENTORY_FILE}" + + stage("Prepare-${machine}") { + withCredentials([ + sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY') ]) { - sh '''#!/bin/bash + sh """#!/bin/bash set -euo pipefail - - echo "[INFO] Syncing Jenkins labels for $SYNC_MACHINE" - echo "JENKINS_URL is: $JENKINS_URL" - - python3 builder-reimage/build/sync_labels.py \ - --file "$SYNC_INVENTORY_FILE" \ - --jenkins_url "$JENKINS_URL" \ - --user "$JOB_BUILDER_USER" \ - --token "$JOB_BUILDER_PASS" \ - --node "$SYNC_TARGET_FQDN" - ''' + + mkdir -p "${WORK_DIR}" + + cp "${SSH_KEY}" "${JENKINS_GIT_KEY_FILE}" + chmod 600 "${JENKINS_GIT_KEY_FILE}" + + bash builder-reimage/build/prepare_env.sh \ + "${TARGET_FQDN}" \ + "${WORK_DIR}" \ + true \ + git@github.com:ceph/ceph-build.git \ + git@github.com:ceph/ceph-cm-ansible.git \ + git@github.com:ceph/ceph-sepia-secrets.git + + # Temporary test helper to use a non-default secrets repo branch. + # Uncomment the lines below only for branch-based inventory validation. + + # export GIT_SSH_COMMAND="ssh -i ${JENKINS_GIT_KEY_FILE} -o StrictHostKeyChecking=no" + # cd "${WORK_DIR}/repos/secret-repo" + # git fetch origin + # git checkout -B FETCH_HEAD + """ } } - } - stage("Detect OS-${machine}") { - def DETECTED_OS = sh( - script: """#!/bin/bash - python3 builder-reimage/build/get_node_os.py \ - "${INVENTORY_FILE}" \ - "${machine}" - """, - returnStdout: true - ).trim() - - echo "Detected OS for ${machine} is ${DETECTED_OS}" - - if (!DETECTED_OS) { - error("OS detection failed for ${machine}") + + stage("Sync Labels-${machine}") { + withCredentials([ + usernamePassword( + credentialsId: 'jenkins-api-token', + usernameVariable: 'JOB_BUILDER_USER', + passwordVariable: 'JOB_BUILDER_PASS' + ) + ]) { + withEnv([ + "SYNC_MACHINE=${machine}", + "SYNC_TARGET_FQDN=${TARGET_FQDN}", + "SYNC_INVENTORY_FILE=${INVENTORY_FILE}" + ]) { + sh '''#!/bin/bash + set -euo pipefail + + echo "[INFO] Syncing Jenkins labels for $SYNC_MACHINE" + echo "JENKINS_URL is: $JENKINS_URL" + + python3 builder-reimage/build/sync_labels.py \ + --file "$SYNC_INVENTORY_FILE" \ + --jenkins_url "$JENKINS_URL" \ + --user "$JOB_BUILDER_USER" \ + --token "$JOB_BUILDER_PASS" \ + --node "$SYNC_TARGET_FQDN" + ''' + } + } } - def parts = DETECTED_OS.split(",") + stage("Detect OS-${machine}") { + def DETECTED_OS = sh( + script: """#!/bin/bash + python3 builder-reimage/build/get_node_os.py \ + "${INVENTORY_FILE}" \ + "${machine}" + """, + returnStdout: true + ).trim() - env.DETECTED_OS = parts[0] - env.LIBVIRT_ENABLED = parts.size() > 1 ? parts[1] : "false" - - echo "Detected OS: ${env.DETECTED_OS}" - echo "Libvirt enabled: ${env.LIBVIRT_ENABLED}" - - } + echo "Detected OS for ${machine} is ${DETECTED_OS}" - stage("Reimage-${machine}") { - if (!params.SKIP_REIMAGE) { - withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) { - sh """#!/bin/bash - set -euo pipefail + if (!DETECTED_OS) { + error("OS detection failed for ${machine}") + } - REPO_DIR="\$WORKSPACE/builder-reimage/build" - VENV_PATH="\$REPO_DIR/.venv" + def parts = DETECTED_OS.split(",") - cd "\$REPO_DIR" + env.DETECTED_OS = parts[0] + env.LIBVIRT_ENABLED = parts.size() > 1 ? parts[1] : "false" + + echo "Detected OS: ${env.DETECTED_OS}" + echo "Libvirt enabled: ${env.LIBVIRT_ENABLED}" - if [ ! -d "\$VENV_PATH" ]; then - echo "Creating virtual environment" - python3 -m venv "\$VENV_PATH" - source "\$VENV_PATH/bin/activate" - pip install --upgrade pip - pip install -r requirements.txt - else - source "\$VENV_PATH/bin/activate" - fi + } - echo "Starting reimage for ${machine} with OS ${env.DETECTED_OS}" + stage("Prepare Jenkins Node-${machine}") { + if (!params.SKIP_REIMAGE) { + withCredentials([ + usernamePassword( + credentialsId: 'jenkins-api-token', + usernameVariable: 'JOB_BUILDER_USER', + passwordVariable: 'JOB_BUILDER_PASS' + ) + ]) { + withEnv([ + "SYNC_MACHINE=${machine}", + "SYNC_TARGET_FQDN=${TARGET_FQDN}", + "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}" + ]) { + sh '''#!/bin/bash + set -euo pipefail + + echo "[INFO] Preparing Jenkins node state for $SYNC_MACHINE" + + python3 builder-reimage/build/jenkins_node_control.py \ + --action prepare_for_reimage \ + --jenkins_url "$JENKINS_URL" \ + --user "$JOB_BUILDER_USER" \ + --token "$JOB_BUILDER_PASS" \ + --node "$SYNC_TARGET_FQDN" \ + --state-file "$SYNC_STATE_FILE" \ + --message "Marked temporary offline for builder reimage activity" + ''' + } + } + } + } - python Jenkins_builder-reimage.py \ - --action reimage \ - --machine ${machine} \ - --os ${env.DETECTED_OS} - """ + stage("Reimage-${machine}") { + if (!params.SKIP_REIMAGE) { + withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) { + sh """#!/bin/bash + set -euo pipefail + + REPO_DIR="\$WORKSPACE/builder-reimage/build" + VENV_PATH="\$REPO_DIR/.venv" + + cd "\$REPO_DIR" + + if [ ! -d "\$VENV_PATH" ]; then + echo "Creating virtual environment" + python3 -m venv "\$VENV_PATH" + source "\$VENV_PATH/bin/activate" + pip install --upgrade pip + pip install -r requirements.txt + else + source "\$VENV_PATH/bin/activate" + fi + + echo "Starting reimage for ${machine} with OS ${env.DETECTED_OS}" + + python Jenkins_builder-reimage.py \ + --action reimage \ + --machine ${machine} \ + --os ${env.DETECTED_OS} + """ + } } } - } - stage("Ansible-${machine}") { - withCredentials([ - string(credentialsId: 'builder-api-token', variable: 'BUILDER_TOKEN'), - string(credentialsId: 'ansible-vault-pass', variable: 'VAULT_PASS'), - sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY') - ]) { + stage("Ansible-${machine}") { + withCredentials([ + string(credentialsId: 'builder-api-token', variable: 'BUILDER_TOKEN'), + string(credentialsId: 'ansible-vault-pass', variable: 'VAULT_PASS'), + sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY') + ]) { - sh """#!/bin/bash - set -euo pipefail + sh """#!/bin/bash + set -euo pipefail - REPO_DIR="\$WORKSPACE/builder-reimage" - BUILD_DIR="\$REPO_DIR/build" - WORK_DIR="${WORK_DIR}" - TARGET_FQDN="${TARGET_FQDN}" + REPO_DIR="\$WORKSPACE/builder-reimage" + BUILD_DIR="\$REPO_DIR/build" + WORK_DIR="${WORK_DIR}" + TARGET_FQDN="${TARGET_FQDN}" - echo "[INFO] Running Ansible for \$TARGET_FQDN" + echo "[INFO] Running Ansible for \$TARGET_FQDN" - mkdir -p "\$WORK_DIR" + mkdir -p "\$WORK_DIR" - cp "${SSH_KEY}" /tmp/jenkins_git_key - chmod 600 /tmp/jenkins_git_key + cp "${SSH_KEY}" "${JENKINS_GIT_KEY_FILE}" + chmod 600 "${JENKINS_GIT_KEY_FILE}" - export GIT_SSH_COMMAND="ssh -i /tmp/jenkins_git_key -o StrictHostKeyChecking=no" + export GIT_SSH_COMMAND="ssh -i ${JENKINS_GIT_KEY_FILE} -o StrictHostKeyChecking=no" - VAULT_FILE="/home/jenkins-build/.vault_pass.txt" - echo "\$VAULT_PASS" > "\$VAULT_FILE" + VAULT_FILE="/home/jenkins-build/.vault_pass.txt" + echo "\$VAULT_PASS" > "\$VAULT_FILE" - cd "\$BUILD_DIR" + cd "\$BUILD_DIR" - bash prepare_env.sh "\$TARGET_FQDN" "\$WORK_DIR" true \ - git@github.com:ceph/ceph-build.git \ - git@github.com:ceph/ceph-cm-ansible.git \ - git@github.com:ceph/ceph-sepia-secrets.git + bash prepare_env.sh "\$TARGET_FQDN" "\$WORK_DIR" true \ + git@github.com:ceph/ceph-build.git \ + git@github.com:ceph/ceph-cm-ansible.git \ + git@github.com:ceph/ceph-sepia-secrets.git - bash ansible_runner.sh \ - "\$TARGET_FQDN" \ - "\$WORK_DIR" \ - "${env.VENV_DIR}" \ - "${env.DETECTED_OS}" \ - "\$BUILDER_TOKEN" \ - "${env.LIBVIRT_ENABLED}" - """ + bash ansible_runner.sh \ + "\$TARGET_FQDN" \ + "\$WORK_DIR" \ + "${env.VENV_DIR}" \ + "${env.DETECTED_OS}" \ + "\$BUILDER_TOKEN" \ + "${env.LIBVIRT_ENABLED}" + """ - archiveArtifacts artifacts: "ci-work-${machine}/ansible-logs/*.log", allowEmptyArchive: true + archiveArtifacts artifacts: "ci-work-${machine}/ansible-logs/*.log", allowEmptyArchive: true + } } + + } finally { + withCredentials([ + usernamePassword( + credentialsId: 'jenkins-api-token', + usernameVariable: 'JOB_BUILDER_USER', + passwordVariable: 'JOB_BUILDER_PASS' + ) + ]) { + withEnv([ + "SYNC_MACHINE=${machine}", + "SYNC_TARGET_FQDN=${TARGET_FQDN}", + "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}" + ]) { + sh '''#!/bin/bash + set +e + + echo "[INFO] Restoring Jenkins node state for $SYNC_MACHINE" + + python3 builder-reimage/build/jenkins_node_control.py \ + --action restore_after_reimage \ + --jenkins_url "$JENKINS_URL" \ + --user "$JOB_BUILDER_USER" \ + --token "$JOB_BUILDER_PASS" \ + --node "$SYNC_TARGET_FQDN" \ + --state-file "$SYNC_STATE_FILE" + + exit 0 + ''' + } + } + + sh """#!/bin/bash + rm -f "${JENKINS_GIT_KEY_FILE}" || true + """ } } } @@ -215,6 +297,45 @@ pipeline { echo 'Build finished' } + aborted { + script { + def machines = params.MACHINE.split(',').collect { it.trim() } + + for (m in machines) { + withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) { + withEnv(["ABORT_MACHINE=${m}"]) { + sh '''#!/bin/bash + set +e + + REPO_DIR="$WORKSPACE/builder-reimage/build" + VENV_PATH="$REPO_DIR/.venv" + + cd "$REPO_DIR" + + if [ ! -d "$VENV_PATH" ]; then + echo "Creating virtual environment for abort cleanup" + python3 -m venv "$VENV_PATH" + source "$VENV_PATH/bin/activate" + pip install --upgrade pip + pip install -r requirements.txt + else + source "$VENV_PATH/bin/activate" + fi + + echo "Aborting MAAS deployment for $ABORT_MACHINE" + + python3 Jenkins_builder-reimage.py \ + --action abort_deploy \ + --machine "$ABORT_MACHINE" + + exit 0 + ''' + } + } + } + } + } + failure { echo 'One or more nodes failed' } diff --git a/builder-reimage/build/get_node_os.py b/builder-reimage/build/get_node_os.py index d594aacc5..ee401bc11 100644 --- a/builder-reimage/build/get_node_os.py +++ b/builder-reimage/build/get_node_os.py @@ -50,5 +50,11 @@ for label in labels: if label == "libvirt": libvirt = "true" +# Validate that an OS label exists +if not os_name: + # Fail early if required installed-os-* label is missing + print(f"[ERROR] No installed-os-* label found for {node_name}") + sys.exit(1) + # Print both values in structured way print(f"{os_name},{libvirt}") diff --git a/builder-reimage/build/jenkins_node_control.py b/builder-reimage/build/jenkins_node_control.py new file mode 100644 index 000000000..d78cc23f7 --- /dev/null +++ b/builder-reimage/build/jenkins_node_control.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 + +""" +Manage Jenkins node offline state for builder reimage. + +Actions: + prepare_for_reimage + Validate node state and mark it temporarily offline if needed. + + restore_after_reimage + Restore node online if it was marked offline by this job. + +Usage: + jenkins_node_control.py --action --jenkins_url --user --token --node --state-file [--message ] +""" + +import argparse +import json +import os +import sys +import time +from urllib.parse import quote + +import requests + + +def get_crumb(session, base_url): + crumb_url = f"{base_url}/crumbIssuer/api/json" + r = session.get(crumb_url) + if r.status_code == 404: + return {} + r.raise_for_status() + data = r.json() + return {data["crumbRequestField"]: data["crumb"]} + + +def normalize_base_url(url): + return url.rstrip("/") + + +def resolve_node_name(session, base_url, short_name): + api_url = f"{base_url}/computer/api/json" + r = session.get(api_url) + r.raise_for_status() + + for node in r.json().get("computer", []): + display_name = node.get("displayName", "") + if display_name == short_name: + return display_name + if display_name.endswith(f"+{short_name}"): + return display_name + if short_name in display_name: + return display_name + + raise RuntimeError(f"Could not resolve Jenkins node name for {short_name}") + + +def get_node_info(session, base_url, node_name): + encoded = quote(node_name, safe="") + api_url = ( + f"{base_url}/computer/{encoded}/api/json" + "?tree=displayName,offline,temporarilyOffline,offlineCauseReason," + "executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]" + ) + r = session.get(api_url) + r.raise_for_status() + return r.json() + + +def node_is_busy(node_info): + for executor in node_info.get("executors", []): + if executor.get("currentExecutable"): + return True + + for executor in node_info.get("oneOffExecutors", []): + if executor.get("currentExecutable"): + return True + + return False + + +def wait_until_idle(session, base_url, node_name, timeout=3600, interval=15): + waited = 0 + + while waited < timeout: + node_info = get_node_info(session, base_url, node_name) + + if not node_is_busy(node_info): + print(f"[INFO] {node_name} is now idle", flush=True) + return True + + print(f"[INFO] {node_name} is currently running a job. Waiting for it to finish before reimage.", flush=True) + time.sleep(interval) + waited += interval + + print(f"[ERROR] Timed out waiting for {node_name} to become idle", flush=True) + return False + + +def mark_offline(session, base_url, node_name, message, headers): + encoded = quote(node_name, safe="") + url = f"{base_url}/computer/{encoded}/toggleOffline" + r = session.post(url, data={"offlineMessage": message}, headers=headers) + r.raise_for_status() + + +def mark_online(session, base_url, node_name, headers): + encoded = quote(node_name, safe="") + url = f"{base_url}/computer/{encoded}/toggleOffline" + r = session.post(url, headers=headers) + r.raise_for_status() + + +def save_state(path, state): + with open(path, "w") as f: + json.dump(state, f) + + +def load_state(path): + if not os.path.exists(path): + return None + with open(path) as f: + return json.load(f) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--action", required=True, choices=["prepare_for_reimage", "restore_after_reimage"]) + parser.add_argument("--jenkins_url", required=True) + parser.add_argument("--user", required=True) + parser.add_argument("--token", required=True) + parser.add_argument("--node", required=True) + parser.add_argument("--state-file", required=True) + parser.add_argument("--message", default="Marked temporary offline for builder reimage activity") + args = parser.parse_args() + + base_url = normalize_base_url(args.jenkins_url) + short_name = args.node.split(".")[0] + + session = requests.Session() + session.auth = (args.user, args.token) + headers = get_crumb(session, base_url) + + if args.action == "prepare_for_reimage": + node_name = resolve_node_name(session, base_url, short_name) + node_info = get_node_info(session, base_url, node_name) + + state = { + "short_name": short_name, + "node_name": node_name, + "was_offline": bool(node_info.get("offline")), + "marked_offline_by_job": False, + } + + if node_info.get("offline"): + print(f"[INFO] {short_name} is already offline in Jenkins", flush=True) + save_state(args.state_file, state) + + if node_is_busy(node_info): + print(f"[INFO] {short_name} is currently running a job. Waiting for it to finish before reimage.", flush=True) + + if not wait_until_idle(session, base_url, node_name): + sys.exit(1) + + sys.exit(0) + + print(f"[ACTION] Marking {short_name} temporarily offline", flush=True) + mark_offline(session, base_url, node_name, args.message, headers) + + state["marked_offline_by_job"] = True + save_state(args.state_file, state) + print(f"[SUCCESS] {short_name} marked temporarily offline", flush=True) + + if node_is_busy(node_info): + print(f"[INFO] {short_name} is currently running a job. Waiting for it to finish before reimage.", flush=True) + + if not wait_until_idle(session, base_url, node_name): + sys.exit(1) + + sys.exit(0) + + if args.action == "restore_after_reimage": + state = load_state(args.state_file) + if not state: + print("[INFO] No Jenkins node state file found. Nothing to restore.") + sys.exit(0) + + if state.get("was_offline"): + print(f"[INFO] {state['short_name']} was already offline before this job. Leaving it offline.") + sys.exit(0) + + if not state.get("marked_offline_by_job"): + print(f"[INFO] {state['short_name']} was not marked offline by this job. Nothing to restore.") + sys.exit(0) + + print(f"[ACTION] Bringing {state['short_name']} back online") + mark_online(session, base_url, state["node_name"], headers) + print(f"[SUCCESS] {state['short_name']} brought back online") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/builder-reimage/config/definitions/builder-reimage.yml b/builder-reimage/config/definitions/builder-reimage.yml index 31aa2cdd9..c07333831 100644 --- a/builder-reimage/config/definitions/builder-reimage.yml +++ b/builder-reimage/config/definitions/builder-reimage.yml @@ -33,7 +33,7 @@ - git: url: https://github.com/ceph/ceph-build.git branches: - - main + - main clean: true pipeline-scm: