# Wait for status
# -------------------------------------------------------------------------
async def wait_for_status(client, system_id, expected):
- expected = expected.lower()
+ if isinstance(expected, str):
+ expected_states = {expected.lower()}
+ else:
+ expected_states = {x.lower() for x in expected}
+
start = time.time()
while time.time() - start < STATUS_WAIT_TIMEOUT:
m = await client.machines.get(system_id=system_id)
- if get_normalized_status(m) == expected:
+ if get_normalized_status(m) in expected_states:
return True
await asyncio.sleep(5)
if get_normalized_status(machine) == "deployed":
await machine.release()
+# -------------------------------------------------------------------------
+# Commission
+# -------------------------------------------------------------------------
+async def commission_machine(machine):
+ if get_normalized_status(machine) == "new":
+ await machine.commission()
+
# -------------------------------------------------------------------------
# Reimage
# -------------------------------------------------------------------------
+
async def reimage_machine(client, hostname, os_release, log_file):
machines = await get_cached_machines(client)
m = next((x for x in machines if x.hostname == hostname), None)
m = await client.machines.get(system_id=m.system_id)
status = get_normalized_status(m)
+ log(f"[INFO] Current state for {hostname}: {status}", log_file)
if status == "deployed":
log(f"[ACTION] Releasing {hostname}", log_file)
if not await wait_for_status(client, m.system_id, "ready"):
return False
+ if status == "new":
+ log(f"[ACTION] Commissioning {hostname}", log_file)
+ await commission_machine(m)
+ if not await wait_for_status(client, m.system_id, "ready"):
+ return False
+
return await deploy_machine(client, hostname, os_release, log_file)
+# -------------------------------------------------------------------------
+# Abort Deploy
+# -------------------------------------------------------------------------
+async def abort_deploy_machine(client, hostname, log_file):
+ machines = await get_cached_machines(client)
+ m = next((x for x in machines if x.hostname == hostname), None)
+
+ if not m:
+ log(f"[WARN] {hostname} not found", log_file)
+ return False
+
+ m = await client.machines.get(system_id=m.system_id)
+ status = get_normalized_status(m)
+
+ if status == "deploying":
+ log(f"[ACTION] Aborting deployment for {hostname}", log_file)
+ try:
+ await m.abort()
+ log(f"[SUCCESS] Deployment aborted for {hostname}", log_file)
+ return True
+ except Exception as e:
+ log(f"[ERROR] Failed to abort deployment for {hostname}: {e}", log_file)
+ return False
+
+ log(f"[INFO] No active deployment to abort for {hostname} (state: {status})", log_file)
+ return True
+
# -------------------------------------------------------------------------
# Main
# -------------------------------------------------------------------------
client = await connect_maas()
client._machines_cache = await client.machines.list()
- # UPDATED: multi-machine support
if args.action == "reimage":
if not args.machine:
print("Missing --machine")
log(f"Success: {success}", log_file)
log(f"Failed: {failed}", log_file)
+ if failed > 0:
+ sys.exit(1)
+
+ sys.exit(0)
+
+ if args.action == "abort_deploy":
+ if not args.machine:
+ print("Missing --machine")
+ return
+
+ machine_list = parse_machine_list(args.machine, args.os)
+
+ success = 0
+ failed = 0
+
+ for hostname, _ in machine_list:
+ log(f"\n[ABORT CHECK] {hostname}", log_file)
+ try:
+ result = await abort_deploy_machine(client, hostname, log_file)
+ if result:
+ success += 1
+ else:
+ failed += 1
+ except Exception as e:
+ log(f"[ERROR] {hostname}: {e}", log_file)
+ failed += 1
+
+ log("\n===== ABORT SUMMARY =====", log_file)
+ log(f"Total: {len(machine_list)}", log_file)
+ log(f"Success: {success}", log_file)
+ log(f"Failed: {failed}", log_file)
+
+ if failed > 0:
+ sys.exit(1)
+
+ sys.exit(0)
+
# -------------------------------------------------------------------------
if __name__ == "__main__":
branches["Node-${machine}"] = {
node('teuthology') {
- stage("Checkout-${machine}") {
- checkout scm
- }
-
def WORK_DIR = "${env.WORKSPACE}/ci-work-${machine}"
def TARGET_FQDN = "${machine}.${env.DOMAIN}"
def INVENTORY_FILE = "${WORK_DIR}/repos/secret-repo/ansible/inventory/group_vars/jenkins_builders.yml"
+ def JENKINS_NODE_STATE_FILE = "${WORK_DIR}/jenkins-node-state.json"
+ def JENKINS_GIT_KEY_FILE = "${WORK_DIR}/jenkins_git_key"
- stage("Prepare-${machine}") {
- withCredentials([
- sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY')
- ]) {
- sh """#!/bin/bash
- set -euo pipefail
-
- mkdir -p "${WORK_DIR}"
-
- cp "${SSH_KEY}" /tmp/jenkins_git_key
- chmod 600 /tmp/jenkins_git_key
+ try {
- bash builder-reimage/build/prepare_env.sh \
- "${TARGET_FQDN}" \
- "${WORK_DIR}" \
- true \
- git@github.com:ceph/ceph-build.git \
- git@github.com:ceph/ceph-cm-ansible.git \
- git@github.com:ceph/ceph-sepia-secrets.git
- """
+ stage("Checkout-${machine}") {
+ checkout scm
}
- }
-
- stage("Sync Labels-${machine}") {
- withCredentials([
- usernamePassword(
- credentialsId: 'jenkins-api-token',
- usernameVariable: 'JOB_BUILDER_USER',
- passwordVariable: 'JOB_BUILDER_PASS'
- )
- ]) {
- withEnv([
- "SYNC_MACHINE=${machine}",
- "SYNC_TARGET_FQDN=${TARGET_FQDN}",
- "SYNC_INVENTORY_FILE=${INVENTORY_FILE}"
+
+ stage("Prepare-${machine}") {
+ withCredentials([
+ sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY')
]) {
- sh '''#!/bin/bash
+ sh """#!/bin/bash
set -euo pipefail
-
- echo "[INFO] Syncing Jenkins labels for $SYNC_MACHINE"
- echo "JENKINS_URL is: $JENKINS_URL"
-
- python3 builder-reimage/build/sync_labels.py \
- --file "$SYNC_INVENTORY_FILE" \
- --jenkins_url "$JENKINS_URL" \
- --user "$JOB_BUILDER_USER" \
- --token "$JOB_BUILDER_PASS" \
- --node "$SYNC_TARGET_FQDN"
- '''
+
+ mkdir -p "${WORK_DIR}"
+
+ cp "${SSH_KEY}" "${JENKINS_GIT_KEY_FILE}"
+ chmod 600 "${JENKINS_GIT_KEY_FILE}"
+
+ bash builder-reimage/build/prepare_env.sh \
+ "${TARGET_FQDN}" \
+ "${WORK_DIR}" \
+ true \
+ git@github.com:ceph/ceph-build.git \
+ git@github.com:ceph/ceph-cm-ansible.git \
+ git@github.com:ceph/ceph-sepia-secrets.git
+
+ # Temporary test helper to use a non-default secrets repo branch.
+ # Uncomment the lines below only for branch-based inventory validation.
+
+ # export GIT_SSH_COMMAND="ssh -i ${JENKINS_GIT_KEY_FILE} -o StrictHostKeyChecking=no"
+ # cd "${WORK_DIR}/repos/secret-repo"
+ # git fetch origin <local_branch_name>
+ # git checkout -B <local_branch_name> FETCH_HEAD
+ """
}
}
- }
- stage("Detect OS-${machine}") {
- def DETECTED_OS = sh(
- script: """#!/bin/bash
- python3 builder-reimage/build/get_node_os.py \
- "${INVENTORY_FILE}" \
- "${machine}"
- """,
- returnStdout: true
- ).trim()
-
- echo "Detected OS for ${machine} is ${DETECTED_OS}"
-
- if (!DETECTED_OS) {
- error("OS detection failed for ${machine}")
+
+ stage("Sync Labels-${machine}") {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'jenkins-api-token',
+ usernameVariable: 'JOB_BUILDER_USER',
+ passwordVariable: 'JOB_BUILDER_PASS'
+ )
+ ]) {
+ withEnv([
+ "SYNC_MACHINE=${machine}",
+ "SYNC_TARGET_FQDN=${TARGET_FQDN}",
+ "SYNC_INVENTORY_FILE=${INVENTORY_FILE}"
+ ]) {
+ sh '''#!/bin/bash
+ set -euo pipefail
+
+ echo "[INFO] Syncing Jenkins labels for $SYNC_MACHINE"
+ echo "JENKINS_URL is: $JENKINS_URL"
+
+ python3 builder-reimage/build/sync_labels.py \
+ --file "$SYNC_INVENTORY_FILE" \
+ --jenkins_url "$JENKINS_URL" \
+ --user "$JOB_BUILDER_USER" \
+ --token "$JOB_BUILDER_PASS" \
+ --node "$SYNC_TARGET_FQDN"
+ '''
+ }
+ }
}
- def parts = DETECTED_OS.split(",")
+ stage("Detect OS-${machine}") {
+ def DETECTED_OS = sh(
+ script: """#!/bin/bash
+ python3 builder-reimage/build/get_node_os.py \
+ "${INVENTORY_FILE}" \
+ "${machine}"
+ """,
+ returnStdout: true
+ ).trim()
- env.DETECTED_OS = parts[0]
- env.LIBVIRT_ENABLED = parts.size() > 1 ? parts[1] : "false"
-
- echo "Detected OS: ${env.DETECTED_OS}"
- echo "Libvirt enabled: ${env.LIBVIRT_ENABLED}"
-
- }
+ echo "Detected OS for ${machine} is ${DETECTED_OS}"
- stage("Reimage-${machine}") {
- if (!params.SKIP_REIMAGE) {
- withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) {
- sh """#!/bin/bash
- set -euo pipefail
+ if (!DETECTED_OS) {
+ error("OS detection failed for ${machine}")
+ }
- REPO_DIR="\$WORKSPACE/builder-reimage/build"
- VENV_PATH="\$REPO_DIR/.venv"
+ def parts = DETECTED_OS.split(",")
- cd "\$REPO_DIR"
+ env.DETECTED_OS = parts[0]
+ env.LIBVIRT_ENABLED = parts.size() > 1 ? parts[1] : "false"
+
+ echo "Detected OS: ${env.DETECTED_OS}"
+ echo "Libvirt enabled: ${env.LIBVIRT_ENABLED}"
- if [ ! -d "\$VENV_PATH" ]; then
- echo "Creating virtual environment"
- python3 -m venv "\$VENV_PATH"
- source "\$VENV_PATH/bin/activate"
- pip install --upgrade pip
- pip install -r requirements.txt
- else
- source "\$VENV_PATH/bin/activate"
- fi
+ }
- echo "Starting reimage for ${machine} with OS ${env.DETECTED_OS}"
+ stage("Prepare Jenkins Node-${machine}") {
+ if (!params.SKIP_REIMAGE) {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'jenkins-api-token',
+ usernameVariable: 'JOB_BUILDER_USER',
+ passwordVariable: 'JOB_BUILDER_PASS'
+ )
+ ]) {
+ withEnv([
+ "SYNC_MACHINE=${machine}",
+ "SYNC_TARGET_FQDN=${TARGET_FQDN}",
+ "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}"
+ ]) {
+ sh '''#!/bin/bash
+ set -euo pipefail
+
+ echo "[INFO] Preparing Jenkins node state for $SYNC_MACHINE"
+
+ python3 builder-reimage/build/jenkins_node_control.py \
+ --action prepare_for_reimage \
+ --jenkins_url "$JENKINS_URL" \
+ --user "$JOB_BUILDER_USER" \
+ --token "$JOB_BUILDER_PASS" \
+ --node "$SYNC_TARGET_FQDN" \
+ --state-file "$SYNC_STATE_FILE" \
+ --message "Marked temporary offline for builder reimage activity"
+ '''
+ }
+ }
+ }
+ }
- python Jenkins_builder-reimage.py \
- --action reimage \
- --machine ${machine} \
- --os ${env.DETECTED_OS}
- """
+ stage("Reimage-${machine}") {
+ if (!params.SKIP_REIMAGE) {
+ withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) {
+ sh """#!/bin/bash
+ set -euo pipefail
+
+ REPO_DIR="\$WORKSPACE/builder-reimage/build"
+ VENV_PATH="\$REPO_DIR/.venv"
+
+ cd "\$REPO_DIR"
+
+ if [ ! -d "\$VENV_PATH" ]; then
+ echo "Creating virtual environment"
+ python3 -m venv "\$VENV_PATH"
+ source "\$VENV_PATH/bin/activate"
+ pip install --upgrade pip
+ pip install -r requirements.txt
+ else
+ source "\$VENV_PATH/bin/activate"
+ fi
+
+ echo "Starting reimage for ${machine} with OS ${env.DETECTED_OS}"
+
+ python Jenkins_builder-reimage.py \
+ --action reimage \
+ --machine ${machine} \
+ --os ${env.DETECTED_OS}
+ """
+ }
}
}
- }
- stage("Ansible-${machine}") {
- withCredentials([
- string(credentialsId: 'builder-api-token', variable: 'BUILDER_TOKEN'),
- string(credentialsId: 'ansible-vault-pass', variable: 'VAULT_PASS'),
- sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY')
- ]) {
+ stage("Ansible-${machine}") {
+ withCredentials([
+ string(credentialsId: 'builder-api-token', variable: 'BUILDER_TOKEN'),
+ string(credentialsId: 'ansible-vault-pass', variable: 'VAULT_PASS'),
+ sshUserPrivateKey(credentialsId: 'ansible-ssh-key', keyFileVariable: 'SSH_KEY')
+ ]) {
- sh """#!/bin/bash
- set -euo pipefail
+ sh """#!/bin/bash
+ set -euo pipefail
- REPO_DIR="\$WORKSPACE/builder-reimage"
- BUILD_DIR="\$REPO_DIR/build"
- WORK_DIR="${WORK_DIR}"
- TARGET_FQDN="${TARGET_FQDN}"
+ REPO_DIR="\$WORKSPACE/builder-reimage"
+ BUILD_DIR="\$REPO_DIR/build"
+ WORK_DIR="${WORK_DIR}"
+ TARGET_FQDN="${TARGET_FQDN}"
- echo "[INFO] Running Ansible for \$TARGET_FQDN"
+ echo "[INFO] Running Ansible for \$TARGET_FQDN"
- mkdir -p "\$WORK_DIR"
+ mkdir -p "\$WORK_DIR"
- cp "${SSH_KEY}" /tmp/jenkins_git_key
- chmod 600 /tmp/jenkins_git_key
+ cp "${SSH_KEY}" "${JENKINS_GIT_KEY_FILE}"
+ chmod 600 "${JENKINS_GIT_KEY_FILE}"
- export GIT_SSH_COMMAND="ssh -i /tmp/jenkins_git_key -o StrictHostKeyChecking=no"
+ export GIT_SSH_COMMAND="ssh -i ${JENKINS_GIT_KEY_FILE} -o StrictHostKeyChecking=no"
- VAULT_FILE="/home/jenkins-build/.vault_pass.txt"
- echo "\$VAULT_PASS" > "\$VAULT_FILE"
+ VAULT_FILE="/home/jenkins-build/.vault_pass.txt"
+ echo "\$VAULT_PASS" > "\$VAULT_FILE"
- cd "\$BUILD_DIR"
+ cd "\$BUILD_DIR"
- bash prepare_env.sh "\$TARGET_FQDN" "\$WORK_DIR" true \
- git@github.com:ceph/ceph-build.git \
- git@github.com:ceph/ceph-cm-ansible.git \
- git@github.com:ceph/ceph-sepia-secrets.git
+ bash prepare_env.sh "\$TARGET_FQDN" "\$WORK_DIR" true \
+ git@github.com:ceph/ceph-build.git \
+ git@github.com:ceph/ceph-cm-ansible.git \
+ git@github.com:ceph/ceph-sepia-secrets.git
- bash ansible_runner.sh \
- "\$TARGET_FQDN" \
- "\$WORK_DIR" \
- "${env.VENV_DIR}" \
- "${env.DETECTED_OS}" \
- "\$BUILDER_TOKEN" \
- "${env.LIBVIRT_ENABLED}"
- """
+ bash ansible_runner.sh \
+ "\$TARGET_FQDN" \
+ "\$WORK_DIR" \
+ "${env.VENV_DIR}" \
+ "${env.DETECTED_OS}" \
+ "\$BUILDER_TOKEN" \
+ "${env.LIBVIRT_ENABLED}"
+ """
- archiveArtifacts artifacts: "ci-work-${machine}/ansible-logs/*.log", allowEmptyArchive: true
+ archiveArtifacts artifacts: "ci-work-${machine}/ansible-logs/*.log", allowEmptyArchive: true
+ }
}
+
+ } finally {
+ withCredentials([
+ usernamePassword(
+ credentialsId: 'jenkins-api-token',
+ usernameVariable: 'JOB_BUILDER_USER',
+ passwordVariable: 'JOB_BUILDER_PASS'
+ )
+ ]) {
+ withEnv([
+ "SYNC_MACHINE=${machine}",
+ "SYNC_TARGET_FQDN=${TARGET_FQDN}",
+ "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}"
+ ]) {
+ sh '''#!/bin/bash
+ set +e
+
+ echo "[INFO] Restoring Jenkins node state for $SYNC_MACHINE"
+
+ python3 builder-reimage/build/jenkins_node_control.py \
+ --action restore_after_reimage \
+ --jenkins_url "$JENKINS_URL" \
+ --user "$JOB_BUILDER_USER" \
+ --token "$JOB_BUILDER_PASS" \
+ --node "$SYNC_TARGET_FQDN" \
+ --state-file "$SYNC_STATE_FILE"
+
+ exit 0
+ '''
+ }
+ }
+
+ sh """#!/bin/bash
+ rm -f "${JENKINS_GIT_KEY_FILE}" || true
+ """
}
}
}
echo 'Build finished'
}
+ aborted {
+ script {
+ def machines = params.MACHINE.split(',').collect { it.trim() }
+
+ for (m in machines) {
+ withCredentials([string(credentialsId: 'maas-api-key', variable: 'MAAS_API_KEY')]) {
+ withEnv(["ABORT_MACHINE=${m}"]) {
+ sh '''#!/bin/bash
+ set +e
+
+ REPO_DIR="$WORKSPACE/builder-reimage/build"
+ VENV_PATH="$REPO_DIR/.venv"
+
+ cd "$REPO_DIR"
+
+ if [ ! -d "$VENV_PATH" ]; then
+ echo "Creating virtual environment for abort cleanup"
+ python3 -m venv "$VENV_PATH"
+ source "$VENV_PATH/bin/activate"
+ pip install --upgrade pip
+ pip install -r requirements.txt
+ else
+ source "$VENV_PATH/bin/activate"
+ fi
+
+ echo "Aborting MAAS deployment for $ABORT_MACHINE"
+
+ python3 Jenkins_builder-reimage.py \
+ --action abort_deploy \
+ --machine "$ABORT_MACHINE"
+
+ exit 0
+ '''
+ }
+ }
+ }
+ }
+ }
+
failure {
echo 'One or more nodes failed'
}
if label == "libvirt":
libvirt = "true"
+# Validate that an OS label exists
+if not os_name:
+ # Fail early if required installed-os-* label is missing
+ print(f"[ERROR] No installed-os-* label found for {node_name}")
+ sys.exit(1)
+
# Print both values in structured way
print(f"{os_name},{libvirt}")
--- /dev/null
+#!/usr/bin/env python3
+
+"""
+Manage Jenkins node offline state for builder reimage.
+
+Actions:
+ prepare_for_reimage
+ Validate node state and mark it temporarily offline if needed.
+
+ restore_after_reimage
+ Restore node online if it was marked offline by this job.
+
+Usage:
+ jenkins_node_control.py --action <action> --jenkins_url <url> --user <user> --token <token> --node <node> --state-file <file> [--message <text>]
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+from urllib.parse import quote
+
+import requests
+
+
+def get_crumb(session, base_url):
+ crumb_url = f"{base_url}/crumbIssuer/api/json"
+ r = session.get(crumb_url)
+ if r.status_code == 404:
+ return {}
+ r.raise_for_status()
+ data = r.json()
+ return {data["crumbRequestField"]: data["crumb"]}
+
+
+def normalize_base_url(url):
+ return url.rstrip("/")
+
+
+def resolve_node_name(session, base_url, short_name):
+ api_url = f"{base_url}/computer/api/json"
+ r = session.get(api_url)
+ r.raise_for_status()
+
+ for node in r.json().get("computer", []):
+ display_name = node.get("displayName", "")
+ if display_name == short_name:
+ return display_name
+ if display_name.endswith(f"+{short_name}"):
+ return display_name
+ if short_name in display_name:
+ return display_name
+
+ raise RuntimeError(f"Could not resolve Jenkins node name for {short_name}")
+
+
+def get_node_info(session, base_url, node_name):
+ encoded = quote(node_name, safe="")
+ api_url = (
+ f"{base_url}/computer/{encoded}/api/json"
+ "?tree=displayName,offline,temporarilyOffline,offlineCauseReason,"
+ "executors[currentExecutable[url]],oneOffExecutors[currentExecutable[url]]"
+ )
+ r = session.get(api_url)
+ r.raise_for_status()
+ return r.json()
+
+
+def node_is_busy(node_info):
+ for executor in node_info.get("executors", []):
+ if executor.get("currentExecutable"):
+ return True
+
+ for executor in node_info.get("oneOffExecutors", []):
+ if executor.get("currentExecutable"):
+ return True
+
+ return False
+
+
+def wait_until_idle(session, base_url, node_name, timeout=3600, interval=15):
+ waited = 0
+
+ while waited < timeout:
+ node_info = get_node_info(session, base_url, node_name)
+
+ if not node_is_busy(node_info):
+ print(f"[INFO] {node_name} is now idle", flush=True)
+ return True
+
+ print(f"[INFO] {node_name} is currently running a job. Waiting for it to finish before reimage.", flush=True)
+ time.sleep(interval)
+ waited += interval
+
+ print(f"[ERROR] Timed out waiting for {node_name} to become idle", flush=True)
+ return False
+
+
+def mark_offline(session, base_url, node_name, message, headers):
+ encoded = quote(node_name, safe="")
+ url = f"{base_url}/computer/{encoded}/toggleOffline"
+ r = session.post(url, data={"offlineMessage": message}, headers=headers)
+ r.raise_for_status()
+
+
+def mark_online(session, base_url, node_name, headers):
+ encoded = quote(node_name, safe="")
+ url = f"{base_url}/computer/{encoded}/toggleOffline"
+ r = session.post(url, headers=headers)
+ r.raise_for_status()
+
+
+def save_state(path, state):
+ with open(path, "w") as f:
+ json.dump(state, f)
+
+
+def load_state(path):
+ if not os.path.exists(path):
+ return None
+ with open(path) as f:
+ return json.load(f)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--action", required=True, choices=["prepare_for_reimage", "restore_after_reimage"])
+ parser.add_argument("--jenkins_url", required=True)
+ parser.add_argument("--user", required=True)
+ parser.add_argument("--token", required=True)
+ parser.add_argument("--node", required=True)
+ parser.add_argument("--state-file", required=True)
+ parser.add_argument("--message", default="Marked temporary offline for builder reimage activity")
+ args = parser.parse_args()
+
+ base_url = normalize_base_url(args.jenkins_url)
+ short_name = args.node.split(".")[0]
+
+ session = requests.Session()
+ session.auth = (args.user, args.token)
+ headers = get_crumb(session, base_url)
+
+ if args.action == "prepare_for_reimage":
+ node_name = resolve_node_name(session, base_url, short_name)
+ node_info = get_node_info(session, base_url, node_name)
+
+ state = {
+ "short_name": short_name,
+ "node_name": node_name,
+ "was_offline": bool(node_info.get("offline")),
+ "marked_offline_by_job": False,
+ }
+
+ if node_info.get("offline"):
+ print(f"[INFO] {short_name} is already offline in Jenkins", flush=True)
+ save_state(args.state_file, state)
+
+ if node_is_busy(node_info):
+ print(f"[INFO] {short_name} is currently running a job. Waiting for it to finish before reimage.", flush=True)
+
+ if not wait_until_idle(session, base_url, node_name):
+ sys.exit(1)
+
+ sys.exit(0)
+
+ print(f"[ACTION] Marking {short_name} temporarily offline", flush=True)
+ mark_offline(session, base_url, node_name, args.message, headers)
+
+ state["marked_offline_by_job"] = True
+ save_state(args.state_file, state)
+ print(f"[SUCCESS] {short_name} marked temporarily offline", flush=True)
+
+ if node_is_busy(node_info):
+ print(f"[INFO] {short_name} is currently running a job. Waiting for it to finish before reimage.", flush=True)
+
+ if not wait_until_idle(session, base_url, node_name):
+ sys.exit(1)
+
+ sys.exit(0)
+
+ if args.action == "restore_after_reimage":
+ state = load_state(args.state_file)
+ if not state:
+ print("[INFO] No Jenkins node state file found. Nothing to restore.")
+ sys.exit(0)
+
+ if state.get("was_offline"):
+ print(f"[INFO] {state['short_name']} was already offline before this job. Leaving it offline.")
+ sys.exit(0)
+
+ if not state.get("marked_offline_by_job"):
+ print(f"[INFO] {state['short_name']} was not marked offline by this job. Nothing to restore.")
+ sys.exit(0)
+
+ print(f"[ACTION] Bringing {state['short_name']} back online")
+ mark_online(session, base_url, state["node_name"], headers)
+ print(f"[SUCCESS] {state['short_name']} brought back online")
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()
- git:
url: https://github.com/ceph/ceph-build.git
branches:
- - main
+ - main
clean: true
pipeline-scm:
projects / ceph-build.git / commitdiff