From: jitendrasahu1803 <jitendra.sahu1803@gmail.com>
Date: Fri, 10 Jul 2026 20:54:19 +0000 (-0400)
Subject: builder-reimage: improve Ansible SSH failure handling
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fe3a8f884aa6116c05f010c765da5b0578ce070e;p=ceph-build.git

builder-reimage: improve Ansible SSH failure handling
---

diff --git a/builder-reimage/build/Jenkinsfile b/builder-reimage/build/Jenkinsfile
index 0bc63bbc2..59b877d0e 100644
--- a/builder-reimage/build/Jenkinsfile
+++ b/builder-reimage/build/Jenkinsfile
@@ -63,6 +63,8 @@ pipeline {
                                                 cp "\$SSH_KEY" "${JENKINS_GIT_KEY_FILE}"
                                                 chmod 600 "${JENKINS_GIT_KEY_FILE}"
 
+                                                # Set GIT_SSH_COMMAND to use the per-node Jenkins git key during prepare_env    
+
                                                 export GIT_SSH_COMMAND="ssh -i ${JENKINS_GIT_KEY_FILE} -o StrictHostKeyChecking=no"
 
                                                 bash builder-reimage/build/prepare_env.sh \
@@ -262,14 +264,26 @@ pipeline {
                                             passwordVariable: 'JOB_BUILDER_PASS'
                                         )
                                     ]) {
+
+                                        // If Ansible SSH/connectivity failed after reimage, do not restore
+                                        // the Jenkins node state. The node may be unreachable or only
+                                        // partially configured, so keep it offline for manual investigation.
+
                                         withEnv([
                                             "SYNC_MACHINE=${machine}",
                                             "SYNC_TARGET_FQDN=${TARGET_FQDN}",
-                                            "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}"
+                                            "SYNC_STATE_FILE=${JENKINS_NODE_STATE_FILE}",
+                                            "SYNC_CONNECTIVITY_FAILED_FILE=${WORK_DIR}/ansible-connectivity-failed"
                                         ]) {
                                             sh '''#!/bin/bash
                                                 set +e
 
+                                                if [ -f "$SYNC_CONNECTIVITY_FAILED_FILE" ]; then
+                                                    echo "[WARN] Skipping Jenkins node restore for $SYNC_MACHINE because Ansible connectivity failed after reimage"
+                                                    echo "[WARN] Keeping node offline for manual investigation"
+                                                    exit 0
+                                                fi
+
                                                 echo "[INFO] Restoring Jenkins node state for $SYNC_MACHINE"
 
                                                 python3 builder-reimage/build/jenkins_node_control.py \
diff --git a/builder-reimage/build/ansible_runner.sh b/builder-reimage/build/ansible_runner.sh
index 32705093c..6f118cfd6 100644
--- a/builder-reimage/build/ansible_runner.sh
+++ b/builder-reimage/build/ansible_runner.sh
@@ -73,6 +73,11 @@ VAULT_ARG="--vault-password-file=${VAULT_FILE}"
 
 mkdir -p "${LOG_DIR}"
 
+CONNECTIVITY_FAILED_FILE="${WORK_DIR}/ansible-connectivity-failed"
+CONNECTIVITY_LOG="${LOG_DIR}/${TARGET_FQDN}-wait-online.log"
+
+rm -f "${CONNECTIVITY_FAILED_FILE}"
+
 # Activate virtualenv if present
 if [[ -f "${WORK_DIR}/${VENV_DIR}/bin/activate" ]]; then
     echo "[ansible_runner] Activating venv: ${WORK_DIR}/${VENV_DIR}"
@@ -93,6 +98,50 @@ echo "[ansible_runner] SSH user selected = ${SSH_USER}"
 echo "[ansible_runner] Using inventory = ${INVENTORY_PATH}"
 echo "[ansible_runner] Using secrets = ${SECRETS_PATH}"
 
+# Wait until the reimaged node is reachable by Ansible.
+# MAAS may mark the node as deployed before SSH/cloud-init is fully ready.
+# This prevents playbooks from starting too early and failing with connection errors.
+wait_for_ansible_connectivity() {
+    local attempts=0
+    local max_attempts=40
+    local interval=15
+
+    echo "[ansible_runner] Waiting for SSH/Ansible connectivity on ${TARGET_FQDN}"
+
+    until [[ $attempts -ge $max_attempts ]]; do
+        attempts=$((attempts + 1))
+
+        # Use Ansible ping to confirm SSH access and remote Python readiness.
+        ANSIBLE_HOST_KEY_CHECKING=False ansible all \
+            -i "${INVENTORY_PATH}" \
+            --limit "${TARGET_FQDN}" \
+            -m ping \
+            -e "ansible_ssh_user=${SSH_USER}" \
+            > "${CONNECTIVITY_LOG}" 2>&1
+
+        rc=$?
+
+        if [[ $rc -eq 0 ]]; then
+            echo "[ansible_runner] ${TARGET_FQDN} is reachable by Ansible"
+            return 0
+        fi
+
+        # Node is deployed but not ready yet, retry after a short wait.
+        echo "[ansible_runner] ${TARGET_FQDN} not ready yet, attempt ${attempts}/${max_attempts}"
+        tail -n 10 "${CONNECTIVITY_LOG}" || true
+
+        sleep "${interval}"
+    done
+
+    # Fail clearly if the node never becomes reachable after deployment.
+    echo "[ansible_runner] ERROR: ${TARGET_FQDN} did not become reachable by Ansible"
+    echo "[ansible_runner] Last connectivity check output:"
+    cat "${CONNECTIVITY_LOG}" || true
+
+    touch "${CONNECTIVITY_FAILED_FILE}"
+    return 1
+}
+
 # Admin user JSON builder
 ADMIN_USERS=(
   "akraitma"
@@ -148,6 +197,12 @@ run_playbook() {
     done
 }
 
+# Ensure the node is actually reachable before running post-reimage playbooks.
+if ! wait_for_ansible_connectivity; then
+    FAILED_PLAYBOOKS+=("precheck-ansible-connectivity")
+    FAILED_LOGS+=("${CONNECTIVITY_LOG}")
+else
+
 ##############################################
 # PLAYBOOK 1 — ansible_managed.yml
 ##############################################
@@ -267,6 +322,8 @@ if ! (
     FAILED_LOGS+=("${LOG_DIR}/${TARGET_FQDN}-play6-main-builder.log")
 fi
 
+fi
+
 echo "========================================"
 echo "[ansible_runner] FINAL EXECUTION SUMMARY"
 echo "========================================"