windows: increase log collection timeout

Some failed Windows jobs are missing the build artifacts and it
seems like the log collection function times out.

https://jenkins.ceph.com/job/ceph-windows-pull-requests/39380/

We'll increase the timeout from 30s to 10m.

While at it, we're updating the scp_download and scp_upload helpers,
checking for timeout errors.

Signed-off-by: Lucian Petrut <lpetrut@cloudbasesolutions.com>

diff --git a/ceph-windows-installer-build/build/build b/ceph-windows-installer-build/build/build
index b8ddd1b17e213a84e4f122179751274b06b9d950..55d9fd28df4ecfd51845852dec6f45d2b2e958c3 100644 (file)
--- a/ceph-windows-installer-build/build/build
+++ b/ceph-windows-installer-build/build/build
@@ -33,7 +33,7 @@ popd
 #
 # Upload ceph-windows-installer repo to the Windows VM
 #
-SSH_TIMEOUT=5m scp_upload $WORKSPACE/ceph-windows-installer /workspace/ceph-windows-installer
+scp_upload $WORKSPACE/ceph-windows-installer /workspace/ceph-windows-installer
 
 #
 # Build the Visual Studio project

diff --git a/scripts/build_utils.sh b/scripts/build_utils.sh
index 63eddd603137b5a9d7a4ae14995547413a649a87..37e10333024a3a65b23df2e4b6c95f14980b4b84 100755 (executable)
--- a/scripts/build_utils.sh
+++ b/scripts/build_utils.sh
@@ -1747,7 +1747,16 @@ function scp_upload() {
     if [[ ! -z $SSH_KNOWN_HOSTS_FILE ]]; then
         SSH_OPTS="$SSH_OPTS -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE"
     fi
-    timeout ${SSH_TIMEOUT:-"30s"} scp -i ${SSH_KEY:-"$HOME/.ssh/id_rsa"} $SSH_OPTS -r $LOCAL_FILE ${SSH_USER}@${SSH_ADDRESS}:${REMOTE_FILE}
+    timeout ${SSH_TIMEOUT:-"10m"} scp -i ${SSH_KEY:-"$HOME/.ssh/id_rsa"} $SSH_OPTS -r $LOCAL_FILE ${SSH_USER}@${SSH_ADDRESS}:${REMOTE_FILE} || {
+        EXIT_CODE=$?
+        # By default, the "timeout" CLI tool always exits with 124 when the
+        # timeout is exceeded. Unless "--preserve-status" argument is used, the
+        # exit code is never set to the exit code of the command that timed out.
+        if [[ $EXIT_CODE -eq 124 ]]; then
+            echo "ERROR: scp upload timed out"
+        fi
+        return $EXIT_CODE
+    }
 }
 
 function scp_download() {
@@ -1765,7 +1774,16 @@ function scp_download() {
     if [[ ! -z $SSH_KNOWN_HOSTS_FILE ]]; then
         SSH_OPTS="$SSH_OPTS -o UserKnownHostsFile=$SSH_KNOWN_HOSTS_FILE"
     fi
-    timeout ${SSH_TIMEOUT:-"30s"} scp -i ${SSH_KEY:-"$HOME/.ssh/id_rsa"} $SSH_OPTS -r ${SSH_USER}@${SSH_ADDRESS}:${REMOTE_FILE} $LOCAL_FILE
+    timeout ${SSH_TIMEOUT:-"10m"} scp -i ${SSH_KEY:-"$HOME/.ssh/id_rsa"} $SSH_OPTS -r ${SSH_USER}@${SSH_ADDRESS}:${REMOTE_FILE} $LOCAL_FILE || {
+        EXIT_CODE=$?
+        # By default, the "timeout" CLI tool always exits with 124 when the
+        # timeout is exceeded. Unless "--preserve-status" argument is used, the
+        # exit code is never set to the exit code of the command that timed out.
+        if [[ $EXIT_CODE -eq 124 ]]; then
+            echo "ERROR: scp download timed out"
+        fi
+        return $EXIT_CODE
+    }
 }
 
 function retrycmd_if_failure() {

diff --git a/scripts/ceph-windows/run_tests b/scripts/ceph-windows/run_tests
index 56d8775d19045b4ac2c09834e0b8207529d4023d..6e2da2d79a455a0e7844e91590d95cd856c0c0c6 100644 (file)
--- a/scripts/ceph-windows/run_tests
+++ b/scripts/ceph-windows/run_tests
@@ -38,7 +38,7 @@ scp_upload $CEPH_KEYRING /ProgramData/ceph/keyring
 #
 # Setup the Ceph Windows build in the Windows VM
 #
-SSH_TIMEOUT=5m scp_upload $WORKSPACE/ceph.zip /ceph.zip
+scp_upload $WORKSPACE/ceph.zip /ceph.zip
 SSH_TIMEOUT=10m ssh_exec powershell.exe "\$ProgressPreference='SilentlyContinue'; Expand-Archive -Path /ceph.zip -DestinationPath / -Force"
 ssh_exec powershell.exe "Add-MpPreference -ExclusionPath 'C:\ceph'"
 ssh_exec powershell.exe "New-Service -Name ceph-rbd -BinaryPathName 'c:\ceph\rbd-wnbd.exe service'"
@@ -83,6 +83,7 @@ function collect_artifacts() {
     ssh_exec curl.exe --retry-max-time 30 --retry 10 -L -o /workspace/collect-event-logs.ps1 $COLLECT_EVENT_LOGS_SCRIPT_URL
     SSH_TIMEOUT=30m ssh_exec powershell.exe /workspace/collect-event-logs.ps1 -LogDirectory /workspace/eventlogs
     scp_download /workspace/eventlogs $WORKSPACE/artifacts/client/eventlogs
+    echo "Successfully retrieved artifacts."
 }
 trap collect_artifacts EXIT