From d552f389b2c1ca955fc56cb07755cad88292b00d Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 5 Nov 2021 11:53:17 +0200 Subject: [PATCH] Update ceph-windows-test job * Create `scripts/ceph-windows` directory with the common scripts for the Ceph Windows testing. * Move `run_tests` script to `scripts/ceph-windows`. * Remove VM image caching from the `run_tests` script. * Add `cleanup_tests` script to `scripts/ceph-windows`, and call this as a post-build step when the job is aborted. * Add `get_chacra_build` build script, and refactor away this logic from the `run_tests` script. * Move cleanup Windows tests function to `scripts/build_utils.sh`. --- ceph-windows-test/build/get_chacra_build | 16 ++++ .../config/definitions/ceph-windows-test.yml | 22 +++-- scripts/build_utils.sh | 26 ++++++ scripts/ceph-windows/cleanup_tests | 6 ++ .../build => scripts/ceph-windows}/run_tests | 87 ++++--------------- 5 files changed, 80 insertions(+), 77 deletions(-) create mode 100644 ceph-windows-test/build/get_chacra_build create mode 100644 scripts/ceph-windows/cleanup_tests rename {ceph-windows-test/build => scripts/ceph-windows}/run_tests (66%) mode change 100755 => 100644 diff --git a/ceph-windows-test/build/get_chacra_build b/ceph-windows-test/build/get_chacra_build new file mode 100644 index 00000000..e6ad1339 --- /dev/null +++ b/ceph-windows-test/build/get_chacra_build @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +CEPH_WINDOWS_VERSION=${CEPH_WINDOWS_VERSION:-"1809"} +CEPH_WINDOWS_SHA1=${CEPH_WINDOWS_SHA1:-"latest"} + +GET_BIN_SCRIPT_URL="https://raw.githubusercontent.com/ceph/ceph-win32-tests/master/get-bin.py" + +# +# Download the Chacra Ceph Windows build +# +cd $WORKSPACE +timeout 1m curl -L -o ./get-chacra-bin.py $GET_BIN_SCRIPT_URL +chmod +x ./get-chacra-bin.py +timeout 10m ./get-chacra-bin.py --distrover $CEPH_WINDOWS_VERSION --sha1 $CEPH_WINDOWS_SHA1 diff --git a/ceph-windows-test/config/definitions/ceph-windows-test.yml b/ceph-windows-test/config/definitions/ceph-windows-test.yml index d3283d0c..004256e0 100644 --- a/ceph-windows-test/config/definitions/ceph-windows-test.yml +++ b/ceph-windows-test/config/definitions/ceph-windows-test.yml @@ -23,17 +23,12 @@ description: "The SHA1 for the Ceph build." default: latest - scm: - - git: - url: https://github.com/ceph/ceph-win32-tests.git - branches: - - master - basedir: ceph-win32-tests - builders: - shell: !include-raw: - ../../build/run_tests + - ../../build/get_chacra_build + - ../../../scripts/build_utils.sh + - ../../../scripts/ceph-windows/run_tests wrappers: - credentials-binding: @@ -46,3 +41,14 @@ artifacts: 'logs/**' allow-empty: true latest-only: false + + - postbuildscript: + builders: + - role: SLAVE + build-on: + - ABORTED + build-steps: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../../scripts/ceph-windows/cleanup_tests diff --git a/scripts/build_utils.sh b/scripts/build_utils.sh index 09fcdb8a..6ba86d29 100644 --- a/scripts/build_utils.sh +++ b/scripts/build_utils.sh @@ -1595,3 +1595,29 @@ docs_pr_only() { fi popd } + +function cleanup_windows_tests_env() { + local VM_NAME=${VM_NAME:-"ceph-win-ltsc2019-${JOB_NAME}-${BUILD_ID}"} + + # Cleanup virsh VM + if sudo virsh list | grep -q $VM_NAME; then + echo "Shutting down VM $VM_NAME" + sudo virsh destroy $VM_NAME + fi + if sudo virsh list --all | grep -q $VM_NAME; then + echo "Deleting VM $VM_NAME" + sudo virsh undefine $VM_NAME --remove-all-storage + fi + # Cleanup Ceph clusters spawned via cephadm + if [[ -x $WORKSPACE/cephadm ]] && [[ -d /var/lib/ceph ]]; then + for FSID in $(sudo ls /var/lib/ceph); do + echo "Removing Ceph cluster $FSID" + sudo $WORKSPACE/cephadm rm-cluster --fsid $FSID --force + done + fi + # Cleanup remaning files / directories + sudo rm -rf \ + $WORKSPACE/ceph.conf $WORKSPACE/keyring $WORKSPACE/cephadm \ + $WORKSPACE/ceph.zip $WORKSPACE/known_hosts \ + /etc/ceph /var/log/ceph /var/lib/ceph /var/run/ceph +} diff --git a/scripts/ceph-windows/cleanup_tests b/scripts/ceph-windows/cleanup_tests new file mode 100644 index 00000000..b37f2604 --- /dev/null +++ b/scripts/ceph-windows/cleanup_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + + +cleanup_windows_tests_env diff --git a/ceph-windows-test/build/run_tests b/scripts/ceph-windows/run_tests old mode 100755 new mode 100644 similarity index 66% rename from ceph-windows-test/build/run_tests rename to scripts/ceph-windows/run_tests index dbf50bff..b3a2db04 --- a/ceph-windows-test/build/run_tests +++ b/scripts/ceph-windows/run_tests @@ -2,75 +2,46 @@ set -o errexit set -o pipefail -CEPH_WINDOWS_VERSION=${CEPH_WINDOWS_VERSION:-"1809"} -CEPH_WINDOWS_SHA1=${CEPH_WINDOWS_SHA1:-"latest"} +if [[ -z $CEPH_WIN_CI_KEY ]]; then + echo "ERROR: The CI SSH private key secret (CEPH_WIN_CI_KEY) is not set" + exit 1 +fi +if [[ ! -f $WORKSPACE/ceph.zip ]]; then + echo "ERROR: The Ceph Windows build zip file doesn't exist at $WORKSPACE/ceph.zip" + exit 1 +fi VM_IMAGE_URL=${VM_IMAGE_URL:-"https://filedump.ceph.com/windows/ceph-win-ltsc2019-ci-image.qcow2"} -VM_IMAGES_DIR=${VM_IMAGES_DIR:-"$WORKSPACE/vm_images"} - CEPHADM_RELEASE=${CEPHADM_RELEASE:-"pacific"} -VM_NAME="ceph-win-ltsc2019-${BUILD_ID}" -SSH_KNOWN_HOSTS="/tmp/ssh_known_hosts_${BUILD_ID}" +VM_NAME="ceph-win-ltsc2019-${JOB_NAME}-${BUILD_ID}" SSH_TIMEOUT="30s" function run_ssh_command() { local VM_IP="$1" shift - timeout $SSH_TIMEOUT ssh -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} administrator@${VM_IP} ${@} + timeout $SSH_TIMEOUT ssh -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${WORKSPACE}/known_hosts administrator@${VM_IP} ${@} } function run_scp_upload_command() { local VM_IP="$1" local LOCAL_FILE="$2" local REMOTE_FILE="$3" - timeout $SSH_TIMEOUT scp -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -r $LOCAL_FILE administrator@${VM_IP}:${REMOTE_FILE} + timeout $SSH_TIMEOUT scp -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${WORKSPACE}/known_hosts -r $LOCAL_FILE administrator@${VM_IP}:${REMOTE_FILE} } function run_scp_download_command() { local VM_IP="$1" local REMOTE_FILE="$2" local LOCAL_FILE="$3" - timeout $SSH_TIMEOUT scp -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${SSH_KNOWN_HOSTS} -r administrator@${VM_IP}:${REMOTE_FILE} $LOCAL_FILE -} - -function cleanup() { - # Cleanup virsh VM - if sudo virsh list | grep -q $VM_NAME; then - echo "Shutting down VM $VM_NAME" - sudo virsh destroy $VM_NAME - fi - if sudo virsh list --all | grep -q $VM_NAME; then - echo "Deleting VM $VM_NAME" - sudo virsh undefine $VM_NAME --remove-all-storage - fi - # Cleanup Ceph clusters spawned via cephadm - if [[ -x $WORKSPACE/cephadm ]] && [[ -d /var/lib/ceph ]]; then - for FSID in $(sudo ls /var/lib/ceph); do - echo "Removing Ceph cluster $FSID" - sudo $WORKSPACE/cephadm rm-cluster --fsid $FSID --force - done - fi - # Cleanup remaning files / directories - sudo rm -rf \ - $WORKSPACE/ceph.conf $WORKSPACE/keyring $WORKSPACE/cephadm \ - $WORKSPACE/ceph.zip $SSH_KNOWN_HOSTS /etc/ceph /var/log/ceph \ - /var/lib/ceph /var/run/ceph + timeout $SSH_TIMEOUT scp -i $CEPH_WIN_CI_KEY -o UserKnownHostsFile=${WORKSPACE}/known_hosts -r administrator@${VM_IP}:${REMOTE_FILE} $LOCAL_FILE } - # # Cleanup before the script exits # -trap cleanup EXIT - -# -# Download the Chacra Ceph Windows build -# -timeout 10m $WORKSPACE/ceph-win32-tests/get-bin.py \ - --distrover $CEPH_WINDOWS_VERSION \ - --sha1 $CEPH_WINDOWS_SHA1 +trap cleanup_windows_tests_env EXIT # # Install requirements (if needed) @@ -133,33 +104,11 @@ sudo chown $USER $WORKSPACE/ceph.conf $WORKSPACE/keyring sudo ceph osd pool create rbd # -# Download the Windows qcow2 image (if necessary) +# Download the Windows qcow2 image # IMAGE_FILE="$(basename $VM_IMAGE_URL)" -mkdir -p $VM_IMAGES_DIR - -echo "Decide if the VM image is up to date" -DOWNLOAD_IMAGE=false -if [[ -e "${VM_IMAGES_DIR}/${IMAGE_FILE}" ]]; then - SHA256="$(curl -L -s ${VM_IMAGE_URL}.sha256)" - if [[ ! -e "${VM_IMAGES_DIR}/${IMAGE_FILE}.sha256" ]]; then - sha256sum ${VM_IMAGES_DIR}/${IMAGE_FILE} | awk '{print $1}' > "${VM_IMAGES_DIR}/${IMAGE_FILE}.sha256" - fi - if [[ "$SHA256" != "$(cat ${VM_IMAGES_DIR}/${IMAGE_FILE}.sha256)" ]]; then - DOWNLOAD_IMAGE=true - fi -else - DOWNLOAD_IMAGE=true -fi -if [[ $DOWNLOAD_IMAGE == true ]]; then - echo "The VM image is outdated. Downloading it from $VM_IMAGE_URL" - curl -L -o ${VM_IMAGES_DIR}/${IMAGE_FILE} $VM_IMAGE_URL - curl -L -o ${VM_IMAGES_DIR}/${IMAGE_FILE}.sha256 ${VM_IMAGE_URL}.sha256 -else - echo "The VM image is up to date" -fi -echo "Copying VM image to be used for VM ${VM_NAME}" -cp ${VM_IMAGES_DIR}/${IMAGE_FILE} ${WORKSPACE}/${IMAGE_FILE} +echo "Downloading VM image from $VM_IMAGE_URL" +curl -L -o ${WORKSPACE}/${IMAGE_FILE} $VM_IMAGE_URL # # Start the Windows testing VM @@ -195,7 +144,7 @@ while true; do sleep $SLEEP_SECS continue } - ssh-keyscan -H $VM_IP &> $SSH_KNOWN_HOSTS || { + ssh-keyscan -H $VM_IP &> ${WORKSPACE}/known_hosts || { echo "SSH is not reachable yet" sleep $SLEEP_SECS continue @@ -226,7 +175,7 @@ run_ssh_command $VM_IP powershell.exe Start-Service -Name ceph-rbd # # Run the Windows tests # -SSH_TIMEOUT=5m run_scp_upload_command $VM_IP $WORKSPACE/ceph-win32-tests /workspace/repos/ceph-win32-tests +SSH_TIMEOUT=5m run_ssh_command $VM_IP git.exe clone https://github.com/ceph/ceph-win32-tests.git /workspace/repos/ceph-win32-tests SSH_TIMEOUT=1h run_ssh_command $VM_IP powershell.exe /workspace/repos/ceph-win32-tests/test_host/run_tests.ps1 -workerCount 4 # -- 2.39.5