From 1b3b97aee91bcbbdcc20eb6b16d9ea8d96c1d5f0 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Thu, 16 Dec 2021 16:12:57 +0200 Subject: [PATCH] Add `ceph-windows-installer-build` job to build Ceph Windows installer The `ceph-windows-installer` project creates a Windows MSI installer that can be used to setup Ceph on Windows. It bundles the `wnbd` driver, and the `ceph` binaries. These are downloaded from Chacra prior to building the installer. --- ceph-windows-installer-build/build/build | 78 ++++++++++++ ceph-windows-installer-build/build/failure | 10 ++ ceph-windows-installer-build/build/setup | 68 +++++++++++ .../ceph-windows-installer-build.yml | 115 ++++++++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 ceph-windows-installer-build/build/build create mode 100644 ceph-windows-installer-build/build/failure create mode 100644 ceph-windows-installer-build/build/setup create mode 100644 ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml diff --git a/ceph-windows-installer-build/build/build b/ceph-windows-installer-build/build/build new file mode 100644 index 00000000..f6e3d5a6 --- /dev/null +++ b/ceph-windows-installer-build/build/build @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +BUILD_CONFIGURATION=${BUILD_CONFIGURATION:-"Release"} + + +# +# Setup the installer requirements +# +mkdir -p $WORKSPACE/ceph-windows-installer/Driver +mkdir -p $WORKSPACE/ceph-windows-installer/Binaries +mkdir -p $WORKSPACE/ceph-windows-installer/Symbols + +mv $WORKSPACE/build/wnbd/driver/* $WORKSPACE/ceph-windows-installer/Driver/ +mv $WORKSPACE/build/wnbd/binaries/* $WORKSPACE/ceph-windows-installer/Binaries/ +mv $WORKSPACE/build/wnbd/symbols/* $WORKSPACE/ceph-windows-installer/Symbols/ + +pushd $WORKSPACE/build/ceph +mkdir -p $WORKSPACE/ceph-windows-installer/Binaries/.debug +for FILE in $(ls *.dll ceph-conf.exe rados.exe rbd.exe rbd-wnbd.exe ceph-dokan.exe); do + cp $FILE $WORKSPACE/ceph-windows-installer/Binaries/ + cp .debug/${FILE}.debug $WORKSPACE/ceph-windows-installer/Binaries/.debug/ +done +popd + +# +# Upload ceph-windows-installer repo to the Windows VM +# +SSH_TIMEOUT=5m scp_upload $WORKSPACE/ceph-windows-installer /workspace/ceph-windows-installer + +# +# Build the Visual Studio project +# +BUILD_CMD="MSBuild.exe %SystemDrive%\\workspace\\ceph-windows-installer\\ceph-windows-installer.sln /p:Platform=x64 /p:Configuration=${BUILD_CONFIGURATION}" +SSH_TIMEOUT=30m ssh_exec "\"%ProgramFiles(x86)%\\Microsoft Visual Studio\\2019\\BuildTools\\VC\\Auxiliary\\Build\\vcvarsall.bat\" x86_amd64 & ${BUILD_CMD}" + +# +# Download the installer +# +scp_download /workspace/ceph-windows-installer/bin/${BUILD_CONFIGURATION}/Ceph.msi $WORKSPACE/ +cp $WORKSPACE/ceph-windows-installer/Driver/wnbd.cer $WORKSPACE/wnbd_code_signing.cer + +# +# Upload the installer to Chacra +# +if [ "$THROWAWAY" = false ]; then + # push binaries to chacra + chacra_binary="$VENV/chacractl binary" + [ "$FORCE" = true ] && chacra_binary="$chacra_binary --force" + + ls $WORKSPACE/Ceph.msi $WORKSPACE/wnbd_code_signing.cer | $chacra_binary create ${chacra_binary_endpoint} + + ceph_version=$(ssh_exec /workspace/ceph-windows-installer/Binaries/rbd.exe -v | awk '{print $3}' | tr -d '[:space:]') + wnbd_version=$(ssh_exec /workspace/ceph-windows-installer/Binaries/wnbd-client.exe -v | grep wnbd-client.exe | cut -d ':' -f2 | tr -d '[:space:]') + vers="ceph-${ceph_version}|wnbd-${wnbd_version}" + + # write json file with build info + cat > $WORKSPACE/repo-extra.json << EOF +{ + "version":"$vers", + "package_manager_version":"", + "build_url":"$BUILD_URL", + "root_build_cause":"$ROOT_BUILD_CAUSE", + "node_name":"$NODE_NAME", + "job_name":"$JOB_NAME" +} +EOF + # post the json to repo-extra json to chacra + curl --fail -X POST -H "Content-Type:application/json" --data "@$WORKSPACE/repo-extra.json" -u $CHACRACTL_USER:$CHACRACTL_KEY ${chacra_url}repos/${chacra_repo_endpoint}/extra/ + # start repo creation + $VENV/chacractl repo update ${chacra_repo_endpoint} + + echo "Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_repo_endpoint}/" +fi + +# update shaman with the completed build status +update_build_status "completed" "ceph-windows-installer" $DISTRO $DISTRO_VERSION $ARCH diff --git a/ceph-windows-installer-build/build/failure b/ceph-windows-installer-build/build/failure new file mode 100644 index 00000000..1243f6d4 --- /dev/null +++ b/ceph-windows-installer-build/build/failure @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +FLAVOR="default" + +BRANCH=`branch_slash_filter $BRANCH` + +# update shaman with the failed build status +failed_build_status "ceph-windows-installer" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH diff --git a/ceph-windows-installer-build/build/setup b/ceph-windows-installer-build/build/setup new file mode 100644 index 00000000..3732c0d4 --- /dev/null +++ b/ceph-windows-installer-build/build/setup @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -o errexit +set -o pipefail + +CEPH_WINDOWS_BRANCH=${CEPH_WINDOWS_BRANCH:-"master"} +CEPH_WINDOWS_SHA1=${CEPH_WINDOWS_SHA1:-"latest"} +WNBD_BRANCH=${WNBD_BRANCH:-"master"} +WNBD_SHA1=${WNBD_SHA1:-"latest"} + +GET_BIN_SCRIPT_URL="https://raw.githubusercontent.com/ceph/ceph-win32-tests/master/get-bin.py" + +DISTRO="windows" +DISTRO_VERSION="1809" +ARCH="x86_64" +FLAVOR="default" + +BRANCH=`branch_slash_filter $BRANCH` +SHA1="$GIT_COMMIT" + +# +# Setup Chacra and Shaman +# +pkgs=( "chacractl>=0.0.21" ) +TEMPVENV=$(create_venv_dir) +VENV=${TEMPVENV}/bin +install_python_packages $TEMPVENV "pkgs[@]" + +# ask shaman which chacra instance to use +chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +# create the .chacractl config file using global variables +make_chacractl_config $chacra_url + +chacra_endpoint="ceph-windows-installer/${BRANCH}/${SHA1}/${DISTRO}/${DISTRO_VERSION}" +chacra_binary_endpoint="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}" +chacra_repo_endpoint="${chacra_endpoint}/flavors/${FLAVOR}" +chacra_check_url="${chacra_binary_endpoint}/Ceph.msi" + +if [ "$THROWAWAY" = false ] ; then + # this exists in scripts/build_utils.sh + check_binary_existence $VENV $chacra_check_url +fi + +# create build status in shaman +update_build_status "started" "ceph-windows-installer" $DISTRO $DISTRO_VERSION $ARCH + +# +# Install requirements (if needed) +# +if ! which unzip >/dev/null; then + sudo apt-get update + sudo apt-get install -y unzip +fi + +# +# Download the Ceph Windows build and the WNBD build from Chacra +# +rm -rf $WORKSPACE/build +mkdir -p $WORKSPACE/build +cd $WORKSPACE/build + +timeout 1m curl -L -o ./get-chacra-bin.py $GET_BIN_SCRIPT_URL +chmod +x ./get-chacra-bin.py + +timeout 10m ./get-chacra-bin.py --project ceph --branchname $CEPH_WINDOWS_BRANCH --sha1 $CEPH_WINDOWS_SHA1 --filename ceph.zip +unzip -q ceph.zip + +timeout 10m ./get-chacra-bin.py --project wnbd --branchname $WNBD_BRANCH --sha1 $WNBD_SHA1 --filename wnbd.zip +unzip -q wnbd.zip diff --git a/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml b/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml new file mode 100644 index 00000000..35b0ad11 --- /dev/null +++ b/ceph-windows-installer-build/config/definitions/ceph-windows-installer-build.yml @@ -0,0 +1,115 @@ +- job: + name: ceph-windows-installer-build + description: 'Builds the Ceph Windows MSI installer.' + node: amd64 && focal && libvirt + project-type: freestyle + defaults: global + concurrent: true + display-name: 'ceph-windows-installer-build' + properties: + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + artifact-days-to-keep: 30 + artifact-num-to-keep: 30 + + parameters: + - string: + name: BRANCH + description: "The git branch (or tag) to build" + default: "master" + + - string: + name: CEPH_WINDOWS_BRANCH + description: "The branch name for the Ceph build." + default: master + + - string: + name: CEPH_WINDOWS_SHA1 + description: "The SHA1 for the Ceph build." + default: latest + + - string: + name: WNBD_BRANCH + description: "The branch name for the WNBD build." + default: master + + - string: + name: WNBD_SHA1 + description: "The SHA1 for the WNBD build." + default: latest + + - bool: + name: THROWAWAY + description: | + Default: False. When True it will not POST binaries to chacra. Artifacts will not be around for long. Useful to test builds. + default: false + + - bool: + name: FORCE + description: | + If this is unchecked, then nothing is built or pushed if they already exist in chacra. This is the default. + + If this is checked, then the binaries will be built and pushed to chacra even if they already exist in chacra. + + scm: + - git: + url: https://github.com/cloudbase/ceph-windows-installer.git + branches: + - $BRANCH + timeout: 20 + wipe-workspace: true + basedir: ceph-windows-installer + + builders: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/setup + - ../../../scripts/ceph-windows/setup_libvirt_vm + - ../../build/build + + wrappers: + - inject-passwords: + global: true + mask-password-params: true + - credentials-binding: + - file: + credential-id: ceph_win_ci_private_key + variable: CEPH_WIN_CI_KEY + - text: + credential-id: chacractl-key + variable: CHACRACTL_KEY + - text: + credential-id: shaman-api-key + variable: SHAMAN_API_KEY + + publishers: + - postbuildscript: + builders: + - role: SLAVE + build-on: + - SUCCESS + - UNSTABLE + - FAILURE + - ABORTED + build-steps: + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../../scripts/ceph-windows/cleanup + + - postbuildscript: + builders: + - role: SLAVE + build-on: + - FAILURE + - ABORTED + build-steps: + - inject: + properties-file: ${WORKSPACE}/build_info + + - shell: + !include-raw: + - ../../../scripts/build_utils.sh + - ../../build/failure -- 2.47.3