From b2a720aebd7dc55762a836fee08ad76dd6ddf48f Mon Sep 17 00:00:00 2001 From: David Galloway Date: Thu, 4 Nov 2021 11:03:11 -0400 Subject: [PATCH] ceph-dev*: Start building Windows binaries for master Signed-off-by: David Galloway --- ceph-dev-build/build/build_mingw | 61 +++++++++++++++ ceph-dev-build/build/setup_mingw | 77 +++++++++++++++++++ ceph-dev-build/build/validate_mingw | 17 ++++ .../config/definitions/ceph-dev-build.yml | 8 ++ .../config/definitions/ceph-dev-cron.yml | 4 +- ceph-dev/config/definitions/ceph-dev.yml | 2 +- 6 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 ceph-dev-build/build/build_mingw create mode 100644 ceph-dev-build/build/setup_mingw create mode 100644 ceph-dev-build/build/validate_mingw diff --git a/ceph-dev-build/build/build_mingw b/ceph-dev-build/build/build_mingw new file mode 100644 index 00000000..b26c0255 --- /dev/null +++ b/ceph-dev-build/build/build_mingw @@ -0,0 +1,61 @@ +#!/bin/bash +set -ex + +# We need Ubuntu Focal to cross-compile Ceph for Windows. +# Previous versions provide broken Mingw packages. +# "DIST" will be set to "windows", so we're currently overriding it with +# a hardcoded value. +tmp_pbuild_script=$(mktemp /tmp/build_mingw_pbuild.XXXXXX) +cat << EOF > $tmp_pbuild_script +#!/bin/sh +# Used by the build script +apt-get install -y sudo git automake wget + +cd /mnt/ceph +CMAKE_BUILD_TYPE=Release BUILD_ZIP=1 CLEAN_BUILD=1 timeout 3h ./win32_build.sh +EOF +chmod a+x $tmp_pbuild_script +sudo pbuilder execute \ + --bindmounts "$(pwd):/mnt/ceph" \ + --distribution "focal" \ + --basetgz $basedir/focal.tgz \ + -- $tmp_pbuild_script +rm $tmp_pbuild_script + +if [ "$THROWAWAY" = false ]; then + # push binaries to chacra + chacra_binary="$VENV/chacractl binary" + chacra_create="$chacra_binary create" + [ "$FORCE" = true ] && chacra_binary="$chacra_binary --force" + + find build -name "*.zip" | + $chacra_create ${chacra_binary_endpoint} + + # write json file with build info + cat > $WORKSPACE/repo-extra.json << EOF +{ + "version":"$vers", + "package_manager_version":"", + "build_url":"$BUILD_URL", + "root_build_cause":"$ROOT_BUILD_CAUSE", + "node_name":"$NODE_NAME", + "job_name":"$JOB_NAME" +} +EOF + # post the json to repo-extra json to chacra + curl -X POST \ + -H "Content-Type:application/json" \ + --data "@$WORKSPACE/repo-extra.json" \ + -u $CHACRACTL_USER:$CHACRACTL_KEY \ + ${chacra_url}repos/${chacra_repo_endpoint}/extra/ + # start repo creation + $VENV/chacractl repo update ${chacra_repo_endpoint} + + echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_repo_endpoint}/ +fi + +# pbuilder will leave root-owned files in shared workspaces +sudo chown -R jenkins-build ${WORKSPACE}/dist + +# update shaman with the completed build status +update_build_status "completed" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH diff --git a/ceph-dev-build/build/setup_mingw b/ceph-dev-build/build/setup_mingw new file mode 100644 index 00000000..0b944127 --- /dev/null +++ b/ceph-dev-build/build/setup_mingw @@ -0,0 +1,77 @@ +#!/bin/bash + +set -ex +HOST=$(hostname --short) +echo "Building on $(hostname)" +echo " DIST=${DIST}" +echo " BPTAG=${BPTAG}" +echo " KEYID=${KEYID}" +echo " WS=$WORKSPACE" +echo " PWD=$(pwd)" +echo " BUILD SOURCE=$COPYARTIFACT_BUILD_NUMBER_CEPH_SETUP" +echo "*****" +env +echo "*****" + +if test $(id -u) != 0 ; then + SUDO=sudo +fi +export LC_ALL=C # the following is vulnerable to i18n + +BRANCH=`branch_slash_filter $BRANCH` + +cd ${WORKSPACE}/dist +vers=$(cat version) +raw_version=`echo $vers | cut -d '-' -f 1` +RELEASE_BRANCH=$(release_from_version $raw_version) + +# unpack the tar.gz that contains the source +tar xzf *.orig.tar.gz +cd $(basename *.orig.tar.gz .orig.tar.gz | sed s/_/-/) +pwd + +raw_version_major=$(echo $vers | cut -d '.' -f 1) +if [ 0${raw_version_major} -lt 16 ]; then + echo The following Ceph release does not support Windows: $RELEASE_BRANCH + exit 1 +fi + +# Normalize variables across rpm/deb builds +NORMAL_DISTRO=$DIST +NORMAL_DISTRO_VERSION="1809" +NORMAL_ARCH=$ARCH + +# create build status in shaman +update_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH + +pkgs=( "chacractl>=0.0.21" ) +TEMPVENV=$(create_venv_dir) +VENV=${TEMPVENV}/bin +install_python_packages $TEMPVENV "pkgs[@]" + +# ask shaman which chacra instance to use +chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +# create the .chacractl config file using global variables +make_chacractl_config $chacra_url + +FLAVOR="default" + +chacra_ref="$BRANCH" +chacra_endpoint="ceph/${chacra_ref}/${SHA1}/${DIST}/${NORMAL_DISTRO_VERSION}" +chacra_binary_endpoint="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}" +chacra_repo_endpoint="${chacra_endpoint}/flavors/${FLAVOR}" +chacra_check_url="${chacra_binary_endpoint}/ceph.zip" + +if [ "$THROWAWAY" = false ] ; then + # this exists in scripts/build_utils.sh + # TODO if this exits we need to post to shaman a success + check_binary_existence $VENV $chacra_check_url +fi + +# We need Ubuntu Focal to cross-compile Ceph for Windows. +# Previous versions provide broken Mingw packages. +# "DIST" will be set to "windows", so we're currently overriding it with +# a hardcoded value. +DIST="focal" +setup_pbuilder use_gcc +DIST="$NORMAL_DISTRO" diff --git a/ceph-dev-build/build/validate_mingw b/ceph-dev-build/build/validate_mingw new file mode 100644 index 00000000..c6174dfb --- /dev/null +++ b/ceph-dev-build/build/validate_mingw @@ -0,0 +1,17 @@ +#!/bin/bash +set -ex + +# We're currently using pbuilder. +( source /etc/os-release + case $ID in + ubuntu) + exit 0 + ;; + *) + exit 1 + ;; + esac) || exit 0 + +if [ "${DIST}" != "windows" ]; then + exit 0 +fi diff --git a/ceph-dev-build/config/definitions/ceph-dev-build.yml b/ceph-dev-build/config/definitions/ceph-dev-build.yml index e657ff9c..1ee340dc 100644 --- a/ceph-dev-build/config/definitions/ceph-dev-build.yml +++ b/ceph-dev-build/config/definitions/ceph-dev-build.yml @@ -54,6 +54,7 @@ - precise - centos6 - leap15 + - windows - axis: type: dynamic name: DIST @@ -105,6 +106,13 @@ - ../../../scripts/build_utils.sh - ../../build/setup_osc - ../../build/build_osc + # mingw build scripts (targeting Windows) + - shell: + !include-raw: + - ../../build/validate_mingw + - ../../../scripts/build_utils.sh + - ../../build/setup_mingw + - ../../build/build_mingw publishers: - postbuildscript: diff --git a/ceph-dev-cron/config/definitions/ceph-dev-cron.yml b/ceph-dev-cron/config/definitions/ceph-dev-cron.yml index f07be81a..3899d218 100644 --- a/ceph-dev-cron/config/definitions/ceph-dev-cron.yml +++ b/ceph-dev-cron/config/definitions/ceph-dev-cron.yml @@ -119,7 +119,7 @@ FLAVOR=crimson ARCHS=x86_64 # build master on: - # default: focal centos8 + # default: focal centos8 windows # crimson: centos8 - conditional-step: condition-kind: regex-match @@ -136,7 +136,7 @@ predefined-parameters: | BRANCH=${GIT_BRANCH} FORCE=True - DISTROS=focal centos8 + DISTROS=focal centos8 windows - project: 'ceph-dev' predefined-parameters: | BRANCH=${GIT_BRANCH} diff --git a/ceph-dev/config/definitions/ceph-dev.yml b/ceph-dev/config/definitions/ceph-dev.yml index a15945ee..3c8a0b4d 100644 --- a/ceph-dev/config/definitions/ceph-dev.yml +++ b/ceph-dev/config/definitions/ceph-dev.yml @@ -25,7 +25,7 @@ - string: name: DISTROS - description: "A list of distros to build for. Available options are: centos7, centos6, focal, bionic, xenial, trusty, precise, wheezy, and jessie" + description: "A list of distros to build for. Available options are: centos7, centos6, focal, bionic, xenial, trusty, precise, wheezy, jessie, and windows" default: "focal bionic centos7 centos8" - string: -- 2.39.5