From: Kefu Chai Date: Sat, 15 May 2021 03:15:01 +0000 (+0800) Subject: scripts/build_utils.sh: pass VENV to helper functions X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F1813%2Fhead;p=ceph-build.git scripts/build_utils.sh: pass VENV to helper functions pass VENV to following functions explicitly: * teardown_vagrant_tests() * start_tox() * build_debs() * install_python_packages() * install_python_packages_no_binary() * pip_download() * check_binary_existence() prepare for the cleanup to avoid creating TEMPVENV everytime the script is included. Signed-off-by: Kefu Chai --- diff --git a/calamari-clients-build/build/build_rpm b/calamari-clients-build/build/build_rpm index 6a8b3d4e0..585563ba9 100644 --- a/calamari-clients-build/build/build_rpm +++ b/calamari-clients-build/build/build_rpm @@ -62,7 +62,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/calamari-clients-${VERSION}-${RPM_R if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/calamari-clients-build/build/setup b/calamari-clients-build/build/setup index c97d4718f..8856866ba 100644 --- a/calamari-clients-build/build/setup +++ b/calamari-clients-build/build/setup @@ -54,7 +54,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/calamari/build/setup b/calamari/build/setup index e014d1ff1..981ec8544 100644 --- a/calamari/build/setup +++ b/calamari/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-ansible-docs-prs/build/build b/ceph-ansible-docs-prs/build/build index 50c9f920b..ee4ec6516 100644 --- a/ceph-ansible-docs-prs/build/build +++ b/ceph-ansible-docs-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd $WORKSPACE/docs/ $VENV/tox -rv diff --git a/ceph-ansible-docs/build/build b/ceph-ansible-docs/build/build index 2af4acb89..b060335bb 100644 --- a/ceph-ansible-docs/build/build +++ b/ceph-ansible-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # trims leading slashes BRANCH=${GIT_BRANCH#*/} diff --git a/ceph-ansible-nightly/build/build b/ceph-ansible-nightly/build/build index 3514dd0f0..6a96d997e 100644 --- a/ceph-ansible-nightly/build/build +++ b/ceph-ansible-nightly/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-ansible-nightly/build/teardown b/ceph-ansible-nightly/build/teardown index 350248e52..02a05ccf6 100644 --- a/ceph-ansible-nightly/build/teardown +++ b/ceph-ansible-nightly/build/teardown @@ -3,4 +3,4 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-ansible-prs/build/build b/ceph-ansible-prs/build/build index c6ab69851..a929dcdb2 100644 --- a/ceph-ansible-prs/build/build +++ b/ceph-ansible-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate @@ -35,6 +35,6 @@ rm -rf $HOME/ansible/facts/* [[ "$ghprbTargetBranch" != stable-3.2 && "$SCENARIO" == shrink_osd_legacy ]] || [[ "$ghprbTargetBranch" =~ stable-3 && "$SCENARIO" == filestore_to_bluestore ]] || [[ "$ghprbTargetBranch" == stable-5.0 && "$DEPLOYMENT" == "non_container" && "$SCENARIO" == update ]] || -start_tox +start_tox $TEMPVENV # Update scenario on stable-5.0 must be enabled back once 15.2.8 is out diff --git a/ceph-ansible-prs/build/teardown b/ceph-ansible-prs/build/teardown index e4434d648..6e250f61c 100644 --- a/ceph-ansible-prs/build/teardown +++ b/ceph-ansible-prs/build/teardown @@ -3,7 +3,7 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache rm -rf $HOME/ansible/facts/* diff --git a/ceph-ansible-rpm/build/build b/ceph-ansible-rpm/build/build index 08ecaf306..1a5ed0da0 100644 --- a/ceph-ansible-rpm/build/build +++ b/ceph-ansible-rpm/build/build @@ -18,7 +18,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-ansible-scenario/build/build b/ceph-ansible-scenario/build/build index 20fe106f0..aa99bd185 100644 --- a/ceph-ansible-scenario/build/build +++ b/ceph-ansible-scenario/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate @@ -17,4 +17,4 @@ update_vagrant_boxes # In the same logic, clean fact cache rm -rf $HOME/ansible/facts/* -start_tox \ No newline at end of file +start_tox diff --git a/ceph-ansible-scenario/build/teardown b/ceph-ansible-scenario/build/teardown index b9e5ebfa9..6e250f61c 100644 --- a/ceph-ansible-scenario/build/teardown +++ b/ceph-ansible-scenario/build/teardown @@ -3,7 +3,7 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache -rm -rf $HOME/ansible/facts/* \ No newline at end of file +rm -rf $HOME/ansible/facts/* diff --git a/ceph-build-pull-requests/build/build b/ceph-build-pull-requests/build/build index b51a923c9..fb6a7159a 100644 --- a/ceph-build-pull-requests/build/build +++ b/ceph-build-pull-requests/build/build @@ -6,7 +6,7 @@ set -e # must pin urllib3 to 1.22 because 1.23 is incompatible with requests, which # is used by jenkins-job-builder pkgs=( "ansible" "jenkins-job-builder>=3.5.0" "urllib3==1.22" "pyopenssl" "ndg-httpsclient" "pyasn1" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # Test every definition if available in the current repository and update the jobs diff --git a/ceph-build/build/build_deb b/ceph-build/build/build_deb index aa573c6e0..312f379ab 100644 --- a/ceph-build/build/build_deb +++ b/ceph-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs ${VENV} ${vers} ${debian_version} diff --git a/ceph-build/build/setup_deb b/ceph-build/build/setup_deb index 6332a498c..6ed8e12eb 100644 --- a/ceph-build/build/setup_deb +++ b/ceph-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" chacra_url=https://chacra.ceph.com/ make_chacractl_config @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-build/build/setup_rpm b/ceph-build/build/setup_rpm index 61edfdd67..f6b63fe1a 100644 --- a/ceph-build/build/setup_rpm +++ b/ceph-build/build/setup_rpm @@ -49,7 +49,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -70,5 +70,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-container-nighlity/build/build b/ceph-container-nighlity/build/build index 589d3fc54..df9a61282 100644 --- a/ceph-container-nighlity/build/build +++ b/ceph-container-nighlity/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-container-prs/build/build b/ceph-container-prs/build/build index 589d3fc54..df9a61282 100644 --- a/ceph-container-prs/build/build +++ b/ceph-container-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-deploy-build/build/build b/ceph-deploy-build/build/build index e89ec9b98..5edbae202 100644 --- a/ceph-deploy-build/build/build +++ b/ceph-deploy-build/build/build @@ -46,7 +46,7 @@ then # this exists in scripts/build_utils.sh for ceph-deploy, binaries have # no architecture so we POST them to 'noarch' for rpms - check_binary_existence $chacra_endpoint/noarch + check_binary_existence $VENV $chacra_endpoint/noarch if [ ! -e setup.py ] ; then echo "Are we in the right directory" @@ -103,7 +103,7 @@ then chacra_endpoint="ceph-deploy/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/universal/all" # this exists in scripts/build_utils.sh - check_binary_existence $chacra_endpoint + check_binary_existence $VENV $chacra_endpoint if [ ! -d debian ] ; then echo "Are we in the right directory" diff --git a/ceph-deploy-build/build/setup b/ceph-deploy-build/build/setup index a1fa6a8c7..1f71004f5 100644 --- a/ceph-deploy-build/build/setup +++ b/ceph-deploy-build/build/setup @@ -22,7 +22,7 @@ if test -f /etc/redhat-release ; then fi pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-deploy-docs/build/build b/ceph-deploy-docs/build/build index 73e99b7a9..1f80aafb4 100644 --- a/ceph-deploy-docs/build/build +++ b/ceph-deploy-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the docs build with tox $VENV/tox -rv -e docs diff --git a/ceph-deploy-pull-requests/build/setup b/ceph-deploy-pull-requests/build/setup index d8dc99073..ced5ae484 100644 --- a/ceph-deploy-pull-requests/build/setup +++ b/ceph-deploy-pull-requests/build/setup @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying diff --git a/ceph-deploy-tag/build/build b/ceph-deploy-tag/build/build index 89bb0ad34..1ea302bf3 100644 --- a/ceph-deploy-tag/build/build +++ b/ceph-deploy-tag/build/build @@ -9,7 +9,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to do all the tagging and release specifying # a local connection and 'localhost' as the host where to execute diff --git a/ceph-dev-build/build/build_deb b/ceph-dev-build/build/build_deb index aa573c6e0..5881c4785 100644 --- a/ceph-dev-build/build/build_deb +++ b/ceph-dev-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs $VENV ${vers} ${debian_version} diff --git a/ceph-dev-build/build/setup_deb b/ceph-dev-build/build/setup_deb index 98888e5ae..9720f518a 100644 --- a/ceph-dev-build/build/setup_deb +++ b/ceph-dev-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-build/build/setup_osc b/ceph-dev-build/build/setup_osc index 4b81c6e09..591d40347 100644 --- a/ceph-dev-build/build/setup_osc +++ b/ceph-dev-build/build/setup_osc @@ -78,7 +78,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -97,5 +97,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-build/build/setup_rpm b/ceph-dev-build/build/setup_rpm index 9146c3007..639d7b625 100644 --- a/ceph-dev-build/build/setup_rpm +++ b/ceph-dev-build/build/setup_rpm @@ -51,7 +51,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -72,5 +72,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-new-build/build/build_deb b/ceph-dev-new-build/build/build_deb index aa573c6e0..312f379ab 100644 --- a/ceph-dev-new-build/build/build_deb +++ b/ceph-dev-new-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs ${VENV} ${vers} ${debian_version} diff --git a/ceph-dev-new-build/build/setup_deb b/ceph-dev-new-build/build/setup_deb index 98888e5ae..9720f518a 100644 --- a/ceph-dev-new-build/build/setup_deb +++ b/ceph-dev-new-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-new-build/build/setup_mingw b/ceph-dev-new-build/build/setup_mingw index 307c3ed69..08ba136fd 100644 --- a/ceph-dev-new-build/build/setup_mingw +++ b/ceph-dev-new-build/build/setup_mingw @@ -45,7 +45,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -63,7 +63,7 @@ chacra_check_url="${chacra_binary_endpoint}/ceph.zip" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi # We need Ubuntu Focal to cross-compile Ceph for Windows. diff --git a/ceph-dev-new-build/build/setup_rpm b/ceph-dev-new-build/build/setup_rpm index 5e6700794..aad8c5d74 100644 --- a/ceph-dev-new-build/build/setup_rpm +++ b/ceph-dev-new-build/build/setup_rpm @@ -49,7 +49,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -70,5 +70,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-iscsi-cli-flake8/build/build b/ceph-iscsi-cli-flake8/build/build index 8c66e74f2..70779665c 100644 --- a/ceph-iscsi-cli-flake8/build/build +++ b/ceph-iscsi-cli-flake8/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi-cli" diff --git a/ceph-iscsi-cli/build/setup b/ceph-iscsi-cli/build/setup index 085780331..c36cffd2d 100644 --- a/ceph-iscsi-cli/build/setup +++ b/ceph-iscsi-cli/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-config-flake8/build/build b/ceph-iscsi-config-flake8/build/build index 426bc9d40..2aa39eaa1 100644 --- a/ceph-iscsi-config-flake8/build/build +++ b/ceph-iscsi-config-flake8/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi-config" diff --git a/ceph-iscsi-config/build/setup b/ceph-iscsi-config/build/setup index 085780331..c36cffd2d 100644 --- a/ceph-iscsi-config/build/setup +++ b/ceph-iscsi-config/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-stable/build/setup b/ceph-iscsi-stable/build/setup index 3989aced8..746f0f1d4 100644 --- a/ceph-iscsi-stable/build/setup +++ b/ceph-iscsi-stable/build/setup @@ -45,7 +45,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" chacra_url="https://chacra.ceph.com/" # create the .chacractl config file using global variables diff --git a/ceph-iscsi-tools/build/setup b/ceph-iscsi-tools/build/setup index 085780331..c36cffd2d 100644 --- a/ceph-iscsi-tools/build/setup +++ b/ceph-iscsi-tools/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-tox/build/build b/ceph-iscsi-tox/build/build index 8a97583d3..e5b00a245 100644 --- a/ceph-iscsi-tox/build/build +++ b/ceph-iscsi-tox/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi" diff --git a/ceph-iscsi/build/setup b/ceph-iscsi/build/setup index 085780331..c36cffd2d 100644 --- a/ceph-iscsi/build/setup +++ b/ceph-iscsi/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-medic-docs/build/build b/ceph-medic-docs/build/build index 5f99f47c7..003ddbbc7 100644 --- a/ceph-medic-docs/build/build +++ b/ceph-medic-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # trims leading slashes BRANCH=`branch_slash_filter ${GIT_BRANCH}` diff --git a/ceph-medic-pull-requests/build/build b/ceph-medic-pull-requests/build/build index ec6f12731..c5385f196 100644 --- a/ceph-medic-pull-requests/build/build +++ b/ceph-medic-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" sudo yum install -y epel-release sudo yum --enablerepo epel install -y python36 diff --git a/ceph-medic-release/build/build_rpm b/ceph-medic-release/build/build_rpm index 125bb4fa2..cef722bcb 100644 --- a/ceph-medic-release/build/build_rpm +++ b/ceph-medic-release/build/build_rpm @@ -16,7 +16,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" make_chacractl_config diff --git a/ceph-medic-rpm/build/build b/ceph-medic-rpm/build/build index 08ecaf306..1a5ed0da0 100644 --- a/ceph-medic-rpm/build/build +++ b/ceph-medic-rpm/build/build @@ -18,7 +18,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-medic-tests/build/build b/ceph-medic-tests/build/build index 2ba71d3e9..3a7453708 100644 --- a/ceph-medic-tests/build/build +++ b/ceph-medic-tests/build/build @@ -1,6 +1,6 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" WORKDIR=$(mktemp -td tox.XXXXXXXXXX) diff --git a/ceph-pr-commits/build/build b/ceph-pr-commits/build/build index 4c134b4d4..0f4ac0bdc 100644 --- a/ceph-pr-commits/build/build +++ b/ceph-pr-commits/build/build @@ -17,7 +17,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "pytest" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE" timeout 600 $VENV/py.test -m "${pytest_mark}" -vs --junitxml="$WORKSPACE/report.xml" "$WORKSPACE/ceph-build/ceph-pr-commits/build/test_commits.py" diff --git a/ceph-qa-suite-pull-requests/build/build b/ceph-qa-suite-pull-requests/build/build index d1bd1238a..08b39bbdf 100644 --- a/ceph-qa-suite-pull-requests/build/build +++ b/ceph-qa-suite-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode diff --git a/ceph-release-rpm/build/build b/ceph-release-rpm/build/build index ec8b5ab8c..aed3cd23d 100644 --- a/ceph-release-rpm/build/build +++ b/ceph-release-rpm/build/build @@ -13,7 +13,7 @@ rm -rf dist rm -rf RPMBUILD pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -37,7 +37,7 @@ if [ "$target" = "sles11sp2" ] ; then fi echo "Target directory is: $target" -check_binary_existence $chacra_baseurl/noarch +check_binary_existence $VENV $chacra_baseurl/noarch # setup rpm build area mkdir -p RPMBUILD/{BUILD,RPMS,SOURCES,SPECS,SRPMS} diff --git a/ceph-tag/build/build b/ceph-tag/build/build index 4fabef2da..d97a3e705 100644 --- a/ceph-tag/build/build +++ b/ceph-tag/build/build @@ -9,7 +9,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to do all the tagging and release specifying # a local connection and 'localhost' as the host where to execute diff --git a/ceph-volume-ansible-prs/build/build b/ceph-volume-ansible-prs/build/build index d8d3e050c..17c9ffa28 100644 --- a/ceph-volume-ansible-prs/build/build +++ b/ceph-volume-ansible-prs/build/build @@ -10,7 +10,7 @@ github_status_setup # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="pending" $VENV/github-status create diff --git a/ceph-volume-ansible-prs/build/teardown b/ceph-volume-ansible-prs/build/teardown index 07ed0e329..6b92cb3ee 100644 --- a/ceph-volume-ansible-prs/build/teardown +++ b/ceph-volume-ansible-prs/build/teardown @@ -4,11 +4,11 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="failure" $VENV/github-status create cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-volume-nightly/build/build b/ceph-volume-nightly/build/build index f4f054801..8f1a68f9b 100644 --- a/ceph-volume-nightly/build/build +++ b/ceph-volume-nightly/build/build @@ -4,7 +4,7 @@ WORKDIR=$(mktemp -td tox.XXXXXXXXXX) # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" prune_stale_vagrant_vms $WORKSPACE/../**/tests delete_libvirt_vms diff --git a/ceph-volume-nightly/build/teardown b/ceph-volume-nightly/build/teardown index d4518338b..ad7de0428 100644 --- a/ceph-volume-nightly/build/teardown +++ b/ceph-volume-nightly/build/teardown @@ -8,4 +8,4 @@ cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional scenarios=$(find . | grep vagrant_ssh_config | xargs -r dirname) # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-volume-pr/build/build b/ceph-volume-pr/build/build index 83833a7a2..8ee52a8e7 100644 --- a/ceph-volume-pr/build/build +++ b/ceph-volume-pr/build/build @@ -8,7 +8,7 @@ github_status_setup # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3") -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd src/ceph-volume diff --git a/ceph-volume-pr/build/teardown b/ceph-volume-pr/build/teardown index 057f0e9f5..ab07894a5 100644 --- a/ceph-volume-pr/build/teardown +++ b/ceph-volume-pr/build/teardown @@ -4,6 +4,6 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="failure" $VENV/github-status create diff --git a/ceph-volume-scenario/build/build b/ceph-volume-scenario/build/build index 2bac2e594..2023af313 100644 --- a/ceph-volume-scenario/build/build +++ b/ceph-volume-scenario/build/build @@ -4,7 +4,7 @@ WORKDIR=$(mktemp -td tox.XXXXXXXXXX) # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" delete_libvirt_vms diff --git a/ceph-volume-scenario/build/teardown b/ceph-volume-scenario/build/teardown index a5c26da80..f03694a55 100644 --- a/ceph-volume-scenario/build/teardown +++ b/ceph-volume-scenario/build/teardown @@ -3,4 +3,4 @@ cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/cephadm-ansible-prs/build/build b/cephadm-ansible-prs/build/build index a2edde822..bd5afe3d4 100644 --- a/cephadm-ansible-prs/build/build +++ b/cephadm-ansible-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/cephadm-ansible-prs/build/teardown b/cephadm-ansible-prs/build/teardown index 9ce6137ff..c4f2c3f89 100644 --- a/cephadm-ansible-prs/build/teardown +++ b/cephadm-ansible-prs/build/teardown @@ -3,7 +3,7 @@ cd "${WORKSPACE}"/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache rm -rf "${HOME}"/ansible/facts/* diff --git a/cephmetrics-pull-requests/setup/setup b/cephmetrics-pull-requests/setup/setup index e7e861606..420fbefc0 100644 --- a/cephmetrics-pull-requests/setup/setup +++ b/cephmetrics-pull-requests/setup/setup @@ -13,4 +13,4 @@ elif test -f /etc/debian_version ; then fi pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" diff --git a/cephmetrics-release/build/setup b/cephmetrics-release/build/setup index e4477e088..576e7fb94 100644 --- a/cephmetrics-release/build/setup +++ b/cephmetrics-release/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config diff --git a/cephmetrics/build/setup b/cephmetrics/build/setup index 04867736f..14fec3fc7 100644 --- a/cephmetrics/build/setup +++ b/cephmetrics/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/chacra-pull-requests/build/build b/chacra-pull-requests/build/build index 6fc7c5924..44fae5c15 100644 --- a/chacra-pull-requests/build/build +++ b/chacra-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying # a local connection and 'localhost' as the host where to execute diff --git a/configshell-fb/build/setup b/configshell-fb/build/setup index 085780331..c36cffd2d 100644 --- a/configshell-fb/build/setup +++ b/configshell-fb/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/diamond-build/build/build_deb b/diamond-build/build/build_deb index e8ef73550..103644ac6 100644 --- a/diamond-build/build/build_deb +++ b/diamond-build/build/build_deb @@ -33,7 +33,7 @@ chacra_check_url="${chacra_endpoint}/diamond_${BPVER}_${ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/diamond-build/build/build_rpm b/diamond-build/build/build_rpm index c63a62a80..4edc8e780 100644 --- a/diamond-build/build/build_rpm +++ b/diamond-build/build/build_rpm @@ -63,7 +63,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/diamond-${VERSION}-0.${DIST}.${ARCH if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/diamond-build/build/setup b/diamond-build/build/setup index c97d4718f..8856866ba 100644 --- a/diamond-build/build/setup +++ b/diamond-build/build/setup @@ -54,7 +54,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/jenkins-job-builder/build/build b/jenkins-job-builder/build/build index 8683e0455..bbc44b7ec 100644 --- a/jenkins-job-builder/build/build +++ b/jenkins-job-builder/build/build @@ -10,7 +10,7 @@ set -euxo pipefail # the following two methods exist in scripts/build_utils.sh pkgs=( "jenkins-job-builder>=3.5.0" ) -install_python_packages "pkgs[@]" latest +install_python_packages $TEMPVENV "pkgs[@]" latest # Wipe out JJB's cache if $FORCE is set. [ "$FORCE" = true ] && rm -rf "$HOME/.cache/jenkins_jobs/" diff --git a/kernel/build/build_deb b/kernel/build/build_deb index aff31ee41..cdd5ddb99 100644 --- a/kernel/build/build_deb +++ b/kernel/build/build_deb @@ -28,7 +28,7 @@ chacra_check_url="${chacra_endpoint}/kernel_${BPVER}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/kernel/build/build_rpm b/kernel/build/build_rpm index 0699c9d38..366699fb3 100644 --- a/kernel/build/build_rpm +++ b/kernel/build/build_rpm @@ -55,7 +55,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/kernel-${VERSION}-${RPM_RELEASE}.${ if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/kernel/build/setup b/kernel/build/setup index 13570b770..b95471e38 100644 --- a/kernel/build/setup +++ b/kernel/build/setup @@ -68,7 +68,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/merfi-pull-requests/build/build b/merfi-pull-requests/build/build index 48947738b..4d2603ceb 100644 --- a/merfi-pull-requests/build/build +++ b/merfi-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode # by default this will run all environments defined diff --git a/mita-deploy/build/build b/mita-deploy/build/build index 82ef308eb..83b0ee176 100644 --- a/mita-deploy/build/build +++ b/mita-deploy/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/deploy/playbooks/" $VENV/ansible-playbook -i "localhost," -c local local_deploy.yml --extra-vars="branch=$BRANCH jenkins_prado_token=$JENKINS_PRADO_TOKEN prado_token=$PRADO_TOKEN" diff --git a/nfs-ganesha-stable/build/setup b/nfs-ganesha-stable/build/setup index 43c019ed5..58c6f0853 100644 --- a/nfs-ganesha-stable/build/setup +++ b/nfs-ganesha-stable/build/setup @@ -45,7 +45,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" NFS_GANESHA_BRANCH=`branch_slash_filter $NFS_GANESHA_BRANCH` CEPH_BRANCH=`branch_slash_filter $CEPH_BRANCH` diff --git a/nfs-ganesha/build/setup b/nfs-ganesha/build/setup index 4146be77f..57c1408cb 100644 --- a/nfs-ganesha/build/setup +++ b/nfs-ganesha/build/setup @@ -42,7 +42,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" NFS_GANESHA_BRANCH=`branch_slash_filter $NFS_GANESHA_BRANCH` CEPH_BRANCH=`branch_slash_filter $CEPH_BRANCH` diff --git a/paddles-pull-requests/build/build b/paddles-pull-requests/build/build index ec83f09e1..9f0922e63 100644 --- a/paddles-pull-requests/build/build +++ b/paddles-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" sudo yum install -y epel-release sudo yum --enablerepo epel install -y python36 diff --git a/radosgw-agent-pull-requests/build/build b/radosgw-agent-pull-requests/build/build index 7ac19b72a..7ff22019b 100644 --- a/radosgw-agent-pull-requests/build/build +++ b/radosgw-agent-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying diff --git a/radosgw-agent/build/build b/radosgw-agent/build/build index e9b935cb5..9be367634 100644 --- a/radosgw-agent/build/build +++ b/radosgw-agent/build/build @@ -15,7 +15,7 @@ ls -l # the following two methods exist in scripts/build_utils.sh pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -37,7 +37,7 @@ if [[ -f /etc/redhat-release || -f /usr/bin/zypper ]] ; then chacra_endpoint="radosgw-agent/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/${DISTRO_VERSION}" - check_binary_existence $chacra_endpoint/noarch + check_binary_existence $VENV $chacra_endpoint/noarch suse=$(uname -n | grep -ic -e suse -e sles || true) if [ $suse -gt 0 ] @@ -75,7 +75,7 @@ else chacra_endpoint="radosgw-agent/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/${DEB_BUILD}/noarch" - check_binary_existence $chacra_endpoint + check_binary_existence $VENV $chacra_endpoint dpkg-source -b . # we no longer sign the .dsc or .changes files (done by default with diff --git a/rtslib-fb/build/setup b/rtslib-fb/build/setup index 085780331..c36cffd2d 100644 --- a/rtslib-fb/build/setup +++ b/rtslib-fb/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/samba/build/setup b/samba/build/setup index 426b73ff2..d7956618f 100644 --- a/samba/build/setup +++ b/samba/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" SAMBA_BRANCH=$(branch_slash_filter $SAMBA_BRANCH) CEPH_BRANCH=$(branch_slash_filter $CEPH_BRANCH) diff --git a/scripts/build_utils.sh b/scripts/build_utils.sh index 7ef1cb93e..a251c7dc9 100644 --- a/scripts/build_utils.sh +++ b/scripts/build_utils.sh @@ -80,12 +80,14 @@ branch_slash_filter() { } pip_download() { + local venv=$1 + shift local package=$1 shift local options=$@ - if ! $VENV/pip download $options --dest="$PIP_SDIST_INDEX" $package; then + if ! $venv/pip download $options --dest="$PIP_SDIST_INDEX" $package; then # pip <8.0.0 does not have "download" command - $VENV/pip install $options \ + $venv/pip install $options \ --upgrade --exists-action=i --cache-dir="$PIP_SDIST_INDEX" \ $package fi @@ -107,6 +109,9 @@ create_virtualenv () { } install_python_packages_no_binary () { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" # Use this function to create a virtualenv and install python packages # without compiling (or using wheels). Pass a list of package names. If # the virtualenv exists it will get re-used since this function can be used @@ -115,19 +120,19 @@ install_python_packages_no_binary () { # Usage (with pip==20.3.4 [the default]): # # to_install=( "ansible" "chacractl>=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" + # install_python_packages_no_binary $TEMPVENV "to_install[@]" # # Usage (with pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" "pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" latest + # install_python_packages_no_binary $TEMPVENV "to_install[@]" latest - create_virtualenv $TEMPVENV + create_virtualenv $venv_dir # Define and ensure the PIP cache PIP_SDIST_INDEX="$HOME/.cache/pip" @@ -139,51 +144,54 @@ install_python_packages_no_binary () { # Updated to 20.3.4 in March 2021 because 10.0.0 is just too old. if [ "$2" == "latest" ]; then echo "Ensuring latest pip is installed" - $VENV/pip install -U pip + $venv/pip install -U pip elif [[ -n $2 && "$2" != "latest" ]]; then echo "Installing $2" - $VENV/pip install "$2" + $venv/pip install "$2" else # This is the default for most jobs. # See ff01d2c5 and fea10f52 echo "Installing pip 20.3.4" - $VENV/pip install "pip==20.3.4" + $venv/pip install "pip==20.3.4" fi echo "Updating setuptools" - pip_download setuptools + pip_download $VENV setuptools pkgs=("${!1}") for package in ${pkgs[@]}; do echo $package # download packages to the local pip cache - pip_download $package --no-binary=:all: + pip_download $VENV $package --no-binary=:all: # install packages from the local pip cache, ignoring pypi - $VENV/pip install --no-binary=:all: --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package + $venv/pip install --no-binary=:all: --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package done } install_python_packages () { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" # Use this function to create a virtualenv and install # python packages. Pass a list of package names. # # Usage (with pip 20.3.4 [the default]): # # to_install=( "ansible" "chacractl>=0.0.21" ) - # install_python_packages "to_install[@]" + # install_python_packages $TEMPVENV "to_install[@]" # # Usage (with pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" "pip=0.0.21" ) - # install_python_packages "to_install[@]" latest + # install_python_packages $TEMPVENV "to_install[@]" latest - create_virtualenv $TEMPVENV + create_virtualenv $venv_dir # Define and ensure the PIP cache PIP_SDIST_INDEX="$HOME/.cache/pip" @@ -195,27 +203,27 @@ install_python_packages () { # Updated to 20.3.4 in March 2021 because 10.0.0 is just too old. if [ "$2" == "latest" ]; then echo "Ensuring latest pip is installed" - $VENV/pip install -U pip + $venv/pip install -U pip elif [[ -n $2 && "$2" != "latest" ]]; then echo "Installing $2" - $VENV/pip install "$2" + $venv/pip install "$2" else # This is the default for most jobs. # See ff01d2c5 and fea10f52 echo "Installing pip 20.3.4" - $VENV/pip install "pip==20.3.4" + $venv/pip install "pip==20.3.4" fi echo "Updating setuptools" - pip_download setuptools + pip_download $venv setuptools pkgs=("${!1}") for package in ${pkgs[@]}; do echo $package # download packages to the local pip cache - pip_download $package + pip_download $venv $package # install packages from the local pip cache, ignoring pypi - $VENV/pip install --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package + $venv/pip install --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package done } @@ -283,11 +291,14 @@ get_rpm_dist() { } check_binary_existence () { - url=$1 + local venv=$1 + shift + local url=$1 + shift # we have to use ! here so thet -e will ignore the error code for the command # because of this, the exit code is also reversed - ! $VENV/chacractl exists binaries/${url} ; exists=$? + ! $venv/chacractl exists binaries/${url} ; exists=$? # if the binary already exists in chacra, do not rebuild if [ $exists -eq 1 ] && [ "$FORCE" = false ] ; then @@ -843,6 +854,8 @@ ceph_build_args_from_flavor() { } build_debs() { + local venv=$1 + shift local vers=$1 shift local debian_version=$1 @@ -912,7 +925,7 @@ build_debs() { find release/$vers/ | \ egrep "*\.(changes|deb|ddeb|dsc|gz)$" | \ egrep -v "(Packages|Sources|Contents)" | \ - $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} + $venv/chacractl binary ${chacra_flags} create ${chacra_endpoint} # write json file with build info cat > $WORKSPACE/repo-extra.json << EOF { @@ -931,7 +944,7 @@ EOF -u $CHACRACTL_USER:$CHACRACTL_KEY \ ${chacra_url}repos/${chacra_repo_endpoint}/extra/ # start repo creation - $VENV/chacractl repo update ${chacra_repo_endpoint} + $venv/chacractl repo update ${chacra_repo_endpoint} echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_repo_endpoint}/ fi @@ -1047,6 +1060,10 @@ update_vagrant_boxes() { } start_tox() { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" + # the $SCENARIO var is injected by the job template. It maps # to an actual, defined, tox environment while true; do @@ -1122,11 +1139,11 @@ start_tox() { ;; esac - for tox_env in $("$VENV"/tox -c "$TOX_INI_FILE" -l) + for tox_env in $("$venv"/tox -c "$TOX_INI_FILE" -l) do if [[ "$ENV_NAME" == "$tox_env" ]]; then # shellcheck disable=SC2116 - if ! eval "$(echo "${TOX_RUN_ENV[@]}")" "$VENV"/tox -c "$TOX_INI_FILE" --workdir="$TEMPVENV" -v -e="$ENV_NAME" -- --provider=libvirt; then + if ! eval "$(echo "${TOX_RUN_ENV[@]}")" "$venv"/tox -c "$TOX_INI_FILE" --workdir="$venv_dir" -v -e="$ENV_NAME" -- --provider=libvirt; then echo "ERROR: Job didn't complete successfully or got stuck for more than 3h." exit 1 fi @@ -1244,6 +1261,8 @@ EOF } collect_ceph_logs() { + local venv=$1 + shift # this is meant to be run in a testing scenario directory # with running vagrant vms. the ansible playbook will connect # to your test nodes and fetch any ceph logs that are present @@ -1257,15 +1276,18 @@ collect_ceph_logs() { write_collect_logs_playbook pkgs=( "ansible" ) - install_python_packages "pkgs[@]" + install_python_packages $TEMPVENV "pkgs[@]" export ANSIBLE_SSH_ARGS='-F ./vagrant_ssh_config' export ANSIBLE_STDOUT_CALLBACK='debug' - $VENV/ansible-playbook -vv -i hosts --limit $limit --extra-vars "archive_path=$WORKSPACE/logs" $WORKSPACE/collect-logs.yml || true + $venv/ansible-playbook -vv -i hosts --limit $limit --extra-vars "archive_path=$WORKSPACE/logs" $WORKSPACE/collect-logs.yml || true fi } teardown_vagrant_tests() { + local venv=$1 + shift + # collect ceph logs and teardown any running vagrant vms # this also cleans up any lingering livirt networks scenarios=$(find . | grep vagrant_ssh_config | xargs -r dirname) @@ -1273,7 +1295,7 @@ teardown_vagrant_tests() { for scenario in $scenarios; do cd $scenario # collect all ceph logs from all test nodes - collect_ceph_logs all + collect_ceph_logs $venv all vagrant destroy -f stat ./fetch > /dev/null 2>&1 && rm -rf ./fetch cd - diff --git a/shaman-pull-requests/build/build b/shaman-pull-requests/build/build index 71fa654c6..85f1137d8 100644 --- a/shaman-pull-requests/build/build +++ b/shaman-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying # a local connection and 'localhost' as the host where to execute diff --git a/tcmu-runner/build/setup b/tcmu-runner/build/setup index 085780331..c36cffd2d 100644 --- a/tcmu-runner/build/setup +++ b/tcmu-runner/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/teuthology-docs/build/build b/teuthology-docs/build/build index 381c06d6c..77628b1ac 100644 --- a/teuthology-docs/build/build +++ b/teuthology-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the docs build with tox $VENV/tox -rv -e docs diff --git a/teuthology-pull-requests/build/build b/teuthology-pull-requests/build/build index 48947738b..4d2603ceb 100644 --- a/teuthology-pull-requests/build/build +++ b/teuthology-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode # by default this will run all environments defined