From df61029beddb4a8a0368885cdd86ba6df7e3daad Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 15 May 2021 11:15:01 +0800 Subject: [PATCH] scripts/build_utils.sh: pass VENV to helper functions pass VENV to following functions explicitly: * teardown_vagrant_tests() * start_tox() * build_debs() * install_python_packages() * install_python_packages_no_binary() * pip_download() * check_binary_existence() prepare for the cleanup to avoid creating TEMPVENV everytime the script is included. Signed-off-by: Kefu Chai --- calamari-clients-build/build/build_rpm | 2 +- calamari-clients-build/build/setup | 2 +- calamari/build/setup | 2 +- ceph-ansible-docs-prs/build/build | 2 +- ceph-ansible-docs/build/build | 2 +- ceph-ansible-nightly/build/build | 2 +- ceph-ansible-nightly/build/teardown | 2 +- ceph-ansible-prs/build/build | 4 +- ceph-ansible-prs/build/teardown | 2 +- ceph-ansible-rpm/build/build | 2 +- ceph-ansible-scenario/build/build | 4 +- ceph-ansible-scenario/build/teardown | 4 +- ceph-build-pull-requests/build/build | 2 +- ceph-build/build/build_deb | 2 +- ceph-build/build/setup_deb | 4 +- ceph-build/build/setup_rpm | 4 +- ceph-container-nighlity/build/build | 2 +- ceph-container-prs/build/build | 2 +- ceph-deploy-build/build/build | 4 +- ceph-deploy-build/build/setup | 2 +- ceph-deploy-docs/build/build | 2 +- ceph-deploy-pull-requests/build/setup | 2 +- ceph-deploy-tag/build/build | 2 +- ceph-dev-build/build/build_deb | 2 +- ceph-dev-build/build/setup_deb | 4 +- ceph-dev-build/build/setup_osc | 4 +- ceph-dev-build/build/setup_rpm | 4 +- ceph-dev-new-build/build/build_deb | 2 +- ceph-dev-new-build/build/setup_deb | 4 +- ceph-dev-new-build/build/setup_mingw | 4 +- ceph-dev-new-build/build/setup_rpm | 4 +- ceph-iscsi-cli-flake8/build/build | 2 +- ceph-iscsi-cli/build/setup | 2 +- ceph-iscsi-config-flake8/build/build | 2 +- ceph-iscsi-config/build/setup | 2 +- ceph-iscsi-stable/build/setup | 2 +- ceph-iscsi-tools/build/setup | 2 +- ceph-iscsi-tox/build/build | 2 +- ceph-iscsi/build/setup | 2 +- ceph-medic-docs/build/build | 2 +- ceph-medic-pull-requests/build/build | 2 +- ceph-medic-release/build/build_rpm | 2 +- ceph-medic-rpm/build/build | 2 +- ceph-medic-tests/build/build | 2 +- ceph-pr-commits/build/build | 2 +- ceph-qa-suite-pull-requests/build/build | 2 +- ceph-release-rpm/build/build | 4 +- ceph-tag/build/build | 2 +- ceph-volume-ansible-prs/build/build | 2 +- ceph-volume-ansible-prs/build/teardown | 4 +- ceph-volume-nightly/build/build | 2 +- ceph-volume-nightly/build/teardown | 2 +- ceph-volume-pr/build/build | 2 +- ceph-volume-pr/build/teardown | 2 +- ceph-volume-scenario/build/build | 2 +- ceph-volume-scenario/build/teardown | 2 +- cephadm-ansible-prs/build/build | 2 +- cephadm-ansible-prs/build/teardown | 2 +- cephmetrics-pull-requests/setup/setup | 2 +- cephmetrics-release/build/setup | 2 +- cephmetrics/build/setup | 2 +- chacra-pull-requests/build/build | 2 +- configshell-fb/build/setup | 2 +- diamond-build/build/build_deb | 2 +- diamond-build/build/build_rpm | 2 +- diamond-build/build/setup | 2 +- jenkins-job-builder/build/build | 2 +- kernel/build/build_deb | 2 +- kernel/build/build_rpm | 2 +- kernel/build/setup | 2 +- merfi-pull-requests/build/build | 2 +- mita-deploy/build/build | 2 +- nfs-ganesha-stable/build/setup | 2 +- nfs-ganesha/build/setup | 2 +- paddles-pull-requests/build/build | 2 +- radosgw-agent-pull-requests/build/build | 2 +- radosgw-agent/build/build | 6 +- rtslib-fb/build/setup | 2 +- samba/build/setup | 2 +- scripts/build_utils.sh | 84 ++++++++++++++++--------- shaman-pull-requests/build/build | 2 +- tcmu-runner/build/setup | 2 +- teuthology-docs/build/build | 2 +- teuthology-pull-requests/build/build | 2 +- 84 files changed, 152 insertions(+), 130 deletions(-) diff --git a/calamari-clients-build/build/build_rpm b/calamari-clients-build/build/build_rpm index 6a8b3d4e..585563ba 100644 --- a/calamari-clients-build/build/build_rpm +++ b/calamari-clients-build/build/build_rpm @@ -62,7 +62,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/calamari-clients-${VERSION}-${RPM_R if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/calamari-clients-build/build/setup b/calamari-clients-build/build/setup index c97d4718..8856866b 100644 --- a/calamari-clients-build/build/setup +++ b/calamari-clients-build/build/setup @@ -54,7 +54,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/calamari/build/setup b/calamari/build/setup index e014d1ff..981ec854 100644 --- a/calamari/build/setup +++ b/calamari/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-ansible-docs-prs/build/build b/ceph-ansible-docs-prs/build/build index 50c9f920..ee4ec651 100644 --- a/ceph-ansible-docs-prs/build/build +++ b/ceph-ansible-docs-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd $WORKSPACE/docs/ $VENV/tox -rv diff --git a/ceph-ansible-docs/build/build b/ceph-ansible-docs/build/build index 2af4acb8..b060335b 100644 --- a/ceph-ansible-docs/build/build +++ b/ceph-ansible-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # trims leading slashes BRANCH=${GIT_BRANCH#*/} diff --git a/ceph-ansible-nightly/build/build b/ceph-ansible-nightly/build/build index 3514dd0f..6a96d997 100644 --- a/ceph-ansible-nightly/build/build +++ b/ceph-ansible-nightly/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-ansible-nightly/build/teardown b/ceph-ansible-nightly/build/teardown index 350248e5..02a05ccf 100644 --- a/ceph-ansible-nightly/build/teardown +++ b/ceph-ansible-nightly/build/teardown @@ -3,4 +3,4 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-ansible-prs/build/build b/ceph-ansible-prs/build/build index c6ab6985..a929dcdb 100644 --- a/ceph-ansible-prs/build/build +++ b/ceph-ansible-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate @@ -35,6 +35,6 @@ rm -rf $HOME/ansible/facts/* [[ "$ghprbTargetBranch" != stable-3.2 && "$SCENARIO" == shrink_osd_legacy ]] || [[ "$ghprbTargetBranch" =~ stable-3 && "$SCENARIO" == filestore_to_bluestore ]] || [[ "$ghprbTargetBranch" == stable-5.0 && "$DEPLOYMENT" == "non_container" && "$SCENARIO" == update ]] || -start_tox +start_tox $TEMPVENV # Update scenario on stable-5.0 must be enabled back once 15.2.8 is out diff --git a/ceph-ansible-prs/build/teardown b/ceph-ansible-prs/build/teardown index e4434d64..6e250f61 100644 --- a/ceph-ansible-prs/build/teardown +++ b/ceph-ansible-prs/build/teardown @@ -3,7 +3,7 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache rm -rf $HOME/ansible/facts/* diff --git a/ceph-ansible-rpm/build/build b/ceph-ansible-rpm/build/build index 08ecaf30..1a5ed0da 100644 --- a/ceph-ansible-rpm/build/build +++ b/ceph-ansible-rpm/build/build @@ -18,7 +18,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-ansible-scenario/build/build b/ceph-ansible-scenario/build/build index 20fe106f..aa99bd18 100644 --- a/ceph-ansible-scenario/build/build +++ b/ceph-ansible-scenario/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate @@ -17,4 +17,4 @@ update_vagrant_boxes # In the same logic, clean fact cache rm -rf $HOME/ansible/facts/* -start_tox \ No newline at end of file +start_tox diff --git a/ceph-ansible-scenario/build/teardown b/ceph-ansible-scenario/build/teardown index b9e5ebfa..6e250f61 100644 --- a/ceph-ansible-scenario/build/teardown +++ b/ceph-ansible-scenario/build/teardown @@ -3,7 +3,7 @@ cd $WORKSPACE/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache -rm -rf $HOME/ansible/facts/* \ No newline at end of file +rm -rf $HOME/ansible/facts/* diff --git a/ceph-build-pull-requests/build/build b/ceph-build-pull-requests/build/build index b51a923c..fb6a7159 100644 --- a/ceph-build-pull-requests/build/build +++ b/ceph-build-pull-requests/build/build @@ -6,7 +6,7 @@ set -e # must pin urllib3 to 1.22 because 1.23 is incompatible with requests, which # is used by jenkins-job-builder pkgs=( "ansible" "jenkins-job-builder>=3.5.0" "urllib3==1.22" "pyopenssl" "ndg-httpsclient" "pyasn1" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # Test every definition if available in the current repository and update the jobs diff --git a/ceph-build/build/build_deb b/ceph-build/build/build_deb index aa573c6e..312f379a 100644 --- a/ceph-build/build/build_deb +++ b/ceph-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs ${VENV} ${vers} ${debian_version} diff --git a/ceph-build/build/setup_deb b/ceph-build/build/setup_deb index 6332a498..6ed8e12e 100644 --- a/ceph-build/build/setup_deb +++ b/ceph-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" chacra_url=https://chacra.ceph.com/ make_chacractl_config @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-build/build/setup_rpm b/ceph-build/build/setup_rpm index 61edfdd6..f6b63fe1 100644 --- a/ceph-build/build/setup_rpm +++ b/ceph-build/build/setup_rpm @@ -49,7 +49,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -70,5 +70,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-container-nighlity/build/build b/ceph-container-nighlity/build/build index 589d3fc5..df9a6128 100644 --- a/ceph-container-nighlity/build/build +++ b/ceph-container-nighlity/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-container-prs/build/build b/ceph-container-prs/build/build index 589d3fc5..df9a6128 100644 --- a/ceph-container-prs/build/build +++ b/ceph-container-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/ceph-deploy-build/build/build b/ceph-deploy-build/build/build index e89ec9b9..5edbae20 100644 --- a/ceph-deploy-build/build/build +++ b/ceph-deploy-build/build/build @@ -46,7 +46,7 @@ then # this exists in scripts/build_utils.sh for ceph-deploy, binaries have # no architecture so we POST them to 'noarch' for rpms - check_binary_existence $chacra_endpoint/noarch + check_binary_existence $VENV $chacra_endpoint/noarch if [ ! -e setup.py ] ; then echo "Are we in the right directory" @@ -103,7 +103,7 @@ then chacra_endpoint="ceph-deploy/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/universal/all" # this exists in scripts/build_utils.sh - check_binary_existence $chacra_endpoint + check_binary_existence $VENV $chacra_endpoint if [ ! -d debian ] ; then echo "Are we in the right directory" diff --git a/ceph-deploy-build/build/setup b/ceph-deploy-build/build/setup index a1fa6a8c..1f71004f 100644 --- a/ceph-deploy-build/build/setup +++ b/ceph-deploy-build/build/setup @@ -22,7 +22,7 @@ if test -f /etc/redhat-release ; then fi pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-deploy-docs/build/build b/ceph-deploy-docs/build/build index 73e99b7a..1f80aafb 100644 --- a/ceph-deploy-docs/build/build +++ b/ceph-deploy-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the docs build with tox $VENV/tox -rv -e docs diff --git a/ceph-deploy-pull-requests/build/setup b/ceph-deploy-pull-requests/build/setup index d8dc9907..ced5ae48 100644 --- a/ceph-deploy-pull-requests/build/setup +++ b/ceph-deploy-pull-requests/build/setup @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying diff --git a/ceph-deploy-tag/build/build b/ceph-deploy-tag/build/build index 89bb0ad3..1ea302bf 100644 --- a/ceph-deploy-tag/build/build +++ b/ceph-deploy-tag/build/build @@ -9,7 +9,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to do all the tagging and release specifying # a local connection and 'localhost' as the host where to execute diff --git a/ceph-dev-build/build/build_deb b/ceph-dev-build/build/build_deb index aa573c6e..5881c478 100644 --- a/ceph-dev-build/build/build_deb +++ b/ceph-dev-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs $VENV ${vers} ${debian_version} diff --git a/ceph-dev-build/build/setup_deb b/ceph-dev-build/build/setup_deb index 98888e5a..9720f518 100644 --- a/ceph-dev-build/build/setup_deb +++ b/ceph-dev-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-build/build/setup_osc b/ceph-dev-build/build/setup_osc index 4b81c6e0..591d4034 100644 --- a/ceph-dev-build/build/setup_osc +++ b/ceph-dev-build/build/setup_osc @@ -78,7 +78,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -97,5 +97,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-build/build/setup_rpm b/ceph-dev-build/build/setup_rpm index 9146c300..639d7b62 100644 --- a/ceph-dev-build/build/setup_rpm +++ b/ceph-dev-build/build/setup_rpm @@ -51,7 +51,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -72,5 +72,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-new-build/build/build_deb b/ceph-dev-new-build/build/build_deb index aa573c6e..312f379a 100644 --- a/ceph-dev-new-build/build/build_deb +++ b/ceph-dev-new-build/build/build_deb @@ -1,4 +1,4 @@ #!/bin/bash set -ex -build_debs ${vers} ${debian_version} +build_debs ${VENV} ${vers} ${debian_version} diff --git a/ceph-dev-new-build/build/setup_deb b/ceph-dev-new-build/build/setup_deb index 98888e5a..9720f518 100644 --- a/ceph-dev-new-build/build/setup_deb +++ b/ceph-dev-new-build/build/setup_deb @@ -59,7 +59,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -75,5 +75,5 @@ chacra_check_url="${chacra_endpoint}/librados2_${bpvers}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-dev-new-build/build/setup_mingw b/ceph-dev-new-build/build/setup_mingw index 307c3ed6..08ba136f 100644 --- a/ceph-dev-new-build/build/setup_mingw +++ b/ceph-dev-new-build/build/setup_mingw @@ -45,7 +45,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -63,7 +63,7 @@ chacra_check_url="${chacra_binary_endpoint}/ceph.zip" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi # We need Ubuntu Focal to cross-compile Ceph for Windows. diff --git a/ceph-dev-new-build/build/setup_rpm b/ceph-dev-new-build/build/setup_rpm index 5e670079..aad8c5d7 100644 --- a/ceph-dev-new-build/build/setup_rpm +++ b/ceph-dev-new-build/build/setup_rpm @@ -49,7 +49,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "ceph" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` @@ -70,5 +70,5 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/flavors/${FLAVOR}/librados2-${vers} if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh # TODO if this exits we need to post to shaman a success - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi diff --git a/ceph-iscsi-cli-flake8/build/build b/ceph-iscsi-cli-flake8/build/build index 8c66e74f..70779665 100644 --- a/ceph-iscsi-cli-flake8/build/build +++ b/ceph-iscsi-cli-flake8/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi-cli" diff --git a/ceph-iscsi-cli/build/setup b/ceph-iscsi-cli/build/setup index 08578033..c36cffd2 100644 --- a/ceph-iscsi-cli/build/setup +++ b/ceph-iscsi-cli/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-config-flake8/build/build b/ceph-iscsi-config-flake8/build/build index 426bc9d4..2aa39eaa 100644 --- a/ceph-iscsi-config-flake8/build/build +++ b/ceph-iscsi-config-flake8/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi-config" diff --git a/ceph-iscsi-config/build/setup b/ceph-iscsi-config/build/setup index 08578033..c36cffd2 100644 --- a/ceph-iscsi-config/build/setup +++ b/ceph-iscsi-config/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-stable/build/setup b/ceph-iscsi-stable/build/setup index 3989aced..746f0f1d 100644 --- a/ceph-iscsi-stable/build/setup +++ b/ceph-iscsi-stable/build/setup @@ -45,7 +45,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" chacra_url="https://chacra.ceph.com/" # create the .chacractl config file using global variables diff --git a/ceph-iscsi-tools/build/setup b/ceph-iscsi-tools/build/setup index 08578033..c36cffd2 100644 --- a/ceph-iscsi-tools/build/setup +++ b/ceph-iscsi-tools/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-iscsi-tox/build/build b/ceph-iscsi-tox/build/build index 8a97583d..e5b00a24 100644 --- a/ceph-iscsi-tox/build/build +++ b/ceph-iscsi-tox/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/ceph-iscsi" diff --git a/ceph-iscsi/build/setup b/ceph-iscsi/build/setup index 08578033..c36cffd2 100644 --- a/ceph-iscsi/build/setup +++ b/ceph-iscsi/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-medic-docs/build/build b/ceph-medic-docs/build/build index 5f99f47c..003ddbbc 100644 --- a/ceph-medic-docs/build/build +++ b/ceph-medic-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # trims leading slashes BRANCH=`branch_slash_filter ${GIT_BRANCH}` diff --git a/ceph-medic-pull-requests/build/build b/ceph-medic-pull-requests/build/build index ec6f1273..c5385f19 100644 --- a/ceph-medic-pull-requests/build/build +++ b/ceph-medic-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" sudo yum install -y epel-release sudo yum --enablerepo epel install -y python36 diff --git a/ceph-medic-release/build/build_rpm b/ceph-medic-release/build/build_rpm index 125bb4fa..cef722bc 100644 --- a/ceph-medic-release/build/build_rpm +++ b/ceph-medic-release/build/build_rpm @@ -16,7 +16,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" make_chacractl_config diff --git a/ceph-medic-rpm/build/build b/ceph-medic-rpm/build/build index 08ecaf30..1a5ed0da 100644 --- a/ceph-medic-rpm/build/build +++ b/ceph-medic-rpm/build/build @@ -18,7 +18,7 @@ make rpm || ( tail -n +1 {root,build}.log && exit 1 ) # Chacra time pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/ceph-medic-tests/build/build b/ceph-medic-tests/build/build index 2ba71d3e..3a745370 100644 --- a/ceph-medic-tests/build/build +++ b/ceph-medic-tests/build/build @@ -1,6 +1,6 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" WORKDIR=$(mktemp -td tox.XXXXXXXXXX) diff --git a/ceph-pr-commits/build/build b/ceph-pr-commits/build/build index 4c134b4d..0f4ac0bd 100644 --- a/ceph-pr-commits/build/build +++ b/ceph-pr-commits/build/build @@ -17,7 +17,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "pytest" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE" timeout 600 $VENV/py.test -m "${pytest_mark}" -vs --junitxml="$WORKSPACE/report.xml" "$WORKSPACE/ceph-build/ceph-pr-commits/build/test_commits.py" diff --git a/ceph-qa-suite-pull-requests/build/build b/ceph-qa-suite-pull-requests/build/build index d1bd1238..08b39bbd 100644 --- a/ceph-qa-suite-pull-requests/build/build +++ b/ceph-qa-suite-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode diff --git a/ceph-release-rpm/build/build b/ceph-release-rpm/build/build index ec8b5ab8..aed3cd23 100644 --- a/ceph-release-rpm/build/build +++ b/ceph-release-rpm/build/build @@ -13,7 +13,7 @@ rm -rf dist rm -rf RPMBUILD pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -37,7 +37,7 @@ if [ "$target" = "sles11sp2" ] ; then fi echo "Target directory is: $target" -check_binary_existence $chacra_baseurl/noarch +check_binary_existence $VENV $chacra_baseurl/noarch # setup rpm build area mkdir -p RPMBUILD/{BUILD,RPMS,SOURCES,SPECS,SRPMS} diff --git a/ceph-tag/build/build b/ceph-tag/build/build index 4fabef2d..d97a3e70 100644 --- a/ceph-tag/build/build +++ b/ceph-tag/build/build @@ -9,7 +9,7 @@ fi # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to do all the tagging and release specifying # a local connection and 'localhost' as the host where to execute diff --git a/ceph-volume-ansible-prs/build/build b/ceph-volume-ansible-prs/build/build index d8d3e050..17c9ffa2 100644 --- a/ceph-volume-ansible-prs/build/build +++ b/ceph-volume-ansible-prs/build/build @@ -10,7 +10,7 @@ github_status_setup # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="pending" $VENV/github-status create diff --git a/ceph-volume-ansible-prs/build/teardown b/ceph-volume-ansible-prs/build/teardown index 07ed0e32..6b92cb3e 100644 --- a/ceph-volume-ansible-prs/build/teardown +++ b/ceph-volume-ansible-prs/build/teardown @@ -4,11 +4,11 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="failure" $VENV/github-status create cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-volume-nightly/build/build b/ceph-volume-nightly/build/build index f4f05480..8f1a68f9 100644 --- a/ceph-volume-nightly/build/build +++ b/ceph-volume-nightly/build/build @@ -4,7 +4,7 @@ WORKDIR=$(mktemp -td tox.XXXXXXXXXX) # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" prune_stale_vagrant_vms $WORKSPACE/../**/tests delete_libvirt_vms diff --git a/ceph-volume-nightly/build/teardown b/ceph-volume-nightly/build/teardown index d4518338..ad7de042 100644 --- a/ceph-volume-nightly/build/teardown +++ b/ceph-volume-nightly/build/teardown @@ -8,4 +8,4 @@ cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional scenarios=$(find . | grep vagrant_ssh_config | xargs -r dirname) # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/ceph-volume-pr/build/build b/ceph-volume-pr/build/build index 83833a7a..8ee52a8e 100644 --- a/ceph-volume-pr/build/build +++ b/ceph-volume-pr/build/build @@ -8,7 +8,7 @@ github_status_setup # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" "github-status>0.0.3") -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd src/ceph-volume diff --git a/ceph-volume-pr/build/teardown b/ceph-volume-pr/build/teardown index 057f0e9f..ab07894a 100644 --- a/ceph-volume-pr/build/teardown +++ b/ceph-volume-pr/build/teardown @@ -4,6 +4,6 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "github-status>0.0.3" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" GITHUB_STATUS_STATE="failure" $VENV/github-status create diff --git a/ceph-volume-scenario/build/build b/ceph-volume-scenario/build/build index 2bac2e59..2023af31 100644 --- a/ceph-volume-scenario/build/build +++ b/ceph-volume-scenario/build/build @@ -4,7 +4,7 @@ WORKDIR=$(mktemp -td tox.XXXXXXXXXX) # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" delete_libvirt_vms diff --git a/ceph-volume-scenario/build/teardown b/ceph-volume-scenario/build/teardown index a5c26da8..f03694a5 100644 --- a/ceph-volume-scenario/build/teardown +++ b/ceph-volume-scenario/build/teardown @@ -3,4 +3,4 @@ cd $WORKSPACE/src/ceph-volume/ceph_volume/tests/functional # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV diff --git a/cephadm-ansible-prs/build/build b/cephadm-ansible-prs/build/build index a2edde82..bd5afe3d 100644 --- a/cephadm-ansible-prs/build/build +++ b/cephadm-ansible-prs/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # XXX this might not be needed source $VENV/activate diff --git a/cephadm-ansible-prs/build/teardown b/cephadm-ansible-prs/build/teardown index 9ce6137f..c4f2c3f8 100644 --- a/cephadm-ansible-prs/build/teardown +++ b/cephadm-ansible-prs/build/teardown @@ -3,7 +3,7 @@ cd "${WORKSPACE}"/tests # the method exists in scripts/build_utils.sh -teardown_vagrant_tests +teardown_vagrant_tests $VENV # clean fact cache rm -rf "${HOME}"/ansible/facts/* diff --git a/cephmetrics-pull-requests/setup/setup b/cephmetrics-pull-requests/setup/setup index e7e86160..420fbefc 100644 --- a/cephmetrics-pull-requests/setup/setup +++ b/cephmetrics-pull-requests/setup/setup @@ -13,4 +13,4 @@ elif test -f /etc/debian_version ; then fi pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" diff --git a/cephmetrics-release/build/setup b/cephmetrics-release/build/setup index e4477e08..576e7fb9 100644 --- a/cephmetrics-release/build/setup +++ b/cephmetrics-release/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config diff --git a/cephmetrics/build/setup b/cephmetrics/build/setup index 04867736..14fec3fc 100644 --- a/cephmetrics/build/setup +++ b/cephmetrics/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/chacra-pull-requests/build/build b/chacra-pull-requests/build/build index 6fc7c592..44fae5c1 100644 --- a/chacra-pull-requests/build/build +++ b/chacra-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying # a local connection and 'localhost' as the host where to execute diff --git a/configshell-fb/build/setup b/configshell-fb/build/setup index 08578033..c36cffd2 100644 --- a/configshell-fb/build/setup +++ b/configshell-fb/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/diamond-build/build/build_deb b/diamond-build/build/build_deb index e8ef7355..103644ac 100644 --- a/diamond-build/build/build_deb +++ b/diamond-build/build/build_deb @@ -33,7 +33,7 @@ chacra_check_url="${chacra_endpoint}/diamond_${BPVER}_${ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/diamond-build/build/build_rpm b/diamond-build/build/build_rpm index c63a62a8..4edc8e78 100644 --- a/diamond-build/build/build_rpm +++ b/diamond-build/build/build_rpm @@ -63,7 +63,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/diamond-${VERSION}-0.${DIST}.${ARCH if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/diamond-build/build/setup b/diamond-build/build/setup index c97d4718..8856866b 100644 --- a/diamond-build/build/setup +++ b/diamond-build/build/setup @@ -54,7 +54,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/jenkins-job-builder/build/build b/jenkins-job-builder/build/build index 8683e045..bbc44b7e 100644 --- a/jenkins-job-builder/build/build +++ b/jenkins-job-builder/build/build @@ -10,7 +10,7 @@ set -euxo pipefail # the following two methods exist in scripts/build_utils.sh pkgs=( "jenkins-job-builder>=3.5.0" ) -install_python_packages "pkgs[@]" latest +install_python_packages $TEMPVENV "pkgs[@]" latest # Wipe out JJB's cache if $FORCE is set. [ "$FORCE" = true ] && rm -rf "$HOME/.cache/jenkins_jobs/" diff --git a/kernel/build/build_deb b/kernel/build/build_deb index aff31ee4..cdd5ddb9 100644 --- a/kernel/build/build_deb +++ b/kernel/build/build_deb @@ -28,7 +28,7 @@ chacra_check_url="${chacra_endpoint}/kernel_${BPVER}_${DEB_ARCH}.deb" if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/kernel/build/build_rpm b/kernel/build/build_rpm index 0699c9d3..366699fb 100644 --- a/kernel/build/build_rpm +++ b/kernel/build/build_rpm @@ -55,7 +55,7 @@ chacra_check_url="${chacra_endpoint}/${ARCH}/kernel-${VERSION}-${RPM_RELEASE}.${ if [ "$THROWAWAY" = false ] ; then # this exists in scripts/build_utils.sh - check_binary_existence $chacra_check_url + check_binary_existence $VENV $chacra_check_url fi HOST=$(hostname --short) diff --git a/kernel/build/setup b/kernel/build/setup index 13570b77..b95471e3 100644 --- a/kernel/build/setup +++ b/kernel/build/setup @@ -68,7 +68,7 @@ CentOS|Fedora|SUSE*|RedHatEnterpriseServer) esac pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/merfi-pull-requests/build/build b/merfi-pull-requests/build/build index 48947738..4d2603ce 100644 --- a/merfi-pull-requests/build/build +++ b/merfi-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode # by default this will run all environments defined diff --git a/mita-deploy/build/build b/mita-deploy/build/build index 82ef308e..83b0ee17 100644 --- a/mita-deploy/build/build +++ b/mita-deploy/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" cd "$WORKSPACE/deploy/playbooks/" $VENV/ansible-playbook -i "localhost," -c local local_deploy.yml --extra-vars="branch=$BRANCH jenkins_prado_token=$JENKINS_PRADO_TOKEN prado_token=$PRADO_TOKEN" diff --git a/nfs-ganesha-stable/build/setup b/nfs-ganesha-stable/build/setup index 43c019ed..58c6f085 100644 --- a/nfs-ganesha-stable/build/setup +++ b/nfs-ganesha-stable/build/setup @@ -45,7 +45,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" NFS_GANESHA_BRANCH=`branch_slash_filter $NFS_GANESHA_BRANCH` CEPH_BRANCH=`branch_slash_filter $CEPH_BRANCH` diff --git a/nfs-ganesha/build/setup b/nfs-ganesha/build/setup index 4146be77..57c1408c 100644 --- a/nfs-ganesha/build/setup +++ b/nfs-ganesha/build/setup @@ -42,7 +42,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" NFS_GANESHA_BRANCH=`branch_slash_filter $NFS_GANESHA_BRANCH` CEPH_BRANCH=`branch_slash_filter $CEPH_BRANCH` diff --git a/paddles-pull-requests/build/build b/paddles-pull-requests/build/build index ec83f09e..9f0922e6 100644 --- a/paddles-pull-requests/build/build +++ b/paddles-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" sudo yum install -y epel-release sudo yum --enablerepo epel install -y python36 diff --git a/radosgw-agent-pull-requests/build/build b/radosgw-agent-pull-requests/build/build index 7ac19b72..7ff22019 100644 --- a/radosgw-agent-pull-requests/build/build +++ b/radosgw-agent-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying diff --git a/radosgw-agent/build/build b/radosgw-agent/build/build index e9b935cb..9be36763 100644 --- a/radosgw-agent/build/build +++ b/radosgw-agent/build/build @@ -15,7 +15,7 @@ ls -l # the following two methods exist in scripts/build_utils.sh pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the .chacractl config file using global variables make_chacractl_config @@ -37,7 +37,7 @@ if [[ -f /etc/redhat-release || -f /usr/bin/zypper ]] ; then chacra_endpoint="radosgw-agent/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/${DISTRO_VERSION}" - check_binary_existence $chacra_endpoint/noarch + check_binary_existence $VENV $chacra_endpoint/noarch suse=$(uname -n | grep -ic -e suse -e sles || true) if [ $suse -gt 0 ] @@ -75,7 +75,7 @@ else chacra_endpoint="radosgw-agent/${chacra_ref}/${GIT_COMMIT}/${DISTRO}/${DEB_BUILD}/noarch" - check_binary_existence $chacra_endpoint + check_binary_existence $VENV $chacra_endpoint dpkg-source -b . # we no longer sign the .dsc or .changes files (done by default with diff --git a/rtslib-fb/build/setup b/rtslib-fb/build/setup index 08578033..c36cffd2 100644 --- a/rtslib-fb/build/setup +++ b/rtslib-fb/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/samba/build/setup b/samba/build/setup index 426b73ff..d7956618 100644 --- a/samba/build/setup +++ b/samba/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" SAMBA_BRANCH=$(branch_slash_filter $SAMBA_BRANCH) CEPH_BRANCH=$(branch_slash_filter $CEPH_BRANCH) diff --git a/scripts/build_utils.sh b/scripts/build_utils.sh index 7ef1cb93..a251c7dc 100644 --- a/scripts/build_utils.sh +++ b/scripts/build_utils.sh @@ -80,12 +80,14 @@ branch_slash_filter() { } pip_download() { + local venv=$1 + shift local package=$1 shift local options=$@ - if ! $VENV/pip download $options --dest="$PIP_SDIST_INDEX" $package; then + if ! $venv/pip download $options --dest="$PIP_SDIST_INDEX" $package; then # pip <8.0.0 does not have "download" command - $VENV/pip install $options \ + $venv/pip install $options \ --upgrade --exists-action=i --cache-dir="$PIP_SDIST_INDEX" \ $package fi @@ -107,6 +109,9 @@ create_virtualenv () { } install_python_packages_no_binary () { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" # Use this function to create a virtualenv and install python packages # without compiling (or using wheels). Pass a list of package names. If # the virtualenv exists it will get re-used since this function can be used @@ -115,19 +120,19 @@ install_python_packages_no_binary () { # Usage (with pip==20.3.4 [the default]): # # to_install=( "ansible" "chacractl>=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" + # install_python_packages_no_binary $TEMPVENV "to_install[@]" # # Usage (with pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" "pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" latest + # install_python_packages_no_binary $TEMPVENV "to_install[@]" latest - create_virtualenv $TEMPVENV + create_virtualenv $venv_dir # Define and ensure the PIP cache PIP_SDIST_INDEX="$HOME/.cache/pip" @@ -139,51 +144,54 @@ install_python_packages_no_binary () { # Updated to 20.3.4 in March 2021 because 10.0.0 is just too old. if [ "$2" == "latest" ]; then echo "Ensuring latest pip is installed" - $VENV/pip install -U pip + $venv/pip install -U pip elif [[ -n $2 && "$2" != "latest" ]]; then echo "Installing $2" - $VENV/pip install "$2" + $venv/pip install "$2" else # This is the default for most jobs. # See ff01d2c5 and fea10f52 echo "Installing pip 20.3.4" - $VENV/pip install "pip==20.3.4" + $venv/pip install "pip==20.3.4" fi echo "Updating setuptools" - pip_download setuptools + pip_download $VENV setuptools pkgs=("${!1}") for package in ${pkgs[@]}; do echo $package # download packages to the local pip cache - pip_download $package --no-binary=:all: + pip_download $VENV $package --no-binary=:all: # install packages from the local pip cache, ignoring pypi - $VENV/pip install --no-binary=:all: --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package + $venv/pip install --no-binary=:all: --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package done } install_python_packages () { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" # Use this function to create a virtualenv and install # python packages. Pass a list of package names. # # Usage (with pip 20.3.4 [the default]): # # to_install=( "ansible" "chacractl>=0.0.21" ) - # install_python_packages "to_install[@]" + # install_python_packages $TEMPVENV "to_install[@]" # # Usage (with pip=0.0.21" ) - # install_python_packages_no_binary "to_install[@]" "pip=0.0.21" ) - # install_python_packages "to_install[@]" latest + # install_python_packages $TEMPVENV "to_install[@]" latest - create_virtualenv $TEMPVENV + create_virtualenv $venv_dir # Define and ensure the PIP cache PIP_SDIST_INDEX="$HOME/.cache/pip" @@ -195,27 +203,27 @@ install_python_packages () { # Updated to 20.3.4 in March 2021 because 10.0.0 is just too old. if [ "$2" == "latest" ]; then echo "Ensuring latest pip is installed" - $VENV/pip install -U pip + $venv/pip install -U pip elif [[ -n $2 && "$2" != "latest" ]]; then echo "Installing $2" - $VENV/pip install "$2" + $venv/pip install "$2" else # This is the default for most jobs. # See ff01d2c5 and fea10f52 echo "Installing pip 20.3.4" - $VENV/pip install "pip==20.3.4" + $venv/pip install "pip==20.3.4" fi echo "Updating setuptools" - pip_download setuptools + pip_download $venv setuptools pkgs=("${!1}") for package in ${pkgs[@]}; do echo $package # download packages to the local pip cache - pip_download $package + pip_download $venv $package # install packages from the local pip cache, ignoring pypi - $VENV/pip install --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package + $venv/pip install --upgrade --exists-action=i --find-links="file://$PIP_SDIST_INDEX" --no-index $package done } @@ -283,11 +291,14 @@ get_rpm_dist() { } check_binary_existence () { - url=$1 + local venv=$1 + shift + local url=$1 + shift # we have to use ! here so thet -e will ignore the error code for the command # because of this, the exit code is also reversed - ! $VENV/chacractl exists binaries/${url} ; exists=$? + ! $venv/chacractl exists binaries/${url} ; exists=$? # if the binary already exists in chacra, do not rebuild if [ $exists -eq 1 ] && [ "$FORCE" = false ] ; then @@ -843,6 +854,8 @@ ceph_build_args_from_flavor() { } build_debs() { + local venv=$1 + shift local vers=$1 shift local debian_version=$1 @@ -912,7 +925,7 @@ build_debs() { find release/$vers/ | \ egrep "*\.(changes|deb|ddeb|dsc|gz)$" | \ egrep -v "(Packages|Sources|Contents)" | \ - $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} + $venv/chacractl binary ${chacra_flags} create ${chacra_endpoint} # write json file with build info cat > $WORKSPACE/repo-extra.json << EOF { @@ -931,7 +944,7 @@ EOF -u $CHACRACTL_USER:$CHACRACTL_KEY \ ${chacra_url}repos/${chacra_repo_endpoint}/extra/ # start repo creation - $VENV/chacractl repo update ${chacra_repo_endpoint} + $venv/chacractl repo update ${chacra_repo_endpoint} echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_repo_endpoint}/ fi @@ -1047,6 +1060,10 @@ update_vagrant_boxes() { } start_tox() { + local venv_dir=$1 + shift + local venv="$venv_dir/bin" + # the $SCENARIO var is injected by the job template. It maps # to an actual, defined, tox environment while true; do @@ -1122,11 +1139,11 @@ start_tox() { ;; esac - for tox_env in $("$VENV"/tox -c "$TOX_INI_FILE" -l) + for tox_env in $("$venv"/tox -c "$TOX_INI_FILE" -l) do if [[ "$ENV_NAME" == "$tox_env" ]]; then # shellcheck disable=SC2116 - if ! eval "$(echo "${TOX_RUN_ENV[@]}")" "$VENV"/tox -c "$TOX_INI_FILE" --workdir="$TEMPVENV" -v -e="$ENV_NAME" -- --provider=libvirt; then + if ! eval "$(echo "${TOX_RUN_ENV[@]}")" "$venv"/tox -c "$TOX_INI_FILE" --workdir="$venv_dir" -v -e="$ENV_NAME" -- --provider=libvirt; then echo "ERROR: Job didn't complete successfully or got stuck for more than 3h." exit 1 fi @@ -1244,6 +1261,8 @@ EOF } collect_ceph_logs() { + local venv=$1 + shift # this is meant to be run in a testing scenario directory # with running vagrant vms. the ansible playbook will connect # to your test nodes and fetch any ceph logs that are present @@ -1257,15 +1276,18 @@ collect_ceph_logs() { write_collect_logs_playbook pkgs=( "ansible" ) - install_python_packages "pkgs[@]" + install_python_packages $TEMPVENV "pkgs[@]" export ANSIBLE_SSH_ARGS='-F ./vagrant_ssh_config' export ANSIBLE_STDOUT_CALLBACK='debug' - $VENV/ansible-playbook -vv -i hosts --limit $limit --extra-vars "archive_path=$WORKSPACE/logs" $WORKSPACE/collect-logs.yml || true + $venv/ansible-playbook -vv -i hosts --limit $limit --extra-vars "archive_path=$WORKSPACE/logs" $WORKSPACE/collect-logs.yml || true fi } teardown_vagrant_tests() { + local venv=$1 + shift + # collect ceph logs and teardown any running vagrant vms # this also cleans up any lingering livirt networks scenarios=$(find . | grep vagrant_ssh_config | xargs -r dirname) @@ -1273,7 +1295,7 @@ teardown_vagrant_tests() { for scenario in $scenarios; do cd $scenario # collect all ceph logs from all test nodes - collect_ceph_logs all + collect_ceph_logs $venv all vagrant destroy -f stat ./fetch > /dev/null 2>&1 && rm -rf ./fetch cd - diff --git a/shaman-pull-requests/build/build b/shaman-pull-requests/build/build index 71fa654c..85f1137d 100644 --- a/shaman-pull-requests/build/build +++ b/shaman-pull-requests/build/build @@ -2,7 +2,7 @@ # the following two methods exist in scripts/build_utils.sh pkgs=( "ansible" "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run ansible to get this current host to meet our requirements, specifying # a local connection and 'localhost' as the host where to execute diff --git a/tcmu-runner/build/setup b/tcmu-runner/build/setup index 08578033..c36cffd2 100644 --- a/tcmu-runner/build/setup +++ b/tcmu-runner/build/setup @@ -36,7 +36,7 @@ echo "*****" export LC_ALL=C # the following is vulnerable to i18n pkgs=( "chacractl>=0.0.21" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # ask shaman which chacra instance to use chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` diff --git a/teuthology-docs/build/build b/teuthology-docs/build/build index 381c06d6..77628b1a 100644 --- a/teuthology-docs/build/build +++ b/teuthology-docs/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # create the docs build with tox $VENV/tox -rv -e docs diff --git a/teuthology-pull-requests/build/build b/teuthology-pull-requests/build/build index 48947738..4d2603ce 100644 --- a/teuthology-pull-requests/build/build +++ b/teuthology-pull-requests/build/build @@ -4,7 +4,7 @@ set -ex # the following two methods exist in scripts/build_utils.sh pkgs=( "tox" ) -install_python_packages "pkgs[@]" +install_python_packages $TEMPVENV "pkgs[@]" # run tox by recreating the environment and in verbose mode # by default this will run all environments defined -- 2.39.5