From: Kefu Chai Date: Wed, 3 Jan 2018 03:15:37 +0000 (+0800) Subject: scripts/build_utils.sh: add security repo X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=94d10053f11945e5a814e334057810a356d85213;p=ceph-build.git scripts/build_utils.sh: add security repo gcc-7 depends on a recent binutils which is offered by security repo Signed-off-by: Kefu Chai --- diff --git a/ceph-build/build/setup_pbuilder b/ceph-build/build/setup_pbuilder index 3ff7a7054..be1d9f033 100755 --- a/ceph-build/build/setup_pbuilder +++ b/ceph-build/build/setup_pbuilder @@ -12,92 +12,4 @@ if test -f /etc/redhat-release ; then exit 0 fi -basedir="/srv/debian-base" - -# Ensure that the basedir directory exists -sudo mkdir -p "$basedir" - -# This used to live in a *file* on /srv/ceph-build as -# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make -# sense to have a file that lives in /srv/ that we then concatenate to get its -# contents. what. -# By using $DIST we are narrowing down to updating only the distro image we -# need, unlike before where we updated everything on every server on every -# build. - -os="debian" -[ "$DIST" = "precise" ] && os="ubuntu" -[ "$DIST" = "saucy" ] && os="ubuntu" -[ "$DIST" = "trusty" ] && os="ubuntu" -[ "$DIST" = "xenial" ] && os="ubuntu" - -if [ $os = "debian" ]; then - mirror="http://www.gtlib.gatech.edu/pub/debian" - # this assumes that newer Debian releases are being added to - # /etc/apt/trusted.gpg that is also the default location for Ubuntu trusted - # keys. The slave should ensure that the needed keys are added accordingly - # to this location. - debootstrapopts='DEBOOTSTRAPOPTS=( "--keyring" "/etc/apt/trusted.gpg" )' - components='COMPONENTS="main contrib"' -elif [ "$ARCH" = "arm64" ]; then - mirror="http://ports.ubuntu.com/ubuntu-ports" - debootstrapopts="" - components='COMPONENTS="main universe"' -else - mirror="http://us.archive.ubuntu.com/ubuntu" - debootstrapopts="" - components='COMPONENTS="main universe"' -fi - -# ensure that the tgz is valid, otherwise remove it so that it can be recreated -# again -pbuild_tar="$basedir/$DIST.tgz" -is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"` -file_size_kb=`du -k "$pbuild_tar" | cut -f1` - -if [ "$is_not_tar" = "1"] ; then - sudo rm -f "$pbuild_tar" -fi - -if [ $file_size_kb -lt 1 ]; then - sudo rm -f "$pbuild_tar" -fi - -# Ordinarily pbuilder only pulls packages from "main". ceph depends on -# packages like python-virtualenv which are in "universe". We have to configure -# pbuilder to look in "universe". Otherwise the build would fail with a message similar -# to: -# The following packages have unmet dependencies: -# pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package. -# Depends: xmlstarlet which is a virtual package. -# Unable to resolve dependencies! Giving up... -echo "$components" > ~/.pbuilderrc -echo "$debootstrapopts" >> ~/.pbuilderrc -# Newer pbuilder versions set $HOME to /nonexistent which breaks all kinds of -# things that rely on a proper (writable) path. Setting this to the system user's $HOME is not enough -# because of how pbuilder uses a chroot environment for builds, using a temporary directory here ensures -# that writes will be successful. -echo "BUILD_HOME=`mktemp -d`" >> ~/.pbuilderrc -# Some Ceph components will want to use cached wheels that may have older versions of buggy executables -# like: /usr/share/python-wheels/pip-8.1.1-py2.py3-none-any.whl which causes errors that are already fixed -# in newer versions. This ticket solves the specific issue in 8.1.1 (which vendors urllib3): -# https://github.com/shazow/urllib3/issues/567 -echo "USENETWORK=yes" >> ~/.pbuilderrc -setup_pbuilder_for_ppa >> ~/.pbuilderrc - -sudo pbuilder --clean - -if [ -e $basedir/$DIST.tgz ]; then - echo updating $DIST base.tgz - sudo pbuilder update \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" \ - --override-config -else - echo building $DIST base.tgz - sudo pbuilder create \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" -fi +setup_pbuilder use_gcc diff --git a/ceph-dev-build/build/setup_pbuilder b/ceph-dev-build/build/setup_pbuilder index 3ff7a7054..be1d9f033 100755 --- a/ceph-dev-build/build/setup_pbuilder +++ b/ceph-dev-build/build/setup_pbuilder @@ -12,92 +12,4 @@ if test -f /etc/redhat-release ; then exit 0 fi -basedir="/srv/debian-base" - -# Ensure that the basedir directory exists -sudo mkdir -p "$basedir" - -# This used to live in a *file* on /srv/ceph-build as -# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make -# sense to have a file that lives in /srv/ that we then concatenate to get its -# contents. what. -# By using $DIST we are narrowing down to updating only the distro image we -# need, unlike before where we updated everything on every server on every -# build. - -os="debian" -[ "$DIST" = "precise" ] && os="ubuntu" -[ "$DIST" = "saucy" ] && os="ubuntu" -[ "$DIST" = "trusty" ] && os="ubuntu" -[ "$DIST" = "xenial" ] && os="ubuntu" - -if [ $os = "debian" ]; then - mirror="http://www.gtlib.gatech.edu/pub/debian" - # this assumes that newer Debian releases are being added to - # /etc/apt/trusted.gpg that is also the default location for Ubuntu trusted - # keys. The slave should ensure that the needed keys are added accordingly - # to this location. - debootstrapopts='DEBOOTSTRAPOPTS=( "--keyring" "/etc/apt/trusted.gpg" )' - components='COMPONENTS="main contrib"' -elif [ "$ARCH" = "arm64" ]; then - mirror="http://ports.ubuntu.com/ubuntu-ports" - debootstrapopts="" - components='COMPONENTS="main universe"' -else - mirror="http://us.archive.ubuntu.com/ubuntu" - debootstrapopts="" - components='COMPONENTS="main universe"' -fi - -# ensure that the tgz is valid, otherwise remove it so that it can be recreated -# again -pbuild_tar="$basedir/$DIST.tgz" -is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"` -file_size_kb=`du -k "$pbuild_tar" | cut -f1` - -if [ "$is_not_tar" = "1"] ; then - sudo rm -f "$pbuild_tar" -fi - -if [ $file_size_kb -lt 1 ]; then - sudo rm -f "$pbuild_tar" -fi - -# Ordinarily pbuilder only pulls packages from "main". ceph depends on -# packages like python-virtualenv which are in "universe". We have to configure -# pbuilder to look in "universe". Otherwise the build would fail with a message similar -# to: -# The following packages have unmet dependencies: -# pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package. -# Depends: xmlstarlet which is a virtual package. -# Unable to resolve dependencies! Giving up... -echo "$components" > ~/.pbuilderrc -echo "$debootstrapopts" >> ~/.pbuilderrc -# Newer pbuilder versions set $HOME to /nonexistent which breaks all kinds of -# things that rely on a proper (writable) path. Setting this to the system user's $HOME is not enough -# because of how pbuilder uses a chroot environment for builds, using a temporary directory here ensures -# that writes will be successful. -echo "BUILD_HOME=`mktemp -d`" >> ~/.pbuilderrc -# Some Ceph components will want to use cached wheels that may have older versions of buggy executables -# like: /usr/share/python-wheels/pip-8.1.1-py2.py3-none-any.whl which causes errors that are already fixed -# in newer versions. This ticket solves the specific issue in 8.1.1 (which vendors urllib3): -# https://github.com/shazow/urllib3/issues/567 -echo "USENETWORK=yes" >> ~/.pbuilderrc -setup_pbuilder_for_ppa >> ~/.pbuilderrc - -sudo pbuilder --clean - -if [ -e $basedir/$DIST.tgz ]; then - echo updating $DIST base.tgz - sudo pbuilder update \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" \ - --override-config -else - echo building $DIST base.tgz - sudo pbuilder create \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" -fi +setup_pbuilder use_gcc diff --git a/ceph-dev-new-build/build/setup_pbuilder b/ceph-dev-new-build/build/setup_pbuilder index 3ff7a7054..be1d9f033 100755 --- a/ceph-dev-new-build/build/setup_pbuilder +++ b/ceph-dev-new-build/build/setup_pbuilder @@ -12,92 +12,4 @@ if test -f /etc/redhat-release ; then exit 0 fi -basedir="/srv/debian-base" - -# Ensure that the basedir directory exists -sudo mkdir -p "$basedir" - -# This used to live in a *file* on /srv/ceph-build as -# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make -# sense to have a file that lives in /srv/ that we then concatenate to get its -# contents. what. -# By using $DIST we are narrowing down to updating only the distro image we -# need, unlike before where we updated everything on every server on every -# build. - -os="debian" -[ "$DIST" = "precise" ] && os="ubuntu" -[ "$DIST" = "saucy" ] && os="ubuntu" -[ "$DIST" = "trusty" ] && os="ubuntu" -[ "$DIST" = "xenial" ] && os="ubuntu" - -if [ $os = "debian" ]; then - mirror="http://www.gtlib.gatech.edu/pub/debian" - # this assumes that newer Debian releases are being added to - # /etc/apt/trusted.gpg that is also the default location for Ubuntu trusted - # keys. The slave should ensure that the needed keys are added accordingly - # to this location. - debootstrapopts='DEBOOTSTRAPOPTS=( "--keyring" "/etc/apt/trusted.gpg" )' - components='COMPONENTS="main contrib"' -elif [ "$ARCH" = "arm64" ]; then - mirror="http://ports.ubuntu.com/ubuntu-ports" - debootstrapopts="" - components='COMPONENTS="main universe"' -else - mirror="http://us.archive.ubuntu.com/ubuntu" - debootstrapopts="" - components='COMPONENTS="main universe"' -fi - -# ensure that the tgz is valid, otherwise remove it so that it can be recreated -# again -pbuild_tar="$basedir/$DIST.tgz" -is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"` -file_size_kb=`du -k "$pbuild_tar" | cut -f1` - -if [ "$is_not_tar" = "1"] ; then - sudo rm -f "$pbuild_tar" -fi - -if [ $file_size_kb -lt 1 ]; then - sudo rm -f "$pbuild_tar" -fi - -# Ordinarily pbuilder only pulls packages from "main". ceph depends on -# packages like python-virtualenv which are in "universe". We have to configure -# pbuilder to look in "universe". Otherwise the build would fail with a message similar -# to: -# The following packages have unmet dependencies: -# pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package. -# Depends: xmlstarlet which is a virtual package. -# Unable to resolve dependencies! Giving up... -echo "$components" > ~/.pbuilderrc -echo "$debootstrapopts" >> ~/.pbuilderrc -# Newer pbuilder versions set $HOME to /nonexistent which breaks all kinds of -# things that rely on a proper (writable) path. Setting this to the system user's $HOME is not enough -# because of how pbuilder uses a chroot environment for builds, using a temporary directory here ensures -# that writes will be successful. -echo "BUILD_HOME=`mktemp -d`" >> ~/.pbuilderrc -# Some Ceph components will want to use cached wheels that may have older versions of buggy executables -# like: /usr/share/python-wheels/pip-8.1.1-py2.py3-none-any.whl which causes errors that are already fixed -# in newer versions. This ticket solves the specific issue in 8.1.1 (which vendors urllib3): -# https://github.com/shazow/urllib3/issues/567 -echo "USENETWORK=yes" >> ~/.pbuilderrc -setup_pbuilder_for_ppa >> ~/.pbuilderrc - -sudo pbuilder --clean - -if [ -e $basedir/$DIST.tgz ]; then - echo updating $DIST base.tgz - sudo pbuilder update \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" \ - --override-config -else - echo building $DIST base.tgz - sudo pbuilder create \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" -fi +setup_pbuilder use_gcc diff --git a/diamond-build/build/setup_pbuilder b/diamond-build/build/setup_pbuilder index 15f2f1657..c6c21c6cd 100755 --- a/diamond-build/build/setup_pbuilder +++ b/diamond-build/build/setup_pbuilder @@ -12,81 +12,4 @@ if test -f /etc/redhat-release ; then exit 0 fi -basedir="/srv/debian-base" - -# Ensure that the basedir directory exists -sudo mkdir -p "$basedir" - -# This used to live in a *file* on /srv/ceph-build as -# /srv/ceph-build/update_pbuilder.sh Now it lives here because it doesn't make -# sense to have a file that lives in /srv/ that we then concatenate to get its -# contents. what. -# By using $DIST we are narrowing down to updating only the distro image we -# need, unlike before where we updated everything on every server on every -# build. - -os="debian" -[ "$DIST" = "precise" ] && os="ubuntu" -[ "$DIST" = "saucy" ] && os="ubuntu" -[ "$DIST" = "trusty" ] && os="ubuntu" -[ "$DIST" = "xenial" ] && os="ubuntu" - -if [ $os = "debian" ]; then - mirror="http://www.gtlib.gatech.edu/pub/debian" - # this assumes that newer Debian releases are being added to - # /etc/apt/trusted.gpg that is also the default location for Ubuntu trusted - # keys. The slave should ensure that the needed keys are added accordingly - # to this location. - debootstrapopts='DEBOOTSTRAPOPTS=( "--keyring" "/etc/apt/trusted.gpg" )' - components='COMPONENTS="main contrib"' -elif [ "$ARCH" = "arm64" ]; then - mirror="http://ports.ubuntu.com/ubuntu-ports" - debootstrapopts="" - components='COMPONENTS="main universe"' -else - mirror="http://us.archive.ubuntu.com/ubuntu" - debootstrapopts="" - components='COMPONENTS="main universe"' -fi - -# ensure that the tgz is valid, otherwise remove it so that it can be recreated -# again -pbuild_tar="$basedir/$DIST.tgz" -is_not_tar=`python -c "exec 'try: import tarfile;print int(not int(tarfile.is_tarfile(\"$pbuild_tar\")))\nexcept IOError: print 1'"` -file_size_kb=`du -k "$pbuild_tar" | cut -f1` - -if $is_not_tar; then - sudo rm -f "$pbuild_tar" -fi - -if [ $file_size_kb -lt 1 ]; then - sudo rm -f "$pbuild_tar" -fi - -# Ordinarily pbuilder only pulls packages from "main". ceph depends on -# packages like python-virtualenv which are in "universe". We have to configure -# pbuilder to look in "universe". Otherwise the build would fail with a message similar -# to: -# The following packages have unmet dependencies: -# pbuilder-satisfydepends-dummy : Depends: python-virtualenv which is a virtual package. -# Depends: xmlstarlet which is a virtual package. -# Unable to resolve dependencies! Giving up... -echo "$components" > ~/.pbuilderrc -echo "$debootstrapopts" >> ~/.pbuilderrc - -sudo pbuilder --clean - -if [ -e $basedir/$DIST.tgz ]; then - echo updating $DIST base.tgz - sudo pbuilder update \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" \ - --override-config -else - echo building $DIST base.tgz - sudo pbuilder create \ - --basetgz $basedir/$DIST.tgz \ - --distribution $DIST \ - --mirror "$mirror" -fi +setup_pbuilder diff --git a/nfs-ganesha-stable/build/build_deb b/nfs-ganesha-stable/build/build_deb index edbba33d3..054d0c760 100644 --- a/nfs-ganesha-stable/build/build_deb +++ b/nfs-ganesha-stable/build/build_deb @@ -33,7 +33,7 @@ dch -v "$NTIRPC_VERSION-1${DIST}" "$NTIRPC_VERSION for download.ceph.com" sudo dpkg-buildpackage -S -us -uc -d ## Setup the pbuilder -setup_pbuilder +setup_pbuilder use_gcc PBUILDDIR="/srv/debian-base" ## Build with pbuilder diff --git a/nfs-ganesha/build/build_deb b/nfs-ganesha/build/build_deb index a0a94d2e5..827ff768b 100644 --- a/nfs-ganesha/build/build_deb +++ b/nfs-ganesha/build/build_deb @@ -41,7 +41,7 @@ NORMAL_ARCH=$ARCH create_build_status "started" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH ## Setup the pbuilder -setup_pbuilder +setup_pbuilder use_gcc cd $WORKSPACE/nfs-ganesha git submodule update --init || git submodule sync diff --git a/scripts/build_utils.sh b/scripts/build_utils.sh index ffe80a7ac..ee9b248ba 100644 --- a/scripts/build_utils.sh +++ b/scripts/build_utils.sh @@ -325,6 +325,8 @@ get_distro_and_target() { setup_pbuilder() { + local use_gcc=$1 + # This function will set the tgz images needed for pbuilder on a given host. It has # some hard-coded values like `/srv/debian-base` because it gets built every # time this file is executed - completely ephemeral. If a Debian host will use @@ -393,6 +395,19 @@ setup_pbuilder() { echo "$components" > ~/.pbuilderrc echo "$debootstrapopts" >> ~/.pbuilderrc + if [ -n "$use_gcc" ]; then + # Newer pbuilder versions set $HOME to /nonexistent which breaks all kinds of + # things that rely on a proper (writable) path. Setting this to the system user's $HOME is not enough + # because of how pbuilder uses a chroot environment for builds, using a temporary directory here ensures + # that writes will be successful. + echo "BUILD_HOME=`mktemp -d`" >> ~/.pbuilderrc + # Some Ceph components will want to use cached wheels that may have older versions of buggy executables + # like: /usr/share/python-wheels/pip-8.1.1-py2.py3-none-any.whl which causes errors that are already fixed + # in newer versions. This ticket solves the specific issue in 8.1.1 (which vendors urllib3): + # https://github.com/shazow/urllib3/issues/567 + echo "USENETWORK=yes" >> ~/.pbuilderrc + setup_pbuilder_for_ppa >> ~/.pbuilderrc + fi sudo pbuilder --clean if [ -e $basedir/$DIST.tgz ]; then @@ -480,6 +495,8 @@ setup_pbuilder_for_new_gcc() { # will fail. because the `build-essential` depends on a certain version # of gcc which is upgraded already by the one in test repo. cat > $hookdir/E05install-gcc-7 <> \ + /etc/apt/sources.list.d/ubuntu-toolchain-r.list echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $DIST main" >> \ /etc/apt/sources.list.d/ubuntu-toolchain-r.list echo "deb [arch=amd64] http://mirror.cs.uchicago.edu/ubuntu-toolchain-r $DIST main" >> \