2 # -*- mode:sh; tab-width:8; indent-tabs-mode:t -*-
4 # Ceph distributed storage system
6 # Copyright (C) 2014, 2015 Red Hat <contact@redhat.com>
8 # Author: Loic Dachary <loic@dachary.org>
10 # This library is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public
12 # License as published by the Free Software Foundation; either
13 # version 2.1 of the License, or (at your option) any later version.
16 DIR=/tmp/install-deps.$$
17 trap "rm -fr $DIR" EXIT
19 if test $(id -u) != 0 ; then
22 export LC_ALL=C # the following is vulnerable to i18n
26 function munge_ceph_spec_in {
29 local for_make_check=$1
32 sed -e 's/@//g' < ceph.spec.in > $OUTFILE
33 # http://rpm.org/user_doc/conditional_builds.html
34 if $with_seastar; then
35 sed -i -e 's/%bcond_with seastar/%bcond_without seastar/g' $OUTFILE
37 if $for_make_check; then
38 sed -i -e 's/%bcond_with make_check/%bcond_without make_check/g' $OUTFILE
42 function munge_debian_control {
47 local for_make_check=$1
52 control="/tmp/control.$$"
53 grep -v babeltrace debian/control > $control
56 if $with_seastar; then
57 sed -i -e 's/^# Crimson[[:space:]]//g' $control
59 if $for_make_check; then
60 sed -i 's/^# Make-Check[[:space:]]/ /g' $control
65 function ensure_decent_gcc_on_ubuntu {
66 # point gcc to the one offered by g++-7 if the used one is not
68 local old=$(gcc -dumpfullversion -dumpversion)
71 if dpkg --compare-versions $old ge ${new}.0; then
75 if [ ! -f /usr/bin/g++-${new} ]; then
76 $SUDO tee /etc/apt/sources.list.d/ubuntu-toolchain-r.list <<EOF
77 deb [lang=none] http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
78 deb [arch=amd64 lang=none] http://mirror.nullivex.com/ppa/ubuntu-toolchain-r-test $codename main
79 deb [arch=amd64 lang=none] http://deb.rug.nl/ppa/mirror/ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu $codename main
81 # import PPA's signing key into APT's keyring
82 cat << ENDOFKEY | $SUDO apt-key add -
83 -----BEGIN PGP PUBLIC KEY BLOCK-----
85 Comment: Hostname: keyserver.ubuntu.com
87 mI0ESuBvRwEEAMi4cDba7xlKaaoXjO1n1HX8RKrkW+HEIl79nSOSJyvzysajs7zUow/OzCQp
88 9NswqrDmNuH1+lPTTRNAGtK8r2ouq2rnXT1mTl23dpgHZ9spseR73s4ZBGw/ag4bpU5dNUSt
89 vfmHhIjVCuiSpNn7cyy1JSSvSs3N2mxteKjXLBf7ABEBAAG0GkxhdW5jaHBhZCBUb29sY2hh
90 aW4gYnVpbGRziLYEEwECACAFAkrgb0cCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRAe
91 k3eiup7yfzGKA/4xzUqNACSlB+k+DxFFHqkwKa/ziFiAlkLQyyhm+iqz80htRZr7Ls/ZRYZl
92 0aSU56/hLe0V+TviJ1s8qdN2lamkKdXIAFfavA04nOnTzyIBJ82EAUT3Nh45skMxo4z4iZMN
93 msyaQpNl/m/lNtOLhR64v5ZybofB2EWkMxUzX8D/FQ==
95 -----END PGP PUBLIC KEY BLOCK-----
97 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y || true
98 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y g++-${new}
109 $SUDO update-alternatives --remove-all gcc || true
110 $SUDO update-alternatives \
111 --install /usr/bin/gcc gcc /usr/bin/gcc-${new} 20 \
112 --slave /usr/bin/g++ g++ /usr/bin/g++-${new}
114 if [ -f /usr/bin/g++-${old} ]; then
115 $SUDO update-alternatives \
116 --install /usr/bin/gcc gcc /usr/bin/gcc-${old} 10 \
117 --slave /usr/bin/g++ g++ /usr/bin/g++-${old}
120 $SUDO update-alternatives --auto gcc
122 # cmake uses the latter by default
123 $SUDO ln -nsf /usr/bin/gcc /usr/bin/${ARCH}-linux-gnu-gcc
124 $SUDO ln -nsf /usr/bin/g++ /usr/bin/${ARCH}-linux-gnu-g++
127 function ensure_python3_sphinx_on_ubuntu {
128 local sphinx_command=/usr/bin/sphinx-build
129 # python-sphinx points $sphinx_command to
130 # ../share/sphinx/scripts/python2/sphinx-build when it's installed
131 # let's "correct" this
132 if test -e $sphinx_command && head -n1 $sphinx_command | grep -q python$; then
133 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove python-sphinx
137 function install_pkg_on_ubuntu {
148 if [ $force = "force" ]; then
152 if ! apt -qq list $pkg 2>/dev/null | grep -q installed; then
153 missing_pkgs+=" $pkg"
157 if test -n "$missing_pkgs"; then
158 local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
159 $SUDO curl --silent --location $shaman_url --output /etc/apt/sources.list.d/$project.list
160 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true
161 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs
165 function install_boost_on_ubuntu {
167 if apt -qq list ceph-libboost1.72-dev 2>/dev/null | grep -q installed; then
168 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove 'ceph-libboost.*1.72.*'
169 $SUDO rm /etc/apt/sources.list.d/ceph-libboost1.72.list
171 local project=libboost
173 local sha1=7aba8a1882670522ee1d1ee1bba0ea170b292dec
174 install_pkg_on_ubuntu \
179 ceph-libboost-atomic$ver-dev \
180 ceph-libboost-chrono$ver-dev \
181 ceph-libboost-container$ver-dev \
182 ceph-libboost-context$ver-dev \
183 ceph-libboost-coroutine$ver-dev \
184 ceph-libboost-date-time$ver-dev \
185 ceph-libboost-filesystem$ver-dev \
186 ceph-libboost-iostreams$ver-dev \
187 ceph-libboost-program-options$ver-dev \
188 ceph-libboost-python$ver-dev \
189 ceph-libboost-random$ver-dev \
190 ceph-libboost-regex$ver-dev \
191 ceph-libboost-system$ver-dev \
192 ceph-libboost-test$ver-dev \
193 ceph-libboost-thread$ver-dev \
194 ceph-libboost-timer$ver-dev
197 function version_lt {
198 test $1 != $(echo -e "$1\n$2" | sort -rV | head -n 1)
205 elif [ $FOR_MAKE_CHECK ]; then
211 if [ x$(uname)x = xFreeBSDx ]; then
212 $SUDO pkg install -yq \
221 devel/boost-python-libs \
226 devel/google-perftools \
228 devel/py-virtualenv \
230 net/openldap24-client \
234 misc/e2fsprogs-libuuid \
241 textproc/xmlstarlet \
252 devel/py-prettytable \
260 security/oath-toolkit \
262 sysutils/fusefs-libs \
264 # Now use pip to install some extra python modules
269 [ $WITH_SEASTAR ] && with_seastar=true || with_seastar=false
270 source /etc/os-release
272 debian|ubuntu|devuan|elementary)
273 echo "Using apt-get to install dependencies"
274 $SUDO apt-get install -y devscripts equivs
275 $SUDO apt-get install -y dpkg-dev
276 ensure_python3_sphinx_on_ubuntu
279 ensure_decent_gcc_on_ubuntu 9 bionic
280 [ ! $NO_BOOST_PKGS ] && install_boost_on_ubuntu bionic
283 [ ! $NO_BOOST_PKGS ] && apt-get install -y libboost1.67-all-dev
286 $SUDO apt-get install -y gcc
289 if ! test -r debian/control ; then
290 echo debian/control is not a readable file
296 control=$(munge_debian_control "$VERSION" "$with_seastar" "$for_make_check" "debian/control")
299 backports="-t $codename-backports"
303 # make a metapackage that expresses the build dependencies,
304 # install it, rm the .deb; then uninstall the package as its
306 $SUDO env DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool="apt-get -y --no-install-recommends $backports" $control || exit 1
307 $SUDO env DEBIAN_FRONTEND=noninteractive apt-get -y remove ceph-build-deps
308 if [ "$control" != "debian/control" ] ; then rm $control; fi
310 centos|fedora|rhel|ol|virtuozzo)
311 builddepcmd="dnf -y builddep --allowerasing"
312 echo "Using dnf to install dependencies"
315 $SUDO dnf install -y dnf-utils
317 centos|rhel|ol|virtuozzo)
318 MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)"
319 $SUDO dnf install -y dnf-utils
320 rpm --quiet --query epel-release || \
321 $SUDO dnf -y install --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm
322 $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION
323 $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org*
324 if test $ID = centos -a $MAJOR_VERSION = 8 ; then
325 $SUDO dnf config-manager --set-enabled PowerTools
326 # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
327 $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
328 $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
329 elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then
330 $SUDO subscription-manager repos --enable "codeready-builder-for-rhel-8-${ARCH}-rpms"
331 $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
332 $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
334 $SUDO dnf copr enable -y tchaikov/gcc-toolset-9 centos-stream-x86_64
337 munge_ceph_spec_in $with_seastar $for_make_check $DIR/ceph.spec
338 # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
339 $SUDO dnf install -y python3-devel
340 $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out
341 [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
342 IGNORE_YUM_BUILDEP_ERRORS="ValueError: SELinux policy is not managed or store cannot be accessed."
343 sed "/$IGNORE_YUM_BUILDEP_ERRORS/d" $DIR/yum-builddep.out | grep -qi "error:" && exit 1
346 echo "Using zypper to install dependencies"
347 zypp_install="zypper --gpg-auto-import-keys --non-interactive install --no-recommends"
348 $SUDO $zypp_install systemd-rpm-macros rpm-build || exit 1
349 munge_ceph_spec_in $with_seastar $for_make_check $DIR/ceph.spec
350 $SUDO $zypp_install $(rpmspec -q --buildrequires $DIR/ceph.spec) || exit 1
353 echo "$ID is unknown, dependencies will have to be installed manually."
359 function populate_wheelhouse() {
363 # although pip comes with virtualenv, having a recent version
364 # of pip matters when it comes to using wheel packages
365 PIP_OPTS="--timeout 300 --exists-action i"
366 pip $PIP_OPTS $install \
367 'setuptools >= 0.8' 'pip >= 7.0' 'wheel >= 0.24' 'tox >= 2.9.1' || return 1
368 if test $# != 0 ; then
369 pip $PIP_OPTS $install $@ || return 1
373 function activate_virtualenv() {
375 local env_dir=$top_srcdir/install-deps-python3
377 if ! test -d $env_dir ; then
378 virtualenv --python=python3 ${env_dir}
379 . $env_dir/bin/activate
380 if ! populate_wheelhouse install ; then
385 . $env_dir/bin/activate
388 function preload_wheels_for_tox() {
393 local require_files=$(ls *requirements*.txt 2>/dev/null) || true
394 local constraint_files=$(ls *constraints*.txt 2>/dev/null) || true
395 local require=$(echo -n "$require_files" | sed -e 's/^/-r /')
396 local constraint=$(echo -n "$constraint_files" | sed -e 's/^/-c /')
397 local md5=wheelhouse/md5
398 if test "$require"; then
399 if ! test -f $md5 || ! md5sum -c $md5 > /dev/null; then
403 if test "$require" && ! test -d wheelhouse ; then
404 type python3 > /dev/null 2>&1 || continue
405 activate_virtualenv $top_srcdir || exit 1
406 populate_wheelhouse "wheel -w $wip_wheelhouse" $require $constraint || exit 1
407 mv $wip_wheelhouse wheelhouse
408 md5sum $require_files $constraint_files > $md5
413 # use pip cache if possible but do not store it outside of the source
415 # see https://pip.pypa.io/en/stable/reference/pip_install.html#caching
416 if $for_make_check; then
417 mkdir -p install-deps-cache
419 export XDG_CACHE_HOME=$top_srcdir/install-deps-cache
420 wip_wheelhouse=wheelhouse-wip
422 # preload python modules so that tox can run without network access
424 find . -name tox.ini | while read ini ; do
425 preload_wheels_for_tox $ini
427 rm -rf $top_srcdir/install-deps-python3
428 rm -rf $XDG_CACHE_HOME
429 type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false)