From 085ee471479ac83d606b096923130d904deb36b3 Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Wed, 15 Feb 2017 16:30:44 -0500 Subject: [PATCH] nfs-ganesha: Debian builds, setup scripts renamed .debs building with pbuilder setup scripts now distro dependent remove $BRANCH variable various minor fixes Signed-off-by: Ali Maredia --- nfs-ganesha/build/build_deb | 90 +++++++++++-------- nfs-ganesha/build/build_rpm | 4 +- nfs-ganesha/build/{setup => setup_deb} | 5 ++ nfs-ganesha/build/setup_rpm | 58 ++++++++++++ .../config/definitions/nfs-ganesha.yml | 9 +- 5 files changed, 121 insertions(+), 45 deletions(-) rename nfs-ganesha/build/{setup => setup_deb} (91%) create mode 100644 nfs-ganesha/build/setup_rpm diff --git a/nfs-ganesha/build/build_deb b/nfs-ganesha/build/build_deb index 4630e2ab..f5dfc891 100644 --- a/nfs-ganesha/build/build_deb +++ b/nfs-ganesha/build/build_deb @@ -6,11 +6,16 @@ if test "$DISTRO" != "debian" -a "$DISTRO" != "ubuntu"; then exit 0 fi +REPO_URL="https://shaman.ceph.com/api/repos/ceph/$CEPH_BRANCH/$CEPH_SHA1/$DISTRO/$DIST/repo" +if `curl --fail -L $REPO_URL`; then + echo "Ceph debian lib repo exists in shaman" +else + echo "Ceph debian lib repo does NOT exist in shaman" + exit 1 +fi -# We need this for system, to build ganesha, and to enable FSAL_CEPH and -# FSAL_RGW in the .spec file when cmake command runs +# We need this for system and to run the cmake sudo apt-get update -sudo apt-get -y install cmake equivs libkrb5-dev bison flex quilt librgw-dev libcephfs-dev # Normalize variables across rpm/deb builds NORMAL_DISTRO=$DISTRO @@ -23,40 +28,42 @@ create_build_status "started" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSIO ## Setup the pbuilder setup_pbuilder +cd $WORKSPACE/nfs-ganesha +git submodule update --init || git submodule sync -## Build the source tarball +PACKAGE_MANAGER_VERSION="`git describe --long | sed 's/V//1'`-1${DIST}" -cd $WORKSPACE/nfs-ganesha +# Version is in format X.XdevX-X-SHA1 +VERSION=`git describe --long | sed -e 's/V//1; s/-//1; s/-//1'` -git submodule update --init || git submodule sync +rm -rf .git + +cd $WORKSPACE + +## Build the source tarball +NFS_GANESHA_ORIG_TAR_GZ="nfs-ganesha_${VERSION}.orig.tar.gz" +tar czf ${NFS_GANESHA_ORIG_TAR_GZ} nfs-ganesha/src + +# remove old version +rm -rf $WORKSPACE/nfs-ganesha -mkdir build -cd build +# unpack just the src +tar xzf ${NFS_GANESHA_ORIG_TAR_GZ} -# make source tarball -cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_GLUSTER=OFF -DUSE_FSAL_CEPH=ON -DUSE_FSAL_RGW=ON $WORKSPACE/nfs-ganesha/src && make dist || exit 1 +cd $WORKSPACE/nfs-ganesha -# move unpacked tarball code into nfs-ganesha-debian/src -NFS_GANESHA_SOURCE_DIR=`grep -om 1 'nfs-ganesha-[[:digit:]]\+\.[[:digit:]]\+-dev-[[:digit:]]\+-[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+-Source' $WORKSPACE/nfs-ganesha/build/CPackSourceConfig.cmake` -VERSION=`grep -om 1 '[[:digit:]]\+\.[[:digit:]]\+-dev-[[:digit:]]\+' $WORKSPACE/nfs-ganesha/build/CPackSourceConfig.cmake` -TAR_VERSION=`grep -om 1 '[[:digit:]]\+\.[[:digit:]]\+-dev' $WORKSPACE/nfs-ganesha/build/CPackSourceConfig.cmake` -NFS_GANESHA_ORIG_TAR_GZ="nfs-ganesha_${TAR_VERSION}.orig.tar.gz" -mkdir nfs-ganesha-${VERSION} -tar xzf ${NFS_GANESHA_SOURCE_DIR}.tar.gz -mv ${NFS_GANESHA_SOURCE_DIR} nfs-ganesha-${VERSION}/src -tar czf ${NFS_GANESHA_ORIG_TAR_GZ} nfs-ganesha-${VERSION}/src -mv $WORKSPACE/nfs-ganesha-debian/debian $WORKSPACE/nfs-ganesha/build/nfs-ganesha-${VERSION}/debian -cd nfs-ganesha-${VERSION} +# add debian directory next to src +mv $WORKSPACE/nfs-ganesha-debian/debian $WORKSPACE/nfs-ganesha/ ## Get some basic information about the system and the repository DEB_ARCH=$(dpkg-architecture -qDEB_BUILD_ARCH) ## Prepare the debian files # Bump the changelog -dch -v "$VERSION" "Dev release ($VERSION)" +dch -v "$VERSION-1${DIST}" "$VERSION for Shaman" -# Create .dsc and source tarball -sudo dpkg-buildpackage -S -us -uc -d +# Create .dsc and source tarball, we don't care about signing changes or source package +sudo dpkg-buildpackage -S -us -uc ## Build with pbuilder echo "Building debs" @@ -67,12 +74,26 @@ sudo pbuilder --clean mkdir -p $WORKSPACE/dist/deb -# use libcephfs and librgw from shaman -REPO_URL="https://shaman.ceph.com/api/repos/ceph/$CEPH_BRANCH/$CEPH_SHA1/$DISTRO/$DIST/repo" +# add missing packages and components to pbuilder +sudo pbuilder update \ + --basetgz $PBUILDDIR/$DIST.tgz \ + --distribution $DIST \ + --extrapackages "apt-transport-https apt-utils ca-certificates" \ + --components "main restricted universe multiverse" + +# add other mirror to pbuilder SHAMAN_MIRROR=`curl -L ${REPO_URL}` -echo "EXTRAPACKAGES=\"libcephfs-dev librgw-dev\"" >> ~/.pbuilderrc -echo "OTHERMIRROR=\"${SHAMAN_MIRROR}\"" >> ~/.pbuilderrc -cat ~/.pbuilderrc +sudo pbuilder update \ + --basetgz $PBUILDDIR/$DIST.tgz \ + --distribution $DIST \ + --othermirror "${SHAMAN_MIRROR}" \ + --override-config + +# use libcephfs and librgw from shaman +sudo pbuilder update \ + --basetgz $PBUILDDIR/$DIST.tgz \ + --distribution $DIST \ + --extrapackages "librados-dev libcephfs-dev librgw-dev" echo "Building debs for $DIST" sudo pbuilder build \ @@ -80,14 +101,11 @@ sudo pbuilder build \ --basetgz $PBUILDDIR/$DIST.tgz \ --buildresult $WORKSPACE/dist/deb/ \ --debbuildopts "-j`grep -c processor /proc/cpuinfo`" \ - $WORKSPACE/nfs-ganesha/build/nfs-ganesha_${VERSION}.dsc + $WORKSPACE/nfs-ganesha_${VERSION}-1${DIST}.dsc ## Upload the created RPMs to chacra chacra_endpoint="nfs-ganesha/${NFS_GANESHA_BRANCH}/${GIT_COMMIT}/${DISTRO}/${DIST}" -# TODO: change this -bpvers="foo" -#bpvers=`gen_debian_version $debian_version $DIST` [ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" @@ -95,13 +113,13 @@ bpvers="foo" if [ "$THROWAWAY" = false ] ; then # push binaries to chacra - find $WORKSPACE/dist/deb | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} + find $WORKSPACE/dist/deb | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH} # write json file with build info # version and package_manager version are needed for teuthology cat > $WORKSPACE/repo-extra.json << EOF { "version":"$VERSION", - "package_manager_version":"$bpvers", + "package_manager_version":"$PACKAGE_MANAGER_VERSION" "build_url":"$BUILD_URL", "root_build_cause":"$ROOT_BUILD_CAUSE", "node_name":"$NODE_NAME", @@ -114,7 +132,7 @@ EOF $VENV/chacractl repo update ${chacra_endpoint} fi -echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} +echo "Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}" # update shaman with the completed build status SHA1=${GIT_COMMIT} diff --git a/nfs-ganesha/build/build_rpm b/nfs-ganesha/build/build_rpm index e4ff1ac2..fd48515a 100644 --- a/nfs-ganesha/build/build_rpm +++ b/nfs-ganesha/build/build_rpm @@ -76,7 +76,7 @@ echo "\"\"\"" >> nfs-ganesha.cfg ## Build the binaries with mock echo "Building RPMs" -sudo mock -r nfs-ganesha.cfg --define "dist .el7" --resultdir=$WORKSPACE/dist/RPMS/ ${SRPM} || ( tail -n +1 $WORKSPACE/dist/RPMS/{root,build}.log && exit 1 ) +sudo mock --verbose -r nfs-ganesha.cfg --define "dist .el7" --resultdir=$WORKSPACE/dist/RPMS/ ${SRPM} || ( tail -n +1 $WORKSPACE/dist/RPMS/{root,build}.log && exit 1 ) VERSION=`grep -om 1 '[[:digit:]]\+\.[[:digit:]]\+-dev-[[:digit:]]\+' $WORKSPACE/nfs-ganesha/build/include/config.h` chacra_endpoint="nfs-ganesha/${NFS_GANESHA_BRANCH}/${GIT_COMMIT}/${DISTRO}/${RELEASE}" @@ -108,7 +108,7 @@ EOF $VENV/chacractl repo update ${chacra_endpoint} fi -echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} +echo "Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint}" # update shaman with the completed build status SHA1=${GIT_COMMIT} diff --git a/nfs-ganesha/build/setup b/nfs-ganesha/build/setup_deb similarity index 91% rename from nfs-ganesha/build/setup rename to nfs-ganesha/build/setup_deb index 78c61c20..160c8a37 100644 --- a/nfs-ganesha/build/setup +++ b/nfs-ganesha/build/setup_deb @@ -14,6 +14,11 @@ set -ex +# Only do actual work when we are a DEB distro +if test "$DISTRO" != "debian" -a "$DISTRO" != "ubuntu"; then + exit 0 +fi + # Make sure we execute at the top level directory before we do anything cd $WORKSPACE diff --git a/nfs-ganesha/build/setup_rpm b/nfs-ganesha/build/setup_rpm new file mode 100644 index 00000000..ef217fae --- /dev/null +++ b/nfs-ganesha/build/setup_rpm @@ -0,0 +1,58 @@ +#! /usr/bin/bash +# +# Ceph distributed storage system +# +# Copyright (C) 2016 Red Hat +# +# Author: Boris Ranto +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# + +set -ex + +# Only do actual work when we are an RPM distro +if test "$DISTRO" != "fedora" -a "$DISTRO" != "centos" -a "$DISTRO" != "rhel"; then + exit 0 +fi + +# Make sure we execute at the top level directory before we do anything +cd $WORKSPACE + +# This will set the DISTRO and MOCK_TARGET variables +get_distro_and_target + +# Perform a clean-up +cd $WORKSPACE/nfs-ganesha +git clean -fxd + +cd $WORKSPACE/nfs-ganesha-debian +git clean -fxd + +# Make sure the dist directory is clean +cd $WORKSPACE +rm -rf dist +mkdir -p dist + +# Print some basic system info +HOST=$(hostname --short) +echo "Building on $(hostname) with the following env" +echo "*****" +env +echo "*****" + +export LC_ALL=C # the following is vulnerable to i18n + +pkgs=( "chacractl>=0.0.4" ) +install_python_packages "pkgs[@]" + +NFS_GANESHA_BRANCH=`branch_slash_filter $NFS_GANESHA_BRANCH` +CEPH_BRANCH=`branch_slash_filter $CEPH_BRANCH` + +# ask shaman which chacra instance to use +chacra_url=`curl -f -u $SHAMAN_API_USER:$SHAMAN_API_KEY https://shaman.ceph.com/api/nodes/next/` +# create the .chacractl config file using global variables +make_chacractl_config $chacra_url diff --git a/nfs-ganesha/config/definitions/nfs-ganesha.yml b/nfs-ganesha/config/definitions/nfs-ganesha.yml index 98afac11..d7de8e49 100644 --- a/nfs-ganesha/config/definitions/nfs-ganesha.yml +++ b/nfs-ganesha/config/definitions/nfs-ganesha.yml @@ -37,11 +37,6 @@ description: "The git branch (or tag) to build" default: "next" - - string: - name: BRANCH - description: "The git branch that will be posted to shaman" - default: $NFS_GANESHA_BRANCH - - string: name: CEPH_SHA1 description: "The SHA1 of the ceph branch" @@ -125,13 +120,13 @@ If this is checked, then the binaries will be built and pushed to chacra even if - shell: !include-raw: - ../../../scripts/build_utils.sh - - ../../build/setup + - ../../build/setup_deb - ../../build/build_deb # rpm build scripts - shell: !include-raw: - ../../../scripts/build_utils.sh - - ../../build/setup + - ../../build/setup_rpm - ../../build/build_rpm publishers: -- 2.39.5