From b6bb8808c32533088852ec16bf313ae4120b3eee Mon Sep 17 00:00:00 2001 From: Ali Maredia Date: Tue, 24 Jan 2017 15:59:58 -0500 Subject: [PATCH] nfs-ganesha: Shaman related additions, various fixes enable "Build" section on shaman enable 'Extras' build info on shaman various other minor fixes Signed-off-by: Ali Maredia --- nfs-ganesha/build/build_deb | 57 +++++++++++--- nfs-ganesha/build/build_rpm | 78 +++++++++++++++---- .../config/definitions/nfs-ganesha.yml | 6 ++ 3 files changed, 115 insertions(+), 26 deletions(-) diff --git a/nfs-ganesha/build/build_deb b/nfs-ganesha/build/build_deb index 85fcb939..2356cbf9 100644 --- a/nfs-ganesha/build/build_deb +++ b/nfs-ganesha/build/build_deb @@ -8,15 +8,29 @@ fi ## Install any setup-time deps -# TODO -- upload install-deps.sh script that installs the setup time deps -# to upstream repo or update this section to install these packages +# TODO: add file to /etc/apt/sources.list.d/ to get repos for librgw-dev and libcephfs-dev and run sudo apt-get update # We need this for system, to build ganesha, and to enable FSAL_CEPH and # FSAL_RGW in the .spec file when cmake command runs -sudo apt-get install -y equivs cmake git libkrb5-dev bison flex librgw2-dev libcephfs-dev +sudo apt-get -y install " +cmake +equivs +libkrb5-dev +bison +flex +librgw-dev +libcephfs-dev +" + +# Normalize variables across rpm/deb builds +NORMAL_DISTRO=$DISTRO +NORMAL_DISTRO_VERSION=$DIST +NORMAL_ARCH=$ARCH + +# create build status in shaman +create_build_status "started" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH ## Setup the pbuilder -# TODO remove if you do not want to use pbuilders setup_pbuilder @@ -30,11 +44,11 @@ mkdir build cd build # make source tarball -cmake -DCMAKE_BUILD_TYPE=Maintainer $WORKSPACE/nfs-ganesha/src && make dist || exit 1 +cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_CEPH=ON -DUSE_FSAL_RGW=ON $WORKSPACE/nfs-ganesha/src && make dist || exit 1 # move unpacked tarball code into nfs-ganesha-debian/src NFS_GANESHA_SOURCE_DIR=`grep -om 1 "nfs-ganesha-[0-9].[0-9]-dev-[0-9]-[0-9].[0-9].[0-9]-Source" CPackSourceConfig.cmake` -tar xf ${NFS_GANESHA_SOURCE_DIR}.tar.gz +tar xzf ${NFS_GANESHA_SOURCE_DIR}.tar.gz mv ${NFS_GANESHA_SOURCE_DIR} src cp -r src $WORKSPACE/nfs-ganesha-debian/ @@ -76,13 +90,38 @@ sudo pbuilder build \ ## Upload the created RPMs to chacra chacra_endpoint="nfs-ganesha/${NFS_GANESHA_BRANCH}/${GIT_COMMIT}/${DISTRO}/${DIST}" +# TODO: change this +bpvers="foo" +#bpvers=`gen_debian_version $debian_version $DIST` [ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" # push binaries to chacra -find $WORKSPACE/dist/deb -name "*.deb" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${DEB_ARCH}/ -# start repo creation -$VENV/chacractl repo update ${chacra_endpoint} +if [ "$THROWAWAY" = false ] ; then + # push binaries to chacra + find $WORKSPACE/dist/deb | egrep "*\.(changes|deb|dsc|gz)$" | egrep -v "(Packages|Sources|Contents)" | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint} + # write json file with build info + # version and package_manager version are needed for teuthology + cat > $WORKSPACE/repo-extra.json << EOF +{ + "version":"$VERSION", + "package_manager_version":"$bpvers", + "build_url":"$BUILD_URL", + "root_build_cause":"$ROOT_BUILD_CAUSE", + "node_name":"$NODE_NAME", + "job_name":"$JOB_NAME" +} +EOF + # post the json to repo-extra json to chacra + curl -X POST -H "Content-Type:application/json" --data "@$WORKSPACE/repo-extra.json" -u $CHACRACTL_USER:$CHACRACTL_KEY ${chacra_url}/repos/${chacra_endpoint}/extra/ + # start repo creation + $VENV/chacractl repo update ${chacra_endpoint} +fi echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} + +# update shaman with the completed build status +update_build_status "completed" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH + +sudo rm -rf $WORKSPACE/dist diff --git a/nfs-ganesha/build/build_rpm b/nfs-ganesha/build/build_rpm index 97bd3966..d076000c 100644 --- a/nfs-ganesha/build/build_rpm +++ b/nfs-ganesha/build/build_rpm @@ -18,13 +18,31 @@ else exit 1 fi -# install ceph libraries so enable FSAL_CEPH and FSAL_RGW in the .spec file -# when cmake command runs +# add shaman repos to /etc/yum.repos.d/ to install ceph libraries so enable +#FSAL_CEPH (enabled by default) and FSAL_RGW in the .spec file when cmake command runs +sudo cp $WORKSPACE/shaman.repo /etc/yum.repos.d/ xargs sudo yum install -y <<< " -librgw2-devel -libcephfs1 +dbus-devel +libacl-devel +libblkid-devel +libcap-devel +libnfsidmap-devel +libwbclient-devel +krb5-devel +librgw-devel +libcephfs-devel " +# 1-17-17 will remove on next push of EPEL 7 stable, can remove in a couple days +sudo yum install -y mock --enablerepo=epel-testing + +# Normalize variables across rpm/deb builds +NORMAL_DISTRO=$DISTRO +NORMAL_DISTRO_VERSION=$RELEASE +NORMAL_ARCH=$ARCH + +# create build status in shaman +create_build_status "started" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH cd $WORKSPACE/nfs-ganesha @@ -33,8 +51,11 @@ git submodule update --init || git submodule sync mkdir build cd build -# generate .spec file and make source tarball -cmake -DCMAKE_BUILD_TYPE=Maintainer $WORKSPACE/nfs-ganesha/src && make dist || exit 1 +# generate .spec file, edit .spec file for correct versions of libs and make source tarball +cmake -DCMAKE_BUILD_TYPE=Maintainer -DUSE_FSAL_CEPH=ON -DUSE_FSAL_RGW=ON $WORKSPACE/nfs-ganesha/src && make dist || exit 1 + +sed -i 's/libcephfs1-devel/libcephfs-devel/' $WORKSPACE/nfs-ganesha/src/nfs-ganesha.spec +sed -i 's/librgw2-devel/librgw-devel/' $WORKSPACE/nfs-ganesha/src/nfs-ganesha.spec ## Create the source rpm echo "Building SRPM" @@ -42,10 +63,10 @@ rpmbuild \ --define "_sourcedir ." \ --define "_specdir $WORKSPACE/dist" \ --define "_builddir $WORKSPACE/dist" \ - --define "_srcrpmdir $WORKSPACE/dist" \ - --define "_rpmdir $WORKSPACE/dist" \ + --define "_srcrpmdir $WORKSPACE/dist/SRPMS" \ + --define "_rpmdir $WORKSPACE/dist/RPMS" \ --nodeps -bs $WORKSPACE/nfs-ganesha/src/nfs-ganesha.spec -SRPM=$(readlink -f $WORKSPACE/dist/*.src.rpm) +SRPM=$(readlink -f $WORKSPACE/dist/SRPMS/*.src.rpm) # add repo file to mock config sudo head -n -1 /etc/mock/${MOCK_TARGET}-${RELEASE}-${ARCH}.cfg > temp.cfg @@ -55,18 +76,41 @@ echo "\"\"\"" >> nfs-ganesha.cfg ## Build the binaries with mock echo "Building RPMs" -sudo mock -r nfs-ganesha.cfg --resultdir=$WORKSPACE/dist/rpm/ ${SRPM} || ( tail -n +1 $WORKSPACE/dist/rpm/{root,build}.log && exit 1 ) - +sudo mock -r nfs-ganesha.cfg --define "dist .el7" --resultdir=$WORKSPACE/dist/RPMS/ ${SRPM} || ( tail -n +1 $WORKSPACE/dist/RPMS/{root,build}.log && exit 1 ) -## Upload the created RPMs to chacra +VERSION=`grep -om 1 "[0-9]\.[0-9]-dev-[0-9]" $WORKSPACE/nfs-ganesha/src/build/include/config.h` chacra_endpoint="nfs-ganesha/${NFS_GANESHA_BRANCH}/${GIT_COMMIT}/${DISTRO}/${RELEASE}" +RPM_RELEASE=`grep Release $WORKSPACE/nfs-ganesha/src/nfs-ganesha.spec | sed 's/Release:[ \t]*//g' | cut -d '%' -f 1` +RPM_VERSION=`grep Version $WORKSPACE/nfs-ganesha/src/nfs-ganesha.spec | sed 's/Version:[ \t]*//g'` +PACKAGE_MANAGER_VERSION="$RPM_VERSION-$RPM_RELEASE" [ "$FORCE" = true ] && chacra_flags="--force" || chacra_flags="" -# push binaries to chacra -find $WORKSPACE/dist/rpm/ | egrep '\.rpm$' | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/$ARCH/ - -# start repo creation -$VENV/chacractl repo update ${chacra_endpoint} +if [ "$THROWAWAY" = false ] ; then + # push binaries to chacra + find $WORKSPACE/dist/SRPMS | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/source + find $WORKSPACE/dist/RPMS/ | grep rpm | $VENV/chacractl binary ${chacra_flags} create ${chacra_endpoint}/${ARCH} + # write json file with build info + # version and package_manager version are needed for teuthology + cat > $WORKSPACE/repo-extra.json << EOF +{ + "version":"$VERSION", + "package_manager_version":"$PACKAGE_MANAGER_VERSION", + "build_url":"$BUILD_URL", + "root_build_cause":"$ROOT_BUILD_CAUSE", + "node_name":"$NODE_NAME", + "job_name":"$JOB_NAME" +} +EOF + # post the json to repo-extra json to chacra + curl -X POST -H "Content-Type:application/json" --data "@$WORKSPACE/repo-extra.json" -u $CHACRACTL_USER:$CHACRACTL_KEY ${chacra_url}/repos/${chacra_endpoint}/extra/ + # start repo creation + $VENV/chacractl repo update ${chacra_endpoint} +fi echo Check the status of the repo at: https://shaman.ceph.com/api/repos/${chacra_endpoint} + +# update shaman with the completed build status +update_build_status "completed" "nfs-ganesha" $NORMAL_DISTRO $NORMAL_DISTRO_VERSION $NORMAL_ARCH + +sudo rm -rf $WORKSPACE/dist diff --git a/nfs-ganesha/config/definitions/nfs-ganesha.yml b/nfs-ganesha/config/definitions/nfs-ganesha.yml index eca6d375..c68074fb 100644 --- a/nfs-ganesha/config/definitions/nfs-ganesha.yml +++ b/nfs-ganesha/config/definitions/nfs-ganesha.yml @@ -56,6 +56,12 @@ description: "A list of architectures to build for. Available options are: x86_64, and arm64" default: "x86_64" + - bool: + name: THROWAWAY + description: " +Default: False. When True it will not POST binaries to chacra. Artifacts will not be around for long. Useful to test builds." + default: false + - bool: name: FORCE description: " -- 2.39.5