buildpackages: add build timeouts.

The build/upload process CAN get stuck sometimes. As we are paying
utility rates for the high-power build server, we should timeout when
it's likely the build is stuck.

Use a default timeout just short of 4 hours, as some providers round up
partial instance-hours for billing.

Signed-off-by: Robin H. Johnson <robin.johnson@dreamhost.com>
(cherry picked from commit a33517a88f27df9c13a013ad3378aab050ddbea7)

diff --git a/tasks/buildpackages/Makefile b/tasks/buildpackages/Makefile
index 3fb269c22f28b8f8e19064214b7a76d0497a1ca5..de20fbb755117a208c4735da5c5f6ca025286685 100644 (file)
--- a/tasks/buildpackages/Makefile
+++ b/tasks/buildpackages/Makefile
@@ -1,6 +1,8 @@
 SHELL=/bin/bash
 D=/tmp/stampsdir
 VPATH=${D}
+TIMEOUT_SERVER_CREATE = 30m
+TIMEOUT_BUILD = 220m # 20 minutes short of 4 hours
 PKG_REPO=packages-repository
 PKG_REPO_OS_TYPE=ubuntu
 PKG_REPO_OS_VERSION=14.04
@@ -30,7 +32,7 @@ ${HOME}/.ssh_agent:
        grep -q ssh_agent ~/.bashrc_teuthology || echo 'source ${HOME}/.ssh_agent' >> ~/.bashrc_teuthology
 
 flock-${PKG_REPO}:
-       openstack server create --image 'teuthology-ubuntu-14.04-${HTTP_ARCH}' ${OPENSTACK_NETWORK} --flavor ${HTTP_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${PKG_REPO_USER_DATA} --wait ${PKG_REPO}
+       timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-ubuntu-14.04-${HTTP_ARCH}' ${OPENSTACK_NETWORK} --flavor ${HTTP_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${PKG_REPO_USER_DATA} --wait ${PKG_REPO}
        sleep 30
        set -ex ; \
        ip=$(call get_ip,${PKG_REPO}) ; \
@@ -54,7 +56,7 @@ ${PKG_REPO}:
 # If it's a weird status, bail out and let the delete fire
 # eg: ERROR status can happen if there is no VM host without enough capacity for the request.
 ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${PKG_REPO}
-       openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-${CEPH_ARCH}' ${OPENSTACK_NETWORK} --flavor ${BUILD_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-user-data.txt --wait $@
+       timeout $(TIMEOUT_SERVER_CREATE) openstack server create --image 'teuthology-${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-${CEPH_ARCH}' ${OPENSTACK_NETWORK} --flavor ${BUILD_FLAVOR} --key-name teuthology --security-group teuthology --property ownedby=${MY_IP} --user-data ${CEPH_OS_TYPE}-${CEPH_OS_VERSION}-user-data.txt --wait $@
        set -ex ; \
        trap "openstack server delete --wait $@" EXIT ; \
        for delay in 30 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 ; do \
@@ -70,7 +72,7 @@ ceph-${CEPH_PKG_TYPE}-${CEPH_DIST}-${CEPH_ARCH}-${CEPH_FLAVOR}-${CEPH_SHA1}: ${P
        for delay in 1 2 4 8 8 8 8 8 8 8 8 8 16 16 16 16 16 32 32 32 64 128 256 512 ; do if ssh -o 'ConnectTimeout=3' $$ip bash -c '"grep -q READYTORUN /var/log/cloud-init*.log"' ; then break ; else sleep $$delay ; fi ; done ; \
        scp make-${CEPH_PKG_TYPE}.sh common.sh ubuntu@$$ip: ; \
        packages_repository=$(call get_ip,${<F}) ; \
-       ssh -tt -A ubuntu@$$ip bash ./make-${CEPH_PKG_TYPE}.sh $$packages_repository ${CEPH_DIST} ${CEPH_GIT_URL} ${CEPH_SHA1} ${CEPH_FLAVOR} ${CEPH_ARCH}
+       timeout $(TIMEOUT_BUILD) ssh -tt -A ubuntu@$$ip bash ./make-${CEPH_PKG_TYPE}.sh $$packages_repository ${CEPH_DIST} ${CEPH_GIT_URL} ${CEPH_SHA1} ${CEPH_FLAVOR} ${CEPH_ARCH}
        mkdir -p ${D}/${@D} ; touch ${D}/$@
 
 clobber: