From: David Galloway <dgallowa@redhat.com>
Date: Thu, 17 Jan 2019 13:43:22 +0000 (-0500)
Subject: sepia-fog-images: Optionally pause the sepia queue
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=99add1e81aaaaea548ce040a0574cda8e4924313;p=ceph-build.git

sepia-fog-images: Optionally pause the sepia queue

Signed-off-by: David Galloway <dgallowa@redhat.com>
---

diff --git a/sepia-fog-images/build/build b/sepia-fog-images/build/build
index e6e9033d0..a419b314a 100755
--- a/sepia-fog-images/build/build
+++ b/sepia-fog-images/build/build
@@ -177,29 +177,34 @@ set -ex
 # Restart dhcpd so servers PXE boot to FOG server
 ssh ubuntu@store01.front.sepia.ceph.com "sudo service dhcpd restart"
 
-# Get FOG 'Deploy' TaskID
-fogdeployid=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/tasktype -d '{"name": "Deploy"}' -X GET | jq -r '.tasktypes[0].id')
-
-# Check for scheduled deploy tasks
-deploytasks=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/task/active -d '{"typeID": "'${fogdeployid}'"}' -X GET | jq -r '.count')
-
-# If there are scheduled or active deploy tasks, pause the queue and let them finish.
-# Capturing a new OS image can interrupt active OS deployments.
-if [ $deploytasks -gt 0 ]; then
-  for type in $MACHINETYPES; do
-    # Only pause the queue for 20 minutes just in case anything goes wrong with the Jenkins job.
-    teuthology-queue --pause 1200 --machine_type $type
-  done
-  pausedqueue=true
-  currentretries=0
-  while [ $deploytasks -gt 0 ]; do
-    echo "$(date) -- $deploytasks FOG deploy tasks still queued.  Sleeping 10sec"
-    sleep 10
-    deploytasks=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/task/active -d '{"typeID": "'${fogdeployid}'"}' -X GET | jq -r '.count')
-    ((++currentretries))
-    # Retry for 30min
-    funRetry $currentretries 180
-  done
+# Only pause the queue if needed
+if [ "$PAUSEQUEUE" == "true" ]; then
+  # Get FOG 'Deploy' TaskID
+  fogdeployid=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/tasktype -d '{"name": "Deploy"}' -X GET | jq -r '.tasktypes[0].id')
+
+  # Check for scheduled deploy tasks
+  deploytasks=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/task/active -d '{"typeID": "'${fogdeployid}'"}' -X GET | jq -r '.count')
+
+  # If there are scheduled or active deploy tasks, pause the queue and let them finish.
+  # Capturing a new OS image can interrupt active OS deployments.
+  if [ $deploytasks -gt 0 ]; then
+    for type in $MACHINETYPES; do
+      # Only pause the queue for 20 minutes just in case anything goes wrong with the Jenkins job.
+      teuthology-queue --pause 1200 --machine_type $type
+    done
+    pausedqueue=true
+    currentretries=0
+    while [ $deploytasks -gt 0 ]; do
+      echo "$(date) -- $deploytasks FOG deploy tasks still queued.  Sleeping 10sec"
+      sleep 10
+      deploytasks=$(curl -s -k -H "fog-api-token: ${FOG_API_TOKEN}" -H "fog-user-token: ${FOG_USER_TOKEN}" http://fog.front.sepia.ceph.com/fog/task/active -d '{"typeID": "'${fogdeployid}'"}' -X GET | jq -r '.count')
+      ((++currentretries))
+      # Retry for 30min
+      funRetry $currentretries 180
+    done
+  fi
+else
+  pausedqueue=false
 fi
 
 # Reboot all hosts so FOG can capture their OSes
diff --git a/sepia-fog-images/config/definitions/sepia-fog-images.yml b/sepia-fog-images/config/definitions/sepia-fog-images.yml
index b52ec7cb1..19fd59355 100644
--- a/sepia-fog-images/config/definitions/sepia-fog-images.yml
+++ b/sepia-fog-images/config/definitions/sepia-fog-images.yml
@@ -32,6 +32,10 @@
           name: CMANSIBLEBRANCH
           default: "master"
           description: "Optionally define a different ceph-cm-ansible branch (useful for testing)"
+      - string:
+          name: PAUSEQUEUE
+          default: "true"
+          description: "Should the teuthology queue be paused?  Recapturing an existing OS image will cause running reimages to fail without pausing the queue.  The queue can remain unpaused when a new distro/version is being captured.  Queue is paused by default."
 
     builders:
       - shell: