fog: Wait 15 minutes for FOG task to complete

When there are running jobs and a large run gets scheduled, FOG
provisioning gets backed up due to its built-in rate limiting.  15
minutes should be a little more lenient and prevent large spikes of dead
jobs when a run is scheduled in an idle queue.

Signed-off-by: David Galloway <dgallowa@redhat.com>

diff --git a/teuthology/provision/fog.py b/teuthology/provision/fog.py
index a9a3cca05e79b082d60c304d49f6e3f7be881813..3e08e038e0667a36b9b06c5018a530b8b2cf0774 100644 (file)
--- a/teuthology/provision/fog.py
+++ b/teuthology/provision/fog.py
@@ -241,7 +241,7 @@ class FOG(object):
         completed)
         """
         self.log.info("Waiting for deploy to finish")
-        with safe_while(sleep=15, tries=40) as proceed:
+        with safe_while(sleep=15, tries=60) as proceed:
             while proceed():
                 if not self.deploy_task_active(task_id):
                     break

diff --git a/teuthology/provision/test/test_fog.py b/teuthology/provision/test/test_fog.py
index d849bee21f45afd94f49fd0b0dcdad2c23a7d968..61c342c43bf34eb4538784801eedd80c41b8a5cd 100644 (file)
--- a/teuthology/provision/test/test_fog.py
+++ b/teuthology/provision/test/test_fog.py
@@ -252,7 +252,7 @@ class TestFOG(object):
 
     @mark.parametrize(
         'tries',
-        [3, 45],
+        [3, 61],
     )
     def test_wait_for_deploy_task(self, tries):
         wait_results = [True for i in range(tries)] + [False]
@@ -262,7 +262,7 @@ class TestFOG(object):
             deploy_task_active=DEFAULT,
         ) as local_mocks:
             local_mocks['deploy_task_active'].side_effect = wait_results
-            if tries >= 40:
+            if tries >= 60:
                 with raises(MaxWhileTries):
                     obj.wait_for_deploy_task(9)
                 return