fog: If deploy fails, cancel the deploy task

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/provision/fog.py b/teuthology/provision/fog.py
index 202ef0cf06176943f2076cd8574cdb74aa801224..5e8d14141fb50d197ce8c2505ff3b91aef509607 100644 (file)
--- a/teuthology/provision/fog.py
+++ b/teuthology/provision/fog.py
@@ -76,12 +76,16 @@ class FOG(object):
         host_id = int(host_data['id'])
         self.set_image(host_id)
         task_id = self.schedule_deploy_task(host_id)
-        # Use power_off/power_on because other methods call _wait_for_login,
-        # which will not work here since the newly-imaged host will have an
-        # incorrect hostname
-        self.remote.console.power_off()
-        self.remote.console.power_on()
-        self.wait_for_deploy_task(task_id)
+        try:
+            # Use power_off/power_on because other methods call
+            # _wait_for_login, which will not work here since the newly-imaged
+            # host will have an incorrect hostname
+            self.remote.console.power_off()
+            self.remote.console.power_on()
+            self.wait_for_deploy_task(task_id)
+        except Exception:
+            self.cancel_deploy_task(task_id)
+            raise
         self._wait_for_ready()
         self._fix_hostname()
         self.log.info("Deploy complete!")
@@ -235,6 +239,15 @@ class FOG(object):
                 if not self.deploy_task_active(task_id):
                     break
 
+    def cancel_deploy_task(self,  task_id):
+        """ Cancel an active deploy task """
+        resp = self.do_request(
+            '/task/cancel',
+            method='DELETE',
+            data='{"id": %i}' % task_id,
+        )
+        resp.raise_for_status()
+
     def _wait_for_ready(self):
         """ Attempt to connect to the machine via SSH """
         with safe_while(sleep=6, tries=50) as proceed:

diff --git a/teuthology/provision/test/test_fog.py b/teuthology/provision/test/test_fog.py
index 339b9eda080027b61042e25744fe8f37913ff3a2..f370cd6e4801a301bc52c8067039a8526cb10a8f 100644 (file)
--- a/teuthology/provision/test/test_fog.py
+++ b/teuthology/provision/test/test_fog.py
@@ -82,7 +82,8 @@ class TestFOG(object):
         assert obj.os_type == 'type'
         assert obj.os_version == '1.0'
 
-    def test_create(self):
+    @mark.parametrize('success', [True, False])
+    def test_create(self, success):
         self.mocks['m_Remote_hostname'].return_value = 'name.fqdn'
         self.mocks['m_Remote_machine_type'].return_value = 'type1'
         obj = self.klass('name.fqdn', 'type', '1.0')
@@ -93,18 +94,27 @@ class TestFOG(object):
             set_image=DEFAULT,
             schedule_deploy_task=DEFAULT,
             wait_for_deploy_task=DEFAULT,
+            cancel_deploy_task=DEFAULT,
             _wait_for_ready=DEFAULT,
             _fix_hostname=DEFAULT,
         ) as local_mocks:
             local_mocks['get_host_data'].return_value = dict(id=host_id)
-            obj.create()
+            if not success:
+                local_mocks['wait_for_deploy_task'].side_effect = RuntimeError
+                with raises(RuntimeError):
+                    obj.create()
+            else:
+                obj.create()
             assert local_mocks['get_host_data'].called_once_with()
             assert local_mocks['set_image'].called_once_with(host_id)
             assert local_mocks['schedule_deploy_task']\
                 .called_once_with(host_id)
             assert local_mocks['wait_for_deploy_task'].called_once_with()
-            assert local_mocks['_wait_for_ready'].called_once_with()
-            assert local_mocks['_fix_hostname'].called_once_with()
+            if success:
+                assert local_mocks['_wait_for_ready'].called_once_with()
+                assert local_mocks['_fix_hostname'].called_once_with()
+            else:
+                assert len(local_mocks['cancel_deploy_task'].call_args_list) == 1
         assert self.mocks['m_Remote_console']\
             .return_value.power_off.called_once_with()
         assert self.mocks['m_Remote_console']\
@@ -260,6 +270,19 @@ class TestFOG(object):
             assert len(local_mocks['deploy_task_active'].call_args_list) == \
                 tries + 1
 
+    def test_cancel_deploy_task(self):
+        obj = self.klass('name.fqdn', 'type', '1.0')
+        with patch.multiple(
+            'teuthology.provision.fog.FOG',
+            do_request=DEFAULT,
+        ) as local_mocks:
+            obj.cancel_deploy_task(10)
+            assert local_mocks['do_request'].called_once_with(
+                '/task/cancel',
+                method='DELETE',
+                data='{"id": 10}',
+            )
+
     @mark.parametrize(
         'tries',
         [1, 51],