From 5eb509cab4970e448bd4016d883d710159228cc4 Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 28 Nov 2017 15:48:15 -0700 Subject: [PATCH] fog: If deploy fails, cancel the deploy task Signed-off-by: Zack Cerza --- teuthology/provision/fog.py | 25 +++++++++++++++------ teuthology/provision/test/test_fog.py | 31 +++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/teuthology/provision/fog.py b/teuthology/provision/fog.py index 202ef0cf06..5e8d14141f 100644 --- a/teuthology/provision/fog.py +++ b/teuthology/provision/fog.py @@ -76,12 +76,16 @@ class FOG(object): host_id = int(host_data['id']) self.set_image(host_id) task_id = self.schedule_deploy_task(host_id) - # Use power_off/power_on because other methods call _wait_for_login, - # which will not work here since the newly-imaged host will have an - # incorrect hostname - self.remote.console.power_off() - self.remote.console.power_on() - self.wait_for_deploy_task(task_id) + try: + # Use power_off/power_on because other methods call + # _wait_for_login, which will not work here since the newly-imaged + # host will have an incorrect hostname + self.remote.console.power_off() + self.remote.console.power_on() + self.wait_for_deploy_task(task_id) + except Exception: + self.cancel_deploy_task(task_id) + raise self._wait_for_ready() self._fix_hostname() self.log.info("Deploy complete!") @@ -235,6 +239,15 @@ class FOG(object): if not self.deploy_task_active(task_id): break + def cancel_deploy_task(self, task_id): + """ Cancel an active deploy task """ + resp = self.do_request( + '/task/cancel', + method='DELETE', + data='{"id": %i}' % task_id, + ) + resp.raise_for_status() + def _wait_for_ready(self): """ Attempt to connect to the machine via SSH """ with safe_while(sleep=6, tries=50) as proceed: diff --git a/teuthology/provision/test/test_fog.py b/teuthology/provision/test/test_fog.py index 339b9eda08..f370cd6e48 100644 --- a/teuthology/provision/test/test_fog.py +++ b/teuthology/provision/test/test_fog.py @@ -82,7 +82,8 @@ class TestFOG(object): assert obj.os_type == 'type' assert obj.os_version == '1.0' - def test_create(self): + @mark.parametrize('success', [True, False]) + def test_create(self, success): self.mocks['m_Remote_hostname'].return_value = 'name.fqdn' self.mocks['m_Remote_machine_type'].return_value = 'type1' obj = self.klass('name.fqdn', 'type', '1.0') @@ -93,18 +94,27 @@ class TestFOG(object): set_image=DEFAULT, schedule_deploy_task=DEFAULT, wait_for_deploy_task=DEFAULT, + cancel_deploy_task=DEFAULT, _wait_for_ready=DEFAULT, _fix_hostname=DEFAULT, ) as local_mocks: local_mocks['get_host_data'].return_value = dict(id=host_id) - obj.create() + if not success: + local_mocks['wait_for_deploy_task'].side_effect = RuntimeError + with raises(RuntimeError): + obj.create() + else: + obj.create() assert local_mocks['get_host_data'].called_once_with() assert local_mocks['set_image'].called_once_with(host_id) assert local_mocks['schedule_deploy_task']\ .called_once_with(host_id) assert local_mocks['wait_for_deploy_task'].called_once_with() - assert local_mocks['_wait_for_ready'].called_once_with() - assert local_mocks['_fix_hostname'].called_once_with() + if success: + assert local_mocks['_wait_for_ready'].called_once_with() + assert local_mocks['_fix_hostname'].called_once_with() + else: + assert len(local_mocks['cancel_deploy_task'].call_args_list) == 1 assert self.mocks['m_Remote_console']\ .return_value.power_off.called_once_with() assert self.mocks['m_Remote_console']\ @@ -260,6 +270,19 @@ class TestFOG(object): assert len(local_mocks['deploy_task_active'].call_args_list) == \ tries + 1 + def test_cancel_deploy_task(self): + obj = self.klass('name.fqdn', 'type', '1.0') + with patch.multiple( + 'teuthology.provision.fog.FOG', + do_request=DEFAULT, + ) as local_mocks: + obj.cancel_deploy_task(10) + assert local_mocks['do_request'].called_once_with( + '/task/cancel', + method='DELETE', + data='{"id": 10}', + ) + @mark.parametrize( 'tries', [1, 51], -- 2.39.5