From ad940ab425c282e494b373298e8f83479f8ee06b Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 31 Oct 2015 04:06:58 +0900 Subject: [PATCH] openstack: more robust server deletion When removing a server and the volumes attached to it, the following can happen: * the server goes in ERROR status instead of being deleted * if removed before the server is deleted, the volume may fail detach and consequently the deletion can fail with "still attached, detach volume first" * some volumes go in ERROR status instead of being deleted This will cause problem when and if an instance is assigned the same IP as an instance in ERROR state. They will both have the same name and confuse teuthology. Instead of assuming deletion is a reliable operation, assume it is a best effort: * rename the server and volumes to REMOVE-ME * delete the server and then the volumes: ignore failures The names instances and volumes in error state no longer conflict with the name of running instances and can be dealt with at a later time, and kept around for the cloud provider support to investigate. Signed-off-by: Loic Dachary --- teuthology/provision.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/teuthology/provision.py b/teuthology/provision.py index e839cb880d..af61b5eefc 100644 --- a/teuthology/provision.py +++ b/teuthology/provision.py @@ -362,18 +362,13 @@ class ProvisionOpenStack(OpenStack): if not self.exists(name_or_id, server_info=server_info): return True volumes = self.list_volumes(name_or_id, server_info=server_info) - if self.get_value(server_info, 'status').lower() == 'error': - log.info( - "Instance %s is in an error state; skipping volume detachment", - name_or_id - ) - else: - for volume in volumes: - misc.sh("openstack server remove volume %s %s" % - (name_or_id, volume)) - misc.sh("openstack server delete " + name_or_id) + server_id = self.get_value(server_info, 'ID') + misc.sh("openstack server set --name REMOVE-ME-" + name_or_id + + " " + server_id) + misc.sh("openstack server delete --wait " + server_id + " || true") for volume in volumes: - misc.sh("openstack volume delete " + volume) + misc.sh("openstack volume set --name REMOVE-ME " + volume) + misc.sh("openstack volume delete " + volume + " || true") return True -- 2.39.5