nuke: --stale-openstack must cope with disappearing volumes / instances

"openstack server/volume show" typically takes a few seconds to
complete due to the provider API slowness/throttling. It is possible that
a volume/instance found with "openstack server list" no longer exists when
running "openstack server show" to get more information about it.

Signed-off-by: Loic Dachary <loic@dachary.org>

diff --git a/teuthology/nuke.py b/teuthology/nuke.py
index cf1799a6a1030fa4886eb8fc9a4f860062382631..cd9d2f8a180f594758c658b80571115cac8369c2 100644 (file)
--- a/teuthology/nuke.py
+++ b/teuthology/nuke.py
@@ -389,7 +389,12 @@ OPENSTACK_DELAY = 30 * 60
 def stale_openstack_instances(ctx, instances, locked_nodes):
     for (name, instance) in instances.iteritems():
         i = OpenStackInstance(name)
-        if (i.get_created() > config['max_job_time'] + OPENSTACK_DELAY):
+        if not i.exists():
+            log.debug("stale-openstack: {instance} disappeared, ignored"
+                      .format(instance=name))
+            continue
+        if (i.get_created() >
+            config['max_job_time'] + OPENSTACK_DELAY):
             log.info(
                 "stale-openstack: destroying instance {instance}"
                 " because it was created {created} seconds ago"
@@ -423,8 +428,13 @@ def openstack_delete_volume(id):
 def stale_openstack_volumes(ctx, volumes):
     now = datetime.datetime.now()
     for volume in volumes:
-        volume = json.loads(sh("openstack volume show -f json " +
-                               volume['ID']))
+        try:
+            volume = json.loads(sh("openstack volume show -f json " +
+                                   volume['ID']))
+        except subprocess.CalledProcessError:
+            log.debug("stale-openstack: {id} disappeared, ignored"
+                      .format(id=volume['ID']))
+            continue
         volume = dict(map(lambda v: (v['Field'], v['Value']), volume))
         created_at = datetime.datetime.strptime(
             volume['created_at'], '%Y-%m-%dT%H:%M:%S.%f')

diff --git a/teuthology/test/test_nuke.py b/teuthology/test/test_nuke.py
index bef19dc4c28dd8411f35e9e28dd0ee12a313dd6f..a4e75f0ab0fe67b4f343490434eee5734443d98d 100644 (file)
--- a/teuthology/test/test_nuke.py
+++ b/teuthology/test/test_nuke.py
@@ -1,5 +1,6 @@
 import json
 import datetime
+import subprocess
 
 from mock import patch, Mock, DEFAULT
 
@@ -68,6 +69,20 @@ class TestNuke(object):
             nuke.stale_openstack_volumes(ctx, volume_list)
             m['openstack_delete_volume'].assert_called_with(id)
 
+        #
+        # A volume that no longer exists is ignored
+        #
+        def sh(cmd):
+            raise subprocess.CalledProcessError('ERROR', 'FAIL')
+
+        with patch.multiple(
+                nuke,
+                sh=sh,
+                openstack_delete_volume=DEFAULT,
+                ) as m:
+            nuke.stale_openstack_volumes(ctx, volume_list)
+            m['openstack_delete_volume'].assert_not_called()
+
     def test_stale_openstack_nodes(self):
         ctx = Mock()
         ctx.teuthology_config = config
@@ -147,6 +162,7 @@ class TestNuke(object):
         #
         with patch.multiple(
                 nuke.OpenStackInstance,
+                exists=lambda _: True,
                 get_created=lambda _: 1,
                 __getitem__=lambda _, key: name,
                 destroy=DEFAULT,
@@ -161,6 +177,7 @@ class TestNuke(object):
         #
         with patch.multiple(
                 nuke.OpenStackInstance,
+                exists=lambda _: True,
                 get_created=lambda _: 1000000000,
                 __getitem__=lambda _, key: name,
                 destroy=DEFAULT,
@@ -172,11 +189,28 @@ class TestNuke(object):
             })
             m['destroy'].assert_called_with()
         #
+        # An instance that turns out to not exist any longer
+        # is ignored.
+        #
+        with patch.multiple(
+                nuke.OpenStackInstance,
+                exists=lambda _: False,
+                __getitem__=lambda _, key: name,
+                destroy=DEFAULT,
+                ) as m:
+            nuke.stale_openstack_instances(ctx, {
+                name: { 'id': 'UUID', },
+            }, {
+                misc.canonicalize_hostname(name, user=None): {},
+            })
+            m['destroy'].assert_not_called()
+        #
         # An instance created but not locked after a while is
         # destroyed.
         #
         with patch.multiple(
                 nuke.OpenStackInstance,
+                exists=lambda _: True,
                 get_created=lambda _: nuke.OPENSTACK_DELAY + 1,
                 __getitem__=lambda _, key: name,
                 destroy=DEFAULT,
@@ -192,6 +226,7 @@ class TestNuke(object):
         #
         with patch.multiple(
                 nuke.OpenStackInstance,
+                exists=lambda _: True,
                 get_created=lambda _: nuke.OPENSTACK_DELAY + 1,
                 __getitem__=lambda _, key: name,
                 destroy=DEFAULT,