provision.fog: Add optional sentinel file

Similar to how provision.cloud.openstack works, wait for a particular
file to be created before considering provisioning to be finished.

Unlike the cloud.openstack module, teuthology doesn't control the entire
process, so to avoid breaking any labs that aren't using a sentinel
file, make the feature optional and default to being inoperative.

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/provision/fog.py b/teuthology/provision/fog.py
index e579bad0c6cdd1aae28f44d81ea7929e7231a7c9..d4ad3444b21201e3a5767a9378c2f97899e5f471 100644 (file)
--- a/teuthology/provision/fog.py
+++ b/teuthology/provision/fog.py
@@ -6,6 +6,7 @@ import re
 
 from datetime import datetime
 from paramiko import SSHException
+from paramiko.ssh_exception import NoValidConnectionsError
 
 import teuthology.orchestra
 
@@ -263,10 +264,16 @@ class FOG(object):
                 except (
                     socket.error,
                     SSHException,
+                    NoValidConnectionsError,
                     MaxWhileTries,
                     EOFError,
                 ):
                     pass
+        sentinel_file = config.fog.get('sentinel_file', None)
+        if sentinel_file:
+            cmd = "while [ ! -e '%s' ]; do sleep 5; done" % sentinel_file
+            self.remote.run(args=cmd, timeout=600)
+        self.log.info("Node is ready")
 
     def _fix_hostname(self):
         """

diff --git a/teuthology/provision/test/test_fog.py b/teuthology/provision/test/test_fog.py
index f4a791f1dc42bf8da62a5c4bf5a63ed7c3c8dfd6..9119f74f1e2c3f663deda714f7db051771437189 100644 (file)
--- a/teuthology/provision/test/test_fog.py
+++ b/teuthology/provision/test/test_fog.py
@@ -4,7 +4,7 @@ from mock import patch, DEFAULT, PropertyMock
 from pytest import raises, mark
 
 from teuthology.config import config
-from teuthology.exceptions import MaxWhileTries
+from teuthology.exceptions import MaxWhileTries, CommandFailedError
 from teuthology.provision import fog
 
 
@@ -287,7 +287,7 @@ class TestFOG(object):
         'tries',
         [1, 101],
     )
-    def test_wait_for_ready(self, tries):
+    def test_wait_for_ready_tries(self, tries):
         connect_results = [MaxWhileTries for i in range(tries)] + [True]
         obj = self.klass('name.fqdn', 'type', '1.0')
         self.mocks['m_Remote_connect'].side_effect = connect_results
@@ -297,3 +297,21 @@ class TestFOG(object):
             return
         obj._wait_for_ready()
         assert len(self.mocks['m_Remote_connect'].call_args_list) == tries + 1
+
+    @mark.parametrize(
+        'sentinel_present',
+        ([False, True]),
+    )
+    def test_wait_for_ready_sentinel(self, sentinel_present):
+        config.fog['sentinel_file'] = '/a_file'
+        obj = self.klass('name.fqdn', 'type', '1.0')
+        if not sentinel_present:
+            self.mocks['m_Remote_run'].side_effect = [
+                CommandFailedError(command='cmd', exitstatus=1)]
+            with raises(CommandFailedError):
+                obj._wait_for_ready()
+        else:
+            obj._wait_for_ready()
+        assert len(self.mocks['m_Remote_run'].call_args_list) == 1
+        assert "'/a_file'" in \
+            self.mocks['m_Remote_run'].call_args_list[0][1]['args']