If conserver fails, fall back to ipmitool

Signed-off-by: Zack Cerza <zack@redhat.com>

diff --git a/teuthology/orchestra/console.py b/teuthology/orchestra/console.py
index 3366ec9ec36303903730ced9423939f93f17ed69..f1244091bf66d8fa4bbb9ece581f83bf4bbf76ab 100644 (file)
--- a/teuthology/orchestra/console.py
+++ b/teuthology/orchestra/console.py
@@ -66,8 +66,15 @@ class PhysicalConsole():
         )
 
     def _get_console(self, readonly=True):
-        cmd = self._console_command(readonly=readonly)
-        child = self._pexpect_spawn(cmd)
+        def start():
+            cmd = self._console_command(readonly=readonly)
+            return self._pexpect_spawn(cmd)
+
+        child = start()
+        if self.has_conserver and not child.isalive():
+            log.error("conserver failed to get the console; will try ipmitool")
+            self.has_conserver = False
+            child = start()
         return child
 
     def _console_command(self, readonly=True):
@@ -267,21 +274,29 @@ class PhysicalConsole():
 
         :returns: a subprocess.Popen object
         """
-        console_cmd = self._console_command()
         pexpect_templ = \
             "import pexpect; " \
             "pexpect.run('{cmd}', logfile=file('{log}', 'w'), timeout=None)"
-        python_cmd = [
-            '/usr/bin/env', 'python', '-c',
-            pexpect_templ.format(
-                cmd=console_cmd,
-                log=dest_path,
-            ),
-        ]
-        proc = subprocess.Popen(
-            python_cmd,
-            env=os.environ,
-        )
+
+        def start():
+            console_cmd = self._console_command()
+            python_cmd = [
+                '/usr/bin/env', 'python', '-c',
+                pexpect_templ.format(
+                    cmd=console_cmd,
+                    log=dest_path,
+                ),
+            ]
+            return subprocess.Popen(
+                python_cmd,
+                env=os.environ,
+            )
+
+        proc = start()
+        if self.has_conserver and proc.poll() is not None:
+            log.error("conserver failed to get the console; will try ipmitool")
+            self.has_conserver = False
+            proc = start()
         return proc
 
 

diff --git a/teuthology/orchestra/test/test_console.py b/teuthology/orchestra/test/test_console.py
index 3ab42163b1b0ed4bdf029a8078a91c42e4db1cf0..248907b021fd9817d18012a7d4a92c8be95d4e41 100644 (file)
--- a/teuthology/orchestra/test/test_console.py
+++ b/teuthology/orchestra/test/test_console.py
@@ -120,6 +120,24 @@ class TestPhysicalConsole(TestConsole):
                 ['ipmitool' in arg for arg in call_args]
             )
 
+    def test_spawn_log_fallback(self):
+        with patch(
+            'teuthology.orchestra.console.subprocess.Popen',
+            autospec=True,
+        ) as m_popen:
+            m_popen.return_value.wait.return_value = 0
+            cons = self.klass(self.hostname)
+            assert cons.has_conserver is True
+            m_popen.reset_mock()
+            m_popen.return_value.poll.return_value = 1
+            cons.spawn_sol_log('/fake/path')
+            assert cons.has_conserver is False
+            assert m_popen.call_count == 2
+            call_args = m_popen.call_args_list[1][0][0]
+            assert any(
+                ['ipmitool' in arg for arg in call_args]
+            )
+
     def test_get_console_conserver(self):
         with patch(
             'teuthology.orchestra.console.subprocess.Popen',
@@ -153,3 +171,23 @@ class TestPhysicalConsole(TestConsole):
             cons._get_console()
             assert m_spawn.call_count == 1
             assert 'ipmitool' in m_spawn.call_args_list[0][0][0]
+
+    def test_get_console_fallback(self):
+        with patch(
+            'teuthology.orchestra.console.subprocess.Popen',
+            autospec=True,
+        ) as m_popen:
+            m_popen.return_value.wait.return_value = 0
+            cons = self.klass(self.hostname)
+        assert cons.has_conserver is True
+        with patch(
+            'teuthology.orchestra.console.pexpect.spawn',
+            autospec=True,
+        ) as m_spawn:
+            cons.has_conserver = True
+            m_spawn.return_value.isalive.return_value = False
+            cons._get_console()
+            assert m_spawn.return_value.isalive.call_count == 1
+            assert m_spawn.call_count == 2
+            assert cons.has_conserver is False
+            assert 'ipmitool' in m_spawn.call_args_list[1][0][0]