ansible: read the ansible failure log and raise a more meaningful error

This works with a custom ansible callback plugin that intercepts task
failures and writes them to a log file (in yaml format) provided by the
ansible task as an environment variable. If the playbook fails, teuthology
reads that failure log and raises an AnsibleFailedError with the actual
output from the failed ansible task.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>

diff --git a/teuthology/exceptions.py b/teuthology/exceptions.py
index 73c59ae8945d1f52c721e5859a656278029cbece..9b7c65991bbaee4092af16d6f879698568f1ed2e 100644 (file)
--- a/teuthology/exceptions.py
+++ b/teuthology/exceptions.py
@@ -52,6 +52,20 @@ class CommandFailedError(Exception):
             )
 
 
+class AnsibleFailedError(Exception):
+
+    """
+    Exception thrown when an ansible playbook fails
+    """
+    def __init__(self, failures):
+        self.failures = failures
+
+    def __str__(self):
+        return "{failures}".format(
+            failures=self.failures,
+        )
+
+
 class CommandCrashedError(Exception):
 
     """

diff --git a/teuthology/task/ansible.py b/teuthology/task/ansible.py
index 43071bb30d4e71cec2dbef06fb28c3b79a3bb5d9..046e364dcb833f99c47b496475f2faeda6d0bb84 100644 (file)
--- a/teuthology/task/ansible.py
+++ b/teuthology/task/ansible.py
@@ -9,7 +9,7 @@ from cStringIO import StringIO
 from tempfile import NamedTemporaryFile
 
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import CommandFailedError
+from teuthology.exceptions import CommandFailedError, AnsibleFailedError
 from teuthology.repo_utils import fetch_repo
 
 from . import Task
@@ -116,6 +116,15 @@ class Ansible(Task):
         if not hasattr(self, 'playbook_file'):
             self.generate_playbook()
 
+    @property
+    def failure_log(self):
+        if not hasattr(self, '_failure_log'):
+            self._failure_log = NamedTemporaryFile(
+                prefix="teuth_ansible_failures_",
+                delete=False,
+            )
+        return self._failure_log
+
     def find_repo(self):
         """
         Locate the repo we're using; cloning it from a remote repo if necessary
@@ -229,6 +238,7 @@ class Ansible(Task):
         """
         environ = os.environ
         environ['ANSIBLE_SSH_PIPELINING'] = '1'
+        environ['ANSIBLE_FAILURE_LOG'] = self.failure_log.name
         environ['ANSIBLE_ROLES_PATH'] = "%s/roles" % self.repo_path
         args = self._build_args()
         command = ' '.join(args)
@@ -242,7 +252,7 @@ class Ansible(Task):
             timeout=None,
         )
         if status != 0:
-            raise CommandFailedError(command, status)
+            self._handle_failure(command, status)
 
         if self.config.get('reconnect', True) is True:
             remotes = self.cluster.remotes.keys()
@@ -250,6 +260,23 @@ class Ansible(Task):
             for remote in remotes:
                 remote.reconnect()
 
+    def _handle_failure(self, command, status):
+        failures = None
+        with open(self.failure_log.name, 'r') as log:
+            failures = yaml.safe_load(log)
+
+        if failures:
+            if self.ctx.archive:
+                self._archive_failures()
+            raise AnsibleFailedError(failures)
+        raise CommandFailedError(command, status)
+
+    def _archive_failures(self):
+        os.rename(
+            self.failure_log.name,
+            "{0}/ansible_failures.yaml".format(self.ctx.archive)
+        )
+
     def _build_args(self):
         """
         Assemble the list of args to be executed