From: Andrew Schoen <aschoen@redhat.com>
Date: Tue, 4 Aug 2015 20:10:25 +0000 (-0500)
Subject: ansible: read the ansible failure log and raise a more meaningful error
X-Git-Tag: 1.1.0~854^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=fdc11d9e5922811533e51968a34d05a2d94feec2;p=teuthology.git

ansible: read the ansible failure log and raise a more meaningful error

This works with a custom ansible callback plugin that intercepts task
failures and writes them to a log file (in yaml format) provided by the
ansible task as an environment variable. If the playbook fails, teuthology
reads that failure log and raises an AnsibleFailedError with the actual
output from the failed ansible task.

Signed-off-by: Andrew Schoen <aschoen@redhat.com>
---

diff --git a/teuthology/exceptions.py b/teuthology/exceptions.py
index 73c59ae89..9b7c65991 100644
--- a/teuthology/exceptions.py
+++ b/teuthology/exceptions.py
@@ -52,6 +52,20 @@ class CommandFailedError(Exception):
             )
 
 
+class AnsibleFailedError(Exception):
+
+    """
+    Exception thrown when an ansible playbook fails
+    """
+    def __init__(self, failures):
+        self.failures = failures
+
+    def __str__(self):
+        return "{failures}".format(
+            failures=self.failures,
+        )
+
+
 class CommandCrashedError(Exception):
 
     """
diff --git a/teuthology/task/ansible.py b/teuthology/task/ansible.py
index 43071bb30..046e364dc 100644
--- a/teuthology/task/ansible.py
+++ b/teuthology/task/ansible.py
@@ -9,7 +9,7 @@ from cStringIO import StringIO
 from tempfile import NamedTemporaryFile
 
 from teuthology.config import config as teuth_config
-from teuthology.exceptions import CommandFailedError
+from teuthology.exceptions import CommandFailedError, AnsibleFailedError
 from teuthology.repo_utils import fetch_repo
 
 from . import Task
@@ -116,6 +116,15 @@ class Ansible(Task):
         if not hasattr(self, 'playbook_file'):
             self.generate_playbook()
 
+    @property
+    def failure_log(self):
+        if not hasattr(self, '_failure_log'):
+            self._failure_log = NamedTemporaryFile(
+                prefix="teuth_ansible_failures_",
+                delete=False,
+            )
+        return self._failure_log
+
     def find_repo(self):
         """
         Locate the repo we're using; cloning it from a remote repo if necessary
@@ -229,6 +238,7 @@ class Ansible(Task):
         """
         environ = os.environ
         environ['ANSIBLE_SSH_PIPELINING'] = '1'
+        environ['ANSIBLE_FAILURE_LOG'] = self.failure_log.name
         environ['ANSIBLE_ROLES_PATH'] = "%s/roles" % self.repo_path
         args = self._build_args()
         command = ' '.join(args)
@@ -242,7 +252,7 @@ class Ansible(Task):
             timeout=None,
         )
         if status != 0:
-            raise CommandFailedError(command, status)
+            self._handle_failure(command, status)
 
         if self.config.get('reconnect', True) is True:
             remotes = self.cluster.remotes.keys()
@@ -250,6 +260,23 @@ class Ansible(Task):
             for remote in remotes:
                 remote.reconnect()
 
+    def _handle_failure(self, command, status):
+        failures = None
+        with open(self.failure_log.name, 'r') as log:
+            failures = yaml.safe_load(log)
+
+        if failures:
+            if self.ctx.archive:
+                self._archive_failures()
+            raise AnsibleFailedError(failures)
+        raise CommandFailedError(command, status)
+
+    def _archive_failures(self):
+        os.rename(
+            self.failure_log.name,
+            "{0}/ansible_failures.yaml".format(self.ctx.archive)
+        )
+
     def _build_args(self):
         """
         Assemble the list of args to be executed