From 89a69f04c3a86e7a12deafd265365f0e6308cbde Mon Sep 17 00:00:00 2001 From: Vallari Agrawal Date: Sat, 1 Oct 2022 16:46:52 +0530 Subject: [PATCH] orch/run: Add unit test XML scanner Scan XML output files with find_unittest_error and raise UnitTestError. Signed-off-by: Vallari Agrawal --- setup.cfg | 1 + teuthology/exceptions.py | 23 ++++++ teuthology/orchestra/run.py | 62 +++++++++++++++- teuthology/orchestra/test/test_run.py | 4 + .../test/xml_files/test_scan_nose.xml | 73 +++++++++++++++++++ 5 files changed, 160 insertions(+), 3 deletions(-) create mode 100644 teuthology/orchestra/test/xml_files/test_scan_nose.xml diff --git a/setup.cfg b/setup.cfg index f5de27bf9..26483eb31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ install_requires = httplib2 humanfriendly lupa + lxml ndg-httpsclient netaddr paramiko diff --git a/teuthology/exceptions.py b/teuthology/exceptions.py index 3939ba53c..bd8c4a20e 100644 --- a/teuthology/exceptions.py +++ b/teuthology/exceptions.py @@ -205,3 +205,26 @@ class NoRemoteError(Exception): def __str__(self): return self.message + + +class UnitTestError(Exception): + """ + Exception thrown on unit test failure + """ + def __init__(self, exitstatus=None, node=None, label=None, message=None): + self.exitstatus = exitstatus + self.node = node + self.label = label + self.message = message + + def __str__(self): + prefix = "Unit test failed" + if self.label: + prefix += " ({label})".format(label=self.label) + if self.node: + prefix += " on {node}".format(node=self.node) + return "{prefix} with status {status}: '{message}'".format( + prefix=prefix, + status=self.exitstatus, + message=self.message, + ) diff --git a/teuthology/orchestra/run.py b/teuthology/orchestra/run.py index f31dfd0d7..085eea715 100644 --- a/teuthology/orchestra/run.py +++ b/teuthology/orchestra/run.py @@ -3,6 +3,7 @@ Paramiko run support """ import io +from pathlib import Path from paramiko import ChannelFile @@ -12,10 +13,11 @@ import socket import pipes import logging import shutil +from lxml import etree from teuthology.contextutil import safe_while from teuthology.exceptions import (CommandCrashedError, CommandFailedError, - ConnectionLostError) + ConnectionLostError, UnitTestError) log = logging.getLogger(__name__) @@ -34,12 +36,13 @@ class RemoteProcess(object): # for orchestra.remote.Remote to place a backreference 'remote', 'label', + 'unittest_xml', ] deadlock_warning = "Using PIPE for %s without wait=False would deadlock" def __init__(self, client, args, check_status=True, hostname=None, - label=None, timeout=None, wait=True, logger=None, cwd=None): + label=None, timeout=None, wait=True, logger=None, cwd=None, unittest_xml=None): """ Create the object. Does not initiate command execution. @@ -58,6 +61,8 @@ class RemoteProcess(object): :param logger: Alternative logger to use (optional) :param cwd: Directory in which the command will be executed (optional) + :param unittest_xml: Absolute path to unit-tests output XML file + (optional) """ self.client = client self.args = args @@ -84,6 +89,7 @@ class RemoteProcess(object): self.returncode = self.exitstatus = None self._wait = wait self.logger = logger or log + self.unittest_xml = unittest_xml or "" def execute(self): """ @@ -178,6 +184,17 @@ class RemoteProcess(object): # signal; sadly SSH does not tell us which signal raise CommandCrashedError(command=self.command) if self.returncode != 0: + if self.unittest_xml: + error_msg = None + try: + error_msg = find_unittest_error(self.unittest_xml) + except Exception as exc: + self.logger.error('Unable to scan logs, exception occurred: {exc}'.format(exc=repr(exc))) + if error_msg: + raise UnitTestError( + exitstatus=self.returncode, node=self.hostname, + label=self.label, message=error_msg + ) raise CommandFailedError( command=self.command, exitstatus=self.returncode, node=self.hostname, label=self.label @@ -221,6 +238,43 @@ class RemoteProcess(object): name=self.hostname, ) +def find_unittest_error(xmlfile_path): + """ + Load the unit test output XML file + and parse for failures and errors. + """ + + if not xmlfile_path: + return "No XML file was passed to process!" + try: + xml_path = Path(xmlfile_path) + if xml_path.is_file(): + tree = etree.parse(xmlfile_path) + failed_testcases = tree.xpath('.//failure/.. | .//error/..') + if len(failed_testcases) == 0: + log.debug("No failures or errors found in unit test's output xml file.") + return None + + error_message = f'Total {len(failed_testcases)} testcase/s did not pass. ' + + # show details of first error/failure for quick inspection + testcase1 = failed_testcases[0] + testcase1_casename = testcase1.get("name", "test-name") + testcase1_suitename = testcase1.get("classname", "suite-name") + testcase1_msg = f'Test `{testcase1_casename}` of `{testcase1_suitename}` did not pass.' + + for child in testcase1: + if child.tag in ['failure', 'error']: + fault_kind = child.tag.upper() + reason = child.get('message', 'NO MESSAGE FOUND IN XML FILE; CHECK LOGS.') + reason = reason[:reason.find('begin captured')] # remove captured logs/stdout + testcase1_msg = f'{fault_kind}: Test `{testcase1_casename}` of `{testcase1_suitename}` because {reason}' + break + + return (error_message + testcase1_msg).replace("\n", " ") + return f'XML output not found at `{xmlfile_path}`!' + except Exception as exc: + raise Exception("Somthing went wrong while searching for error in XML file: " + repr(exc)) class Raw(object): @@ -394,6 +448,7 @@ def run( quiet=False, timeout=None, cwd=None, + unittest_xml=None, # omit_sudo is used by vstart_runner.py omit_sudo=False ): @@ -429,6 +484,7 @@ def run( :param timeout: timeout value for args to complete on remote channel of paramiko :param cwd: Directory in which the command should be executed. + :param unittest_xml: Absolute path to unit-tests output XML file. """ try: transport = client.get_transport() @@ -446,7 +502,7 @@ def run( log.info("Running command with timeout %d", timeout) r = RemoteProcess(client, args, check_status=check_status, hostname=name, label=label, timeout=timeout, wait=wait, logger=logger, - cwd=cwd) + cwd=cwd, unittest_xml=unittest_xml) r.execute() r.setup_stdin(stdin) r.setup_output_stream(stderr, 'stderr', quiet) diff --git a/teuthology/orchestra/test/test_run.py b/teuthology/orchestra/test/test_run.py index 074d90b05..14b4b98f0 100644 --- a/teuthology/orchestra/test/test_run.py +++ b/teuthology/orchestra/test/test_run.py @@ -263,6 +263,10 @@ class TestRun(object): run.copy_and_close('', MagicMock()) run.copy_and_close(b'', MagicMock()) + def test_find_unittest_error(self): + unittest_xml = "xml_files/test_scan_nose.xml" + error_msg = run.find_unittest_error(unittest_xml) + assert error_msg == "Total 1 testcase/s did not pass. FAILURE: Test `test_set_bucket_tagging` of `s3tests_boto3.functional.test_s3` because 'NoSuchTagSetError' != 'NoSuchTagSet' -------------------- >> " class TestQuote(object): def test_quote_simple(self): diff --git a/teuthology/orchestra/test/xml_files/test_scan_nose.xml b/teuthology/orchestra/test/xml_files/test_scan_nose.xml new file mode 100644 index 000000000..d0fab4816 --- /dev/null +++ b/teuthology/orchestra/test/xml_files/test_scan_nose.xml @@ -0,0 +1,73 @@ + + + + + +> begin captured logging << -------------------- +botocore.hooks: DEBUG: Event choose-service-name: calling handler +PUT +/test-client.0-2txq2dyjghs0vdf-335 + +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb +botocore.auth: DEBUG: Signature: +8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 +botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 +botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'' +botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler +botocore.retryhandler: DEBUG: No retry needed. +GET +/test-client.0-2txq2dyjghs0vdf-335 +tagging= +host:smithi196.front.sepia.ceph.com +x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +x-amz-date:20220929T065029Z + +host;x-amz-content-sha256;x-amz-date +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +botocore.auth: DEBUG: StringToSign: +AWS4-HMAC-SHA256 +20220929T065029Z +20220929/us-east-1/s3/aws4_request +8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 +botocore.auth: DEBUG: Signature: +a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 +botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler +botocore.endpoint: DEBUG: Sending http request: +urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 +urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 +botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} +botocore.parsers: DEBUG: Response body: +b'NoSuchTagSetErrortest-client.0-2txq2dyjghs0vdf-335tx00000ebc589e4bcad8d86-0063354035-1157-default1157-default-default' +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler +botocore.retryhandler: DEBUG: No retry needed. +botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler > +--------------------- >> end captured logging << ---------------------]]> + \ No newline at end of file -- 2.47.3