# python-novaclient
lupa==1.14.1
# via teuthology (pyproject.toml)
+lxml==4.9.3
+ # via teuthology (pyproject.toml)
markupsafe==2.0.1
# via jinja2
mock==4.0.3
httplib2
humanfriendly
lupa
+ lxml
ndg-httpsclient
netaddr
paramiko
def __str__(self):
return self.message
+
+
+class UnitTestError(Exception):
+ """
+ Exception thrown on unit test failure
+ """
+ def __init__(self, exitstatus=None, node=None, label=None, message=None):
+ self.exitstatus = exitstatus
+ self.node = node
+ self.label = label
+ self.message = message
+
+ def __str__(self):
+ prefix = "Unit test failed"
+ if self.label:
+ prefix += " ({label})".format(label=self.label)
+ if self.node:
+ prefix += " on {node}".format(node=self.node)
+ if self.exitstatus:
+ prefix += " with status {status}".format(status=self.exitstatus)
+ return "{prefix}: '{message}'".format(
+ prefix=prefix,
+ message=self.message,
+ )
from teuthology.orchestra.opsys import OS
import teuthology.provision
from teuthology import misc
-from teuthology.exceptions import CommandFailedError
+from teuthology.exceptions import CommandFailedError, UnitTestError
+from teuthology.util.scanner import UnitTestScanner
from teuthology.misc import host_shortname
import errno
import re
r.remote = self
return r
+ def run_unit_test(self, xml_path_regex, output_yaml, **kwargs):
+ try:
+ r = self.run(**kwargs)
+ except CommandFailedError as exc:
+ if xml_path_regex:
+ error_msg = UnitTestScanner(remote=self).scan_and_write(xml_path_regex, output_yaml)
+ if error_msg:
+ raise UnitTestError(
+ exitstatus=exc.exitstatus, node=exc.node,
+ label=exc.label, message=error_msg
+ )
+ raise exc
+ return r
+
def _sftp_put_file(self, local_path, remote_path):
"""
Use the paramiko.SFTPClient to put a file. Returns the remote filename.
sftp.get(remote_path, local_path)
return local_path
- def _sftp_open_file(self, remote_path):
+ def _sftp_open_file(self, remote_path, mode=None):
"""
Use the paramiko.SFTPClient to open a file. Returns a
paramiko.SFTPFile object.
"""
sftp = self.ssh.open_sftp()
+ if mode:
+ return sftp.open(remote_path, mode)
return sftp.open(remote_path)
def _sftp_get_size(self, remote_path):
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="nosetests" tests="644" errors="0" failures="1" skip="79">
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_response" time="3.205"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_origin_wildcard" time="3.081"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_cors_header_option" time="3.119"></testcase>
+<testcase classname="s3tests_boto3.functional.test_s3" name="test_set_bucket_tagging" time="0.059"><failure type="builtins.AssertionError" message="'NoSuchTagSetError' != 'NoSuchTagSet' -------------------- >> begin captured logging << -------------------- botocore.hooks: DEBUG: Event choose-service-name: calling handler <function handle_service_name_alias at 0x7fb6def62d30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function add_generate_presigned_post at 0x7fb6defdad30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function lazy_call.<locals>._handler at 0x7fb6debf5d30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function add_generate_presigned_url at 0x7fb6defdaaf0> botocore.endpoint: DEBUG: Setting s3 timeout as (60, 60) botocore.client: DEBUG: Registering retry handlers for service: s3 botocore.hooks: DEBUG: Event before-parameter-build.s3.CreateBucket: calling handler <function validate_bucket_name at 0x7fb6def01430> botocore.hooks: DEBUG: Event before-parameter-build.s3.CreateBucket: calling handler <bound method S3RegionRedirector.redirect_from_cache of <botocore.utils.S3RegionRedirector object at 0x7fb6de659d30>> botocore.hooks: DEBUG: Event before-parameter-build.s3.CreateBucket: calling handler <bound method S3ArnParamHandler.handle_arn of <botocore.utils.S3ArnParamHandler object at 0x7fb6de659580>> botocore.hooks: DEBUG: Event before-parameter-build.s3.CreateBucket: calling handler <function generate_idempotent_uuid at 0x7fb6def01280> botocore.hooks: DEBUG: Event before-call.s3.CreateBucket: calling handler <function add_expect_header at 0x7fb6def01790> botocore.hooks: DEBUG: Event before-call.s3.CreateBucket: calling handler <bound method S3RegionRedirector.set_request_url of <botocore.utils.S3RegionRedirector object at 0x7fb6de659d30>> botocore.hooks: DEBUG: Event before-call.s3.CreateBucket: calling handler <function add_recursion_detection_header at 0x7fb6deef7ee0> botocore.hooks: DEBUG: Event before-call.s3.CreateBucket: calling handler <function inject_api_version_header_if_needed at 0x7fb6def02af0> botocore.endpoint: DEBUG: Making request for OperationModel(name=CreateBucket) with params: {'url_path': '/test-client.0-2txq2dyjghs0vdf-335', 'query_string': {}, 'method': 'PUT', 'headers': {'User-Agent': 'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82'}, 'body': b'', 'url': 'http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335', 'context': {'client_region': 'us-east-1', 'client_config': <botocore.config.Config object at 0x7fb6de1298b0>, 'has_streaming_input': False, 'auth_type': None, 'signing': {'bucket': 'test-client.0-2txq2dyjghs0vdf-335'}}} botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler <bound method RequestSigner.handler of <botocore.signers.RequestSigner object at 0x7fb6de129e20>> botocore.hooks: DEBUG: Event choose-signer.s3.CreateBucket: calling handler <bound method S3EndpointSetter.set_signer of <botocore.utils.S3EndpointSetter object at 0x7fb6de6598b0>> botocore.hooks: DEBUG: Event choose-signer.s3.CreateBucket: calling handler <function set_operation_specific_signer at 0x7fb6def01160> botocore.hooks: DEBUG: Event before-sign.s3.CreateBucket: calling handler <bound method S3EndpointSetter.set_endpoint of <botocore.utils.S3EndpointSetter object at 0x7fb6de6598b0>> botocore.utils: DEBUG: Using S3 path style addressing. botocore.auth: DEBUG: Calculating signature using v4 auth. botocore.auth: DEBUG: CanonicalRequest: PUT /test-client.0-2txq2dyjghs0vdf-335 host:smithi196.front.sepia.ceph.com x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 x-amz-date:20220929T065029Z host;x-amz-content-sha256;x-amz-date e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 botocore.auth: DEBUG: StringToSign: AWS4-HMAC-SHA256 20220929T065029Z 20220929/us-east-1/s3/aws4_request ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb botocore.auth: DEBUG: Signature: 8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45 botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler <function add_retry_headers at 0x7fb6def031f0> botocore.endpoint: DEBUG: Sending http request: <AWSPreparedRequest stream_output=False, method=PUT, url=http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335, headers={'User-Agent': b'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82', 'X-Amz-Date': b'20220929T065029Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=EBIAVLDBAMGWPWFMDQHA/20220929/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45', 'amz-sdk-invocation-id': b'1795d992-7f27-43c5-ae48-06ef5ea88c6c', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0'}> urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0 botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} botocore.parsers: DEBUG: Response body: b'' botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler <botocore.retryhandler.RetryHandler object at 0x7fb6de6597c0> botocore.retryhandler: DEBUG: No retry needed. botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler <bound method S3RegionRedirector.redirect_from_error of <botocore.utils.S3RegionRedirector object at 0x7fb6de659d30>> botocore.hooks: DEBUG: Event choose-service-name: calling handler <function handle_service_name_alias at 0x7fb6def62d30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function add_generate_presigned_post at 0x7fb6defdad30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function lazy_call.<locals>._handler at 0x7fb6debf5d30> botocore.hooks: DEBUG: Event creating-client-class.s3: calling handler <function add_generate_presigned_url at 0x7fb6defdaaf0> botocore.endpoint: DEBUG: Setting s3 timeout as (60, 60) botocore.client: DEBUG: Registering retry handlers for service: s3 botocore.hooks: DEBUG: Event before-parameter-build.s3.GetBucketTagging: calling handler <function validate_bucket_name at 0x7fb6def01430> botocore.hooks: DEBUG: Event before-parameter-build.s3.GetBucketTagging: calling handler <bound method S3RegionRedirector.redirect_from_cache of <botocore.utils.S3RegionRedirector object at 0x7fb6dec1eaf0>> botocore.hooks: DEBUG: Event before-parameter-build.s3.GetBucketTagging: calling handler <bound method S3ArnParamHandler.handle_arn of <botocore.utils.S3ArnParamHandler object at 0x7fb6dec1ec10>> botocore.hooks: DEBUG: Event before-parameter-build.s3.GetBucketTagging: calling handler <function generate_idempotent_uuid at 0x7fb6def01280> botocore.hooks: DEBUG: Event before-call.s3.GetBucketTagging: calling handler <function add_expect_header at 0x7fb6def01790> botocore.hooks: DEBUG: Event before-call.s3.GetBucketTagging: calling handler <bound method S3RegionRedirector.set_request_url of <botocore.utils.S3RegionRedirector object at 0x7fb6dec1eaf0>> botocore.hooks: DEBUG: Event before-call.s3.GetBucketTagging: calling handler <function add_recursion_detection_header at 0x7fb6deef7ee0> botocore.hooks: DEBUG: Event before-call.s3.GetBucketTagging: calling handler <function inject_api_version_header_if_needed at 0x7fb6def02af0> botocore.endpoint: DEBUG: Making request for OperationModel(name=GetBucketTagging) with params: {'url_path': '/test-client.0-2txq2dyjghs0vdf-335?tagging', 'query_string': {}, 'method': 'GET', 'headers': {'User-Agent': 'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82'}, 'body': b'', 'url': 'http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335?tagging', 'context': {'client_region': 'us-east-1', 'client_config': <botocore.config.Config object at 0x7fb6de2c5220>, 'has_streaming_input': False, 'auth_type': None, 'signing': {'bucket': 'test-client.0-2txq2dyjghs0vdf-335'}}} botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler <bound method RequestSigner.handler of <botocore.signers.RequestSigner object at 0x7fb6de2c5dc0>> botocore.hooks: DEBUG: Event choose-signer.s3.GetBucketTagging: calling handler <bound method S3EndpointSetter.set_signer of <botocore.utils.S3EndpointSetter object at 0x7fb6dec1e760>> botocore.hooks: DEBUG: Event choose-signer.s3.GetBucketTagging: calling handler <function set_operation_specific_signer at 0x7fb6def01160> botocore.hooks: DEBUG: Event before-sign.s3.GetBucketTagging: calling handler <bound method S3EndpointSetter.set_endpoint of <botocore.utils.S3EndpointSetter object at 0x7fb6dec1e760>> botocore.utils: DEBUG: Using S3 path style addressing. botocore.auth: DEBUG: Calculating signature using v4 auth. botocore.auth: DEBUG: CanonicalRequest: GET /test-client.0-2txq2dyjghs0vdf-335 tagging= host:smithi196.front.sepia.ceph.com x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 x-amz-date:20220929T065029Z host;x-amz-content-sha256;x-amz-date e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 botocore.auth: DEBUG: StringToSign: AWS4-HMAC-SHA256 20220929T065029Z 20220929/us-east-1/s3/aws4_request 8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575 botocore.auth: DEBUG: Signature: a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1 botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler <function add_retry_headers at 0x7fb6def031f0> botocore.endpoint: DEBUG: Sending http request: <AWSPreparedRequest stream_output=False, method=GET, url=http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335?tagging, headers={'User-Agent': b'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82', 'X-Amz-Date': b'20220929T065029Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=EBIAVLDBAMGWPWFMDQHA/20220929/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1', 'amz-sdk-invocation-id': b'c4afeb5a-6a87-4e29-83ef-b96195038217', 'amz-sdk-request': b'attempt=1'}> urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80 urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248 botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'} botocore.parsers: DEBUG: Response body: b'<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchTagSetError</Code><BucketName>test-client.0-2txq2dyjghs0vdf-335</BucketName><RequestId>tx00000ebc589e4bcad8d86-0063354035-1157-default</RequestId><HostId>1157-default-default</HostId></Error>' botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler <botocore.retryhandler.RetryHandler object at 0x7fb6dec1ebe0> botocore.retryhandler: DEBUG: No retry needed. botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler <bound method S3RegionRedirector.redirect_from_error of <botocore.utils.S3RegionRedirector object at 0x7fb6dec1eaf0>> --------------------- >> end captured logging << ---------------------"><![CDATA[ File "/usr/lib/python3.8/unittest/case.py", line 60, in testPartExecutor
+ yield
+ File "/usr/lib/python3.8/unittest/case.py", line 676, in run
+ self._callTestMethod(testMethod)
+ File "/usr/lib/python3.8/unittest/case.py", line 633, in _callTestMethod
+ method()
+ File "/home/ubuntu/cephtest/s3-tests-client.0/virtualenv/lib/python3.8/site-packages/nose/case.py", line 198, in runTest
+ self.test(*self.arg)
+ File "/home/ubuntu/cephtest/s3-tests-client.0/s3tests_boto3/functional/test_s3.py", line 7692, in test_set_bucket_tagging
+ eq(error_code, 'NoSuchTagSet')
+ File "/home/ubuntu/cephtest/s3-tests-client.0/virtualenv/lib/python3.8/site-packages/nose/tools/trivial.py", line 29, in eq_
+ raise AssertionError(msg or "%r != %r" % (a, b))
+'NoSuchTagSetError' != 'NoSuchTagSet'
+-------------------- >> begin captured logging << --------------------
+botocore.hooks: DEBUG: Event choose-service-name: calling handler <function handle_service_name_alias at 0x7fb6def62d30>
+PUT
+/test-client.0-2txq2dyjghs0vdf-335
+
+host:smithi196.front.sepia.ceph.com
+x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+x-amz-date:20220929T065029Z
+
+host;x-amz-content-sha256;x-amz-date
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+botocore.auth: DEBUG: StringToSign:
+AWS4-HMAC-SHA256
+20220929T065029Z
+20220929/us-east-1/s3/aws4_request
+ddfd952c0ac842cff08711f6b1425bec213bd1f69ae5ae6f37afb7a2f66e7fcb
+botocore.auth: DEBUG: Signature:
+8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45
+botocore.hooks: DEBUG: Event request-created.s3.CreateBucket: calling handler <function add_retry_headers at 0x7fb6def031f0>
+botocore.endpoint: DEBUG: Sending http request: <AWSPreparedRequest stream_output=False, method=PUT, url=http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335, headers={'User-Agent': b'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82', 'X-Amz-Date': b'20220929T065029Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=EBIAVLDBAMGWPWFMDQHA/20220929/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=8b7f685e9b8a9a807437088da293390ac21ed9a10acf51903a8da2281bdc9c45', 'amz-sdk-invocation-id': b'1795d992-7f27-43c5-ae48-06ef5ea88c6c', 'amz-sdk-request': b'attempt=1', 'Content-Length': '0'}>
+urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80
+urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "PUT /test-client.0-2txq2dyjghs0vdf-335 HTTP/1.1" 200 0
+botocore.parsers: DEBUG: Response headers: {'x-amz-request-id': 'tx00000e29af2294ab8b56c-0063354035-1157-default', 'Content-Length': '0', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'}
+botocore.parsers: DEBUG: Response body:
+b''
+botocore.hooks: DEBUG: Event needs-retry.s3.CreateBucket: calling handler <botocore.retryhandler.RetryHandler object at 0x7fb6de6597c0>
+botocore.retryhandler: DEBUG: No retry needed.
+GET
+/test-client.0-2txq2dyjghs0vdf-335
+tagging=
+host:smithi196.front.sepia.ceph.com
+x-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+x-amz-date:20220929T065029Z
+
+host;x-amz-content-sha256;x-amz-date
+e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+botocore.auth: DEBUG: StringToSign:
+AWS4-HMAC-SHA256
+20220929T065029Z
+20220929/us-east-1/s3/aws4_request
+8a096d01796a8a6afca50c1bc3bc5c9098917c26a6dba7e752412ce31041c575
+botocore.auth: DEBUG: Signature:
+a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1
+botocore.hooks: DEBUG: Event request-created.s3.GetBucketTagging: calling handler <function add_retry_headers at 0x7fb6def031f0>
+botocore.endpoint: DEBUG: Sending http request: <AWSPreparedRequest stream_output=False, method=GET, url=http://smithi196.front.sepia.ceph.com:80/test-client.0-2txq2dyjghs0vdf-335?tagging, headers={'User-Agent': b'Boto3/1.24.82 Python/3.8.10 Linux/5.4.0-126-generic Botocore/1.27.82', 'X-Amz-Date': b'20220929T065029Z', 'X-Amz-Content-SHA256': b'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', 'Authorization': b'AWS4-HMAC-SHA256 Credential=EBIAVLDBAMGWPWFMDQHA/20220929/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a58a94727b0c0d6d43e8783c91499ce9a9758260aa09a286524c0eb1bc4883d1', 'amz-sdk-invocation-id': b'c4afeb5a-6a87-4e29-83ef-b96195038217', 'amz-sdk-request': b'attempt=1'}>
+urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): smithi196.front.sepia.ceph.com:80
+urllib3.connectionpool: DEBUG: http://smithi196.front.sepia.ceph.com:80 "GET /test-client.0-2txq2dyjghs0vdf-335?tagging HTTP/1.1" 404 248
+botocore.parsers: DEBUG: Response headers: {'Content-Length': '248', 'x-amz-request-id': 'tx00000ebc589e4bcad8d86-0063354035-1157-default', 'Accept-Ranges': 'bytes', 'Content-Type': 'application/xml', 'Date': 'Thu, 29 Sep 2022 06:50:29 GMT', 'Connection': 'Keep-Alive'}
+botocore.parsers: DEBUG: Response body:
+b'<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchTagSetError</Code><BucketName>test-client.0-2txq2dyjghs0vdf-335</BucketName><RequestId>tx00000ebc589e4bcad8d86-0063354035-1157-default</RequestId><HostId>1157-default-default</HostId></Error>'
+botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler <botocore.retryhandler.RetryHandler object at 0x7fb6dec1ebe0>
+botocore.retryhandler: DEBUG: No retry needed.
+botocore.hooks: DEBUG: Event needs-retry.s3.GetBucketTagging: calling handler <bound method S3RegionRedirector.redirect_from_error of <botocore.utils.S3RegionRedirector object at 0x7fb6dec1eaf0>>
+--------------------- >> end captured logging << ---------------------]]></failure></testcase>
+</testsuite>
\ No newline at end of file
--- /dev/null
+import logging
+import yaml
+from typing import Optional, Tuple
+from collections import defaultdict
+from lxml import etree
+
+log = logging.getLogger(__name__)
+
+
+class Scanner():
+ def __init__(self, remote=None) -> None:
+ self.summary_data = []
+ self.remote = remote
+
+ def _parse(self, file_content) -> Tuple[str, dict]:
+ """
+ This parses file_content and returns:
+ :returns: a message string
+ :returns: data dictionary with additional info
+
+ Just an abstract method in Scanner class,
+ to be defined in inherited classes.
+ """
+ raise NotImplementedError
+
+ def scan_file(self, path: str) -> Optional[str]:
+ if not path:
+ return None
+ try:
+ file = self.remote._sftp_open_file(path, 'r')
+ file_content = file.read()
+ txt, data = self._parse(file_content)
+ if data:
+ data["file_path"] = path
+ self.summary_data += [data]
+ file.close()
+ return txt
+ except Exception as exc:
+ log.error(str(exc))
+
+ def scan_all_files(self, path_regex: str) -> [str]:
+ """
+ Scans all files matching path_regex
+ and collect additional data in self.summary_data
+
+ :param path_regex: Regex string to find all the files which have to be scanned.
+ Example: /path/to/dir/*.xml
+ """
+ (_, stdout, _) = self.remote.ssh.exec_command(f'ls -d {path_regex}', timeout=200)
+
+ files = stdout.read().decode().split('\n')
+
+ extracted_txts = []
+ for fpath in files:
+ txt = self.scan_file(fpath)
+ if txt:
+ extracted_txts += [txt]
+ return extracted_txts
+
+ def write_summary(self, yaml_path: str) -> None:
+ """
+ Create yaml file locally
+ with self.summary_data.
+ """
+ if self.summary_data and yaml_path:
+ with open(yaml_path, 'a') as f:
+ yaml.safe_dump(self.summary_data, f, default_flow_style=False)
+ else:
+ log.info("summary_data or yaml_file is empty!")
+
+
+class UnitTestScanner(Scanner):
+ def __init__(self, remote=None) -> None:
+ super().__init__(remote)
+
+ def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+ xml_tree = etree.fromstring(file_content)
+
+ failed_testcases = xml_tree.xpath('.//failure/.. | .//error/..')
+ if len(failed_testcases) == 0:
+ return None, None
+
+ exception_txt = ""
+ error_data = defaultdict(list)
+ for testcase in failed_testcases:
+ testcase_name = testcase.get("name", "test-name")
+ testcase_suitename = testcase.get("classname", "suite-name")
+ for child in testcase:
+ if child.tag in ['failure', 'error']:
+ fault_kind = child.tag
+ reason = child.get('message', 'No message found in xml output, check logs.')
+ short_reason = reason[:200]
+ error_data[testcase_suitename] += [{
+ "kind": fault_kind,
+ "testcase": testcase_name,
+ "message": reason,
+ }]
+ if not exception_txt:
+ exception_txt = f'{fault_kind.upper()}: Test `{testcase_name}` of `{testcase_suitename}`. Reason: {short_reason}.'
+
+ return exception_txt, { "failed_testsuites": dict(error_data), "num_of_failures": len(failed_testcases) }
+
+ def scan_and_write(self, path_regex: str, summary_path: str) -> Optional[str]:
+ """
+ Scan all files matching 'path_regex'
+ and write summary in 'summary_path'.
+ """
+ try:
+ errors = self.scan_all_files(path_regex)
+ self.write_summary(summary_path)
+ if errors:
+ return errors[0]
+ except Exception as scanner_exc:
+ log.error(str(scanner_exc))
+
+
+class ValgrindScanner(Scanner):
+ def __init__(self, remote=None) -> None:
+ super().__init__(remote)
+
+ def _parse(self, file_content: str) -> Tuple[Optional[str], Optional[dict]]:
+ xml_tree = etree.fromstring(file_content)
+ if not xml_tree:
+ return None, None
+
+ error_tree = xml_tree.find('error')
+ if error_tree is None:
+ return None, None
+
+ error_data = {
+ "kind": error_tree.findtext("kind"),
+ "traceback": [],
+ }
+ for frame in error_tree.xpath("stack/frame"):
+ if len(error_data["traceback"]) >= 5:
+ break
+ curr_frame = {
+ "file": f"{frame.findtext('dir', '')}/{frame.findtext('file', '')}",
+ "line": frame.findtext("line", ''),
+ "function": frame.findtext("fn", ''),
+ }
+ error_data["traceback"].append(curr_frame)
+
+ traceback_functions = "\n".join(
+ frame.get("function", "N/A")
+ for frame in error_data["traceback"][:3]
+ )
+ exception_text = f"valgrind error: {error_data['kind']}\n{traceback_functions}"
+ return exception_text, error_data
projects / teuthology.git / commitdiff