scrape: searching backtrace with gzip

Scrape script that trying to find backtrace in gzip log files
can hit TypeError: a bytes-like object is required, not 'str' error
and fail to collect results. the gzip file need to be decoded.

Fixes: https://tracker.ceph.com/issues/64402
Signed-off-by: Nitzan Mordechai <nmordec@redhat.com>

diff --git a/teuthology/scrape.py b/teuthology/scrape.py
index 92e52f32219eae553e265f042a62e845dc629167..33a38e81c06e4e6506eb3ea7251d41be6075f598 100644 (file)
--- a/teuthology/scrape.py
+++ b/teuthology/scrape.py
@@ -3,7 +3,7 @@
 
 import difflib
 from errno import ENOENT
-from gzip import GzipFile
+import gzip
 import sys
 import os
 import yaml
@@ -361,6 +361,8 @@ class Job(object):
             return
 
         for line in grep(tlog_path, "command crashed with signal"):
+            if not line:
+                continue
             log.debug("Found a crash indication: {0}".format(line))
             # tasks.ceph.osd.1.plana82.stderr
             match = re.search(r"tasks.ceph.([^\.]+).([^\.]+).([^\.]+).stderr", line)
@@ -387,7 +389,8 @@ class Job(object):
                 ))
                 continue
 
-            bt, ass = self._search_backtrace(GzipFile(gzipped_log_path))
+            with gzip.open(gzipped_log_path, 'rt',  errors='ignore') as f:
+                bt, ass = self._search_backtrace(f)
             if ass and not self.assertion:
                 self.assertion = ass
             if bt:

diff --git a/teuthology/test/test_scrape.py b/teuthology/test/test_scrape.py
index f8a03520eced0c96cb3ec71848b2962b094a145b..ed281b4b2ea9163c6f324d63f581a492216a5954 100644 (file)
--- a/teuthology/test/test_scrape.py
+++ b/teuthology/test/test_scrape.py
@@ -1,5 +1,7 @@
 from __future__ import with_statement
 
+import glob
+import gzip
 import os
 import shutil
 import tempfile
@@ -12,7 +14,8 @@ class FakeResultDir(object):
     def __init__(self,
                 failure_reason="Dummy reason",
                 assertion="FAILED assert 1 == 2\n",
-                blank_backtrace=False
+                blank_backtrace=False,
+                assertion_osd=False,
     ):
         self.failure_reason = failure_reason
         self.assertion = assertion
@@ -35,6 +38,15 @@ class FakeResultDir(object):
                 f.write(self.assertion)
             f.write(" NOTE: a copy of the executable dummy text\n")
 
+        if assertion_osd:
+            host = "host1"
+            rem_log_dir = os.path.join(self.path, "remote", host, "log")
+            os.makedirs(rem_log_dir, exist_ok=True)
+            ceph_mon_log = os.path.join(rem_log_dir, "ceph-osd.0.log")
+            with open(ceph_mon_log, "w") as f:
+                f.write("ceph version 1000\n")
+                f.write(self.assertion)
+
     def __enter__(self):
         return self
 
@@ -165,3 +177,29 @@ class TestScrape(object):
         assert os.path.exists(os.path.join(d.path, "scrape.log"))
 
         shutil.rmtree(d.path)
+
+    def test_gzip_backtrace_decode(self):
+        with FakeResultDir(assertion="FAILED assert dummy backtrace line",
+                        blank_backtrace=True,
+                        assertion_osd=True) as d:
+
+            with open(os.path.join(d.path, "teuthology.log"), "a") as root_log:
+                root_log.write(
+                    "command crashed with signal SIGSEGV tasks.ceph.osd.0.host1.stderr\n"
+                )
+
+            pattern = os.path.join(d.path, "**", "ceph-osd.0.log")
+            raws = glob.glob(pattern, recursive=True)
+            assert len(raws) == 1, f"expected one raw log, found: {raws}"
+            raw_log = raws[0]
+            gz_log = raw_log + ".gz"
+
+            with gzip.open(gz_log, "wb") as out:
+                out.write(open(raw_log, "rb").read())
+            os.remove(raw_log)
+
+            assert not os.path.exists(raw_log)
+            assert os.path.exists(gz_log)
+
+            job = scrape.Job(d.path, 1)
+            assert job.get_assertion() == "FAILED assert dummy backtrace line"
\ No newline at end of file