Improve logs filtering

Author: mbriand

swattool/logsview.py

@@ -10,6 +10,7 @@
 import requests
 from simple_term_menu import TerminalMenu  # type: ignore
 
+from . import swatbotrest
 from . import swatbuild
 from . import utils
 from .webrequests import Session
@@ -27,6 +28,39 @@
 WHITE = "\x1b[1;37m"
 
 
+class _Highlight:
+    # pylint: disable=too-few-public-methods
+    def __init__(self, keyword: str, color: str, in_menu: bool):
+        self.keyword = keyword
+        self.color = color
+        self.in_menu = in_menu
+
+
+class _Filter:
+    # pylint: disable=too-few-public-methods
+    def __init__(self, pat: re.Pattern, enabled: bool, color: Optional[str],
+                 in_menu: bool):
+        self.pat = pat
+        self.enabled = enabled
+        self.color = color
+        self.in_menu = in_menu
+
+    def match(self, line: str) -> tuple[bool, Optional[_Highlight]]:
+        """Check if the filter matches a given line."""
+        if not self.enabled:
+            return (False, None)
+
+        match = self.pat.match(line)
+        if not match:
+            return (False, None)
+
+        if not self.color:
+            return (True, None)
+
+        hl = _Highlight(match.group("keyword"), self.color, self.in_menu)
+        return (True, hl)
+
+
 def show_logs_menu(build: swatbuild.Build) -> bool:
     """Show a menu allowing to select log file to analyze."""
     def get_failure_line(failure, logname):
@@ -51,22 +85,22 @@ def get_failure_line(failure, logname):
 
 
 def _format_log_line(linenum: int, text: str, colorized_line: Optional[int],
-                     highlight_lines: dict[int, tuple[str, str]]):
+                     highlight_lines: dict[int, _Highlight]):
     if linenum == colorized_line:
         if linenum in highlight_lines:
-            linecolor = highlight_lines[linenum][1]
+            linecolor = highlight_lines[linenum].color
         else:
             linecolor = CYAN
         text = f"{linecolor}{text}{RESET}"
     elif linenum in highlight_lines:
-        pat = highlight_lines[linenum][0]
-        color = highlight_lines[linenum][1]
+        pat = highlight_lines[linenum].keyword
+        color = highlight_lines[linenum].color
         text = re.sub(pat, f"{color}{pat}{RESET}", text)
     return text
 
 
 def _format_log_preview_line(linenum: int, text: str, colorized_line: int,
-                             highlight_lines: dict[int, tuple[str, str]]):
+                             highlight_lines: dict[int, _Highlight]):
     preview_text = text.replace('\t', '    ')
     formatted_text = _format_log_line(linenum, preview_text, colorized_line,
                                       highlight_lines)
@@ -85,33 +119,55 @@ def _get_preview_window(linenum: int, lines: list[str], preview_height: int
 
 
 def _format_log_preview(linenum: int, lines: list[str],
-                        highlight_lines: dict[int, tuple[str, str]],
+                        highlight_lines: dict[int, _Highlight],
                         preview_height: int) -> str:
     start, end = _get_preview_window(linenum, lines, preview_height)
     lines = [_format_log_preview_line(i, t, linenum, highlight_lines)
              for i, t in enumerate(lines[start: end], start=start + 1)]
     return "\n".join(lines)
 
 
-def _get_log_highlights(loglines: list[str]) -> dict[int, tuple[str, str]]:
-    pats = [(re.compile(r"(.*\s|^)(?P<keyword>\S*error):", flags=re.I),
-             RED),
-            (re.compile(r"(.*\s|^)(?P<keyword>\S*warning):", flags=re.I),
-             YELLOW),
-            ]
+def _get_log_highlights(loglines: list[str], failure: swatbuild.Failure
+                        ) -> dict[int, _Highlight]:
+    status = failure.build.status
+    test = failure.build.test
+    filters = [
+        # Toaster specific rules:
+        #  - Do nothing on "except xxxError:" (likely python code output).
+        #  - Match on "selenium .*exception:".
+        #  - Match on generic errors, but do not show in menu.
+        _Filter(re.compile(r".*except\s*\S*error:", flags=re.I),
+                test == "toaster", None, False),
+        _Filter(re.compile(r"(.*\s|^)(?P<keyword>selenium\.\S*exception):",
+                           flags=re.I),
+                test == "toaster", RED, status == swatbotrest.Status.ERROR),
+        _Filter(re.compile(r"(.*\s|^)(?P<keyword>\S*error):", flags=re.I),
+                test == "toaster", RED, status == swatbotrest.Status.ERROR),
+
+        # Generic rules:
+        #  - Match on "error:", show in menu if build status is error.
+        #  - Match on "warning:", show in menu if build status is warning.
+        _Filter(re.compile(r"(.*\s|^)(?P<keyword>\S*error):", flags=re.I),
+                True, RED, status == swatbotrest.Status.ERROR),
+        _Filter(re.compile(r"(.*\s|^)(?P<keyword>\S*warning):",
+                           flags=re.I),
+                True, YELLOW, status == swatbotrest.Status.WARNING),
+    ]
 
     highlight_lines = {}
     for linenum, line in enumerate(loglines, start=1):
-        for (pat, color) in pats:
-            match = pat.match(line)
-            if match:
-                highlight_lines[linenum] = (match.group("keyword"), color)
+        for filtr in filters:
+            matched, highlight = filtr.match(line)
+            if matched:
+                if highlight:
+                    highlight_lines[linenum] = highlight
+                break
 
     return highlight_lines
 
 
 def _show_log(loglines: list[str], selected_line: Optional[int],
-              highlight_lines: dict[int, tuple[str, str]],
+              highlight_lines: dict[int, _Highlight],
               preview_height: Optional[int]):
     colorlines = [_format_log_line(i, t, selected_line, highlight_lines)
                   for i, t in enumerate(loglines, start=1)]
@@ -150,20 +206,21 @@ def show_log_menu(failure: swatbuild.Failure, logname: str) -> bool:
 
     utils.clear()
     loglines = logdata.splitlines()
-    highlight_lines = _get_log_highlights(loglines)
+    highlights = _get_log_highlights(loglines, failure)
 
     entries = ["View entire log file|",
                "View entire log file in default editor|",
-               *[f"On line {line: 6d}: {highlight_lines[line][0]}|{line}"
-                 for line in sorted(highlight_lines)]
+               *[f"On line {line: 6d}: {highlights[line].keyword}|{line}"
+                 for line in sorted(highlights)
+                 if highlights[line].in_menu]
                ]
 
     preview_size = 0.6
     termheight = shutil.get_terminal_size((80, 20)).lines
     preview_height = int(preview_size * termheight)
 
     def preview(line):
-        return _format_log_preview(int(line), loglines, highlight_lines,
+        return _format_log_preview(int(line), loglines, highlights,
                                    preview_height)
 
     title = f"{failure.build.format_short_description()}: " \
@@ -178,9 +235,9 @@ def preview(line):
             return True
 
         if entry == 0:
-            _show_log(loglines, None, highlight_lines, None)
+            _show_log(loglines, None, highlights, None)
         elif entry == 1:
             utils.launch_in_system_defaultshow_in_less(logdata)
         else:
             _, _, num = entries[entry].partition('|')
-            _show_log(loglines, int(num), highlight_lines, preview_height)
+            _show_log(loglines, int(num), highlights, preview_height)