Integrate flaky test detection into test results finisher

rollouts/__init__.py

@@ -5,6 +5,8 @@
 # Declare the feature variants and parameters via Django Admin
 LIST_REPOS_GENERATOR_BY_OWNER_ID = Feature("list_repos_generator")
 
+FLAKY_TEST_DETECTION = Feature("flaky_test_detection")
+
 # Eventually we want all repos to use this
 # This flag will just help us with the rollout process
 USE_LABEL_INDEX_IN_REPORT_PROCESSING_BY_REPO_ID = Feature(

services/test_results.py

@@ -7,7 +7,7 @@
 from shared.yaml import UserYaml
 from sqlalchemy import desc
 
-from database.enums import ReportType
+from database.enums import FlakeSymptomType, ReportType
 from database.models import Commit, CommitReport, RepositoryFlag, TestInstance, Upload
 from services.report import BaseReportService
 from services.repository import (
@@ -119,12 +119,19 @@
     test_id: str
 
 
+@dataclass
+class TestResultsNotificationFlake:
+    flake_type: list[FlakeSymptomType]
+    is_new_flake: bool
+
+
 @dataclass
 class TestResultsNotificationPayload:
     failed: int
     passed: int
     skipped: int
     failures: List[TestResultsNotificationFailure]
+    flaky_tests: dict[str, TestResultsNotificationFlake] | None = None
 
 
 class TestResultsNotifier:
@@ -164,24 +171,53 @@
     def generate_test_description(
         self,
         fail: TestResultsNotificationFailure,
+        flake: TestResultsNotificationFlake | None = None,
     ):
         envs = [f"- {env}" for env in fail.envs] or ["- default"]
         env_section = "<br>".join(envs)
         test_description = (
-            f"Testsuite:<br>{fail.testsuite}<br>"
-            f"Test name:<br>{fail.testname}<br>"
-            f"Envs:<br>{env_section}<br>"
+            "<pre>"
+            f"**Testsuite:**<br>{fail.testsuite}<br><br>"
+            f"**Test name:**<br>{fail.testname}<br><br>"
+            f"**Envs:**<br>{env_section}<br>"
+            "</pre>"
         )
+
+        if flake is not None:
+            if flake.is_new_flake:
+                test_description = f":snowflake::card_index_dividers: **Newly Detected Flake**<br>{test_description}"
+            else:
+                test_description = f":snowflake::card_index_dividers: **Known Flaky Test**<br>{test_description}"
+
         return test_description
 
     def generate_failure_info(
         self,
         fail: TestResultsNotificationFailure,
+        flake: TestResultsNotificationFlake | None = None,
     ):
         if fail.failure_message is not None:
-            return fail.failure_message
+            failure_message = fail.failure_message
         else:
-            return "No failure message available"
+            failure_message = "No failure message available"
+
+        if flake is not None:
+
+            flake_messages = []
+            if FlakeSymptomType.FAILED_IN_DEFAULT_BRANCH in flake.flake_type:
+                flake_messages.append("Failure on default branch")
+            if FlakeSymptomType.CONSECUTIVE_DIFF_OUTCOMES in flake.flake_type:
+                flake_messages.append("Differing outcomes on the same commit")
+            if FlakeSymptomType.UNRELATED_MATCHING_FAILURES in flake.flake_type:
+                flake_messages.append("Matching failures on unrelated branches")
+            flake_section = "".join(
+                [
+                    f":snowflake: :card_index_dividers: **{msg}**<br>"
+                    for msg in flake_messages
+                ]
+            )
+            return f"{flake_section}<pre>{failure_message}</pre>"
+        return f"<pre>{failure_message}</pre>"
 
     def build_message(self, payload: TestResultsNotificationPayload) -> str:
         message = []
@@ -193,9 +229,38 @@
         ]
 
         completed = payload.failed + payload.passed + payload.skipped
-        results = f"Completed {completed} tests with **`{payload.failed} failed`**, {payload.passed} passed and {payload.skipped} skipped."
+        if payload.flaky_tests:
+            new_flakes = 0
+            existing_flakes = 0
+
+            for failure in payload.failures:
+                flake = payload.flaky_tests.get(failure.test_id, None)
+                if flake is not None:
+                    if flake.is_new_flake:
+                        new_flakes += 1
+                    else:
+                        existing_flakes += 1
+            existing_flake_section = (
+                f"{existing_flakes} known flaky" if existing_flakes else ""
+            )
+            comma_section = ", " if new_flakes and existing_flakes else ""
+            new_flake_section = (
+                f"{new_flakes} newly detected flaky" if new_flakes else ""
+            )
+            flake_section = (
+                f"({existing_flake_section}{comma_section}{new_flake_section})"
+                if (existing_flakes or new_flakes)
+                else ""
+            )
 
-        message.append(results)
+            results = [
+                f"Completed {completed} tests with **`{payload.failed} failed`**{flake_section}, {payload.passed} passed and {payload.skipped} skipped.",
+                f"- Total :snowflake:**{len(payload.flaky_tests)} flaky tests.**",
+            ]
+            message += results
+        else:
+            results = f"Completed {completed} tests with **`{payload.failed} failed`**, {payload.passed} passed and {payload.skipped} skipped."
+            message.append(results)
 
         details = [
             "<details><summary>View the full list of failed tests</summary>",
@@ -207,12 +272,12 @@
         message += details
 
         for fail in payload.failures:
-
-            test_description = self.generate_test_description(fail)
-            failure_information = self.generate_failure_info(fail)
-            single_test_row = (
-                f"| <pre>{test_description}</pre> | <pre>{failure_information}</pre> |"
-            )
+            flake = None
+            if payload.flaky_tests is not None:
+                flake = payload.flaky_tests.get(fail.test_id, None)
+            test_description = self.generate_test_description(fail, flake)
+            failure_information = self.generate_failure_info(fail, flake)
+            single_test_row = f"| {test_description} | {failure_information} |"
             message.append(single_test_row)
 
         return "\n".join(message)

services/tests/test_flake_detector.py

@@ -161,6 +161,32 @@ def test_flake_consecutive_differing_outcomes_no_main_branch_specified(dbsession
     assert flaky_tests[test_id] == {FlakeSymptomType.CONSECUTIVE_DIFF_OUTCOMES}
 
 
+def test_flake_consecutive_differing_outcomes_no_main_branch_specified(dbsession):
+    repoid = create_repo(
+        dbsession,
+    )
+    commitid = create_commit(dbsession, repoid, "not_main")
+    reportid = create_report(dbsession, commitid)
+    reportid2 = create_report(dbsession, commitid)
+    uploadid = create_upload(dbsession, reportid)
+    uploadid2 = create_upload(dbsession, reportid2)
+    test_id = create_test(dbsession, repoid)
+    ti1 = create_test_instance(
+        dbsession, test_id, uploadid, str(Outcome.Failure), "failure message"
+    )
+    _ = create_test_instance(dbsession, test_id, uploadid2, str(Outcome.Pass), None)
+
+    dbfd = DefaultBranchFailureDetector(dbsession, repoid)
+    umd = UnrelatedMatchesDetector()
+    dod = DiffOutcomeDetector()
+
+    fd = FlakeDetectionEngine(dbsession, repoid, [dbfd, dod, umd], None)
+    flaky_tests = fd.detect_flakes()
+
+    assert test_id in flaky_tests
+    assert flaky_tests[test_id] == {FlakeSymptomType.CONSECUTIVE_DIFF_OUTCOMES}
+
+
 def test_flake_matching_failures_on_unrelated_branches(dbsession):
     repoid = create_repo(
         dbsession,