feat: upsert daily test rollup when we process a test instance (#699)

database/models/reports.py

@@ -350,3 +350,31 @@ class Flake(CodecovBaseModel, MixinBaseClassNoExternalID):
     fail_count = Column(types.Integer)
     start_date = Column(types.DateTime)
     end_date = Column(types.DateTime, nullable=True)
+
+
+class DailyTestRollup(CodecovBaseModel, MixinBaseClassNoExternalID):
+    __tablename__ = "reports_dailytestrollups"
+
+    test_id = Column(types.Text, ForeignKey("reports_test.id"))
+    test = relationship(Test, backref=backref("dailytestrollups"))
+    date = Column(types.Date)
+    repoid = Column(types.Integer)
+    branch = Column(types.Text)
+
+    fail_count = Column(types.Integer)
+    skip_count = Column(types.Integer)
+    pass_count = Column(types.Integer)
+    last_duration_seconds = Column(types.Float)
+    avg_duration_seconds = Column(types.Float)
+    latest_run = Column(types.DateTime)
+    commits_where_fail = Column(types.ARRAY(types.Text))
+
+    __table_args__ = (
+        UniqueConstraint(
+            "repoid",
+            "date",
+            "branch",
+            "test_id",
+            name="reports_dailytestrollups_repoid_date_branch_test",
+        ),
+    )

requirements.in

@@ -1,5 +1,5 @@
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
-https://github.com/codecov/shared/archive/cdf263f5173c16585030bcab40e639045c69e199.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/5cc5f48e7d847d3d47ff875a682192465161155f.tar.gz#egg=shared
 https://github.com/codecov/test-results-parser/archive/1507de2241601d678e514c08b38426e48bb6d47d.tar.gz#egg=test-results-parser
 https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring
 asgiref>=3.7.2

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.12
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 amqp==5.2.0
     # via kombu
@@ -357,7 +357,7 @@ sentry-sdk[celery]==2.13.0
     # via
     #   -r requirements.in
     #   shared
-shared @ https://github.com/codecov/shared/archive/cdf263f5173c16585030bcab40e639045c69e199.tar.gz
+shared @ https://github.com/codecov/shared/archive/5cc5f48e7d847d3d47ff875a682192465161155f.tar.gz
     # via -r requirements.in
 six==1.16.0
     # via

tasks/test_results_processor.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import zlib
+from datetime import date, datetime
 from io import BytesIO
 from sys import getsizeof
 from typing import List
@@ -13,6 +14,7 @@
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.orm import Session
 from test_results_parser import (
+    Outcome,
     ParserError,
     Testrun,
     parse_junit_xml,
@@ -21,7 +23,7 @@
 )
 
 from app import celery_app
-from database.models import Repository, Test, TestInstance, Upload
+from database.models import DailyTestRollup, Repository, Test, TestInstance, Upload
 from services.archive import ArchiveService
 from services.test_results import generate_flags_hash, generate_test_id
 from services.yaml import read_yaml_field
@@ -110,6 +112,7 @@ def _bulk_write_tests_to_db(
         with sentry_sdk.metrics.timing(key="test_results.processor.write_to_db"):
             test_data = []
             test_instance_data = []
+            daily_totals = dict()
             for testrun in parsed_testruns:
                 # Build up the data for bulk insert
                 name = testrun.name
@@ -143,12 +146,105 @@ def _bulk_write_tests_to_db(
                     )
                 )
 
+                def update_daily_total():
+                    daily_totals[test_id]["last_duration_seconds"] = duration_seconds
+                    daily_totals[test_id]["avg_duration_seconds"] = (
+                        daily_totals[test_id]["avg_duration_seconds"]
+                        * (
+                            daily_totals[test_id]["pass_count"]
+                            + daily_totals[test_id]["fail_count"]
+                        )
+                        + duration_seconds
+                    ) / (
+                        daily_totals[test_id]["pass_count"]
+                        + daily_totals[test_id]["fail_count"]
+                        + 1
+                    )
+
+                    if outcome == str(Outcome.Pass):
+                        daily_totals[test_id]["pass_count"] += 1
+                    elif outcome == str(Outcome.Failure) or outcome == str(
+                        Outcome.Error
+                    ):
+                        daily_totals[test_id]["fail_count"] += 1
+                    elif outcome == str(Outcome.Skip):
+                        daily_totals[test_id]["skip_count"] += 1
+
+                def create_daily_total():
+                    daily_totals[test_id] = {
+                        "test_id": test_id,
+                        "repoid": repoid,
+                        "last_duration_seconds": duration_seconds,
+                        "avg_duration_seconds": duration_seconds,
+                        "pass_count": 1 if outcome == str(Outcome.Pass) else 0,
+                        "fail_count": 1
+                        if outcome == str(Outcome.Failure)
+                        or outcome == str(Outcome.Error)
+                        else 0,
+                        "skip_count": 1 if outcome == str(Outcome.Skip) else 0,
+                        "branch": branch,
+                        "date": date.today(),
+                        "latest_run": datetime.now(),
+                        "commits_where_fail": [commitid]
+                        if (
+                            outcome == str(Outcome.Failure)
+                            or outcome == str(Outcome.Error)
+                        )
+                        else [],
+                    }
+
+                if outcome != str(Outcome.Skip):
+                    if test_id in daily_totals:
+                        update_daily_total()
+                    else:
+                        create_daily_total()
+
             # Save Tests
             insert_on_conflict_do_nothing = (
                 insert(Test.__table__).values(test_data).on_conflict_do_nothing()
             )
             db_session.execute(insert_on_conflict_do_nothing)
             db_session.flush()
+
+            # Upsert Daily Test Totals
+            stmt = insert(DailyTestRollup.__table__).values(list(daily_totals.values()))
+            stmt = stmt.on_conflict_do_update(
+                index_elements=[
+                    "repoid",
+                    "branch",
+                    "test_id",
+                    "date",
+                ],
+                set_={
+                    "last_duration_seconds": stmt.excluded.last_duration_seconds,
+                    "avg_duration_seconds": (
+                        DailyTestRollup.__table__.c.avg_duration_seconds
+                        * (
+                            DailyTestRollup.__table__.c.pass_count
+                            + DailyTestRollup.__table__.c.fail_count
+                        )
+                        + stmt.excluded.avg_duration_seconds
+                    )
+                    / (
+                        DailyTestRollup.__table__.c.pass_count
+                        + DailyTestRollup.__table__.c.fail_count
+                        + 1
+                    ),
+                    "latest_run": stmt.excluded.latest_run,
+                    "pass_count": DailyTestRollup.__table__.c.pass_count
+                    + stmt.excluded.pass_count,
+                    "skip_count": DailyTestRollup.__table__.c.skip_count
+                    + stmt.excluded.skip_count,
+                    "fail_count": DailyTestRollup.__table__.c.fail_count
+                    + stmt.excluded.fail_count,
+                    "commits_where_fail": DailyTestRollup.__table__.c.commits_where_fail
+                    + stmt.excluded.commits_where_fail,
+                },
+            )
+
+            db_session.execute(stmt)
+            db_session.flush()
+
             # Save TestInstances
             insert_test_instances = insert(TestInstance.__table__).values(
                 test_instance_data

tasks/tests/samples/sample_multi_test_part_1.txt

@@ -0,0 +1,11 @@
+{
+    "test_results_files": [
+        {
+            "filename": "codecov-demo/multi-test.xml",
+            "format": "base64+compressed",
+            "data": "eJy1U01vgkAQvfsrJtvEaCywi18FBdOkNqaHnpp6bDawICkCYZe2/vsOi1aqxvTSTYDM25n3HrM788XXNoUPUcokzzzCTEpAZEEeJlnskUpFxh1Z+J25ElLJKsGP3wFcRwAyvhUeKXY1gsVlmZfSI8gT8SStSoEBIyDfk6IQod7QtRpVSV1LTWpPiOatVw1KxbeFR2xqjww6NWznhQ1daruMmZOpMxk6Bh25FLk2uVSNg9fH1fP66WG9csw0D3hK/B9G7TbgUkCQcimbfO3Cqt9vUZUFChtgFjsCx90fvGWUshav5t7/JqBpyWPMQgVRKmDQjdXMg3H3htEZDAA+NwLzGHhQ8izEJxY9eguM9okfiugXrW5E20Ov754l1AKwlzulPKifFS3h1OCFlMEVs52LRs9b6Y5duNdSGC/rezG39s1qHY11OJv/P64/a0VVmu4gyHEuRHiQgBO00Zia9hWFJtZzgkNktafoG8+FAfk=",
+            "labels": ""
+        }
+    ],
+    "metadata": {}
+}

tasks/tests/samples/sample_multi_test_part_2.txt

@@ -0,0 +1,11 @@
+{
+    "test_results_files": [
+        {
+            "filename": "codecov-demo/multi-test.xml",
+            "format": "base64+compressed",
+            "data": "eJzNU8tugzAQvOcrVq4UJUoBm7wKCUSVmirqoaeqOVYWGIJKAGHTNn/fxeRBk7TnWgK0492ZweudL762KXyIUiZ55hFmUgIiC/IwyWKPVCoy7sjC78yVkEpWCX78DuA6AZDxrfBIsasRLC7LvJQeQZ6IJ2lVCgwYAfmeFIUI9Yau1ahK6lpqUntCNG+9alAqvi08YlN7ZNCpYTsvbOhS22XMnEydydAx6MilyLXJpWocvD6untdPD+uVY6Z5wFPiHxm124BLAUHKpWzytQurfr9FVRYoPACz2BE47R7xllFqt3mtA/E/0oqqNN1BkGNTRXiQgDO00Riak5aCZtw3DbAFkseYg7yiVMCgG6uZB+PuDaMzGAB8bpAKcQ9KnoX4xKJHb4HRPvFDEf2g1W1t/2Wv714k1AKwlzunPKhfFC3h3OCVlMEfZjtXjV42yx27cK+lMF7Wt3xu7Q/r9yY1sZ4THCKrPUXfoXQB+w==",
+            "labels": ""
+        }
+    ],
+    "metadata": {}
+}

tasks/tests/samples/sample_test.txt

@@ -1 +1 @@
-{"test_results_files": [{"filename": "codecov-demo/temp.junit.xml", "format": "base64+compressed", "data": "eJy1VMluwjAQvfMVI1dCoBbHZiklJEFVS4V66Kkqx8okBqw6i2KHwt/XWSChnCrRXDLjefNm8Uuc2T6UsOOpEnHkIooJAh75cSCijYsyve49oJnnaK60yoR5NWyIWMhdlBzyE5OWpnGqXGQY1kzILOXGoQjUl0gSHhSBItdFQ2OJPJdgMuqXjtIsTFzUJ/1Bj9IeuX+n1KZjmwwxoZSMDWwbK13W/HhZvC1fn5eLCZaxzyQq2/KZ4uBLplQJY4nAmocJNhA/k0zHKc5xn7WPqimKYxYEjc6Iad66DrHKVjplvv4f9jCTWiTy0GQnV2MPxE4E/Lxzz6muGMzFKbbJaZXiqQajIHBdIHjUvqFkCrcA31tugEUA2lJP11nkayM3eKobKIsA00D2lAz9CV9NSHujpx16B/3uievI9mceU/sChryAr6ExZKdrtwpw+VQjXeSVPVVjtubn6HoBp8iVlnDGd9VFtFpGFFYuCqsWgfVLFDg52ANiw2Mxpyk3zz94x6qU4DnWUW2VWfwkmrbyfgBbcXMH", "labels": ""}], "metadata": {}}
+{"test_results_files": [{"filename": "codecov-demo/temp.junit.xml", "format": "base64+compressed", "data": "eJy1VMluwjAQvfMVI1dCoBbHZiklJEFVS4V66Kkqx8okBqw6i2KHwt/XWSChnCrRXDLjefNm8Uuc2T6UsOOpEnHkIooJAh75cSCijYsyve49oJnnaK60yoR5NWyIWMhdlBzyE5OWpnGqXGQY1kzILOXGoQjUl0gSHhSBItdFQ2OJPJdgMuqXjtIsTFzUJ/1Bj9IeuX+n1KZjmwwxoZSMDWwbK13W/HhZvC1fn5eLCZaxzyQq2/KZ4uBLplQJY4nAmocJNhA/k0zHKc5xn7WPqimKYxYEjc6Iad66DrHKVjplvv4f9jCTWiTy0GQnV2MPxE4E/Lxzz6muGMzFKbbJaZXiqQajIHBdIHjUvqFkCrcA31tugEUA2lJP11nkayM3eKobKIsA00D2lAz9CV9NSHujpx16B/3uievI9mceU/sChryAr6ExZKdrtwpw+VQjXeSVPVVjtubn6HoBp8iVlnDGd9VFtFpGFFYuCqsWgfVLFDg52ANiw2Mxpyk3zz94x6qU4DnWUW2VWfwkmrbyfgBbcXMH", "labels": ""}], "metadata": {}}