From 27fd7a54cda2834273963a4333c3cb522467d3dc Mon Sep 17 00:00:00 2001 From: joseph-sentry <136376984+joseph-sentry@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:24:14 -0400 Subject: [PATCH] feat: upsert daily test rollup when we process a test instance (#699) --- database/models/reports.py | 28 ++++ requirements.in | 2 +- requirements.txt | 4 +- tasks/test_results_processor.py | 98 ++++++++++- .../samples/sample_multi_test_part_1.txt | 11 ++ .../samples/sample_multi_test_part_2.txt | 11 ++ tasks/tests/samples/sample_test.txt | 2 +- .../unit/test_test_results_processor_task.py | 155 +++++++++++++++++- 8 files changed, 304 insertions(+), 7 deletions(-) create mode 100644 tasks/tests/samples/sample_multi_test_part_1.txt create mode 100644 tasks/tests/samples/sample_multi_test_part_2.txt diff --git a/database/models/reports.py b/database/models/reports.py index 98b55b712..b8017d0fd 100644 --- a/database/models/reports.py +++ b/database/models/reports.py @@ -350,3 +350,31 @@ class Flake(CodecovBaseModel, MixinBaseClassNoExternalID): fail_count = Column(types.Integer) start_date = Column(types.DateTime) end_date = Column(types.DateTime, nullable=True) + + +class DailyTestRollup(CodecovBaseModel, MixinBaseClassNoExternalID): + __tablename__ = "reports_dailytestrollups" + + test_id = Column(types.Text, ForeignKey("reports_test.id")) + test = relationship(Test, backref=backref("dailytestrollups")) + date = Column(types.Date) + repoid = Column(types.Integer) + branch = Column(types.Text) + + fail_count = Column(types.Integer) + skip_count = Column(types.Integer) + pass_count = Column(types.Integer) + last_duration_seconds = Column(types.Float) + avg_duration_seconds = Column(types.Float) + latest_run = Column(types.DateTime) + commits_where_fail = Column(types.ARRAY(types.Text)) + + __table_args__ = ( + UniqueConstraint( + "repoid", + "date", + "branch", + "test_id", + name="reports_dailytestrollups_repoid_date_branch_test", + ), + ) diff --git a/requirements.in b/requirements.in index 1c3152b0d..195531c64 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,5 @@ https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem -https://github.com/codecov/shared/archive/cdf263f5173c16585030bcab40e639045c69e199.tar.gz#egg=shared +https://github.com/codecov/shared/archive/5cc5f48e7d847d3d47ff875a682192465161155f.tar.gz#egg=shared https://github.com/codecov/test-results-parser/archive/1507de2241601d678e514c08b38426e48bb6d47d.tar.gz#egg=test-results-parser https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2f887936a98a668dda42.tar.gz#egg=timestring asgiref>=3.7.2 diff --git a/requirements.txt b/requirements.txt index 341c8cf76..bd327684b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile # amqp==5.2.0 # via kombu @@ -357,7 +357,7 @@ sentry-sdk[celery]==2.13.0 # via # -r requirements.in # shared -shared @ https://github.com/codecov/shared/archive/cdf263f5173c16585030bcab40e639045c69e199.tar.gz +shared @ https://github.com/codecov/shared/archive/5cc5f48e7d847d3d47ff875a682192465161155f.tar.gz # via -r requirements.in six==1.16.0 # via diff --git a/tasks/test_results_processor.py b/tasks/test_results_processor.py index 159b19281..abc6b5f52 100644 --- a/tasks/test_results_processor.py +++ b/tasks/test_results_processor.py @@ -2,6 +2,7 @@ import json import logging import zlib +from datetime import date, datetime from io import BytesIO from sys import getsizeof from typing import List @@ -13,6 +14,7 @@ from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session from test_results_parser import ( + Outcome, ParserError, Testrun, parse_junit_xml, @@ -21,7 +23,7 @@ ) from app import celery_app -from database.models import Repository, Test, TestInstance, Upload +from database.models import DailyTestRollup, Repository, Test, TestInstance, Upload from services.archive import ArchiveService from services.test_results import generate_flags_hash, generate_test_id from services.yaml import read_yaml_field @@ -110,6 +112,7 @@ def _bulk_write_tests_to_db( with sentry_sdk.metrics.timing(key="test_results.processor.write_to_db"): test_data = [] test_instance_data = [] + daily_totals = dict() for testrun in parsed_testruns: # Build up the data for bulk insert name = testrun.name @@ -143,12 +146,105 @@ def _bulk_write_tests_to_db( ) ) + def update_daily_total(): + daily_totals[test_id]["last_duration_seconds"] = duration_seconds + daily_totals[test_id]["avg_duration_seconds"] = ( + daily_totals[test_id]["avg_duration_seconds"] + * ( + daily_totals[test_id]["pass_count"] + + daily_totals[test_id]["fail_count"] + ) + + duration_seconds + ) / ( + daily_totals[test_id]["pass_count"] + + daily_totals[test_id]["fail_count"] + + 1 + ) + + if outcome == str(Outcome.Pass): + daily_totals[test_id]["pass_count"] += 1 + elif outcome == str(Outcome.Failure) or outcome == str( + Outcome.Error + ): + daily_totals[test_id]["fail_count"] += 1 + elif outcome == str(Outcome.Skip): + daily_totals[test_id]["skip_count"] += 1 + + def create_daily_total(): + daily_totals[test_id] = { + "test_id": test_id, + "repoid": repoid, + "last_duration_seconds": duration_seconds, + "avg_duration_seconds": duration_seconds, + "pass_count": 1 if outcome == str(Outcome.Pass) else 0, + "fail_count": 1 + if outcome == str(Outcome.Failure) + or outcome == str(Outcome.Error) + else 0, + "skip_count": 1 if outcome == str(Outcome.Skip) else 0, + "branch": branch, + "date": date.today(), + "latest_run": datetime.now(), + "commits_where_fail": [commitid] + if ( + outcome == str(Outcome.Failure) + or outcome == str(Outcome.Error) + ) + else [], + } + + if outcome != str(Outcome.Skip): + if test_id in daily_totals: + update_daily_total() + else: + create_daily_total() + # Save Tests insert_on_conflict_do_nothing = ( insert(Test.__table__).values(test_data).on_conflict_do_nothing() ) db_session.execute(insert_on_conflict_do_nothing) db_session.flush() + + # Upsert Daily Test Totals + stmt = insert(DailyTestRollup.__table__).values(list(daily_totals.values())) + stmt = stmt.on_conflict_do_update( + index_elements=[ + "repoid", + "branch", + "test_id", + "date", + ], + set_={ + "last_duration_seconds": stmt.excluded.last_duration_seconds, + "avg_duration_seconds": ( + DailyTestRollup.__table__.c.avg_duration_seconds + * ( + DailyTestRollup.__table__.c.pass_count + + DailyTestRollup.__table__.c.fail_count + ) + + stmt.excluded.avg_duration_seconds + ) + / ( + DailyTestRollup.__table__.c.pass_count + + DailyTestRollup.__table__.c.fail_count + + 1 + ), + "latest_run": stmt.excluded.latest_run, + "pass_count": DailyTestRollup.__table__.c.pass_count + + stmt.excluded.pass_count, + "skip_count": DailyTestRollup.__table__.c.skip_count + + stmt.excluded.skip_count, + "fail_count": DailyTestRollup.__table__.c.fail_count + + stmt.excluded.fail_count, + "commits_where_fail": DailyTestRollup.__table__.c.commits_where_fail + + stmt.excluded.commits_where_fail, + }, + ) + + db_session.execute(stmt) + db_session.flush() + # Save TestInstances insert_test_instances = insert(TestInstance.__table__).values( test_instance_data diff --git a/tasks/tests/samples/sample_multi_test_part_1.txt b/tasks/tests/samples/sample_multi_test_part_1.txt new file mode 100644 index 000000000..fc0a34ff9 --- /dev/null +++ b/tasks/tests/samples/sample_multi_test_part_1.txt @@ -0,0 +1,11 @@ +{ + "test_results_files": [ + { + "filename": "codecov-demo/multi-test.xml", + "format": "base64+compressed", + "data": "eJy1U01vgkAQvfsrJtvEaCywi18FBdOkNqaHnpp6bDawICkCYZe2/vsOi1aqxvTSTYDM25n3HrM788XXNoUPUcokzzzCTEpAZEEeJlnskUpFxh1Z+J25ElLJKsGP3wFcRwAyvhUeKXY1gsVlmZfSI8gT8SStSoEBIyDfk6IQod7QtRpVSV1LTWpPiOatVw1KxbeFR2xqjww6NWznhQ1daruMmZOpMxk6Bh25FLk2uVSNg9fH1fP66WG9csw0D3hK/B9G7TbgUkCQcimbfO3Cqt9vUZUFChtgFjsCx90fvGWUshav5t7/JqBpyWPMQgVRKmDQjdXMg3H3htEZDAA+NwLzGHhQ8izEJxY9eguM9okfiugXrW5E20Ov754l1AKwlzulPKifFS3h1OCFlMEVs52LRs9b6Y5duNdSGC/rezG39s1qHY11OJv/P64/a0VVmu4gyHEuRHiQgBO00Zia9hWFJtZzgkNktafoG8+FAfk=", + "labels": "" + } + ], + "metadata": {} +} diff --git a/tasks/tests/samples/sample_multi_test_part_2.txt b/tasks/tests/samples/sample_multi_test_part_2.txt new file mode 100644 index 000000000..3fc9e5b52 --- /dev/null +++ b/tasks/tests/samples/sample_multi_test_part_2.txt @@ -0,0 +1,11 @@ +{ + "test_results_files": [ + { + "filename": "codecov-demo/multi-test.xml", + "format": "base64+compressed", + "data": "eJzNU8tugzAQvOcrVq4UJUoBm7wKCUSVmirqoaeqOVYWGIJKAGHTNn/fxeRBk7TnWgK0492ZweudL762KXyIUiZ55hFmUgIiC/IwyWKPVCoy7sjC78yVkEpWCX78DuA6AZDxrfBIsasRLC7LvJQeQZ6IJ2lVCgwYAfmeFIUI9Yau1ahK6lpqUntCNG+9alAqvi08YlN7ZNCpYTsvbOhS22XMnEydydAx6MilyLXJpWocvD6untdPD+uVY6Z5wFPiHxm124BLAUHKpWzytQurfr9FVRYoPACz2BE47R7xllFqt3mtA/E/0oqqNN1BkGNTRXiQgDO00Riak5aCZtw3DbAFkseYg7yiVMCgG6uZB+PuDaMzGAB8bpAKcQ9KnoX4xKJHb4HRPvFDEf2g1W1t/2Wv714k1AKwlzunPKhfFC3h3OCVlMEfZjtXjV42yx27cK+lMF7Wt3xu7Q/r9yY1sZ4THCKrPUXfoXQB+w==", + "labels": "" + } + ], + "metadata": {} +} diff --git a/tasks/tests/samples/sample_test.txt b/tasks/tests/samples/sample_test.txt index 8241fe733..c8da9986a 100644 --- a/tasks/tests/samples/sample_test.txt +++ b/tasks/tests/samples/sample_test.txt @@ -1 +1 @@ -{"test_results_files": [{"filename": "codecov-demo/temp.junit.xml", "format": "base64+compressed", "data": "eJy1VMluwjAQvfMVI1dCoBbHZiklJEFVS4V66Kkqx8okBqw6i2KHwt/XWSChnCrRXDLjefNm8Uuc2T6UsOOpEnHkIooJAh75cSCijYsyve49oJnnaK60yoR5NWyIWMhdlBzyE5OWpnGqXGQY1kzILOXGoQjUl0gSHhSBItdFQ2OJPJdgMuqXjtIsTFzUJ/1Bj9IeuX+n1KZjmwwxoZSMDWwbK13W/HhZvC1fn5eLCZaxzyQq2/KZ4uBLplQJY4nAmocJNhA/k0zHKc5xn7WPqimKYxYEjc6Iad66DrHKVjplvv4f9jCTWiTy0GQnV2MPxE4E/Lxzz6muGMzFKbbJaZXiqQajIHBdIHjUvqFkCrcA31tugEUA2lJP11nkayM3eKobKIsA00D2lAz9CV9NSHujpx16B/3uievI9mceU/sChryAr6ExZKdrtwpw+VQjXeSVPVVjtubn6HoBp8iVlnDGd9VFtFpGFFYuCqsWgfVLFDg52ANiw2Mxpyk3zz94x6qU4DnWUW2VWfwkmrbyfgBbcXMH", "labels": ""}], "metadata": {}} \ No newline at end of file +{"test_results_files": [{"filename": "codecov-demo/temp.junit.xml", "format": "base64+compressed", "data": "eJy1VMluwjAQvfMVI1dCoBbHZiklJEFVS4V66Kkqx8okBqw6i2KHwt/XWSChnCrRXDLjefNm8Uuc2T6UsOOpEnHkIooJAh75cSCijYsyve49oJnnaK60yoR5NWyIWMhdlBzyE5OWpnGqXGQY1kzILOXGoQjUl0gSHhSBItdFQ2OJPJdgMuqXjtIsTFzUJ/1Bj9IeuX+n1KZjmwwxoZSMDWwbK13W/HhZvC1fn5eLCZaxzyQq2/KZ4uBLplQJY4nAmocJNhA/k0zHKc5xn7WPqimKYxYEjc6Iad66DrHKVjplvv4f9jCTWiTy0GQnV2MPxE4E/Lxzz6muGMzFKbbJaZXiqQajIHBdIHjUvqFkCrcA31tugEUA2lJP11nkayM3eKobKIsA00D2lAz9CV9NSHujpx16B/3uievI9mceU/sChryAr6ExZKdrtwpw+VQjXeSVPVVjtubn6HoBp8iVlnDGd9VFtFpGFFYuCqsWgfVLFDg52ANiw2Mxpyk3zz94x6qU4DnWUW2VWfwkmrbyfgBbcXMH", "labels": ""}], "metadata": {}} diff --git a/tasks/tests/unit/test_test_results_processor_task.py b/tasks/tests/unit/test_test_results_processor_task.py index f19606f80..e10f91c44 100644 --- a/tasks/tests/unit/test_test_results_processor_task.py +++ b/tasks/tests/unit/test_test_results_processor_task.py @@ -1,11 +1,13 @@ +from datetime import date, datetime, timedelta from pathlib import Path import pytest from shared.storage.exceptions import FileNotInStorageError from test_results_parser import Outcome +from time_machine import travel from database.models import CommitReport -from database.models.reports import Test, TestInstance +from database.models.reports import DailyTestRollup, Test, TestInstance from database.tests.factories import CommitFactory, UploadFactory from services.test_results import generate_test_id from tasks.test_results_processor import ( @@ -386,6 +388,8 @@ def test_test_result_processor_task_delete_archive( dbsession.query(TestInstance).filter_by(outcome=str(Outcome.Failure)).all() ) + assert result == expected_result + assert len(tests) == 1 assert len(test_instances) == 4 assert len(failures) == 4 @@ -395,7 +399,6 @@ def test_test_result_processor_task_delete_archive( == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) assert test_instances[0].test.id == tests[0].id - assert expected_result == result assert "Deleting uploaded file as requested" in caplog.text with pytest.raises(FileNotInStorageError): mock_storage.read_file("archive", url) @@ -625,3 +628,151 @@ def test_upload_processor_task_call_existing_test_diff_flags_hash( ) assert expected_result == result assert commit.message == "hello world" + + @pytest.mark.integration + def test_upload_processor_task_call_daily_test_totals( + self, + mocker, + mock_configuration, + dbsession, + codecov_vcr, + mock_storage, + mock_redis, + celery_app, + ): + traveller = travel("1970-1-1T00:00:00Z", tick=False) + traveller.start() + first_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/a84d445c-9c1e-434f-8275-f18f1f320f81.txt" + with open(here.parent.parent / "samples" / "sample_multi_test_part_1.txt") as f: + content = f.read() + mock_storage.write_file("archive", first_url, content) + + first_commit = CommitFactory.create( + message="hello world", + commitid="cd76b0821854a780b60012aed85af0a8263004ad", + repository__owner__unencrypted_oauth_token="test7lk5ndmtqzxlx06rip65nac9c7epqopclnoy", + repository__owner__username="joseph-sentry", + repository__owner__service="github", + repository__name="codecov-demo", + branch="first_branch", + ) + dbsession.add(first_commit) + dbsession.flush() + + first_report_row = CommitReport(commit_id=first_commit.id_) + dbsession.add(first_report_row) + dbsession.flush() + + upload = UploadFactory.create(storage_path=first_url, report=first_report_row) + dbsession.add(upload) + dbsession.flush() + + repoid = upload.report.commit.repoid + redis_queue = [{"url": first_url, "upload_pk": upload.id_}] + mocker.patch.object(TestResultsProcessorTask, "app", celery_app) + + result = TestResultsProcessorTask().run_impl( + dbsession, + repoid=repoid, + commitid=first_commit.commitid, + commit_yaml={"codecov": {"max_report_age": False}}, + arguments_list=redis_queue, + ) + expected_result = [ + { + "successful": True, + } + ] + + rollups = dbsession.query(DailyTestRollup).all() + + assert [r.branch for r in rollups] == [ + "first_branch", + "first_branch", + ] + + assert [r.date for r in rollups] == [ + date.today(), + date.today(), + ] + + traveller.stop() + + traveller = travel("1970-1-2T00:00:00Z", tick=False) + traveller.start() + + second_commit = CommitFactory.create( + message="hello world 2", + commitid="bd76b0821854a780b60012aed85af0a8263004ad", + repository=first_commit.repository, + branch="second_branch", + ) + dbsession.add(second_commit) + dbsession.flush() + + second_report_row = CommitReport(commit_id=second_commit.id_) + dbsession.add(second_report_row) + dbsession.flush() + + second_url = "v4/raw/2019-05-22/C3C4715CA57C910D11D5EB899FC86A7E/4c4e4654ac25037ae869caeb3619d485970b6304/b84d445c-9c1e-434f-8275-f18f1f320f81.txt" + with open(here.parent.parent / "samples" / "sample_multi_test_part_2.txt") as f: + content = f.read() + mock_storage.write_file("archive", second_url, content) + upload = UploadFactory.create(storage_path=second_url, report=second_report_row) + dbsession.add(upload) + dbsession.flush() + + redis_queue = [{"url": second_url, "upload_pk": upload.id_}] + + result = TestResultsProcessorTask().run_impl( + dbsession, + repoid=repoid, + commitid=second_commit.commitid, + commit_yaml={"codecov": {"max_report_age": False}}, + arguments_list=redis_queue, + ) + expected_result = [ + { + "successful": True, + } + ] + + rollups: list[DailyTestRollup] = dbsession.query(DailyTestRollup).all() + + assert result == expected_result + + assert [r.branch for r in rollups] == [ + "first_branch", + "first_branch", + "second_branch", + "second_branch", + ] + + assert [r.date for r in rollups] == [ + date.today() - timedelta(days=1), + date.today() - timedelta(days=1), + date.today(), + date.today(), + ] + + assert [r.fail_count for r in rollups] == [1, 0, 0, 1] + assert [r.pass_count for r in rollups] == [1, 1, 2, 0] + assert [r.skip_count for r in rollups] == [0, 0, 0, 0] + + assert [r.commits_where_fail for r in rollups] == [ + ["cd76b0821854a780b60012aed85af0a8263004ad"], + [], + [], + ["bd76b0821854a780b60012aed85af0a8263004ad"], + ] + + assert [r.latest_run for r in rollups] == [ + datetime(1970, 1, 1, 0, 0), + datetime(1970, 1, 1, 0, 0), + datetime(1970, 1, 2, 0, 0), + datetime(1970, 1, 2, 0, 0), + ] + assert [r.avg_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6] + assert [r.last_duration_seconds for r in rollups] == [0.001, 7.2, 0.002, 3.6] + + traveller.stop()