Merge branch 'main' of github.com:codecov/worker into joseph/new-feat…

…ures-comment

.github/workflows/ci.yml

@@ -13,6 +13,8 @@ on:
 permissions:
   contents: "read"
   id-token: "write"
+  issues: "write"
+  pull-requests: "write"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -22,11 +24,11 @@ jobs:
   lint:
     name: Run Lint
 
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
 
   build:
     name: Build Worker
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
 
     secrets: inherit
     with:
@@ -35,14 +37,14 @@ jobs:
   codecovstartup:
     name: Codecov Startup
     needs: build
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
 
     secrets: inherit
 
   test:
     name: Test
     needs: [build]
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
 
     secrets: inherit
     with:
@@ -52,7 +54,7 @@ jobs:
     name: Build Self Hosted Worker
     needs: [build, test]
 
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
     secrets: inherit
     with:
       repo: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-worker' }}
@@ -61,7 +63,7 @@ jobs:
     name: Push Staging Image
     needs: [build, test]
     if: ${{ github.event_name == 'push' && (github.event.ref == 'refs/heads/main' || github.event.ref == 'refs/heads/staging') && github.repository_owner == 'codecov' }}
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
     secrets: inherit
     with:
       environment: staging
@@ -71,7 +73,7 @@ jobs:
     name: Push Production Image
     needs: [build, test]
     if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }}
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
     secrets: inherit
     with:
       environment: production
@@ -82,7 +84,7 @@ jobs:
     needs: [build-self-hosted, test]
     secrets: inherit
     if: ${{ github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }}
-    uses: codecov/gha-workflows/.github/workflows/[email protected].23
+    uses: codecov/gha-workflows/.github/workflows/[email protected].26
     with:
       push_rolling: true
       repo: ${{ vars.CODECOV_IMAGE_V2 || 'codecov/self-hosted-worker' }}

.gitignore

@@ -51,6 +51,8 @@ nosetests.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+*.junit.xml
+junit.xml
 
 # Translations
 *.mo

Makefile

@@ -199,6 +199,9 @@ push.self-hosted-rolling:
 	docker push ${DOCKERHUB_REPO}:rolling_no_dependencies
 	docker push ${DOCKERHUB_REPO}:rolling
 
+shell:
+	docker-compose exec worker bash
+
 test_env.up:
 	env | grep GITHUB > .testenv; true
 	TIMESERIES_ENABLED=${TIMESERIES_ENABLED} docker-compose up -d
@@ -213,6 +216,7 @@ test_env.install_cli:
 	pip install --no-cache-dir codecov-cli==$(CODECOV_CLI_VERSION)
 
 test_env.container_prepare:
+	apt-get update
 	apt-get install -y git build-essential netcat-traditional
 	make test_env.install_cli
 	git config --global --add safe.directory /worker
@@ -232,8 +236,8 @@ test_env.upload:
 	docker-compose exec worker make test_env.container_upload_test_results CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL}
 
 test_env.container_upload:
-	codecovcli -u ${CODECOV_URL} do-upload --flag unit --file unit.coverage.xml
-	codecovcli -u ${CODECOV_URL} do-upload --flag integration --file integration.coverage.xml
+	codecovcli -u ${CODECOV_URL} do-upload --flag unit --file unit.coverage.xml --disable-search
+	codecovcli -u ${CODECOV_URL} do-upload --flag integration --file integration.coverage.xml --disable-search
 
 test_env.container_upload_test_results:
 	codecovcli -v -u ${CODECOV_URL} do-upload --report-type test_results || true

VERSION

@@ -1 +1 @@
-24.10.1
+24.11.1

conftest.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from pathlib import Path
 
@@ -8,9 +9,8 @@
 from shared.storage.memory import MemoryStorageService
 from shared.torngit import Github as GithubHandler
 from sqlalchemy import event
-from sqlalchemy.exc import OperationalError
 from sqlalchemy.orm import Session
-from sqlalchemy_utils import create_database, database_exists
+from sqlalchemy_utils import database_exists
 
 from celery_config import initialize_logging
 from database.base import Base
@@ -39,7 +39,7 @@ def pytest_itemcollected(item):
 
 
 @pytest.fixture(scope="session")
-def engine(request, sqlalchemy_connect_url, app_config):
+def engine(request, sqlalchemy_db, sqlalchemy_connect_url, app_config):
     """Engine configuration.
     See http://docs.sqlalchemy.org/en/latest/core/engines.html
     for more details.
@@ -66,30 +66,87 @@ def engine(request, sqlalchemy_connect_url, app_config):
         engine.url.database = "{}_{}".format(engine.url.database, xdist_suffix)
         engine = create_engine(engine.url)  # override engine
 
-    def fin():
-        print("Disposing engine")
-        engine.dispose()
+    # Check that the DB exist and migrate the unmigrated SQLALchemy models as a stop-gap
+    database_url = sqlalchemy_connect_url
+    if not database_exists(database_url):
+        raise RuntimeError(f"SQLAlchemy cannot connect to DB at {database_url}")
+
+    Base.metadata.tables["profiling_profilingcommit"].create(bind=engine)
+    Base.metadata.tables["profiling_profilingupload"].create(bind=engine)
+    Base.metadata.tables["timeseries_measurement"].create(bind=engine)
+    Base.metadata.tables["timeseries_dataset"].create(bind=engine)
+
+    Base.metadata.tables["compare_commitcomparison"].create(bind=engine)
+    Base.metadata.tables["compare_flagcomparison"].create(bind=engine)
+    Base.metadata.tables["compare_componentcomparison"].create(bind=engine)
+
+    Base.metadata.tables["labelanalysis_labelanalysisrequest"].create(bind=engine)
+    Base.metadata.tables["labelanalysis_labelanalysisprocessingerror"].create(
+        bind=engine
+    )
 
-    request.addfinalizer(fin)
-    return engine
+    Base.metadata.tables["staticanalysis_staticanalysissuite"].create(bind=engine)
+    Base.metadata.tables["staticanalysis_staticanalysissinglefilesnapshot"].create(
+        bind=engine
+    )
+    Base.metadata.tables["staticanalysis_staticanalysissuitefilepath"].create(
+        bind=engine
+    )
+
+    yield engine
+
+    print("Disposing engine")
+    engine.dispose()
 
 
 @pytest.fixture(scope="session")
-def db(engine, sqlalchemy_connect_url):
-    database_url = sqlalchemy_connect_url
+def sqlalchemy_db(request: pytest.FixtureRequest, django_db_blocker, django_db_setup):
+    # Bootstrap the DB by running the Django bootstrap version.
+    from django.conf import settings
+    from django.db import connections
+    from django.test.utils import setup_databases, teardown_databases
+
+    with django_db_blocker.unblock():
+        # Temporarily reset the database to the SQLAlchemy DBs to run the migrations.
+        original_test_name = settings.DATABASES["default"]["TEST"]["NAME"]
+        settings.DATABASES["default"]["TEST"]["NAME"] = "test_postgres_sqlalchemy"
+        db_cfg = setup_databases(
+            verbosity=request.config.option.verbose,
+            interactive=False,
+        )
+        settings.DATABASES["default"]["TEST"]["NAME"] = original_test_name
+
+        # Hack to get the default connection for the test database to _actually_ be the
+        # Django database that the django_db should actually use. It was set to the SQLAlchemy database,
+        # but this makes sure that the default Django DB connection goes to the Django database.
+        # Since the database was already created and migrated in the django_db_setup fixture,
+        # we set keepdb=True to avoid recreating the database and rerunning the migrations.
+        connections.configure_settings(settings.DATABASES)
+        connections["default"].creation.create_test_db(
+            verbosity=request.config.option.verbose,
+            autoclobber=True,
+            keepdb=True,
+        )
+
+    yield
+
+    # Cleanup with Django version as well
     try:
-        if not database_exists(database_url):
-            create_database(database_url)
-    except OperationalError:
-        pytest.skip("No available db")
-    connection = engine.connect()
-    connection.execute("DROP SCHEMA IF EXISTS public CASCADE;")
-    connection.execute("CREATE SCHEMA public;")
-    Base.metadata.create_all(engine)
+        with django_db_blocker.unblock():
+            settings.DATABASES["default"]["TEST"]["NAME"] = "test_postgres_sqlalchemy"
+            teardown_databases(db_cfg, verbosity=request.config.option.verbose)
+            settings.DATABASES["default"]["TEST"]["NAME"] = original_test_name
+    except Exception as exc:  # noqa: BLE001
+        request.node.warn(
+            pytest.PytestWarning(
+                f"Error when trying to teardown test databases: {exc!r}"
+            )
+        )
 
 
 @pytest.fixture
-def dbsession(db, engine):
+def dbsession(sqlalchemy_db, engine):
+    """Sets up the SQLAlchemy dbsession."""
     connection = engine.connect()
 
     connection_transaction = connection.begin()
@@ -134,7 +191,6 @@ def mock_configuration(mocker):
                 "secret_access_key": "codecov-default-secret",
                 "verify_ssl": False,
             },
-            "redis_url": "redis://redis:@localhost:6379/",
             "smtp": {
                 "host": "mailhog",
                 "port": 1025,
@@ -164,6 +220,9 @@ def empty_configuration(mocker):
 
 @pytest.fixture
 def codecov_vcr(request):
+    vcr_log = logging.getLogger("vcr")
+    vcr_log.setLevel(logging.ERROR)
+
     current_path = Path(request.node.fspath)
     current_path_name = current_path.name.replace(".py", "")
     cassete_path = current_path.parent / "cassetes" / current_path_name
@@ -236,7 +295,7 @@ def side_effect(*args, **kwargs):
 @pytest.fixture
 def with_sql_functions(dbsession):
     dbsession.execute(
-        """CREATE FUNCTION array_append_unique(anyarray, anyelement) RETURNS anyarray
+        """CREATE OR REPLACE FUNCTION array_append_unique(anyarray, anyelement) RETURNS anyarray
                 LANGUAGE sql IMMUTABLE
                 AS $_$
             select case when $2 is null
@@ -315,19 +374,6 @@ def mock_submit_fn(metric, start, end):
     )
 
 
-@pytest.fixture(autouse=True)
-def mock_metric_context(mocker, request):
-    if request.node.get_closest_marker("real_metric_context"):
-        return
-
-    from helpers.telemetry import MetricContext
-
-    def populate(self):
-        self.populated = True
-
-    return mocker.patch.object(MetricContext, "populate", populate)
-
-
 @pytest.fixture(autouse=True)
 def mock_feature(mocker, request):
     if request.node.get_closest_marker("real_feature"):

database/models/core.py

@@ -16,7 +16,7 @@
 from helpers.config import should_write_data_to_storage_config_check
 
 
-class User(CodecovBaseModel):
+class User(CodecovBaseModel, MixinBaseClass):
     __tablename__ = "users"
     id_ = Column("id", types.BigInteger, primary_key=True)
 
@@ -136,7 +136,7 @@ class Repository(CodecovBaseModel):
     updatestamp = Column(types.DateTime)
     yaml = Column(postgresql.JSON)
     deleted = Column(types.Boolean, nullable=False, default=False)
-    branch = Column(types.Text)
+    branch = Column(types.Text, default="main")
     image_token = Column(
         types.Text,
         default=lambda: "".join(
@@ -185,7 +185,7 @@ class GithubAppInstallation(CodecovBaseModel, MixinBaseClass):
     # replacement for owner.integration_id
     # installation id GitHub sends us in the installation-related webhook events
     installation_id = Column(types.Integer, server_default=FetchedValue())
-    name = Column(types.Text, server_default=FetchedValue())
+    name = Column(types.Text, default=GITHUB_APP_INSTALLATION_DEFAULT_NAME)
     # if null, all repos are covered by this installation
     # otherwise, it's a list of repo.id values
     repository_service_ids = Column(
@@ -200,7 +200,7 @@ class GithubAppInstallation(CodecovBaseModel, MixinBaseClass):
         Owner, foreign_keys=[ownerid], back_populates="github_app_installations"
     )
 
-    is_suspended = Column(types.Boolean, server_default=FetchedValue())
+    is_suspended = Column(types.Boolean, default=False)
 
     def repository_queryset(self, dbsession: Session):
         """Returns a query set of repositories covered by this installation"""
@@ -365,7 +365,7 @@ class Branch(CodecovBaseModel):
     __tablename__ = "branches"
 
     repoid = Column(types.Integer, ForeignKey("repos.repoid"), primary_key=True)
-    updatestamp = Column(types.DateTime)
+    updatestamp = Column(types.DateTime, default=datetime.now)
     branch = Column(types.Text, nullable=False, primary_key=True)
     base = Column(types.Text)
     head = Column(types.Text, nullable=False)

database/models/reports.py

@@ -289,6 +289,11 @@ def id(self):
     # for example: the same test being run on windows vs. mac
     flags_hash = Column(types.String(256), nullable=False)
 
+    framework = Column(types.String(100), nullable=True)
+
+    computed_name = Column(types.Text, nullable=True)
+    filename = Column(types.Text, nullable=True)
+
     __table_args__ = (
         UniqueConstraint(
             "repoid",
@@ -329,9 +334,18 @@ class TestResultReportTotals(CodecovBaseModel, MixinBaseClass):
     error = Column(types.String(100), nullable=True)
 
 
-class ReducedError(CodecovBaseModel, MixinBaseClass):
+class ReducedError(CodecovBaseModel):
     __tablename__ = "reports_reducederror"
+    id_ = Column("id", types.BigInteger, primary_key=True)
     message = Column(types.Text)
+    created_at = Column(types.DateTime(timezone=True), default=get_utc_now)
+    updated_at = Column(
+        types.DateTime(timezone=True), onupdate=get_utc_now, default=get_utc_now
+    )
+
+    @property
+    def id(self):
+        return self.id_
 
 
 class Flake(CodecovBaseModel, MixinBaseClassNoExternalID):