Merge branch 'main' into gio/pull-flare-storage

celery_config.py

@@ -16,10 +16,12 @@
 )
 from shared.celery_config import (
     BaseCeleryConfig,
+    brolly_stats_rollup_task_name,
     gh_app_webhook_check_task_name,
     health_check_task_name,
     profiling_finding_task_name,
 )
+from shared.config import get_config
 
 from celery_task_router import route_task
 from helpers.cache import RedisBackend, cache
@@ -97,7 +99,7 @@ def init_celery_tracing(*args, **kwargs):
 trial_expiration_cron_task_name = "app.cron.plan.TrialExpirationCronTask"
 
 
-class CeleryWorkerConfig(BaseCeleryConfig):
+def _beat_schedule():
     beat_schedule = {
         "hourly_check": {
             "task": hourly_check_task_name,
@@ -137,4 +139,19 @@ class CeleryWorkerConfig(BaseCeleryConfig):
         },
     }
 
+    if get_config("setup", "telemetry", "enabled", default=True):
+        beat_schedule["brolly_stats_rollup"] = {
+            "task": brolly_stats_rollup_task_name,
+            "schedule": crontab(hour="2"),
+            "kwargs": {
+                "cron_task_generation_time_iso": BeatLazyFunc(get_utc_now_as_iso_format)
+            },
+        }
+
+    return beat_schedule
+
+
+class CeleryWorkerConfig(BaseCeleryConfig):
+    beat_schedule = _beat_schedule()
+
     task_routes = route_task

conftest.py

@@ -103,7 +103,6 @@ def dbsession(db, engine):
     @event.listens_for(session, "after_transaction_end")
     def restart_savepoint(session, transaction):
         if transaction.nested and not transaction._parent.nested:
-
             # ensure that state is expired the way
             # session.commit() at the top level normally does
             # (optional step)
@@ -139,6 +138,9 @@ def mock_configuration(mocker):
         "setup": {
             "codecov_url": "https://codecov.io",
             "encryption_secret": "zp^P9*i8aR3",
+            "telemetry": {
+                "endpoint_override": "abcde",
+            },
         },
     }
     mock_config.set_params(our_config)

database/models/core.py

@@ -1,10 +1,11 @@
 import random
 import string
+import uuid
 from datetime import datetime
 
 from sqlalchemy import Column, ForeignKey, Index, UniqueConstraint, types
 from sqlalchemy.dialects import postgresql
-from sqlalchemy.orm import backref, relationship
+from sqlalchemy.orm import backref, relationship, validates
 from sqlalchemy.schema import FetchedValue
 
 import database.models
@@ -14,6 +15,30 @@
 from helpers.config import should_write_data_to_storage_config_check
 
 
+class User(CodecovBaseModel):
+    __tablename__ = "users"
+    id_ = Column("id", types.BigInteger, primary_key=True)
+
+    # This field is case-insensitive but we don't have a way to represent that
+    # here. Options to address:
+    # - Upgrade sqlalchemy and use `postgresql.CITEXT(100)` as the field type
+    # - Add a case-insensitive collation to postgres[1] and use it here + in `codecov-api`
+    #
+    # [1] https://www.postgresql.org/docs/current/collation.html
+    email = Column(types.String(100), nullable=True)
+
+    name = Column(types.String(100), nullable=True)
+    is_staff = Column(types.Boolean, default=False)
+    is_superuser = Column(types.Boolean, default=False)
+    external_id = Column(postgresql.UUID(as_uuid=True), unique=True, default=uuid.uuid4)
+
+    @validates("external_id")
+    def validate_external_id(self, key, value):
+        if self.external_id:
+            raise ValueError("`external_id` cannot be modified")
+        return value
+
+
 class Owner(CodecovBaseModel):
     __tablename__ = "owners"
     ownerid = Column(types.Integer, primary_key=True)
@@ -70,6 +95,9 @@ class Owner(CodecovBaseModel):
         passive_deletes=True,
     )
 
+    # TODO: Create association between `User` and `Owner` mirroring `codecov-api`
+    # https://github.com/codecov/codecov-api/blob/204f7fd7e37896efe0259e4bc91aad20601087e0/codecov_auth/models.py#L196-L202
+
     __table_args__ = (
         Index("owner_service_ids", "service", "service_id", unique=True),
         Index("owner_service_username", "service", "username", unique=True),
@@ -414,3 +442,10 @@ class OrganizationLevelToken(MixinBaseClass, CodecovBaseModel):
     token = Column(postgresql.UUID)
     valid_until = Column(types.DateTime)
     token_type = Column(types.String)
+
+
+class Constants(CodecovBaseModel):
+    __tablename__ = "constants"
+
+    key = Column(types.String, primary_key=True)
+    value = Column(types.String)

database/tests/factories/core.py

@@ -15,6 +15,19 @@ def encrypt_oauth_token(val):
     return encryptor.encode(val)
 
 
+class UserFactory(Factory):
+    class Meta:
+        model = models.User
+
+    id_ = factory.Sequence(lambda n: n)
+
+    name = factory.Faker("name")
+    email = factory.Faker("email")
+    is_staff = False
+    is_superuser = False
+    external_id = factory.LazyFunction(lambda: uuid4())
+
+
 class OwnerFactory(Factory):
     class Meta:
         model = models.Owner
@@ -261,3 +274,11 @@ class Meta:
     token = factory.LazyFunction(lambda: uuid4().hex)
     token_type = "upload"
     owner = factory.SubFactory(OwnerFactory)
+
+
+class ConstantsFactory(Factory):
+    class Meta:
+        model = models.Constants
+
+    key = ""
+    value = ""

database/tests/unit/test_model_utils.py

@@ -130,17 +130,5 @@ def test_archive_setter_archive_field(self, db, mocker):
         assert test_class._archive_field == None
         assert test_class._archive_field_storage_path == "path/to/written/object"
         assert test_class.archive_field == some_json
-        # Writing cleans the cache
-        assert mock_read_file.call_count == 1
-        mock_read_file.assert_called_with("path/to/written/object")
-        mock_write_file.assert_called_with(
-            commit_id=commit.commitid,
-            table="test_table",
-            field="archive_field",
-            external_id=test_class.external_id,
-            data=some_json,
-            encoder=ReportEncoder,
-        )
-        mock_archive_service.return_value.delete_file.assert_called_with(
-            "path/to/old/data"
-        )
+        # Cache is updated on write
+        assert mock_read_file.call_count == 0

database/tests/unit/test_models.py

@@ -403,14 +403,11 @@ def test_set_files_array_to_storage(self, dbsession, mocker, mock_configuration)
         # Retrieve the set value
         files_array = retrieved_instance.files_array
         assert files_array == self.sample_files_array
-        assert mock_archive_service.call_count == 2
-        mock_archive_service.return_value.read_file.assert_has_calls(
-            [call("https://storage-url/path/to/item.json")]
-        )
-        # Check that caching (still) works within the instance
+        assert mock_archive_service.call_count == 1
+        # Check that caching works within the instance
         files_array = retrieved_instance.files_array
-        assert mock_archive_service.call_count == 2
-        assert mock_archive_service.return_value.read_file.call_count == 1
+        assert mock_archive_service.call_count == 1
+        assert mock_archive_service.return_value.read_file.call_count == 0
 
 
 class TestCommitModel(object):

database/utils.py

@@ -83,8 +83,8 @@ def __set_name__(self, owner, name):
         self.public_name = name
         self.db_field_name = "_" + name
         self.archive_field_name = "_" + name + "_storage_path"
+        self.cached_value_property_name = f"__{self.public_name}_cached_value"
 
-    @lru_cache(maxsize=1)
     def _get_value_from_archive(self, obj):
         repository = obj.get_repository()
         archive_service = ArchiveService(repository=repository)
@@ -113,10 +113,16 @@ def _get_value_from_archive(self, obj):
         return self.default_value
 
     def __get__(self, obj, objtype=None):
+        cached_value = getattr(obj, self.cached_value_property_name, None)
+        if cached_value:
+            return cached_value
         db_field = getattr(obj, self.db_field_name)
         if db_field is not None:
-            return self.rehydrate_fn(obj, db_field)
-        return self._get_value_from_archive(obj)
+            value = self.rehydrate_fn(obj, db_field)
+        else:
+            value = self._get_value_from_archive(obj)
+        setattr(obj, self.cached_value_property_name, value)
+        return value
 
     def __set__(self, obj, value):
         # Set the new value
@@ -137,6 +143,6 @@ def __set__(self, obj, value):
                 archive_service.delete_file(old_file_path)
             setattr(obj, self.archive_field_name, path)
             setattr(obj, self.db_field_name, None)
-            self._get_value_from_archive.cache_clear()
         else:
             setattr(obj, self.db_field_name, value)
+        setattr(obj, self.cached_value_property_name, value)

dockerscripts/Dockerfile.requirements

@@ -37,7 +37,7 @@ FROM            python:3.10.8-alpine3.16
 
 RUN             apk add --upgrade --no-cache apk-tools && \
                 apk -U upgrade busybox expat libretls  && \
-                apk add --no-cache postgresql-libs gcc libxslt-dev make
+                apk add --no-cache postgresql-libs gcc libxslt-dev curl make
 
 WORKDIR         /pip-packages/
 COPY            --from=build /pip-packages/ /pip-packages/

requirements.in

@@ -1,4 +1,4 @@
-git+ssh://[email protected]/codecov/shared.git@51e113113f1e9b1d111135dffa5cdb77bd6e486f#egg=shared
+git+ssh://[email protected]/codecov/shared.git@6a8a33248804a9c101d34f417efda7c11e4bbe63#egg=shared
 git+ssh://[email protected]/codecov/[email protected]#egg=codecovopentelem
 boto3
 celery
@@ -24,6 +24,7 @@ python-json-logger
 PyYAML
 redis>=4.4.4
 requests
+respx
 sentry-sdk
 SQLAlchemy-Utils
 SQLAlchemy

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile requirements.in
@@ -116,6 +116,7 @@ httplib2==0.19.0
 httpx==0.23.0
     # via
     #   -r requirements.in
+    #   respx
     #   shared
 idna==2.10
     # via
@@ -240,6 +241,8 @@ requests==2.28.0
     #   google-api-core
     #   google-cloud-storage
     #   stripe
+respx==0.20.2
+    # via -r requirements.in
 rfc3986[idna2008]==1.4.0
     # via httpx
 rsa==4.7.2
@@ -248,7 +251,7 @@ s3transfer==0.3.4
     # via boto3
 sentry-sdk==1.19.1
     # via -r requirements.in
-shared @ git+ssh://[email protected]/codecov/shared.git@51e113113f1e9b1d111135dffa5cdb77bd6e486f
+shared @ git+ssh://[email protected]/codecov/shared.git@6a8a33248804a9c101d34f417efda7c11e4bbe63
     # via -r requirements.in
 six==1.15.0
     # via
@@ -283,7 +286,7 @@ text-unidecode==1.3
     # via faker
 timestring==1.6.4
     # via -r requirements.in
-tlslite-ng==0.8.0-alpha39
+tlslite-ng==0.8.0a39
     # via shared
 tomli==2.0.1
     # via pytest

services/notification/notifiers/mixins/message/helpers.py

@@ -116,6 +116,53 @@ def make_metrics(before, after, relative, show_complexity, yaml):
     return "".join(("|", coverage, complexity, icon))
 
 
+def make_patch_only_metrics(before, after, relative, show_complexity, yaml):
+    if after is None:
+        # e.g. missing flags
+        coverage = " `?` |"
+
+    elif after is False:
+        # e.g. file deleted
+        coverage = " |"
+
+    else:
+        layout = " `{relative}` |"
+        coverage = layout.format(
+            relative=format_number_to_str(
+                yaml, relative.coverage if relative else 0, style="{0}%", if_null="\xF8"
+            ),
+        )
+    return "".join(("|", coverage))
+
+
+def get_metrics_function(hide_project_coverage):
+    if hide_project_coverage:
+        metrics = make_patch_only_metrics
+    else:
+        metrics = make_metrics
+    return metrics
+
+
+def get_table_header(hide_project_coverage, show_complexity):
+    if not hide_project_coverage:
+        table_header = (
+            "| Coverage \u0394 |"
+            + (" Complexity \u0394 |" if show_complexity else "")
+            + " |"
+        )
+    else:
+        table_header = "| Coverage |"
+
+    return table_header
+
+
+def get_table_layout(hide_project_coverage, show_complexity):
+    if hide_project_coverage:
+        return "|---|---|"
+    else:
+        return "|---|---|---|" + ("---|" if show_complexity else "")
+
+
 def format_number_to_str(
     yml, value, if_zero=None, if_null=None, plus=False, style="{0}"
 ) -> str: