Merge branch 'main' into matt/add-upload-flow-logs

database/models/core.py

@@ -238,7 +238,7 @@ def should_write_to_storage(self) -> bool:
     _report_json_storage_path = Column("report_storage_path", types.Text, nullable=True)
     report_json = ArchiveField(
         should_write_to_storage_fn=should_write_to_storage,
-        default_value={},
+        default_value_class=dict,
     )
 
 
@@ -331,6 +331,36 @@ def get_head_commit_notifications(self):
             )
         return []
 
+    def get_repository(self):
+        return self.repository
+
+    def get_commitid(self):
+        return None
+
+    @property
+    def external_id(self):
+        return self.pullid
+
+    @property
+    def id(self):
+        return self.id_
+
+    def should_write_to_storage(self) -> bool:
+        if self.repository is None or self.repository.owner is None:
+            return False
+        is_codecov_repo = self.repository.owner.username == "codecov"
+        return should_write_data_to_storage_config_check(
+            master_switch_key="pull_flare",
+            is_codecov_repo=is_codecov_repo,
+            repoid=self.repository.repoid,
+        )
+
+    _flare = Column("flare", postgresql.JSON)
+    _flare_storage_path = Column("flare_storage_path", types.Text, nullable=True)
+    flare = ArchiveField(
+        should_write_to_storage_fn=should_write_to_storage, default_value_class=dict
+    )
+
 
 class CommitNotification(CodecovBaseModel):
     __tablename__ = "commit_notifications"

database/models/reports.py

@@ -170,7 +170,7 @@ def _should_write_to_storage(self) -> bool:
     files_array = ArchiveField(
         should_write_to_storage_fn=_should_write_to_storage,
         rehydrate_fn=rehydrate_encoded_data,
-        default_value=[],
+        default_value_class=list,
     )
 
 

database/tests/unit/test_model_utils.py

@@ -36,9 +36,7 @@ def __init__(
             self._archive_field_storage_path = archive_value
             self.should_write_to_gcs = should_write_to_gcs
 
-        archive_field = ArchiveField(
-            should_write_to_storage_fn=should_write_to_storage, default_value=None
-        )
+        archive_field = ArchiveField(should_write_to_storage_fn=should_write_to_storage)
 
     class ClassWithArchiveFieldMissingMethods:
         commit: Commit

database/utils.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from functools import lru_cache
-from typing import Any, Callable
+from typing import Any, Callable, Optional
 
 from shared.storage.exceptions import FileNotInStorageError
 from shared.utils.ReportEncoder import ReportEncoder
@@ -18,16 +18,23 @@ def __subclasscheck__(cls, subclass):
             and callable(subclass.get_repository)
             and hasattr(subclass, "get_commitid")
             and callable(subclass.get_commitid)
+            and hasattr(subclass, "external_id")
         )
 
 
 class ArchiveFieldInterface(metaclass=ArchiveFieldInterfaceMeta):
     """Any class that uses ArchiveField must implement this interface"""
 
+    external_id: str
+
     def get_repository(self):
+        """Returns the repository object associated with self"""
         raise NotImplementedError()
 
-    def get_commitid(self) -> str:
+    def get_commitid(self) -> Optional[str]:
+        """Returns the commitid associated with self.
+        If no commitid is associated return None.
+        """
         raise NotImplementedError()
 
 
@@ -61,9 +68,9 @@ def __init__(
         should_write_to_storage_fn: Callable[[object], bool],
         rehydrate_fn: Callable[[object, object], Any] = lambda self, x: x,
         json_encoder=ReportEncoder,
-        default_value=None,
+        default_value_class=lambda: None,
     ):
-        self.default_value = default_value
+        self.default_value_class = default_value_class
         self.rehydrate_fn = rehydrate_fn
         self.should_write_to_storage_fn = should_write_to_storage_fn
         self.json_encoder = json_encoder
@@ -96,14 +103,14 @@ def _get_value_from_archive(self, obj):
                     ),
                 )
         else:
-            log.info(
+            log.debug(
                 "Both db_field and archive_field are None",
                 extra=dict(
                     object_id=obj.id,
                     commit=obj.get_commitid(),
                 ),
             )
-        return self.default_value
+        return self.default_value_class()
 
     def __get__(self, obj, objtype=None):
         cached_value = getattr(obj, self.cached_value_property_name, None)

services/archive.py

@@ -20,6 +20,9 @@
 class MinioEndpoints(Enum):
     chunks = "{version}/repos/{repo_hash}/commits/{commitid}/{chunks_file_name}.txt"
     json_data = "{version}/repos/{repo_hash}/commits/{commitid}/json_data/{table}/{field}/{external_id}.json"
+    json_data_no_commit = (
+        "{version}/repos/{repo_hash}/json_data/{table}/{field}/{external_id}.json"
+    )
     profiling_summary = "{version}/repos/{repo_hash}/profilingsummaries/{profiling_commit_id}/{location}"
     raw = "v4/raw/{date}/{repo_hash}/{commit_sha}/{reportid}.txt"
     profiling_collection = "{version}/repos/{repo_hash}/profilingcollections/{profiling_commit_id}/{location}"
@@ -200,20 +203,26 @@ def write_json_data_to_storage(
         *,
         encoder=ReportEncoder,
     ):
-        path = MinioEndpoints.json_data.get_path(
-            version="v4",
-            repo_hash=self.storage_hash,
-            commitid=commit_id,
-            table=table,
-            field=field,
-            external_id=external_id,
-        )
-        stringified_data = json.dumps(data, cls=encoder)
-        if table == "reports_reportdetails" and not ("meta" in stringified_data):
-            log.warning(
-                "Saving files_array data without meta",
-                extra=dict(commit=commit_id, data=stringified_data, path=path),
+        if commit_id is None:
+            # Some classes don't have a commit associated with them
+            # For example Pull belongs to multiple commits.
+            path = MinioEndpoints.json_data_no_commit.get_path(
+                version="v4",
+                repo_hash=self.storage_hash,
+                table=table,
+                field=field,
+                external_id=external_id,
             )
+        else:
+            path = MinioEndpoints.json_data.get_path(
+                version="v4",
+                repo_hash=self.storage_hash,
+                commitid=commit_id,
+                table=table,
+                field=field,
+                external_id=external_id,
+            )
+        stringified_data = json.dumps(data, cls=encoder)
         self.write_file(path, stringified_data)
         return path
 

services/notification/notifiers/mixins/message/helpers.py

@@ -116,6 +116,53 @@ def make_metrics(before, after, relative, show_complexity, yaml):
     return "".join(("|", coverage, complexity, icon))
 
 
+def make_patch_only_metrics(before, after, relative, show_complexity, yaml):
+    if after is None:
+        # e.g. missing flags
+        coverage = " `?` |"
+
+    elif after is False:
+        # e.g. file deleted
+        coverage = " |"
+
+    else:
+        layout = " `{relative}` |"
+        coverage = layout.format(
+            relative=format_number_to_str(
+                yaml, relative.coverage if relative else 0, style="{0}%", if_null="\xF8"
+            ),
+        )
+    return "".join(("|", coverage))
+
+
+def get_metrics_function(hide_project_coverage):
+    if hide_project_coverage:
+        metrics = make_patch_only_metrics
+    else:
+        metrics = make_metrics
+    return metrics
+
+
+def get_table_header(hide_project_coverage, show_complexity):
+    if not hide_project_coverage:
+        table_header = (
+            "| Coverage \u0394 |"
+            + (" Complexity \u0394 |" if show_complexity else "")
+            + " |"
+        )
+    else:
+        table_header = "| Coverage |"
+
+    return table_header
+
+
+def get_table_layout(hide_project_coverage, show_complexity):
+    if hide_project_coverage:
+        return "|---|---|"
+    else:
+        return "|---|---|---|" + ("---|" if show_complexity else "")
+
+
 def format_number_to_str(
     yml, value, if_zero=None, if_null=None, plus=False, style="{0}"
 ) -> str:

services/notification/notifiers/mixins/message/sections.py

@@ -20,6 +20,9 @@
     diff_to_string,
     ellipsis,
     escape_markdown,
+    get_metrics_function,
+    get_table_header,
+    get_table_layout,
     make_metrics,
     sort_by_importance,
 )
@@ -464,11 +467,13 @@ async def do_write_section(self, comparison, diff, changes, links, behind_by=Non
         head_report = comparison.head.report
         if base_report is None:
             base_report = Report()
+        hide_project_coverage = self.settings.get("hide_project_coverage", False)
+        make_metrics_fn = get_metrics_function(hide_project_coverage)
         files_in_diff = [
             (
                 _diff["type"],
                 path,
-                make_metrics(
+                make_metrics_fn(
                     get_totals_from_file_in_reports(base_report, path) or False,
                     get_totals_from_file_in_reports(head_report, path) or False,
                     _diff["totals"],
@@ -487,12 +492,8 @@ async def do_write_section(self, comparison, diff, changes, links, behind_by=Non
             c.path for c in changes or []
         )
         if files_in_diff:
-            table_header = (
-                "| Coverage \u0394 |"
-                + (" Complexity \u0394 |" if self.show_complexity else "")
-                + " |"
-            )
-            table_layout = "|---|---|---|" + ("---|" if self.show_complexity else "")
+            table_header = get_table_header(hide_project_coverage, self.show_complexity)
+            table_layout = get_table_layout(hide_project_coverage, self.show_complexity)
             # add table headers
             yield (
                 "| [Files Changed]({0}?src=pr&el=tree) {1}".format(
@@ -539,7 +540,7 @@ def tree_cell(typ, path, metrics, _=None):
                     )
                 )
 
-        if changes:
+        if changes and not hide_project_coverage:
             len_changes_not_in_diff = len(all_files or []) - len(files_in_diff or [])
             if files_in_diff and len_changes_not_in_diff > 0:
                 yield ("")
@@ -628,7 +629,10 @@ async def do_write_section(self, comparison, diff, changes, links, behind_by=Non
                 + ("---|" if has_carriedforward_flags else "")
             )
 
-            yield ("| Flag " + table_header)
+            yield (
+                "| [Flag]({href}/flags?src=pr&el=flags) ".format(href=links["pull"])
+                + table_header
+            )
             yield (table_layout)
             for flag in sorted(flags, key=lambda f: f["name"]):
                 carriedforward, carriedforward_from = (
@@ -660,7 +664,10 @@ async def do_write_section(self, comparison, diff, changes, links, behind_by=Non
 
                 yield (
                     "| {name} {metrics}{cf}".format(
-                        name=flag["name"],
+                        name="[{flag_name}]({href}/flags?src=pr&el=flag)".format(
+                            flag_name=flag["name"],
+                            href=links["pull"],
+                        ),
                         metrics=make_metrics(
                             flag["before"],
                             flag["after"],
@@ -721,13 +728,18 @@ async def do_write_section(
         )
 
         # Table header and layout
-        yield "| Components | Coverage \u0394 | |"
+        yield "| [Components]({href}/components?src=pr&el=components) | Coverage \u0394 | |".format(
+            href=links["pull"],
+        )
         yield "|---|---|---|"
         # The interesting part
         for component_data in component_data_to_show:
             yield (
                 "| {name} {metrics}".format(
-                    name=component_data["name"],
+                    name="[{component_name}]({href}/components?src=pr&el=component)".format(
+                        component_name=component_data["name"],
+                        href=links["pull"],
+                    ),
                     metrics=make_metrics(
                         component_data["before"],
                         component_data["after"],