Surface AC Systems in Vendor Consents section of the TCF Overlay (#4266)

CHANGELOG.md

@@ -20,6 +20,7 @@ The types of changes are:
 
 ### Added
 - Added a `FidesPreferenceToggled` event to Fides.js to track when user preferences change without being saved [#4253](https://github.com/ethyca/fides/pull/4253)
+- Add AC Systems to the TCF Overlay under Vendor Consents section [#4266](https://github.com/ethyca/fides/pull/4266/)
 
 ### Fixed
 - Stacks that do not have any purposes will no longer render an empty purpose block [#4278](https://github.com/ethyca/fides/pull/4278)

src/fides/api/util/tcf/tc_model.py

@@ -11,7 +11,12 @@
     TCFVendorConsentRecord,
     TCFVendorLegitimateInterestsRecord,
 )
-from fides.api.util.tcf.tcf_experience_contents import TCFExperienceContents, load_gvl
+from fides.api.util.tcf.tcf_experience_contents import (
+    AC_PREFIX,
+    GVL_PREFIX,
+    TCFExperienceContents,
+    load_gvl,
+)
 
 CMP_ID: int = 407
 CMP_VERSION = 1
@@ -20,8 +25,6 @@
 FORBIDDEN_LEGITIMATE_INTEREST_PURPOSE_IDS = [1, 3, 4, 5, 6]
 gvl: Dict = load_gvl()
 
-ac_prefix = "gacp."
-
 
 def universal_vendor_id_to_gvl_id(universal_vendor_id: str) -> int:
     """Converts a universal gvl vendor id to a vendor id
@@ -31,9 +34,9 @@ def universal_vendor_id_to_gvl_id(universal_vendor_id: str) -> int:
 
     We store vendor ids as a universal vendor id internally, but need to strip this off when building TC strings.
     """
-    if universal_vendor_id.startswith(ac_prefix):
+    if universal_vendor_id.startswith(AC_PREFIX):
         raise ValueError("Skipping AC Vendor ID")
-    return int(universal_vendor_id.lstrip("gvl."))
+    return int(universal_vendor_id.lstrip(GVL_PREFIX))
 
 
 class TCModel(FidesSchema):

src/fides/api/util/tcf/tc_string.py

@@ -13,7 +13,7 @@
 from fides.api.util.tcf.tc_model import TCModel, convert_tcf_contents_to_tc_model
 
 # Number of bits allowed for certain sections that are used in multiple places
-from fides.api.util.tcf.tcf_experience_contents import TCFExperienceContents
+from fides.api.util.tcf.tcf_experience_contents import GVL_PREFIX, TCFExperienceContents
 
 USE_NON_STANDARD_TEXT_BITS = 1
 SPECIAL_FEATURE_BITS = 12
@@ -23,7 +23,7 @@
 
 def add_gvl_prefix(vendor_id: str) -> str:
     """Add gvl prefix to create a universal gvl identifier for the given vendor id"""
-    return "gvl." + vendor_id
+    return GVL_PREFIX + vendor_id
 
 
 class TCField(FidesSchema):

src/fides/api/util/tcf/tcf_experience_contents.py

@@ -18,6 +18,7 @@
 from fideslang.models import LegalBasisForProcessingEnum
 from fideslang.validation import FidesKey
 from loguru import logger
+from sqlalchemy import and_, or_
 from sqlalchemy.engine.row import Row  # type:ignore[import]
 from sqlalchemy.orm import Query, Session
 
@@ -47,6 +48,9 @@
     "gvl.json",
 )
 
+AC_PREFIX = "gacp."
+GVL_PREFIX = "gvl."
+
 PURPOSE_DATA_USES: List[str] = []
 for purpose in MAPPED_PURPOSES.values():
     PURPOSE_DATA_USES.extend(purpose.data_uses)
@@ -91,6 +95,16 @@
     List[TCFSpecialFeatureRecord],
 ]
 
+# Common SQLAlchemy filters used below
+AC_SYSTEM_FILTER = System.vendor_id.startswith(AC_PREFIX)
+CONSENT_LEGAL_BASIS_FILTER = (
+    PrivacyDeclaration.legal_basis_for_processing == LegalBasisForProcessingEnum.CONSENT
+)
+LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER = (
+    PrivacyDeclaration.legal_basis_for_processing
+    == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST
+)
+
 
 class ConsentRecordType(Enum):
     """*ALL* of the relevant consent items that can be served or have preferences saved against.
@@ -196,15 +210,20 @@ def get_matching_privacy_declarations(db: Session) -> Query:
             PrivacyDeclaration.legal_basis_for_processing,
             PrivacyDeclaration.features,
         )
-        .join(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id)
+        .outerjoin(PrivacyDeclaration, System.id == PrivacyDeclaration.system_id)
         .filter(
-            PrivacyDeclaration.data_use.in_(ALL_GVL_DATA_USES),
-            PrivacyDeclaration.legal_basis_for_processing.in_(
-                [
-                    LegalBasisForProcessingEnum.CONSENT,
-                    LegalBasisForProcessingEnum.LEGITIMATE_INTEREST,
-                ]
-            ),
+            or_(
+                and_(
+                    PrivacyDeclaration.data_use.in_(ALL_GVL_DATA_USES),
+                    PrivacyDeclaration.legal_basis_for_processing.in_(
+                        [
+                            LegalBasisForProcessingEnum.CONSENT,
+                            LegalBasisForProcessingEnum.LEGITIMATE_INTEREST,
+                        ]
+                    ),
+                ),
+                AC_SYSTEM_FILTER,
+            )
         )
         .order_by(
             PrivacyDeclaration.created_at.desc()
@@ -242,7 +261,7 @@ def get_matching_data_uses_or_features(
         )
 
     unique_features: Set[str] = set()
-    for feature_name in record.features:
+    for feature_name in record.features or []:
         if feature_name in relevant_uses_or_features:
             unique_features.add(feature_name)
     return unique_features
@@ -422,9 +441,45 @@ def build_purpose_or_feature_section_and_update_vendor_map(
                 privacy_declaration_row=privacy_declaration_row,
             )
 
+        # Separately, add AC Vendor to Vendor Consent Map if applicable
+        add_ac_vendor_to_vendor_consent_map(
+            vendor_map=vendor_map,
+            tcf_vendor_component_type=tcf_vendor_component_type,
+            privacy_declaration_row=privacy_declaration_row,
+        )
+
     return non_vendor_record_map, vendor_map
 
 
+def add_ac_vendor_to_vendor_consent_map(
+    vendor_map: Dict[str, VendorRecord],
+    tcf_vendor_component_type: VendorSectionType,
+    privacy_declaration_row: Row,
+) -> None:
+    """
+    Add systems with ac.*-prefixed vendor records to the Vendor Consent section if applicable.
+
+    FE shows Consent toggle only for AC vendors, and they are not required to have Privacy Declarations
+    """
+    vendor_id, _ = get_system_identifiers(privacy_declaration_row)
+
+    if not (vendor_id and vendor_id.startswith(AC_PREFIX)):
+        return
+
+    if not tcf_vendor_component_type == TCFVendorConsentRecord:
+        return
+
+    if vendor_id in vendor_map:
+        return
+
+    vendor_map[vendor_id] = TCFVendorConsentRecord(
+        id=vendor_id,
+        name=privacy_declaration_row.system_name,
+        description=privacy_declaration_row.system_description,
+        has_vendor_id=True,
+    )
+
+
 def populate_vendor_relationships_basic_attributes(
     vendor_map: Dict[str, TCFVendorRelationships],
     matching_privacy_declarations: Query,
@@ -491,7 +546,8 @@ def get_tcf_contents(
 
     matching_privacy_declarations: Query = get_matching_privacy_declarations(db)
 
-    # Collect purposes with a legal basis of consent and update system map
+    # Collect purposes with a legal basis of consent *or* AC systems (which aren't required to have privacy
+    # declarations) and update system map
     (
         purpose_consent_map,
         updated_vendor_consent_map,
@@ -502,8 +558,10 @@ def get_tcf_contents(
         vendor_subsection_name="purpose_consents",
         vendor_map=vendor_consent_map,
         matching_privacy_declaration_query=matching_privacy_declarations.filter(
-            PrivacyDeclaration.legal_basis_for_processing
-            == LegalBasisForProcessingEnum.CONSENT
+            or_(
+                CONSENT_LEGAL_BASIS_FILTER,
+                AC_SYSTEM_FILTER,
+            )
         ),
     )
 
@@ -518,8 +576,7 @@ def get_tcf_contents(
         vendor_subsection_name="purpose_legitimate_interests",
         vendor_map=vendor_legitimate_interests_map,
         matching_privacy_declaration_query=matching_privacy_declarations.filter(
-            PrivacyDeclaration.legal_basis_for_processing
-            == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST
+            LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER
         ),
     )
 
@@ -705,18 +762,14 @@ def get_relevant_systems_for_tcf_attribute(  # pylint: disable=too-many-return-s
 
     if tcf_field == TCFComponentType.purpose_consent.value:
         return systems_that_match_tcf_data_uses(
-            starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.CONSENT
-            ),
+            starting_privacy_declarations.filter(CONSENT_LEGAL_BASIS_FILTER),
             purpose_data_uses,
         )
 
     if tcf_field == TCFComponentType.purpose_legitimate_interests.value:
         return systems_that_match_tcf_data_uses(
             starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST
+                LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER
             ),
             purpose_data_uses,
         )
@@ -740,36 +793,33 @@ def get_relevant_systems_for_tcf_attribute(  # pylint: disable=too-many-return-s
     if tcf_field == TCFComponentType.vendor_consent.value:
         return systems_that_match_vendor_string(
             starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.CONSENT
+                or_(
+                    CONSENT_LEGAL_BASIS_FILTER,
+                    AC_SYSTEM_FILTER,
+                )
             ),
             tcf_value,  # type: ignore[arg-type]
         )
 
     if tcf_field == TCFComponentType.vendor_legitimate_interests.value:
         return systems_that_match_vendor_string(
             starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST
+                LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER
             ),
             tcf_value,  # type: ignore[arg-type]
         )
 
     if tcf_field == TCFComponentType.system_consent.value:
         return systems_that_match_system_id(
-            starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.CONSENT
-            ),
+            starting_privacy_declarations.filter(CONSENT_LEGAL_BASIS_FILTER),
             tcf_value
             # type: ignore[arg-type]
         )
 
     if tcf_field == TCFComponentType.system_legitimate_interests.value:
         return systems_that_match_system_id(
             starting_privacy_declarations.filter(
-                PrivacyDeclaration.legal_basis_for_processing
-                == LegalBasisForProcessingEnum.LEGITIMATE_INTEREST
+                LEGITIMATE_INTEREST_LEGAL_BASIS_FILTER
             ),
             tcf_value,  # type: ignore[arg-type]
         )

tests/fixtures/application_fixtures.py

@@ -2831,8 +2831,8 @@ def tcf_system(db: Session) -> System:
 
 
 @pytest.fixture(scope="function")
-def ac_system(db: Session) -> System:
-    """Test AC System - will be fleshed out further later"""
+def ac_system_with_privacy_declaration(db: Session) -> System:
+    """Test AC System with a privacy declaration"""
     system = System.create(
         db=db,
         data={
@@ -2859,6 +2859,23 @@ def ac_system(db: Session) -> System:
     return system
 
 
+@pytest.fixture(scope="function")
+def ac_system_without_privacy_declaration(db: Session) -> System:
+    """Test AC System without privacy declaration"""
+    system = System.create(
+        db=db,
+        data={
+            "fides_key": f"ac_system{uuid.uuid4()}",
+            "vendor_id": "gacp.100",
+            "name": f"Test AC System",
+            "organization_fides_key": "default_organization",
+            "system_type": "Service",
+        },
+    )
+
+    return system
+
+
 # Detailed systems with attributes for TC string testing
 # Please don't update them!