Setting requires_primary_keys for select connectors + updating tests

src/fides/api/service/connectors/base_connector.py

@@ -135,5 +135,11 @@ def execute_standalone_retrieval_query(
 
     @property
     def requires_primary_keys(self) -> bool:
-        """Indicates if datasets linked to this connector require primary keys for erasures. Defaults to True."""
+        """
+        Indicates if datasets linked to this connector require primary keys for erasures.
+        Defaults to True.
+        """
+
+        # Defaulting to true for now so we can keep the default behavior and
+        # incrementally determine the need for primary keys across all connectors
         return True

src/fides/api/service/connectors/bigquery_connector.py

@@ -35,6 +35,7 @@ class BigQueryConnector(SQLConnector):
 
     @property
     def requires_primary_keys(self) -> bool:
+        """BigQuery does not have the concept of primary keys so they're not required for erasures."""
         return False
 
     # Overrides BaseConnector.build_uri

src/fides/api/service/connectors/postgres_connector.py

@@ -19,6 +19,11 @@ class PostgreSQLConnector(SQLConnector):
 
     secrets_schema = PostgreSQLSchema
 
+    @property
+    def requires_primary_keys(self) -> bool:
+        """Postgres allows arbitrary columns in the WHERE clause for updates so primary keys are not required."""
+        return False
+
     def build_uri(self) -> str:
         """Build URI of format postgresql://[user[:password]@][netloc][:port][/dbname]"""
         config = self.secrets_schema(**self.configuration.secrets or {})

src/fides/api/service/connectors/query_configs/query_config.py

@@ -430,7 +430,7 @@ def get_update_stmt(
     def get_update_clauses(
         self,
         update_value_map: Dict[str, Any],
-        non_empty_reference_fields: Dict[str, Field],
+        where_clause_fields: Dict[str, Field],
     ) -> List[str]:
         """Returns a list of update clauses for the update statement."""
 
@@ -567,7 +567,7 @@ def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]
     def get_update_clauses(
         self,
         update_value_map: Dict[str, Any],
-        non_empty_reference_fields: Dict[str, Field],
+        where_clause_fields: Dict[str, Field],
     ) -> List[str]:
         """Returns a list of update clauses for the update statement."""
         return self.format_key_map_for_update_stmt(update_value_map)

src/fides/api/service/connectors/saas_connector.py

@@ -72,7 +72,9 @@
 class SaaSConnector(BaseConnector[AuthenticatedClient], Contextualizable):
     """A connector type to integrate with third-party SaaS APIs"""
 
+    @property
     def requires_primary_keys(self) -> bool:
+        """SaaS connectors work with HTTP requests, so the database concept of primary keys does not apply."""
         return False
 
     def get_log_context(self) -> Dict[LoggerContextKeys, Any]:

src/fides/api/service/connectors/scylla_connector.py

@@ -28,6 +28,11 @@ class ScyllaConnectorMissingKeyspace(Exception):
 class ScyllaConnector(BaseConnector[Cluster]):
     """Scylla Connector"""
 
+    @property
+    def requires_primary_keys(self) -> bool:
+        """ScyllaDB requires primary keys for erasures."""
+        return True
+
     def build_uri(self) -> str:
         """
         Builds URI - Not yet implemented

src/fides/api/service/connectors/scylla_query_config.py

@@ -77,14 +77,19 @@ def format_key_map_for_update_stmt(self, param_map: Dict[str, Any]) -> List[str]
     def get_update_clauses(
         self,
         update_value_map: Dict[str, Any],
-        non_empty_reference_fields: Dict[str, Field],
+        where_clause_fields: Dict[str, Field],
     ) -> List[str]:
-        """Returns a list of update clauses for the update statement."""
+        """Returns a list of update clauses for the update statement.
+
+        Omits primary key fields from updates since ScyllaDB prohibits
+        updating primary key fields.
+        """
+
         return self.format_key_map_for_update_stmt(
             {
                 key: value
                 for key, value in update_value_map.items()
-                if key not in non_empty_reference_fields
+                if key not in where_clause_fields
             }
         )
 

src/fides/data/sample_project/sample_resources/postgres_example_test_dataset.yml

@@ -11,6 +11,8 @@ dataset:
             data_categories: [user.contact.address.street]
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: state
             data_categories: [user.contact.address.state]
           - name: street
@@ -36,6 +38,8 @@ dataset:
               data_type: string
           - name: id
             data_categories: [user.unique_id]
+            fides_meta:
+              primary_key: True
           - name: name
             data_categories: [user.name]
             fides_meta:
@@ -58,6 +62,8 @@ dataset:
               data_type: string
           - name: id
             data_categories: [user.unique_id]
+            fides_meta:
+              primary_key: True
           - name: name
             data_categories: [user.name]
             fides_meta:
@@ -74,6 +80,8 @@ dataset:
                   direction: from
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: time
             data_categories: [user.sensor]
 
@@ -88,6 +96,8 @@ dataset:
                   direction: from
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: shipping_address_id
             data_categories: [system.operations]
             fides_meta:
@@ -138,6 +148,8 @@ dataset:
                   direction: from
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: name
             data_categories: [user.financial]
           - name: preferred
@@ -147,6 +159,8 @@ dataset:
         fields:
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: name
             data_categories: [system.operations]
           - name: price
@@ -161,6 +175,8 @@ dataset:
               data_type: string
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: month
             data_categories: [system.operations]
           - name: name
@@ -193,6 +209,8 @@ dataset:
                   direction: from
           - name: id
             data_categories: [system.operations]
+            fides_meta:
+              primary_key: True
           - name: opened
             data_categories: [system.operations]