chore: sync from 5168

Makefile

@@ -225,6 +225,14 @@ synchronize-from-nmp-mr: ## Sync from NMP MR. Usage: make synchronize-from-nmp-m
 ifndef MR
 	$(error MR is required. Usage: make synchronize-from-nmp-mr MR=5603)
 endif
+ifeq ($(shell git rev-parse --abbrev-ref HEAD), main)
+	@echo "~~~~~~"
+	@echo "you are on the main branch"
+	@echo "creating a new branch from main"
+	git checkout -b $$USER/sync-$(MR)-from-nmp
+endif
+	@echo "~~~~~~"
+	@echo "synchronizing the changes from the NMP MR $(MR) to the nemo_safe_synthesizer package"
 	bash tools/sync-from-mr.sh $(MR)
 
 .PHONY: synchronize-py-files-from-nmp

src/nemo_safe_synthesizer/artifacts/analyzers/field_features.py

@@ -199,15 +199,16 @@ def describe_field(field_name: str, data: Series) -> FieldFeatures:
 
     non_na_data = data.dropna()
     non_na_count = int(non_na_data.count())
-
-    unique_count = int(non_na_data.nunique())
+    unique_values_list = non_na_data.unique().tolist()
+    unique_count = len(unique_values_list)
     missing_count = total_count - non_na_count
 
     lengths = [len(str(entry)) for entry in non_na_data]
     features = FieldFeatures(
         name=field_name,
         type=FieldType.OTHER,
         count=non_na_count,
+        unique_values_list=unique_values_list,
         unique_count=unique_count,
         unique_percent=(round(unique_count / non_na_count * 100, 4) if non_na_count > 0 else 0),
         missing_count=missing_count,

src/nemo_safe_synthesizer/artifacts/base/fields.py

@@ -5,7 +5,7 @@
 
 from enum import StrEnum
 from functools import cached_property
-from typing import Union
+from typing import Any, Union
 
 from pydantic import BaseModel, Field
 
@@ -28,6 +28,11 @@ class FieldFeatures(BaseModel):
     Number of non-empty values.
     """
 
+    unique_values_list: list[Any]
+    """
+    List of unique values.
+    """
+
     unique_count: int
     """
     Number of unique values.

src/nemo_safe_synthesizer/evaluation/components/pii_replay.py

@@ -19,7 +19,23 @@
 
 
 class PIIReplayData(BaseModel):
+    """
+    Contains data for each PII column, listed in the PII Replay section of the SQS report.
+
+    Args:
+        column_name: The name of the column with PII data.
+        column_assigned_type: The assigned type for the column, whether it's text, unique identifier, date, email, etc.
+        pii_type: Type of the PII data in the column. For non-text fields, this is the same as column_assigned_type. For text fields, it is the PII entities detected within the text such as race, SSN, address, etc.
+        total_ref_data: Total number of rows in the reference data that contain PII values.
+        unique_ref_data: Total number of rows in the reference data that contain unique PII values.
+        total_synth_data: Total number of output rows that contain PII present in the reference data.
+        unique_synth_data: Total number of output rows that contain unique PII present in the reference data.
+        unique_synth_data_percentage: Percentage of unique PII in the output data that matches unique entity data in the reference dataset.
+
+    """
+
     column_name: str = Field()
+    column_assigned_type: str = Field()
     pii_type: str = Field(default=UNKNOWN_ENTITY)
     total_ref_data: int = Field(default=0)
     unique_ref_data: int = Field(default=0)
@@ -56,10 +72,11 @@ def from_evaluation_dataset(evaluation_dataset, config: SafeSynthesizerParameter
         classified_entities = []
         for col, column_statistics in evaluation_dataset.column_statistics.items():
             entity_names = column_statistics.detected_entity_counts.keys()
+            entity_assigned_type = column_statistics.assigned_type
             # Scope down to user supplied set of entities if there is one
             if pii_replay_entities:
                 entity_names = set(entity_names).intersection(set(pii_replay_entities))
-            classified_entities += [(col, entity_name) for entity_name in entity_names]
+            classified_entities += [(col, entity_name, entity_assigned_type) for entity_name in entity_names]
             # But add user specified set of columns as needed if there is one
             if pii_replay_columns:
                 for user_specified_col in set(pii_replay_columns).difference(
@@ -68,9 +85,7 @@ def from_evaluation_dataset(evaluation_dataset, config: SafeSynthesizerParameter
                     classified_entities.append((user_specified_col, UNKNOWN_ENTITY))
 
         pii_replay_data = []
-        for ce in classified_entities:
-            col, entity_name = ce[0], ce[1]
-
+        for col, entity_name, entity_assigned_type in classified_entities:
             # UNKNOWN_ENTITY case, use the entire column
             ref_entity_count = len(evaluation_dataset.reference[col])
             ref_entity_unique_values = evaluation_dataset.reference[col].unique()
@@ -94,6 +109,7 @@ def from_evaluation_dataset(evaluation_dataset, config: SafeSynthesizerParameter
             pii_replay_data.append(
                 PIIReplayData(
                     column_name=col,
+                    column_assigned_type=entity_assigned_type,
                     pii_type=entity_name,
                     total_ref_data=ref_entity_count,
                     unique_ref_data=ref_entity_unique_count,

src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py

@@ -161,6 +161,11 @@ def from_dataframes(
                 ),
                 config=config,
             )
+        # drop columns with assigned type "text" from pii_replay.
+        # TODO: This will be removed once text entities is added to the PII Replay section of the evaluation report.
+        pii_replay.pii_replay_data = [
+            datum for datum in pii_replay.pii_replay_data if datum.column_assigned_type != "text"
+        ]
         components.append(pii_replay)
 
         dataset_statistics = DatasetStatistics.from_evaluation_dataset(evaluation_dataset)