diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/__init__.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/__init__.py index 584161e3d..338a5771a 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/__init__.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/__init__.py @@ -2,25 +2,25 @@ # SPDX-License-Identifier: Apache-2.0 # Make all checks available to import from `data_checks` module. -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( +from .base import ( DataCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.dataset_size import ( +from .dataset_size import ( DatasetSizeCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.high_float_precision import ( +from .high_float_precision import ( HighFloatPrecisionCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.missing_data import ( +from .missing_data import ( MissingDataCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.sparse_data import ( +from .sparse_data import ( SparseDataCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.surrounding_whitespaces import ( +from .surrounding_whitespaces import ( SurroundingWhitespacesCheck, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.text_data import ( +from .text_data import ( TextDataCheck, ) diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/base.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/base.py index 217de80d9..216d9b5ea 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/base.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/base.py @@ -6,14 +6,14 @@ from abc import ABC, abstractmethod from typing import Any -from nemo_safe_synthesizer.artifacts.base.analyzer import ( +from ...base.analyzer import ( AnalyzerContext, ArtifactAnalyzer, ) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( +from ...base.data_checks import ( DataCheckResult, ) -from nemo_safe_synthesizer.artifacts.base.metrics import timed +from ...base.metrics import timed class DataChecksAnalyzer(ArtifactAnalyzer): diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/dataset_size.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/dataset_size.py index 9f82215cd..db64485f5 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/dataset_size.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/dataset_size.py @@ -5,14 +5,14 @@ from typing import TYPE_CHECKING -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import DataCheck -from nemo_safe_synthesizer.artifacts.base.data_checks import ( +from ...base.data_checks import ( DataCheckResult, DataCheckWarning, ) +from .base import DataCheck if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import AnalyzerContext + from ...base.analyzer import AnalyzerContext _RECORD_COUNT_LIMIT = 50_000 _FIELD_COUNT_LIMIT = 30 diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/high_float_precision.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/high_float_precision.py index 4e84a0d74..aea5260a2 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/high_float_precision.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/high_float_precision.py @@ -5,20 +5,22 @@ from typing import TYPE_CHECKING -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( +from ...base.data_checks import ( + DataCheckResult, + DataCheckWarning, +) +from ...base.fields import ( + FieldType, +) +from .base import ( DataCheck, plural_verb, warning_explain_prefix, ) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( - DataCheckResult, - DataCheckWarning, -) -from nemo_safe_synthesizer.artifacts.base.fields import FieldType if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import AnalyzerContext - from nemo_safe_synthesizer.artifacts.base.fields import FieldFeatures + from ...base.analyzer import AnalyzerContext + from ...base.fields import FieldFeatures _FLOAT_PRECISION_LIMIT = 6 diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/missing_data.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/missing_data.py index 597bc8713..1d2d6f026 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/missing_data.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/missing_data.py @@ -5,19 +5,19 @@ from typing import TYPE_CHECKING -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( +from ...base.data_checks import ( + DataCheckResult, + DataCheckWarning, +) +from .base import ( DataCheck, plural_verb, warning_explain_prefix, ) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( - DataCheckResult, - DataCheckWarning, -) if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import AnalyzerContext - from nemo_safe_synthesizer.artifacts.base.fields import FieldFeatures + from ...base.analyzer import AnalyzerContext + from ...base.fields import FieldFeatures _MISSING_PERCENTAGE_LIMIT = 50 diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/sparse_data.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/sparse_data.py index aaabd1b16..476ebee5a 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/sparse_data.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/sparse_data.py @@ -7,24 +7,24 @@ import numpy as np -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( - DataCheck, - plural_verb, - warning_explain_prefix, -) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( +from ...base.data_checks import ( DataCheckResult, DataCheckWarning, ) -from nemo_safe_synthesizer.artifacts.base.fields import ( +from ...base.fields import ( FieldType, ) +from .base import ( + DataCheck, + plural_verb, + warning_explain_prefix, +) if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import ( + from ...base.analyzer import ( AnalyzerContext, ) - from nemo_safe_synthesizer.artifacts.base.fields import ( + from ...base.fields import ( FieldFeatures, ) diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/surrounding_whitespaces.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/surrounding_whitespaces.py index df548a743..b0f4c69e9 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/surrounding_whitespaces.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/surrounding_whitespaces.py @@ -8,18 +8,18 @@ from pandas import Series from pandas.core.dtypes.common import is_string_or_object_np_dtype -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( +from ...base.data_checks import ( + DataCheckResult, + DataCheckWarning, +) +from .base import ( DataCheck, plural_verb, warning_explain_prefix, ) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( - DataCheckResult, - DataCheckWarning, -) if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import AnalyzerContext + from ...base.analyzer import AnalyzerContext _VALUES_WITH_WHITESPACE_PERCENTAGE_LIMIT = 20 diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/text_data.py b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/text_data.py index 754de8935..8e3363c5e 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/text_data.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/data_checks/text_data.py @@ -5,19 +5,19 @@ from typing import TYPE_CHECKING -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( +from ...base.data_checks import ( + DataCheckResult, + DataCheckWarning, +) +from ...base.fields import FieldType +from .base import ( DataCheck, plural_verb, warning_explain_prefix, ) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( - DataCheckResult, - DataCheckWarning, -) -from nemo_safe_synthesizer.artifacts.base.fields import FieldType if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.analyzer import AnalyzerContext + from ...base.analyzer import AnalyzerContext _TEXT_FIELDS_COUNT_LIMIT = 2 diff --git a/src/nemo_safe_synthesizer/artifacts/analyzers/field_features.py b/src/nemo_safe_synthesizer/artifacts/analyzers/field_features.py index 76a806c1b..3b59e510b 100644 --- a/src/nemo_safe_synthesizer/artifacts/analyzers/field_features.py +++ b/src/nemo_safe_synthesizer/artifacts/analyzers/field_features.py @@ -12,19 +12,19 @@ from pandas import Series from pandas.core.dtypes.common import is_float_dtype, is_numeric_dtype -from nemo_safe_synthesizer.artifacts.base.analyzer import ( +from ..base.analyzer import ( AnalyzerContext, ArtifactAnalyzer, ) -from nemo_safe_synthesizer.artifacts.base.fields import ( +from ..base.fields import ( FieldFeatures, FieldFeaturesInfo, FieldType, ) -from nemo_safe_synthesizer.artifacts.base.metrics import timed +from ..base.metrics import timed if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.metadata import ( + from ..base.metadata import ( DatasetMetadata, ) diff --git a/src/nemo_safe_synthesizer/artifacts/base/analyzer.py b/src/nemo_safe_synthesizer/artifacts/base/analyzer.py index 7fdbd3c36..3a6f3b2dd 100644 --- a/src/nemo_safe_synthesizer/artifacts/base/analyzer.py +++ b/src/nemo_safe_synthesizer/artifacts/base/analyzer.py @@ -9,20 +9,20 @@ from pandas import DataFrame from pydantic import BaseModel, ConfigDict, Field -from nemo_safe_synthesizer.artifacts.base.name_anonymizer import NoopNameAnonymizer +from .name_anonymizer import NoopNameAnonymizer if TYPE_CHECKING: - from nemo_safe_synthesizer.artifacts.base.data_checks import ( + from .data_checks import ( DataCheckResult, ) - from nemo_safe_synthesizer.artifacts.base.fields import ( + from .fields import ( FieldFeatures, FieldFeaturesInfo, ) - from nemo_safe_synthesizer.artifacts.base.metadata import ( + from .metadata import ( DatasetMetadata, ) - from nemo_safe_synthesizer.artifacts.base.name_anonymizer import NameAnonymizer + from .name_anonymizer import NameAnonymizer class AnalyzerContext(BaseModel): diff --git a/src/nemo_safe_synthesizer/artifacts/base/metadata.py b/src/nemo_safe_synthesizer/artifacts/base/metadata.py index 9e9a02bd5..b160f1fbc 100644 --- a/src/nemo_safe_synthesizer/artifacts/base/metadata.py +++ b/src/nemo_safe_synthesizer/artifacts/base/metadata.py @@ -7,7 +7,7 @@ from pydantic import Field -from nemo_safe_synthesizer.config.base import NSSBaseModel +from ...config.base import NSSBaseModel class FieldAttribute(StrEnum): diff --git a/src/nemo_safe_synthesizer/artifacts/data_checks_processor.py b/src/nemo_safe_synthesizer/artifacts/data_checks_processor.py index c5514fc31..d8f84a06b 100644 --- a/src/nemo_safe_synthesizer/artifacts/data_checks_processor.py +++ b/src/nemo_safe_synthesizer/artifacts/data_checks_processor.py @@ -5,25 +5,15 @@ from pandas import DataFrame -from nemo_safe_synthesizer.artifacts.analyzers.data_checks import ( +from ..observability import get_logger +from .analyzers.data_checks import ( create_all_checks, ) -from nemo_safe_synthesizer.artifacts.analyzers.data_checks.base import ( - DataChecksAnalyzer, -) -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - FieldFeaturesAnalyzer, -) -from nemo_safe_synthesizer.artifacts.base.analyzer import ( - AnalyzerContext, - ArtifactAnalyzer, - DataFrameBasicAnalyzer, -) -from nemo_safe_synthesizer.artifacts.base.data_checks import ( - DataCheckResults, -) -from nemo_safe_synthesizer.artifacts.base.name_anonymizer import NoopNameAnonymizer -from nemo_safe_synthesizer.observability import get_logger +from .analyzers.data_checks.base import DataChecksAnalyzer +from .analyzers.field_features import FieldFeaturesAnalyzer +from .base.analyzer import AnalyzerContext, ArtifactAnalyzer, DataFrameBasicAnalyzer +from .base.data_checks import DataCheckResults +from .base.name_anonymizer import NoopNameAnonymizer logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/cli/artifacts.py b/src/nemo_safe_synthesizer/cli/artifacts.py index 4b0efe4c7..b9df157ee 100644 --- a/src/nemo_safe_synthesizer/cli/artifacts.py +++ b/src/nemo_safe_synthesizer/cli/artifacts.py @@ -8,7 +8,7 @@ import click -from nemo_safe_synthesizer.cli.artifact_structure import BoundDir, PathT, Workdir +from .artifact_structure import BoundDir, PathT, Workdir @click.group(invoke_without_command=True) diff --git a/src/nemo_safe_synthesizer/cli/run.py b/src/nemo_safe_synthesizer/cli/run.py index c9d9d3c2a..12e0fc0c9 100644 --- a/src/nemo_safe_synthesizer/cli/run.py +++ b/src/nemo_safe_synthesizer/cli/run.py @@ -30,7 +30,7 @@ ) if TYPE_CHECKING: - from nemo_safe_synthesizer.sdk.library_builder import SafeSynthesizer + from ..sdk.library_builder import SafeSynthesizer def common_run_options(f): @@ -216,7 +216,7 @@ def run( run_logger.debug("running with: ", extra={"config": config.model_dump()}) with traced_user("SafeSynthesizer"): - from nemo_safe_synthesizer.sdk.library_builder import SafeSynthesizer + from ..sdk.library_builder import SafeSynthesizer ss: SafeSynthesizer = SafeSynthesizer(config=config, workdir=workdir).with_data_source(df) ss.run() @@ -270,7 +270,7 @@ def run_train( settings=settings, phase="train", ) - from nemo_safe_synthesizer.sdk.library_builder import SafeSynthesizer + from ..sdk.library_builder import SafeSynthesizer with traced_user("SafeSynthesizer"): SafeSynthesizer(config, workdir=workdir).with_data_source(df).process_data().train() @@ -347,7 +347,7 @@ def run_generate( auto_discover_adapter=auto_discover_adapter, wandb_resume_job_id=wandb_resume_job_id, ) - from nemo_safe_synthesizer.sdk.library_builder import SafeSynthesizer + from ..sdk.library_builder import SafeSynthesizer final_output_file = settings.output_file or workdir.output_file with traced_user("SafeSynthesizer"): diff --git a/src/nemo_safe_synthesizer/cli/utils.py b/src/nemo_safe_synthesizer/cli/utils.py index a94d57f0e..25d825f67 100644 --- a/src/nemo_safe_synthesizer/cli/utils.py +++ b/src/nemo_safe_synthesizer/cli/utils.py @@ -272,7 +272,7 @@ def common_setup( config = merge_overrides(settings.config_path, synthesis_overrides) # 6. Initialize wandb (uses workdir for run ID files) - initialize_wandb_run(workdir, resume_job_id=wandb_resume_job_id) + initialize_wandb_run(workdir, resume_job_id=wandb_resume_job_id, cfg=config) return run_logger, config, df, workdir diff --git a/src/nemo_safe_synthesizer/cli/wandb_setup.py b/src/nemo_safe_synthesizer/cli/wandb_setup.py index 8741fb8e3..a62a6f10b 100644 --- a/src/nemo_safe_synthesizer/cli/wandb_setup.py +++ b/src/nemo_safe_synthesizer/cli/wandb_setup.py @@ -132,6 +132,9 @@ def update_wandb_config( cfg: SafeSynthesizerParameters to log additional_configs: Additional key-value pairs to log """ + if wandb.run is None: + return + if additional_configs is None: additional_configs = {} @@ -141,7 +144,11 @@ def update_wandb_config( wandb.config.update(config_dict, allow_val_change=True) -def initialize_wandb_run(workdir: Workdir, resume_job_id: str | None = None) -> None: +def initialize_wandb_run( + workdir: Workdir, + resume_job_id: str | None = None, + cfg: SafeSynthesizerParameters | None = None, +) -> None: """Initialize or resume a wandb run with consistent configuration. This function handles four cases (in priority order): @@ -153,6 +160,7 @@ def initialize_wandb_run(workdir: Workdir, resume_job_id: str | None = None) -> Args: workdir: Workdir structure containing paths for run ID files resume_job_id: Optional wandb run ID or path to file containing the ID + cfg: Optional SafeSynthesizerParameters to log to wandb config """ settings = WandbSettings() @@ -187,7 +195,6 @@ def initialize_wandb_run(workdir: Workdir, resume_job_id: str | None = None) -> if wandb.run is not None: run_id_file.parent.mkdir(parents=True, exist_ok=True) run_id_file.write_text(wandb.run.id, encoding="utf-8") - update_wandb_config(additional_configs=additional_configs) # Case 2: Explicit resume_job_id provided (ID or file path) elif resume_job_id is not None: @@ -224,7 +231,6 @@ def initialize_wandb_run(workdir: Workdir, resume_job_id: str | None = None) -> else: logger.info(f"Creating new wandb run: {workdir.run_name}") run_id_file.parent.mkdir(parents=True, exist_ok=True) - update_wandb_config(additional_configs=additional_configs) wandb.init( project=wandb_project, name=workdir.run_name, @@ -236,6 +242,9 @@ def initialize_wandb_run(workdir: Workdir, resume_job_id: str | None = None) -> run_id_file.write_text(wandb.run.id, encoding="utf-8") logger.info(f"Saved wandb run ID to {workdir.wandb_run_id_file}") + # Log config to wandb (only for new runs - resumed runs already have config) + update_wandb_config(cfg, additional_configs=additional_configs) + # Log run info logger.info(f"Wandb run name: {wandb.run.name if wandb.run else 'None'}") logger.info(f"Wandb run id: {wandb.run.id if wandb.run else 'None'}") diff --git a/src/nemo_safe_synthesizer/data_processing/actions/data_actions.py b/src/nemo_safe_synthesizer/data_processing/actions/data_actions.py index ed5db6675..01d6650e0 100644 --- a/src/nemo_safe_synthesizer/data_processing/actions/data_actions.py +++ b/src/nemo_safe_synthesizer/data_processing/actions/data_actions.py @@ -36,12 +36,13 @@ model_validator, ) -from nemo_safe_synthesizer import utils -from nemo_safe_synthesizer.data_processing.actions.distributions import ( +from ... import utils +from ...observability import get_logger +from .distributions import ( DatetimeDistributionT, DistributionT, ) -from nemo_safe_synthesizer.data_processing.actions.utils import ( +from .utils import ( ActionCtx, DataSourceT, MetadataColumns, @@ -51,7 +52,6 @@ guess_datetime_format, type_alias_fn, ) -from nemo_safe_synthesizer.observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/data_processing/actions/utils.py b/src/nemo_safe_synthesizer/data_processing/actions/utils.py index e8b490639..604ef8fad 100644 --- a/src/nemo_safe_synthesizer/data_processing/actions/utils.py +++ b/src/nemo_safe_synthesizer/data_processing/actions/utils.py @@ -22,10 +22,15 @@ import numpy as np import pandas as pd -from pydantic import BaseModel, ConfigDict, Field, PrivateAttr +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, +) from typing_extensions import Self -from nemo_safe_synthesizer.data_processing.actions.dates import parse_date +from .dates import parse_date T = TypeVar("T") @@ -86,14 +91,14 @@ class TransformsUtil: """ def __init__(self, seed: Optional[int] = None) -> None: - from nemo_safe_synthesizer.pii_replacer.data_editor.edit import ( + from ...pii_replacer.data_editor.edit import ( Environment, ) self.env = Environment(locales=None, seed=seed, globals_config={}, entity_extractor=None) def execute_col_updates(self, col: str, df: pd.DataFrame, updates: list[str]) -> pd.DataFrame: - from nemo_safe_synthesizer.pii_replacer.data_editor.edit import ( + from ...pii_replacer.data_editor.edit import ( ProgressLog, Step, ) @@ -111,7 +116,7 @@ def execute_col_updates(self, col: str, df: pd.DataFrame, updates: list[str]) -> return Step.execute(df, {}, {}, step, self.env, ProgressLog(30), None) def execute_updates(self, df: pd.DataFrame, updates: list[TransformsUpdate]) -> pd.DataFrame: - from nemo_safe_synthesizer.pii_replacer.data_editor.edit import ( + from ...pii_replacer.data_editor.edit import ( ProgressLog, Step, ) @@ -133,7 +138,7 @@ def execute_updates(self, df: pd.DataFrame, updates: list[TransformsUpdate]) -> return Step.execute(df, {}, {}, step, self.env, ProgressLog(30), None) def execute_drop_condition(self, batch: pd.DataFrame, conditions: list) -> pd.DataFrame: - from nemo_safe_synthesizer.pii_replacer.data_editor.edit import ( + from ...pii_replacer.data_editor.edit import ( ProgressLog, Step, ) diff --git a/src/nemo_safe_synthesizer/data_processing/assembler.py b/src/nemo_safe_synthesizer/data_processing/assembler.py index 846cea7a0..6ae42c8ce 100644 --- a/src/nemo_safe_synthesizer/data_processing/assembler.py +++ b/src/nemo_safe_synthesizer/data_processing/assembler.py @@ -20,27 +20,27 @@ from tqdm.auto import tqdm from transformers import PreTrainedTokenizer -from nemo_safe_synthesizer import utils -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.data_processing.record_utils import ( +from .. import utils +from ..config.parameters import SafeSynthesizerParameters +from ..data_processing.record_utils import ( extract_records_from_jsonl_string, records_to_jsonl, ) -from nemo_safe_synthesizer.data_processing.stats import ( +from ..data_processing.stats import ( RunningStatistics, Statistics, ) -from nemo_safe_synthesizer.defaults import ( +from ..defaults import ( DEFAULT_CACHE_PREFIX, TRAIN_SET_SIZE_BUFFER, ) -from nemo_safe_synthesizer.errors import ( +from ..errors import ( GenerationError, ParameterError, ) -from nemo_safe_synthesizer.holdout.holdout import grouped_train_test_split, naive_train_test_split -from nemo_safe_synthesizer.llm.metadata import ModelMetadata -from nemo_safe_synthesizer.observability import get_logger +from ..holdout.holdout import grouped_train_test_split, naive_train_test_split +from ..llm.metadata import ModelMetadata +from ..observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/data_processing/record_utils.py b/src/nemo_safe_synthesizer/data_processing/record_utils.py index 58f5d8777..c7aef0efc 100644 --- a/src/nemo_safe_synthesizer/data_processing/record_utils.py +++ b/src/nemo_safe_synthesizer/data_processing/record_utils.py @@ -11,7 +11,7 @@ import jsonschema import pandas as pd -from nemo_safe_synthesizer.observability import get_logger +from ..observability import get_logger RECORD_REGEX_PATTERN = r"{.+?}(?:\n|$)" RECORD_REGEX_PATTEN_LOOKAHEAD = r"{.+?}(?=\n|$)" diff --git a/src/nemo_safe_synthesizer/data_processing/records/base.py b/src/nemo_safe_synthesizer/data_processing/records/base.py index d53c2b6d1..060a93376 100644 --- a/src/nemo_safe_synthesizer/data_processing/records/base.py +++ b/src/nemo_safe_synthesizer/data_processing/records/base.py @@ -8,7 +8,7 @@ from numbers import Number from typing import Iterable, List, Union -from nemo_safe_synthesizer.data_processing.records.value_path import ( +from .value_path import ( ValuePath, value_path_to_json_path, ) diff --git a/src/nemo_safe_synthesizer/data_processing/records/fragment.py b/src/nemo_safe_synthesizer/data_processing/records/fragment.py index 921975e1b..433d7c194 100644 --- a/src/nemo_safe_synthesizer/data_processing/records/fragment.py +++ b/src/nemo_safe_synthesizer/data_processing/records/fragment.py @@ -11,8 +11,8 @@ from datetime import datetime from typing import List, Tuple, Union -from nemo_safe_synthesizer.pii_replacer.ner.entity import Score -from nemo_safe_synthesizer.pii_replacer.ner.predictor import NERPrediction +from ...pii_replacer.ner.entity import Score +from ...pii_replacer.ner.predictor import NERPrediction class MetadataError(Exception): diff --git a/src/nemo_safe_synthesizer/data_processing/records/json_record.py b/src/nemo_safe_synthesizer/data_processing/records/json_record.py index 4b1ab71d8..1a182d77f 100644 --- a/src/nemo_safe_synthesizer/data_processing/records/json_record.py +++ b/src/nemo_safe_synthesizer/data_processing/records/json_record.py @@ -11,8 +11,8 @@ from itertools import chain, starmap from typing import Optional, Tuple -import nemo_safe_synthesizer.data_processing.records.base as base -from nemo_safe_synthesizer.data_processing.records.value_path import ( +from . import base +from .value_path import ( value_path, value_path_to_field_name, ) diff --git a/src/nemo_safe_synthesizer/evaluation/components/attribute_inference_protection.py b/src/nemo_safe_synthesizer/evaluation/components/attribute_inference_protection.py index 74ceb6cde..6640d339f 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/attribute_inference_protection.py +++ b/src/nemo_safe_synthesizer/evaluation/components/attribute_inference_protection.py @@ -13,28 +13,22 @@ from typing import cast import category_encoders as ce -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - describe_field, -) -from nemo_safe_synthesizer.config.evaluate import QUASI_IDENTIFIER_COUNT -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, - PrivacyGrade, -) -from nemo_safe_synthesizer.observability import get_logger from pandas.api.types import is_float_dtype from pydantic import ConfigDict, Field from sentence_transformers import SentenceTransformer, util from sklearn.preprocessing import QuantileTransformer +from ...artifacts.analyzers.field_features import describe_field +from ...config.evaluate import QUASI_IDENTIFIER_COUNT +from ...config.parameters import SafeSynthesizerParameters +from ...observability import get_logger +from ..components.component import Component +from ..data_model.evaluation_dataset import EvaluationDataset +from ..data_model.evaluation_score import EvaluationScore, PrivacyGrade +from . import multi_modal_figures as figures + faiss_available = False try: import faiss diff --git a/src/nemo_safe_synthesizer/evaluation/components/column_distribution.py b/src/nemo_safe_synthesizer/evaluation/components/column_distribution.py index 638fabd0d..54cae87b4 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/column_distribution.py +++ b/src/nemo_safe_synthesizer/evaluation/components/column_distribution.py @@ -5,27 +5,20 @@ from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - FieldType, -) -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_field import ( - EvaluationField, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics from plotly.graph_objects import Figure from pydantic import BaseModel, Field +from ...artifacts.analyzers.field_features import FieldType +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_field import EvaluationField +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...observability import get_logger +from ...pii_replacer.transform_result import ColumnStatistics +from . import multi_modal_figures as figures + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/component.py b/src/nemo_safe_synthesizer/evaluation/components/component.py index 4db40150c..fb0dc22a8 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/component.py +++ b/src/nemo_safe_synthesizer/evaluation/components/component.py @@ -7,17 +7,14 @@ from functools import cached_property from typing import Any -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import pandas as pd -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) from pydantic import BaseModel, Field +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_score import EvaluationScore +from . import multi_modal_figures as figures + class Component(ABC, BaseModel): name: str = Field( diff --git a/src/nemo_safe_synthesizer/evaluation/components/composite_score.py b/src/nemo_safe_synthesizer/evaluation/components/composite_score.py index 5b3573aea..bdef586bf 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/composite_score.py +++ b/src/nemo_safe_synthesizer/evaluation/components/composite_score.py @@ -5,12 +5,9 @@ from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, - PrivacyGrade, -) +from ..components.component import Component +from ..data_model.evaluation_score import EvaluationScore, PrivacyGrade +from . import multi_modal_figures as figures class CompositeScore(Component): diff --git a/src/nemo_safe_synthesizer/evaluation/components/correlation.py b/src/nemo_safe_synthesizer/evaluation/components/correlation.py index f43db28f4..ada3b9cd8 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/correlation.py +++ b/src/nemo_safe_synthesizer/evaluation/components/correlation.py @@ -5,25 +5,20 @@ from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import pandas as pd -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.constants import JOB_COUNT -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_field import ( - EvaluationField, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.evaluation.statistics import stats -from nemo_safe_synthesizer.observability import get_logger from pydantic import ConfigDict, Field +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.constants import JOB_COUNT +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_field import EvaluationField +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...evaluation.statistics import stats +from ...observability import get_logger +from . import multi_modal_figures as figures + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/data_privacy_score.py b/src/nemo_safe_synthesizer/evaluation/components/data_privacy_score.py index 7e1e09830..694dcb8d8 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/data_privacy_score.py +++ b/src/nemo_safe_synthesizer/evaluation/components/data_privacy_score.py @@ -3,12 +3,16 @@ from __future__ import annotations -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.components.composite_score import CompositeScore -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import EvaluationScore, PrivacyGrade -from nemo_safe_synthesizer.observability import get_logger from pydantic import Field +from ...observability import get_logger +from ..data_model.evaluation_score import ( + EvaluationScore, + PrivacyGrade, +) +from .component import Component +from .composite_score import CompositeScore + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/dataset_statistics.py b/src/nemo_safe_synthesizer/evaluation/components/dataset_statistics.py index c9546d537..0cc955a8c 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/dataset_statistics.py +++ b/src/nemo_safe_synthesizer/evaluation/components/dataset_statistics.py @@ -5,17 +5,14 @@ from functools import cached_property -import nemo_safe_synthesizer.evaluation.statistics.stats as stats -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) from pydantic import Field +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...evaluation.statistics import stats + class DatasetStatistics(Component): name: str = Field(default="Dataset Statistics") diff --git a/src/nemo_safe_synthesizer/evaluation/components/deep_structure.py b/src/nemo_safe_synthesizer/evaluation/components/deep_structure.py index ea1d73819..918c16aed 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/deep_structure.py +++ b/src/nemo_safe_synthesizer/evaluation/components/deep_structure.py @@ -5,28 +5,23 @@ from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import pandas as pd from category_encoders.count import CountEncoder -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( +from pydantic import ConfigDict, Field + +from ...artifacts.analyzers.field_features import ( FieldType, describe_field, ) -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_field import ( - EvaluationField, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.evaluation.statistics import stats -from nemo_safe_synthesizer.observability import get_logger -from pydantic import ConfigDict, Field +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_field import EvaluationField +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...evaluation.statistics import stats +from ...observability import get_logger +from . import multi_modal_figures as figures logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/membership_inference_protection.py b/src/nemo_safe_synthesizer/evaluation/components/membership_inference_protection.py index 619b6d866..513a1a6f3 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/membership_inference_protection.py +++ b/src/nemo_safe_synthesizer/evaluation/components/membership_inference_protection.py @@ -7,29 +7,23 @@ from statistics import mean import category_encoders as ce -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import numpy.typing as npt import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - describe_field, -) -from nemo_safe_synthesizer.config.evaluate import DEFAULT_RECORD_COUNT -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, - PrivacyGrade, -) -from nemo_safe_synthesizer.observability import get_logger from pydantic import ConfigDict, Field from sentence_transformers import SentenceTransformer, util from sklearn.metrics import accuracy_score, precision_score from sklearn.preprocessing import QuantileTransformer +from ...artifacts.analyzers.field_features import describe_field +from ...config.evaluate import DEFAULT_RECORD_COUNT +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_score import EvaluationScore, PrivacyGrade +from ...observability import get_logger +from . import multi_modal_figures as figures + faiss_available = False try: import faiss diff --git a/src/nemo_safe_synthesizer/evaluation/components/multi_modal_figures.py b/src/nemo_safe_synthesizer/evaluation/components/multi_modal_figures.py index c2fac833d..c5d4fc16c 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/multi_modal_figures.py +++ b/src/nemo_safe_synthesizer/evaluation/components/multi_modal_figures.py @@ -7,13 +7,14 @@ import numpy as np import pandas as pd import plotly.graph_objects as go -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( +from numpy import cos, pi, sin +from plotly.subplots import make_subplots + +from ...evaluation.data_model.evaluation_score import ( EvaluationScore, PrivacyGrade, ) -from nemo_safe_synthesizer.evaluation.statistics.stats import get_numeric_distribution_bins -from numpy import cos, pi, sin -from plotly.subplots import make_subplots +from ...evaluation.statistics.stats import get_numeric_distribution_bins _REPORT_PALETTE = ["#3C2ED1", "#1AA2E6"] _GRAPH_BARGAP = 0.2 # gap between bars of adjacent location coordinates diff --git a/src/nemo_safe_synthesizer/evaluation/components/pii_replay.py b/src/nemo_safe_synthesizer/evaluation/components/pii_replay.py index daa953d37..972b0b6ff 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/pii_replay.py +++ b/src/nemo_safe_synthesizer/evaluation/components/pii_replay.py @@ -6,14 +6,13 @@ import math from functools import cached_property -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.observability import get_logger from pydantic import BaseModel, Field +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...observability import get_logger + logger = get_logger(__name__) UNKNOWN_ENTITY: str = "none" diff --git a/src/nemo_safe_synthesizer/evaluation/components/sqs_score.py b/src/nemo_safe_synthesizer/evaluation/components/sqs_score.py index 7ac0ee619..30b65a5ca 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/sqs_score.py +++ b/src/nemo_safe_synthesizer/evaluation/components/sqs_score.py @@ -3,17 +3,16 @@ from __future__ import annotations -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( +from pydantic import Field + +from ...artifacts.analyzers.field_features import ( FieldType, ) -from nemo_safe_synthesizer.evaluation.components.column_distribution import ColumnDistribution -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.components.composite_score import CompositeScore -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.observability import get_logger -from pydantic import Field +from ...evaluation.components.column_distribution import ColumnDistribution +from ...evaluation.components.component import Component +from ...evaluation.components.composite_score import CompositeScore +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/text_semantic_similarity.py b/src/nemo_safe_synthesizer/evaluation/components/text_semantic_similarity.py index 34bbbffd9..3e357ba4a 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/text_semantic_similarity.py +++ b/src/nemo_safe_synthesizer/evaluation/components/text_semantic_similarity.py @@ -6,28 +6,9 @@ import logging from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import numpy.typing as npt import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - FieldType, -) -from nemo_safe_synthesizer.config.evaluate import DEFAULT_RECORD_COUNT -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.constants import ( - MIN_RECORDS_FOR_TEXT_AND_PRIVACY_METRICS, - MIN_RECORDS_FOR_TEXT_METRICS_WITHOUT_WARNING, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.evaluation.statistics import stats -from nemo_safe_synthesizer.observability import get_logger from numpy.linalg import norm from pydantic import BaseModel, ConfigDict, Field from scipy.stats import ks_2samp @@ -40,6 +21,20 @@ wait_exponential, ) +from ...artifacts.analyzers.field_features import FieldType +from ...config.evaluate import DEFAULT_RECORD_COUNT +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.constants import ( + MIN_RECORDS_FOR_TEXT_AND_PRIVACY_METRICS, + MIN_RECORDS_FOR_TEXT_METRICS_WITHOUT_WARNING, +) +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...evaluation.statistics import stats +from ...observability import get_logger +from . import multi_modal_figures as figures + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/components/text_structure_similarity.py b/src/nemo_safe_synthesizer/evaluation/components/text_structure_similarity.py index 14114371c..f317a8895 100644 --- a/src/nemo_safe_synthesizer/evaluation/components/text_structure_similarity.py +++ b/src/nemo_safe_synthesizer/evaluation/components/text_structure_similarity.py @@ -6,26 +6,19 @@ import re from functools import cached_property -import nemo_safe_synthesizer.evaluation.components.multi_modal_figures as figures import numpy as np import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - FieldType, -) -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_field import ( - EvaluationField, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( - EvaluationScore, -) -from nemo_safe_synthesizer.observability import get_logger from pydantic import BaseModel, ConfigDict, Field +from ...artifacts.analyzers.field_features import FieldType +from ...config.parameters import SafeSynthesizerParameters +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import EvaluationDataset +from ...evaluation.data_model.evaluation_field import EvaluationField +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...observability import get_logger +from . import multi_modal_figures as figures + logger = get_logger(__name__) _SENTENCE_REGEX = re.compile(r"[^\.。︀?？؟⸮!！…:।෴።။]+") diff --git a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_dataset.py b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_dataset.py index 30de14740..da72897d7 100644 --- a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_dataset.py +++ b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_dataset.py @@ -5,22 +5,16 @@ import random -import nemo_safe_synthesizer.evaluation.statistics.stats as stats import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( - FieldType, -) -from nemo_safe_synthesizer.config.evaluate import ( - DEFAULT_RECORD_COUNT, - DEFAULT_SQS_REPORT_COLUMNS, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_field import ( - EvaluationField, -) -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics from pydantic import BaseModel, ConfigDict, Field, model_validator +from ...artifacts.analyzers.field_features import FieldType +from ...config.evaluate import DEFAULT_RECORD_COUNT, DEFAULT_SQS_REPORT_COLUMNS +from ...evaluation.data_model.evaluation_field import EvaluationField +from ...evaluation.statistics import stats +from ...observability import get_logger +from ...pii_replacer.transform_result import ColumnStatistics + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_field.py b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_field.py index 7738a4e2f..73beac499 100644 --- a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_field.py +++ b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_field.py @@ -8,17 +8,18 @@ import numpy as np import pandas as pd -from nemo_safe_synthesizer.artifacts.analyzers.field_features import ( +from pandas.api.types import is_integer_dtype +from pydantic import BaseModel, Field + +from ...artifacts.analyzers.field_features import ( FieldFeatures, FieldType, describe_field, ) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import EvaluationScore -from nemo_safe_synthesizer.evaluation.statistics import stats -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics -from pandas.api.types import is_integer_dtype -from pydantic import BaseModel, Field +from ...evaluation.data_model.evaluation_score import EvaluationScore +from ...evaluation.statistics import stats +from ...observability import get_logger +from ...pii_replacer.transform_result import ColumnStatistics logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_report.py b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_report.py index 1e9c7a2b8..9521b67ba 100644 --- a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_report.py +++ b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_report.py @@ -6,12 +6,13 @@ import re from functools import cached_property -from nemo_safe_synthesizer.evaluation.components.component import Component -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( +from pydantic import BaseModel, Field + +from ...evaluation.components.component import Component +from ...evaluation.data_model.evaluation_dataset import ( EvaluationDataset, ) -from nemo_safe_synthesizer.observability import get_logger -from pydantic import BaseModel, Field +from ...observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_score.py b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_score.py index 91b43e9aa..d39d1838d 100644 --- a/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_score.py +++ b/src/nemo_safe_synthesizer/evaluation/data_model/evaluation_score.py @@ -5,11 +5,10 @@ from enum import Enum -from nemo_safe_synthesizer.observability import get_logger - -# from types import TracebackType from pydantic import BaseModel, Field +from ...observability import get_logger + logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/evaluator.py b/src/nemo_safe_synthesizer/evaluation/evaluator.py index f391fb71c..797ac45c8 100644 --- a/src/nemo_safe_synthesizer/evaluation/evaluator.py +++ b/src/nemo_safe_synthesizer/evaluation/evaluator.py @@ -6,15 +6,13 @@ import pandas as pd -from nemo_safe_synthesizer.cli.artifact_structure import Workdir -from nemo_safe_synthesizer.config import SafeSynthesizerResults, SafeSynthesizerSummary, SafeSynthesizerTiming -from nemo_safe_synthesizer.config.parameters import ( - SafeSynthesizerParameters, -) -from nemo_safe_synthesizer.evaluation.reports.multimodal.multimodal_report import MultimodalReport -from nemo_safe_synthesizer.generation.results import GenerateJobResults -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics +from ..cli.artifact_structure import Workdir +from ..config import SafeSynthesizerResults, SafeSynthesizerSummary, SafeSynthesizerTiming +from ..config.parameters import SafeSynthesizerParameters +from ..generation.results import GenerateJobResults +from ..observability import get_logger +from ..pii_replacer.transform_result import ColumnStatistics +from .reports.multimodal.multimodal_report import MultimodalReport logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/render.py b/src/nemo_safe_synthesizer/evaluation/render.py index dc57865cd..610c32760 100644 --- a/src/nemo_safe_synthesizer/evaluation/render.py +++ b/src/nemo_safe_synthesizer/evaluation/render.py @@ -9,13 +9,11 @@ from jinja2 import Environment, FunctionLoader, select_autoescape -from nemo_safe_synthesizer.evaluation.data_model.evaluation_report import ( - EvaluationReport, -) -from nemo_safe_synthesizer.observability import get_logger +from ..observability import get_logger +from .data_model.evaluation_report import EvaluationReport if TYPE_CHECKING: - from nemo_safe_synthesizer.cli.artifact_structure import Workdir + from ..cli.artifact_structure import Workdir logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py b/src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py index 7bf60c79b..7a4cf6326 100644 --- a/src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py +++ b/src/nemo_safe_synthesizer/evaluation/reports/multimodal/multimodal_report.py @@ -8,55 +8,44 @@ from typing import Any import pandas as pd -from nemo_safe_synthesizer.config.evaluate import ( +from pydantic import Field + +from ....config.evaluate import ( DEFAULT_RECORD_COUNT, DEFAULT_SQS_REPORT_COLUMNS, ) -from nemo_safe_synthesizer.config.parameters import SafeSynthesizerParameters -from nemo_safe_synthesizer.evaluation.assets.text.multi_modal_tooltips import tooltips -from nemo_safe_synthesizer.evaluation.components.attribute_inference_protection import ( - AttributeInferenceProtection, -) -from nemo_safe_synthesizer.evaluation.components.column_distribution import ( +from ....config.parameters import SafeSynthesizerParameters +from ....evaluation.assets.text.multi_modal_tooltips import tooltips +from ....evaluation.components.attribute_inference_protection import AttributeInferenceProtection +from ....evaluation.components.column_distribution import ( ColumnDistribution, ColumnDistributionPlotRow, ) -from nemo_safe_synthesizer.evaluation.components.correlation import ( +from ....evaluation.components.correlation import ( Correlation, ) -from nemo_safe_synthesizer.evaluation.components.data_privacy_score import DataPrivacyScore -from nemo_safe_synthesizer.evaluation.components.dataset_statistics import ( +from ....evaluation.components.data_privacy_score import DataPrivacyScore +from ....evaluation.components.dataset_statistics import ( DatasetStatistics, ) -from nemo_safe_synthesizer.evaluation.components.deep_structure import ( +from ....evaluation.components.deep_structure import ( DeepStructure, ) -from nemo_safe_synthesizer.evaluation.components.membership_inference_protection import ( +from ....evaluation.components.membership_inference_protection import ( MembershipInferenceProtection, ) -from nemo_safe_synthesizer.evaluation.components.pii_replay import PIIReplay -from nemo_safe_synthesizer.evaluation.components.sqs_score import ( - SQSScore, -) -from nemo_safe_synthesizer.evaluation.components.text_semantic_similarity import ( - TextSemanticSimilarity, -) -from nemo_safe_synthesizer.evaluation.components.text_structure_similarity import ( - TextStructureSimilarity, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_dataset import ( - EvaluationDataset, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_report import ( - EvaluationReport, -) -from nemo_safe_synthesizer.evaluation.data_model.evaluation_score import ( +from ....evaluation.components.pii_replay import PIIReplay +from ....evaluation.components.sqs_score import SQSScore +from ....evaluation.components.text_semantic_similarity import TextSemanticSimilarity +from ....evaluation.components.text_structure_similarity import TextStructureSimilarity +from ....evaluation.data_model.evaluation_dataset import EvaluationDataset +from ....evaluation.data_model.evaluation_report import EvaluationReport +from ....evaluation.data_model.evaluation_score import ( EvaluationScore, PrivacyGrade, ) -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics -from pydantic import Field +from ....observability import get_logger +from ....pii_replacer.transform_result import ColumnStatistics logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/data_editor/__init__.py b/src/nemo_safe_synthesizer/pii_replacer/data_editor/__init__.py index 2becae977..513687107 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/data_editor/__init__.py +++ b/src/nemo_safe_synthesizer/pii_replacer/data_editor/__init__.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.data_editor.detect import * # noqa: F403 -from nemo_safe_synthesizer.pii_replacer.data_editor.edit import * # noqa: F403 -from nemo_safe_synthesizer.pii_replacer.data_editor.environment import * # noqa: F403 -from nemo_safe_synthesizer.pii_replacer.data_editor.filters import * # noqa: F403 -from nemo_safe_synthesizer.pii_replacer.data_editor.transform_test_utils import * # noqa: F403 +from ..data_editor.detect import * # noqa: F403 +from ..data_editor.edit import * # noqa: F403 +from ..data_editor.environment import * # noqa: F403 +from ..data_editor.filters import * # noqa: F403 +from ..data_editor.transform_test_utils import * # noqa: F403 diff --git a/src/nemo_safe_synthesizer/pii_replacer/data_editor/detect.py b/src/nemo_safe_synthesizer/pii_replacer/data_editor/detect.py index acbb9b28c..8379614d5 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/data_editor/detect.py +++ b/src/nemo_safe_synthesizer/pii_replacer/data_editor/detect.py @@ -20,11 +20,11 @@ from openai import OpenAI from pydantic import ConfigDict, TypeAdapter, ValidationError -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner import ner_mp -from nemo_safe_synthesizer.pii_replacer.ner.factory import LabelSetPredictorFilter, NERFactory -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction -from nemo_safe_synthesizer.pii_replacer.ner.pipeline import Pipeline +from ...observability import get_logger +from ..ner import ner_mp +from ..ner.factory import LabelSetPredictorFilter, NERFactory +from ..ner.ner import NERPrediction +from ..ner.pipeline import Pipeline logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/data_editor/edit.py b/src/nemo_safe_synthesizer/pii_replacer/data_editor/edit.py index 288d6bb85..422dfeca5 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/data_editor/edit.py +++ b/src/nemo_safe_synthesizer/pii_replacer/data_editor/edit.py @@ -17,9 +17,9 @@ from jinja2 import Template from jinja2.exceptions import TemplateError, TemplateSyntaxError -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.data_editor.detect import DEFAULT_ENTITIES, EntityExtractor -from nemo_safe_synthesizer.pii_replacer.data_editor.environment import ( +from ...observability import get_logger +from .detect import DEFAULT_ENTITIES, EntityExtractor +from .environment import ( Environment, SafeSynthesizerFakerMethodNotFound, ) diff --git a/src/nemo_safe_synthesizer/pii_replacer/data_editor/transform_test_utils.py b/src/nemo_safe_synthesizer/pii_replacer/data_editor/transform_test_utils.py index ca03d9e1c..78be6f66b 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/data_editor/transform_test_utils.py +++ b/src/nemo_safe_synthesizer/pii_replacer/data_editor/transform_test_utils.py @@ -8,13 +8,13 @@ import pandas as pd -from nemo_safe_synthesizer.pii_replacer.data_editor.detect import ( +from ..ner.ner import NERPrediction +from .detect import ( UNKNOWN_ENTITY, ColumnClassifier, EntityExtractorGliner, IAPIClassifierConfig, ) -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction class EntityExtractorMock(EntityExtractorGliner): diff --git a/src/nemo_safe_synthesizer/pii_replacer/nemo_pii.py b/src/nemo_safe_synthesizer/pii_replacer/nemo_pii.py index 49e4b8b2d..9b9416358 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/nemo_pii.py +++ b/src/nemo_safe_synthesizer/pii_replacer/nemo_pii.py @@ -8,10 +8,12 @@ import pandas as pd from openai import OpenAI -from nemo_safe_synthesizer.artifacts.analyzers.field_features import describe_field -from nemo_safe_synthesizer.artifacts.base.fields import FieldType -from nemo_safe_synthesizer.config.replace_pii import PiiReplacerConfig -from nemo_safe_synthesizer.pii_replacer.data_editor.detect import ( +from ..artifacts.analyzers.field_features import describe_field +from ..artifacts.base.fields import FieldType +from ..config.replace_pii import PiiReplacerConfig +from ..pii_replacer.data_editor.edit import Editor, TransformFnAccounting +from ..pii_replacer.transform_result import ColumnStatistics, TransformResult +from .data_editor.detect import ( DEFAULT_ENTITIES, UNKNOWN_ENTITY, ClassifyConfig, @@ -22,8 +24,6 @@ EntityExtractorRegexp, NerReport, ) -from nemo_safe_synthesizer.pii_replacer.data_editor.edit import Editor, TransformFnAccounting -from nemo_safe_synthesizer.pii_replacer.transform_result import ColumnStatistics, TransformResult def classify_config_from_params( @@ -197,27 +197,49 @@ def classify_df(self, df: pd.DataFrame) -> dict[str, dict[str, str]]: type as values. The "entities" key maps to a dictionary with column names as keys and entity names as values. """ + # Pre-initialize with defaults + entities = {} + columns = {item: None for item in df.columns} + try: - column_classifier = get_column_classifier() - columns = column_classifier.detect_types(df, self.classify_config.valid_entities) - - entities = { - name: (entity if entity != UNKNOWN_ENTITY and entity in self.classify_config.valid_entities else None) - for (name, entity) in columns.items() - } - except Exception: - logging.error("Could not perform classify, falling back to default entities.", exc_info=False) - entities = {} - columns = {item: None for item in df.columns} - - # Use field type detection to identify text columns if not already - # assigned an entity. These text columns are where NER is used if - # enabled during transform_df. - fields = [describe_field(field_name, df[field_name]) for field_name in df.columns] - for field in fields: - existing_type = columns.get(field.name, None) - if (existing_type is None or existing_type.lower() == "none") and field.type == FieldType.TEXT: - columns[field.name] = "text" + # Only attempt classification if enabled + if self.pii_replacer_config.globals.classify.enable_classify is not False: + column_classifier = None + + # Try to initialize the column classifier + try: + column_classifier = get_column_classifier() + except Exception: + logging.error( + "Could not initialize column classifier, falling back to default entities.", exc_info=False + ) + + # Try to perform classification if we successfully got a classifier + if column_classifier is not None: + try: + columns = column_classifier.detect_types(df, self.classify_config.valid_entities) + + entities = { + name: ( + entity + if entity != UNKNOWN_ENTITY and entity in self.classify_config.valid_entities + else None + ) + for (name, entity) in columns.items() + } + except Exception: + logging.error("Could not perform classify, falling back to default entities.", exc_info=False) + else: + logging.info("Column classification is disabled (enable_classify=False), skipping classify call.") + finally: + # Use field type detection to identify text columns if not already + # assigned an entity. These text columns are where NER is used if + # enabled during transform_df. + fields = [describe_field(field_name, df[field_name]) for field_name in df.columns] + for field in fields: + existing_type = columns.get(field.name, None) + if (existing_type is None or existing_type.lower() == "none") and field.type == FieldType.TEXT: + columns[field.name] = "text" return { "columns": columns, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/__init__.py b/src/nemo_safe_synthesizer/pii_replacer/ner/__init__.py index fb362ffba..af1896564 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/__init__.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/__init__.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.ner import NER, PipelineResult, Timings +from ..ner.ner import NER, PipelineResult, Timings __all__ = ["NER", "Timings", "PipelineResult"] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/custom.py b/src/nemo_safe_synthesizer/pii_replacer/ner/custom.py index 3f33e1d46..74d6992bd 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/custom.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/custom.py @@ -34,11 +34,11 @@ import yaml -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner.entity import Score -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern as RegexPattern -from nemo_safe_synthesizer.pii_replacer.ner.regex import PhraseMatcherBuilder, RegexPredictor +from ...observability import get_logger +from .entity import Score +from .predictor import Predictor +from .regex import Pattern as RegexPattern +from .regex import PhraseMatcherBuilder, RegexPredictor logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/datetime.py b/src/nemo_safe_synthesizer/pii_replacer/ner/datetime.py index 77cdf564e..01b1ca3dc 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/datetime.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/datetime.py @@ -11,14 +11,14 @@ from dateparser.date import get_date_from_timestamp from dateparser.search import search_dates -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.pii_replacer.ner.entity import ( +from ...data_processing.records.json_record import JSONRecord +from .entity import ( Entity, Score, ) -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor, PredictorContext -from nemo_safe_synthesizer.pii_replacer.ner.regex import ( +from .ner import NERPrediction +from .predictor import Predictor, PredictorContext +from .regex import ( create_exact_field_matcher, split_header_contexts, ) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/factory.py b/src/nemo_safe_synthesizer/pii_replacer/ner/factory.py index 93a5f3ba2..6e786765f 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/factory.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/factory.py @@ -10,15 +10,15 @@ from enum import StrEnum from typing import List, Optional, Set, Union -from nemo_safe_synthesizer.data_processing.records.base import normalize_labels -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner.metadata import FieldLabelCondition -from nemo_safe_synthesizer.pii_replacer.ner.ner import NER -from nemo_safe_synthesizer.pii_replacer.ner.ner_mp import NERParallel -from nemo_safe_synthesizer.pii_replacer.ner.nlp import SpacyPredictor -from nemo_safe_synthesizer.pii_replacer.ner.pipeline import Pipeline, regex_pipeline -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor -from nemo_safe_synthesizer.pii_replacer.ner.regex import RegexPredictor +from ...data_processing.records.base import normalize_labels +from ...observability import get_logger +from .metadata import FieldLabelCondition +from .ner import NER +from .ner_mp import NERParallel +from .nlp import SpacyPredictor +from .pipeline import Pipeline, regex_pipeline +from .predictor import Predictor +from .regex import RegexPredictor logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/fasttext.py b/src/nemo_safe_synthesizer/pii_replacer/ner/fasttext.py index 85104a654..337607cc2 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/fasttext.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/fasttext.py @@ -5,7 +5,7 @@ from typing import Union -from nemo_safe_synthesizer.pii_replacer.ner.models import ( +from .models import ( ModelManifest, ObjectRef, Visibility, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/helpers.py b/src/nemo_safe_synthesizer/pii_replacer/ner/helpers.py index 42c1b4a67..0f23b2fb5 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/helpers.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/helpers.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity +from .entity import Entity spacy = None diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/labels.py b/src/nemo_safe_synthesizer/pii_replacer/ner/labels.py index 3f8cc1b96..bba54be37 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/labels.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/labels.py @@ -6,9 +6,9 @@ import re from typing import Iterator, List, Pattern, Set -from nemo_safe_synthesizer.data_processing.records.base import normalize_label, normalize_labels -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity +from ...data_processing.records.base import normalize_label, normalize_labels +from ...observability import get_logger +from .entity import Entity logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/metadata.py b/src/nemo_safe_synthesizer/pii_replacer/ner/metadata.py index eb020808f..1db4e618c 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/metadata.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/metadata.py @@ -9,9 +9,9 @@ from math import ceil from typing import List, Optional, Set, Union -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.pii_replacer.ner.ner import NER, PipelineResult -from nemo_safe_synthesizer.pii_replacer.ner.ner_mp import NERParallel +from ...data_processing.records.json_record import JSONRecord +from .ner import NER, PipelineResult +from .ner_mp import NERParallel class FieldAttribute(StrEnum): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/model.py b/src/nemo_safe_synthesizer/pii_replacer/ner/model.py index c927fb93d..1657caf1a 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/model.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/model.py @@ -7,11 +7,11 @@ import re from typing import Pattern, Tuple -import nemo_safe_synthesizer.pii_replacer.ner.ner as ner -import nemo_safe_synthesizer.pii_replacer.ner.pipeline as pipeline -import nemo_safe_synthesizer.pii_replacer.ner.regex as regex -from nemo_safe_synthesizer.data_processing.records.fragment import create_ner_api_response -from nemo_safe_synthesizer.pii_replacer.ner.entity import Score +from ...data_processing.records.fragment import create_ner_api_response +from ...pii_replacer.ner.entity import Score +from ...pii_replacer.ner.ner import ner +from ...pii_replacer.ner.pipeline import pipeline +from ...pii_replacer.ner.regex import regex InputData = str | dict | list[str] | list[dict] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/models.py b/src/nemo_safe_synthesizer/pii_replacer/ner/models.py index 2539a4954..88f3e64e4 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/models.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/models.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional -from nemo_safe_synthesizer.observability import get_logger +from ...observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/ner.py b/src/nemo_safe_synthesizer/pii_replacer/ner/ner.py index bf9844a54..77c8ac776 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/ner.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/ner.py @@ -14,16 +14,16 @@ from dataclasses import field as dataclasses_field from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Union -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.data_processing.records.value_path import ( +from ...data_processing.records.json_record import JSONRecord +from ...data_processing.records.value_path import ( ValuePath, value_path_to_json_path, ) -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner.utils import InData, input_to_json_records +from ...observability import get_logger +from .utils import InData, input_to_json_records if TYPE_CHECKING: - from nemo_safe_synthesizer.pii_replacer.ner.pipeline import Pipeline + from .pipeline import Pipeline else: Pipeline = None diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/ner_mp.py b/src/nemo_safe_synthesizer/pii_replacer/ner/ner_mp.py index 986116cc4..cf902b8d6 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/ner_mp.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/ner_mp.py @@ -11,11 +11,11 @@ import joblib.externals.loky as loky -import nemo_safe_synthesizer.pii_replacer.ner.pipeline as pipeline -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner import NER, PipelineResult, Timings -from nemo_safe_synthesizer.pii_replacer.ner.utils import InData +from ...data_processing.records.json_record import JSONRecord +from ...observability import get_logger +from . import pipeline +from .ner import NER, PipelineResult, Timings +from .utils import InData logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/nlp.py b/src/nemo_safe_synthesizer/pii_replacer/ner/nlp.py index 77a7ccfb8..7fc634492 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/nlp.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/nlp.py @@ -9,21 +9,21 @@ from time import perf_counter from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union -from nemo_safe_synthesizer.data_processing.records.base import KVPair -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.data_processing.records.value_path import ValuePath -from nemo_safe_synthesizer.pii_replacer.ner.const import const -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.fasttext import FTEntityMatcher -from nemo_safe_synthesizer.pii_replacer.ner.models import ( +from ...data_processing.records.base import KVPair +from ...data_processing.records.json_record import JSONRecord +from ...data_processing.records.value_path import ValuePath +from .const import const +from .entity import Entity, Score +from .fasttext import FTEntityMatcher +from .models import ( ModelManifest, ObjectRef, Visibility, get_cache_manager, ) -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor -from nemo_safe_synthesizer.pii_replacer.ner.utils import is_string_a_number +from .ner import NERPrediction +from .predictor import Predictor +from .utils import is_string_a_number spacy = None Doc = None diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/person_name.py b/src/nemo_safe_synthesizer/pii_replacer/ner/person_name.py index 69a3ca5b6..76884d82b 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/person_name.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/person_name.py @@ -17,17 +17,17 @@ except ImportError: from typing import Pattern -from nemo_safe_synthesizer.data_processing.records.base import KVPair, tokenize_header -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.models import ( +from ...data_processing.records.base import KVPair, tokenize_header +from ...data_processing.records.json_record import JSONRecord +from .entity import Entity, Score +from .models import ( ModelManifest, ObjectRef, Visibility, get_cache_manager, ) -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor +from .ner import NERPrediction +from .predictor import Predictor DEFAULT_MANIFEST = ModelManifest( model="person_name_wordlist", diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/pipeline.py b/src/nemo_safe_synthesizer/pii_replacer/ner/pipeline.py index ad39413c1..25af43cd6 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/pipeline.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/pipeline.py @@ -8,14 +8,14 @@ from pathlib import Path from typing import Dict, Iterator, List, Sequence, Type, Union -import nemo_safe_synthesizer.pii_replacer.ner.person_name as person_name -from nemo_safe_synthesizer.observability import get_logger -from nemo_safe_synthesizer.pii_replacer.ner.custom import get_predictors_from_yaml -from nemo_safe_synthesizer.pii_replacer.ner.datetime import BirthDateTime, DateTime -from nemo_safe_synthesizer.pii_replacer.ner.ner import NER -from nemo_safe_synthesizer.pii_replacer.ner.predictor import Predictor -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor -from nemo_safe_synthesizer.pii_replacer.ner.regexes import rules +from ...observability import get_logger +from . import person_name +from .custom import get_predictors_from_yaml +from .datetime import BirthDateTime, DateTime +from .ner import NER +from .predictor import Predictor +from .regex import Pattern, RegexPredictor +from .regexes import rules logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/predictor.py b/src/nemo_safe_synthesizer/pii_replacer/ner/predictor.py index 38c773401..e009fc5e6 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/predictor.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/predictor.py @@ -6,9 +6,9 @@ from re import Pattern from typing import List, Optional, Tuple, Union -from nemo_safe_synthesizer.data_processing.records.base import KVPair -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERPrediction +from ...data_processing.records.base import KVPair +from ...data_processing.records.json_record import JSONRecord +from .ner import NERPrediction DEFAULT_CONTEXT_SPAN_SIZE = 16 diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regex.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regex.py index 8a87e43db..87beb8949 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regex.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regex.py @@ -17,11 +17,11 @@ except ImportError: from typing import Pattern as RePattern -from nemo_safe_synthesizer.data_processing.records.base import KVPair -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.ner import NERError, NERPrediction -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan, Predictor, is_context_matched +from ...data_processing.records.base import KVPair +from ...data_processing.records.json_record import JSONRecord +from .entity import Entity, Score +from .ner import NERError, NERPrediction +from .predictor import ContextSpan, Predictor, is_context_matched def split_header_contexts( diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/__init__.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/__init__.py index 54fe2e219..7f5791104 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/__init__.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/__init__.py @@ -3,7 +3,7 @@ from typing import Sequence, Type -from nemo_safe_synthesizer.pii_replacer.ner.regex import RegexPredictor +from ..regex import RegexPredictor # TODO: are relative imports acceptable in nmp repo? from .aba_routing_number import AbaRoutingNumber diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/aba_routing_number.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/aba_routing_number.py index da2d0144e..4c1395dc2 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/aba_routing_number.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/aba_routing_number.py @@ -3,9 +3,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor ABA_ROUTING_NUMBER_REGEX_1 = r"\b[0,1,2,3,6,7,8]\d{3}-\d{4}-\d\b" ABA_ROUTING_NUMBER_REGEX_2 = r"\b[0,1,2,3,6,7,8]\d{8}\b" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/age.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/age.py index 7d4fcc31d..272ad6db4 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/age.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/age.py @@ -7,8 +7,8 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..regex import Pattern, RegexPredictor # https://github.com/Gretellabs/monogretel/issues/190 HEADERS = ["age", "ages"] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/credit_card.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/credit_card.py index b80e14af0..8073449c8 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/credit_card.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/credit_card.py @@ -5,9 +5,9 @@ from stdnum import luhn -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor CC_REGEX = ( r"^(?:4[0-9]{12}(?:[0-9]{3})?|[25][1-7][0-9]{14}|6(?:011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(?:0[0-5]" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/domain_name.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/domain_name.py index dfc92b3c6..7695b0eff 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/domain_name.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/domain_name.py @@ -5,9 +5,9 @@ import tldextract -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import ( +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import ( Pattern, RegexPredictor, create_exact_field_matcher, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/email.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/email.py index 8f116bca4..2638086f1 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/email.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/email.py @@ -3,8 +3,8 @@ import tldextract -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class Email(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/facebook.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/facebook.py index 5df0fde02..fbb23f582 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/facebook.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/facebook.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class Facebook(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/generic_key.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/generic_key.py index 79d5d8d3e..6910b93d1 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/generic_key.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/generic_key.py @@ -3,9 +3,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor REG = [ r"api.{0,3}(key|token|secret)", diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/github.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/github.py index 88583c690..8d9ba2934 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/github.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/github.py @@ -3,9 +3,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor GIT_TOKEN = re.compile(r"(?:^|\b|_)git(?:\b|_|$)") COMMIT_URL = r"https://github.com/.*?/.*?/commit/{}" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google.py index a2018287a..482c7c39c 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class Google(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google_olc.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google_olc.py index 9cd2be812..18bf8114e 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google_olc.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/google_olc.py @@ -1,8 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor CODE_DIGIT = "23456789CFGHJMPQRVWX" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/iban.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/iban.py index d49badc9a..fe49288c3 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/iban.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/iban.py @@ -20,8 +20,8 @@ except ImportError: import regex as re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor IBAN_GENERIC_REGEX = r"\b[A-Z]{2}[0-9]{2}[ ]?([a-zA-Z0-9][ ]?){11,28}\b" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/imei.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/imei.py index 711126919..2887ecad1 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/imei.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/imei.py @@ -4,9 +4,9 @@ from stdnum import imei -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor LABELS = ["imei", "hardware", "meid", "imeis", "meids"] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/ip_address.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/ip_address.py index 21e09cf94..bec614d20 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/ip_address.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/ip_address.py @@ -4,8 +4,8 @@ import ipaddress -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor NEG_HEADERS = ["ver", "version", "versions"] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/jwt.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/jwt.py index a7c28627c..aa07a859d 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/jwt.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/jwt.py @@ -5,8 +5,8 @@ import base64 import json -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class JWT(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/lat_lon.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/lat_lon.py index 42f437f53..d96f7eeeb 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/lat_lon.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/lat_lon.py @@ -4,8 +4,8 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..regex import Pattern, RegexPredictor LAT_RE = r"[-+]?(90(\.0+)?|([1-8]?\d)(\.\d+)?)" LAT_LABELS = [ diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/md5.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/md5.py index e281720ad..6b49f2144 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/md5.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/md5.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor MD5_REGEX = r"^(?=.*[a-fA-F])(?=.*[0-9])[a-fA-F0-9]{32}$" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/race_ethnicity.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/race_ethnicity.py index 7ecec3207..26e5b21d5 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/race_ethnicity.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/race_ethnicity.py @@ -6,8 +6,8 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import ( +from ..entity import Entity, Score +from ..regex import ( Pattern, RegexPredictor, create_exact_field_matcher, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sendgrid.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sendgrid.py index 43fb4af2a..6de4bc256 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sendgrid.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sendgrid.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class SendGrid(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sex_gender.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sex_gender.py index 65c7c4d48..2fcf3a936 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sex_gender.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sex_gender.py @@ -4,8 +4,8 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import ( +from ..entity import Entity, Score +from ..regex import ( Pattern, RegexPredictor, create_exact_field_matcher, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha256.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha256.py index b657fe1e2..2657aaba4 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha256.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha256.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor SHA256_REGEX = r"^(?=.*[a-zA-Z])(?=.*[0-9])[a-zA-Z0-9]{64}$" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha512.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha512.py index f604b723c..41a21ab30 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha512.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/sha512.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor SHA512_REGEX = r"^(?=.*[a-zA-Z])(?=.*[0-9])[a-zA-Z0-9]{128}$" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/slack.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/slack.py index 34b9b5dc3..618e509df 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/slack.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/slack.py @@ -8,8 +8,8 @@ We set both score values to 1.0 here because these are such precise matches. """ -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class SlackSecrets(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/square.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/square.py index 08acce974..e248e3465 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/square.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/square.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class SquareAPIKeys(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/stripe.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/stripe.py index cac839732..51de3003e 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/stripe.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/stripe.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class StripeAPIKey(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/swift.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/swift.py index 3b8d81e3e..0ff5bb647 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/swift.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/swift.py @@ -4,9 +4,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor -from nemo_safe_synthesizer.pii_replacer.ner.regexes.iban import regex_per_country +from ..entity import Entity +from ..regex import Pattern, RegexPredictor +from .iban import regex_per_country # NOTE that https://docs.microsoft.com/en-us/exchange/policy-and-compliance/data-loss-prevention/sensitive-information-types?view=exchserver-2019#swift-code # noqa # and https://en.wikipedia.org/wiki/ISO_9362 specify different specs. Using ISO_9362 spec. diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/twilio.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/twilio.py index de509c6a3..e738e1da8 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/twilio.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/twilio.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor class TwilioAPIKeys(RegexPredictor): diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/url.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/url.py index 764bf40d7..f71297da6 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/url.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/url.py @@ -4,9 +4,9 @@ import tldextract -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor URL_REGEX = r"https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,4}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)" # noqa diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_phone.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_phone.py index fa8f64935..8387f0849 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_phone.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_phone.py @@ -4,9 +4,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor US_PHONE_REGEX = r"((\+?1)|(001))?-?\(?[0-9]{3}\)?[-.*\s]?[0-9]{3}[-.*\s]?[0-9]{4}(x\d+)?" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_ssn.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_ssn.py index 5c0f55ba8..e9c622f15 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_ssn.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_ssn.py @@ -4,8 +4,8 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.regex import ( +from ..entity import Entity +from ..regex import ( Pattern, RegexPredictor, create_exact_field_matcher, diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_zipcode.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_zipcode.py index 8e01ac0d7..6da2d2bd9 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_zipcode.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/us_zipcode.py @@ -4,9 +4,9 @@ import re -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity -from nemo_safe_synthesizer.pii_replacer.ner.predictor import ContextSpan -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity +from ..predictor import ContextSpan +from ..regex import Pattern, RegexPredictor US_ZIPCODE_REGEX = r"\b[0-9]{5}(?:-[0-9]{4})?\b" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/uuid.py b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/uuid.py index 7deb9d505..813f1147c 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/uuid.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/regexes/uuid.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -from nemo_safe_synthesizer.pii_replacer.ner.entity import Entity, Score -from nemo_safe_synthesizer.pii_replacer.ner.regex import Pattern, RegexPredictor +from ..entity import Entity, Score +from ..regex import Pattern, RegexPredictor UUID_REGEX = r"^[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[1-5][0-9a-fA-F]{3}-?[89abAB][0-9a-fA-F]{3}-?[0-9a-fA-F]{12}$" diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/report/metadata.py b/src/nemo_safe_synthesizer/pii_replacer/ner/report/metadata.py index 6601e9a8b..355ed2b85 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/report/metadata.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/report/metadata.py @@ -7,12 +7,12 @@ from pydantic.v1 import Field -from nemo_safe_synthesizer.pii_replacer.ner.metadata import ( +from ..metadata import ( DatasetMetadata, EntityMetadata, FieldMetadata, ) -from nemo_safe_synthesizer.pii_replacer.ner.report.base import ReportBaseModel +from .base import ReportBaseModel def convert_to_report(metadata: DatasetMetadata) -> DatasetMetadataReport: diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/report/results.py b/src/nemo_safe_synthesizer/pii_replacer/ner/report/results.py index 921f4ce18..badbbbc02 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/report/results.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/report/results.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional -from nemo_safe_synthesizer.data_processing.records.value_path import value_path_to_json_path +from ....data_processing.records.value_path import value_path_to_json_path Prediction = Dict[str, Any] diff --git a/src/nemo_safe_synthesizer/pii_replacer/ner/utils.py b/src/nemo_safe_synthesizer/pii_replacer/ner/utils.py index e4b65ac30..277ca6827 100644 --- a/src/nemo_safe_synthesizer/pii_replacer/ner/utils.py +++ b/src/nemo_safe_synthesizer/pii_replacer/ner/utils.py @@ -3,7 +3,7 @@ from typing import List, Union -from nemo_safe_synthesizer.data_processing.records.json_record import JSONRecord +from ...data_processing.records.json_record import JSONRecord """Valid input data includes a str, tuple or dict""" InData = Union[str, list, dict, JSONRecord] diff --git a/src/nemo_safe_synthesizer/privacy/dp_transformers/dp_utils.py b/src/nemo_safe_synthesizer/privacy/dp_transformers/dp_utils.py index 222f06ed2..ce27005e9 100644 --- a/src/nemo_safe_synthesizer/privacy/dp_transformers/dp_utils.py +++ b/src/nemo_safe_synthesizer/privacy/dp_transformers/dp_utils.py @@ -32,19 +32,19 @@ ) from transformers.trainer import TRAINING_ARGS_NAME -import nemo_safe_synthesizer.privacy.dp_transformers.linear # imported for side effects # noqa -from nemo_safe_synthesizer.privacy.dp_transformers.privacy_args import ( +from . import linear # imported for side effects # noqa +from .privacy_args import ( PrivacyArguments, SafeSynthesizerAccountant, ) -from nemo_safe_synthesizer.privacy.dp_transformers.sampler import ( +from .sampler import ( PoissonEntitySampler, ShuffledEntitySampler, ) if utils.is_safetensors_available(): import safetensors.torch -from nemo_safe_synthesizer.observability import get_logger +from ...observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/privacy/dp_transformers/privacy_args.py b/src/nemo_safe_synthesizer/privacy/dp_transformers/privacy_args.py index 74928aa85..5d340d666 100644 --- a/src/nemo_safe_synthesizer/privacy/dp_transformers/privacy_args.py +++ b/src/nemo_safe_synthesizer/privacy/dp_transformers/privacy_args.py @@ -16,7 +16,7 @@ from prv_accountant import Accountant as PRVAccountant from scipy import optimize -from nemo_safe_synthesizer.observability import get_logger +from ...observability import get_logger logger = get_logger() diff --git a/src/nemo_safe_synthesizer/privacy/dp_transformers/sampler.py b/src/nemo_safe_synthesizer/privacy/dp_transformers/sampler.py index 8e937b18a..2aab61626 100644 --- a/src/nemo_safe_synthesizer/privacy/dp_transformers/sampler.py +++ b/src/nemo_safe_synthesizer/privacy/dp_transformers/sampler.py @@ -11,7 +11,7 @@ from opacus.utils.uniform_sampler import UniformWithReplacementSampler from torch.utils.data.sampler import BatchSampler, RandomSampler, Sampler -from nemo_safe_synthesizer.observability import get_logger +from ...observability import get_logger logger = get_logger(__name__) diff --git a/src/nemo_safe_synthesizer/results.py b/src/nemo_safe_synthesizer/results.py index 4357fc4b4..22e22c43c 100644 --- a/src/nemo_safe_synthesizer/results.py +++ b/src/nemo_safe_synthesizer/results.py @@ -3,10 +3,10 @@ import pandas as pd -from nemo_safe_synthesizer.config import SafeSynthesizerResults, SafeSynthesizerSummary, SafeSynthesizerTiming -from nemo_safe_synthesizer.evaluation.render import render_report -from nemo_safe_synthesizer.evaluation.reports.multimodal.multimodal_report import MultimodalReport -from nemo_safe_synthesizer.generation.results import GenerateJobResults +from .config import SafeSynthesizerResults, SafeSynthesizerSummary, SafeSynthesizerTiming +from .evaluation.render import render_report +from .evaluation.reports.multimodal.multimodal_report import MultimodalReport +from .generation.results import GenerateJobResults def make_nss_summary( diff --git a/src/nemo_safe_synthesizer/training/huggingface_backend.py b/src/nemo_safe_synthesizer/training/huggingface_backend.py index 5f2e54fd7..744e16af1 100644 --- a/src/nemo_safe_synthesizer/training/huggingface_backend.py +++ b/src/nemo_safe_synthesizer/training/huggingface_backend.py @@ -28,8 +28,6 @@ ) from transformers.trainer_pt_utils import get_model_param_count -from nemo_safe_synthesizer.utils import write_json - from .. import utils from ..config.autoconfig import AutoConfigResolver from ..data_processing.assembler import TrainingExampleAssembler @@ -63,6 +61,7 @@ ProgressBarCallback, SafeSynthesizerWorkerCallback, ) +from ..utils import write_json logger = get_logger(__name__) diff --git a/tests/pii_replacer/test_nemo_pii.py b/tests/pii_replacer/test_nemo_pii.py index 6d7685a8a..88d884afb 100644 --- a/tests/pii_replacer/test_nemo_pii.py +++ b/tests/pii_replacer/test_nemo_pii.py @@ -79,6 +79,48 @@ def test_nemo_pii_classify_df_no_column_classifier(_build_entity_extractor): assert result["entities"] == {} +@patch("nemo_safe_synthesizer.pii_replacer.nemo_pii.build_entity_extractor", return_value=MagicMock()) +def test_nemo_pii_classify_disabled(_build_entity_extractor): + """Test that when enable_classify is False, no external API calls are made.""" + df = pd.read_csv(Path(__file__).parent / "fake_people_dataset.csv") + + # Create a config with classify disabled + from nemo_safe_synthesizer.config.replace_pii import PiiReplacerConfig + + config = PiiReplacerConfig.get_default_config() + config.globals.classify.enable_classify = False + + mock_column_classifier = MagicMock() + mock_column_classifier.detect_types.return_value = { + "fname": "first_name", + "lname": "last_name", + } + + with patch( + "nemo_safe_synthesizer.pii_replacer.nemo_pii.get_column_classifier", return_value=mock_column_classifier + ) as mock_get_classifier: + nemo_pii = NemoPII(config=config) + result = nemo_pii.classify_df(df) + + # Verify the classifier was never instantiated (no API call made) + mock_get_classifier.assert_not_called() + + # Verify we still get a valid result with text columns detected via local field detection + expected = { + "columns": { + "fname": None, + "lname": None, + "email": None, + "full address": "text", + "height": None, + "date of birth": None, + "notes": "text", + }, + "entities": {}, + } + assert result == expected + + @patch("nemo_safe_synthesizer.pii_replacer.nemo_pii.build_entity_extractor", return_value=MagicMock()) def test_nemo_pii_default_config_national_id(_build_entity_extractor): # Added to confirm a typo fix works in the condition for national_id