Fixes #1232 reorganize document filter document modifier dir

.github/workflows/config/.secrets.baseline

@@ -180,13 +180,6 @@
         "is_verified": false,
         "line_number": 33
       },
-      {
-        "type": "Secret Keyword",
-        "filename": "docs/curate-text/synthetic/llm-client.md",
-        "hashed_secret": "2083c49ad8d63838a4d18f1de0c419f06eb464db",
-        "is_verified": false,
-        "line_number": 44
-      },
       {
         "type": "Secret Keyword",
         "filename": "docs/curate-text/synthetic/llm-client.md",
@@ -202,15 +195,6 @@
         "line_number": 165
       }
     ],
-    "docs/curate-text/synthetic/multilingual-qa.md": [
-      {
-        "type": "Secret Keyword",
-        "filename": "docs/curate-text/synthetic/multilingual-qa.md",
-        "hashed_secret": "2083c49ad8d63838a4d18f1de0c419f06eb464db",
-        "is_verified": false,
-        "line_number": 30
-      }
-    ],
     "tests/models/client/test_openai_client.py": [
       {
         "type": "Secret Keyword",
@@ -248,5 +232,5 @@
       }
     ]
   },
-  "generated_at": "2026-02-11T21:26:53Z"
+  "generated_at": "2026-02-26T00:35:18Z"
 }

benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py

@@ -49,17 +49,16 @@
 from nemo_curator.stages.text.download.base import URLGenerator
 from nemo_curator.stages.text.download.base.iterator import DocumentIterateExtractStage
 from nemo_curator.stages.text.download.base.url_generation import URLGenerationStage
-from nemo_curator.stages.text.filters import (
-    FastTextLangId,
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
+from nemo_curator.stages.text.filters.heuristic import (
     PunctuationFilter,
-    RepeatedLinesFilter,
-    RepeatingTopNGramsFilter,
     UrlsFilter,
     WordCountFilter,
 )
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatedLinesFilter, RepeatingTopNGramsFilter
 from nemo_curator.stages.text.io.writer import JsonlWriter, ParquetWriter
 from nemo_curator.stages.text.modules.add_id import AddId
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
 from nemo_curator.tasks import DocumentBatch, _EmptyTask
 from nemo_curator.tasks.utils import TaskPerfUtils
 

benchmarking/scripts/modifier_benchmark.py

@@ -30,15 +30,15 @@
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import ParquetReader
 from nemo_curator.stages.text.io.writer import ParquetWriter
-from nemo_curator.stages.text.modifiers import (
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.string import (
     BoilerPlateStringModifier,
     MarkdownRemover,
     NewlineNormalizer,
     QuotationRemover,
-    UnicodeReformatter,
     UrlRemover,
 )
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
 
 
 def run_modify_benchmark(

docs/about/concepts/audio/text-integration.md

@@ -37,11 +37,11 @@ Audio-text integration in NeMo Curator operates on several levels:
 
 ```{mermaid}
 flowchart LR
-    A[Audio Files] --> B[InferenceAsrNemoStage] 
+    A[Audio Files] --> B[InferenceAsrNemoStage]
     B --> C[AudioToDocumentStage]
     C --> D[ScoreFilter<br/>Text Processing]
     D --> E[Integrated Output]
-    
+
     style A fill:#e1f5fe
     style C fill:#ffcc02
     style E fill:#fff3e0
@@ -58,7 +58,7 @@ flowchart LR
     B --> E[Cross-Modal<br/>Quality Assessment]
     D --> E
     E --> F[Filtered Output]
-    
+
     style A fill:#e1f5fe
     style C fill:#e8f5e8
     style F fill:#fff3e0
@@ -110,15 +110,15 @@ flowchart TD
     F --> G[Filter<br/>Metadata-based Filtering]
     G --> H[Text Enhancement Stages]
     H --> I[Processed Dataset]
-    
+
     style A fill:#e1f5fe
     style E fill:#fff3e0
     style I fill:#e8f5e8
-    
+
     classDef audioStage fill:#bbdefb
     classDef conversionStage fill:#ffcc02
     classDef textStage fill:#c8e6c9
-    
+
     class B,C,D audioStage
     class E conversionStage
     class F,G,H textStage
@@ -143,15 +143,15 @@ flowchart TD
     E --> F[GetPairwiseWerStage<br/>Cross-Modal Metrics]
     F --> G[ScoreFilter<br/>Consistency Filtering]
     G --> H[Validated Dataset]
-    
+
     style A fill:#e8f5e8
     style D fill:#fff3e0
     style H fill:#e1f5fe
-    
+
     classDef textStage fill:#c8e6c9
     classDef matchingStage fill:#ffcc02
     classDef audioStage fill:#bbdefb
-    
+
     class B,C textStage
     class D matchingStage
     class E,F,G audioStage
@@ -180,7 +180,7 @@ converter = AudioToDocumentStage()
 # Example input AudioBatch data
 audio_data = {
     "audio_filepath": "/audio.wav",
-    "text": "ground truth", 
+    "text": "ground truth",
     "pred_text": "asr prediction",
     "wer": 15.2,
     "duration": 3.4
@@ -207,7 +207,7 @@ For practical usage examples and step-by-step implementation, refer to {doc}`/cu
 # Stage 1: Initial loading
 stage1_output = {"audio_filepath": "/audio.wav", "text": "transcription"}
 
-# Stage 2: ASR inference  
+# Stage 2: ASR inference
 stage2_output = {**stage1_output, "pred_text": "asr result"}
 
 # Stage 3: Quality assessment
@@ -231,7 +231,7 @@ NeMo Curator provides these audio quality assessment capabilities:
 
 **Duration and Speech Rate Analysis**:
 
-- Duration validation using `GetAudioDurationStage`  
+- Duration validation using `GetAudioDurationStage`
 - Speech rate calculation using `get_wordrate()` function
 - Character rate calculation using `get_charrate()` function
 
@@ -267,8 +267,8 @@ NeMo Curator provides these audio quality assessment capabilities:
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage
 from nemo_curator.stages.audio.io.convert import AudioToDocumentStage
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
-from nemo_curator.filters import WordCountFilter  # Example filter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter  # Example filter
 
 # Define a text quality filter
 text_quality_filter = WordCountFilter(min_words=10)
@@ -285,7 +285,7 @@ audio_results = audio_pipeline.run(executor)
 
 # Then process text
 text_pipeline = Pipeline(
-    name="text_processing", 
+    name="text_processing",
     stages=[
         ScoreFilter(filter_obj=text_quality_filter)
     ]

docs/about/concepts/text/data-processing-concepts.md

@@ -103,11 +103,11 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatedLinesFilter
+from nemo_curator.stages.text.filters.heuristic import (
     WordCountFilter,
     NonAlphaNumericFilter,
-    RepeatedLinesFilter,
     PunctuationFilter,
     BoilerPlateStringFilter
 )
@@ -186,8 +186,8 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Modify
-from nemo_curator.stages.text.modifiers import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
 
 # Start Ray client
 ray_client = RayClient()

docs/curate-text/load-data/read-existing.md

@@ -25,8 +25,8 @@ Use Curator's `JsonlReader` and `ParquetReader` to read existing datasets into a
 from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Initialize Ray client
 ray_client = RayClient()
@@ -70,8 +70,8 @@ ray_client.stop()
 from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import ParquetReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
 
 # Initialize Ray client
 ray_client = RayClient()

docs/curate-text/process-data/content-processing/index.md

@@ -65,8 +65,9 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
 
 # Initialize Ray client
 ray_client = RayClient()

docs/curate-text/process-data/content-processing/text-cleaning.md

@@ -13,7 +13,7 @@ modality: "text-only"
 
 Remove undesirable text such as improperly decoded Unicode characters, inconsistent line spacing, or excessive URLs from documents being pre-processed for your dataset using NeMo Curator.
 
-One common issue in text datasets is improper Unicode character encoding, which can result in garbled or unreadable text, particularly with special characters like apostrophes, quotes, or diacritical marks. For example, the input sentence `"The Mona Lisa doesn't have eyebrows."` from a given document may not have included a properly encoded apostrophe (`'`), resulting in the sentence decoding as `"The Mona Lisa doesn’t have eyebrows."`. 
+One common issue in text datasets is improper Unicode character encoding, which can result in garbled or unreadable text, particularly with special characters like apostrophes, quotes, or diacritical marks. For example, the input sentence `"The Mona Lisa doesn't have eyebrows."` from a given document may not have included a properly encoded apostrophe (`'`), resulting in the sentence decoding as `"The Mona Lisa doesn’t have eyebrows."`.
 
 NeMo Curator enables you to easily run this document through the default `UnicodeReformatter` module to detect and remove the unwanted text, or you can define your own custom Unicode text cleaner tailored to your needs.
 
@@ -42,8 +42,9 @@ from nemo_curator.core.client import RayClient
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
 
 def main():
     # Initialize Ray client
@@ -55,15 +56,15 @@ def main():
         name="text_cleaning_pipeline",
         description="Clean text data using Unicode reformatter, newline normalizer, and URL remover"
     )
-    
+
     # Add reader stage
     pipeline.add_stage(JsonlReader(file_paths="books/"))
-    
+
     # Add processing stages
     pipeline.add_stage(Modify(UnicodeReformatter()))
     pipeline.add_stage(Modify(NewlineNormalizer()))
     pipeline.add_stage(Modify(UrlRemover()))
-    
+
     # Add writer stage
     pipeline.add_stage(JsonlWriter(path="cleaned_books/"))
 
@@ -72,7 +73,7 @@ def main():
 
     # Stop Ray client
     ray_client.stop()
-    
+
 if __name__ == "__main__":
     main()
 ```
@@ -87,7 +88,7 @@ You can create your own custom text cleaner by extending the `DocumentModifier`
 ```python
 import re
 
-from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier
+from nemo_curator.stages.text.modifiers import DocumentModifier
 
 URL_REGEX = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE)
 

docs/curate-text/process-data/language-management/index.md

@@ -32,8 +32,8 @@ Language management in NeMo Curator typically follows this pattern using the Pip
 ```python
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
 
 # 1) Build the pipeline
 pipeline = Pipeline(name="language_management")

docs/curate-text/process-data/language-management/language.md

@@ -50,9 +50,9 @@ The following example demonstrates how to create a language identification pipel
 """Language identification using Curator."""
 
 from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
 from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
+from nemo_curator.stages.text.filters import ScoreFilter
 
 def create_language_identification_pipeline(data_dir: str) -> Pipeline:
     """Create a pipeline for language identification."""

docs/curate-text/process-data/quality-assessment/classifier.md

@@ -87,8 +87,8 @@ results = pipeline.run()
 from nemo_curator.pipeline import Pipeline
 from nemo_curator.stages.text.io.reader import JsonlReader
 from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
 
 # Create pipeline with FastText filter (requires pre-trained model)
 pipeline = Pipeline(name="fasttext_quality_pipeline")
@@ -123,7 +123,7 @@ You can configure quality classifiers and filters with different parameters:
 
 ```python
 from nemo_curator.stages.text.classifiers import QualityClassifier
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
 
 # DeBERTa quality classifier configurations
 basic_deberta_classifier = QualityClassifier(