diff --git a/.github/workflows/config/.secrets.baseline b/.github/workflows/config/.secrets.baseline
index 619b37c481..d024d5f39f 100644
--- a/.github/workflows/config/.secrets.baseline
+++ b/.github/workflows/config/.secrets.baseline
@@ -180,13 +180,6 @@
"is_verified": false,
"line_number": 33
},
- {
- "type": "Secret Keyword",
- "filename": "docs/curate-text/synthetic/llm-client.md",
- "hashed_secret": "2083c49ad8d63838a4d18f1de0c419f06eb464db",
- "is_verified": false,
- "line_number": 44
- },
{
"type": "Secret Keyword",
"filename": "docs/curate-text/synthetic/llm-client.md",
@@ -202,15 +195,6 @@
"line_number": 165
}
],
- "docs/curate-text/synthetic/multilingual-qa.md": [
- {
- "type": "Secret Keyword",
- "filename": "docs/curate-text/synthetic/multilingual-qa.md",
- "hashed_secret": "2083c49ad8d63838a4d18f1de0c419f06eb464db",
- "is_verified": false,
- "line_number": 30
- }
- ],
"tests/models/client/test_openai_client.py": [
{
"type": "Secret Keyword",
@@ -248,5 +232,5 @@
}
]
},
- "generated_at": "2026-02-11T21:26:53Z"
+ "generated_at": "2026-02-26T00:35:18Z"
}
diff --git a/benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py b/benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py
index 37066e9951..bb925f27b7 100644
--- a/benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py
+++ b/benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py
@@ -49,17 +49,16 @@
from nemo_curator.stages.text.download.base import URLGenerator
from nemo_curator.stages.text.download.base.iterator import DocumentIterateExtractStage
from nemo_curator.stages.text.download.base.url_generation import URLGenerationStage
-from nemo_curator.stages.text.filters import (
- FastTextLangId,
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
+from nemo_curator.stages.text.filters.heuristic import (
PunctuationFilter,
- RepeatedLinesFilter,
- RepeatingTopNGramsFilter,
UrlsFilter,
WordCountFilter,
)
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatedLinesFilter, RepeatingTopNGramsFilter
from nemo_curator.stages.text.io.writer import JsonlWriter, ParquetWriter
from nemo_curator.stages.text.modules.add_id import AddId
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
from nemo_curator.tasks import DocumentBatch, _EmptyTask
from nemo_curator.tasks.utils import TaskPerfUtils
diff --git a/benchmarking/scripts/modifier_benchmark.py b/benchmarking/scripts/modifier_benchmark.py
index 3226b47e92..fd4a47e8e1 100644
--- a/benchmarking/scripts/modifier_benchmark.py
+++ b/benchmarking/scripts/modifier_benchmark.py
@@ -30,15 +30,15 @@
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import ParquetReader
from nemo_curator.stages.text.io.writer import ParquetWriter
-from nemo_curator.stages.text.modifiers import (
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.string import (
BoilerPlateStringModifier,
MarkdownRemover,
NewlineNormalizer,
QuotationRemover,
- UnicodeReformatter,
UrlRemover,
)
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
def run_modify_benchmark(
diff --git a/docs/about/concepts/audio/text-integration.md b/docs/about/concepts/audio/text-integration.md
index e32a5e726c..25a5f8a99a 100644
--- a/docs/about/concepts/audio/text-integration.md
+++ b/docs/about/concepts/audio/text-integration.md
@@ -37,11 +37,11 @@ Audio-text integration in NeMo Curator operates on several levels:
```{mermaid}
flowchart LR
- A[Audio Files] --> B[InferenceAsrNemoStage]
+ A[Audio Files] --> B[InferenceAsrNemoStage]
B --> C[AudioToDocumentStage]
C --> D[ScoreFilter
Text Processing]
D --> E[Integrated Output]
-
+
style A fill:#e1f5fe
style C fill:#ffcc02
style E fill:#fff3e0
@@ -58,7 +58,7 @@ flowchart LR
B --> E[Cross-Modal
Quality Assessment]
D --> E
E --> F[Filtered Output]
-
+
style A fill:#e1f5fe
style C fill:#e8f5e8
style F fill:#fff3e0
@@ -110,15 +110,15 @@ flowchart TD
F --> G[Filter
Metadata-based Filtering]
G --> H[Text Enhancement Stages]
H --> I[Processed Dataset]
-
+
style A fill:#e1f5fe
style E fill:#fff3e0
style I fill:#e8f5e8
-
+
classDef audioStage fill:#bbdefb
classDef conversionStage fill:#ffcc02
classDef textStage fill:#c8e6c9
-
+
class B,C,D audioStage
class E conversionStage
class F,G,H textStage
@@ -143,15 +143,15 @@ flowchart TD
E --> F[GetPairwiseWerStage
Cross-Modal Metrics]
F --> G[ScoreFilter
Consistency Filtering]
G --> H[Validated Dataset]
-
+
style A fill:#e8f5e8
style D fill:#fff3e0
style H fill:#e1f5fe
-
+
classDef textStage fill:#c8e6c9
classDef matchingStage fill:#ffcc02
classDef audioStage fill:#bbdefb
-
+
class B,C textStage
class D matchingStage
class E,F,G audioStage
@@ -180,7 +180,7 @@ converter = AudioToDocumentStage()
# Example input AudioBatch data
audio_data = {
"audio_filepath": "/audio.wav",
- "text": "ground truth",
+ "text": "ground truth",
"pred_text": "asr prediction",
"wer": 15.2,
"duration": 3.4
@@ -207,7 +207,7 @@ For practical usage examples and step-by-step implementation, refer to {doc}`/cu
# Stage 1: Initial loading
stage1_output = {"audio_filepath": "/audio.wav", "text": "transcription"}
-# Stage 2: ASR inference
+# Stage 2: ASR inference
stage2_output = {**stage1_output, "pred_text": "asr result"}
# Stage 3: Quality assessment
@@ -231,7 +231,7 @@ NeMo Curator provides these audio quality assessment capabilities:
**Duration and Speech Rate Analysis**:
-- Duration validation using `GetAudioDurationStage`
+- Duration validation using `GetAudioDurationStage`
- Speech rate calculation using `get_wordrate()` function
- Character rate calculation using `get_charrate()` function
@@ -267,8 +267,8 @@ NeMo Curator provides these audio quality assessment capabilities:
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage
from nemo_curator.stages.audio.io.convert import AudioToDocumentStage
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
-from nemo_curator.filters import WordCountFilter # Example filter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Example filter
# Define a text quality filter
text_quality_filter = WordCountFilter(min_words=10)
@@ -285,7 +285,7 @@ audio_results = audio_pipeline.run(executor)
# Then process text
text_pipeline = Pipeline(
- name="text_processing",
+ name="text_processing",
stages=[
ScoreFilter(filter_obj=text_quality_filter)
]
diff --git a/docs/about/concepts/text/data-processing-concepts.md b/docs/about/concepts/text/data-processing-concepts.md
index 71875c3ca6..1d2c76db72 100644
--- a/docs/about/concepts/text/data-processing-concepts.md
+++ b/docs/about/concepts/text/data-processing-concepts.md
@@ -103,11 +103,11 @@ from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatedLinesFilter
+from nemo_curator.stages.text.filters.heuristic import (
WordCountFilter,
NonAlphaNumericFilter,
- RepeatedLinesFilter,
PunctuationFilter,
BoilerPlateStringFilter
)
@@ -186,8 +186,8 @@ from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Modify
-from nemo_curator.stages.text.modifiers import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
# Start Ray client
ray_client = RayClient()
diff --git a/docs/curate-text/load-data/read-existing.md b/docs/curate-text/load-data/read-existing.md
index 91fa337808..ae95842531 100644
--- a/docs/curate-text/load-data/read-existing.md
+++ b/docs/curate-text/load-data/read-existing.md
@@ -25,8 +25,8 @@ Use Curator's `JsonlReader` and `ParquetReader` to read existing datasets into a
from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
# Initialize Ray client
ray_client = RayClient()
@@ -70,8 +70,8 @@ ray_client.stop()
from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import ParquetReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
# Initialize Ray client
ray_client = RayClient()
diff --git a/docs/curate-text/process-data/content-processing/index.md b/docs/curate-text/process-data/content-processing/index.md
index f89642116e..b42cda946f 100644
--- a/docs/curate-text/process-data/content-processing/index.md
+++ b/docs/curate-text/process-data/content-processing/index.md
@@ -65,8 +65,9 @@ from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
# Initialize Ray client
ray_client = RayClient()
diff --git a/docs/curate-text/process-data/content-processing/text-cleaning.md b/docs/curate-text/process-data/content-processing/text-cleaning.md
index ea55a6641d..cc0aa12b4c 100644
--- a/docs/curate-text/process-data/content-processing/text-cleaning.md
+++ b/docs/curate-text/process-data/content-processing/text-cleaning.md
@@ -13,7 +13,7 @@ modality: "text-only"
Remove undesirable text such as improperly decoded Unicode characters, inconsistent line spacing, or excessive URLs from documents being pre-processed for your dataset using NeMo Curator.
-One common issue in text datasets is improper Unicode character encoding, which can result in garbled or unreadable text, particularly with special characters like apostrophes, quotes, or diacritical marks. For example, the input sentence `"The Mona Lisa doesn't have eyebrows."` from a given document may not have included a properly encoded apostrophe (`'`), resulting in the sentence decoding as `"The Mona Lisa doesn’t have eyebrows."`.
+One common issue in text datasets is improper Unicode character encoding, which can result in garbled or unreadable text, particularly with special characters like apostrophes, quotes, or diacritical marks. For example, the input sentence `"The Mona Lisa doesn't have eyebrows."` from a given document may not have included a properly encoded apostrophe (`'`), resulting in the sentence decoding as `"The Mona Lisa doesn’t have eyebrows."`.
NeMo Curator enables you to easily run this document through the default `UnicodeReformatter` module to detect and remove the unwanted text, or you can define your own custom Unicode text cleaner tailored to your needs.
@@ -42,8 +42,9 @@ from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer
-from nemo_curator.stages.text.modules import Modify
+from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
+from nemo_curator.stages.text.modifiers import Modify
def main():
# Initialize Ray client
@@ -55,15 +56,15 @@ def main():
name="text_cleaning_pipeline",
description="Clean text data using Unicode reformatter, newline normalizer, and URL remover"
)
-
+
# Add reader stage
pipeline.add_stage(JsonlReader(file_paths="books/"))
-
+
# Add processing stages
pipeline.add_stage(Modify(UnicodeReformatter()))
pipeline.add_stage(Modify(NewlineNormalizer()))
pipeline.add_stage(Modify(UrlRemover()))
-
+
# Add writer stage
pipeline.add_stage(JsonlWriter(path="cleaned_books/"))
@@ -72,7 +73,7 @@ def main():
# Stop Ray client
ray_client.stop()
-
+
if __name__ == "__main__":
main()
```
@@ -87,7 +88,7 @@ You can create your own custom text cleaner by extending the `DocumentModifier`
```python
import re
-from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier
+from nemo_curator.stages.text.modifiers import DocumentModifier
URL_REGEX = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE)
diff --git a/docs/curate-text/process-data/language-management/index.md b/docs/curate-text/process-data/language-management/index.md
index 347976538a..c61b4e0321 100644
--- a/docs/curate-text/process-data/language-management/index.md
+++ b/docs/curate-text/process-data/language-management/index.md
@@ -32,8 +32,8 @@ Language management in NeMo Curator typically follows this pattern using the Pip
```python
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
# 1) Build the pipeline
pipeline = Pipeline(name="language_management")
diff --git a/docs/curate-text/process-data/language-management/language.md b/docs/curate-text/process-data/language-management/language.md
index 1289fa51e7..dceaf4d764 100644
--- a/docs/curate-text/process-data/language-management/language.md
+++ b/docs/curate-text/process-data/language-management/language.md
@@ -50,9 +50,9 @@ The following example demonstrates how to create a language identification pipel
"""Language identification using Curator."""
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.filters import FastTextLangId
+from nemo_curator.stages.text.filters.fasttext import FastTextLangId
from nemo_curator.stages.text.io.reader import JsonlReader
-from nemo_curator.stages.text.modules import ScoreFilter
+from nemo_curator.stages.text.filters import ScoreFilter
def create_language_identification_pipeline(data_dir: str) -> Pipeline:
"""Create a pipeline for language identification."""
diff --git a/docs/curate-text/process-data/quality-assessment/classifier.md b/docs/curate-text/process-data/quality-assessment/classifier.md
index 107c1afaf6..68c1a61f1e 100644
--- a/docs/curate-text/process-data/quality-assessment/classifier.md
+++ b/docs/curate-text/process-data/quality-assessment/classifier.md
@@ -87,8 +87,8 @@ results = pipeline.run()
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
# Create pipeline with FastText filter (requires pre-trained model)
pipeline = Pipeline(name="fasttext_quality_pipeline")
@@ -123,7 +123,7 @@ You can configure quality classifiers and filters with different parameters:
```python
from nemo_curator.stages.text.classifiers import QualityClassifier
-from nemo_curator.stages.text.filters import FastTextQualityFilter
+from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter
# DeBERTa quality classifier configurations
basic_deberta_classifier = QualityClassifier(
diff --git a/docs/curate-text/process-data/quality-assessment/heuristic.md b/docs/curate-text/process-data/quality-assessment/heuristic.md
index 49d2fbc3da..00623e1fb1 100644
--- a/docs/curate-text/process-data/quality-assessment/heuristic.md
+++ b/docs/curate-text/process-data/quality-assessment/heuristic.md
@@ -26,7 +26,7 @@ These filters assess quality using measurable document characteristics such as:
For details on filter structure and the filtering process, refer to {ref}`Data Processing Concepts `.
----
+---
## Usage
@@ -37,12 +37,9 @@ For details on filter structure and the filtering process, refer to {ref}`Data P
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
- WordCountFilter,
- RepeatingTopNGramsFilter,
- PunctuationFilter
-)
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter, PunctuationFilter
# Create pipeline
pipeline = Pipeline(name="heuristic_filtering")
@@ -89,10 +86,10 @@ results = pipeline.run()
:::{tab-item} Configuration
```python
# Example configuration for common heuristic filters
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters.heuristic import (
WordCountFilter,
PunctuationFilter,
- RepeatingTopNGramsFilter,
SymbolsToWordsFilter,
CommonEnglishWordsFilter
)
@@ -200,17 +197,17 @@ stages:
file_paths: ${input_path}
fields: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WordCountFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WordCountFilter
min_words: 50
max_words: 100000
text_field: ${text_field}
score_field: word_count
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.PunctuationFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.PunctuationFilter
max_num_sentences_without_endmark_ratio: 0.85
text_field: ${text_field}
score_field: null
@@ -236,8 +233,9 @@ When building filter chains, follow these best practices:
```python
# Efficient ordering - place fast filters first
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter, UrlsFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter, UrlsFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
pipeline = Pipeline(name="efficient_filtering")
# Fast filters first
@@ -266,8 +264,8 @@ strict_filter = WordCountFilter(min_words=100, max_words=10000)
:::{tab-item} Language Considerations
```python
# Chinese text filter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import SymbolsToWordsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import SymbolsToWordsFilter
cn_filter = ScoreFilter(
filter_obj=SymbolsToWordsFilter(max_symbol_to_word_ratio=0.15, lang="zh"),
@@ -280,12 +278,12 @@ cn_filter = ScoreFilter(
```python
# Comprehensive quality filter pipeline
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
- WordCountFilter,
- PunctuationFilter,
- CommonEnglishWordsFilter,
- RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters.heuristic import (
+ WordCountFilter,
+ PunctuationFilter,
+ CommonEnglishWordsFilter,
)
quality_pipeline = Pipeline(name="comprehensive_quality")
@@ -330,8 +328,9 @@ Use `Score` to add score columns to your data without removing any documents:
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Score
-from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import Score
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
# Create scoring pipeline (no filtering)
pipeline = Pipeline(name="score_analysis")
@@ -408,8 +407,9 @@ After analyzing distributions, apply filters with your chosen thresholds:
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter
pipeline = Pipeline(name="filtering_pipeline")
pipeline.add_stage(JsonlReader(file_paths="input_data/", fields=["text", "id"]))
diff --git a/docs/curate-text/process-data/quality-assessment/index.md b/docs/curate-text/process-data/quality-assessment/index.md
index 8979d5b02b..1acf224589 100644
--- a/docs/curate-text/process-data/quality-assessment/index.md
+++ b/docs/curate-text/process-data/quality-assessment/index.md
@@ -30,8 +30,8 @@ The `ScoreFilter` is at the center of filtering in NeMo Curator. It applies a fi
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
# Create pipeline
pipeline = Pipeline(name="quality_filtering")
@@ -78,10 +78,10 @@ For more specific use cases, NeMo Curator provides two specialized modules:
- Takes a scoring function that evaluates text and returns a score
- Adds the score to a specified metadata field
- Useful for analysis or multi-stage filtering pipelines
-
+
```python
# Example: Score documents without filtering
-from nemo_curator.stages.text.modules import Score
+from nemo_curator.stages.text.filters import Score
scoring_step = Score(
WordCountFilter().score_document, # Use just the scoring part
@@ -95,10 +95,10 @@ scored_dataset = scoring_step.process(dataset)
- Takes a filter function that evaluates metadata and returns True/False
- Only uses existing metadata fields (doesn't compute new scores)
- Efficient for filtering on pre-computed metrics
-
+
```python
# Example: Filter using pre-computed scores
-from nemo_curator.stages.text.modules import Filter
+from nemo_curator.stages.text.filters import Filter
filter_step = Filter(
lambda score: score >= 100, # Keep documents with score >= 100
@@ -111,7 +111,7 @@ You can combine these modules in pipelines:
```python
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import Score, Filter
+from nemo_curator.stages.text.filters import Score, Filter
# Assume `word_counter` and `symbol_counter` are callables that return numeric scores
pipeline = Pipeline(name="multi_stage_filtering")
pipeline.add_stage(Score(word_counter, score_field="word_count"))
@@ -171,8 +171,8 @@ NeMo Curator provides programmatic interfaces for document filtering through the
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter
# Create and configure pipeline
pipeline = Pipeline(name="document_filtering")
diff --git a/docs/curate-text/process-data/specialized-processing/code.md b/docs/curate-text/process-data/specialized-processing/code.md
index a83c72074d..d6ecbeea42 100644
--- a/docs/curate-text/process-data/specialized-processing/code.md
+++ b/docs/curate-text/process-data/specialized-processing/code.md
@@ -26,8 +26,8 @@ Here's an example of applying code filters to a dataset:
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import (
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import (
PythonCommentToCodeFilter,
NumberOfLinesOfCodeFilter,
AlphaFilter
@@ -196,8 +196,11 @@ When filtering code datasets, consider these best practices:
1. **Language-specific configurations**: Adjust thresholds based on the programming language
```python
- from nemo_curator.stages.text.modules import ScoreFilter
- from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, GeneralCommentToCodeFilter
+ from nemo_curator.stages.text.filters import ScoreFilter
+ from nemo_curator.stages.text.filters.heuristic.code import (
+ PythonCommentToCodeFilter,
+ GeneralCommentToCodeFilter,
+ )
# Python tends to have more comments than C
python_comment_filter = ScoreFilter(
@@ -213,8 +216,8 @@ When filtering code datasets, consider these best practices:
2. **Preserve code structure**: Ensure filters don't inadvertently remove valid coding patterns
```python
- from nemo_curator.stages.text.modules import ScoreFilter
- from nemo_curator.stages.text.filters import GeneralCommentToCodeFilter
+ from nemo_curator.stages.text.filters import ScoreFilter
+ from nemo_curator.stages.text.filters.heuristic.code import GeneralCommentToCodeFilter
# Some languages naturally have low comment ratios
assembly_filter = ScoreFilter(
@@ -230,13 +233,13 @@ When filtering code datasets, consider these best practices:
```python
# First check if the content is actually Python using FastText language ID
- from nemo_curator.stages.text.filters import FastTextLangId
+ from nemo_curator.stages.text.filters.fasttext import FastTextLangId
from nemo_curator.pipeline import Pipeline
- from nemo_curator.stages.text.modules import ScoreFilter
-
+ from nemo_curator.stages.text.filters import ScoreFilter
+
# Create pipeline for Python code filtering with language detection
pipeline = Pipeline(name="python_code_filtering")
-
+
# Add language detection stage
pipeline.add_stage(ScoreFilter(
filter_obj=FastTextLangId(
@@ -246,7 +249,7 @@ When filtering code datasets, consider these best practices:
text_field="content",
score_field="language"
))
-
+
# Then apply Python-specific filters
pipeline.add_stage(ScoreFilter(
filter_obj=PythonCommentToCodeFilter(),
@@ -263,17 +266,17 @@ When filtering code datasets, consider these best practices:
```python
# Track filter statistics by running individual filters and measuring results
from nemo_curator.stages.text.io.reader import JsonlReader
-
+
# Load dataset for testing
reader = JsonlReader(file_paths="test_data/*.jsonl")
-
+
# Test individual filters to measure rejection rates
filters_to_test = {
"python_comment": PythonCommentToCodeFilter(),
"line_count": NumberOfLinesOfCodeFilter(min_lines=5, max_lines=1000),
"alpha_content": AlphaFilter(min_alpha_ratio=0.3)
}
-
+
# Note: Actual statistics collection would require running the pipeline
# and analyzing the results to determine optimal thresholds
```
@@ -286,8 +289,12 @@ When filtering code datasets, consider these best practices:
```python
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import NumberOfLinesOfCodeFilter, XMLHeaderFilter, GeneralCommentToCodeFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import (
+ NumberOfLinesOfCodeFilter,
+ XMLHeaderFilter,
+ GeneralCommentToCodeFilter,
+)
# Create pipeline to filter non-functional code snippets
pipeline = Pipeline(name="code_cleaning")
@@ -317,8 +324,12 @@ pipeline.add_stage(ScoreFilter(
```python
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import AlphaFilter, TokenizerFertilityFilter, HTMLBoilerplateFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import (
+ AlphaFilter,
+ TokenizerFertilityFilter,
+ HTMLBoilerplateFilter,
+)
# Create pipeline for training data preparation
pipeline = Pipeline(name="training_data_prep")
diff --git a/docs/curate-text/process-data/specialized-processing/index.md b/docs/curate-text/process-data/specialized-processing/index.md
index a3c597f023..c527e0ffab 100644
--- a/docs/curate-text/process-data/specialized-processing/index.md
+++ b/docs/curate-text/process-data/specialized-processing/index.md
@@ -54,8 +54,8 @@ Specialized filters for programming content and source code
```python
from nemo_curator.pipeline import Pipeline
-from nemo_curator.stages.text.modules import ScoreFilter
-from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.code import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter
from nemo_curator.stages.text.io.reader import JsonlReader
# Filter Python code based on quality metrics
@@ -76,7 +76,7 @@ code_pipeline = Pipeline(
),
ScoreFilter(
filter_obj=NumberOfLinesOfCodeFilter(min_lines=5, max_lines=1000),
- text_field="content",
+ text_field="content",
score_field="line_count"
)
])
diff --git a/docs/curate-text/synthetic/llm-client.md b/docs/curate-text/synthetic/llm-client.md
index 4f70f1a450..fbc696dc11 100644
--- a/docs/curate-text/synthetic/llm-client.md
+++ b/docs/curate-text/synthetic/llm-client.md
@@ -41,7 +41,7 @@ client = AsyncOpenAIClient(
Set your API key as an environment variable to avoid hardcoding credentials:
```bash
-export NVIDIA_API_KEY="nvapi-..."
+export NVIDIA_API_KEY=""
```
The underlying OpenAI client automatically uses the `OPENAI_API_KEY` environment variable if no `api_key` is provided. For NVIDIA APIs, explicitly pass the key:
diff --git a/docs/curate-text/synthetic/multilingual-qa.md b/docs/curate-text/synthetic/multilingual-qa.md
index e47997acb8..8797205dc7 100644
--- a/docs/curate-text/synthetic/multilingual-qa.md
+++ b/docs/curate-text/synthetic/multilingual-qa.md
@@ -27,7 +27,7 @@ A pipeline that:
- **NeMo Curator**: Installed with text extras
```bash
-export NVIDIA_API_KEY="nvapi-..."
+export NVIDIA_API_KEY=""
```
## Quick Start
@@ -147,8 +147,7 @@ pipeline.add_stage(
If your prompt includes language prefixes, you can filter to keep only specific languages:
```python
-from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
+from nemo_curator.stages.text.filters import DocumentFilter, ScoreFilter
class BeginsWithLanguageFilter(DocumentFilter):
diff --git a/docs/curate-text/synthetic/nemotron-cc/index.md b/docs/curate-text/synthetic/nemotron-cc/index.md
index 4a6a3e7c9d..c759e5833a 100644
--- a/docs/curate-text/synthetic/nemotron-cc/index.md
+++ b/docs/curate-text/synthetic/nemotron-cc/index.md
@@ -33,18 +33,18 @@ flowchart TB
C --> D[Segment Filter]
D --> E[Document Joiner]
end
-
+
subgraph "LLM Generation"
E --> F[Task-Specific Stage
WikiPara/DiverseQA/Distill/etc.]
end
-
+
subgraph "Postprocessing"
F --> G[Token Count Filter]
G --> H[Markdown Remover]
H --> I[Task-Specific Cleanup]
I --> J[Quality Filter]
end
-
+
J --> K[Output Dataset]
```
@@ -190,7 +190,7 @@ For documents with high quality scores, use tasks that leverage the existing qua
- **KnowledgeList**: Extract structured facts
```python
-from nemo_curator.stages.text.modules.score_filter import Filter
+from nemo_curator.stages.text.filters import Filter
# Filter for high-quality documents (score > 11)
pipeline.add_stage(
diff --git a/docs/get-started/text.md b/docs/get-started/text.md
index 49b801611f..fa40ad27b9 100644
--- a/docs/get-started/text.md
+++ b/docs/get-started/text.md
@@ -66,7 +66,7 @@ Install the latest version directly from GitHub:
git clone https://github.com/NVIDIA-NeMo/Curator.git
cd Curator
uv sync --extra text_cuda12 --all-groups
-source .venv/bin/activate
+source .venv/bin/activate
```
```{note}
@@ -127,8 +127,8 @@ Here's a simple example to get started with NeMo Curator's pipeline-based archit
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.text.io.reader import JsonlReader
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
-from nemo_curator.stages.text.filters import WordCountFilter, NonAlphaNumericFilter
+from nemo_curator.stages.text.filters import ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import WordCountFilter, NonAlphaNumericFilter
# Create a pipeline for text curation
pipeline = Pipeline(
diff --git a/nemo_curator/config/text/code_filter_pipeline.yaml b/nemo_curator/config/text/code_filter_pipeline.yaml
index f31ed7fd87..c038337995 100644
--- a/nemo_curator/config/text/code_filter_pipeline.yaml
+++ b/nemo_curator/config/text/code_filter_pipeline.yaml
@@ -50,23 +50,23 @@ stages:
blocksize: null
fields: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.code.PythonCommentToCodeFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.code.PythonCommentToCodeFilter
min_comment_to_code_ratio: 0.001
max_comment_to_code_ratio: 0.85
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.code.NumberOfLinesOfCodeFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.code.NumberOfLinesOfCodeFilter
min_lines: 5
max_lines: 20000
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.code.TokenizerFertilityFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.code.TokenizerFertilityFilter
path_to_tokenizer: ${path_to_tokenizer}
min_char_to_token_ratio: 2
text_field: ${text_field}
diff --git a/nemo_curator/config/text/fasttext_filter_pipeline.yaml b/nemo_curator/config/text/fasttext_filter_pipeline.yaml
index cd69a32f79..fcfda9f800 100644
--- a/nemo_curator/config/text/fasttext_filter_pipeline.yaml
+++ b/nemo_curator/config/text/fasttext_filter_pipeline.yaml
@@ -47,16 +47,16 @@ stages:
blocksize: null
fields: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.fasttext_filter.FastTextLangId
+ _target_: nemo_curator.stages.text.filters.fasttext.fasttext_filters.FastTextLangId
model_path: ${fasttext_langid_model_path}
min_langid_score: 0.3
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.fasttext_filter.FastTextQualityFilter
+ _target_: nemo_curator.stages.text.filters.fasttext.fasttext_filters.FastTextQualityFilter
model_path: ${fasttext_quality_model_path}
# The label used for high-quality documents
label: "__label__hq"
diff --git a/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml b/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml
index c1bb816ed0..3fa1c1c23b 100644
--- a/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml
+++ b/nemo_curator/config/text/heuristic_filter_english_pipeline.yaml
@@ -47,193 +47,193 @@ stages:
blocksize: null
fields: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.NonAlphaNumericFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.NonAlphaNumericFilter
max_non_alpha_numeric_to_text_ratio: 0.25
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.SymbolsToWordsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.SymbolsToWordsFilter
max_symbol_to_word_ratio: 0.1
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.NumbersFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.NumbersFilter
max_number_to_text_ratio: 0.15
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.UrlsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.UrlsFilter
max_url_to_text_ratio: 0.2
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WhiteSpaceFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WhiteSpaceFilter
max_white_space_ratio: 0.25
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.ParenthesesFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.ParenthesesFilter
max_parentheses_ratio: 0.1
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.BoilerPlateStringFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.BoilerPlateStringFilter
remove_if_at_top_or_bottom: True
max_boilerplate_string_ratio: 0.4
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedLinesFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedLinesFilter
max_repeated_line_fraction: 0.7
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedParagraphsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedParagraphsFilter
max_repeated_paragraphs_ratio: 0.7
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedLinesByCharFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedLinesByCharFilter
max_repeated_lines_char_ratio: 0.8
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedParagraphsByCharFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedParagraphsByCharFilter
max_repeated_paragraphs_char_ratio: 0.8
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WordCountFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WordCountFilter
min_words: 50
max_words: 100000
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.PunctuationFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.PunctuationFilter
max_num_sentences_without_endmark_ratio: 0.85
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WordsWithoutAlphabetsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WordsWithoutAlphabetsFilter
min_words_with_alphabets: 0.8
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.CommonEnglishWordsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.CommonEnglishWordsFilter
min_num_common_words: 2
stop_at_false: True
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.MeanWordLengthFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.MeanWordLengthFilter
max_mean_word_length: 10
min_mean_word_length: 3
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.LongWordFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.LongWordFilter
max_word_length: 1000
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.EllipsisFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.EllipsisFilter
max_num_lines_ending_with_ellipsis_ratio: 0.3
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 2
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 2
max_repeating_ngram_ratio: 0.2
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 3
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 3
max_repeating_ngram_ratio: 0.18
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 4
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 4
max_repeating_ngram_ratio: 0.16
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 5
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 5
max_repeating_duplicate_ngram_ratio: 0.15
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 6
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 6
max_repeating_duplicate_ngram_ratio: 0.14
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 7
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 7
max_repeating_duplicate_ngram_ratio: 0.13
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 8
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 8
max_repeating_duplicate_ngram_ratio: 0.12
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 9
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 9
max_repeating_duplicate_ngram_ratio: 0.11
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 10
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 10
max_repeating_duplicate_ngram_ratio: 0.10
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.BulletsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.BulletsFilter
max_bullet_lines_ratio: 0.9
text_field: ${text_field}
score_field: null
diff --git a/nemo_curator/config/text/heuristic_filter_non_english_pipeline.yaml b/nemo_curator/config/text/heuristic_filter_non_english_pipeline.yaml
index eae5b7c5ed..26a2ba7233 100644
--- a/nemo_curator/config/text/heuristic_filter_non_english_pipeline.yaml
+++ b/nemo_curator/config/text/heuristic_filter_non_english_pipeline.yaml
@@ -47,176 +47,176 @@ stages:
blocksize: null
fields: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.SymbolsToWordsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.SymbolsToWordsFilter
max_symbol_to_word_ratio: 0.1
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.NumbersFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.NumbersFilter
max_number_to_text_ratio: 0.15
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.UrlsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.UrlsFilter
max_url_to_text_ratio: 0.2
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WhiteSpaceFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WhiteSpaceFilter
max_white_space_ratio: 0.25
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.ParenthesesFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.ParenthesesFilter
max_parentheses_ratio: 0.1
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.BoilerPlateStringFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.BoilerPlateStringFilter
remove_if_at_top_or_bottom: True
max_boilerplate_string_ratio: 0.4
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedLinesFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedLinesFilter
max_repeated_line_fraction: 0.7
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedParagraphsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedParagraphsFilter
max_repeated_paragraphs_ratio: 0.7
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedLinesByCharFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedLinesByCharFilter
max_repeated_lines_char_ratio: 0.8
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatedParagraphsByCharFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatedParagraphsByCharFilter
max_repeated_paragraphs_char_ratio: 0.8
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.WordCountFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.WordCountFilter
min_words: 50
max_words: 100000
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# NOTE: This filter tends to remove many documents and will need to
# be tuned per language.
- _target_: nemo_curator.stages.text.filters.heuristic_filter.PunctuationFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.PunctuationFilter
max_num_sentences_without_endmark_ratio: 0.85
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.MeanWordLengthFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.MeanWordLengthFilter
max_mean_word_length: 10
min_mean_word_length: 3
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.LongWordFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.LongWordFilter
max_word_length: 1000
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.EllipsisFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.EllipsisFilter
max_num_lines_ending_with_ellipsis_ratio: 0.3
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 2
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 2
max_repeating_ngram_ratio: 0.2
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 3
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 3
max_repeating_ngram_ratio: 0.18
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Top N-Gram filters for N-gram 4
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingTopNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingTopNGramsFilter
n: 4
max_repeating_ngram_ratio: 0.16
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 5
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 5
max_repeating_duplicate_ngram_ratio: 0.15
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 6
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 6
max_repeating_duplicate_ngram_ratio: 0.14
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 7
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 7
max_repeating_duplicate_ngram_ratio: 0.13
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 8
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 8
max_repeating_duplicate_ngram_ratio: 0.12
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 9
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 9
max_repeating_duplicate_ngram_ratio: 0.11
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
# Duplicate N-gram filters for N-gram 10
- _target_: nemo_curator.stages.text.filters.heuristic_filter.RepeatingDuplicateNGramsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.repetition.RepeatingDuplicateNGramsFilter
n: 10
max_repeating_duplicate_ngram_ratio: 0.10
text_field: ${text_field}
score_field: null
- - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter
+ - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter
filter_obj:
- _target_: nemo_curator.stages.text.filters.heuristic_filter.BulletsFilter
+ _target_: nemo_curator.stages.text.filters.heuristic.string.BulletsFilter
max_bullet_lines_ratio: 0.9
text_field: ${text_field}
score_field: null
diff --git a/nemo_curator/stages/text/classifiers/aegis.py b/nemo_curator/stages/text/classifiers/aegis.py
index 529f1e7170..6fb2da2196 100644
--- a/nemo_curator/stages/text/classifiers/aegis.py
+++ b/nemo_curator/stages/text/classifiers/aegis.py
@@ -29,10 +29,10 @@
from nemo_curator.backends.base import NodeInfo, WorkerMetadata
from nemo_curator.stages.base import CompositeStage, ProcessingStage
+from nemo_curator.stages.text.filters import Filter
from nemo_curator.stages.text.models.model import ModelStage
from nemo_curator.stages.text.models.tokenizer import TokenizerStage
from nemo_curator.stages.text.models.utils import ATTENTION_MASK_FIELD, INPUT_ID_FIELD, format_name_with_suffix
-from nemo_curator.stages.text.modules.score_filter import Filter
from nemo_curator.tasks import DocumentBatch
from .aegis_utils import AEGIS_LABELS, format_aegis
diff --git a/nemo_curator/stages/text/classifiers/base.py b/nemo_curator/stages/text/classifiers/base.py
index baed4b133a..e967434ea1 100644
--- a/nemo_curator/stages/text/classifiers/base.py
+++ b/nemo_curator/stages/text/classifiers/base.py
@@ -26,10 +26,10 @@
from transformers import AutoConfig, AutoModel
from nemo_curator.stages.base import CompositeStage, ProcessingStage
+from nemo_curator.stages.text.filters import Filter
from nemo_curator.stages.text.models.model import ModelStage
from nemo_curator.stages.text.models.tokenizer import TokenizerStage
from nemo_curator.stages.text.models.utils import ATTENTION_MASK_FIELD, INPUT_ID_FIELD
-from nemo_curator.stages.text.modules.score_filter import Filter
from nemo_curator.tasks import DocumentBatch
diff --git a/nemo_curator/stages/text/classifiers/fineweb_edu.py b/nemo_curator/stages/text/classifiers/fineweb_edu.py
index ac85531035..772b074e24 100644
--- a/nemo_curator/stages/text/classifiers/fineweb_edu.py
+++ b/nemo_curator/stages/text/classifiers/fineweb_edu.py
@@ -23,10 +23,10 @@
from transformers import AutoModelForSequenceClassification
from nemo_curator.stages.base import CompositeStage, ProcessingStage
+from nemo_curator.stages.text.filters import Filter
from nemo_curator.stages.text.models.model import ModelStage
from nemo_curator.stages.text.models.tokenizer import TokenizerStage
from nemo_curator.stages.text.models.utils import ATTENTION_MASK_FIELD, INPUT_ID_FIELD, format_name_with_suffix
-from nemo_curator.stages.text.modules.score_filter import Filter
from nemo_curator.tasks import DocumentBatch
from .constants import DEBERTA_TOKENIZER_PADDING_SIDE
diff --git a/nemo_curator/stages/text/filters/__init__.py b/nemo_curator/stages/text/filters/__init__.py
index 925521a214..f2f6fccc6b 100644
--- a/nemo_curator/stages/text/filters/__init__.py
+++ b/nemo_curator/stages/text/filters/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,82 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .code import (
- AlphaFilter,
- GeneralCommentToCodeFilter,
- HTMLBoilerplateFilter,
- NumberOfLinesOfCodeFilter,
- PerExtensionFilter,
- PythonCommentToCodeFilter,
- TokenizerFertilityFilter,
- XMLHeaderFilter,
-)
from .doc_filter import DocumentFilter
-from .fasttext_filter import FastTextLangId, FastTextQualityFilter
-from .heuristic_filter import (
- BoilerPlateStringFilter,
- BulletsFilter,
- CommonEnglishWordsFilter,
- EllipsisFilter,
- HistogramFilter,
- LongWordFilter,
- MeanWordLengthFilter,
- NonAlphaNumericFilter,
- NumbersFilter,
- ParenthesesFilter,
- PornographicUrlsFilter,
- PunctuationFilter,
- RepeatedLinesByCharFilter,
- RepeatedLinesFilter,
- RepeatedParagraphsByCharFilter,
- RepeatedParagraphsFilter,
- RepeatingDuplicateNGramsFilter,
- RepeatingTopNGramsFilter,
- SubstringFilter,
- SymbolsToWordsFilter,
- TokenCountFilter,
- UrlsFilter,
- WhiteSpaceFilter,
- WordCountFilter,
- WordsWithoutAlphabetsFilter,
-)
+from .score_filter import Filter, Score, ScoreFilter
__all__ = [
- "AlphaFilter",
- "BoilerPlateStringFilter",
- "BulletsFilter",
- "CommonEnglishWordsFilter",
"DocumentFilter",
- "EllipsisFilter",
- "FastTextLangId",
- "FastTextQualityFilter",
- "GeneralCommentToCodeFilter",
- "HTMLBoilerplateFilter",
- "HistogramFilter",
- "LongWordFilter",
- "MeanWordLengthFilter",
- "NonAlphaNumericFilter",
- "NumberOfLinesOfCodeFilter",
- "NumbersFilter",
- "ParenthesesFilter",
- "PerExtensionFilter",
- "PornographicUrlsFilter",
- "PunctuationFilter",
- "PythonCommentToCodeFilter",
- "RepeatedLinesByCharFilter",
- "RepeatedLinesFilter",
- "RepeatedParagraphsByCharFilter",
- "RepeatedParagraphsFilter",
- "RepeatingDuplicateNGramsFilter",
- "RepeatingTopNGramsFilter",
- "SubstringFilter",
- "SymbolsToWordsFilter",
- "TokenCountFilter",
- "TokenizerFertilityFilter",
- "UrlsFilter",
- "WhiteSpaceFilter",
- "WordCountFilter",
- "WordsWithoutAlphabetsFilter",
- "XMLHeaderFilter",
- "import_filter",
+ "Filter",
+ "Score",
+ "ScoreFilter",
]
diff --git a/nemo_curator/stages/text/filters/doc_filter.py b/nemo_curator/stages/text/filters/doc_filter.py
index 96a6745d63..25568461ab 100644
--- a/nemo_curator/stages/text/filters/doc_filter.py
+++ b/nemo_curator/stages/text/filters/doc_filter.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/filters/fasttext/__init__.py b/nemo_curator/stages/text/filters/fasttext/__init__.py
new file mode 100644
index 0000000000..0dd5b0ebb5
--- /dev/null
+++ b/nemo_curator/stages/text/filters/fasttext/__init__.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .fasttext_filters import FastTextLangId, FastTextQualityFilter
+
+__all__ = [
+ "FastTextLangId",
+ "FastTextQualityFilter",
+]
+
+
+def __getattr__(name: str) -> type["FastTextLangId"] | type["FastTextQualityFilter"]:
+ if name == "FastTextLangId":
+ from .fasttext_filters import FastTextLangId
+
+ return FastTextLangId
+ if name == "FastTextQualityFilter":
+ from .fasttext_filters import FastTextQualityFilter
+
+ return FastTextQualityFilter
+ msg = f"module {__name__!r} has no attribute {name!r}"
+ raise AttributeError(msg)
diff --git a/nemo_curator/stages/text/filters/fasttext_filter.py b/nemo_curator/stages/text/filters/fasttext/fasttext_filters.py
similarity index 98%
rename from nemo_curator/stages/text/filters/fasttext_filter.py
rename to nemo_curator/stages/text/filters/fasttext/fasttext_filters.py
index f092a038e2..78a8c9e7e1 100644
--- a/nemo_curator/stages/text/filters/fasttext_filter.py
+++ b/nemo_curator/stages/text/filters/fasttext/fasttext_filters.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/filters/heuristic/__init__.py b/nemo_curator/stages/text/filters/heuristic/__init__.py
new file mode 100644
index 0000000000..dc919e5b60
--- /dev/null
+++ b/nemo_curator/stages/text/filters/heuristic/__init__.py
@@ -0,0 +1,53 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .string import (
+ BoilerPlateStringFilter,
+ BulletsFilter,
+ CommonEnglishWordsFilter,
+ EllipsisFilter,
+ LongWordFilter,
+ MeanWordLengthFilter,
+ NonAlphaNumericFilter,
+ NumbersFilter,
+ ParenthesesFilter,
+ PornographicUrlsFilter,
+ PunctuationFilter,
+ SubstringFilter,
+ SymbolsToWordsFilter,
+ UrlsFilter,
+ WhiteSpaceFilter,
+ WordCountFilter,
+ WordsWithoutAlphabetsFilter,
+)
+
+__all__ = [
+ "BoilerPlateStringFilter",
+ "BulletsFilter",
+ "CommonEnglishWordsFilter",
+ "EllipsisFilter",
+ "LongWordFilter",
+ "MeanWordLengthFilter",
+ "NonAlphaNumericFilter",
+ "NumbersFilter",
+ "ParenthesesFilter",
+ "PornographicUrlsFilter",
+ "PunctuationFilter",
+ "SubstringFilter",
+ "SymbolsToWordsFilter",
+ "UrlsFilter",
+ "WhiteSpaceFilter",
+ "WordCountFilter",
+ "WordsWithoutAlphabetsFilter",
+]
diff --git a/nemo_curator/stages/text/filters/heuristic/code/__init__.py b/nemo_curator/stages/text/filters/heuristic/code/__init__.py
new file mode 100644
index 0000000000..c8ecc49895
--- /dev/null
+++ b/nemo_curator/stages/text/filters/heuristic/code/__init__.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .code import (
+ AlphaFilter,
+ GeneralCommentToCodeFilter,
+ HTMLBoilerplateFilter,
+ NumberOfLinesOfCodeFilter,
+ PerExtensionFilter,
+ PythonCommentToCodeFilter,
+ TokenizerFertilityFilter,
+ XMLHeaderFilter,
+)
+
+__all__ = [
+ "AlphaFilter",
+ "GeneralCommentToCodeFilter",
+ "HTMLBoilerplateFilter",
+ "NumberOfLinesOfCodeFilter",
+ "PerExtensionFilter",
+ "PythonCommentToCodeFilter",
+ "TokenizerFertilityFilter",
+ "XMLHeaderFilter",
+]
diff --git a/nemo_curator/stages/text/filters/code.py b/nemo_curator/stages/text/filters/heuristic/code/code.py
similarity index 99%
rename from nemo_curator/stages/text/filters/code.py
rename to nemo_curator/stages/text/filters/heuristic/code/code.py
index 24aefd1571..571b6c869a 100644
--- a/nemo_curator/stages/text/filters/code.py
+++ b/nemo_curator/stages/text/filters/heuristic/code/code.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/filters/heuristic/repetition/__init__.py b/nemo_curator/stages/text/filters/heuristic/repetition/__init__.py
new file mode 100644
index 0000000000..5a27ad9a05
--- /dev/null
+++ b/nemo_curator/stages/text/filters/heuristic/repetition/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .repetition import (
+ RepeatedLinesByCharFilter,
+ RepeatedLinesFilter,
+ RepeatedParagraphsByCharFilter,
+ RepeatedParagraphsFilter,
+ RepeatingDuplicateNGramsFilter,
+ RepeatingTopNGramsFilter,
+)
+
+__all__ = [
+ "RepeatedLinesByCharFilter",
+ "RepeatedLinesFilter",
+ "RepeatedParagraphsByCharFilter",
+ "RepeatedParagraphsFilter",
+ "RepeatingDuplicateNGramsFilter",
+ "RepeatingTopNGramsFilter",
+]
diff --git a/nemo_curator/stages/text/filters/heuristic/repetition/repetition.py b/nemo_curator/stages/text/filters/heuristic/repetition/repetition.py
new file mode 100644
index 0000000000..44383d7244
--- /dev/null
+++ b/nemo_curator/stages/text/filters/heuristic/repetition/repetition.py
@@ -0,0 +1,203 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
+from nemo_curator.stages.text.utils.text_utils import (
+ get_ngrams,
+ get_paragraphs,
+ get_sentences,
+ get_word_splitter,
+)
+
+
+class RepeatedLinesFilter(DocumentFilter):
+ """
+ If the document shrinks by > 30% in terms of number of lines after
+ removing duplicate lines, then discard.
+ Source: Gopher (Rae et al., 2021)
+ """
+
+ def __init__(self, max_repeated_line_fraction: float = 0.7):
+ super().__init__()
+ self._cutoff = max_repeated_line_fraction
+ self._name = "repeated_lines"
+
+ def score_document(self, text: str) -> float:
+ sentences = self._sentences
+ if sentences is None:
+ sentences = get_sentences(text)
+ return len(set(sentences)) / len(sentences)
+
+ def keep_document(self, score: float) -> bool:
+ return score >= self._cutoff
+
+
+class RepeatedParagraphsFilter(DocumentFilter):
+ """
+ If the document shrinks by > 30% in terms of number of lines after
+ removing duplicate paragraphs, then discard.
+ Source: Gopher (Rae et al., 2021)
+ """
+
+ def __init__(self, max_repeated_paragraphs_ratio: float = 0.7):
+ super().__init__()
+ self._max_repeated_paragraphs_ratio = max_repeated_paragraphs_ratio
+ self._name = "repeated_paragraphs"
+
+ def score_document(self, text: str) -> float:
+ paragraphs = self._paragraphs
+ if paragraphs is None:
+ paragraphs = get_paragraphs(text)
+ return len(set(paragraphs)) / len(paragraphs)
+
+ def keep_document(self, score: float) -> bool:
+ return score >= self._max_repeated_paragraphs_ratio
+
+
+class RepeatedLinesByCharFilter(DocumentFilter):
+ """
+ If the document shrinks by > 20% in terms of number of lines
+ after removing duplicate lines, then discard.
+ Source: Gopher (Rae et al., 2021)
+ """
+
+ def __init__(self, max_repeated_lines_char_ratio: float = 0.8):
+ super().__init__()
+ self._cutoff = max_repeated_lines_char_ratio
+ self._name = "repeated_lines_char"
+
+ def score_document(self, text: str) -> float:
+ sentences = self._sentences
+ if sentences is None:
+ sentences = get_sentences(text)
+
+ return len("".join(set(sentences))) / len("".join(sentences))
+
+ def keep_document(self, score: float) -> bool:
+ return score >= self._cutoff
+
+
+class RepeatedParagraphsByCharFilter(DocumentFilter):
+ """
+ If the document shrinks by > 10% in terms of number of lines after
+ removing duplicate paragraphs, then discard.
+ Source: Gopher (Rae et al., 2021)
+ """
+
+ def __init__(self, max_repeated_paragraphs_char_ratio: float = 0.8):
+ super().__init__()
+ self._cutoff = max_repeated_paragraphs_char_ratio
+ self._name = "repeated_paragraphs_char"
+
+ def score_document(self, text: str) -> float:
+ paragraphs = self._paragraphs
+ if paragraphs is None:
+ paragraphs = get_paragraphs(text)
+
+ return len("".join(set(paragraphs))) / len("".join(paragraphs))
+
+ def keep_document(self, score: float) -> bool:
+ return score >= self._cutoff
+
+
+class RepeatingTopNGramsFilter(DocumentFilter):
+ """
+ If the document shrinks by > x% in terms of number of characters after
+ removing the top n-grams, then discard.
+ Source: Gopher (Rae et al., 2021)
+
+ For Chinese and Japanese text, we use external libraries to split the text
+ because these languages are not separated by spaces. For all other languages,
+ such as English, we assume words are separated by spaces.
+ """
+
+ def __init__(self, n: int = 2, max_repeating_ngram_ratio: float = 0.2, lang: str = "en"):
+ super().__init__()
+ self._n = n
+ self._cutoff = max_repeating_ngram_ratio
+ self._max_ratio = 1.0
+ self._word_splitter = get_word_splitter(lang)
+ self._name = f"repeating_top_{n}grams"
+
+ def score_document(self, text: str) -> float:
+ ngrams = self._ngrams
+ if ngrams is None:
+ split_text = self._word_splitter(text.strip())
+ if len(split_text) < self._n:
+ return self._max_ratio
+ ngrams = get_ngrams(split_text, self._n)
+ unique_ngrams = set(ngrams)
+ # Find the most frequent ngram in the zipped ngram list
+ counts = {ngram: {"freq": 0, "num_chars": sum(len(word) for word in ngram)} for ngram in unique_ngrams}
+ for ngram in ngrams:
+ counts[ngram]["freq"] += 1
+ most_frqnt_ngram = " ".join(max(counts, key=lambda x: counts[x]["freq"]))
+ # Find the number of characters the most frequent ngram
+ # contributes to the document
+ nchar = len(text)
+ len_diff = nchar - len(text.replace(most_frqnt_ngram, ""))
+ # Remove if the document is empty
+ return len_diff / nchar if nchar > 0 else 1.0
+
+ def keep_document(self, score: float) -> bool:
+ return score <= self._cutoff
+
+
+class RepeatingDuplicateNGramsFilter(DocumentFilter):
+ """
+ If the document shrinks by > x% in terms of number of characters
+ after removing all duplicate n-grams, then discard.
+ Source: Gopher (Rae et al., 2021)
+
+ For Chinese and Japanese text, we use external libraries to split the text
+ because these languages are not separated by spaces. For all other languages,
+ such as English, we assume words are separated by spaces.
+ """
+
+ def __init__(self, n: int = 2, max_repeating_duplicate_ngram_ratio: float = 0.2, lang: str = "en"):
+ super().__init__()
+ self._n = n
+ self._cutoff = max_repeating_duplicate_ngram_ratio
+ self._max_ratio = 1.0
+ self._word_splitter = get_word_splitter(lang)
+ self._name = f"repeating_dup_{n}gram"
+
+ def score_document(self, text: str) -> float:
+ ngrams = self._ngrams
+ if ngrams is None:
+ split_text = self._word_splitter(text.strip())
+ if len(split_text) < self._n:
+ return self._max_ratio
+ ngrams = get_ngrams(split_text, self._n)
+
+ counts = {}
+ duplicated_nchar = 0
+ overlapping_ngrams = 0
+ for ngram in ngrams:
+ counts[ngram] = counts.get(ngram, 0) + 1
+ if counts[ngram] > 1:
+ # Count the number of characters in this ngram that haven't been counted already
+ duplicated_ngrams = sum(len(gram) for gram in ngram[overlapping_ngrams:])
+ # Count the spaces between the ngrams
+ nspaces = min(self._n - overlapping_ngrams, self._n - 1)
+ duplicated_nchar += duplicated_ngrams + nspaces
+ overlapping_ngrams = self._n
+ overlapping_ngrams = max(overlapping_ngrams - 1, 0)
+
+ nchar = len(text)
+ # Remove if the document is empty
+ return duplicated_nchar / nchar if nchar > 0 else 1.0
+
+ def keep_document(self, score: float) -> bool:
+ return score <= self._cutoff
diff --git a/nemo_curator/stages/text/filters/heuristic_filter.py b/nemo_curator/stages/text/filters/heuristic/string.py
similarity index 54%
rename from nemo_curator/stages/text/filters/heuristic_filter.py
rename to nemo_curator/stages/text/filters/heuristic/string.py
index cfccce2c09..ce1a0d2b4e 100644
--- a/nemo_curator/stages/text/filters/heuristic_filter.py
+++ b/nemo_curator/stages/text/filters/heuristic/string.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,15 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import os.path
-import tarfile
from typing import Literal
-import huggingface_hub
-import requests
-from platformdirs import user_cache_dir
-from transformers import AutoTokenizer
-
from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
from nemo_curator.stages.text.utils.constants import (
bullet_list,
@@ -37,7 +30,6 @@
white_space_list,
)
from nemo_curator.stages.text.utils.text_utils import (
- get_ngrams,
get_paragraphs,
get_sentences,
get_word_splitter,
@@ -325,188 +317,6 @@ def keep_document(self, score: float) -> bool:
return self._min_cutoff <= score <= self._max_cutoff
-class RepeatedLinesFilter(DocumentFilter):
- """
- If the document shrinks by > 30% in terms of number of lines after
- removing duplicate lines, then discard.
- Source: Gopher (Rae et al., 2021)
- """
-
- def __init__(self, max_repeated_line_fraction: float = 0.7):
- super().__init__()
- self._cutoff = max_repeated_line_fraction
- self._name = "repeated_lines"
-
- def score_document(self, text: str) -> float:
- sentences = self._sentences
- if sentences is None:
- sentences = get_sentences(text)
- return len(set(sentences)) / len(sentences)
-
- def keep_document(self, score: float) -> bool:
- return score >= self._cutoff
-
-
-class RepeatedParagraphsFilter(DocumentFilter):
- """
- If the document shrinks by > 30% in terms of number of lines after
- removing duplicate paragraphs, then discard.
- Source: Gopher (Rae et al., 2021)
- """
-
- def __init__(self, max_repeated_paragraphs_ratio: float = 0.7):
- super().__init__()
- self._max_repeated_paragraphs_ratio = max_repeated_paragraphs_ratio
- self._name = "repeated_paragraphs"
-
- def score_document(self, text: str) -> float:
- paragraphs = self._paragraphs
- if paragraphs is None:
- paragraphs = get_paragraphs(text)
- return len(set(paragraphs)) / len(paragraphs)
-
- def keep_document(self, score: float) -> bool:
- return score >= self._max_repeated_paragraphs_ratio
-
-
-class RepeatedLinesByCharFilter(DocumentFilter):
- """
- If the document shrinks by > 20% in terms of number of lines
- after removing duplicate lines, then discard.
- Source: Gopher (Rae et al., 2021)
- """
-
- def __init__(self, max_repeated_lines_char_ratio: float = 0.8):
- super().__init__()
- self._cutoff = max_repeated_lines_char_ratio
- self._name = "repeated_lines_char"
-
- def score_document(self, text: str) -> float:
- sentences = self._sentences
- if sentences is None:
- sentences = get_sentences(text)
-
- return len("".join(set(sentences))) / len("".join(sentences))
-
- def keep_document(self, score: float) -> bool:
- return score >= self._cutoff
-
-
-class RepeatedParagraphsByCharFilter(DocumentFilter):
- """
- If the document shrinks by > 10% in terms of number of lines after
- removing duplicate paragraphs, then discard.
- Source: Gopher (Rae et al., 2021)
- """
-
- def __init__(self, max_repeated_paragraphs_char_ratio: float = 0.8):
- super().__init__()
- self._cutoff = max_repeated_paragraphs_char_ratio
- self._name = "repeated_paragraphs_char"
-
- def score_document(self, text: str) -> float:
- paragraphs = self._paragraphs
- if paragraphs is None:
- paragraphs = get_paragraphs(text)
-
- return len("".join(set(paragraphs))) / len("".join(paragraphs))
-
- def keep_document(self, score: float) -> bool:
- return score >= self._cutoff
-
-
-class RepeatingTopNGramsFilter(DocumentFilter):
- """
- If the document shrinks by > x% in terms of number of characters after
- removing the top n-grams, then discard.
- Source: Gopher (Rae et al., 2021)
-
- For Chinese and Japanese text, we use external libraries to split the text
- because these languages are not separated by spaces. For all other languages,
- such as English, we assume words are separated by spaces.
- """
-
- def __init__(self, n: int = 2, max_repeating_ngram_ratio: float = 0.2, lang: str = "en"):
- super().__init__()
- self._n = n
- self._cutoff = max_repeating_ngram_ratio
- self._max_ratio = 1.0
- self._word_splitter = get_word_splitter(lang)
- self._name = f"repeating_top_{n}grams"
-
- def score_document(self, text: str) -> float:
- ngrams = self._ngrams
- if ngrams is None:
- split_text = self._word_splitter(text.strip())
- if len(split_text) < self._n:
- return self._max_ratio
- ngrams = get_ngrams(split_text, self._n)
- unique_ngrams = set(ngrams)
- # Find the most frequent ngram in the zipped ngram list
- counts = {ngram: {"freq": 0, "num_chars": sum(len(word) for word in ngram)} for ngram in unique_ngrams}
- for ngram in ngrams:
- counts[ngram]["freq"] += 1
- most_frqnt_ngram = " ".join(max(counts, key=lambda x: counts[x]["freq"]))
- # Find the number of characters the most frequent ngram
- # contributes to the document
- nchar = len(text)
- len_diff = nchar - len(text.replace(most_frqnt_ngram, ""))
- # Remove if the document is empty
- return len_diff / nchar if nchar > 0 else 1.0
-
- def keep_document(self, score: float) -> bool:
- return score <= self._cutoff
-
-
-class RepeatingDuplicateNGramsFilter(DocumentFilter):
- """
- If the document shrinks by > x% in terms of number of characters
- after removing all duplicate n-grams, then discard.
- Source: Gopher (Rae et al., 2021)
-
- For Chinese and Japanese text, we use external libraries to split the text
- because these languages are not separated by spaces. For all other languages,
- such as English, we assume words are separated by spaces.
- """
-
- def __init__(self, n: int = 2, max_repeating_duplicate_ngram_ratio: float = 0.2, lang: str = "en"):
- super().__init__()
- self._n = n
- self._cutoff = max_repeating_duplicate_ngram_ratio
- self._max_ratio = 1.0
- self._word_splitter = get_word_splitter(lang)
- self._name = f"repeating_dup_{n}gram"
-
- def score_document(self, text: str) -> float:
- ngrams = self._ngrams
- if ngrams is None:
- split_text = self._word_splitter(text.strip())
- if len(split_text) < self._n:
- return self._max_ratio
- ngrams = get_ngrams(split_text, self._n)
-
- counts = {}
- duplicated_nchar = 0
- overlapping_ngrams = 0
- for ngram in ngrams:
- counts[ngram] = counts.get(ngram, 0) + 1
- if counts[ngram] > 1:
- # Count the number of characters in this ngram that haven't been counted already
- duplicated_ngrams = sum(len(gram) for gram in ngram[overlapping_ngrams:])
- # Count the spaces between the ngrams
- nspaces = min(self._n - overlapping_ngrams, self._n - 1)
- duplicated_nchar += duplicated_ngrams + nspaces
- overlapping_ngrams = self._n
- overlapping_ngrams = max(overlapping_ngrams - 1, 0)
-
- nchar = len(text)
- # Remove if the document is empty
- return duplicated_nchar / nchar if nchar > 0 else 1.0
-
- def keep_document(self, score: float) -> bool:
- return score <= self._cutoff
-
-
class PunctuationFilter(DocumentFilter):
"""
If more than 85% of the sentences do not end with a
@@ -638,71 +448,6 @@ def keep_document(self, score: int) -> bool:
return score != 1
-class TokenCountFilter(DocumentFilter):
- """
- If the document contains more or less than a specified number of tokens, then discard.
- """
-
- def __init__(
- self,
- tokenizer: AutoTokenizer | None = None,
- hf_model_name: str | None = None,
- hf_token: str | None = None,
- min_tokens: int = 0,
- max_tokens: int = float("inf"),
- ):
- """
- Args:
- tokenizer (AutoTokenizer | None): The pre-loaded tokenizer to use to count the tokens.
- If None, the tokenizer will be initialized from the hf_model_name.
- hf_model_name (str | None): The name of the Hugging Face model to use to count the tokens.
- If None, the pre-loaded tokenizer must be provided via the tokenizer argument.
- hf_token (str | None): The token to use to access the Hugging Face model, if needed.
- min_tokens (int): The minimum number of tokens the document must contain.
- Set to 0 to disable the minimum token count filter.
- max_tokens (int): The maximum number of tokens the document can contain.
- Set to infinity to disable the maximum token count filter.
- """
- super().__init__()
-
- if tokenizer is None and hf_model_name is None:
- msg = "Either tokenizer or hf_model_name must be provided"
- raise ValueError(msg)
- if tokenizer is not None and hf_model_name is not None:
- msg = "Either tokenizer or hf_model_name must be provided, not both"
- raise ValueError(msg)
-
- self._token_count_filter_tokenizer = tokenizer
- self._hf_model_name = hf_model_name
- self._hf_token = hf_token
- self._min_tokens = min_tokens
- self._max_tokens = max_tokens
- self._name = "token_count"
-
- def model_check_or_download(self) -> None:
- if self._hf_model_name is not None:
- # Use snapshot_download to download all files without loading the model into memory.
- huggingface_hub.snapshot_download(
- repo_id=self._hf_model_name,
- token=self._hf_token,
- local_files_only=False, # Download if not cached
- resume_download=True, # Resume interrupted downloads
- )
-
- def load_tokenizer(self) -> None:
- if self._hf_model_name is not None:
- self._token_count_filter_tokenizer = AutoTokenizer.from_pretrained(
- self._hf_model_name, local_files_only=True
- )
-
- def score_document(self, text: str) -> int:
- tokens = self._token_count_filter_tokenizer.encode(text)
- return len(tokens)
-
- def keep_document(self, score: int) -> bool:
- return self._min_tokens <= score <= self._max_tokens
-
-
class SubstringFilter(DocumentFilter):
"""
Keeps documents that contain a substring in a given position.
@@ -735,103 +480,3 @@ def score_document(self, text: str) -> int:
def keep_document(self, score: int) -> bool:
return score == 1
-
-
-class HistogramFilter(DocumentFilter):
- """Histogram filter used by the NLLB paper (https://arxiv.org/pdf/2207.04672). See p30 for details.
-
- The high-level idea of histogram filter can be described as a cheap version of language ID.
- Basically, it checks what ratio of characters in the data instance are included in the character historgrams collected from trusted data in the corresponding language.
- If the ratio is too low, then there is a good chance that there is a language ID mismatch and the data instance should be discarded.
-
- Written with reference to the original fairseq implementation at:
- https://github.com/facebookresearch/fairseq/blob/main/examples/m2m_100/process_data/clean_histogram.py.
- """
-
- def __init__(
- self,
- lang: str | None = "en",
- threshold: float | None = 0.8,
- cache_dir: str | None = "",
- threshold_char: str | None = "]",
- ):
- """Args:
- lang (str, optional): Expected language of the segment. This will decide which histogram will be loaded. Defaults to "en".
- threshold (float, optional): Threshold for ratio of characters in the histogram. Defaults to 0.8.
- cache_dir (str, optional): Cache dir download histogram files. Defaults to "".
- threshold_char (str, optional): Formatter character of the histogram files. You should not change this unless you rebuilt your own histogram. Defaults to "]".
- """
- super().__init__()
- self._lang = lang
- self._threshold = threshold
- self._cache_dir = cache_dir if cache_dir else user_cache_dir()
- self._threshold_char = threshold_char
- self._name = "histogram"
-
- if not os.path.isdir(os.path.join(self._cache_dir, "histograms")):
- self._download_histograms()
-
- self._read_hist()
-
- def _download_histograms(self) -> None:
- """Download and process histograms from default repo.
-
- Raises:
- requests.exceptions.RequestException: If download fails.
- """
-
- # Send a GET request to the URL
- response = requests.get("https://dl.fbaipublicfiles.com/m2m_100/histograms.tar.gz") # noqa: S113
-
- # Check if the request was successful
- if response.status_code != 200: # noqa: PLR2004
- msg = f"Failed to download histogram file. Status code: {response.status_code}"
- raise requests.exceptions.RequestException(msg)
-
- # Open a file to write the content
- os.makedirs(self._cache_dir, exist_ok=True)
- download_dest_path = os.path.join(self._cache_dir, "histograms.tar.gz")
- with open(download_dest_path, "wb") as file:
- file.write(response.content)
-
- extract_path = os.path.join(self._cache_dir, "histograms")
- with tarfile.open(download_dest_path, "r:gz") as tar:
- # Extract all the contents into the specified directory
- tar.extractall(path=extract_path) # noqa: S202
-
- def _read_hist(self) -> None:
- """Load histogram files."""
-
- self._histogram = []
- with open(
- os.path.join(
- self._cache_dir,
- "histograms",
- "checkpoint",
- "edunov",
- "cc60_multilingual",
- "clean_hists",
- self._lang,
- )
- ) as f:
- for line in f:
- c = line[0]
- if c == self._threshold_char:
- break
- self._histogram.append(c)
- self._histogram = set(self._histogram)
-
- def score_document(self, text: str) -> float:
- """Compute histogram token ratio of a text data instance according to the loaded histogram.
-
- Args:
- text (str): Text data instance.
-
- Returns:
- float: Ratio of tokens included in the histogram.
- """
- cnt = len([c for c in text.strip() if c in self._histogram])
- return 1 if cnt / len(text) > self._threshold else 0
-
- def keep_document(self, score: float) -> bool:
- return score == 1
diff --git a/nemo_curator/stages/text/filters/histogram/__init__.py b/nemo_curator/stages/text/filters/histogram/__init__.py
new file mode 100644
index 0000000000..aff1dbe37d
--- /dev/null
+++ b/nemo_curator/stages/text/filters/histogram/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .histogram import HistogramFilter
+
+__all__ = [
+ "HistogramFilter",
+]
+
+
+def __getattr__(name: str) -> type["HistogramFilter"]:
+ if name == "HistogramFilter":
+ from .histogram import HistogramFilter
+
+ return HistogramFilter
+ msg = f"module {__name__!r} has no attribute {name!r}"
+ raise AttributeError(msg)
diff --git a/nemo_curator/stages/text/filters/histogram/histogram.py b/nemo_curator/stages/text/filters/histogram/histogram.py
new file mode 100644
index 0000000000..dd5434c4e5
--- /dev/null
+++ b/nemo_curator/stages/text/filters/histogram/histogram.py
@@ -0,0 +1,121 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import tarfile
+
+import requests
+from platformdirs import user_cache_dir
+
+from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
+
+
+class HistogramFilter(DocumentFilter):
+ """Histogram filter used by the NLLB paper (https://arxiv.org/pdf/2207.04672). See p30 for details.
+
+ The high-level idea of histogram filter can be described as a cheap version of language ID.
+ Basically, it checks what ratio of characters in the data instance are included in the character historgrams collected from trusted data in the corresponding language.
+ If the ratio is too low, then there is a good chance that there is a language ID mismatch and the data instance should be discarded.
+
+ Written with reference to the original fairseq implementation at:
+ https://github.com/facebookresearch/fairseq/blob/main/examples/m2m_100/process_data/clean_histogram.py.
+ """
+
+ def __init__(
+ self,
+ lang: str | None = "en",
+ threshold: float | None = 0.8,
+ cache_dir: str | None = "",
+ threshold_char: str | None = "]",
+ ):
+ """Args:
+ lang (str, optional): Expected language of the segment. This will decide which histogram will be loaded. Defaults to "en".
+ threshold (float, optional): Threshold for ratio of characters in the histogram. Defaults to 0.8.
+ cache_dir (str, optional): Cache dir download histogram files. Defaults to "".
+ threshold_char (str, optional): Formatter character of the histogram files. You should not change this unless you rebuilt your own histogram. Defaults to "]".
+ """
+ super().__init__()
+ self._lang = lang
+ self._threshold = threshold
+ self._cache_dir = cache_dir if cache_dir else user_cache_dir()
+ self._threshold_char = threshold_char
+ self._name = "histogram"
+
+ if not os.path.isdir(os.path.join(self._cache_dir, "histograms")):
+ self._download_histograms()
+
+ self._read_hist()
+
+ def _download_histograms(self) -> None:
+ """Download and process histograms from default repo.
+
+ Raises:
+ requests.exceptions.RequestException: If download fails.
+ """
+
+ # Send a GET request to the URL
+ response = requests.get("https://dl.fbaipublicfiles.com/m2m_100/histograms.tar.gz") # noqa: S113
+
+ # Check if the request was successful
+ if response.status_code != 200: # noqa: PLR2004
+ msg = f"Failed to download histogram file. Status code: {response.status_code}"
+ raise requests.exceptions.RequestException(msg)
+
+ # Open a file to write the content
+ os.makedirs(self._cache_dir, exist_ok=True)
+ download_dest_path = os.path.join(self._cache_dir, "histograms.tar.gz")
+ with open(download_dest_path, "wb") as file:
+ file.write(response.content)
+
+ extract_path = os.path.join(self._cache_dir, "histograms")
+ with tarfile.open(download_dest_path, "r:gz") as tar:
+ # Extract all the contents into the specified directory
+ tar.extractall(path=extract_path) # noqa: S202
+
+ def _read_hist(self) -> None:
+ """Load histogram files."""
+
+ self._histogram = []
+ with open(
+ os.path.join(
+ self._cache_dir,
+ "histograms",
+ "checkpoint",
+ "edunov",
+ "cc60_multilingual",
+ "clean_hists",
+ self._lang,
+ )
+ ) as f:
+ for line in f:
+ c = line[0]
+ if c == self._threshold_char:
+ break
+ self._histogram.append(c)
+ self._histogram = set(self._histogram)
+
+ def score_document(self, text: str) -> float:
+ """Compute histogram token ratio of a text data instance according to the loaded histogram.
+
+ Args:
+ text (str): Text data instance.
+
+ Returns:
+ float: Ratio of tokens included in the histogram.
+ """
+ cnt = len([c for c in text.strip() if c in self._histogram])
+ return 1 if cnt / len(text) > self._threshold else 0
+
+ def keep_document(self, score: float) -> bool:
+ return score == 1
diff --git a/nemo_curator/stages/text/modules/score_filter.py b/nemo_curator/stages/text/filters/score_filter.py
similarity index 99%
rename from nemo_curator/stages/text/modules/score_filter.py
rename to nemo_curator/stages/text/filters/score_filter.py
index 7bc1394b0b..5ea1b76825 100644
--- a/nemo_curator/stages/text/modules/score_filter.py
+++ b/nemo_curator/stages/text/filters/score_filter.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/filters/token/__init__.py b/nemo_curator/stages/text/filters/token/__init__.py
new file mode 100644
index 0000000000..d8b5f91aa6
--- /dev/null
+++ b/nemo_curator/stages/text/filters/token/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .token_count import TokenCountFilter
+
+__all__ = [
+ "TokenCountFilter",
+]
+
+
+def __getattr__(name: str) -> type["TokenCountFilter"]:
+ if name == "TokenCountFilter":
+ from .token_count import TokenCountFilter
+
+ return TokenCountFilter
+ msg = f"module {__name__!r} has no attribute {name!r}"
+ raise AttributeError(msg)
diff --git a/nemo_curator/stages/text/filters/token/token_count.py b/nemo_curator/stages/text/filters/token/token_count.py
new file mode 100644
index 0000000000..da39907de2
--- /dev/null
+++ b/nemo_curator/stages/text/filters/token/token_count.py
@@ -0,0 +1,83 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import huggingface_hub
+from transformers import AutoTokenizer
+
+from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
+
+
+class TokenCountFilter(DocumentFilter):
+ """
+ If the document contains more or less than a specified number of tokens, then discard.
+ """
+
+ def __init__(
+ self,
+ tokenizer: AutoTokenizer | None = None,
+ hf_model_name: str | None = None,
+ hf_token: str | None = None,
+ min_tokens: int = 0,
+ max_tokens: int = float("inf"),
+ ):
+ """
+ Args:
+ tokenizer (AutoTokenizer | None): The pre-loaded tokenizer to use to count the tokens.
+ If None, the tokenizer will be initialized from the hf_model_name.
+ hf_model_name (str | None): The name of the Hugging Face model to use to count the tokens.
+ If None, the pre-loaded tokenizer must be provided via the tokenizer argument.
+ hf_token (str | None): The token to use to access the Hugging Face model, if needed.
+ min_tokens (int): The minimum number of tokens the document must contain.
+ Set to 0 to disable the minimum token count filter.
+ max_tokens (int): The maximum number of tokens the document can contain.
+ Set to infinity to disable the maximum token count filter.
+ """
+ super().__init__()
+
+ if tokenizer is None and hf_model_name is None:
+ msg = "Either tokenizer or hf_model_name must be provided"
+ raise ValueError(msg)
+ if tokenizer is not None and hf_model_name is not None:
+ msg = "Either tokenizer or hf_model_name must be provided, not both"
+ raise ValueError(msg)
+
+ self._token_count_filter_tokenizer = tokenizer
+ self._hf_model_name = hf_model_name
+ self._hf_token = hf_token
+ self._min_tokens = min_tokens
+ self._max_tokens = max_tokens
+ self._name = "token_count"
+
+ def model_check_or_download(self) -> None:
+ if self._hf_model_name is not None:
+ # Use snapshot_download to download all files without loading the model into memory.
+ huggingface_hub.snapshot_download(
+ repo_id=self._hf_model_name,
+ token=self._hf_token,
+ local_files_only=False, # Download if not cached
+ resume_download=True, # Resume interrupted downloads
+ )
+
+ def load_tokenizer(self) -> None:
+ if self._hf_model_name is not None:
+ self._token_count_filter_tokenizer = AutoTokenizer.from_pretrained(
+ self._hf_model_name, local_files_only=True
+ )
+
+ def score_document(self, text: str) -> int:
+ tokens = self._token_count_filter_tokenizer.encode(text)
+ return len(tokens)
+
+ def keep_document(self, score: int) -> bool:
+ return self._min_tokens <= score <= self._max_tokens
diff --git a/nemo_curator/stages/text/modifiers/__init__.py b/nemo_curator/stages/text/modifiers/__init__.py
index 9586fa4803..6fecfc95a1 100644
--- a/nemo_curator/stages/text/modifiers/__init__.py
+++ b/nemo_curator/stages/text/modifiers/__init__.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,26 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from .c4 import BoilerPlateStringModifier
from .doc_modifier import DocumentModifier
-from .fasttext import FastTextLabelModifier
-from .line_remover import LineRemover
-from .markdown_remover import MarkdownRemover
-from .newline_normalizer import NewlineNormalizer
-from .quotation_remover import QuotationRemover
-from .slicer import Slicer
-from .unicode_reformatter import UnicodeReformatter
-from .url_remover import UrlRemover
+from .modifier import Modify
+from .string import (
+ BoilerPlateStringModifier,
+ LineRemover,
+ MarkdownRemover,
+ NewlineNormalizer,
+ QuotationRemover,
+ Slicer,
+ UrlRemover,
+)
__all__ = [
"BoilerPlateStringModifier",
"DocumentModifier",
- "FastTextLabelModifier",
"LineRemover",
"MarkdownRemover",
+ "Modify",
"NewlineNormalizer",
"QuotationRemover",
"Slicer",
- "UnicodeReformatter",
"UrlRemover",
]
diff --git a/nemo_curator/stages/text/modifiers/doc_modifier.py b/nemo_curator/stages/text/modifiers/doc_modifier.py
index 76791fbab5..df12ece1fb 100644
--- a/nemo_curator/stages/text/modifiers/doc_modifier.py
+++ b/nemo_curator/stages/text/modifiers/doc_modifier.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/fasttext/__init__.py b/nemo_curator/stages/text/modifiers/fasttext/__init__.py
new file mode 100644
index 0000000000..226df04b56
--- /dev/null
+++ b/nemo_curator/stages/text/modifiers/fasttext/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .fasttext_label import FastTextLabelModifier
+
+__all__ = [
+ "FastTextLabelModifier",
+]
+
+
+def __getattr__(name: str) -> type["FastTextLabelModifier"]:
+ if name == "FastTextLabelModifier":
+ from .fasttext_label import FastTextLabelModifier
+
+ return FastTextLabelModifier
+ msg = f"module {__name__!r} has no attribute {name!r}"
+ raise AttributeError(msg)
diff --git a/nemo_curator/stages/text/modifiers/fasttext.py b/nemo_curator/stages/text/modifiers/fasttext/fasttext_label.py
similarity index 93%
rename from nemo_curator/stages/text/modifiers/fasttext.py
rename to nemo_curator/stages/text/modifiers/fasttext/fasttext_label.py
index 49ef2ef9b6..5bbe7af4bd 100644
--- a/nemo_curator/stages/text/modifiers/fasttext.py
+++ b/nemo_curator/stages/text/modifiers/fasttext/fasttext_label.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modules/modifier.py b/nemo_curator/stages/text/modifiers/modifier.py
similarity index 99%
rename from nemo_curator/stages/text/modules/modifier.py
rename to nemo_curator/stages/text/modifiers/modifier.py
index b8a69deea8..ee14710eec 100644
--- a/nemo_curator/stages/text/modules/modifier.py
+++ b/nemo_curator/stages/text/modifiers/modifier.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/string/__init__.py b/nemo_curator/stages/text/modifiers/string/__init__.py
new file mode 100644
index 0000000000..6dd09c79ab
--- /dev/null
+++ b/nemo_curator/stages/text/modifiers/string/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .c4 import BoilerPlateStringModifier
+from .line_remover import LineRemover
+from .markdown_remover import MarkdownRemover
+from .newline_normalizer import NewlineNormalizer
+from .quotation_remover import QuotationRemover
+from .slicer import Slicer
+from .url_remover import UrlRemover
+
+__all__ = [
+ "BoilerPlateStringModifier",
+ "LineRemover",
+ "MarkdownRemover",
+ "NewlineNormalizer",
+ "QuotationRemover",
+ "Slicer",
+ "UrlRemover",
+]
diff --git a/nemo_curator/stages/text/modifiers/c4.py b/nemo_curator/stages/text/modifiers/string/c4.py
similarity index 98%
rename from nemo_curator/stages/text/modifiers/c4.py
rename to nemo_curator/stages/text/modifiers/string/c4.py
index a524a64d7a..205b343096 100644
--- a/nemo_curator/stages/text/modifiers/c4.py
+++ b/nemo_curator/stages/text/modifiers/string/c4.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/line_remover.py b/nemo_curator/stages/text/modifiers/string/line_remover.py
similarity index 94%
rename from nemo_curator/stages/text/modifiers/line_remover.py
rename to nemo_curator/stages/text/modifiers/string/line_remover.py
index 0d16da4826..cb18c0f33a 100644
--- a/nemo_curator/stages/text/modifiers/line_remover.py
+++ b/nemo_curator/stages/text/modifiers/string/line_remover.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/markdown_remover.py b/nemo_curator/stages/text/modifiers/string/markdown_remover.py
similarity index 96%
rename from nemo_curator/stages/text/modifiers/markdown_remover.py
rename to nemo_curator/stages/text/modifiers/string/markdown_remover.py
index 9716012bab..b144ddbbab 100644
--- a/nemo_curator/stages/text/modifiers/markdown_remover.py
+++ b/nemo_curator/stages/text/modifiers/string/markdown_remover.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/newline_normalizer.py b/nemo_curator/stages/text/modifiers/string/newline_normalizer.py
similarity index 94%
rename from nemo_curator/stages/text/modifiers/newline_normalizer.py
rename to nemo_curator/stages/text/modifiers/string/newline_normalizer.py
index e60e228ddb..3e20e160da 100644
--- a/nemo_curator/stages/text/modifiers/newline_normalizer.py
+++ b/nemo_curator/stages/text/modifiers/string/newline_normalizer.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/quotation_remover.py b/nemo_curator/stages/text/modifiers/string/quotation_remover.py
similarity index 95%
rename from nemo_curator/stages/text/modifiers/quotation_remover.py
rename to nemo_curator/stages/text/modifiers/string/quotation_remover.py
index 43dd5c820a..19e82316d4 100644
--- a/nemo_curator/stages/text/modifiers/quotation_remover.py
+++ b/nemo_curator/stages/text/modifiers/string/quotation_remover.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/slicer.py b/nemo_curator/stages/text/modifiers/string/slicer.py
similarity index 98%
rename from nemo_curator/stages/text/modifiers/slicer.py
rename to nemo_curator/stages/text/modifiers/string/slicer.py
index 0366ee3232..e47e06487f 100644
--- a/nemo_curator/stages/text/modifiers/slicer.py
+++ b/nemo_curator/stages/text/modifiers/string/slicer.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/url_remover.py b/nemo_curator/stages/text/modifiers/string/url_remover.py
similarity index 93%
rename from nemo_curator/stages/text/modifiers/url_remover.py
rename to nemo_curator/stages/text/modifiers/string/url_remover.py
index 11a588a739..4f4b88e7fc 100644
--- a/nemo_curator/stages/text/modifiers/url_remover.py
+++ b/nemo_curator/stages/text/modifiers/string/url_remover.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modifiers/unicode/__init__.py b/nemo_curator/stages/text/modifiers/unicode/__init__.py
new file mode 100644
index 0000000000..4359f759c7
--- /dev/null
+++ b/nemo_curator/stages/text/modifiers/unicode/__init__.py
@@ -0,0 +1,31 @@
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from .unicode_reformatter import UnicodeReformatter
+
+__all__ = [
+ "UnicodeReformatter",
+]
+
+
+def __getattr__(name: str) -> type["UnicodeReformatter"]:
+ if name == "UnicodeReformatter":
+ from .unicode_reformatter import UnicodeReformatter
+
+ return UnicodeReformatter
+ msg = f"module {__name__!r} has no attribute {name!r}"
+ raise AttributeError(msg)
diff --git a/nemo_curator/stages/text/modifiers/unicode_reformatter.py b/nemo_curator/stages/text/modifiers/unicode/unicode_reformatter.py
similarity index 99%
rename from nemo_curator/stages/text/modifiers/unicode_reformatter.py
rename to nemo_curator/stages/text/modifiers/unicode/unicode_reformatter.py
index 13bcc9f3a8..c3cfb2458a 100644
--- a/nemo_curator/stages/text/modifiers/unicode_reformatter.py
+++ b/nemo_curator/stages/text/modifiers/unicode/unicode_reformatter.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/nemo_curator/stages/text/modules/__init__.py b/nemo_curator/stages/text/modules/__init__.py
index 724f6612a9..95471bf286 100644
--- a/nemo_curator/stages/text/modules/__init__.py
+++ b/nemo_curator/stages/text/modules/__init__.py
@@ -14,16 +14,10 @@
from .add_id import AddId
from .joiner import DocumentJoiner
-from .modifier import Modify
-from .score_filter import Filter, Score, ScoreFilter
from .splitter import DocumentSplitter
__all__ = [
"AddId",
"DocumentJoiner",
"DocumentSplitter",
- "Filter",
- "Modify",
- "Score",
- "ScoreFilter",
]
diff --git a/tests/config/test_run.py b/tests/config/test_run.py
index 84309fa2a7..a79b447214 100644
--- a/tests/config/test_run.py
+++ b/tests/config/test_run.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -147,16 +147,16 @@ def test_pipeline_with_parquet_writer_stage():
def test_pipeline_with_hydra_instantiated_stage():
- from nemo_curator.stages.text.filters.heuristic_filter import NonAlphaNumericFilter
- from nemo_curator.stages.text.modules.score_filter import ScoreFilter
+ from nemo_curator.stages.text.filters import ScoreFilter
+ from nemo_curator.stages.text.filters.heuristic import NonAlphaNumericFilter
cfg = OmegaConf.create(
{
"stages": [
{
- "_target_": "nemo_curator.stages.text.modules.score_filter.ScoreFilter",
+ "_target_": "nemo_curator.stages.text.filters.score_filter.ScoreFilter",
"filter_obj": {
- "_target_": "nemo_curator.stages.text.filters.heuristic_filter.NonAlphaNumericFilter",
+ "_target_": "nemo_curator.stages.text.filters.heuristic.string.NonAlphaNumericFilter",
"max_non_alpha_numeric_to_text_ratio": 0.25,
},
"text_field": "text",
@@ -179,8 +179,8 @@ def test_pipeline_with_hydra_instantiated_stage():
def test_pipeline_with_multiple_stages():
- from nemo_curator.stages.text.modifiers.url_remover import UrlRemover
- from nemo_curator.stages.text.modules.modifier import Modify
+ from nemo_curator.stages.text.modifiers import Modify
+ from nemo_curator.stages.text.modifiers.string import UrlRemover
cfg = OmegaConf.create(
{
@@ -193,8 +193,8 @@ def test_pipeline_with_multiple_stages():
"fields": None,
},
{
- "_target_": "nemo_curator.stages.text.modules.modifier.Modify",
- "modifier_fn": {"_target_": "nemo_curator.stages.text.modifiers.url_remover.UrlRemover"},
+ "_target_": "nemo_curator.stages.text.modifiers.modifier.Modify",
+ "modifier_fn": {"_target_": "nemo_curator.stages.text.modifiers.string.url_remover.UrlRemover"},
"input_fields": "text",
},
{
diff --git a/tests/stages/text/modules/test_filters.py b/tests/stages/text/modules/test_filters.py
index 9d2c1632b8..3ed3c96484 100644
--- a/tests/stages/text/modules/test_filters.py
+++ b/tests/stages/text/modules/test_filters.py
@@ -18,42 +18,45 @@
import pandas as pd
import pytest
-from nemo_curator.stages.text.filters import (
- AlphaFilter,
+from nemo_curator.stages.text.filters import DocumentFilter, Filter, Score, ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import (
BoilerPlateStringFilter,
BulletsFilter,
CommonEnglishWordsFilter,
- DocumentFilter,
EllipsisFilter,
- GeneralCommentToCodeFilter,
- HistogramFilter,
- HTMLBoilerplateFilter,
LongWordFilter,
MeanWordLengthFilter,
NonAlphaNumericFilter,
- NumberOfLinesOfCodeFilter,
NumbersFilter,
ParenthesesFilter,
- PerExtensionFilter,
PornographicUrlsFilter,
PunctuationFilter,
- PythonCommentToCodeFilter,
- RepeatedLinesByCharFilter,
- RepeatedLinesFilter,
- RepeatedParagraphsByCharFilter,
- RepeatedParagraphsFilter,
- RepeatingDuplicateNGramsFilter,
- RepeatingTopNGramsFilter,
SubstringFilter,
SymbolsToWordsFilter,
- TokenCountFilter,
UrlsFilter,
WhiteSpaceFilter,
WordCountFilter,
WordsWithoutAlphabetsFilter,
+)
+from nemo_curator.stages.text.filters.heuristic.code import (
+ AlphaFilter,
+ GeneralCommentToCodeFilter,
+ HTMLBoilerplateFilter,
+ NumberOfLinesOfCodeFilter,
+ PerExtensionFilter,
+ PythonCommentToCodeFilter,
XMLHeaderFilter,
)
-from nemo_curator.stages.text.modules import Filter, Score, ScoreFilter
+from nemo_curator.stages.text.filters.heuristic.repetition import (
+ RepeatedLinesByCharFilter,
+ RepeatedLinesFilter,
+ RepeatedParagraphsByCharFilter,
+ RepeatedParagraphsFilter,
+ RepeatingDuplicateNGramsFilter,
+ RepeatingTopNGramsFilter,
+)
+from nemo_curator.stages.text.filters.histogram import HistogramFilter
+from nemo_curator.stages.text.filters.token import TokenCountFilter
from nemo_curator.tasks import DocumentBatch
diff --git a/tests/stages/text/modules/test_modifiers.py b/tests/stages/text/modules/test_modifiers.py
index 124142f450..a0c685b56a 100644
--- a/tests/stages/text/modules/test_modifiers.py
+++ b/tests/stages/text/modules/test_modifiers.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -15,18 +15,17 @@
import pandas as pd
import pytest
-from nemo_curator.stages.text.modifiers import (
- DocumentModifier,
+from nemo_curator.stages.text.modifiers import DocumentModifier, Modify
+from nemo_curator.stages.text.modifiers.modifier import _normalize_input_fields, _normalize_output_fields
+from nemo_curator.stages.text.modifiers.string import (
LineRemover,
MarkdownRemover,
NewlineNormalizer,
QuotationRemover,
Slicer,
- UnicodeReformatter,
UrlRemover,
)
-from nemo_curator.stages.text.modules import Modify
-from nemo_curator.stages.text.modules.modifier import _normalize_input_fields, _normalize_output_fields
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
from nemo_curator.tasks import DocumentBatch
diff --git a/tutorials/synthetic/nemotron_cc/nemotron_cc_pipelines.py b/tutorials/synthetic/nemotron_cc/nemotron_cc_pipelines.py
index ea9d24e548..6607130b96 100644
--- a/tutorials/synthetic/nemotron_cc/nemotron_cc_pipelines.py
+++ b/tutorials/synthetic/nemotron_cc/nemotron_cc_pipelines.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -21,14 +21,17 @@
DiverseQAPostProcessingStage,
KnowledgeListPostProcessingStage,
)
-from nemo_curator.stages.text.filters.heuristic_filter import SubstringFilter, TokenCountFilter
-from nemo_curator.stages.text.modifiers.line_remover import LineRemover
-from nemo_curator.stages.text.modifiers.markdown_remover import MarkdownRemover
-from nemo_curator.stages.text.modifiers.quotation_remover import QuotationRemover
-from nemo_curator.stages.text.modifiers.slicer import Slicer
+from nemo_curator.stages.text.filters import Filter, ScoreFilter
+from nemo_curator.stages.text.filters.heuristic import SubstringFilter
+from nemo_curator.stages.text.filters.token import TokenCountFilter
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.string import (
+ LineRemover,
+ MarkdownRemover,
+ QuotationRemover,
+ Slicer,
+)
from nemo_curator.stages.text.modules.joiner import DocumentJoiner
-from nemo_curator.stages.text.modules.modifier import Modify
-from nemo_curator.stages.text.modules.score_filter import Filter, ScoreFilter
from nemo_curator.stages.text.modules.splitter import DocumentSplitter
diff --git a/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_high_quality_example_pipeline.py b/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_high_quality_example_pipeline.py
index 46d3df800c..8533c7ccb3 100644
--- a/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_high_quality_example_pipeline.py
+++ b/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_high_quality_example_pipeline.py
@@ -47,10 +47,10 @@
NEMOTRON_CC_DISTILL_SYSTEM_PROMPT,
NEMOTRON_CC_SYSTEM_PROMPT,
)
+from nemo_curator.stages.text.filters import Filter
from nemo_curator.stages.text.io.reader.parquet import ParquetReader
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
from nemo_curator.stages.text.io.writer.parquet import ParquetWriter
-from nemo_curator.stages.text.modules.score_filter import Filter
from nemo_curator.tasks.document import DocumentBatch
# Threshold used to bucket and filter input examples
@@ -261,32 +261,92 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915
input_tasks = None
if args.mock:
input_data = [
- { "text": "The Amazon rainforest contains an unparalleled diversity of plant and animal species. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 12 },
- { "text": "Isaac Newton formulated the laws of motion and universal gravitation. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 4 },
- { "text": "The Great Wall of China is a historic fortification built to protect ancient Chinese states. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 17 },
- { "text": "Mercury is the smallest planet in the Solar System and orbits closest to the Sun. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 1 },
- { "text": "The Parthenon is a classical Greek temple dedicated to the goddess Athena. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 9 },
- { "text": "Giraffes are the tallest living terrestrial animals, native to African savannas. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 6 },
- { "text": "Marie Curie made pioneering contributions to the study of radioactivity. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 14 },
- { "text": "The Pacific Ocean covers more area than all landmasses combined. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 3 },
- { "text": "The Rosetta Stone provided the key to deciphering ancient Egyptian hieroglyphs. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 18 },
- { "text": "The cheetah is capable of reaching speeds over 100 kilometers per hour. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 8 },
- { "text": "Mount Everest is the highest peak on Earth, located in the Himalayas. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 2 },
- { "text": "The Sahara Desert spans much of North Africa and is the largest hot desert in the world. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 5 },
- { "text": "Leonardo da Vinci was an influential artist and inventor during the Italian Renaissance. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 19 },
- { "text": "Photosynthesis enables plants to convert sunlight into chemical energy. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 7 },
- { "text": "The Taj Mahal is an iconic mausoleum built by Mughal emperor Shah Jahan. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 0 },
- { "text": "The human brain contains billions of neurons that communicate through electrical signals. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 11 },
- { "text": "The Roman Empire was one of the most powerful civilizations of the ancient world. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 10 },
- { "text": "The Hubble Space Telescope has captured detailed images of distant galaxies. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 15 },
- { "text": "The Eiffel Tower was constructed for the 1889 Exposition Universelle in Paris. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 4 },
- { "text": "Antarctica contains the vast majority of the Earth's freshwater ice. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.", "bucketed_results": 9 },
+ {
+ "text": "The Amazon rainforest contains an unparalleled diversity of plant and animal species. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 12,
+ },
+ {
+ "text": "Isaac Newton formulated the laws of motion and universal gravitation. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 4,
+ },
+ {
+ "text": "The Great Wall of China is a historic fortification built to protect ancient Chinese states. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 17,
+ },
+ {
+ "text": "Mercury is the smallest planet in the Solar System and orbits closest to the Sun. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 1,
+ },
+ {
+ "text": "The Parthenon is a classical Greek temple dedicated to the goddess Athena. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 9,
+ },
+ {
+ "text": "Giraffes are the tallest living terrestrial animals, native to African savannas. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 6,
+ },
+ {
+ "text": "Marie Curie made pioneering contributions to the study of radioactivity. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 14,
+ },
+ {
+ "text": "The Pacific Ocean covers more area than all landmasses combined. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 3,
+ },
+ {
+ "text": "The Rosetta Stone provided the key to deciphering ancient Egyptian hieroglyphs. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 18,
+ },
+ {
+ "text": "The cheetah is capable of reaching speeds over 100 kilometers per hour. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 8,
+ },
+ {
+ "text": "Mount Everest is the highest peak on Earth, located in the Himalayas. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 2,
+ },
+ {
+ "text": "The Sahara Desert spans much of North Africa and is the largest hot desert in the world. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 5,
+ },
+ {
+ "text": "Leonardo da Vinci was an influential artist and inventor during the Italian Renaissance. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 19,
+ },
+ {
+ "text": "Photosynthesis enables plants to convert sunlight into chemical energy. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 7,
+ },
+ {
+ "text": "The Taj Mahal is an iconic mausoleum built by Mughal emperor Shah Jahan. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 0,
+ },
+ {
+ "text": "The human brain contains billions of neurons that communicate through electrical signals. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 11,
+ },
+ {
+ "text": "The Roman Empire was one of the most powerful civilizations of the ancient world. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 10,
+ },
+ {
+ "text": "The Hubble Space Telescope has captured detailed images of distant galaxies. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 15,
+ },
+ {
+ "text": "The Eiffel Tower was constructed for the 1889 Exposition Universelle in Paris. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 4,
+ },
+ {
+ "text": "Antarctica contains the vast majority of the Earth's freshwater ice. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields. This topic is widely studied and holds significant relevance in scientific and historical contexts. It illustrates important principles, involves complex interactions, and helps researchers develop a deeper understanding of natural systems and cultural developments. Over many years, scholars, explorers, and scientists have contributed insights that enrich our collective knowledge, enabling future generations to continue studying and appreciating its broader importance across different fields.",
+ "bucketed_results": 9,
+ },
]
# Divide input_data into batches of `batch_size` each
# Simulate `num_input_tasks` input tasks
batch_size = 10
num_input_tasks = 100
- input_batches = [input_data[i:i + batch_size] for i in range(0, len(input_data), batch_size)]
+ input_batches = [input_data[i : i + batch_size] for i in range(0, len(input_data), batch_size)]
input_tasks = []
id_counter = 0
for i in range(num_input_tasks // len(input_batches)):
@@ -430,7 +490,6 @@ def main() -> None: # noqa: C901, PLR0912, PLR0915
print(pipeline.describe())
print("\n" + "=" * 50 + "\n")
-
# Create executor
executor = XennaExecutor()
diff --git a/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_low_quality_example_pipeline.py b/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_low_quality_example_pipeline.py
index 51ae958316..07c86d6c80 100644
--- a/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_low_quality_example_pipeline.py
+++ b/tutorials/synthetic/nemotron_cc/nemotron_cc_sdg_low_quality_example_pipeline.py
@@ -34,10 +34,10 @@
NEMOTRON_CC_SYSTEM_PROMPT,
WIKIPEDIA_REPHRASING_PROMPT_TEMPLATE,
)
+from nemo_curator.stages.text.filters import Filter
from nemo_curator.stages.text.io.reader.parquet import ParquetReader
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
from nemo_curator.stages.text.io.writer.parquet import ParquetWriter
-from nemo_curator.stages.text.modules.score_filter import Filter
from nemo_curator.tasks.document import DocumentBatch
# Threshold used to bucket and filter input examples
diff --git a/tutorials/synthetic/synthetic_data_generation_example.py b/tutorials/synthetic/synthetic_data_generation_example.py
index 579229530a..655df05bf7 100644
--- a/tutorials/synthetic/synthetic_data_generation_example.py
+++ b/tutorials/synthetic/synthetic_data_generation_example.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -37,9 +37,8 @@
from nemo_curator.models.client.openai_client import AsyncOpenAIClient
from nemo_curator.pipeline import Pipeline
from nemo_curator.stages.synthetic.qa_multilingual_synthetic import QAMultilingualSyntheticStage
-from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
+from nemo_curator.stages.text.filters import DocumentFilter, ScoreFilter
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
-from nemo_curator.stages.text.modules.score_filter import ScoreFilter
class BeginsWithLanguageFilter(DocumentFilter):
diff --git a/tutorials/text/llama-nemotron-data-curation/filters/heuristic_filters.py b/tutorials/text/llama-nemotron-data-curation/filters/heuristic_filters.py
index 2625b38784..68b99c13b0 100644
--- a/tutorials/text/llama-nemotron-data-curation/filters/heuristic_filters.py
+++ b/tutorials/text/llama-nemotron-data-curation/filters/heuristic_filters.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/tutorials/text/llama-nemotron-data-curation/main.py b/tutorials/text/llama-nemotron-data-curation/main.py
index be4421bb2f..a0d3725f7c 100644
--- a/tutorials/text/llama-nemotron-data-curation/main.py
+++ b/tutorials/text/llama-nemotron-data-curation/main.py
@@ -31,9 +31,9 @@
from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
+from nemo_curator.stages.text.filters import ScoreFilter
from nemo_curator.stages.text.io.reader.jsonl import JsonlReader
from nemo_curator.stages.text.io.writer.jsonl import JsonlWriter
-from nemo_curator.stages.text.modules import ScoreFilter
from nemo_curator.utils.file_utils import get_all_file_paths_under
diff --git a/tutorials/text/peft-curation/main.py b/tutorials/text/peft-curation/main.py
index a2d5d47ed6..ef31982bc4 100644
--- a/tutorials/text/peft-curation/main.py
+++ b/tutorials/text/peft-curation/main.py
@@ -28,9 +28,10 @@
from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
+from nemo_curator.stages.text.filters import ScoreFilter
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modifiers import UnicodeReformatter
-from nemo_curator.stages.text.modules import Modify, ScoreFilter
+from nemo_curator.stages.text.modifiers import Modify
+from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter
def main(args: argparse.Namespace) -> None:
diff --git a/tutorials/text/peft-curation/stages.py b/tutorials/text/peft-curation/stages.py
index 4a7a9283be..b6375f8599 100644
--- a/tutorials/text/peft-curation/stages.py
+++ b/tutorials/text/peft-curation/stages.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -28,8 +28,8 @@
DocumentIterator,
URLGenerator,
)
-from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
-from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier
+from nemo_curator.stages.text.filters import DocumentFilter
+from nemo_curator.stages.text.modifiers import DocumentModifier
class EnronEmailsURLGenerator(URLGenerator):
diff --git a/tutorials/text/tinystories/main.py b/tutorials/text/tinystories/main.py
index 4b63cb89c4..088371a4af 100644
--- a/tutorials/text/tinystories/main.py
+++ b/tutorials/text/tinystories/main.py
@@ -24,8 +24,9 @@
from nemo_curator.core.client import RayClient
from nemo_curator.pipeline import Pipeline
+from nemo_curator.stages.text.filters import ScoreFilter
from nemo_curator.stages.text.io.writer import JsonlWriter
-from nemo_curator.stages.text.modules import Modify, ScoreFilter
+from nemo_curator.stages.text.modifiers import Modify
def main(args: argparse.Namespace) -> None:
diff --git a/tutorials/text/tinystories/stages.py b/tutorials/text/tinystories/stages.py
index 1858ebb28d..c823061988 100644
--- a/tutorials/text/tinystories/stages.py
+++ b/tutorials/text/tinystories/stages.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -26,8 +26,8 @@
DocumentIterator,
URLGenerator,
)
-from nemo_curator.stages.text.filters.doc_filter import DocumentFilter
-from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier
+from nemo_curator.stages.text.filters import DocumentFilter
+from nemo_curator.stages.text.modifiers import DocumentModifier
class TinyStoriesURLGenerator(URLGenerator):