diff --git a/fern/versions/v26.04/pages/about/concepts/audio/text-integration.mdx b/fern/versions/v26.04/pages/about/concepts/audio/text-integration.mdx index 2a2025d29d..f22d57d9cc 100644 --- a/fern/versions/v26.04/pages/about/concepts/audio/text-integration.mdx +++ b/fern/versions/v26.04/pages/about/concepts/audio/text-integration.mdx @@ -265,8 +265,8 @@ NeMo Curator provides these audio quality assessment capabilities: from nemo_curator.pipeline import Pipeline from nemo_curator.stages.audio.inference.asr_nemo import InferenceAsrNemoStage from nemo_curator.stages.audio.io.convert import AudioToDocumentStage -from nemo_curator.stages.text.modules.score_filter import ScoreFilter -from nemo_curator.filters import WordCountFilter # Example filter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Example filter # Define a text quality filter text_quality_filter = WordCountFilter(min_words=10) diff --git a/fern/versions/v26.04/pages/about/concepts/text/data-processing-concepts.mdx b/fern/versions/v26.04/pages/about/concepts/text/data-processing-concepts.mdx index a3e635dfc7..2e7e69119b 100644 --- a/fern/versions/v26.04/pages/about/concepts/text/data-processing-concepts.mdx +++ b/fern/versions/v26.04/pages/about/concepts/text/data-processing-concepts.mdx @@ -86,7 +86,7 @@ from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter +from nemo_curator.stages.text.filters import ScoreFilter from nemo_curator.stages.text.filters import ( WordCountFilter, NonAlphaNumericFilter, @@ -167,8 +167,8 @@ from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import Modify -from nemo_curator.stages.text.modifiers import UnicodeReformatter +from nemo_curator.stages.text.modifiers import Modify +from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter # Start Ray client ray_client = RayClient() diff --git a/fern/versions/v26.04/pages/about/release-notes/index.mdx b/fern/versions/v26.04/pages/about/release-notes/index.mdx index ae1a9ec90d..4b1cb5149a 100644 --- a/fern/versions/v26.04/pages/about/release-notes/index.mdx +++ b/fern/versions/v26.04/pages/about/release-notes/index.mdx @@ -57,6 +57,16 @@ Upgraded Cosmos-Xenna from 0.1.2 to 0.2.0 with a simplified resource model and i - **Xenna-managed CUDA devices**: Xenna now manages CUDA device visibility directly, replacing the previous Ray-managed approach. - **Ray 2.54**: Updated Ray dependency to version 2.54 for compatibility with Cosmos-Xenna 0.2.0. +### Filter and Modifier Directory Reorganization (PR #1472) + +Reorganized the `DocumentFilter` and `DocumentModifier` directory structures to avoid eagerly importing heavy dependencies: + +- **Lazy imports**: Importing `DocumentFilter` or `DocumentModifier` no longer pulls in heavyweight dependencies like HuggingFace Transformers, fastText, or histogram libraries. +- **Grouped by dependency weight**: Filters are now organized into `heuristic/`, `token/`, `histogram/`, and `fasttext/` subdirectories. +- **Modifiers reorganized**: Modifiers are now grouped into `string/`, `unicode/`, and `fasttext/` subdirectories. +- **`ScoreFilter`/`Filter`/`Score` moved**: These stages moved from `stages.text.modules` to `stages.text.filters`. +- **`Modify` moved**: Moved from `stages.text.modules` to `stages.text.modifiers`. + ### Fused Document Iterate and Extract Stages (PR #1458) The data acquisition pipeline now uses a three-stage architecture instead of four, fusing the iterate and extract steps into a single `DocumentIterateExtractStage`. This reduces memory overhead and improves pipeline performance: diff --git a/fern/versions/v26.04/pages/curate-text/load-data/read-existing.mdx b/fern/versions/v26.04/pages/curate-text/load-data/read-existing.mdx index ef898a9069..45f1a4ce86 100644 --- a/fern/versions/v26.04/pages/curate-text/load-data/read-existing.mdx +++ b/fern/versions/v26.04/pages/curate-text/load-data/read-existing.mdx @@ -22,8 +22,8 @@ Use Curator's `JsonlReader` and `ParquetReader` to read existing datasets into a from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Initialize Ray client ray_client = RayClient() @@ -67,8 +67,8 @@ ray_client.stop() from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import ParquetReader -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Initialize Ray client ray_client = RayClient() diff --git a/fern/versions/v26.04/pages/curate-text/process-data/content-processing/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/content-processing/index.mdx index 50caef5712..709c4b2734 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/content-processing/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/content-processing/index.mdx @@ -57,8 +57,9 @@ from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer -from nemo_curator.stages.text.modules import Modify +from nemo_curator.stages.text.modifiers import Modify +from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer +from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter # Initialize Ray client ray_client = RayClient() diff --git a/fern/versions/v26.04/pages/curate-text/process-data/content-processing/text-cleaning.mdx b/fern/versions/v26.04/pages/curate-text/process-data/content-processing/text-cleaning.mdx index da3b3fad55..c2f668f1bb 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/content-processing/text-cleaning.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/content-processing/text-cleaning.mdx @@ -40,8 +40,9 @@ from nemo_curator.core.client import RayClient from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modifiers import UnicodeReformatter, UrlRemover, NewlineNormalizer -from nemo_curator.stages.text.modules import Modify +from nemo_curator.stages.text.modifiers import Modify +from nemo_curator.stages.text.modifiers.string import UrlRemover, NewlineNormalizer +from nemo_curator.stages.text.modifiers.unicode import UnicodeReformatter def main(): # Initialize Ray client @@ -85,7 +86,7 @@ You can create your own custom text cleaner by extending the `DocumentModifier` ```python import re -from nemo_curator.stages.text.modifiers.doc_modifier import DocumentModifier +from nemo_curator.stages.text.modifiers import DocumentModifier URL_REGEX = re.compile(r"https?://\S+|www\.\S+", flags=re.IGNORECASE) diff --git a/fern/versions/v26.04/pages/curate-text/process-data/language-management/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/language-management/index.mdx index 1216aee7c5..ce84c43f4c 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/language-management/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/language-management/index.mdx @@ -30,8 +30,8 @@ Language management in NeMo Curator typically follows this pattern using the Pip ```python from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import FastTextLangId +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.fasttext import FastTextLangId # 1) Build the pipeline pipeline = Pipeline(name="language_management") diff --git a/fern/versions/v26.04/pages/curate-text/process-data/language-management/language.mdx b/fern/versions/v26.04/pages/curate-text/process-data/language-management/language.mdx index a03d50c2bb..8a4714870a 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/language-management/language.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/language-management/language.mdx @@ -47,9 +47,9 @@ The following example demonstrates how to create a language identification pipel """Language identification using Curator.""" from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.filters import FastTextLangId +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.fasttext import FastTextLangId from nemo_curator.stages.text.io.reader import JsonlReader -from nemo_curator.stages.text.modules import ScoreFilter def create_language_identification_pipeline(data_dir: str) -> Pipeline: """Create a pipeline for language identification.""" diff --git a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/classifier.mdx b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/classifier.mdx index fb871dcbf8..e380c99a1a 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/classifier.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/classifier.mdx @@ -83,8 +83,8 @@ results = pipeline.run() from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import FastTextQualityFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter # Create pipeline with FastText filter (requires pre-trained model) pipeline = Pipeline(name="fasttext_quality_pipeline") @@ -119,7 +119,7 @@ You can configure quality classifiers and filters with different parameters: ```python from nemo_curator.stages.text.classifiers import QualityClassifier -from nemo_curator.stages.text.filters import FastTextQualityFilter +from nemo_curator.stages.text.filters.fasttext import FastTextQualityFilter # DeBERTa quality classifier configurations basic_deberta_classifier = QualityClassifier( diff --git a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/heuristic.mdx b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/heuristic.mdx index 979909c6a4..1eba38e531 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/heuristic.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/heuristic.mdx @@ -35,12 +35,9 @@ For details on filter structure and the filtering process, refer to [Data Proces from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import ( - WordCountFilter, - RepeatingTopNGramsFilter, - PunctuationFilter -) +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter, PunctuationFilter +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter # Create pipeline pipeline = Pipeline(name="heuristic_filtering") @@ -87,13 +84,13 @@ results = pipeline.run() ```python # Example configuration for common heuristic filters -from nemo_curator.stages.text.filters import ( +from nemo_curator.stages.text.filters.heuristic import ( WordCountFilter, PunctuationFilter, - RepeatingTopNGramsFilter, SymbolsToWordsFilter, CommonEnglishWordsFilter ) +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter # Define filter configurations filters_config = [ @@ -197,17 +194,17 @@ stages: file_paths: ${input_path} fields: null - - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter + - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter filter_obj: - _target_: nemo_curator.stages.text.filters.heuristic_filter.WordCountFilter + _target_: nemo_curator.stages.text.filters.heuristic.string.WordCountFilter min_words: 50 max_words: 100000 text_field: ${text_field} score_field: word_count - - _target_: nemo_curator.stages.text.modules.score_filter.ScoreFilter + - _target_: nemo_curator.stages.text.filters.score_filter.ScoreFilter filter_obj: - _target_: nemo_curator.stages.text.filters.heuristic_filter.PunctuationFilter + _target_: nemo_curator.stages.text.filters.heuristic.string.PunctuationFilter max_num_sentences_without_endmark_ratio: 0.85 text_field: ${text_field} score_field: null @@ -232,8 +229,9 @@ When building filter chains, follow these best practices: ```python # Efficient ordering - place fast filters first from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter, UrlsFilter, RepeatingTopNGramsFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter, UrlsFilter +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter pipeline = Pipeline(name="efficient_filtering") # Fast filters first @@ -262,8 +260,8 @@ strict_filter = WordCountFilter(min_words=100, max_words=10000) ```python # Chinese text filter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import SymbolsToWordsFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import SymbolsToWordsFilter cn_filter = ScoreFilter( filter_obj=SymbolsToWordsFilter(max_symbol_to_word_ratio=0.15, lang="zh"), @@ -276,13 +274,13 @@ cn_filter = ScoreFilter( ```python # Comprehensive quality filter pipeline from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import ( - WordCountFilter, - PunctuationFilter, - CommonEnglishWordsFilter, - RepeatingTopNGramsFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import ( + WordCountFilter, + PunctuationFilter, + CommonEnglishWordsFilter, ) +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter quality_pipeline = Pipeline(name="comprehensive_quality") @@ -325,8 +323,9 @@ Use `Score` to add score columns to your data without removing any documents: from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import Score -from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter +from nemo_curator.stages.text.filters import Score +from nemo_curator.stages.text.filters.heuristic import WordCountFilter +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter # Create scoring pipeline (no filtering) pipeline = Pipeline(name="score_analysis") @@ -403,8 +402,9 @@ After analyzing distributions, apply filters with your chosen thresholds: from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter, RepeatingTopNGramsFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter +from nemo_curator.stages.text.filters.heuristic.repetition import RepeatingTopNGramsFilter pipeline = Pipeline(name="filtering_pipeline") pipeline.add_stage(JsonlReader(file_paths="input_data/", fields=["text", "id"])) diff --git a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/index.mdx index 8205b3bc1c..fabac0d140 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/quality-assessment/index.mdx @@ -27,8 +27,8 @@ The `ScoreFilter` is at the center of filtering in NeMo Curator. It applies a fi from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Create pipeline pipeline = Pipeline(name="quality_filtering") @@ -78,7 +78,7 @@ For more specific use cases, NeMo Curator provides two specialized modules: ```python # Example: Score documents without filtering -from nemo_curator.stages.text.modules import Score +from nemo_curator.stages.text.filters import Score scoring_step = Score( WordCountFilter().score_document, # Use just the scoring part @@ -95,7 +95,7 @@ scored_dataset = scoring_step.process(dataset) ```python # Example: Filter using pre-computed scores -from nemo_curator.stages.text.modules import Filter +from nemo_curator.stages.text.filters import Filter filter_step = Filter( lambda score: score >= 100, # Keep documents with score >= 100 @@ -108,7 +108,7 @@ You can combine these modules in pipelines: ```python from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import Score, Filter +from nemo_curator.stages.text.filters import Score, Filter # Assume `word_counter` and `symbol_counter` are callables that return numeric scores pipeline = Pipeline(name="multi_stage_filtering") pipeline.add_stage(Score(word_counter, score_field="word_count")) @@ -158,8 +158,8 @@ NeMo Curator provides programmatic interfaces for document filtering through the from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter # Create and configure pipeline pipeline = Pipeline(name="document_filtering") diff --git a/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/code.mdx b/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/code.mdx index c67a9725c2..de72cec10d 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/code.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/code.mdx @@ -27,8 +27,8 @@ Here's an example of applying code filters to a dataset: from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import ( +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic.code import ( PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter, AlphaFilter @@ -197,8 +197,8 @@ When filtering code datasets, consider these best practices: 1. **Language-specific configurations**: Adjust thresholds based on the programming language ```python - from nemo_curator.stages.text.modules import ScoreFilter - from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, GeneralCommentToCodeFilter + from nemo_curator.stages.text.filters import ScoreFilter + from nemo_curator.stages.text.filters.heuristic.code import PythonCommentToCodeFilter, GeneralCommentToCodeFilter # Python tends to have more comments than C python_comment_filter = ScoreFilter( @@ -214,8 +214,8 @@ When filtering code datasets, consider these best practices: 2. **Preserve code structure**: Ensure filters don't inadvertently remove valid coding patterns ```python - from nemo_curator.stages.text.modules import ScoreFilter - from nemo_curator.stages.text.filters import GeneralCommentToCodeFilter + from nemo_curator.stages.text.filters import ScoreFilter + from nemo_curator.stages.text.filters.heuristic.code import GeneralCommentToCodeFilter # Some languages naturally have low comment ratios assembly_filter = ScoreFilter( @@ -231,9 +231,9 @@ When filtering code datasets, consider these best practices: ```python # First check if the content is actually Python using FastText language ID - from nemo_curator.stages.text.filters import FastTextLangId + from nemo_curator.stages.text.filters.fasttext import FastTextLangId from nemo_curator.pipeline import Pipeline - from nemo_curator.stages.text.modules import ScoreFilter + from nemo_curator.stages.text.filters import ScoreFilter # Create pipeline for Python code filtering with language detection pipeline = Pipeline(name="python_code_filtering") @@ -286,8 +286,8 @@ When filtering code datasets, consider these best practices: ```python from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import NumberOfLinesOfCodeFilter, XMLHeaderFilter, GeneralCommentToCodeFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic.code import NumberOfLinesOfCodeFilter, XMLHeaderFilter, GeneralCommentToCodeFilter # Create pipeline to filter non-functional code snippets pipeline = Pipeline(name="code_cleaning") @@ -317,8 +317,8 @@ pipeline.add_stage(ScoreFilter( ```python from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import AlphaFilter, TokenizerFertilityFilter, HTMLBoilerplateFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic.code import AlphaFilter, TokenizerFertilityFilter, HTMLBoilerplateFilter # Create pipeline for training data preparation pipeline = Pipeline(name="training_data_prep") diff --git a/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/index.mdx b/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/index.mdx index 28644b1e0a..6564ebed70 100644 --- a/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/process-data/specialized-processing/index.mdx @@ -47,8 +47,8 @@ languages ```python from nemo_curator.pipeline import Pipeline -from nemo_curator.stages.text.modules import ScoreFilter -from nemo_curator.stages.text.filters import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic.code import PythonCommentToCodeFilter, NumberOfLinesOfCodeFilter from nemo_curator.stages.text.io.reader import JsonlReader # Filter Python code based on quality metrics diff --git a/fern/versions/v26.04/pages/curate-text/synthetic/multilingual-qa.mdx b/fern/versions/v26.04/pages/curate-text/synthetic/multilingual-qa.mdx index 903c79b53e..d209972663 100644 --- a/fern/versions/v26.04/pages/curate-text/synthetic/multilingual-qa.mdx +++ b/fern/versions/v26.04/pages/curate-text/synthetic/multilingual-qa.mdx @@ -145,8 +145,8 @@ pipeline.add_stage( If your prompt includes language prefixes, you can filter to keep only specific languages: ```python -from nemo_curator.stages.text.filters.doc_filter import DocumentFilter -from nemo_curator.stages.text.modules.score_filter import ScoreFilter +from nemo_curator.stages.text.filters import DocumentFilter +from nemo_curator.stages.text.filters import ScoreFilter class BeginsWithLanguageFilter(DocumentFilter): """Filter documents based on language prefix codes.""" diff --git a/fern/versions/v26.04/pages/curate-text/synthetic/nemotron-cc/index.mdx b/fern/versions/v26.04/pages/curate-text/synthetic/nemotron-cc/index.mdx index 148ac3dac7..695cbd2e17 100644 --- a/fern/versions/v26.04/pages/curate-text/synthetic/nemotron-cc/index.mdx +++ b/fern/versions/v26.04/pages/curate-text/synthetic/nemotron-cc/index.mdx @@ -156,7 +156,7 @@ For documents with high quality scores, use tasks that leverage the existing qua - **KnowledgeList**: Extract structured facts ```python -from nemo_curator.stages.text.modules.score_filter import Filter +from nemo_curator.stages.text.filters import Filter # Filter for high-quality documents (score >11) pipeline.add_stage( diff --git a/fern/versions/v26.04/pages/get-started/text.mdx b/fern/versions/v26.04/pages/get-started/text.mdx index e41277bda3..03c29b7ee9 100644 --- a/fern/versions/v26.04/pages/get-started/text.mdx +++ b/fern/versions/v26.04/pages/get-started/text.mdx @@ -125,8 +125,8 @@ Here's a simple example to get started with NeMo Curator's pipeline-based archit from nemo_curator.pipeline import Pipeline from nemo_curator.stages.text.io.reader import JsonlReader from nemo_curator.stages.text.io.writer import JsonlWriter -from nemo_curator.stages.text.modules.score_filter import ScoreFilter -from nemo_curator.stages.text.filters import WordCountFilter, NonAlphaNumericFilter +from nemo_curator.stages.text.filters import ScoreFilter +from nemo_curator.stages.text.filters.heuristic import WordCountFilter, NonAlphaNumericFilter # Create a pipeline for text curation pipeline = Pipeline(