Skip to content
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions benchmarking/scripts/arxiv_e2e_pipeline_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@
from nemo_curator.stages.text.download.arxiv.extract import ArxivExtractor
from nemo_curator.stages.text.download.arxiv.iterator import ArxivIterator
from nemo_curator.stages.text.download.base import URLGenerator
from nemo_curator.stages.text.download.base.extract import DocumentExtractStage
from nemo_curator.stages.text.download.base.iterator import DocumentIterateStage
from nemo_curator.stages.text.download.base.stage import DocumentIterateExtractStage
from nemo_curator.stages.text.download.base.url_generation import URLGenerationStage
from nemo_curator.stages.text.filters import (
FastTextLangId,
Expand Down Expand Up @@ -119,20 +118,15 @@ def __post_init__(self) -> None:
limit=self.url_limit,
)

# Iterate stage (extracts records from tar files)
iterate_stage = DocumentIterateStage(
# Iterate-extract stage (extracts records from tar files and cleans LaTeX to text)
iterate_extract_stage = DocumentIterateExtractStage(
iterator=ArxivIterator(log_frequency=self.log_frequency),
record_limit=self.record_limit,
add_filename_column=self.add_filename_column,
)

# Extract stage (cleans LaTeX to text)
extract_stage = DocumentExtractStage(
extractor=ArxivExtractor(),
record_limit=self.record_limit,
add_filename_column=self.add_filename_column,
)

self.stages = [url_stage, iterate_stage, extract_stage]
self.stages = [url_stage, iterate_extract_stage]
self.name = "local_arxiv_extract"
super().__init__()

Expand Down
5 changes: 2 additions & 3 deletions docs/about/concepts/text/data-acquisition-concepts.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,11 @@ This guide covers the core concepts for acquiring and processing text data from

## Overview

Data acquisition in NeMo Curator follows a four-stage architecture:
Data acquisition in NeMo Curator follows a three-stage architecture:

1. **Generate URLs**: Discover and generate download URLs from minimal input
2. **Download**: Retrieve raw data files from remote sources
3. **Iterate**: Extract individual records from downloaded containers
4. **Extract**: Convert raw content to clean, structured text
3. **Iterate** and **Extract**: Extract individual records from downloaded containers and convert raw content to clean, structured text

This process transforms diverse remote data sources into a standardized `DocumentBatch` that can be used throughout the text curation pipeline.

Expand Down
2 changes: 1 addition & 1 deletion docs/curate-text/load-data/custom.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ Create custom data loading pipelines using Curator. This guide shows how to buil

## How It Works

Curator uses the same **4-step pipeline pattern** described in {ref}`Data Acquisition Concepts <about-concepts-text-data-acquisition>` for custom data loading. Each step uses an abstract base class with corresponding processing stages that compose into pipelines.
Curator uses the same **3-step pipeline pattern** described in {ref}`Data Acquisition Concepts <about-concepts-text-data-acquisition>` for custom data loading. Each step uses an abstract base class with corresponding processing stages that compose into pipelines.

---

Expand Down
5 changes: 2 additions & 3 deletions nemo_curator/stages/text/download/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@

## 📁 Structure Overview

The framework follows a **4-step pipeline pattern** where each step is implemented as an abstract base class with corresponding stages:
The framework follows a **3-step pipeline pattern** where each step is implemented as an abstract base class with corresponding stages:

```
1. URLGenerator → URLGenerationStage (URLs from config/input)
2. DocumentDownloader → DocumentDownloadStage (local files from URLs)
3. DocumentIterator → DocumentIterateStage (raw records from files)
4. DocumentExtractor → DocumentExtractStage (structured data from records)
3. DocumentIterator and DocumentExtractor → DocumentIterateExtractStage (structured data from files)
```

## 🛠️ Implementation Steps
Expand Down
74 changes: 1 addition & 73 deletions nemo_curator/stages/text/download/base/extract.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -13,16 +13,8 @@
# limitations under the License.

from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any

Comment thread
sarahyurick marked this conversation as resolved.
Resolved
import pandas as pd
from loguru import logger

from nemo_curator.stages.base import ProcessingStage
from nemo_curator.tasks import DocumentBatch
from nemo_curator.utils.column_utils import resolve_filename_column


class DocumentExtractor(ABC):
"""Abstract base class for document extractors.
Expand All @@ -45,67 +37,3 @@ def input_columns(self) -> list[str]:
def output_columns(self) -> list[str]:
"""Define output columns - produces DocumentBatch with records."""
...


@dataclass
class DocumentExtractStage(ProcessingStage[DocumentBatch, DocumentBatch]):
"""Stage that extracts structured content from raw records.

Takes DocumentBatch with raw content and produces DocumentBatch with extracted content.
This is for cases where iteration and extraction are separate steps.
"""

extractor: DocumentExtractor
add_filename_column: bool | str = True

def __post_init__(self):
"""Initialize the stage."""
self.filename_col = resolve_filename_column(self.add_filename_column)
self.name = f"extract_{self.extractor.__class__.__name__.lower()}"

def inputs(self) -> tuple[list[str], list[str]]:
"""Define input requirements - expects DocumentBatch with dict records."""
return (["data"], self.extractor.input_columns() + ([self.filename_col] if self.add_filename_column else [])) # type: ignore[reportReturnType]

def outputs(self) -> tuple[list[str], list[str]]:
"""Define output - produces DocumentBatch with processed records."""
return (["data"], self.extractor.output_columns() + ([self.filename_col] if self.add_filename_column else [])) # type: ignore[reportReturnType]

def process(self, task: DocumentBatch) -> DocumentBatch:
"""Extract structured content from raw records.

Args:
task (DocumentBatch): Batch containing records

Returns:
DocumentBatch: Batch containing extracted records
"""
extracted_records = []

for _, row in task.data.iterrows():
# Convert pandas Series to dict
record_dict = row.to_dict()

# Extract structured content
extracted = self.extractor.extract(record_dict)
if extracted is not None:
if self.add_filename_column:
if self.filename_col in extracted:
msg = f"Since add_filename_col is specified, we'll overwrite ({self.filename_col}) from the input data."
logger.warning(msg)

extracted[self.filename_col] = record_dict[self.filename_col] # type: ignore[reportReturnType]
extracted_records.append(extracted)

# Convert to DataFrame
df = pd.DataFrame(extracted_records)

return DocumentBatch(
task_id=task.task_id,
dataset_name=task.dataset_name,
data=df,
_metadata={
**task._metadata,
},
_stage_perf=task._stage_perf,
)
79 changes: 1 addition & 78 deletions nemo_curator/stages/text/download/base/iterator.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -12,19 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import os
from abc import ABC, abstractmethod
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Any
Comment thread
sarahyurick marked this conversation as resolved.
Resolved

import pandas as pd
from loguru import logger

from nemo_curator.stages.base import ProcessingStage
from nemo_curator.tasks import DocumentBatch, FileGroupTask
from nemo_curator.utils.column_utils import resolve_filename_column


class DocumentIterator(ABC):
"""Abstract base class for document iterators.
Expand All @@ -42,71 +33,3 @@ def iterate(self, file_path: str) -> Iterator[dict[str, Any]]:
def output_columns(self) -> list[str]:
"""Define output columns - produces DocumentBatch with records."""
...


@dataclass
class DocumentIterateStage(ProcessingStage[FileGroupTask, DocumentBatch]):
"""Stage that iterates through downloaded files and extracts records.

Takes local file paths and produces a DocumentBatch with records.
All iterators yield dict[str, str] records uniformly.
"""

iterator: DocumentIterator
record_limit: int | None = None
add_filename_column: bool | str = True

def __post_init__(self):
"""Initialize the stage."""
self.filename_col = resolve_filename_column(self.add_filename_column)
self.name = f"iterate_{self.iterator.__class__.__name__.lower()}"

def inputs(self) -> tuple[list[str], list[str]]:
"""Define input requirements - expects FileGroupTask with local file paths."""
return (["data"], [])

def outputs(self) -> tuple[list[str], list[str]]:
"""Define output - produces DocumentBatch with records."""
return (["data"], self.iterator.output_columns() + ([self.filename_col] if self.add_filename_column else [])) # type: ignore[reportReturnType]

def process(self, task: FileGroupTask) -> DocumentBatch:
"""Iterate through files and extract records.

Args:
task (FileGroupTask): Task containing local file paths

Returns:
DocumentBatch: Batch containing records
"""
records = []

for file_path in task.data:
try:
record_count = 0
iterator_result = self.iterator.iterate(file_path)
if iterator_result is not None:
for record_dict in iterator_result:
if self.record_limit and record_count >= self.record_limit:
break
if self.add_filename_column:
# TODO: Support cloud storage https://github.com/NVIDIA-NeMo/Curator/issues/779
record_dict[self.filename_col] = os.path.basename(file_path) # type: ignore[reportReturnType]
records.append(record_dict)
record_count += 1

except Exception as e: # noqa: BLE001
logger.error(f"Error iterating {file_path}: {e}")
continue

# Convert to DataFrame
df = pd.DataFrame(records)

return DocumentBatch(
task_id=task.task_id,
dataset_name=task.dataset_name,
data=df,
_metadata={
**task._metadata,
},
_stage_perf=task._stage_perf,
)
Loading
Loading