diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index 467e87c7c9..3dbb1514d9 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -62,7 +62,7 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.10", "3.12"] - folder: ["backends", "config", "core", "models", "pipelines", "stages-audio", "stages-common", "stages-deduplication", "stages-image", "stages-synthetic", "stages-text", "stages-video", "tasks", "utils"] + folder: ["backends", "config", "core", "models", "pipelines", "stages-audio", "stages-common", "stages-deduplication", "stages-image", "stages-math_stages", "stages-synthetic", "stages-text", "stages-video", "tasks", "utils"] needs: [pre-flight, cicd-wait-in-queue] runs-on: ${{ matrix.os }} name: Unit_Test_${{ matrix.folder}}_CPU_python-${{ matrix.python-version }} diff --git a/nemo_curator/models/vllm_model.py b/nemo_curator/models/vllm_model.py new file mode 100644 index 0000000000..c7186830d9 --- /dev/null +++ b/nemo_curator/models/vllm_model.py @@ -0,0 +1,206 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any + +from loguru import logger + +from nemo_curator.models.base import ModelInterface +from nemo_curator.utils.gpu_utils import get_gpu_count, get_max_model_len_from_config + +try: + from vllm import LLM, SamplingParams + + VLLM_AVAILABLE = True +except ImportError: + VLLM_AVAILABLE = False + + class LLM: + pass + + class SamplingParams: + pass + + +class VLLMModel(ModelInterface): + """Generic vLLM language model wrapper for text generation.""" + + def __init__( # noqa: PLR0913 + self, + model: str, + max_model_len: int | None = None, + tensor_parallel_size: int | None = None, + max_num_batched_tokens: int = 4096, + temperature: float = 0.7, + top_p: float = 0.8, + top_k: int = 20, + min_p: float = 0.0, + max_tokens: int | None = None, + cache_dir: str | None = None, + ): + """ + Initialize the vLLM model wrapper. + + Args: + model: Model identifier (e.g., "microsoft/phi-4") + max_model_len: Maximum model context length. If not specified, + will be auto-detected from HuggingFace AutoConfig. + tensor_parallel_size: Number of GPUs for tensor parallelism. + If not specified, auto-detects available GPUs. + max_num_batched_tokens: Maximum tokens per batch. Defaults to + 4096. + temperature: Sampling temperature. Defaults to 0.7. + top_p: Top-p sampling parameter. Defaults to 0.8. + top_k: Top-k sampling parameter. Defaults to 20. + min_p: Min-p sampling parameter (for Qwen3). Defaults to 0.0. + max_tokens: Maximum tokens to generate. Defaults to None. + cache_dir: Cache directory for model weights. Defaults to None. + """ + self.model = model + self.max_model_len = max_model_len + self.tensor_parallel_size = tensor_parallel_size + self.max_num_batched_tokens = max_num_batched_tokens + self.temperature = temperature + self.top_p = top_p + self.top_k = top_k + self.min_p = min_p + self.max_tokens = max_tokens + self.cache_dir = cache_dir + self._llm: LLM | None = None + self._sampling_params: SamplingParams | None = None + self._final_max_model_len: int | None = None + self._is_qwen3: bool = False + + @property + def model_id_names(self) -> list[str]: + """Return the model identifier.""" + return [self.model] + + def setup(self) -> None: + """Set up the vLLM model and sampling parameters.""" + if not VLLM_AVAILABLE: + msg = ( + "vLLM is required for VLLMModel. " + "Please install it: pip install vllm" + ) + raise ImportError(msg) + + # Fetch max_model_len from user param or auto-detect from HuggingFace AutoConfig + if self.max_model_len is not None: + final_max_model_len = self.max_model_len + else: + final_max_model_len = get_max_model_len_from_config(self.model, cache_dir=self.cache_dir) + + # Set tensor_parallel_size as user param or auto-detect from GPU count + final_tp_size = self.tensor_parallel_size if self.tensor_parallel_size is not None else get_gpu_count() + + # Set max_num_batched_tokens as user param or use default + final_max_batched = self.max_num_batched_tokens + + llm_kwargs: dict[str, Any] = { + "model": self.model, + "enforce_eager": False, + "trust_remote_code": True, + "tensor_parallel_size": final_tp_size, + "max_num_batched_tokens": final_max_batched, + } + + if final_max_model_len is not None: + llm_kwargs["max_model_len"] = final_max_model_len + + if self.cache_dir is not None: + llm_kwargs["download_dir"] = self.cache_dir + + logger.info( + f"Initializing vLLM with: model={self.model}, " + f"max_model_len={final_max_model_len}, " + f"tensor_parallel_size={final_tp_size}, " + f"max_num_batched_tokens={final_max_batched}" + ) + + self._llm = LLM(**llm_kwargs) + self._final_max_model_len = final_max_model_len + + max_gen_tokens = ( + self.max_tokens + if self.max_tokens is not None + else final_max_model_len + ) + if max_gen_tokens is None: + logger.warning( + "max_tokens is None and max_model_len could not be auto-detected. " + "vLLM will use its default (typically 16 tokens), which may be too few." + ) + is_qwen3 = "Qwen3" in self.model or "qwen3" in self.model.lower() + + sampling_kwargs: dict[str, Any] = { + "temperature": self.temperature, + "max_tokens": max_gen_tokens, + } + + if is_qwen3: + sampling_kwargs.update( + { + "top_p": self.top_p, + "top_k": self.top_k, + "min_p": self.min_p, + } + ) + else: + sampling_kwargs["top_p"] = self.top_p + + self._sampling_params = SamplingParams(**sampling_kwargs) + self._is_qwen3 = is_qwen3 + + def generate( + self, + prompts: list[str], + ) -> list[str]: + """ + Generate text from prompts. + + Args: + prompts: List of prompt strings or list of message dicts + (for chat template). + + Returns: + List of generated text strings. + + Raises: + RuntimeError: If the model is not set up or generation fails. + """ + if self._llm is None or self._sampling_params is None: + msg = "Model not initialized. Call setup() first." + raise RuntimeError(msg) + + try: + outputs = self._llm.generate( + prompts, + sampling_params=self._sampling_params, + use_tqdm=False, + ) + return [ + out.outputs[0].text if out.outputs else "" + for out in outputs + ] + except (RuntimeError, ValueError, TypeError) as e: + msg = f"Error generating text: {e}" + raise RuntimeError(msg) from e + + def get_tokenizer(self) -> Any: # noqa: ANN401 + """Get the tokenizer from the LLM instance.""" + if self._llm is None: + msg = "Model not initialized. Call setup() first." + raise RuntimeError(msg) + return self._llm.get_tokenizer() diff --git a/nemo_curator/stages/math/__init__.py b/nemo_curator/stages/math/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nemo_curator/stages/math/classifiers/__init__.py b/nemo_curator/stages/math/classifiers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nemo_curator/stages/math/classifiers/finemath.py b/nemo_curator/stages/math/classifiers/finemath.py new file mode 100644 index 0000000000..75269b99a6 --- /dev/null +++ b/nemo_curator/stages/math/classifiers/finemath.py @@ -0,0 +1,204 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import torch +from transformers import AutoModelForSequenceClassification + +from nemo_curator.stages.base import CompositeStage, ProcessingStage +from nemo_curator.stages.text.classifiers.constants import ( + DEBERTA_TOKENIZER_PADDING_SIDE, +) +from nemo_curator.stages.text.models.model import ModelStage +from nemo_curator.stages.text.models.tokenizer import TokenizerStage +from nemo_curator.stages.text.models.utils import ( + ATTENTION_MASK_FIELD, + INPUT_ID_FIELD, + format_name_with_suffix, +) +from nemo_curator.tasks import DocumentBatch + +FINEMATH_MODEL_ID = "HuggingFaceTB/finemath-classifier" +MAX_SEQ_LENGTH = 512 + + +class CenterCropTextStage(ProcessingStage[DocumentBatch, DocumentBatch]): + """ + Pre-tokenization stage that center-crops the text field to a fixed number + of characters to keep central context. + """ + + def __init__(self, text_field: str = "text", center_crop_chars: int = 10_000): + self.text_field = text_field + self.center_crop_chars = max(0, int(center_crop_chars)) + self.name = format_name_with_suffix(FINEMATH_MODEL_ID, suffix="_center_crop") + + def inputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.text_field] + + def outputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.text_field] + + @staticmethod + def _mid_slice(s: str, n: int) -> str: + m = len(s) // 2 + b, e = max(0, m - n), min(m + n, len(s)) + return s[b:e] + + def process(self, batch: DocumentBatch) -> DocumentBatch: + df = batch.to_pandas() + if self.text_field in df.columns and self.center_crop_chars > 0: + df[self.text_field] = ( + df[self.text_field].astype(str).map(lambda t: self._mid_slice(t, self.center_crop_chars)) + ) + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=df, + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) + + +class FineMathModelStage(ModelStage): + """ + Hugging Face sequence classification model stage for FineMath. + + Outputs columns: + - finemath_scores (float list) + - finemath_int_scores (int list) + """ + + def __init__( # noqa: PLR0913 + self, + model_identifier: str, + cache_dir: str | None = None, + float_score_column: str = "finemath_scores", + int_score_column: str = "finemath_int_scores", + model_inference_batch_size: int = 256, + has_seq_order: bool = True, + autocast: bool = True, + ): + super().__init__( + model_identifier=model_identifier, + cache_dir=cache_dir, + has_seq_order=has_seq_order, + model_inference_batch_size=model_inference_batch_size, + padding_side=DEBERTA_TOKENIZER_PADDING_SIDE, + unpack_inference_batch=True, + ) + self.float_score_column = float_score_column + self.int_score_column = int_score_column + self.autocast = autocast + + def outputs(self) -> tuple[list[str], list[str]]: + return ( + ["data"], + [self.float_score_column, self.int_score_column], + ) + + @staticmethod + def _configure_forward(model: torch.nn.Module) -> torch.nn.Module: + original_forward = model.forward + + @torch.no_grad() + def custom_forward(*args, **kwargs) -> torch.Tensor: + # autocast is handled by parent ModelStage.process() + output = original_forward(*args, **kwargs) + return output.logits.squeeze(-1).float() + + model.forward = custom_forward + return model + + def _setup(self, local_files_only: bool = True) -> None: + model = AutoModelForSequenceClassification.from_pretrained( + self.model_identifier, + cache_dir=self.cache_dir, + local_files_only=local_files_only, + ).cuda() + self.model = self._configure_forward(model) + + def process_model_output( + self, outputs: torch.Tensor, _: dict[str, torch.Tensor] | None = None + ) -> dict[str, np.ndarray]: + logits = outputs.cpu().numpy() + float_scores = np.clip(logits, 0.0, 5.0) + int_scores = np.round(float_scores).astype(int) + return { + self.float_score_column: float_scores, + self.int_score_column: int_scores, + } + + def create_output_dataframe(self, df_cpu: pd.DataFrame, collected_output: dict[str, np.ndarray]) -> pd.DataFrame: + df_cpu = df_cpu.drop(columns=[INPUT_ID_FIELD, ATTENTION_MASK_FIELD]) + df_cpu[self.float_score_column] = collected_output[self.float_score_column] + df_cpu[self.int_score_column] = collected_output[self.int_score_column] + return df_cpu + + +@dataclass(kw_only=True) +class FineMathClassifier(CompositeStage[DocumentBatch, DocumentBatch]): + """ + FineMath composite: TokenizerStage -> FineMathModelStage. + """ + + cache_dir: str | None = None + float_score_column: str = "finemath_scores" + int_score_column: str = "finemath_int_scores" + text_field: str = "text" + max_chars: int | None = None + max_seq_length: int = MAX_SEQ_LENGTH + sort_by_length: bool = False + model_inference_batch_size: int = 1024 + autocast: bool = True + center_crop_chars: int | None = 10_000 + + def __post_init__(self) -> None: + super().__init__() + stages: list[ProcessingStage] = [] + + if self.center_crop_chars is not None and self.center_crop_chars > 0: + stages.append(CenterCropTextStage(text_field=self.text_field, center_crop_chars=self.center_crop_chars)) + + stages.extend( + [ + TokenizerStage( + model_identifier=FINEMATH_MODEL_ID, + cache_dir=self.cache_dir, + text_field=self.text_field, + max_chars=self.max_chars, + max_seq_length=self.max_seq_length, + padding_side=DEBERTA_TOKENIZER_PADDING_SIDE, + sort_by_length=self.sort_by_length, + ), + FineMathModelStage( + model_identifier=FINEMATH_MODEL_ID, + cache_dir=self.cache_dir, + float_score_column=self.float_score_column, + int_score_column=self.int_score_column, + model_inference_batch_size=self.model_inference_batch_size, + has_seq_order=self.sort_by_length, + autocast=self.autocast, + ), + ] + ) + self.stages = stages + self.name = format_name_with_suffix(FINEMATH_MODEL_ID) + + def decompose(self) -> list[ProcessingStage]: + return self.stages diff --git a/nemo_curator/stages/math/download/__init__.py b/nemo_curator/stages/math/download/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nemo_curator/stages/math/download/extract.py b/nemo_curator/stages/math/download/extract.py new file mode 100644 index 0000000000..d8d0cb901e --- /dev/null +++ b/nemo_curator/stages/math/download/extract.py @@ -0,0 +1,258 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import threading +from dataclasses import dataclass, field +from typing import Any + +import magic +import pandas as pd +from loguru import logger +from resiliparse.parse.encoding import bytes_to_str, detect_encoding + +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.stages.text.download.base.extract import DocumentExtractor +from nemo_curator.tasks import DocumentBatch +from nemo_curator.utils.column_utils import resolve_filename_column + +from .html_extractors.lynx import LynxExtractor +from .mime_types import HTML_MAGIC_TYPES, HTML_MIME_TYPES, TEXT_MAGIC_TYPES, TEXT_MIME_TYPES + + +def _remove_xml_encoding_declaration(text: str) -> str: + return re.sub(r"^\s*<\?xml.*\?>", "", text) + + +def _decode_bytes(binary_content: bytes | None) -> str | None: + if binary_content is None: + return None + try: + content = bytes_to_str(binary_content, "utf-8") + except (UnicodeDecodeError, UnicodeError, LookupError): + encoding = detect_encoding(binary_content) + if encoding is None or encoding == "utf-8": + return None + try: + content = bytes_to_str(binary_content, encoding) + except (UnicodeDecodeError, UnicodeError, LookupError): + return None + return _remove_xml_encoding_declaration(content) + + +def _is_notebook(content: str) -> bool: + try: + data = json.loads(content) + return ( + isinstance(data, dict) + and "nbformat" in data + and "nbformat_minor" in data + and "cells" in data + and isinstance(data["cells"], list) + ) + except (json.JSONDecodeError, TypeError, ValueError): + return False + + +def _notebook_to_text(content: str) -> str: + data = json.loads(content) + out = "" + for cell in data.get("cells", []): + t = cell.get("cell_type") + if t in ["code", "markdown", "raw"]: + out += "".join(cell.get("source", [])) + if t == "code" and "outputs" in cell: + for o in cell["outputs"]: + if o.get("output_type") == "stream": + out += "".join(o.get("text", [])) + elif o.get("output_type") in ["execute_result", "display_data"]: + d = o.get("data", {}) + if "text/plain" in d: + out += "".join(d["text/plain"]) + elif o.get("output_type") == "text": + out += "".join(o.get("text", [])) + return out + + +@dataclass +class MathContentExtractor(DocumentExtractor): + """Extractor that decodes bytes, detects type, and extracts text using Lynx for HTML.""" + + binary_column: str = "binary_content" + url_column: str = "url" + mime_type_column: str = "mime_type" + lynx_timeout_sec: int = 20 + + # Lazily-initialized, avoid unpickleable objects during deepcopy in with_() + _lynx: Any | None = field(default=None, init=False, repr=False) + _magic: Any | None = field(default=None, init=False, repr=False) + _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) + + def __post_init__(self): + self._lynx = None + self._magic = None + self._lock = threading.Lock() + + def input_columns(self) -> list[str]: + return [self.binary_column, self.url_column, self.mime_type_column] + + def output_columns(self) -> list[str]: + return ["text", self.url_column, "type", "magic_mime_type"] + + def extract(self, record: dict[str, Any]) -> dict[str, Any] | None: + binary = record.get(self.binary_column) + url = record.get(self.url_column) + mime_type = record.get(self.mime_type_column) + + # Compute magic mime from bytes if available (lazy init) + magic_mime_type = None + if isinstance(binary, (bytes, bytearray)): + try: + if self._magic is None: + with self._lock: + if self._magic is None: + self._magic = magic.Magic(mime=True) + magic_mime_type = self._magic.from_buffer(binary) + except Exception as e: # noqa: BLE001 + logger.debug(f"Magic MIME detection failed: {e}") + magic_mime_type = None + + content = _decode_bytes(binary if isinstance(binary, (bytes, bytearray)) else None) + if not content: + return None + + doc_type = self._determine_type(content, magic_mime_type, mime_type, url) + + if doc_type == "notebook": + return { + "text": _notebook_to_text(content), + self.url_column: url, + "type": doc_type, + "magic_mime_type": magic_mime_type, + } + if doc_type == "html": + # lazy init lynx extractor + if self._lynx is None: + with self._lock: + if self._lynx is None: + self._lynx = LynxExtractor(timeout_sec=self.lynx_timeout_sec) + return { + "text": self._lynx.extract_text(content), + self.url_column: url, + "type": doc_type, + "magic_mime_type": magic_mime_type, + } + return { + "text": content, + self.url_column: url, + "type": doc_type, + "magic_mime_type": magic_mime_type, + } + + def _is_html_document(self, text: str) -> bool: + has_html_open = re.search(r"]*>", text, re.IGNORECASE) + has_html_close = re.search(r"", text, re.IGNORECASE) + has_head_open = re.search(r"]*>", text, re.IGNORECASE) + has_head_close = re.search(r"", text, re.IGNORECASE) + has_body_open = re.search(r"]*>", text, re.IGNORECASE) + has_body_close = re.search(r"", text, re.IGNORECASE) + return all([has_html_open, has_head_open, has_body_open, has_head_close, has_html_close, has_body_close]) + + def _determine_type( + self, content: str | None, magic_mime_type: str | None, mime_type: str | None, url: str | None + ) -> str: + if not content: + return "text" + + # Notebook takes precedence + if self._is_notebook_type(content, magic_mime_type, url): + return "notebook" + + result: str | None = None + + if magic_mime_type is None: + if mime_type in TEXT_MIME_TYPES: + result = "text" + elif mime_type in HTML_MIME_TYPES or self._is_html_document(content): + result = "html" + else: + result = "html" + elif magic_mime_type in HTML_MAGIC_TYPES or (mime_type and mime_type in HTML_MIME_TYPES): + result = "html" + elif mime_type in TEXT_MIME_TYPES or magic_mime_type in TEXT_MAGIC_TYPES: + result = "text" + else: + result = "html" + + return result or "html" + + def _is_notebook_type(self, content: str, magic_mime_type: str | None, url: str | None) -> bool: + """Check if content is a Jupyter notebook.""" + try: + return ((magic_mime_type == "application/json") or (url and url.endswith(".ipynb"))) and _is_notebook( + content + ) + except (TypeError, AttributeError, ValueError): + return False + + +@dataclass +class MathExtractStage(ProcessingStage[DocumentBatch, DocumentBatch]): + """Processing stage that applies a DocumentExtractor row-by-row to a DocumentBatch. + + Designed for use after CommonCrawlWARCReader, where binary content has already + been fetched into a DocumentBatch. Each row is passed to the extractor and rows + where extraction returns None are filtered out. + """ + + extractor: DocumentExtractor + add_filename_column: bool | str = False + + def __post_init__(self) -> None: + self.filename_col = resolve_filename_column(self.add_filename_column) + self.name = f"extract_{self.extractor.__class__.__name__.lower()}" + + def inputs(self) -> tuple[list[str], list[str]]: + return ["data"], self.extractor.input_columns() + + def outputs(self) -> tuple[list[str], list[str]]: + cols = self.extractor.output_columns() + if self.filename_col: + cols = [*cols, self.filename_col] + return ["data"], cols + + def process(self, batch: DocumentBatch) -> DocumentBatch: + df = batch.to_pandas() + records = [] + for _, row in df.iterrows(): + row_dict = row.to_dict() + extracted = self.extractor.extract(row_dict) + if extracted is None: + continue + if self.filename_col and self.filename_col in row_dict: + extracted[self.filename_col] = row_dict[self.filename_col] + records.append(extracted) + + output_cols = self.extractor.output_columns() + if self.filename_col: + output_cols = [*output_cols, self.filename_col] + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=pd.DataFrame(records) if records else pd.DataFrame(columns=output_cols), + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) diff --git a/nemo_curator/stages/math/download/html_extractors/__init__.py b/nemo_curator/stages/math/download/html_extractors/__init__.py new file mode 100644 index 0000000000..6dc801df83 --- /dev/null +++ b/nemo_curator/stages/math/download/html_extractors/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .lynx import LynxExtractor + +__all__ = [ + "LynxExtractor", +] diff --git a/nemo_curator/stages/math/download/html_extractors/lynx.py b/nemo_curator/stages/math/download/html_extractors/lynx.py new file mode 100644 index 0000000000..a97588d3b0 --- /dev/null +++ b/nemo_curator/stages/math/download/html_extractors/lynx.py @@ -0,0 +1,68 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import shutil +import subprocess + +import ftfy + + +class LynxExtractor: + """Extract text from HTML using the lynx command-line browser.""" + + def __init__(self, timeout_sec: int = 20): + self.timeout_sec = timeout_sec + # Validate lynx executable exists at initialization + lynx_path = shutil.which("lynx") + if not lynx_path: + error_msg = "lynx executable not found in PATH" + raise RuntimeError(error_msg) + + def extract_text(self, html: str) -> str: + """Extract text from HTML content. + + Returns empty string on any failure (timeout, encoding errors, etc). + """ + if not html: + return "" + + try: + proc = subprocess.run( # noqa: UP022 + [ # noqa: S607 + "lynx", + "-dump", + "-stdin", + "-nolist", + "-width=10000", + "-assume_charset=utf-8", + "-display_charset=utf-8", + "-localhost", + "-force_html", + ], + input=html.encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=self.timeout_sec, + ) + except (subprocess.TimeoutExpired, subprocess.SubprocessError, OSError, UnicodeEncodeError): + return "" + + if proc.returncode == 0: + try: + return proc.stdout.decode("utf-8") + except (UnicodeDecodeError, UnicodeError): + return ftfy.fix_text(proc.stdout.decode("utf-8", errors="replace")) + + return "" diff --git a/nemo_curator/stages/math/download/mime_types.py b/nemo_curator/stages/math/download/mime_types.py new file mode 100644 index 0000000000..c120428624 --- /dev/null +++ b/nemo_curator/stages/math/download/mime_types.py @@ -0,0 +1,110 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MIME types from HTTP headers that indicate text content (no HTML extraction needed) +TEXT_MIME_TYPES: set[str] = { + "text/x-web-markdown", + "text/x-verilog", + "text/x-rst", + "text/x-ruby", + "text/x-rsrc", + "text/x-python", + "text/x-perl", + "text/x-pascal", + "text/x-objcsrc", + "text/x-ml", + "text/x-matlab", + "text/x-log", + "text/x-haskell", + "text/x-fortran", + "text/x-expect", + "text/x-diff", + "text/x-csrc", + "text/x-common-lisp", + "text/x-chdr", + "text/x-cgi", + "text/x-c++src", + "text/x-basic", + "text/vtt", + "text/x-assembly", + "text/troff", + "text/plain", + "message/rfc822", + "message/news", + "application/mathematica", + "application/mbox", + "application/postscript", + "application/x-elc", + "application/x-matlab-data", + "application/x-sas", + "application/x-sh", + "application/x-subrip", + "application/x-tex", + "application/x-tika-msoffice", +} + +# MIME types from HTTP headers that indicate HTML content (needs extraction) +HTML_MIME_TYPES: set[str] = { + "text/x-php", + "text/x-jsp", + "text/x-coldfusion", + "text/html", + "message/x-emlx", + "text/asp", + "image/svg+xml", + "application/xml", + "application/atom+xml", + "application/rdf+xml", + "application/rss+xml", + "application/x-bibtex-text-file", + "application/xhtml+xml", +} + +# Magic MIME types (from libmagic) that indicate text content +TEXT_MAGIC_TYPES: set[str] = { + "text/x-shellscript", + "text/x-perl", + "text/x-lisp", + "text/x-java", + "text/x-fortran", + "text/x-diff", + "application/postscript", + "application/x-matlab-data", + "message/news", + "message/rfc822", + "text/plain", + "text/texmacs", + "text/x-Algol68", +} + +# Magic MIME types (from libmagic) that indicate HTML content +HTML_MAGIC_TYPES: set[str] = { + "text/xml", + "text/x-tex", + "text/x-php", + "text/x-ruby", + "text/x-script.python", + "text/x-objective-c", + "text/x-forth", + "text/x-c", + "text/x-c++", + "text/csv", + "text/html", + "application/octet-stream", + "application/x-appleworks3", + "application/x-bytecode.python", + "application/x-setupscript", + "application/x-wine-extension-ini", + "image/svg+xml", +} diff --git a/nemo_curator/stages/math/modifiers/__init__.py b/nemo_curator/stages/math/modifiers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/nemo_curator/stages/math/modifiers/chunking.py b/nemo_curator/stages/math/modifiers/chunking.py new file mode 100644 index 0000000000..618e3679f1 --- /dev/null +++ b/nemo_curator/stages/math/modifiers/chunking.py @@ -0,0 +1,132 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd +from transformers import AutoTokenizer + +from nemo_curator.backends.base import NodeInfo, WorkerMetadata +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.stages.text.models.utils import format_name_with_suffix +from nemo_curator.tasks import DocumentBatch + + +class TokenSplitterStage(ProcessingStage[DocumentBatch, DocumentBatch]): + """ + Token-based text chunking stage that splits long texts into smaller chunks + while preserving paragraph boundaries. + """ + + def __init__( # noqa: PLR0913 + self, + model_name: str, + max_length_tokens: int = 8000, + separator: str = "\n\n", + text_field: str = "text", + chunk_id_field: str = "chunk_id", + n_tokens_field: str = "n_tokens", + ): + self.model_name = model_name + self.max_length_tokens = max_length_tokens + self.separator = separator + self.text_field = text_field + self.chunk_id_field = chunk_id_field + self.n_tokens_field = n_tokens_field + self._tokenizer = None + self.name = format_name_with_suffix(self.model_name, suffix="_token_splitter") + + def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + """Download model weights to local cache once per physical node.""" + from huggingface_hub import snapshot_download + + snapshot_download(repo_id=self.model_name, local_files_only=False) + + def setup(self, _worker_metadata: WorkerMetadata | None = None) -> None: + """Load tokenizer from local cache per worker.""" + self._tokenizer = AutoTokenizer.from_pretrained(self.model_name, local_files_only=True) + + def inputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.text_field] + + def outputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.text_field, self.chunk_id_field, self.n_tokens_field] + + def process(self, batch: DocumentBatch) -> DocumentBatch: + """Process a batch of documents and split them into token-based chunks.""" + df = batch.to_pandas() + + records = [] + for _, row in df.iterrows(): + row_dict = row.to_dict() + text = str(row_dict.get(self.text_field, "")) + if self.text_field in row_dict: + row_dict.pop(self.text_field) + + raw_paragraphs = text.split(self.separator) + paragraphs = [] + for para_idx, para in enumerate(raw_paragraphs): + if para.strip(): + is_last = para_idx == len(raw_paragraphs) - 1 + para_to_add = para if is_last else para + self.separator + paragraphs.append(para_to_add) + + chunks = [] + current_paragraphs = [] + token_count = 0 + + for para_text in paragraphs: + tokens = self._tokenizer.encode(para_text, add_special_tokens=False) + n_tokens = len(tokens) + + if token_count + n_tokens > self.max_length_tokens and token_count > 0: + chunk_text = "".join(current_paragraphs) + chunk_dict = { + self.text_field: chunk_text, + self.chunk_id_field: len(chunks), + self.n_tokens_field: token_count, + **row_dict, + } + chunks.append(chunk_dict) + + current_paragraphs = [] + token_count = 0 + + current_paragraphs.append(para_text) + token_count += n_tokens + + if current_paragraphs: + chunk_text = "".join(current_paragraphs) + chunk_dict = { + self.text_field: chunk_text, + self.chunk_id_field: len(chunks), + self.n_tokens_field: token_count, + **row_dict, + } + chunks.append(chunk_dict) + + records.extend(chunks) + + if records: + output_df = pd.DataFrame(records) + else: + output_cols = [self.text_field, self.chunk_id_field, self.n_tokens_field] + output_cols.extend(c for c in df.columns if c != self.text_field) + output_df = pd.DataFrame(columns=output_cols) + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=output_df, + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) diff --git a/nemo_curator/stages/math/modifiers/llm_cleanup.py b/nemo_curator/stages/math/modifiers/llm_cleanup.py new file mode 100644 index 0000000000..00ca48f6c1 --- /dev/null +++ b/nemo_curator/stages/math/modifiers/llm_cleanup.py @@ -0,0 +1,206 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict + +import pandas as pd +from loguru import logger + +from nemo_curator.backends.base import NodeInfo, WorkerMetadata +from nemo_curator.models.vllm_model import VLLMModel +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.models.utils import format_name_with_suffix +from nemo_curator.tasks import DocumentBatch + + +class LLMCleanupStage(ProcessingStage[DocumentBatch, DocumentBatch]): + """ + LLM-based text cleanup stage using vLLM for distributed inference. + + This stage uses a VLLMModel wrapper to generate cleaned text from input prompts. + It handles filtering, sorting, prompt formatting, and output field management. + """ + + def __init__( # noqa: PLR0913 + self, + model: str | VLLMModel, + system_prompt: str, + text_field: str = "text", + output_field: str = "cleaned_text", + max_model_len: int | None = None, + classification: bool = False, + temperature: float = 0.7, + top_p: float = 0.8, + top_k: int = 20, + min_p: float = 0.0, + max_tokens: int | None = None, + cache_dir: str | None = None, + n_tokens_field: str = "n_tokens", + ): + """ + Initialize the LLM cleanup stage. + + Args: + model: Model identifier string (e.g., "microsoft/phi-4") or VLLMModel instance. + system_prompt: Prompt template string with {text} placeholder. + text_field: Name of the input text field. Defaults to "text". + output_field: Name of the output field. Defaults to "cleaned_text". + max_model_len: Maximum model context length. If not specified, vLLM will auto-detect. + classification: If True, output to "label" field instead of output_field. Defaults to False. + temperature: Sampling temperature. Defaults to 0.7. + top_p: Top-p sampling parameter. Defaults to 0.8. + top_k: Top-k sampling parameter. Defaults to 20. + min_p: Min-p sampling parameter (for Qwen3). Defaults to 0.0. + max_tokens: Maximum tokens to generate. Defaults to None. + cache_dir: Cache directory for model weights. Defaults to None. + n_tokens_field: Name of the n_tokens field. Defaults to "n_tokens". + """ + self._model_kwargs = { + "model": model if isinstance(model, str) else model.model, + "max_model_len": max_model_len, + "temperature": temperature, + "top_p": top_p, + "top_k": top_k, + "min_p": min_p, + "max_tokens": max_tokens, + "cache_dir": cache_dir, + } + if isinstance(model, VLLMModel): + self._model = model + self.model_name = model.model + else: + self._model = None + self.model_name = model + + self.system_prompt = system_prompt + self.text_field = text_field + self.output_field = output_field + self.max_model_len = max_model_len + self.classification = classification + self.n_tokens_field = n_tokens_field + self.resources = Resources(cpus=1.0, gpus=1.0) + self.name = format_name_with_suffix(self.model_name, suffix="_llm_cleanup") + self._final_max_model_len = None + + def inputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.text_field] + + def outputs(self) -> tuple[list[str], list[str]]: + if self.classification: + return ["data"], ["label"] + return ["data"], [self.output_field] + + def _initialize_model(self) -> None: + """Create and initialize the VLLMModel.""" + if self._model is None: + self._model = VLLMModel(**self._model_kwargs) + self._model.setup() + if hasattr(self._model, "_final_max_model_len"): + self._final_max_model_len = self._model._final_max_model_len + else: + self._final_max_model_len = self.max_model_len + + def setup_on_node(self, _node_info: NodeInfo | None = None, _worker_metadata: WorkerMetadata | None = None) -> None: + """Download weights and initialize vLLM once per node to avoid torch.compile race conditions.""" + cache_dir = self._model_kwargs.get("cache_dir") if self._model is None else self._model.cache_dir + + from huggingface_hub import snapshot_download + + snapshot_download(repo_id=self.model_name, cache_dir=cache_dir, local_files_only=False) + self._initialize_model() + + def setup(self, _: WorkerMetadata | None = None) -> None: + """Load tokenizer per worker. Falls back to full init if setup_on_node was not called.""" + if self._model is None or not hasattr(self._model, "_llm") or self._model._llm is None: + self._initialize_model() + self._tokenizer = self._model.get_tokenizer() + + def process(self, batch: DocumentBatch) -> DocumentBatch: + df = batch.to_pandas() + + if self.n_tokens_field in df.columns: + final_max_model_len = ( + self._final_max_model_len if self._final_max_model_len is not None else self.max_model_len + ) + if final_max_model_len is None: + msg = "max_model_len must be set when processing chunked data (n_tokens field present)" + raise ValueError(msg) + threshold = int(0.8 * final_max_model_len) + df = df[df[self.n_tokens_field] < threshold].copy() + if len(df) == 0: + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=pd.DataFrame(columns=df.columns), + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) + df = df.sort_values(by=self.n_tokens_field, kind="stable", ignore_index=True) + df = df.drop(columns=[self.n_tokens_field]) + + # Qwen3 supports /no_think as an inline prompt switch to disable chain-of-thought. + # Qwen3.5+ dropped this; use enable_thinking=False in chat template instead. + is_qwen3_family = "Qwen3" in self.model_name or "qwen3" in self.model_name.lower() + is_qwen3_only = is_qwen3_family and "Qwen3." not in self.model_name and "qwen3." not in self.model_name.lower() + + prompts = [] + for _, row in df.iterrows(): + text = str(row[self.text_field]) if pd.notna(row[self.text_field]) else "" + user_prompt = self.system_prompt.format_map(defaultdict(str, text=text)) + + if is_qwen3_only: + user_prompt = user_prompt + " /no_think" + system_content = " /no_think" + else: + system_content = "" + + messages = [ + {"role": "system", "content": system_content}, + {"role": "user", "content": user_prompt}, + ] + + if self._tokenizer is not None: + try: + formatted_prompt = self._tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + **({"enable_thinking": False} if is_qwen3_family else {}), + ) + prompts.append(formatted_prompt) + except (AttributeError, ValueError, TypeError, KeyError) as e: + logger.warning(f"apply_chat_template failed, using raw prompt: {e}") + prompts.append(user_prompt) + else: + prompts.append(user_prompt) + + generated_texts = self._model.generate(prompts) + + output_df = df.copy() + + if self.classification: + output_df["label"] = generated_texts + if self.text_field in output_df.columns: + output_df = output_df.drop(columns=[self.text_field]) + else: + output_df[self.output_field] = generated_texts + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=output_df, + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) diff --git a/nemo_curator/stages/math/modifiers/merge_chunks.py b/nemo_curator/stages/math/modifiers/merge_chunks.py new file mode 100644 index 0000000000..908513d3bd --- /dev/null +++ b/nemo_curator/stages/math/modifiers/merge_chunks.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd +from loguru import logger + +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.tasks import DocumentBatch + + +class ChunkMergeStage(ProcessingStage[DocumentBatch, DocumentBatch]): + """ + Merges chunked documents back into one row per document. + + After LLM cleanup, the pipeline has multiple rows per document (one per + chunk). This stage deduplicates, filters invalid chunks, sorts by chunk + order, and concatenates text back into a single row per document. + """ + + def __init__( # noqa: PLR0913 + self, + text_field: str = "cleaned_text", + raw_text_field: str | None = "text", + chunk_id_field: str = "chunk_id", + groupby_columns: list[str] | None = None, + no_content_markers: list[str] | None = None, + sum_columns: list[str] | None = None, + max_text_length: int = 900_000, + separator: str = "\n", + ): + self.text_field = text_field + self.raw_text_field = raw_text_field + self.chunk_id_field = chunk_id_field + self.groupby_columns = groupby_columns or ["url"] + self.no_content_markers = no_content_markers or [ + "NO USEFUL CONTENT", + '"NO USEFUL CONTENT"', + ] + self.sum_columns = sum_columns or ["num_generated_tokens", "num_input_tokens"] + self.max_text_length = max_text_length + self.separator = separator + self.name = "chunk_merge" + + def inputs(self) -> tuple[list[str], list[str]]: + required_cols = [self.text_field, self.chunk_id_field, *self.groupby_columns] + if self.raw_text_field: + required_cols.append(self.raw_text_field) + return ["data"], required_cols + + def outputs(self) -> tuple[list[str], list[str]]: + output_cols = [self.text_field, *self.groupby_columns] + if self.raw_text_field: + output_cols.append(self.raw_text_field) + return ["data"], output_cols + + def process(self, batch: DocumentBatch) -> DocumentBatch: + """Merge chunked rows back into one row per document.""" + df = batch.to_pandas() + + if df.empty: + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=df, + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) + + rows_before = len(df) + + # Deduplicate by (groupby_columns + chunk_id) + dedup_cols = [*self.groupby_columns, self.chunk_id_field] + df = df.drop_duplicates(subset=dedup_cols, keep="first") + + # Filter rows where text matches no-content markers, is null, empty, or newline + df = df[~df[self.text_field].isin(self.no_content_markers)] + df = df[df[self.text_field].notna()] + df = df[~df[self.text_field].isin(["", "\n"])] + + if df.empty: + logger.info(f"All {rows_before} rows filtered out during chunk merge") + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=pd.DataFrame(columns=df.columns), + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) + + # Sort by groupby_columns + chunk_id for correct ordering + sort_cols = [*self.groupby_columns, self.chunk_id_field] + df = df.sort_values(sort_cols).reset_index(drop=True) + + # Build aggregation: concat text fields, sum token counts, first() for metadata + agg_dict: dict[str, tuple[str, str]] = {} + text_fields_to_concat = [self.text_field] + if self.raw_text_field and self.raw_text_field in df.columns: + text_fields_to_concat.append(self.raw_text_field) + + sum_cols_present = [c for c in self.sum_columns if c in df.columns] + + for col in df.columns: + if col in self.groupby_columns: + continue + if col in text_fields_to_concat: + agg_dict[col] = (col, lambda x, _sep=self.separator: _sep.join(x.astype(str))) + elif col in sum_cols_present: + agg_dict[col] = (col, "sum") + else: + agg_dict[col] = (col, "first") + + merged = df.groupby(self.groupby_columns, sort=False).agg(**agg_dict).reset_index() + + # Post-filter: remove null, empty, or newline-only merged text, and rows exceeding max_text_length + merged = merged[ + merged[self.text_field].notna() + & (merged[self.text_field] != "") + & (merged[self.text_field] != "\n") + & (merged[self.text_field].str.len() <= self.max_text_length) + ] + + logger.info(f"Chunk merge: {rows_before} rows -> {len(merged)} documents") + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=merged.reset_index(drop=True), + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) diff --git a/nemo_curator/stages/text/download/arxiv/download.py b/nemo_curator/stages/text/download/arxiv/download.py index edad8d3ba8..c339d17615 100644 --- a/nemo_curator/stages/text/download/arxiv/download.py +++ b/nemo_curator/stages/text/download/arxiv/download.py @@ -18,6 +18,7 @@ from loguru import logger from nemo_curator.stages.text.download import DocumentDownloader +from nemo_curator.stages.text.download.utils import check_s5cmd_installed class ArxivDownloader(DocumentDownloader): @@ -25,7 +26,7 @@ class ArxivDownloader(DocumentDownloader): def __init__(self, download_dir: str, verbose: bool = False): super().__init__(download_dir, verbose) - if not self._check_s5cmd_installed(): + if not check_s5cmd_installed(): msg = "s5cmd is not installed. Please install it from https://github.com/peak/s5cmd" raise RuntimeError(msg) diff --git a/nemo_curator/stages/text/download/base/download.py b/nemo_curator/stages/text/download/base/download.py index 282ab16966..38f91e7434 100644 --- a/nemo_curator/stages/text/download/base/download.py +++ b/nemo_curator/stages/text/download/base/download.py @@ -13,7 +13,6 @@ # limitations under the License. import os -import subprocess from abc import ABC, abstractmethod from dataclasses import dataclass from typing import Any @@ -39,15 +38,6 @@ def __init__(self, download_dir: str, verbose: bool = False): self._verbose = verbose os.makedirs(download_dir, exist_ok=True) - def _check_s5cmd_installed(self) -> bool: - """Check if s5cmd is installed.""" - try: - subprocess.run(["s5cmd", "version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) # noqa: S607 - except FileNotFoundError: - return False - else: - return True - @abstractmethod def _get_output_filename(self, url: str) -> str: """Generate output filename from URL. diff --git a/nemo_curator/stages/text/download/common_crawl/download.py b/nemo_curator/stages/text/download/common_crawl/download.py index 88e7945129..039333446e 100644 --- a/nemo_curator/stages/text/download/common_crawl/download.py +++ b/nemo_curator/stages/text/download/common_crawl/download.py @@ -12,13 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import concurrent.futures +import gzip +import io import os import subprocess -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse +import pandas as pd +import requests from loguru import logger +from warcio.archiveiterator import ArchiveIterator +from nemo_curator.stages.base import ProcessingStage from nemo_curator.stages.text.download import DocumentDownloader +from nemo_curator.stages.text.download.utils import check_s5cmd_installed +from nemo_curator.tasks import DocumentBatch + +# Common Crawl base URL for HTTPS access +CC_BASE_URL = "https://data.commoncrawl.org/" + +# HTTP status codes +HTTP_OK = 200 +HTTP_PARTIAL_CONTENT = 206 class CommonCrawlWARCDownloader(DocumentDownloader): @@ -38,7 +54,7 @@ def __init__(self, download_dir: str, use_aws_to_download: bool = False, verbose """ super().__init__(download_dir, verbose) self.use_aws_to_download = use_aws_to_download - if self.use_aws_to_download and not self._check_s5cmd_installed(): + if self.use_aws_to_download and not check_s5cmd_installed(): msg = "s5cmd is not installed. Please install it from https://github.com/peak/s5cmd" raise RuntimeError(msg) @@ -89,3 +105,189 @@ def _download_to_path(self, url: str, path: str) -> tuple[bool, str | None]: else: error_msg = result.stderr.decode("utf-8") if result.stderr else "Unknown error" return False, error_msg + + +class CommonCrawlWARCReader(ProcessingStage[DocumentBatch, DocumentBatch]): + """ + Reads WARC records directly from Common Crawl using HTTPS range requests. + + This stage fetches raw HTML content from Common Crawl's public servers + using byte-range requests. No AWS credentials or s5cmd required. + """ + + def __init__( # noqa: PLR0913 + self, + warc_filename_col: str = "warc_filename", + warc_record_offset_col: str = "warc_record_offset", + warc_record_length_col: str = "warc_record_length", + binary_content_col: str = "binary_content", + drop_failed: bool = True, + max_workers: int = 16, + timeout: int = 30, + max_retries: int = 3, + ): + """ + Initialize the WARC reader. + + Args: + warc_filename_col: Column name for WARC filename. + warc_record_offset_col: Column name for byte offset. + warc_record_length_col: Column name for record length. + binary_content_col: Output column name for fetched content. + drop_failed: If True, drop rows where fetch failed. + max_workers: Number of parallel threads for fetching. + timeout: HTTP request timeout in seconds. + max_retries: Number of retries for failed requests. + """ + self.warc_filename_col = warc_filename_col + self.warc_record_offset_col = warc_record_offset_col + self.warc_record_length_col = warc_record_length_col + self.binary_content_col = binary_content_col + self.drop_failed = drop_failed + self.max_workers = max_workers + self.timeout = timeout + self.max_retries = max_retries + self.name = "CommonCrawlWARCReader" + self._session = None + + def inputs(self) -> tuple[list[str], list[str]]: + return ( + ["data"], + [self.warc_filename_col, self.warc_record_offset_col, self.warc_record_length_col], + ) + + def outputs(self) -> tuple[list[str], list[str]]: + return ["data"], [self.binary_content_col] + + def _get_session(self) -> requests.Session: + """Get or create a requests session for connection pooling.""" + if self._session is None: + self._session = requests.Session() + # Configure connection pooling for better performance + adapter = requests.adapters.HTTPAdapter( + pool_connections=self.max_workers, + pool_maxsize=self.max_workers * 2, + max_retries=self.max_retries, + ) + self._session.mount("https://", adapter) + self._session.mount("http://", adapter) + return self._session + + def _read_warc_record(self, row: pd.Series) -> bytes | None: # noqa: C901, PLR0911 + """Fetch a single WARC record using HTTPS range request. + + This method: + 1. Fetches gzip-compressed WARC record bytes via HTTP range request + 2. Decompresses the gzip content + 3. Parses the WARC record format using warcio + 4. Extracts and returns the HTTP response body (the actual content) + """ + filename = None + offset = None + try: + filename = row[self.warc_filename_col] + offset = int(row[self.warc_record_offset_col]) + length = int(row[self.warc_record_length_col]) + + # Build the URL + url = urljoin(CC_BASE_URL, filename) + + # HTTP Range header (inclusive end byte) + end_byte = offset + length - 1 + headers = {"Range": f"bytes={offset}-{end_byte}"} + + response = self._get_session().get( + url, + headers=headers, + timeout=self.timeout, + ) + + # 206 Partial Content is the expected response for range requests + if response.status_code == HTTP_PARTIAL_CONTENT: + raw_bytes = response.content + elif response.status_code == HTTP_OK: + # Server ignored range request, returned full file (unusual but handle it) + logger.warning(f"Server returned full file instead of range for {filename}") + raw_bytes = response.content[offset : offset + length] + else: + logger.warning(f"Failed to fetch WARC record {filename}: HTTP {response.status_code}") + return None + + # Decompress gzip content (WARC files from CC are .warc.gz) + try: + decompressed = gzip.decompress(raw_bytes) + except gzip.BadGzipFile: + # Content might not be gzip-compressed, use as-is + decompressed = raw_bytes + + # Parse the WARC record using warcio to extract HTTP response body + try: + stream = io.BytesIO(decompressed) + archive_iterator = ArchiveIterator(stream) + for record in archive_iterator: + if record.rec_type == "response": + # Return the HTTP response body (content after HTTP headers) + return record.content_stream().read() + except Exception as e: # noqa: BLE001 + logger.debug(f"Failed to parse WARC record {filename}: {e}, returning decompressed bytes") + return decompressed + else: + # If no response record found, return the decompressed bytes as-is + logger.debug(f"No response record found in WARC for {filename}, returning raw content") + return decompressed + + except requests.exceptions.Timeout: + logger.warning(f"Timeout fetching WARC record {filename} at offset {offset}") + return None + except requests.exceptions.RequestException as e: + logger.warning(f"Failed to fetch WARC record {filename} at offset {offset}: {e}") + return None + except Exception as e: # noqa: BLE001 + logger.warning(f"Unexpected error fetching WARC record: {e}") + return None + + def _read_warc_records_batch(self, df_partition: pd.DataFrame) -> list[bytes | None]: + """Fetch multiple records in parallel using ThreadPoolExecutor.""" + results = [None] * len(df_partition) + rows = list(df_partition.iterrows()) + + def fetch_row(row_data: tuple[int, pd.Series]) -> tuple[int, bytes | None]: + idx, row = row_data + return idx, self._read_warc_record(row) + + # Use a thread pool to parallelize the HTTP requests + # Requests are IO bound, so threads work well here + with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor: + futures = [executor.submit(fetch_row, (i, row)) for i, (_, row) in enumerate(rows)] + + for future in concurrent.futures.as_completed(futures): + try: + i, result = future.result() + results[i] = result + except Exception as e: # noqa: BLE001, PERF203 + logger.warning(f"Error in thread pool: {e}") + + return results + + def process(self, batch: DocumentBatch) -> DocumentBatch: + df = batch.to_pandas() + + if self.warc_filename_col in df.columns: + # Use batched/parallel processing for the partition + df[self.binary_content_col] = self._read_warc_records_batch(df) + + if self.drop_failed: + # Drop rows where binary_content is None + initial_count = len(df) + df = df.dropna(subset=[self.binary_content_col]) + dropped_count = initial_count - len(df) + if dropped_count > 0: + logger.info(f"Dropped {dropped_count}/{initial_count} rows due to failed WARC fetch.") + + return DocumentBatch( + task_id=batch.task_id, + dataset_name=batch.dataset_name, + data=df, + _metadata=batch._metadata, + _stage_perf=batch._stage_perf, + ) diff --git a/nemo_curator/stages/text/download/utils.py b/nemo_curator/stages/text/download/utils.py index 6e17b39e41..3d9969d943 100644 --- a/nemo_curator/stages/text/download/utils.py +++ b/nemo_curator/stages/text/download/utils.py @@ -12,12 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +import subprocess import unicodedata import pycld2 as cld2 from charset_normalizer import detect as charset_normalizer_detect +def check_s5cmd_installed() -> bool: + """Check if s5cmd is installed. + + s5cmd is a command-line tool for interacting with S3-compatible storage. + This function checks if it's available in the system PATH. + + Returns: + True if s5cmd is installed and accessible, False otherwise. + """ + try: + subprocess.run(["s5cmd", "version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False) # noqa: S607 + except FileNotFoundError: + return False + else: + return True + + def remove_control_characters(text: str) -> str: """Remove control characters from text. Control characters are non-printable characters in the Unicode standard that control how text is displayed or processed. diff --git a/nemo_curator/utils/gpu_utils.py b/nemo_curator/utils/gpu_utils.py new file mode 100644 index 0000000000..c82351eedd --- /dev/null +++ b/nemo_curator/utils/gpu_utils.py @@ -0,0 +1,70 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +from loguru import logger +from transformers import AutoConfig + + +def get_gpu_count() -> int: + """ + Get number of available CUDA GPUs as a power of 2. + + Many models require tensor parallelism to use power-of-2 GPU counts. + This returns the largest power of 2 <= available GPU count. + + Returns: + Power of 2 GPU count, minimum 1. + + Raises: + RuntimeError: If no CUDA GPUs are detected. + """ + count = torch.cuda.device_count() + if count == 0: + msg = "No CUDA GPUs detected. At least one GPU is required for vLLM inference." + raise RuntimeError(msg) + tp_size = 2 ** int(math.log2(count)) if count >= 2 else 1 # noqa: PLR2004 + logger.info( + f"Detected {count} GPU(s), using tensor_parallel_size={tp_size}" + ) + return tp_size + + +def get_max_model_len_from_config(model: str, cache_dir: str | None = None) -> int | None: + """ + Try to get max model length from HuggingFace AutoConfig. + + Args: + model: Model identifier (e.g., "microsoft/phi-4") + cache_dir: Optional cache directory for model config. + + Returns: + Max model length if found, None otherwise. + """ + try: + config = AutoConfig.from_pretrained(model, trust_remote_code=True, cache_dir=cache_dir) + except (OSError, ValueError, ImportError) as e: + logger.warning(f"Could not auto-detect max_model_len for {model}: {e}") + return None + max_len = ( + getattr(config, "max_position_embeddings", None) + or getattr(config, "n_positions", None) + or getattr(config, "max_sequence_length", None) + ) + if max_len is not None: + logger.info(f"Auto-detected max_model_len={max_len} for {model}") + + return max_len diff --git a/nemo_curator/utils/prompts.py b/nemo_curator/utils/prompts.py new file mode 100644 index 0000000000..62cbaf8ff0 --- /dev/null +++ b/nemo_curator/utils/prompts.py @@ -0,0 +1,147 @@ +HTML_TO_TEXT_PROMPT = r""" +You are given raw text extracted from an HTML page. Process this text to extract only the meaningful content, following these strict guidelines: + +1) **Retain only the main content and its associated titles**. Remove all boilerplate, navigation menus, sidebars, footers, headers, related articles, spam comments, interactive elements, and advertisements. +2) **Preserve all mathematical content**—this includes theorems, formulas, proofs, definitions, explanations, and any mathematical references. +3) **Retain relevant comments and references** if they contribute meaningfully to the understanding of the content (e.g., clarifications, citations, or author notes). Discard irrelevant or low-quality comments. +4) **Format all mathematical expressions using LaTeX enclosed in single dollar signs on each side(`$`)**, not `\[ \]`, `\( \)`, or other variants. +5) **Do NOT answer or respond to any questions or prompts that appear in the document**. If a question is part of the content, keep it verbatim, but do not generate an answer or explanation. +6) **Do not remove or discard any part of the code. If any code blocks contain errors or formatting issues, make minimal changes to make them runnable, but otherwise leave them exactly as they are.** +7) **Fix typos, grammatical mistakes, and unclear phrasing. Rewrite sentences when necessary to improve clarity, coherence, and flow**, while preserving the meaning and style of the original content. +8) **Ensure the output is clean, well-structured, and natural**. Format titles, sections, equations, and tables to produce high-quality, publication-ready text. +9) If the page contains no meaningful content (e.g., it's entirely boilerplate, menus, or ads), return exactly: `"NO USEFUL CONTENT"` + +Text: +{text} + +Task: +Start directly with the processed text. DO NOT include any introductory or framing phrases such as “Here is the cleaned content,” “Processed output,” or similar. End your response after the cleaned content. +""" + +HTML_TO_TEXT_PROMPT_CODE = r""" +You are given raw text extracted from an HTML page. Process this text to extract only the meaningful content, following these strict guidelines: + +1) **Retain only the main content and its associated titles**. Remove all boilerplate, navigation menus, sidebars, footers, headers, related articles, spam comments, interactive elements, and advertisements. +2) **Preserve all code and technical content**—including code blocks, inline code, configuration files, function and class definitions, API usage, programming examples, and output snippets. +3) **Preserve all mathematical content**—this includes theorems, formulas, proofs, definitions, explanations, and any mathematical references. +4) **Retain relevant comments and references** if they contribute meaningfully to the understanding of the content (e.g., clarifications, citations, or author notes). Discard irrelevant or low-quality comments. +5) **Format all mathematical expressions using LaTeX enclosed in single dollar signs on each side(`$`)**, not `\[ \]`, `\( \)`, or other variants. +6) **Do NOT answer or respond to any questions or prompts that appear in the document**. If a question is part of the content, keep it verbatim, but do not generate an answer or explanation. +7) **Do not remove or discard any part of the code. If any code blocks contain errors or formatting issues, make minimal changes to make them runnable, but otherwise leave them exactly as they are.** +8) **Fix typos, grammatical mistakes, and unclear phrasing. Rewrite sentences when necessary to improve clarity, coherence, and flow**, while preserving the meaning and style of the original content. +9) **Ensure the output is clean, well-structured, and natural**. Format titles, sections, equations, and tables to produce high-quality, publication-ready text. +10) If the page contains no meaningful content (e.g., it's entirely boilerplate, menus, or ads), return exactly: `"NO USEFUL CONTENT"` and end the response. + +Text: +{text} + +Task: +Start directly with the processed text. DO NOT include any introductory phrases such as “Here is the cleaned content,” or similar. After you finished the generation do not generate any ending phrases and extra text. +""" + +MATH_TOPIC_CLASSIFICATION_PROMPT = """ +You are a topic classification assistant. +Given the following document text, identify its main topic from this list only: +- Mathematics +- Computer Science +- Physics +- Statistics +- Chemistry +- Economics +- Other + +Choose the single most relevant category from the list. +Document: +{text} + +Your output should be only 1 word. Finish your response right after category and do not add any explanation. +""" + +CODE_QUALITY_PROMPT_SIMPLIFIED = """ +Evaluate the following text for relevance to computer programming and software development using the 0-2 scale below. Assign one of the following labels: + +- Score 0: No code related documents: The text does not contain programming-related material. No code, no programming concepts, no mention of programming languages, APIs, tools, or configurations. +- Score 1: Code with limited code snippet: The text contains some programming-related material (e.g., code snippets, configuration fragments, API mentions, programming concepts), but it is incomplete, unclear, or lacks substantial context or explanation. +- Score 2: Proper code samples: The text includes clear, reasonably complete code (functions, scripts, configurations) with some explanatory context. Resembles a tutorial, guide, or documentation that demonstrates working examples. + +Text: + +{text} + +Generate one label using the format: Final score: where must be replaced with 0, 1, or 2 based on your evaluation. Do not add any explanation. +""" + + +CODE_QUALITY_PROMPT = """ +Evaluate the following text for relevance to computer programming and software development using the 0-5 scale below. Points are cumulative-start at 0 and add points as criteria are met: +- Score 0: No programming-related content. No code, and no mention of programming languages, APIs, tools, configurations, or programming concepts. +- Score 1: Mentions any programming-related material-such as code snippets, configuration files, function definitions, API usage, or discussion of tools or programming concepts-even if low-quality, auto-generated, or boilerplate. +- Score 2: Refers to specific programming topics or tasks, such as languages, libraries, data structures, file formats, or tools-even if off-topic, unclear, or confusing. +- Score 3: Shows problem-solving or implementation details (e.g., full functions, configuration examples, tool usage steps or command-line workflows). Code related forum answers or walkthroughs qualify even without code, if implementation is clearly described. +- Score 4: Contains actual, clear, and reasonably complete code (e.g., functions, scripts, configurations) with some explanatory context. Resembles a tutorial or how-to guide with working code. +- Score 5: High-quality educational material-such as tutorials or documentation-with clean code, useful context, clear structure, and easy-to-follow explanations. + +Question-answer formats (e.g., StackOverflow posts) are acceptable if they meet the criteria. + +Text: +{text} + +After examining the text: +- Briefly justify your total score (max 100 words). +- Conclude with the score using the format: Final score: +""" + +# MIND dataset prompts. See https://arxiv.org/pdf/2410.12881 +mind_two_profs = """ +Convert the context below as a multi-turn discussions between two professors. Make sure that their discussions strictly adhere to the context below and remains faithful to information in the context. Please DO NOT add any new information/reference other than the context. + +{text} +""" + +mind_teacher_student = """Convert the context below as a multi-turn discussions between a teacher and a student. The student has questions about the context and the teacher solves each of them step-by-step.\ +Make sure that their discussions strictly adhere to the context below and remains faithful \ +to information in the context. Please DO NOT add any new information/reference other than the context. + +{text} +""" + +mind_two_students = """Convert the context below as a multi-turn discussions between two students who are working on their assignment related to the given context. \ +Make sure that their discussions strictly adhere to the context below and remains faithful to information in the context. \ +Please DO NOT add any new information/reference other than the context. + +{text} +""" + +mind_interview = """Conduct an interview-style conversation where one participant acts as the interviewer, asking questions exclusively related to the content provided, while the other participant serves as the subject matter expert, providing detailed responses based on the content. \ +Make sure that their discussions strictly adhere to the context below and remains faithful to information in the context. \ +Please DO NOT add any new information/reference other than the context. + +{text} +""" + +mind_problem_solving = """Convert the context below as a multi-turn problem-solving conversation where participants +analyze challenges or scenarios presented in the content and brainstorm solutions within the context of the provided +material, avoiding speculation or unrelated discussions. Make sure that their conversation strictly adhere to the +context below and remains faithful to information in the context. Please DO NOT add any new information/reference other +than the context. + +{text} +""" + +mind_layman_knowall = """Imagine you are presenting the content below step-by-step to a layman. While you are presenting, +the layman has a lot of followup questions regarding your presentation. You answer the questions step-by-step with chain-of-thoughts. +Design this interaction between you and the layman as a multi-turn conversational manner. \ +Make sure that the interaction strictly adhere to the context below and remains faithful to information in the context. \ +Please DO NOT add any new information/reference other than the context. + +{text} +""" + +mind_debate = """Convert the context below as a multi-turn debate-style conversation where the participants present arguments +and counterarguments based solely on the content provided, without introducing external information or personal opinions. Each +participant defends others arguments step-by-step with chain-of-thoughts. \ +Make sure that the conversation strictly adhere to the context below and remains faithful to information in the context. \ +Please DO NOT add any new information/reference other than the context. + +{text} +""" diff --git a/pyproject.toml b/pyproject.toml index 1a555bed3d..ba7b951a32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -154,6 +154,18 @@ video_cuda12 = [ "torchaudio", ] +# Math Curation Dependencies +math_cpu = [ + "nemo_curator[text_cpu]", # Math examples use text processing utilities +] + +math_cuda12 = [ + "nemo_curator[math_cpu]", + "nemo_curator[cuda12]", + "nemo_curator[deduplication_cuda12]", + "vllm>=0.13; (platform_machine == 'x86_64' and platform_system != 'Darwin')", +] + # Synthetic Data Generation (SDG) Dependencies sdg_cpu = [ "data-designer==0.4.0", @@ -163,6 +175,7 @@ sdg_cpu = [ all = [ "nemo_curator[audio_cuda12]", "nemo_curator[image_cuda12]", + "nemo_curator[math_cuda12]", "nemo_curator[sdg_cpu]", "nemo_curator[text_cuda12]", "nemo_curator[video_cuda12]", diff --git a/tests/stages/math_stages/__init__.py b/tests/stages/math_stages/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/stages/math_stages/classifiers/__init__.py b/tests/stages/math_stages/classifiers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/stages/math_stages/classifiers/test_finemath_classifier.py b/tests/stages/math_stages/classifiers/test_finemath_classifier.py new file mode 100644 index 0000000000..4af265a3d2 --- /dev/null +++ b/tests/stages/math_stages/classifiers/test_finemath_classifier.py @@ -0,0 +1,353 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import numpy as np +import pandas as pd +import pytest +import torch + +from nemo_curator.stages.math.classifiers.finemath import ( + FINEMATH_MODEL_ID, + MAX_SEQ_LENGTH, + CenterCropTextStage, + FineMathClassifier, + FineMathModelStage, +) +from nemo_curator.stages.text.models.tokenizer import TokenizerStage +from nemo_curator.tasks import DocumentBatch + + +class TestCenterCropTextStage: + """Test the CenterCropTextStage class.""" + + def test_mid_slice_function(self) -> None: + """Test the _mid_slice static method.""" + # Test with short string (no cropping needed — crop window exceeds length) + short_text = "Hello World" # 11 characters, mid=5 + result = CenterCropTextStage._mid_slice(short_text, 100) + # m=5, b=max(0, 5-100)=0, e=min(5+100, 11)=11 + assert result == "Hello World" # s[0:11] + + # Test with long string (cropping needed) + long_text = "0123456789" * 10 # 100 characters, mid=50 + result = CenterCropTextStage._mid_slice(long_text, 10) + # m=50, b=max(0, 50-10)=40, e=min(50+10, 100-1)=60 + assert len(result) == 20 # s[40:60] + expected = long_text[40:60] # Get the actual slice from the long text + assert result == expected + + # Test edge case with empty string + result = CenterCropTextStage._mid_slice("", 10) + assert result == "" + + def test_process_with_cropping(self) -> None: + """Test process method with text that needs cropping.""" + stage = CenterCropTextStage(center_crop_chars=5) + + # Create test data with long text + long_text = "0123456789ABCDEFGHIJ" # 20 characters, mid=10 + df = pd.DataFrame({"text": [long_text, "short"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Long text: m=10, b=max(0, 10-5)=5, e=min(10+5, 20)=15 + # Should get s[5:15] = "56789ABCDE" + cropped_text = result.data["text"].iloc[0] + assert len(cropped_text) == 10 + assert cropped_text == "56789ABCDE" + + # Short text: "short" has 5 chars, mid=2, b=max(0, 2-5)=0, e=min(2+5, 5)=5 + # Should get s[0:5] = "short" + assert result.data["text"].iloc[1] == "short" + + def test_process_no_cropping_needed(self) -> None: + """Test process method when no cropping is needed.""" + stage = CenterCropTextStage(center_crop_chars=100) + + df = pd.DataFrame({"text": ["Short text", "Another short text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # With large crop_chars, text should not be cropped + # "Short text" (10 chars): m=5, b=0, e=min(5+100, 10)=10, so s[0:10]="Short text" + assert result.data["text"].iloc[0] == "Short text" + # "Another short text" (18 chars): m=9, b=0, e=min(9+100, 18)=18, so s[0:18] + assert result.data["text"].iloc[1] == "Another short text" + + def test_process_zero_crop_chars(self) -> None: + """Test process method with zero crop characters.""" + stage = CenterCropTextStage(center_crop_chars=0) + + df = pd.DataFrame({"text": ["Any text here"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should remain unchanged when crop_chars is 0 + assert result.data["text"].iloc[0] == "Any text here" + + def test_process_missing_text_field(self) -> None: + """Test process method when text field is missing.""" + stage = CenterCropTextStage(text_field="missing_field") + + df = pd.DataFrame({"other_field": ["Some text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should return unchanged when field is missing + assert "other_field" in result.data.columns + assert "missing_field" not in result.data.columns + + +class TestFineMathModelStage: + """Test the FineMathModelStage class.""" + + def test_configure_forward(self) -> None: + """Test _configure_forward method modifies model forward function.""" + # Create a mock model + mock_model = mock.Mock() + mock_logits = mock.Mock() + mock_logits.squeeze.return_value.float.return_value = torch.tensor([1.5, 2.5, 3.5]) + mock_output = mock.Mock() + mock_output.logits = mock_logits + mock_model.forward.return_value = mock_output + + # Configure the forward function + configured_model = FineMathModelStage._configure_forward(mock_model) + + # Test that the forward function was modified + assert configured_model is mock_model + + # Test calling the modified forward function + with mock.patch("torch.no_grad"): + configured_model.forward(input_ids=torch.tensor([1, 2, 3])) + + # Verify the result is processed correctly + mock_logits.squeeze.assert_called_once_with(-1) + mock_logits.squeeze.return_value.float.assert_called_once() + + def test_configure_forward_logits_processing(self) -> None: + """Test _configure_forward correctly processes logits (squeeze + float).""" + mock_model = mock.Mock() + mock_logits = mock.Mock() + mock_logits.squeeze.return_value.float.return_value = torch.tensor([1.5]) + mock_output = mock.Mock() + mock_output.logits = mock_logits + mock_model.forward.return_value = mock_output + + # Autocast is now handled by parent ModelStage.process(), not _configure_forward + configured_model = FineMathModelStage._configure_forward(mock_model) + + # Test calling the modified forward function + with mock.patch("torch.no_grad"): + configured_model.forward(input_ids=torch.tensor([1])) + + # Verify logits are squeezed and converted to float + mock_logits.squeeze.assert_called_once_with(-1) + mock_logits.squeeze.return_value.float.assert_called_once() + + def test_process_model_output(self) -> None: + """Test process_model_output method.""" + stage = FineMathModelStage(model_identifier="test-model") + + # Create mock tensor output + mock_tensor = mock.Mock() + mock_tensor.cpu.return_value.numpy.return_value = np.array([1.2, 3.8, 5.5, -0.5, 2.0]) + + result = stage.process_model_output(mock_tensor) + + # Check that scores are clamped to [0, 5] range + expected_float_scores = np.array([1.2, 3.8, 5.0, 0.0, 2.0]) # Clamped to [0, 5] + expected_int_scores = np.array([1, 4, 5, 0, 2]) # round(clip(score, 0, 5)) + + np.testing.assert_array_almost_equal(result["finemath_scores"], expected_float_scores) + np.testing.assert_array_equal(result["finemath_int_scores"], expected_int_scores) + + def test_process_model_output_custom_columns(self) -> None: + """Test process_model_output with custom column names.""" + stage = FineMathModelStage( + model_identifier="test-model", float_score_column="custom_float", int_score_column="custom_int" + ) + + mock_tensor = mock.Mock() + mock_tensor.cpu.return_value.numpy.return_value = np.array([2.5]) + + result = stage.process_model_output(mock_tensor) + + assert "custom_float" in result + assert "custom_int" in result + np.testing.assert_array_almost_equal(result["custom_float"], [2.5]) + np.testing.assert_array_equal(result["custom_int"], [2]) + + def test_create_output_dataframe(self) -> None: + """Test create_output_dataframe method.""" + stage = FineMathModelStage(model_identifier="test-model") + + # Create input DataFrame with tokenizer columns + input_df = pd.DataFrame( + { + "text": ["Sample text 1", "Sample text 2"], + "input_ids": [[1, 2, 3], [4, 5, 6]], + "attention_mask": [[1, 1, 1], [1, 1, 0]], + "other_column": ["value1", "value2"], + } + ) + + # Create collected output + collected_output = {"finemath_scores": [2.5, 3.8], "finemath_int_scores": [3, 4]} + + result_df = stage.create_output_dataframe(input_df, collected_output) + + # Check that tokenizer columns are dropped + assert "input_ids" not in result_df.columns + assert "attention_mask" not in result_df.columns + + # Check that other columns are preserved + assert "text" in result_df.columns + assert "other_column" in result_df.columns + + # Check that score columns are added + assert "finemath_scores" in result_df.columns + assert "finemath_int_scores" in result_df.columns + + # Verify values + assert result_df["finemath_scores"].tolist() == [2.5, 3.8] + assert result_df["finemath_int_scores"].tolist() == [3, 4] + + +class TestFineMathClassifier: + """Test the FineMathClassifier composite stage.""" + + def test_post_init_creates_stages(self) -> None: + """Test that __post_init__ creates the correct stages.""" + classifier = FineMathClassifier() + + # Should have 3 stages: CenterCropTextStage, TokenizerStage and FineMathModelStage + assert len(classifier.stages) == 3 + + # Check center crop stage + center_crop_stage = classifier.stages[0] + assert isinstance(center_crop_stage, CenterCropTextStage) + assert center_crop_stage.text_field == "text" + assert center_crop_stage.center_crop_chars == 10_000 + + # Check tokenizer stage + tokenizer_stage = classifier.stages[1] + assert tokenizer_stage.model_identifier == FINEMATH_MODEL_ID + assert tokenizer_stage.text_field == "text" + assert tokenizer_stage.max_seq_length == MAX_SEQ_LENGTH + + # Check model stage + model_stage = classifier.stages[2] + assert isinstance(model_stage, FineMathModelStage) + assert model_stage.model_identifier == FINEMATH_MODEL_ID + assert model_stage.float_score_column == "finemath_scores" + assert model_stage.int_score_column == "finemath_int_scores" + + def test_name_generation(self) -> None: + """Test that the classifier name is generated correctly.""" + classifier = FineMathClassifier() + + # Name should be based on the model identifier with format_name_with_suffix + # "HuggingFaceTB/finemath-classifier" -> "finemath_classifier_classifier" + expected_name = "finemath_classifier_classifier" + assert classifier.name == expected_name + + @pytest.fixture + def math_dataset(self) -> DocumentBatch: + """Create a sample dataset with mathematical content.""" + text = [ + "The quadratic formula is x = (-b ± √(b² - 4ac)) / 2a", + "In calculus, the derivative of x² is 2x", + "The Pythagorean theorem states that a² + b² = c²", + "Linear algebra deals with vector spaces and matrices", + "This is just regular text without mathematical content", + ] + df = pd.DataFrame({"text": text}) + return DocumentBatch( + data=df, + task_id="math_batch_1", + dataset_name="math_test_1", + ) + + def test_classifier_structure_with_math_dataset(self, math_dataset: DocumentBatch) -> None: + """Test classifier structure with mathematical dataset.""" + classifier = FineMathClassifier() + + # Check that input columns match dataset + input_columns = classifier.inputs()[1] + assert all(col in math_dataset.data.columns for col in input_columns) + + # Check decomposition + stages = classifier.decompose() + assert len(stages) == 3 + + # Verify stage types + assert isinstance(stages[0], CenterCropTextStage) + assert isinstance(stages[1], TokenizerStage) + assert isinstance(stages[2], FineMathModelStage) + + def test_classifier_with_different_text_field(self) -> None: + """Test classifier with different text field name.""" + classifier = FineMathClassifier(text_field="content") + + # Create dataset with different text field + df = pd.DataFrame({"content": ["Mathematical equation: E = mc²"]}) + dataset = DocumentBatch(data=df, task_id="test", dataset_name="test") + + # Check that input columns match + input_columns = classifier.inputs()[1] + assert "content" in input_columns + assert all(col in dataset.data.columns for col in input_columns) + + def test_edge_case_empty_dataset(self) -> None: + """Test classifier behavior with empty dataset.""" + classifier = FineMathClassifier() + + # Create empty dataset + df = pd.DataFrame({"text": []}) + empty_dataset = DocumentBatch(data=df, task_id="empty", dataset_name="empty") + + # Should still have correct input/output structure + input_columns = classifier.inputs()[1] + assert all(col in empty_dataset.data.columns for col in input_columns) + + output_columns = classifier.outputs()[1] + expected_outputs = ["finemath_scores", "finemath_int_scores"] + assert output_columns == expected_outputs + + def test_score_clamping_edge_cases(self) -> None: + """Test score processing with edge case values.""" + stage = FineMathModelStage(model_identifier="test-model") + + # Test extreme values + mock_tensor = mock.Mock() + extreme_values = np.array([10.0, -5.0, 0.0, 5.0, 2.5, 4.9, 5.1]) + mock_tensor.cpu.return_value.numpy.return_value = extreme_values + + result = stage.process_model_output(mock_tensor) + + # Float scores should be clamped to [0, 5] + expected_float = np.array([5.0, 0.0, 0.0, 5.0, 2.5, 4.9, 5.0]) + np.testing.assert_array_almost_equal(result["finemath_scores"], expected_float) + + # Int scores should be clamped then rounded: round(clip(score, 0, 5)) + # [10.0, -5.0, 0.0, 5.0, 2.5, 4.9, 5.1] -> [5, 0, 0, 5, 2, 5, 5] + expected_int = np.array([5, 0, 0, 5, 2, 5, 5]) + np.testing.assert_array_equal(result["finemath_int_scores"], expected_int) diff --git a/tests/stages/math_stages/download/__init__.py b/tests/stages/math_stages/download/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/stages/math_stages/download/conftest.py b/tests/stages/math_stages/download/conftest.py new file mode 100644 index 0000000000..f7a7a5a61d --- /dev/null +++ b/tests/stages/math_stages/download/conftest.py @@ -0,0 +1,222 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest + + +@pytest.fixture +def simple_html() -> str: + """Simple HTML content for basic testing.""" + return "Content" + + +@pytest.fixture +def html_with_content() -> str: + """HTML with paragraph content for testing.""" + return "

Test content

" + + +@pytest.fixture +def complex_html() -> str: + """Complex HTML structure for comprehensive testing.""" + return """ + + + Test + + +

Content

+ + + """ + + +@pytest.fixture +def math_html() -> str: + """HTML containing mathematical content.""" + return """ + + + Mathematical Formulas + + + +

Quadratic Formula

+

The quadratic formula is:

+
+ $$x = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}$$ +
+

Where a, b, and c are coefficients.

+ +

Example

+

For the equation $x^2 + 5x + 6 = 0$:

+
    +
  • a = 1
    • +
  • b = 5
    • +
  • c = 6
    • +
+ + + """ + + +@pytest.fixture +def basic_notebook_json() -> str: + """Basic notebook JSON for testing.""" + return json.dumps( + { + "nbformat": 4, + "nbformat_minor": 2, + "cells": [{"cell_type": "code", "source": ["print('hello')"], "outputs": []}], + } + ) + + +@pytest.fixture +def complex_notebook_json() -> str: + """Complex notebook with multiple cell types and outputs - used for comprehensive testing.""" + notebook_content = { + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + { + "cell_type": "markdown", + "source": ["# Mathematical Analysis\n", "This notebook demonstrates mathematical concepts."], + }, + { + "cell_type": "code", + "source": ["import numpy as np\n", "x = np.array([1, 2, 3, 4, 5])\n", "print(f'Mean: {np.mean(x)}')"], + "outputs": [{"output_type": "stream", "text": ["Mean: 3.0\n"]}], + }, + { + "cell_type": "code", + "source": [ + "# Calculate standard deviation\n", + "std_dev = np.std(x)\n", + "print(f'Standard deviation: {std_dev}')", + ], + "outputs": [{"output_type": "execute_result", "data": {"text/plain": ["1.4142135623730951"]}}], + }, + ], + } + return json.dumps(notebook_content) + + +@pytest.fixture +def comprehensive_notebook_json() -> str: + """Comprehensive notebook with multiple cell types and outputs for text conversion testing.""" + notebook_content = { + "nbformat": 4, + "nbformat_minor": 2, + "cells": [ + {"cell_type": "markdown", "source": ["# Title\n", "This is markdown content."]}, + { + "cell_type": "code", + "source": ["import numpy as np\n", "print('Hello World')"], + "outputs": [ + {"output_type": "stream", "text": ["Hello World\n"]}, + {"output_type": "execute_result", "data": {"text/plain": ["42"]}}, + ], + }, + {"cell_type": "raw", "source": ["Raw cell content"]}, + { + "cell_type": "code", + "source": ["x = 5"], + "outputs": [{"output_type": "display_data", "data": {"text/plain": [""]}}], + }, + ], + } + return json.dumps(notebook_content) + + +@pytest.fixture +def empty_notebook_json() -> str: + """Empty notebook for edge case testing.""" + return json.dumps({"nbformat": 4, "nbformat_minor": 2, "cells": []}) + + +@pytest.fixture +def plain_text() -> str: + """Plain text content for testing.""" + return "Plain text content" + + +@pytest.fixture +def unknown_content() -> str: + """Unknown content type for fallback testing.""" + return "Some unknown content that doesn't match any specific type" + + +@pytest.fixture +def sample_text_content() -> str: + """Sample plain text content for testing.""" + return "This is plain text content." + + +@pytest.fixture +def sample_html_content() -> str: + """Sample HTML content for testing.""" + return "

Test

" + + +@pytest.fixture +def sample_test_content() -> str: + """Generic test content string.""" + return "test content" + + +@pytest.fixture +def extracted_text_responses() -> dict[str, str]: + """Common extracted text responses for mocking.""" + return {"html": "Extracted HTML text", "lynx": "Extracted text", "generic": "Extracted HTML"} + + +@pytest.fixture +def sample_urls() -> dict[str, str]: + """Common URLs used in tests.""" + return { + "notebook": "http://example.com/notebook.ipynb", + "html": "http://example.com/page.html", + "text": "http://example.com/file.txt", + "empty": "http://example.com/empty.txt", + "none": "http://example.com/none.txt", + } + + +@pytest.fixture +def test_records(sample_text_content: str, sample_urls: dict): + """Sample record structures for different content types.""" + return { + "notebook": { + "binary_content": b'{"nbformat": 4, "nbformat_minor": 2, "cells": []}', + "url": sample_urls["notebook"], + "mime_type": "application/json", + }, + "html": { + "binary_content": b"

Test content

", + "url": sample_urls["html"], + "mime_type": "text/html", + }, + "text": { + "binary_content": sample_text_content.encode("utf-8"), + "url": sample_urls["text"], + "mime_type": "text/plain", + }, + "empty": {"binary_content": b"", "url": sample_urls["empty"], "mime_type": "text/plain"}, + "none": {"binary_content": None, "url": sample_urls["none"], "mime_type": "text/plain"}, + } diff --git a/tests/stages/math_stages/download/test_extract_stage.py b/tests/stages/math_stages/download/test_extract_stage.py new file mode 100644 index 0000000000..2a6d1ace98 --- /dev/null +++ b/tests/stages/math_stages/download/test_extract_stage.py @@ -0,0 +1,377 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from unittest import mock + +import pandas as pd +import pytest + +from nemo_curator.stages.math.download.extract import MathContentExtractor, MathExtractStage +from nemo_curator.stages.text.download.base.extract import DocumentExtractor +from nemo_curator.tasks import DocumentBatch + + +class MockMathExtractor(DocumentExtractor): + """Mock implementation of MathContentExtractor for testing.""" + + def __init__(self, fail_on_url: str | None = None): + self.fail_on_url = fail_on_url + + def extract(self, record: dict[str, Any]) -> dict[str, Any] | None: + """Mock extraction that returns predictable results.""" + url = record.get("url", "") + + # Simulate failure for specific URLs + if self.fail_on_url and url == self.fail_on_url: + return None + + # Simple routing based on URL + if "html" in url: + return {"text": "Extracted HTML text", "url": url, "type": "html", "magic_mime_type": "text/html"} + elif "ipynb" in url: + return { + "text": "Extracted notebook text", + "url": url, + "type": "notebook", + "magic_mime_type": "application/json", + } + else: + return {"text": "Plain text content", "url": url, "type": "text", "magic_mime_type": "text/plain"} + + def input_columns(self) -> list[str]: + return ["binary_content", "url", "mime_type"] + + def output_columns(self) -> list[str]: + return ["text", "url", "type", "magic_mime_type"] + + +class TestMathContentExtractorStage: + """Tests for MathContentExtractor with MathExtractStage.""" + + @pytest.mark.parametrize( + ("url", "expected_type", "expected_text"), + [ + ("http://example.com/page.html", "html", "Extracted HTML text"), + ("http://example.com/notebook.ipynb", "notebook", "Extracted notebook text"), + ("http://example.com/file.txt", "text", "Plain text content"), + ], + ) + def test_process_content_types(self, url: str, expected_type: str, expected_text: str) -> None: + """Test processing different content types using mock extractor.""" + extractor = MockMathExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Create input DataFrame with single record + input_data = pd.DataFrame([{"binary_content": b"test content", "url": url, "mime_type": "test/type"}]) + + input_task = DocumentBatch( + task_id="test_content_type", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + result = stage.process(input_task) + + # Verify result structure + assert isinstance(result, DocumentBatch) + assert len(result.data) == 1 + + row = result.data.iloc[0] + assert row["type"] == expected_type + assert row["text"] == expected_text + assert row["url"] == url + + def test_process_with_extraction_failures(self) -> None: + """Test processing when some records fail extraction.""" + # Use mock extractor that fails on specific URL + extractor = MockMathExtractor(fail_on_url="http://example.com/bad.txt") + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Create input DataFrame with some failing records + input_data = pd.DataFrame( + [ + {"binary_content": b"good content", "url": "http://example.com/good.txt", "mime_type": "text/plain"}, + {"binary_content": b"fail content", "url": "http://example.com/bad.txt", "mime_type": "text/plain"}, + { + "binary_content": b"another good content", + "url": "http://example.com/good2.txt", + "mime_type": "text/plain", + }, + ] + ) + + input_task = DocumentBatch( + task_id="test_extraction_failures", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + result = stage.process(input_task) + + # Should only have 2 records (failed extraction records are filtered out) + df = result.data + assert len(df) == 2 + + # Check that only successful records remain + urls = df["url"].tolist() + assert "http://example.com/good.txt" in urls + assert "http://example.com/good2.txt" in urls + assert "http://example.com/bad.txt" not in urls + + @mock.patch("magic.Magic") + def test_process_with_magic_failures(self, mock_magic_class: mock.Mock) -> None: + """Test processing when magic MIME detection fails for some records.""" + # Setup magic to fail for some records based on content + mock_magic_instance = mock.Mock() + + class MagicDetectionError(Exception): + """Custom exception for magic detection failures in tests.""" + + def magic_side_effect(binary_content: bytes) -> str: + if b"magic_fail" in binary_content: + error_msg = "Magic detection failed" + raise MagicDetectionError(error_msg) + else: + return "text/plain" + + mock_magic_instance.from_buffer.side_effect = magic_side_effect + mock_magic_class.return_value = mock_magic_instance + + extractor = MathContentExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Use real binary content that can be decoded naturally + input_data = pd.DataFrame( + [ + { + "binary_content": b"good content", # Will decode successfully + "url": "http://example.com/good.txt", + "mime_type": "text/plain", + }, + { + "binary_content": b"magic_fail content", # Will cause magic to fail + "url": "http://example.com/magic_fail.txt", + "mime_type": "text/plain", + }, + ] + ) + + input_task = DocumentBatch( + task_id="test_magic_failures", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + result = stage.process(input_task) + + # Both records should be processed (magic failure doesn't prevent processing) + df = result.data + assert len(df) == 2 + + # Check that magic_mime_type is None for failed record + magic_fail_row = df[df["url"] == "http://example.com/magic_fail.txt"].iloc[0] + assert magic_fail_row["magic_mime_type"] is None + assert magic_fail_row["text"] == "magic_fail content" # Content should still be decoded + + # Check that magic_mime_type is set for successful record + good_row = df[df["url"] == "http://example.com/good.txt"].iloc[0] + assert good_row["magic_mime_type"] == "text/plain" + assert good_row["text"] == "good content" + + def test_process_empty_batch(self) -> None: + """Test processing an empty document batch.""" + extractor = MockMathExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + input_data = pd.DataFrame() + input_task = DocumentBatch( + task_id="empty_task", dataset_name="test_dataset", data=input_data, _metadata={"source": "test"} + ) + + result = stage.process(input_task) + + assert isinstance(result, DocumentBatch) + assert result.task_id == "empty_task" + assert result.dataset_name == "test_dataset" + assert len(result.data) == 0 + assert result._metadata == {"source": "test"} + + def test_stage_with_mock_extractor_smoke(self) -> None: + """Smoke test for stage processing using mock extractor.""" + extractor = MockMathExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Test basic stage properties + assert stage.name == "extract_mockmathextractor" + assert stage.inputs() == (["data"], ["binary_content", "url", "mime_type"]) + assert stage.outputs() == (["data"], ["text", "url", "type", "magic_mime_type"]) + + # Test processing + input_data = pd.DataFrame( + [{"binary_content": b"test", "url": "http://example.com/test.html", "mime_type": "text/html"}] + ) + + input_task = DocumentBatch(task_id="smoke_test", dataset_name="test", data=input_data, _metadata={}) + + result = stage.process(input_task) + + assert isinstance(result, DocumentBatch) + assert len(result.data) == 1 + assert result.data.iloc[0]["type"] == "html" + + def test_process_with_filename_column(self) -> None: + """Test processing with filename column enabled.""" + extractor = MathContentExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=True) + + # Use real binary content that can be decoded naturally + test_content = "Test content" + binary_content = test_content.encode("utf-8") + + input_data = pd.DataFrame( + [ + { + "binary_content": binary_content, + "url": "http://example.com/test.txt", + "mime_type": "text/plain", + "file_name": "test_file.txt", + } + ] + ) + + input_task = DocumentBatch( + task_id="test_with_filename", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + # Mock only external system boundaries + with mock.patch("magic.Magic") as mock_magic_class: + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "text/plain" + mock_magic_class.return_value = mock_magic_instance + + result = stage.process(input_task) + + # Should preserve filename column + df = result.data + assert len(df) == 1 + assert "file_name" in df.columns + assert df["file_name"].iloc[0] == "test_file.txt" + assert df["text"].iloc[0] == test_content + + def test_process_real_notebook_content(self, complex_notebook_json: str) -> None: + """Test processing with realistic notebook content - integration style.""" + extractor = MathContentExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Use real binary content that can be decoded naturally + binary_content = complex_notebook_json.encode("utf-8") + input_data = pd.DataFrame( + [ + { + "binary_content": binary_content, + "url": "http://example.com/math_analysis.ipynb", + "mime_type": "application/json", + } + ] + ) + + input_task = DocumentBatch( + task_id="test_real_notebook", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + # Only mock external system boundaries, not internal methods + with mock.patch("magic.Magic") as mock_magic_class: + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "application/json" + mock_magic_class.return_value = mock_magic_instance + + result = stage.process(input_task) + + # Verify notebook processing + df = result.data + assert len(df) == 1 + + row = df.iloc[0] + assert row["type"] == "notebook" + assert row["magic_mime_type"] == "application/json" + + # Check that notebook content was extracted + extracted_text = row["text"] + assert "Mathematical Analysis" in extracted_text + assert "import numpy as np" in extracted_text + assert "Mean: 3.0" in extracted_text + assert "1.4142135623730951" in extracted_text + + def test_process_real_html_with_math(self, math_html: str) -> None: + """Test processing with realistic HTML containing mathematical content.""" + extractor = MathContentExtractor() + stage = MathExtractStage(extractor=extractor, add_filename_column=False) + + # Use real binary content that can be decoded naturally + binary_content = math_html.encode("utf-8") + input_data = pd.DataFrame( + [ + { + "binary_content": binary_content, + "url": "http://example.com/quadratic_formula.html", + "mime_type": "text/html", + } + ] + ) + + input_task = DocumentBatch( + task_id="test_html_math", dataset_name="test_dataset", data=input_data, _metadata={} + ) + + # Mock external systems only - lynx and magic + mock_lynx = mock.Mock() + mock_lynx.extract_text.return_value = """Mathematical Formulas + +Quadratic Formula + +The quadratic formula is: + +x = (-b ± √(b² - 4ac)) / 2a + +Where a, b, and c are coefficients. + +Example + +For the equation x² + 5x + 6 = 0: + +• a = 1 +• b = 5 +• c = 6""" + + with ( + mock.patch("nemo_curator.stages.math.download.extract.LynxExtractor", return_value=mock_lynx), + mock.patch("magic.Magic") as mock_magic_class, + ): + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "text/html" + mock_magic_class.return_value = mock_magic_instance + + result = stage.process(input_task) + + # Verify HTML processing + df = result.data + assert len(df) == 1 + + row = df.iloc[0] + assert row["type"] == "html" + assert row["magic_mime_type"] == "text/html" + + # Check that mathematical content was extracted by lynx + extracted_text = row["text"] + assert "Quadratic Formula" in extracted_text + assert "coefficients" in extracted_text + assert "a = 1" in extracted_text + + # Verify lynx was called with the decoded HTML content + mock_lynx.extract_text.assert_called_once_with(math_html) diff --git a/tests/stages/math_stages/download/test_lynx_extractor.py b/tests/stages/math_stages/download/test_lynx_extractor.py new file mode 100644 index 0000000000..8366d511ee --- /dev/null +++ b/tests/stages/math_stages/download/test_lynx_extractor.py @@ -0,0 +1,179 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +from unittest import mock + +from nemo_curator.stages.math.download.html_extractors.lynx import LynxExtractor + + +class TestLynxExtractor: + """Test the LynxExtractor class.""" + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_success(self, mock_run: mock.Mock, mock_which: mock.Mock, html_with_content: str) -> None: + """Test successful lynx text extraction.""" + # Mock successful subprocess call + mock_process = mock.Mock() + mock_process.returncode = 0 + mock_process.stdout = b"Extracted text content" + mock_run.return_value = mock_process + + extractor = LynxExtractor(timeout_sec=15) + + result = extractor.extract_text(html_with_content) + + assert result == "Extracted text content" + mock_run.assert_called_once_with( + [ + "lynx", + "-dump", + "-stdin", + "-nolist", + "-width=10000", + "-assume_charset=utf-8", + "-display_charset=utf-8", + "-localhost", + "-force_html", + ], + input=html_with_content.encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + timeout=15, + ) + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_timeout(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + """Test LynxExtractor timeout handling.""" + mock_run.side_effect = subprocess.TimeoutExpired(["lynx"], timeout=20) + + extractor = LynxExtractor() + + result = extractor.extract_text(simple_html) + + assert result == "" + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_failure(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + """Test LynxExtractor when lynx returns non-zero exit code.""" + mock_process = mock.Mock() + mock_process.returncode = 1 + mock_run.return_value = mock_process + + extractor = LynxExtractor() + + result = extractor.extract_text(simple_html) + + assert result == "" + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_empty_input(self, mock_run: mock.Mock, mock_which: mock.Mock) -> None: + """Test LynxExtractor with empty input.""" + extractor = LynxExtractor() + + result = extractor.extract_text("") + + assert result == "" + mock_run.assert_not_called() # Should return early without calling subprocess + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_decode_error(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + """Test LynxExtractor with decode error handling.""" + mock_process = mock.Mock() + mock_process.returncode = 0 + # Invalid UTF-8 bytes that will cause decode error + mock_process.stdout = b"\xff\xfe" + mock_run.return_value = mock_process + + extractor = LynxExtractor() + + result = extractor.extract_text(simple_html) + + # Should handle decode error gracefully with ftfy and error replacement + assert isinstance(result, str) + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_extract_text_with_math_content(self, mock_run: mock.Mock, mock_which: mock.Mock, math_html: str) -> None: + """Test LynxExtractor with mathematical content.""" + # Simulate lynx extracting LaTeX/math content + mock_process = mock.Mock() + mock_process.returncode = 0 + mock_process.stdout = "Quadratic Formula\n\nThe quadratic formula is:\n\nx = (-b ± √(b² - 4ac)) / 2a\n\nWhere a, b, and c are coefficients.".encode() + mock_run.return_value = mock_process + + extractor = LynxExtractor() + + result = extractor.extract_text(math_html) + + assert "Quadratic Formula" in result + assert "coefficients" in result + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_subprocess_error(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + """Test LynxExtractor with subprocess error handling.""" + mock_run.side_effect = subprocess.SubprocessError("Subprocess failed") + + extractor = LynxExtractor() + + result = extractor.extract_text(simple_html) + + assert result == "" + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + @mock.patch("subprocess.run") + def test_lynx_extractor_os_error(self, mock_run: mock.Mock, mock_which: mock.Mock, simple_html: str) -> None: + """Test LynxExtractor with OS error handling.""" + mock_run.side_effect = OSError("System error") + + extractor = LynxExtractor() + + result = extractor.extract_text(simple_html) + + assert result == "" + mock_run.assert_called_once() + + @mock.patch("shutil.which", return_value="/usr/bin/lynx") + def test_lynx_extractor_unicode_encode_error(self, mock_which: mock.Mock) -> None: + """Test LynxExtractor with Unicode encode error handling.""" + extractor = LynxExtractor() + + # Create HTML with characters that might cause encoding issues + problematic_html = "Test content with problematic chars: \udcff" + + # Mock subprocess.run to raise UnicodeEncodeError during input processing + def mock_run_with_encode_error(*_args, **kwargs) -> mock.Mock: + # Simulate the error happening when trying to encode the input + if "input" in kwargs: + encoding = "utf-8" + error_msg = "invalid start byte" + raise UnicodeEncodeError(encoding, "test", 0, 1, error_msg) + return mock.Mock() + + with mock.patch("subprocess.run", side_effect=mock_run_with_encode_error): + result = extractor.extract_text(problematic_html) + + assert result == "" diff --git a/tests/stages/math_stages/download/test_math_content_extractor.py b/tests/stages/math_stages/download/test_math_content_extractor.py new file mode 100644 index 0000000000..3be868bbef --- /dev/null +++ b/tests/stages/math_stages/download/test_math_content_extractor.py @@ -0,0 +1,282 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + +from nemo_curator.stages.math.download.extract import MathContentExtractor + + +class TestMathContentExtractor: + """Test the MathContentExtractor class.""" + + def test_extract_edge_cases_none_binary_content(self, test_records: dict) -> None: + """Test extraction with None binary content - integration style.""" + extractor = MathContentExtractor() + record = test_records["none"] # Has None binary_content + + result = extractor.extract(record) + assert result is None + + def test_extract_edge_cases_empty_binary_content(self, test_records: dict) -> None: + """Test extraction with empty binary content - integration style.""" + extractor = MathContentExtractor() + # Create record with empty binary content that will decode to empty string + record = test_records["text"].copy() + record["binary_content"] = b"" # Empty bytes will decode to empty string + + with mock.patch("magic.Magic") as mock_magic: + mock_magic.return_value.from_buffer.return_value = "text/plain" + result = extractor.extract(record) + + assert result is None # Empty content should return None + + @pytest.mark.parametrize( + ("record_type", "expected_type"), + [ + ("notebook", "notebook"), + ("html", "html"), + ("text", "text"), + ], + ) + def test_extract_content_types( # noqa: PLR0913 + self, + test_records: dict, + record_type: str, + expected_type: str, + basic_notebook_json: str, + html_with_content: str, + plain_text: str, + extracted_text_responses: dict, + ) -> None: + """Test extraction of different content types.""" + record = test_records[record_type].copy() + + # Use real binary content that can be decoded naturally + if record_type == "notebook": + record["binary_content"] = basic_notebook_json.encode("utf-8") + with mock.patch("magic.Magic") as mock_magic: + mock_magic.return_value.from_buffer.return_value = "application/json" + extractor = MathContentExtractor() + result = extractor.extract(record) + elif record_type == "html": + record["binary_content"] = html_with_content.encode("utf-8") + mock_lynx = mock.Mock() + mock_lynx.extract_text.return_value = extracted_text_responses["html"] + with ( + mock.patch("nemo_curator.stages.math.download.extract.LynxExtractor", return_value=mock_lynx), + mock.patch("magic.Magic") as mock_magic, + ): + mock_magic.return_value.from_buffer.return_value = "text/html" + extractor = MathContentExtractor() + result = extractor.extract(record) + else: # text + record["binary_content"] = plain_text.encode("utf-8") + with mock.patch("magic.Magic") as mock_magic: + mock_magic.return_value.from_buffer.return_value = "text/plain" + extractor = MathContentExtractor() + result = extractor.extract(record) + + assert result is not None + assert result["type"] == expected_type + assert result["url"] == record["url"] + assert "text" in result + + @mock.patch("magic.Magic") + def test_extract_with_magic_mime_detection( + self, mock_magic_class: mock.Mock, sample_text_content: str, sample_urls: dict + ) -> None: + """Test extraction with magic MIME type detection.""" + # Mock magic.Magic instance + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "text/plain" + mock_magic_class.return_value = mock_magic_instance + + extractor = MathContentExtractor() + record = { + "binary_content": sample_text_content.encode("utf-8"), + "url": sample_urls["text"], + "mime_type": "text/plain", + } + + result = extractor.extract(record) + + assert result is not None + assert result["magic_mime_type"] == "text/plain" + assert result["text"] == sample_text_content + assert result["type"] == "text" + assert result["url"] == sample_urls["text"] + mock_magic_class.assert_called_once_with(mime=True) + mock_magic_instance.from_buffer.assert_called_once_with(record["binary_content"]) + + @mock.patch("magic.Magic") + def test_extract_magic_mime_exception( + self, mock_magic_class: mock.Mock, sample_text_content: str, sample_urls: dict + ) -> None: + """Test extraction when magic MIME detection fails.""" + # Mock magic.Magic to raise exception + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.side_effect = Exception("Magic failed") + mock_magic_class.return_value = mock_magic_instance + + extractor = MathContentExtractor() + record = { + "binary_content": sample_text_content.encode("utf-8"), + "url": sample_urls["text"], + "mime_type": "text/plain", + } + + result = extractor.extract(record) + + assert result is not None + assert result["magic_mime_type"] is None + assert result["text"] == sample_text_content + assert result["type"] == "text" # Should still determine type correctly + + @pytest.mark.parametrize( + ("record_setup", "expected_type"), + [ + # Test notebook detection by URL + ({"url": "http://example.com/notebook.ipynb", "content": "basic_notebook_json"}, "notebook"), + # Test notebook detection by MIME type + ({"mime_type": "application/json", "content": "basic_notebook_json"}, "notebook"), + # Test HTML detection by MIME type + ({"mime_type": "text/html", "content": "html_with_content"}, "html"), + # Test text detection by MIME type + ({"mime_type": "text/plain", "content": "plain_text"}, "text"), + # Test HTML detection by structure (fallback) + ({"content": "complex_html"}, "html"), + # Test fallback to HTML for unknown content + ({"content": "unknown_content"}, "html"), + ], + ) + def test_extract_type_detection( # noqa: PLR0913 + self, + record_setup: dict, + expected_type: str, + test_records: dict, + basic_notebook_json: str, + html_with_content: str, + plain_text: str, + complex_html: str, + unknown_content: str, + extracted_text_responses: dict, + ) -> None: + """Test type detection through public extract method - integration style.""" + # Get base record and override with test-specific values + record = test_records["text"].copy() # Start with text record as base + + # Apply setup overrides + if "url" in record_setup: + record["url"] = record_setup["url"] + if "mime_type" in record_setup: + record["mime_type"] = record_setup["mime_type"] + + # Set up real binary content based on content type + content_map = { + "basic_notebook_json": basic_notebook_json, + "html_with_content": html_with_content, + "plain_text": plain_text, + "complex_html": complex_html, + "unknown_content": unknown_content, + } + content = content_map[record_setup["content"]] + record["binary_content"] = content.encode("utf-8") # Use real binary content + + extractor = MathContentExtractor() + + # Mock external dependencies based on expected type + if expected_type == "notebook": + with mock.patch("magic.Magic") as mock_magic: + mock_magic.return_value.from_buffer.return_value = "application/json" + result = extractor.extract(record) + elif expected_type == "html": + mock_lynx = mock.Mock() + mock_lynx.extract_text.return_value = extracted_text_responses["generic"] + with ( + mock.patch("nemo_curator.stages.math.download.extract.LynxExtractor", return_value=mock_lynx), + mock.patch("magic.Magic") as mock_magic, + ): + mock_magic.return_value.from_buffer.return_value = "text/html" + result = extractor.extract(record) + else: # text + with mock.patch("magic.Magic") as mock_magic: + mock_magic.return_value.from_buffer.return_value = "text/plain" + result = extractor.extract(record) + + assert result is not None + assert result["type"] == expected_type + + def test_lazy_initialization_lynx( + self, sample_html_content: str, sample_urls: dict, extracted_text_responses: dict + ) -> None: + """Test lazy initialization of lynx extractor.""" + extractor = MathContentExtractor() + + # Initially None + assert extractor._lynx is None + + with ( + mock.patch("nemo_curator.stages.math.download.extract.LynxExtractor") as mock_lynx_class, + mock.patch("magic.Magic") as mock_magic_class, + ): + mock_lynx_instance = mock.Mock() + mock_lynx_instance.extract_text.return_value = extracted_text_responses["lynx"] + mock_lynx_class.return_value = mock_lynx_instance + + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "text/html" + mock_magic_class.return_value = mock_magic_instance + + record = { + "binary_content": sample_html_content.encode("utf-8"), + "url": sample_urls["html"], + "mime_type": "text/html", + } + + result = extractor.extract(record) + + # Verify lynx was initialized with correct timeout + mock_lynx_class.assert_called_once_with(timeout_sec=20) + assert extractor._lynx is mock_lynx_instance + assert result["text"] == extracted_text_responses["lynx"] + assert result["type"] == "html" + + def test_lazy_initialization_magic(self, sample_test_content: str, sample_urls: dict) -> None: + """Test lazy initialization of magic MIME detector.""" + extractor = MathContentExtractor() + + # Initially None + assert extractor._magic is None + + with mock.patch("magic.Magic") as mock_magic_class: + mock_magic_instance = mock.Mock() + mock_magic_instance.from_buffer.return_value = "text/plain" + mock_magic_class.return_value = mock_magic_instance + + record = { + "binary_content": sample_test_content.encode("utf-8"), + "url": sample_urls["text"], + "mime_type": "text/plain", + } + + result = extractor.extract(record) + + # Verify magic was initialized correctly + mock_magic_class.assert_called_once_with(mime=True) + assert extractor._magic is mock_magic_instance + assert result["magic_mime_type"] == "text/plain" + assert result["text"] == sample_test_content + assert result["type"] == "text" diff --git a/tests/stages/math_stages/modifiers/__init__.py b/tests/stages/math_stages/modifiers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/stages/math_stages/modifiers/test_chunking.py b/tests/stages/math_stages/modifiers/test_chunking.py new file mode 100644 index 0000000000..e3e3eeb839 --- /dev/null +++ b/tests/stages/math_stages/modifiers/test_chunking.py @@ -0,0 +1,234 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import Mock, patch + +import pandas as pd +import pytest + +from nemo_curator.stages.math.modifiers.chunking import TokenSplitterStage +from nemo_curator.tasks import DocumentBatch + + +@pytest.fixture +def mock_tokenizer(): + """Create a mock tokenizer that simulates tokenization.""" + tokenizer = Mock() + + def mock_encode(text: str, add_special_tokens: bool = False) -> list[int]: # noqa: ARG001 + # Simple tokenization: each word = 1 token, spaces = 0 tokens + # This is a simplified approximation + words = text.split() + # Add some base tokens for special characters + return list(range(100, 100 + len(words))) + + tokenizer.encode = mock_encode + return tokenizer + + +@pytest.fixture(autouse=True) +def setup_mocks(mock_tokenizer: Mock) -> None: # type: ignore[no-untyped-def] + """Automatically setup mocks for AutoTokenizer.""" + with patch("nemo_curator.stages.math.modifiers.chunking.AutoTokenizer") as mock_auto_tokenizer: + mock_auto_tokenizer.from_pretrained.return_value = mock_tokenizer + yield {"auto_tokenizer": mock_auto_tokenizer} + + +class TestTokenSplitterStage: + """Test the TokenSplitterStage class.""" + + def test_process_single_short_text(self): + """Test process method with a single short text that doesn't need chunking.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame({"text": ["Short text here"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert len(result.data) == 1 + assert result.data["text"].iloc[0] == "Short text here" + assert result.data["chunk_id"].iloc[0] == 0 + assert result.data["n_tokens"].iloc[0] > 0 + assert "text" in result.data.columns + + def test_process_text_with_chunking(self): + """Test process method with text that needs chunking.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=5) + stage.setup() + + # Create text with multiple paragraphs that will exceed token limit + text = "Paragraph one.\n\nParagraph two.\n\nParagraph three.\n\nParagraph four." + df = pd.DataFrame({"text": [text]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should create multiple chunks + assert len(result.data) > 1 + assert "chunk_id" in result.data.columns + assert "n_tokens" in result.data.columns + # Chunk IDs should be sequential starting from 0 + chunk_ids = result.data["chunk_id"].tolist() + assert chunk_ids == list(range(len(chunk_ids))) + + def test_process_preserves_metadata(self): + """Test that process preserves original metadata fields.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame( + { + "text": ["Some text"], + "url": ["https://example.com"], + "title": ["Test Title"], + "metadata": ["extra info"], + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # All original columns should be preserved + assert "url" in result.data.columns + assert "title" in result.data.columns + assert "metadata" in result.data.columns + assert result.data["url"].iloc[0] == "https://example.com" + assert result.data["title"].iloc[0] == "Test Title" + + def test_process_multiple_documents(self): + """Test process method with multiple documents.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame( + { + "text": ["First document text", "Second document text", "Third document text"], + "doc_id": [1, 2, 3], + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Each document should produce at least one chunk + assert len(result.data) >= 3 + # All doc_ids should be preserved + assert set(result.data["doc_id"].tolist()) == {1, 2, 3} + + def test_process_empty_text(self): + """Test process method with empty text.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame({"text": [""]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Empty text should produce no chunks (empty paragraphs are filtered) + assert len(result.data) == 0 + + def test_process_text_with_only_whitespace(self): + """Test process method with text containing only whitespace.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame({"text": [" \n\n \n\n "]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Whitespace-only paragraphs are filtered out + assert len(result.data) == 0 + + def test_process_custom_separator(self): + """Test process method with custom separator.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=5, separator="\n") + stage.setup() + + text = "Line one\nLine two\nLine three\nLine four" + df = pd.DataFrame({"text": [text]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should create chunks based on single newline separator + assert len(result.data) > 0 + # Verify separator is preserved in chunks (except last) + for _, row in result.data.iterrows(): + if row["chunk_id"] < len(result.data) - 1: + # Non-last chunks should end with separator + assert row["text"].endswith("\n") + + def test_process_chunk_id_sequential(self): + """Test that chunk_id is sequential for each document.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=5) + stage.setup() + + text = "Para one.\n\nPara two.\n\nPara three.\n\nPara four.\n\nPara five." + df = pd.DataFrame({"text": [text]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Chunk IDs should be sequential: 0, 1, 2, ... + chunk_ids = result.data["chunk_id"].tolist() + assert chunk_ids == list(range(len(chunk_ids))) + + def test_process_n_tokens_calculated(self): + """Test that n_tokens is correctly calculated for each chunk.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + df = pd.DataFrame({"text": ["Some text here"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # n_tokens should be positive + assert all(result.data["n_tokens"] > 0) + # n_tokens should match the token count of the chunk text + # Note: The actual tokenization happens during processing, so we verify + # that n_tokens is set and positive + assert result.data["n_tokens"].iloc[0] > 0 + + def test_process_last_paragraph_no_separator(self): + """Test that last paragraph doesn't get separator appended.""" + stage = TokenSplitterStage(model_name="test-model", max_length_tokens=100) + stage.setup() + + text = "First paragraph.\n\nSecond paragraph." + df = pd.DataFrame({"text": [text]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Last chunk should not end with separator + last_chunk_text = result.data.iloc[-1]["text"] + assert not last_chunk_text.endswith("\n\n") + + def test_process_missing_text_field(self): + """Test process method when text field is missing.""" + stage = TokenSplitterStage(model_name="test-model", text_field="missing_field") + stage.setup() + + df = pd.DataFrame({"other_field": ["Some text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should handle missing field gracefully (empty string) + assert len(result.data) == 0 # Empty text produces no chunks diff --git a/tests/stages/math_stages/modifiers/test_llm_cleanup.py b/tests/stages/math_stages/modifiers/test_llm_cleanup.py new file mode 100644 index 0000000000..54f82a7d5f --- /dev/null +++ b/tests/stages/math_stages/modifiers/test_llm_cleanup.py @@ -0,0 +1,386 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any +from unittest.mock import Mock, patch + +import pandas as pd +import pytest + +# Import after setting up patches to ensure mocks are in place +from nemo_curator.stages.math.modifiers.llm_cleanup import LLMCleanupStage +from nemo_curator.tasks import DocumentBatch + + +class MockLLMOutput: + """Mock output from vLLM's generate method.""" + + def __init__(self, text: str): + self.outputs = [Mock(text=text)] + + +class MockLLM: + """Mock vLLM LLM class.""" + + def __init__(self, *args, **kwargs): + self.model = kwargs.get("model", "test-model") + self.max_model_len = kwargs.get("max_model_len", 32000) + + def generate(self, prompts: list[str], sampling_params=None, use_tqdm=False): # noqa: ANN001 + """Mock generate method that returns cleaned text.""" + results = [] + for prompt in prompts: + # Extract text from prompt - look for "Text:" marker first + if "Text:" in prompt: + # Extract text after "Text:" marker + text_start = prompt.find("Text:") + len("Text:") + text_end = prompt.find("\n", text_start) + if text_end == -1: + text_end = len(prompt) + original_text = prompt[text_start:text_end].strip() + cleaned_text = f"Cleaned: {original_text}" + else: + # For prompts like "Clean this text: Original text here" + # Try to extract text after the last colon + # Split by newlines first to handle multi-line prompts + lines = prompt.split("\n") + last_line = lines[-1] if lines else prompt + if ":" in last_line: + parts = last_line.split(":") + if len(parts) > 1: + original_text = parts[-1].strip() + cleaned_text = f"Cleaned: {original_text}" if original_text else "Cleaned output" + else: + cleaned_text = "Cleaned output" + else: + # Fallback: use a default cleaned output + cleaned_text = "Cleaned output" + results.append(MockLLMOutput(cleaned_text)) + return results + + +class MockSamplingParams: + """Mock SamplingParams class.""" + + def __init__(self, **kwargs): + self.temperature = kwargs.get("temperature", 0.7) + self.top_p = kwargs.get("top_p", 0.8) + self.top_k = kwargs.get("top_k") + self.min_p = kwargs.get("min_p") + self.max_tokens = kwargs.get("max_tokens") + + +class MockVLLMModel: + """Mock VLLMModel class that prevents real vLLM initialization.""" + + def __init__(self, *args, **kwargs): + # Store all kwargs as attributes + self.model = kwargs.get("model", "test-model") + self.max_model_len = kwargs.get("max_model_len") + self.temperature = kwargs.get("temperature", 0.7) + self.top_p = kwargs.get("top_p", 0.8) + self.top_k = kwargs.get("top_k", 20) + self.min_p = kwargs.get("min_p", 0.0) + self.max_tokens = kwargs.get("max_tokens") + self.cache_dir = kwargs.get("cache_dir") + self._llm = None + self._sampling_params = None + + def model_id_names(self): + return [self.model] + + def setup(self): + """Mock setup that initializes mock LLM - never calls real vLLM.""" + # Initialize mocks directly without calling real vLLM + self._llm = MockLLM(model=self.model, max_model_len=self.max_model_len) + # Create sampling params with all parameters that might be set + sampling_kwargs = { + "temperature": self.temperature, + "max_tokens": self.max_tokens if self.max_tokens is not None else self.max_model_len, + } + is_qwen3 = "Qwen3" in self.model or "qwen3" in self.model.lower() + if is_qwen3: + sampling_kwargs.update( + { + "top_p": self.top_p, + "top_k": self.top_k, + "min_p": self.min_p, + } + ) + else: + sampling_kwargs["top_p"] = self.top_p + self._sampling_params = MockSamplingParams(**sampling_kwargs) + self._final_max_model_len = self.max_model_len + + def generate(self, prompts: list[str]) -> list[str]: + """Mock generate method that returns cleaned text.""" + if self._llm is None or self._sampling_params is None: + msg = "Model not initialized. Call setup() first." + raise RuntimeError(msg) + outputs = self._llm.generate(prompts, self._sampling_params, use_tqdm=False) + return [out.outputs[0].text for out in outputs] + + def get_tokenizer(self): + """Mock get_tokenizer method that returns a mock tokenizer.""" + if self._llm is None: + msg = "Model not initialized. Call setup() first." + raise RuntimeError(msg) + + # Return a mock tokenizer with apply_chat_template method + # The template should extract the user message content for the mock to work correctly + def mock_apply_chat_template(messages: list[dict[str, str]], **kwargs: Any) -> str: # noqa: ARG001, ANN401 + # Extract user message content if available, otherwise return formatted string + for msg in messages: + if isinstance(msg, dict) and msg.get("role") == "user": + return msg.get("content", "") + return str(messages) + + mock_tokenizer = Mock() + mock_tokenizer.apply_chat_template = Mock(side_effect=mock_apply_chat_template) + return mock_tokenizer + + +@pytest.fixture(autouse=True) +def setup_mocks(): + """Automatically setup mocks for VLLMModel.""" + # Patch VLLMModel where it's imported in llm_cleanup module + # Also patch vLLM classes to prevent any real initialization attempts + with ( + patch("nemo_curator.stages.math.modifiers.llm_cleanup.VLLMModel", MockVLLMModel), + patch("nemo_curator.models.vllm_model.LLM", MockLLM), + patch("nemo_curator.models.vllm_model.SamplingParams", MockSamplingParams), + patch("nemo_curator.models.vllm_model.VLLM_AVAILABLE", True), + ): + yield + + +class TestLLMCleanupStage: + """Test the LLMCleanupStage class.""" + + def test_process_basic_cleanup(self): + """Test process method for basic text cleanup.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean this text: {text}") + stage.setup() + + df = pd.DataFrame({"text": ["Original text here"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert len(result.data) == 1 + assert "cleaned_text" in result.data.columns + assert "text" in result.data.columns # Original text preserved + assert result.data["cleaned_text"].iloc[0].startswith("Cleaned:") + + def test_process_classification_mode(self): + """Test process method in classification mode.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Classify: {text}", classification=True) + stage.setup() + + df = pd.DataFrame({"text": ["Some text to classify"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert "label" in result.data.columns + assert "text" not in result.data.columns # Text removed in classification mode + assert len(result.data) == 1 + + def test_process_multiple_texts(self): + """Test process method with multiple texts.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean: {text}") + stage.setup() + + df = pd.DataFrame({"text": ["First text", "Second text", "Third text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert len(result.data) == 3 + assert "cleaned_text" in result.data.columns + assert all(result.data["cleaned_text"].iloc[i].startswith("Cleaned:") for i in range(3)) + + def test_process_preserves_metadata(self): + """Test that process preserves original metadata fields.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean: {text}") + stage.setup() + + df = pd.DataFrame( + { + "text": ["Some text"], + "url": ["https://example.com"], + "title": ["Test Title"], + "metadata": ["extra info"], + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert "url" in result.data.columns + assert "title" in result.data.columns + assert "metadata" in result.data.columns + assert result.data["url"].iloc[0] == "https://example.com" + + def test_process_null_text(self): + """Test process method with null/NaN text values.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean: {text}") + stage.setup() + + df = pd.DataFrame({"text": [None, pd.NA, "Valid text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert len(result.data) == 3 + # Null values should be converted to empty strings, which result in "Cleaned output" + # The mock tokenizer extracts user message content, which for null text would be empty + assert result.data["cleaned_text"].iloc[0] == "Cleaned output" + assert result.data["cleaned_text"].iloc[1] == "Cleaned output" + assert result.data["cleaned_text"].iloc[2] == "Cleaned: Valid text" + + def test_process_filter_by_n_tokens(self): + """Test process method with n_tokens filtering enabled.""" + stage = LLMCleanupStage( + model="test-model", + system_prompt="Clean: {text}", + max_model_len=1000, + ) + stage.setup() + + # Create data with n_tokens field + df = pd.DataFrame( + { + "text": ["Short text", "Very long text that exceeds threshold"], + "n_tokens": [100, 900], # Second exceeds 80% of 1000 = 800 + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should filter out chunks exceeding threshold + assert len(result.data) == 1 # Only the short text should remain + assert result.data["text"].iloc[0] == "Short text" + + def test_process_filter_by_n_tokens_all_filtered(self): + """Test process method when all texts are filtered out.""" + stage = LLMCleanupStage( + model="test-model", + system_prompt="Clean: {text}", + max_model_len=1000, + ) + stage.setup() + + df = pd.DataFrame( + { + "text": ["Very long text 1", "Very long text 2"], + "n_tokens": [900, 950], + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should return empty DataFrame + assert len(result.data) == 0 + assert result.task_id == batch.task_id + assert result.dataset_name == batch.dataset_name + + def test_process_sort_by_n_tokens(self): + """Test that process sorts by n_tokens when available.""" + stage = LLMCleanupStage( + model="test-model", + system_prompt="Clean: {text}", + max_model_len=1000, # Required when n_tokens field is present + ) + stage.setup() + + df = pd.DataFrame( + { + "text": ["Text 1", "Text 2", "Text 3"], + "n_tokens": [300, 100, 200], + } + ) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + # Should be sorted by n_tokens (ascending) + # Note: n_tokens column is dropped after filtering/sorting, so we check text order instead + text_order = result.data["text"].tolist() + assert text_order == ["Text 2", "Text 3", "Text 1"] + + def test_process_prompt_formatting(self): + """Test that prompts are correctly formatted with text.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Process this: {text}") + stage.setup() + + df = pd.DataFrame({"text": ["Input text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + # Mock the generate method to capture prompts + original_generate = stage._model.generate + captured_prompts = [] + + def capture_prompts(prompts: list[str]) -> list[str]: + captured_prompts.extend(prompts) + return original_generate(prompts) + + stage._model.generate = capture_prompts + + stage.process(batch) + + # Verify prompt was formatted correctly + assert len(captured_prompts) == 1 + assert "Process this:" in captured_prompts[0] + assert "Input text" in captured_prompts[0] + + def test_process_error_handling(self): + """Test error handling in process method.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean: {text}") + stage.setup() + + # Make generate raise an exception + error_msg = "LLM generation failed" + + def failing_generate(prompts: list[str]) -> None: # noqa: ARG001 + raise RuntimeError(error_msg) + + stage._model.generate = failing_generate + + df = pd.DataFrame({"text": ["Some text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + with pytest.raises(RuntimeError, match="LLM generation failed"): + stage.process(batch) + + def test_process_empty_output(self): + """Test process method when LLM returns empty output.""" + stage = LLMCleanupStage(model="test-model", system_prompt="Clean: {text}") + stage.setup() + + # Mock generate to return empty outputs (as strings, not MockLLMOutput objects) + def empty_generate(prompts: list[str]) -> list[str]: # noqa: ARG001 + return [""] + + stage._model.generate = empty_generate + + df = pd.DataFrame({"text": ["Some text"]}) + batch = DocumentBatch(data=df, task_id="test", dataset_name="test") + + result = stage.process(batch) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "" diff --git a/tests/stages/math_stages/modifiers/test_merge_chunks.py b/tests/stages/math_stages/modifiers/test_merge_chunks.py new file mode 100644 index 0000000000..fe3fd1a349 --- /dev/null +++ b/tests/stages/math_stages/modifiers/test_merge_chunks.py @@ -0,0 +1,236 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pandas as pd + +from nemo_curator.stages.math.modifiers.merge_chunks import ChunkMergeStage +from nemo_curator.tasks import DocumentBatch + + +def _make_batch(df: pd.DataFrame) -> DocumentBatch: + return DocumentBatch(data=df, task_id="test", dataset_name="test") + + +class TestChunkMergeStage: + """Test the ChunkMergeStage class.""" + + def test_process_basic_merge(self): + """Two documents with 3 chunks each merge into 2 rows.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 3 + ["b.com"] * 3, + "chunk_id": [0, 1, 2, 0, 1, 2], + "cleaned_text": ["A0", "A1", "A2", "B0", "B1", "B2"], + "text": ["rA0", "rA1", "rA2", "rB0", "rB1", "rB2"], + "type": ["html"] * 6, + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 2 + urls = set(result.data["url"]) + assert urls == {"a.com", "b.com"} + + row_a = result.data[result.data["url"] == "a.com"].iloc[0] + assert row_a["cleaned_text"] == "A0\nA1\nA2" + assert row_a["text"] == "rA0\nrA1\nrA2" + + def test_process_chunk_ordering(self): + """Chunks with out-of-order chunk_ids are sorted correctly before merge.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 3, + "chunk_id": [2, 0, 1], + "cleaned_text": ["C2", "C0", "C1"], + "text": ["r2", "r0", "r1"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "C0\nC1\nC2" + assert result.data["text"].iloc[0] == "r0\nr1\nr2" + + def test_process_filter_no_useful_content(self): + """Chunks with 'NO USEFUL CONTENT' markers are dropped before merge.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 3, + "chunk_id": [0, 1, 2], + "cleaned_text": ["Good text", "NO USEFUL CONTENT", '"NO USEFUL CONTENT"'], + "text": ["raw0", "raw1", "raw2"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "Good text" + + def test_process_filter_empty_text(self): + """Null, empty, and newline-only chunks are filtered out.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 4, + "chunk_id": [0, 1, 2, 3], + "cleaned_text": ["Keep this", None, "", "\n"], + "text": ["raw0", "raw1", "raw2", "raw3"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "Keep this" + + def test_process_dedup_chunks(self): + """Duplicate (url, chunk_id) rows are deduplicated, keeping first.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 4, + "chunk_id": [0, 1, 1, 2], + "cleaned_text": ["C0", "C1-first", "C1-dup", "C2"], + "text": ["r0", "r1-first", "r1-dup", "r2"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "C0\nC1-first\nC2" + + def test_process_max_text_length(self): + """Merged text exceeding max_text_length is dropped.""" + long_text = "x" * 500 + df = pd.DataFrame( + { + "url": ["a.com"] * 3 + ["b.com"], + "chunk_id": [0, 1, 2, 0], + "cleaned_text": [long_text, long_text, long_text, "short"], + "text": ["r0", "r1", "r2", "r3"], + } + ) + stage = ChunkMergeStage(max_text_length=1000) + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["url"].iloc[0] == "b.com" + + def test_process_metadata_preserved(self): + """Metadata columns (url, finemath_scores, type) survive merge via first().""" + df = pd.DataFrame( + { + "url": ["a.com"] * 2, + "chunk_id": [0, 1], + "cleaned_text": ["C0", "C1"], + "text": ["r0", "r1"], + "type": ["html", "html"], + "finemath_scores": [4.5, 4.5], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + row = result.data.iloc[0] + assert row["type"] == "html" + assert row["finemath_scores"] == 4.5 + assert row["url"] == "a.com" + + def test_process_all_chunks_filtered(self): + """If all chunks of a document are invalid, the document is dropped entirely.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 3 + ["b.com"], + "chunk_id": [0, 1, 2, 0], + "cleaned_text": ["NO USEFUL CONTENT", "", None, "Valid"], + "text": ["r0", "r1", "r2", "r3"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["url"].iloc[0] == "b.com" + + def test_process_custom_groupby_columns(self): + """Custom groupby columns (e.g., warc_filename + url) produce separate documents.""" + df = pd.DataFrame( + { + "warc_filename": ["w1", "w1", "w2", "w2"], + "url": ["a.com", "a.com", "a.com", "a.com"], + "chunk_id": [0, 1, 0, 1], + "cleaned_text": ["W1C0", "W1C1", "W2C0", "W2C1"], + "text": ["r0", "r1", "r2", "r3"], + } + ) + stage = ChunkMergeStage(groupby_columns=["warc_filename", "url"]) + result = stage.process(_make_batch(df)) + + # Same URL but different warc_filename -> 2 separate documents + assert len(result.data) == 2 + + def test_process_custom_separator(self): + """Custom separator is used when concatenating text.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 2, + "chunk_id": [0, 1], + "cleaned_text": ["Part1", "Part2"], + "text": ["r0", "r1"], + } + ) + stage = ChunkMergeStage(separator="\n\n") + result = stage.process(_make_batch(df)) + + assert result.data["cleaned_text"].iloc[0] == "Part1\n\nPart2" + + def test_process_no_raw_text_field(self): + """When raw_text_field is None, only cleaned_text is concatenated.""" + df = pd.DataFrame( + { + "url": ["a.com"] * 2, + "chunk_id": [0, 1], + "cleaned_text": ["C0", "C1"], + } + ) + stage = ChunkMergeStage(raw_text_field=None) + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + assert result.data["cleaned_text"].iloc[0] == "C0\nC1" + + def test_process_token_counts_summed(self): + """Token count columns (num_generated_tokens, num_input_tokens) are summed, not first().""" + df = pd.DataFrame( + { + "url": ["a.com"] * 3, + "chunk_id": [0, 1, 2], + "cleaned_text": ["C0", "C1", "C2"], + "text": ["r0", "r1", "r2"], + "num_generated_tokens": [100, 200, 150], + "num_input_tokens": [50, 80, 70], + "type": ["html", "html", "html"], + } + ) + stage = ChunkMergeStage() + result = stage.process(_make_batch(df)) + + assert len(result.data) == 1 + row = result.data.iloc[0] + assert row["num_generated_tokens"] == 450 + assert row["num_input_tokens"] == 200 + assert row["type"] == "html" # metadata still uses first() diff --git a/tests/stages/text/download/arxiv/test_download.py b/tests/stages/text/download/arxiv/test_download.py index 57faf111f5..1e1ee69ee9 100644 --- a/tests/stages/text/download/arxiv/test_download.py +++ b/tests/stages/text/download/arxiv/test_download.py @@ -20,6 +20,7 @@ import pytest from nemo_curator.stages.text.download.arxiv.download import ArxivDownloader +from nemo_curator.stages.text.download.utils import check_s5cmd_installed class FakeCompletedProcess: @@ -34,10 +35,10 @@ def fake_run_success(cmd: list[str], stdout: str, stderr: str) -> subprocess.Com class TestArxivDownloader: """Test suite for ArxivDownloader.""" - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) @mock.patch("subprocess.run", return_value=mock.Mock(returncode=0)) @pytest.mark.parametrize("verbose", [True, False]) - def test_download_to_path(self, mock_run: mock.Mock, mock_s5cmd: mock.Mock, tmp_path: Path, verbose: bool) -> None: + def test_download_to_path(self, mock_run: mock.Mock, mock_s5cmd_check: mock.Mock, tmp_path: Path, verbose: bool) -> None: """Test _download_to_path with s5cmd.""" downloader = ArxivDownloader(str(tmp_path), verbose=verbose) @@ -59,9 +60,9 @@ def test_download_to_path(self, mock_run: mock.Mock, mock_s5cmd: mock.Mock, tmp_ stderr=stderr, ) - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) @mock.patch("subprocess.run", return_value=mock.Mock(returncode=1, stderr=b"Failed to download")) - def test_download_to_path_failed(self, mock_run: mock.Mock, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + def test_download_to_path_failed(self, mock_run: mock.Mock, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test _download_to_path with failed download.""" downloader = ArxivDownloader(str(tmp_path), verbose=False) @@ -78,24 +79,21 @@ def test_download_to_path_failed(self, mock_run: mock.Mock, mock_s5cmd: mock.Moc stderr=subprocess.DEVNULL, ) - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_init_with_s5cmd(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: - """Test _check_s5cmd_installed when s5cmd is available.""" - downloader = ArxivDownloader(str(tmp_path), verbose=False) - - with mock.patch("subprocess.run") as mock_run: + def test_check_s5cmd_installed_true(self) -> None: + """Test check_s5cmd_installed when s5cmd is available.""" + with mock.patch("nemo_curator.stages.text.download.utils.subprocess.run") as mock_run: mock_run.return_value = None - result = downloader._check_s5cmd_installed() + result = check_s5cmd_installed() assert result is True - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=False) - def test_init_without_s5cmd(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=False) + def test_init_without_s5cmd(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test initialization but s5cmd not installed.""" with pytest.raises(RuntimeError, match="s5cmd is not installed"): ArxivDownloader(str(tmp_path), verbose=False) - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_get_output_filename(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_get_output_filename(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test conversion of URL to output filename.""" downloader = ArxivDownloader(str(tmp_path), verbose=False) @@ -105,7 +103,7 @@ def test_get_output_filename(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> Non assert result == url def test_arxiv_downloader_existing_file(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - with mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True): + with mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True): # Create a temporary download directory and simulate an already-downloaded tar file. download_dir = tmp_path / "downloads" download_dir.mkdir() diff --git a/tests/stages/text/download/arxiv/test_stage.py b/tests/stages/text/download/arxiv/test_stage.py index f7ee0ff066..83b97c6477 100644 --- a/tests/stages/text/download/arxiv/test_stage.py +++ b/tests/stages/text/download/arxiv/test_stage.py @@ -28,8 +28,8 @@ class TestArxivDownloadExtractStage: """Test suite for ArxivDownloadExtractStage.""" - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_decomposition(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_decomposition(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test that ArxivDownloadExtractStage can be decomposed into constituent stages.""" download_dir = str(tmp_path / "downloads") stage = ArxivDownloadExtractStage(download_dir=download_dir) @@ -58,16 +58,16 @@ def test_arxiv_stage_decomposition(self, mock_s5cmd: mock.Mock, tmp_path: Path) assert isinstance(iterate_extract_stage.iterator, ArxivIterator) assert isinstance(iterate_extract_stage.extractor, ArxivExtractor) - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_name(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_name(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test that stage name is as expected.""" download_dir = str(tmp_path / "downloads") stage = ArxivDownloadExtractStage(download_dir=download_dir) assert stage.name == "arxiv_pipeline" - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_description(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_description(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test that stage description is as expected.""" download_dir = str(tmp_path / "downloads") @@ -75,8 +75,8 @@ def test_arxiv_stage_description(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> description = stage.get_description() assert description == "Arxiv pipeline" - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_parameters_propagation(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_parameters_propagation(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test that parameters are properly propagated to constituent stages.""" download_dir = str(tmp_path / "downloads") @@ -108,8 +108,8 @@ def test_arxiv_stage_parameters_propagation(self, mock_s5cmd: mock.Mock, tmp_pat assert iterate_extract_stage.record_limit == 100 assert iterate_extract_stage.filename_col == "custom_filename" - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_inputs_outputs(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_inputs_outputs(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test stage inputs and outputs specification.""" download_dir = str(tmp_path / "downloads") @@ -125,8 +125,8 @@ def test_arxiv_stage_inputs_outputs(self, mock_s5cmd: mock.Mock, tmp_path: Path) # Should produce DocumentBatch with extracted text (from extract stage) + filename column assert outputs == (["data"], ["text", "file_name"]) - @mock.patch.object(ArxivDownloader, "_check_s5cmd_installed", return_value=True) - def test_arxiv_stage_initialization_validation(self, mock_s5cmd: mock.Mock, tmp_path: Path) -> None: + @mock.patch("nemo_curator.stages.text.download.arxiv.download.check_s5cmd_installed", return_value=True) + def test_arxiv_stage_initialization_validation(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test that stage initialization validates parameters correctly.""" download_dir = str(tmp_path / "downloads") diff --git a/tests/stages/text/download/common_crawl/test_download.py b/tests/stages/text/download/common_crawl/test_download.py index fe4e6a8765..981e63de46 100644 --- a/tests/stages/text/download/common_crawl/test_download.py +++ b/tests/stages/text/download/common_crawl/test_download.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ import pytest from nemo_curator.stages.text.download.common_crawl.download import CommonCrawlWARCDownloader +from nemo_curator.stages.text.download.utils import check_s5cmd_installed class TestCommonCrawlWARCDownloader: @@ -42,7 +43,7 @@ def test_download_to_path_wget(self, mock_run: mock.Mock, tmp_path: Path) -> Non stderr=subprocess.PIPE, ) - @mock.patch.object(CommonCrawlWARCDownloader, "_check_s5cmd_installed", return_value=True) + @mock.patch("nemo_curator.stages.text.download.common_crawl.download.check_s5cmd_installed", return_value=True) @mock.patch("subprocess.run", return_value=mock.Mock(returncode=0)) def test_download_to_path_s3(self, mock_run: mock.Mock, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test _download_to_path with s5cmd (use_aws_to_download=True).""" @@ -117,24 +118,20 @@ def test_download_to_path_failed(self, mock_run: mock.Mock, tmp_path: Path) -> N stderr=subprocess.PIPE, ) - def test_check_s5cmd_installed_true(self, tmp_path: Path) -> None: - """Test _check_s5cmd_installed when s5cmd is available.""" - downloader = CommonCrawlWARCDownloader(str(tmp_path), use_aws_to_download=False, verbose=False) - - with mock.patch("subprocess.run") as mock_run: + def test_check_s5cmd_installed_true(self) -> None: + """Test check_s5cmd_installed when s5cmd is available.""" + with mock.patch("nemo_curator.stages.text.download.utils.subprocess.run") as mock_run: mock_run.return_value = None - result = downloader._check_s5cmd_installed() + result = check_s5cmd_installed() assert result is True - def test_check_s5cmd_installed_false(self, tmp_path: Path) -> None: - """Test _check_s5cmd_installed when s5cmd is not available.""" - downloader = CommonCrawlWARCDownloader(str(tmp_path), use_aws_to_download=False, verbose=False) - - with mock.patch("subprocess.run", side_effect=FileNotFoundError): - result = downloader._check_s5cmd_installed() + def test_check_s5cmd_installed_false(self) -> None: + """Test check_s5cmd_installed when s5cmd is not available.""" + with mock.patch("nemo_curator.stages.text.download.utils.subprocess.run", side_effect=FileNotFoundError): + result = check_s5cmd_installed() assert result is False - @mock.patch.object(CommonCrawlWARCDownloader, "_check_s5cmd_installed", return_value=False) + @mock.patch("nemo_curator.stages.text.download.common_crawl.download.check_s5cmd_installed", return_value=False) def test_init_aws_download_without_s5cmd(self, mock_s5cmd_check: mock.Mock, tmp_path: Path) -> None: """Test initialization with AWS download but s5cmd not installed.""" with pytest.raises(RuntimeError, match="s5cmd is not installed"): diff --git a/tutorials/math/0_download.py b/tutorials/math/0_download.py new file mode 100644 index 0000000000..f5d686a70b --- /dev/null +++ b/tutorials/math/0_download.py @@ -0,0 +1,318 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Download math datasets from HuggingFace Hub. + +See --help for usage and README.md for full documentation. +For authentication, set HF_TOKEN or run: huggingface-cli login +""" + +import argparse +import json +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from pathlib import Path + +from huggingface_hub import hf_hub_download, list_repo_files +from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError +from loguru import logger + + +@dataclass +class DownloadConfig: + """Configuration for dataset download.""" + + output_dir: Path + max_files: int | None = None + force: bool = False + workers: int = 1 + + +def load_datasets_config(config_path: Path) -> dict: + """Load dataset configurations from JSON file.""" + with open(config_path) as f: + config = json.load(f) + + # Filter out schema/comment entries + return {k: v for k, v in config.items() if not k.startswith("_")} + + +def parse_huggingface_path(hf_path: str) -> tuple[str, str | None]: + """ + Parse HuggingFace path into repo_id and optional subset/config. + + Examples: + "HuggingFaceTB/finemath/finemath-4plus" -> ("HuggingFaceTB/finemath", "finemath-4plus") + "open-web-math/open-web-math" -> ("open-web-math/open-web-math", None) + "OpenCoder-LLM/opc-fineweb-math-corpus/infiwebmath-4plus" + -> ("OpenCoder-LLM/opc-fineweb-math-corpus", "infiwebmath-4plus") + """ + # Constants for path validation + repo_parts = 2 + repo_with_subset_parts = 3 + + parts = hf_path.split("/") + if len(parts) == repo_parts: + return hf_path, None + elif len(parts) == repo_with_subset_parts: + repo_id = f"{parts[0]}/{parts[1]}" + subset = parts[2] + return repo_id, subset + else: + msg = f"Invalid HuggingFace path format: {hf_path}" + raise ValueError(msg) + + +def get_parquet_files(repo_id: str, subset: str | None = None) -> list[str]: + """Get list of parquet files from a HuggingFace repository.""" + try: + all_files = list_repo_files(repo_id, repo_type="dataset") + except (HfHubHTTPError, RepositoryNotFoundError) as e: + logger.error(f"Failed to list files in {repo_id}: {e}") + raise + + # Filter for parquet files + parquet_files = [f for f in all_files if f.endswith(".parquet")] + + # If subset specified, filter to that subdirectory + if subset: + # Try common patterns: subset/, data/subset/, train/subset/ + subset_patterns = [ + f"{subset}/", + f"data/{subset}/", + f"train/{subset}/", + ] + subset_files = [] + for pattern in subset_patterns: + subset_files.extend([f for f in parquet_files if f.startswith(pattern)]) + + # Use subset files if found, otherwise fallback to files containing subset name + parquet_files = subset_files or [f for f in parquet_files if subset in f] + + if not parquet_files: + logger.warning(f"No parquet files found in {repo_id}" + (f" for subset {subset}" if subset else "")) + + return sorted(parquet_files) + + +def download_single_file( + repo_id: str, + file_path: str, + dataset_dir: Path, + force: bool = False, +) -> tuple[str, Path | None, str | None]: + """ + Download a single file from HuggingFace Hub. + + Args: + repo_id: HuggingFace repository ID + file_path: Path to file within the repository + dataset_dir: Local directory to save the file + force: Force re-download even if file exists + + Returns: + Tuple of (file_path, local_path or None, error_message or None) + """ + try: + # Download directly to target directory (preserves repo structure) + downloaded = hf_hub_download( + repo_id=repo_id, + filename=file_path, + repo_type="dataset", + local_dir=dataset_dir, + force_download=force, + ) + return (file_path, Path(downloaded), None) + except (HfHubHTTPError, RepositoryNotFoundError, OSError) as e: + return (file_path, None, str(e)) + + +def download_dataset( + dataset_name: str, + config: dict, + download_config: DownloadConfig, +) -> Path: + """ + Download a dataset from HuggingFace Hub. + + Args: + dataset_name: Name of the dataset (key in datasets.json) + config: Dataset configuration dict + download_config: Configuration object with output_dir, max_files, force, workers + + Returns: + Path to the downloaded dataset directory + """ + hf_path = config["huggingface"] + repo_id, subset = parse_huggingface_path(hf_path) + + # Create output directory using dataset name (lowercase, underscores) + dataset_dir_name = dataset_name.lower() + dataset_dir = download_config.output_dir / dataset_dir_name + dataset_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Downloading {dataset_name} from {repo_id}" + (f" (subset: {subset})" if subset else "")) + logger.info(f"Output directory: {dataset_dir}") + + # Get list of parquet files + parquet_files = get_parquet_files(repo_id, subset) + + if download_config.max_files: + parquet_files = parquet_files[: download_config.max_files] + logger.info(f"Limiting download to {download_config.max_files} files") + + total_files = len(parquet_files) + logger.info(f"Found {total_files} parquet files to download (workers: {download_config.workers})") + + downloaded_files = [] + failed_files = [] + + with ThreadPoolExecutor(max_workers=download_config.workers) as executor: + futures = { + executor.submit(download_single_file, repo_id, fp, dataset_dir, download_config.force): fp + for fp in parquet_files + } + + for completed, future in enumerate(as_completed(futures), 1): + file_path = futures[future] + filename = Path(file_path).name + + try: + _, local_path, error = future.result() + if error: + logger.error(f"[{completed}/{total_files}] Failed {filename}: {error}") + failed_files.append((file_path, error)) + elif local_path: + logger.info(f"[{completed}/{total_files}] Downloaded {filename}") + downloaded_files.append(local_path) + except (RuntimeError, OSError) as e: + logger.error(f"[{completed}/{total_files}] Failed {filename}: {e}") + failed_files.append((file_path, str(e))) + + # Summary + logger.info(f"Successfully downloaded {len(downloaded_files)} files to {dataset_dir}") + if failed_files: + logger.warning(f"Failed to download {len(failed_files)} files:") + for fp, err in failed_files: + logger.warning(f" - {fp}: {err}") + + return dataset_dir + + +def list_datasets(config: dict) -> None: + """Print available datasets and their info.""" + print("\nAvailable datasets:\n") + print(f"{'Name':<20} {'HuggingFace Path':<50} {'Needs CC Lookup'}") + print("-" * 85) + + for name, cfg in config.items(): + hf_path = cfg.get("huggingface", "N/A") + needs_lookup = "Yes" if cfg.get("needs_cc_lookup", False) else "No" + print(f"{name:<20} {hf_path:<50} {needs_lookup}") + + print("\nUsage: python 0_download.py --dataset --output-dir ./data") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Download math datasets from HuggingFace Hub") + + parser.add_argument( + "--dataset", + nargs="+", + help="Dataset name(s) to download (from datasets.json)", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(os.environ.get("MATH_DATA_DIR", ".")) / "raw", + help="Base output directory for downloaded data (default: $MATH_DATA_DIR/raw or ./raw)", + ) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "datasets.json", + help="Path to datasets.json configuration file", + ) + parser.add_argument( + "--max-files", + type=int, + help="Maximum number of files to download per dataset (for testing)", + ) + parser.add_argument( + "--force", + action="store_true", + help="Force re-download even if files already exist", + ) + parser.add_argument( + "--workers", + type=int, + default=1, + help="Number of parallel download workers (default: 1, recommended: 4-8 for large datasets)", + ) + parser.add_argument( + "--list", + action="store_true", + help="List available datasets and exit", + ) + + args = parser.parse_args() + + # Load configuration + config = load_datasets_config(args.config) + + if args.list: + list_datasets(config) + return + + if not args.dataset: + parser.error("--dataset is required (or use --list to see available datasets)") + + # Validate dataset names + for dataset_name in args.dataset: + if dataset_name not in config: + available = ", ".join(config.keys()) + parser.error(f"Unknown dataset: {dataset_name}\nAvailable: {available}") + + # Download each dataset + args.output_dir.mkdir(parents=True, exist_ok=True) + + for dataset_name in args.dataset: + logger.info(f"\n{'=' * 60}") + logger.info(f"Processing: {dataset_name}") + logger.info(f"{'=' * 60}") + + try: + download_config = DownloadConfig( + output_dir=args.output_dir, + max_files=args.max_files, + force=args.force, + workers=args.workers, + ) + dataset_dir = download_dataset( + dataset_name=dataset_name, + config=config[dataset_name], + download_config=download_config, + ) + logger.info(f"Dataset ready at: {dataset_dir}") + except (HfHubHTTPError, RepositoryNotFoundError, OSError) as e: + logger.error(f"Failed to download {dataset_name}: {e}") + raise + + logger.info("\nAll downloads complete!") + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/1_cc_index_lookup.py b/tutorials/math/1_cc_index_lookup.py new file mode 100644 index 0000000000..865cd178b3 --- /dev/null +++ b/tutorials/math/1_cc_index_lookup.py @@ -0,0 +1,275 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import cudf +import ray +from loguru import logger + +from nemo_curator.core.client import RayClient +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.base import ProcessingStage +from nemo_curator.stages.file_partitioning import FilePartitioningStage +from nemo_curator.stages.resources import Resources +from nemo_curator.tasks import FileGroupTask +from nemo_curator.utils.file_utils import get_all_file_paths_under, get_fs + + +@dataclass +class CCIndexLookupConfig: + """Configuration for CC Index lookup.""" + + input_path: str + output_path: str + cc_index_path: str + crawls: list[str] | None = None + url_col: str = "url" + blocksize: str = "512MiB" + + +CC_INDEX_COLS = [ + "url", + "warc_filename", + "warc_record_offset", + "warc_record_length", + "content_mime_type", + "http_status", +] + + +class CCIndexLookupStage(ProcessingStage[FileGroupTask, FileGroupTask]): + """ + Stage that joins CC Index files against broadcast query URLs using cuDF: + - cudf.read_parquet() for GPU-native reading + - cudf.merge() for GPU-accelerated join + - Query URLs broadcast via Ray object store + """ + + name = "CCIndexLookup" + resources = Resources(gpus=1.0) + + def __init__( + self, + query_urls_ref: ray.ObjectRef, + output_path: str, + url_col: str = "url", + write_kwargs: dict[str, Any] | None = None, + ): + super().__init__() + self.query_urls_ref = query_urls_ref + self.output_path = output_path + self.url_col = url_col + self.write_kwargs = write_kwargs or {} + self._query_df: cudf.DataFrame | None = None + + self.output_fs = get_fs(output_path, self.write_kwargs.get("storage_options")) + self.output_fs.makedirs(output_path, exist_ok=True) + + def setup(self, _worker_metadata: dict | None = None) -> None: + """Load broadcast query URLs from Ray object store.""" + if self._query_df is None: + self._query_df = ray.get(self.query_urls_ref) + logger.info(f"Loaded {len(self._query_df):,} query URLs from broadcast") + + def process(self, task: FileGroupTask) -> FileGroupTask: + """Process CC Index files and join against query URLs.""" + if self._query_df is None: + msg = "Query URLs not loaded. Call setup() first." + raise RuntimeError(msg) + + output_files = [] + total_input = 0 + total_matched = 0 + + for cc_index_file in task.data: + cc_df = cudf.read_parquet(cc_index_file, columns=CC_INDEX_COLS) + total_input += len(cc_df) + + matched = cc_df.merge(self._query_df, on="url", how="inner") + total_matched += len(matched) + + if len(matched) == 0: + continue + + # Convert category columns to strings (cudf parquet writer limitation) + for col in matched.columns: + if matched[col].dtype.name == "category": + matched[col] = matched[col].astype(str) + + # Write output + output_file = self.output_fs.sep.join([self.output_path, Path(cc_index_file).stem + "_enriched.parquet"]) + matched.to_parquet(output_file, **self.write_kwargs) + output_files.append(output_file) + + logger.debug(f"Processed {len(task.data)} files: {total_input:,} -> {total_matched:,} rows") + + return FileGroupTask( + task_id=task.task_id, + dataset_name=task.dataset_name, + data=output_files, + _metadata={ + "input_rows": total_input, + "matched_rows": total_matched, + }, + ) + + +def collect_unique_urls(input_path: str, url_col: str = "url") -> cudf.DataFrame: + """Collect unique URLs from input dataset using cuDF.""" + logger.info(f"Collecting unique URLs from: {input_path}") + + input_files = get_all_file_paths_under(input_path, keep_extensions=[".parquet"]) + if not input_files: + msg = f"No parquet files found at {input_path}" + raise FileNotFoundError(msg) + + logger.info(f"Found {len(input_files)} input files") + + dfs = [cudf.read_parquet(f, columns=[url_col]) for f in input_files] + combined = cudf.concat(dfs, ignore_index=True) + unique_urls = combined.drop_duplicates(subset=[url_col]) + + if url_col != "url": + unique_urls = unique_urls.rename(columns={url_col: "url"}) + + logger.info(f"Collected {len(unique_urls):,} unique URLs") + return unique_urls + + +def get_available_crawls(cc_index_base: str) -> list[str]: + """Auto-detect available crawl directories.""" + if not os.path.exists(cc_index_base): + return [] + crawl_dirs = [d for d in os.listdir(cc_index_base) if d.startswith("crawl=")] + return sorted([d.split("=")[1] for d in crawl_dirs]) + + +def get_cc_index_files(cc_index_base: str, crawls: list[str] | None = None) -> list[str]: + """Collect all CC Index parquet files for specified crawls (or all if None).""" + if crawls is None: + crawls = get_available_crawls(cc_index_base) + if not crawls: + msg = f"No crawl directories found at {cc_index_base}" + raise FileNotFoundError(msg) + logger.info(f"Auto-detected {len(crawls)} crawls: {crawls}") + + all_files = [] + for crawl in crawls: + crawl_path = os.path.join(cc_index_base, f"crawl={crawl}", "subset=warc") + if os.path.exists(crawl_path): + files = get_all_file_paths_under(crawl_path, keep_extensions=[".parquet"]) + all_files.extend(files) + logger.info(f"Found {len(files)} CC Index files for {crawl}") + else: + logger.warning(f"CC Index path not found: {crawl_path}") + + if not all_files: + msg = f"No CC Index files found for crawls {crawls}" + raise FileNotFoundError(msg) + + return all_files + + +def run_cc_index_lookup(config: CCIndexLookupConfig) -> None: + ray_client = RayClient() + ray_client.start() + + try: + # Step 1: Collect query URLs and broadcast + query_urls = collect_unique_urls(config.input_path, config.url_col) + query_urls_ref = ray.put(query_urls) + logger.info("Query URLs broadcast to Ray object store") + + # Step 2: Collect CC Index files from all crawls + cc_files = get_cc_index_files(config.cc_index_path, config.crawls) + logger.info(f"Total CC Index files: {len(cc_files)}") + + # Step 3: Build pipeline + pipeline = Pipeline( + name="cc_index_lookup", + stages=[ + FilePartitioningStage( + file_paths=cc_files, + blocksize=config.blocksize, + ), + CCIndexLookupStage( + query_urls_ref=query_urls_ref, + output_path=config.output_path, + url_col=config.url_col, + ), + ], + ) + + logger.info(pipeline.describe()) + + # Step 4: Run pipeline + result_tasks = pipeline.run() + + # Summarize + total_input = sum(t._metadata.get("input_rows", 0) for t in result_tasks) + total_matched = sum(t._metadata.get("matched_rows", 0) for t in result_tasks) + + logger.info("=" * 60) + logger.info(f"Query URLs: {len(query_urls):,}") + logger.info(f"CC Index rows: {total_input:,}") + logger.info(f"Matched rows: {total_matched:,}") + logger.info(f"Output: {config.output_path}") + logger.info("=" * 60) + + finally: + ray_client.stop() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Enrich dataset with WARC metadata using Curator's cuDF pattern.", + ) + parser.add_argument("--input", required=True, help="Input directory with parquet files") + parser.add_argument("--output", required=True, help="Output directory") + parser.add_argument( + "--cc-index-path", + required=True, + help="CC Index path (/crawl=CC-MAIN-YYYY-WW/subset=warc/)", + ) + parser.add_argument( + "--crawls", + nargs="+", + default=None, + help="Crawl IDs (default: auto-detect all available crawls)", + ) + parser.add_argument("--url-col", default="url", help="URL column name") + parser.add_argument("--blocksize", default="512MiB", help="File block size") + + args = parser.parse_args() + + config = CCIndexLookupConfig( + input_path=args.input, + output_path=args.output, + cc_index_path=args.cc_index_path, + crawls=args.crawls, + url_col=args.url_col, + blocksize=args.blocksize, + ) + run_cc_index_lookup(config) + + logger.info(f"Next: python 2_text_preprocess.py --input {args.output}/*.parquet --fetch-cc") + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/2_text_preprocess.py b/tutorials/math/2_text_preprocess.py new file mode 100644 index 0000000000..61c3947c2f --- /dev/null +++ b/tutorials/math/2_text_preprocess.py @@ -0,0 +1,157 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from dataclasses import dataclass + +import ray.data +from loguru import logger + +from nemo_curator.core.client import RayClient +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.math.download.extract import MathContentExtractor, MathExtractStage +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.download.common_crawl.download import CommonCrawlWARCReader +from nemo_curator.stages.text.io.reader import ParquetReader +from nemo_curator.stages.text.io.writer import JsonlWriter + + +@dataclass +class TextPreprocessConfig: + """Configuration for text preprocessing.""" + + input_glob: str + output_dir: str + fetch_cc: bool = False + warc_filename_col: str = "warc_filename" + warc_record_offset_col: str = "warc_record_offset" + warc_record_length_col: str = "warc_record_length" + + +def build_pipeline(config: TextPreprocessConfig) -> Pipeline: + p = Pipeline(name="math_text_preprocess", description="Decode (binary) → type → html via lynx → text") + + p.add_stage( + ParquetReader(file_paths=config.input_glob).with_( + { + "file_partitioning": {"resources": Resources(cpus=1.0)}, + "parquet_reader": {"resources": Resources(cpus=1.0)}, + } + ) + ) + + if config.fetch_cc: + logger.info("Adding CommonCrawlWARCReader stage to fetch content from S3.") + p.add_stage( + CommonCrawlWARCReader( + warc_filename_col=config.warc_filename_col, + warc_record_offset_col=config.warc_record_offset_col, + warc_record_length_col=config.warc_record_length_col, + ).with_(resources=Resources(cpus=0.5)) # Lightweight network op + ) + + p.add_stage( + MathExtractStage( + extractor=MathContentExtractor( + binary_column="binary_content", url_column="url", mime_type_column="content_mime_type" + ), + add_filename_column=False, + ).with_(resources=Resources(cpus=1.0)) + ) + + p.add_stage(JsonlWriter(path=config.output_dir).with_(resources=Resources(cpus=1.0))) + + return p + + +def report_extraction_stats(output_dir: str) -> None: + """Optional: Report extraction statistics by reading output with Ray Data.""" + try: + from nemo_curator.utils.file_utils import get_all_file_paths_under + + jsonl_files = get_all_file_paths_under(output_dir, keep_extensions=[".jsonl"]) + if not jsonl_files: + logger.debug(f"No JSONL files found in {output_dir}") + return + + ds = ray.data.read_json(jsonl_files) + total = ds.count() + html_docs = ds.filter(lambda row: row.get("type") == "html").count() + html_failed = ds.filter( + lambda row: row.get("type") == "html" and (not row.get("text") or row.get("text").strip() == "") + ).count() + + logger.info( + f"Extraction stats: {total} total documents, {html_docs} HTML, {html_failed} HTML extraction failures" + ) + except Exception as e: # noqa: BLE001 + logger.debug(f"Could not compute stats (optional): {e}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run math text preprocessing on Parquet files") + parser.add_argument("--input", required=True, help="Glob or directory for Parquet input files") + parser.add_argument("--output", required=True, help="Output directory for JSONL results") + parser.add_argument("--report-stats", action="store_true", help="Report extraction statistics after processing") + parser.add_argument( + "--fetch-cc", + action="store_true", + help="Fetch raw content from Common Crawl S3 using WARC metadata (requires 'warc_filename', 'warc_record_offset', 'warc_record_length' columns).", + ) + parser.add_argument( + "--warc-filename-col", + default="warc_filename", + help="Column name for WARC filename (default: 'warc_filename')", + ) + parser.add_argument( + "--offset-col", + default="warc_record_offset", + help="Column name for WARC record offset (default: 'warc_record_offset')", + ) + parser.add_argument( + "--length-col", + default="warc_record_length", + help="Column name for WARC record length (default: 'warc_record_length')", + ) + + args = parser.parse_args() + + ray_client = RayClient() + ray_client.start() + + try: + config = TextPreprocessConfig( + input_glob=args.input, + output_dir=args.output, + fetch_cc=args.fetch_cc, + warc_filename_col=args.warc_filename_col, + warc_record_offset_col=args.offset_col, + warc_record_length_col=args.length_col, + ) + pipeline = build_pipeline(config) + logger.info(pipeline.describe()) + + pipeline.run() + + logger.info("Pipeline completed successfully.") + + # Optional: Report extraction statistics + if args.report_stats: + report_extraction_stats(args.output) + finally: + ray_client.stop() + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/3_llm_cleanup.py b/tutorials/math/3_llm_cleanup.py new file mode 100644 index 0000000000..3bf2768eab --- /dev/null +++ b/tutorials/math/3_llm_cleanup.py @@ -0,0 +1,266 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import os +from datetime import datetime + +import pandas as pd +from loguru import logger + +from nemo_curator.core.client import RayClient +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.math.modifiers.chunking import TokenSplitterStage +from nemo_curator.stages.math.modifiers.llm_cleanup import LLMCleanupStage +from nemo_curator.stages.math.modifiers.merge_chunks import ChunkMergeStage +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.io.reader import JsonlReader, ParquetReader +from nemo_curator.stages.text.io.writer import JsonlWriter +from nemo_curator.stages.text.modifiers import Modify +from nemo_curator.utils import prompts + + +def fill_null_text(text: str | None) -> str: + """Fill null/NaN text values with empty string.""" + if pd.isna(text) or text is None: + return "" + return str(text) + + +def build_pipeline( # noqa: PLR0913 + input_files: list[str], + reader_type: str, + output_dir: str, + model: str, + prompt: str, + chunk_length: int | None = None, + chunk_data: bool = False, + classification: bool = False, + max_model_len: int | None = None, + temperature: float = 0.7, + top_p: float = 0.8, + top_k: int = 20, + min_p: float = 0.0, + max_tokens: int | None = None, + cache_dir: str | None = None, + groupby_columns: list[str] | None = None, + max_text_length: int = 900_000, +) -> Pipeline: + """Build the LLM cleanup pipeline.""" + p = Pipeline( + name="math_cleanup_webpages_with_llm", + description="Clean up HTML/text content using LLM (with optional chunking)", + ) + + # Reader stage + if reader_type == "parquet": + p.add_stage( + ParquetReader(file_paths=input_files).with_( + { + "file_partitioning": {"resources": Resources(cpus=0.5)}, + "parquet_reader": {"resources": Resources(cpus=0.5)}, + } + ) + ) + else: + p.add_stage( + JsonlReader(file_paths=input_files).with_( + { + "file_partitioning": {"resources": Resources(cpus=0.5)}, + "jsonl_reader": {"resources": Resources(cpus=0.5)}, + } + ) + ) + + p.add_stage( + Modify(modifier_fn=fill_null_text, input_fields="text", output_fields="text").with_( + resources=Resources(cpus=1.0) + ) + ) + + # Optional chunking stage + if chunk_data and chunk_length: + p.add_stage( + TokenSplitterStage( + model_name=model, + text_field="text", + max_length_tokens=chunk_length, + ).with_(resources=Resources(cpus=1.0)) + ) + + # Get prompt from prompts module + try: + system_prompt = getattr(prompts, prompt) + except AttributeError: + logger.warning( + f"Prompt '{prompt}' not found in prompts module, using as literal string. " + f"Available: {[p for p in dir(prompts) if p.isupper()]}" + ) + system_prompt = prompt + + # LLM cleanup stage + p.add_stage( + LLMCleanupStage( + model=model, + system_prompt=system_prompt, + text_field="text", + output_field="cleaned_text", + max_model_len=max_model_len, + classification=classification, + temperature=temperature, + top_p=top_p, + top_k=top_k, + min_p=min_p, + max_tokens=max_tokens, + cache_dir=cache_dir, + ).with_(resources=Resources(cpus=1.0, gpus=1.0)) + ) + + # Optional chunk merge stage + if chunk_data and chunk_length: + p.add_stage( + ChunkMergeStage( + text_field="cleaned_text", + raw_text_field="text", + chunk_id_field="chunk_id", + groupby_columns=groupby_columns, + max_text_length=max_text_length, + ).with_(resources=Resources(cpus=1.0)) + ) + + # Writer stage + p.add_stage(JsonlWriter(path=output_dir).with_(resources=Resources(cpus=1.0))) + + return p + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Clean up webpages using LLM with optional chunking", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--input", required=True, help="Input directory or glob pattern for JSONL/Parquet files") + parser.add_argument("--output", required=True, help="Output directory for cleaned JSONL files") + parser.add_argument("--model", required=True, help="Model identifier (e.g., microsoft/phi-4)") + parser.add_argument( + "--prompt", + required=True, + help="""Prompt name from prompts module (e.g., HTML_TO_TEXT_PROMPT). + Must match one of the prompts from the prompts module.""", + ) + parser.add_argument( + "--input_filetype", + choices=["jsonl", "parquet"], + default="jsonl", + help="Input file type", + ) + parser.add_argument( + "--chunk_data", + action="store_true", + help="Enable token-based chunking before LLM processing", + ) + parser.add_argument( + "--chunk_length", + type=int, + default=None, + help="Maximum tokens per chunk when chunk_data is enabled", + ) + parser.add_argument( + "--classification", + action="store_true", + help="Use classification mode (outputs 'label' instead of 'cleaned_text')", + ) + parser.add_argument( + "--max_model_len", + type=int, + default=None, + help="Maximum model context length. If not specified, vLLM will auto-detect from model config.", + ) + parser.add_argument("--temperature", type=float, default=0.7, help="Sampling temperature") + parser.add_argument("--top_p", type=float, default=0.8, help="Top-p sampling parameter") + parser.add_argument("--top_k", type=int, default=20, help="Top-k sampling parameter") + parser.add_argument("--min_p", type=float, default=0.0, help="Min-p sampling parameter (for Qwen3)") + parser.add_argument("--max_tokens", type=int, default=None, help="Maximum tokens to generate") + parser.add_argument("--cache_dir", type=str, default=None, help="Cache directory for model weights") + parser.add_argument( + "--groupby", + nargs="+", + default=["url"], + help="Columns to group by for chunk merging (e.g., url, warc_filename)", + ) + parser.add_argument("--max_text_length", type=int, default=900_000, help="Maximum merged text length in chars") + + args = parser.parse_args() + + if args.chunk_data and not args.chunk_length: + parser.error("--chunk_length is required when --chunk_data is enabled") + if args.chunk_data and not args.max_model_len: + parser.error("--max_model_len is required when --chunk_data is enabled for models not in the spec") + + if os.path.isdir(args.input): + if args.input_filetype == "parquet": + input_files = glob.glob(os.path.join(args.input, "**/*.parquet"), recursive=True) + else: + input_files = glob.glob(os.path.join(args.input, "**/*.jsonl"), recursive=True) + input_files.extend(glob.glob(os.path.join(args.input, "**/*.json"), recursive=True)) + else: + input_files = glob.glob(args.input) + + if not input_files: + logger.error(f"No input files found matching: {args.input}") + return + + logger.info(f"Found {len(input_files)} input files") + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") # noqa: DTZ005 + output_dir = os.path.join(args.output, f"cleanup_{timestamp}") + + ray_client = RayClient() + ray_client.start() + + try: + pipeline = build_pipeline( + input_files=input_files, + reader_type=args.input_filetype, + output_dir=output_dir, + model=args.model, + prompt=args.prompt, + chunk_length=args.chunk_length, + chunk_data=args.chunk_data, + classification=args.classification, + max_model_len=args.max_model_len, + temperature=args.temperature, + top_p=args.top_p, + top_k=args.top_k, + min_p=args.min_p, + max_tokens=args.max_tokens, + cache_dir=args.cache_dir, + groupby_columns=args.groupby, + max_text_length=args.max_text_length, + ) + + logger.info(pipeline.describe()) + + pipeline.run() + + logger.info("Pipeline completed successfully.") + logger.info(f"Output written to: {output_dir}") + + finally: + ray_client.stop() + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/4_quality_classifier.py b/tutorials/math/4_quality_classifier.py new file mode 100644 index 0000000000..3b8aca2f44 --- /dev/null +++ b/tutorials/math/4_quality_classifier.py @@ -0,0 +1,93 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +from loguru import logger + +from nemo_curator.core.client import RayClient +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.math.classifiers.finemath import FineMathClassifier +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.io.reader import JsonlReader +from nemo_curator.stages.text.io.writer import JsonlWriter + + +def build_pipeline(input_glob: str, output_dir: str, text_field: str = "text") -> Pipeline: + p = Pipeline( + name="math_quality_classifier", + description="Classify mathematical content quality using the FineMath model", + ) + + # Reader (composite): tune both file partitioning and jsonl_reader stages + p.add_stage( + JsonlReader(file_paths=input_glob).with_( + { + "file_partitioning": {"resources": Resources(cpus=1.0)}, + "jsonl_reader": {"resources": Resources(cpus=1.0)}, + } + ) + ) + + # Classifier (composite): tune tokenizer and model sub-stages + p.add_stage( + FineMathClassifier(text_field=text_field).with_( + { + "finemath_classifier_tokenizer": {"resources": Resources(cpus=1.0)}, + "finemath_classifier_model": {"resources": Resources(cpus=1.0, gpus=1.0)}, + } + ) + ) + + # Writer (single stage) + p.add_stage(JsonlWriter(path=output_dir).with_(resources=Resources(cpus=1.0))) + + return p + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run Math (FineMath) classifier on JSONL files", + ) + parser.add_argument( + "--input", + required=True, + help="Glob or directory for JSONL input files", + ) + parser.add_argument( + "--output", + required=True, + help="Output directory for JSONL results", + ) + parser.add_argument( + "--text-field", + default="text", + help="Column to classify (default: 'text', use 'cleaned_text' after LLM cleanup)", + ) + args = parser.parse_args() + + ray_client = RayClient() + ray_client.start() + + try: + pipeline = build_pipeline(args.input, args.output, text_field=args.text_field) + logger.info(pipeline.describe()) + + pipeline.run() + finally: + ray_client.stop() + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/5_deduplication.py b/tutorials/math/5_deduplication.py new file mode 100644 index 0000000000..3eebb344eb --- /dev/null +++ b/tutorials/math/5_deduplication.py @@ -0,0 +1,186 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from pathlib import Path + +from loguru import logger + +from nemo_curator.core.client import RayClient +from nemo_curator.stages.deduplication.fuzzy.workflow import FuzzyDeduplicationWorkflow +from nemo_curator.stages.text.deduplication.removal_workflow import TextDuplicatesRemovalWorkflow + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Run fuzzy deduplication on Parquet or JSONL files", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--input", + type=str, + required=True, + help="Input directory path for Parquet/JSONL files", + ) + parser.add_argument( + "--cache_dir", + type=str, + required=True, + help="Cache directory for deduplication intermediates (must be empty between runs)", + ) + parser.add_argument( + "--duplicate_ids_dir", + type=str, + required=True, + help="Output directory for duplicate IDs and id generator mapping", + ) + parser.add_argument( + "--output", + type=str, + required=True, + help="Output directory for deduplicated data", + ) + parser.add_argument( + "--text_field", + type=str, + default="text", + help="Field containing the text to deduplicate", + ) + parser.add_argument( + "--input_filetype", + type=str, + choices=["parquet", "jsonl"], + default="jsonl", + help="Input file type (auto-detected if not specified)", + ) + parser.add_argument( + "--input_blocksize", + type=str, + default="1GiB", + help="Size of input blocks to read", + ) + parser.add_argument( + "--bands_per_iteration", + type=int, + default=5, + help="Number of bands to shuffle concurrently (reduce if OOM)", + ) + # MinHash + LSH parameters + parser.add_argument( + "--seed", + type=int, + default=42, + help="Seed for minhash permutations", + ) + parser.add_argument( + "--char_ngrams", + type=int, + default=24, + help="Size of character n-grams for MinHash (recommended: >= 20)", + ) + parser.add_argument( + "--num_bands", + type=int, + default=20, + help="Number of bands/buckets for LSH", + ) + parser.add_argument( + "--minhashes_per_band", + type=int, + default=13, + help="Number of hashes per band", + ) + parser.add_argument( + "--use_64_bit_hash", + action="store_true", + default=False, + help="Use 64-bit hash function (default: 32-bit)", + ) + + args = parser.parse_args() + + cache_files = list(Path(args.cache_dir).glob("*")) + if cache_files: + logger.warning( + f"Cache directory {args.cache_dir} is not empty. " + "It's recommended to clear it between runs to avoid conflicts." + ) + + if args.input_filetype == "parquet": + input_file_extensions = [".parquet"] + elif args.input_filetype == "jsonl": + input_file_extensions = [".jsonl", ".json"] + + ray_client = RayClient() + ray_client.start() + + try: + logger.info("Running fuzzy deduplication workflow to identify duplicate IDs...") + fuzzy_workflow = FuzzyDeduplicationWorkflow( + input_path=args.input, + cache_path=args.cache_dir, + output_path=args.duplicate_ids_dir, + input_filetype=args.input_filetype, + input_file_extensions=input_file_extensions, + input_blocksize=args.input_blocksize, + text_field=args.text_field, + perform_removal=False, + char_ngrams=args.char_ngrams, + num_bands=args.num_bands, + minhashes_per_band=args.minhashes_per_band, + use_64_bit_hash=args.use_64_bit_hash, + bands_per_iteration=args.bands_per_iteration, + seed=args.seed, + ) + fuzzy_workflow.run() + + duplicate_ids_path = os.path.join(args.duplicate_ids_dir, "FuzzyDuplicateIds") + id_generator_path = os.path.join(args.duplicate_ids_dir, "fuzzy_id_generator.json") + + # Check if duplicates were found + if not os.path.exists(duplicate_ids_path): + logger.info("No duplicates found. Copying input to output directory...") + import shutil + + if os.path.exists(args.output): + logger.warning(f"Removing existing output directory: {args.output}") + shutil.rmtree(args.output) + shutil.copytree(args.input, args.output) + logger.info(f"All documents are unique. Copied {args.input} → {args.output}") + else: + logger.info("Running text duplicates removal workflow to remove duplicates...") + removal_workflow = TextDuplicatesRemovalWorkflow( + input_path=args.input, + ids_to_remove_path=duplicate_ids_path, + output_path=args.output, + input_filetype=args.input_filetype, + output_filetype=args.input_filetype, + input_file_extensions=input_file_extensions, + id_field="_curator_dedup_id", + duplicate_id_field="_curator_dedup_id", + input_blocksize=args.input_blocksize, + id_generator_path=id_generator_path, + ) + removal_workflow.run() + + logger.info("Pipeline completed successfully.") + logger.info(f"Deduplication complete! Deduplicated output: {args.output}") + + finally: + ray_client.stop() + + +if __name__ == "__main__": + main() diff --git a/tutorials/math/README.md b/tutorials/math/README.md new file mode 100644 index 0000000000..16e926a64e --- /dev/null +++ b/tutorials/math/README.md @@ -0,0 +1,519 @@ +# Math Data Curation Pipeline + +This example demonstrates a complete pipeline for curating mathematical content from Common Crawl, including Common Crawl Index lookup, text preprocessing, quality classification, deduplication, and LLM-based cleanup. + +## Install +Use uv to create the project environment and install Curator with the math extra: + +```bash +uv sync --extra math_cuda12 +source .venv/bin/activate +``` + +**Note:** GPU detection - if `nvidia-smi` shows GPUs but examples log "No gpus found", `pynvml` may need to be reinstalled: +```bash +uv pip install --force-reinstall pynvml +``` + +## Prerequisites + +### System Dependencies +- A100 or above GPU(s) with CUDA for the Hugging Face model and vLLM +- Python environment with `nemo-curator[math_cuda12]` installed (uv sync above) +- Lynx system dependency for HTML rendering to text: + - Ubuntu/Debian: `sudo apt-get update && sudo apt-get install -y lynx` + - RHEL/Fedora: `sudo dnf install -y lynx` (or `sudo yum install -y lynx`) + - Conda: `conda install -c conda-forge lynx` + +### Common Crawl (CC) Index Requirements + +The index lookup script (`1_cc_index_lookup.py`) uses **cuDF** against a local CC Index in **parquet format**. + +**Key points:** +- **Local CC Index required**: Download CC Index parquet files locally for GPU-accelerated joins +- **Parquet format**: The script expects parquet files with columns: `url`, `warc_filename`, `warc_record_offset`, `warc_record_length`, `content_mime_type`, `http_status` +- **Hive partitioning**: Files must be in `/crawl=CC-MAIN-YYYY-WW/subset=warc/*.parquet` structure +- **Distributed processing**: Uses Ray for distributed execution across multiple GPUs + +**CC Index Access Options:** + +| Method | Access | Format | Recommended | +|--------|--------|--------|-------------| +| **Columnar Index (S3)** | Requires AWS credentials | Parquet (ready to use) | ✅ Yes | +| **CDX Index (HTTPS)** | Public, no auth needed | Gzip JSON (needs conversion) | Fallback only | + +**Option 1: Columnar Index via S3 (Recommended)** + +The CC Index is available as a pre-built parquet table on S3. This is the fastest approach and requires AWS credentials: + +```bash +# Download parquet files for a specific crawl (~300 partitions, ~1GB each) +aws s3 cp s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-10/subset=warc/ \ + /local/path/cc-index/crawl=CC-MAIN-2024-10/subset=warc/ --recursive + +# For testing: download just a few partition files +aws s3 cp s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-2024-10/subset=warc/part-00000.parquet \ + /local/path/cc-index/crawl=CC-MAIN-2024-10/subset=warc/ +``` + +**Option 2: CDX Index via HTTPS** + +You can also download CDX files via HTTPS and convert them to parquet. CDX files are gzip-compressed JSON lines. + +```bash +# Get the list of CDX files for a crawl: +curl -s https://data.commoncrawl.org/crawl-data/CC-MAIN-2024-10/cc-index.paths.gz | gunzip +``` + +CDX to parquet conversion requires parsing JSON and extracting the required columns with proper types: +- `url` (string), `warc_filename` (string from `filename` field) +- `warc_record_offset` (int64 from `offset`), `warc_record_length` (int64 from `length`) +- `content_mime_type` (string from `mime`), `http_status` (string from `status`) + +## Understanding Index Lookup + +Some datasets (OpenWebMath, InfiWebMath, MegaMath) only have URLs without WARC metadata. To fetch their content from Common Crawl, you first need to look up each URL's location in the Common Crawl (CC) Index. + +The CC Index is [**publicly available on S3**](https://commoncrawl.org/access-the-data). + +### How the Lookup Process Works + +``` +┌─────────────────────────┐ ┌───────────────────────────┐ +│ Your Dataset │ │ CC Index on S3 │ +│ (e.g., OpenWebMath) │ │ (queried directly) │ +├─────────────────────────┤ ├───────────────────────────┤ +│ url │ │ url │ +│ text (optional) │ │ filename (WARC path) │ +│ ...other columns │ │ offset (byte position) │ +└───────────┬─────────────┘ │ length (record size) │ + │ │ mime, status │ + │ └─────────────┬─────────────┘ + │ │ + └──────────────┬──────────────────────┘ + │ + INNER JOIN ON url + │ + ▼ + ┌──────────────────────────────────────┐ + │ Enriched Dataset │ + ├──────────────────────────────────────┤ + │ url ← original │ + │ text ← original │ + │ ...other columns ← original │ + │ warc_filename ← from index │ + │ warc_record_offset ← from index │ + │ warc_record_length ← from index │ + │ content_mime_type ← from index │ + │ http_status ← from index │ + └──────────────────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────┐ + │ 2_text_preprocess.py --fetch-cc │ + │ Uses WARC metadata to fetch actual │ + │ content from Common Crawl S3 │ + └──────────────────────────────────────┘ +``` + +**Key Point:** The inner join means only URLs found in the CC Index are kept. URLs not in the specified crawl(s) are dropped. + +### CC Index Location + +The CC Index is available in two formats: + +| Format | Location | Notes | +|--------|----------|-------| +| **Columnar (Parquet)** | `s3://commoncrawl/cc-index/table/cc-main/warc/crawl=CC-MAIN-YYYY-WW/` | Requires AWS credentials | +| **CDX (Gzip JSON)** | `https://data.commoncrawl.org/cc-index/collections/CC-MAIN-YYYY-WW/indexes/` | Public HTTPS access | + +Available crawl IDs: https://index.commoncrawl.org/ + +## Dataset Configuration + +Dataset metadata is stored in `datasets.json`. This file is used by `0_download.py` to download datasets from HuggingFace Hub and describes which datasets require CC Index lookup. + +Each dataset entry contains: +- `huggingface`: Source repository path for downloading +- `needs_cc_lookup`: Whether CC Index lookup is required (datasets without WARC metadata) +- `url_col`: Column name containing URLs (for datasets needing CC lookup) + +### Pre-configured Datasets + +| Dataset | HuggingFace Source | Has WARC Metadata | CC Index Lookup Required | +|---------|-------------------|-------------------|-------------------------| +| `FINEMATH_4PLUS` | HuggingFaceTB/finemath | ✅ Yes | No | +| `FINEMATH_3PLUS` | HuggingFaceTB/finemath | ✅ Yes | No | +| `OPENWEBMATH` | open-web-math/open-web-math | ❌ No | Yes | +| `OPC_FINEWEB_MATH` | OpenCoder-LLM/opc-fineweb-math-corpus | ❌ No | Yes | +| `MEGAMATH_PRO` | LLM360/MegaMath | ❌ No | Yes | +| `MEGAMATH_WEB` | LLM360/MegaMath | ❌ No | Yes | + +### Custom Dataset Configuration + +To add your own dataset, edit `datasets.json`: + +```json +{ + "MY_DATASET": { + "huggingface": "my-org/my-dataset", + "needs_cc_lookup": true, + "url_col": "url" + } +} +``` + +For datasets with WARC metadata already included, set `"needs_cc_lookup": false` and omit `url_col`. + + +## Complete Pipeline Flow + +```mermaid +flowchart TD + subgraph download["Download (Optional)"] + HF[("HuggingFace Hub")] + DL["0_download.py"] + RAW["Raw Parquet Files
$MATH_DATA_DIR/raw/"] + end + + subgraph datasets["Input Datasets"] + D1["FineMath 3+/4+
Has WARC metadata"] + D2["OpenWebMath
InfiWebMath
MegaMath
URL only"] + end + + subgraph step1["Step 1: CC Index Lookup"] + CC_INDEX[("CC Index on S3
s3://commoncrawl/cc-index/")] + LOOKUP["1_cc_index_lookup.py"] + ENRICHED["Enriched Dataset
+ warc_filename
+ offset, length"] + end + + subgraph step2["Step 2: Text Preprocessing"] + CC_S3[("Common Crawl S3
s3://commoncrawl/crawl-data/")] + EXTRACT["2_text_preprocess.py
--fetch-cc"] + PREPROCESSED["Preprocessed Data
text, url, type"] + end + + subgraph step3["Step 3: LLM Cleanup"] + LLM["3_llm_cleanup.py
vLLM + Phi-4"] + FINAL["Cleaned Data
+ cleaned_text"] + end + + subgraph step4["Step 4: Quality Classification"] + CLASSIFY["4_quality_classifier.py
FineMath model"] + CLASSIFIED["Classified Data
+ finemath_scores"] + end + + subgraph step5["Step 5: Deduplication"] + DEDUP["5_deduplication.py
Fuzzy matching"] + DEDUPED["Deduplicated Data"] + end + + %% Download flow (optional) + HF -->|"Download"| DL + DL --> RAW + RAW --> D1 + RAW --> D2 + + %% Flow for datasets WITH WARC metadata + D1 -->|"Has warc_filename,
offset, length"| EXTRACT + + %% Flow for datasets WITHOUT WARC metadata + D2 -->|"URL only"| LOOKUP + LOOKUP <-->|"Query"| CC_INDEX + LOOKUP --> ENRICHED + ENRICHED --> EXTRACT + + %% Common flow after preprocessing + EXTRACT <-->|"Fetch from Common Crawl
HTTPS range requests"| CC_S3 + EXTRACT --> PREPROCESSED + PREPROCESSED --> LLM + LLM --> FINAL + FINAL --> CLASSIFY + CLASSIFY --> CLASSIFIED + CLASSIFIED --> DEDUP + DEDUP --> DEDUPED + + %% ========================================== + %% NVIDIA Color Scheme + %% ========================================== + %% Primary Green (#76b900): Python scripts / processing tools + %% Dark Gray (#666666): Input and final datasets + %% Light Gray (#999999): Intermediate outputs + %% Purple (#7b68ee): Data stores, S3 buckets (cylinders) + %% All boxes have black borders (#000000) + + classDef nvidiaGreen fill:#76b900,stroke:#000000,color:white,stroke-width:2px + classDef nvidiaGray fill:#666666,stroke:#000000,color:white,stroke-width:2px + classDef nvidiaLightGray fill:#999999,stroke:#000000,color:white,stroke-width:2px + classDef nvidiaPurple fill:#c9b8e6,stroke:#000000,color:#333,stroke-width:2px + + %% Apply NVIDIA colors to nodes + %% Input datasets - Gray + class D1,D2 nvidiaGray + + %% External data stores - Purple (cylinders) + class HF,CC_INDEX,CC_S3 nvidiaPurple + + %% Processing steps (Python scripts) - NVIDIA Green + class DL,LOOKUP,EXTRACT,LLM,CLASSIFY,DEDUP nvidiaGreen + + %% Intermediate outputs - Light Gray + class RAW,ENRICHED,PREPROCESSED,FINAL,CLASSIFIED nvidiaLightGray + + %% Final output - Gray + class DEDUPED nvidiaGray + + %% Subgraph styling (Note: subgraph styling support varies by renderer) + style download fill:transparent,stroke:#000000,stroke-width:2px,stroke-dasharray:5 5,color:#333 + style datasets fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + style step1 fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + style step2 fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + style step3 fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + style step4 fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + style step5 fill:transparent,stroke:#000000,stroke-width:2px,color:#333 + + %% Link/Arrow styling (Note: linkStyle support varies by renderer) + linkStyle default stroke:#76b900,stroke-width:2px +``` + +### Pipeline Summary + +| Step | Script | Input | Output | Required For | +|------|--------|-------|--------|--------------| +| 0 | `0_download.py` | HuggingFace | Raw parquet files | Optional (if data not already available) | +| 1 | `1_cc_index_lookup.py` | URLs | URLs + WARC metadata | Datasets without WARC metadata | +| 2 | `2_text_preprocess.py` | WARC metadata | Extracted text | All datasets | +| 3 | `3_llm_cleanup.py` | Preprocessed text | Cleaned text (merged) | Optional | +| 4 | `4_quality_classifier.py` | Text | Text + quality scores | All datasets | +| 5 | `5_deduplication.py` | Scored text | Deduplicated text | All datasets | + +### Working Directory Setup + +```bash +# Create working directories +export MATH_DATA_DIR=/tmp/math_pipeline +mkdir -p $MATH_DATA_DIR/{raw,enriched,preprocessed,cleaned,classified,dedup_cache,dedup_ids,deduplicated} +``` + +## Download Dataset from HuggingFace (Optional) + +**Skip this step if you already have the dataset downloaded locally.** + +The `0_download.py` script downloads math datasets from HuggingFace Hub. It reads dataset configurations from `datasets.json` and downloads parquet files to `$MATH_DATA_DIR/raw//`. + +### Authentication (Optional) + +For gated datasets or higher download rate limits, authenticate with HuggingFace: + +```bash +# Option 1: Environment variable +export HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxx + +# Option 2: CLI login (saves token to ~/.cache/huggingface/token) +huggingface-cli login +``` + +Get your token at: https://huggingface.co/settings/tokens + +### Download Commands + +```bash +# List available datasets +python tutorials/math/0_download.py --list + +# Download a specific dataset +python tutorials/math/0_download.py \ + --dataset FINEMATH_4PLUS \ + --output-dir $MATH_DATA_DIR/raw + +# Download multiple datasets +python tutorials/math/0_download.py \ + --dataset FINEMATH_4PLUS OPENWEBMATH \ + --output-dir $MATH_DATA_DIR/raw + +# Download only a few files for testing +python tutorials/math/0_download.py \ + --dataset FINEMATH_4PLUS \ + --output-dir $MATH_DATA_DIR/raw \ + --max-files 5 + +# Parallel download with 8 workers (recommended for large datasets) +python tutorials/math/0_download.py \ + --dataset FINEMATH_4PLUS \ + --output-dir $MATH_DATA_DIR/raw \ + --workers 8 +``` + +**Output structure:** +``` +$MATH_DATA_DIR/raw/ +├── finemath_4plus/ +│ ├── train-00000-of-00XXX.parquet +│ ├── train-00001-of-00XXX.parquet +│ └── ... +└── openwebmath/ + └── ... +``` + +**Dataset sizes (approximate):** +| Dataset | Tokens | Notes | +|---------|--------|-------| +| FINEMATH_4PLUS | ~9.5B | High-quality math (score ≥4) | +| FINEMATH_3PLUS | ~30B | Good quality math (score ≥3) | +| OPENWEBMATH | ~14B | Requires CC Index lookup | + +## Step 1: CC Index Lookup (For Datasets Without WARC Metadata) + +**Skip this step if your dataset already has WARC metadata** (like FineMath). + +For datasets that only have URLs (OpenWebMath, InfiWebMath, MegaMath), enrich them with WARC metadata by joining against a local CC Index. + +### Download CC Index + +Download CC Index parquet files for the crawl(s) you need. See [CC Index Requirements](#common-crawl-cc-index-requirements) for download options and required schema. + +### Run CC Index Lookup + +```bash +# Run CC Index lookup (auto-detects all available crawls) +python tutorials/math/1_cc_index_lookup.py \ + --input $MATH_DATA_DIR/raw/openwebmath \ + --output $MATH_DATA_DIR/enriched \ + --cc-index-path $CC_INDEX_DIR + +# Or specify specific crawls +python tutorials/math/1_cc_index_lookup.py \ + --input $MATH_DATA_DIR/raw/openwebmath \ + --output $MATH_DATA_DIR/enriched \ + --cc-index-path $CC_INDEX_DIR \ + --crawls CC-MAIN-2024-10 CC-MAIN-2024-18 +``` + +**Output columns added:** + +| Column | Description | Example | +|--------|-------------|---------| +| `warc_filename` | Path to WARC file | `crawl-data/CC-MAIN-2024-10/.../CC-MAIN-...warc.gz` | +| `warc_record_offset` | Byte offset in WARC | `123456789` | +| `warc_record_length` | Record size in bytes | `45678` | +| `content_mime_type` | MIME type | `text/html` | +| `http_status` | HTTP status code | `200` | + +## Step 2: Text Preprocessing (decode → type-detect → extract) + +Extract and preprocess text from raw web data: + +```bash +# For datasets with WARC metadata (FineMath, or after CC Index lookup) +# Uses --fetch-cc to download content from Common Crawl S3 +python tutorials/math/2_text_preprocess.py \ + --input "$MATH_DATA_DIR/enriched/*.parquet" \ + --output $MATH_DATA_DIR/preprocessed \ + --fetch-cc + +# For local data with binary_content column already present (no fetch needed) +python tutorials/math/2_text_preprocess.py \ + --input "$MATH_DATA_DIR/local/*.parquet" \ + --output $MATH_DATA_DIR/preprocessed + +# Optional: Add --report-stats to see extraction statistics +python tutorials/math/2_text_preprocess.py \ + --input "$MATH_DATA_DIR/enriched/*.parquet" \ + --output $MATH_DATA_DIR/preprocessed \ + --fetch-cc \ + --report-stats +``` + +**Input**: Parquet files with either: +- `warc_filename`, `warc_record_offset`, `warc_record_length` columns → use `--fetch-cc` to download from Common Crawl +- `binary_content` (bytes) column → content already present, no fetch needed + +**Output**: JSONL files with columns: `text`, `url`, `type` + +## Step 3: LLM Cleanup + +Clean and refine text using a large language model. This step uses vLLM for efficient inference and requires a GPU. When `--chunk_data` is enabled, documents are split into chunks, cleaned by the LLM, then merged back into one row per document. + +```bash +python tutorials/math/3_llm_cleanup.py \ + --input $MATH_DATA_DIR/preprocessed \ + --output $MATH_DATA_DIR/cleaned \ + --model microsoft/phi-4 \ + --prompt HTML_TO_TEXT_PROMPT \ + --chunk_data \ + --chunk_length 5000 \ + --max_model_len 16384 \ + --input_filetype jsonl +``` + +**Input**: JSONL files from Step 2 + +**Output**: JSONL files with `cleaned_text` (LLM-processed text). When chunking is enabled, chunks are automatically merged back into one row per document. + +**Key flags**: +- `--chunk_data` / `--chunk_length`: Enable token-based chunking before LLM processing +- `--groupby`: Columns to group by for chunk merging (default: `url`) +- `--max_text_length`: Maximum merged text length in chars (default: 900,000) +- `--classification`: Output classification labels instead of cleaned text +- `--temperature`, `--top_p`, `--top_k`, `--min_p`: Sampling parameters + +## Step 4: Quality Classification + +Classify mathematical content quality using the FineMath model: + +```bash +python tutorials/math/4_quality_classifier.py \ + --input "$MATH_DATA_DIR/cleaned/*.jsonl" \ + --output $MATH_DATA_DIR/classified +``` + +**Input**: JSONL files from Step 3 + +**Output**: JSONL files with additional columns: +- `finemath_scores`: float scores (0..5) +- `finemath_int_scores`: integer scores (0..5) + +## Step 5: Deduplication + +Remove duplicate content using fuzzy deduplication: + +```bash +python tutorials/math/5_deduplication.py \ + --input $MATH_DATA_DIR/classified \ + --cache_dir $MATH_DATA_DIR/dedup_cache \ + --duplicate_ids_dir $MATH_DATA_DIR/dedup_ids \ + --output $MATH_DATA_DIR/deduplicated \ + --input_filetype jsonl +``` + +**Input**: JSONL files from Step 4 + +**Output**: Deduplicated JSONL files + +**Process**: Deduplication takes place in two stages: +1. First stage: Duplicate IDs are identified and saved to `duplicate_ids_dir` +2. Second stage: Duplicates are removed from the dataset + +**Note**: The `cache_dir` must be empty between runs. + +## Alternative Prompts and Use Cases + +The LLM cleanup step supports various specialized prompts for different mathematical content processing needs: + +### Content Cleaning Prompts + +**`HTML_TO_TEXT_PROMPT`** (default): Extract main content, preserve math, standardize equations to LaTeX `$...$`, remove boilerplate + +**`HTML_TO_TEXT_PROMPT_CODE`**: For pages mixing math and significant code (e.g., computational math tutorials) + +```bash +python tutorials/math/3_llm_cleanup.py \ + --input $MATH_DATA_DIR/deduplicated \ + --output $MATH_DATA_DIR/cleaned_code \ + --model microsoft/phi-4 \ + --prompt HTML_TO_TEXT_PROMPT_CODE \ + --chunk_data \ + --chunk_length 5000 \ + --max_model_len 16384 \ + --input_filetype jsonl +``` diff --git a/tutorials/math/datasets.json b/tutorials/math/datasets.json new file mode 100644 index 0000000000..927e6142f2 --- /dev/null +++ b/tutorials/math/datasets.json @@ -0,0 +1,32 @@ +{ + "_comment": "Math dataset configurations for download. Run with: python 0_download.py --list", + + "FINEMATH_4PLUS": { + "huggingface": "HuggingFaceTB/finemath/finemath-4plus", + "needs_cc_lookup": false + }, + "FINEMATH_3PLUS": { + "huggingface": "HuggingFaceTB/finemath/finemath-3plus", + "needs_cc_lookup": false + }, + "OPENWEBMATH": { + "huggingface": "open-web-math/open-web-math", + "needs_cc_lookup": true, + "url_col": "url" + }, + "OPC_FINEWEB_MATH": { + "huggingface": "OpenCoder-LLM/opc-fineweb-math-corpus", + "needs_cc_lookup": true, + "url_col": "url" + }, + "MEGAMATH_PRO": { + "huggingface": "LLM360/MegaMath/megamath-web-pro", + "needs_cc_lookup": true, + "url_col": "url" + }, + "MEGAMATH_WEB": { + "huggingface": "LLM360/MegaMath/megamath-web", + "needs_cc_lookup": true, + "url_col": "url" + } +} diff --git a/uv.lock b/uv.lock index 7a5bcbd391..ffa3edd85e 100644 --- a/uv.lock +++ b/uv.lock @@ -575,21 +575,12 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/75/aa/abcd75e9600987a0bc6cfe9b6b2ff3f0e2cb08c170addc6e76035b5c4cb3/blake3-1.0.8.tar.gz", hash = "sha256:513cc7f0f5a7c035812604c2c852a0c1468311345573de647e310aca4ab165ba", size = 117308, upload-time = "2025-10-14T06:47:48.83Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/df/0825da1cde7ca63a8bcdc785ca7f8647b025e9497eef18c75bb9754dbd26/blake3-1.0.8-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9e1d70bf76c02846d0868a3d413eb6c430b76a315e12f1b2e59b5cf56c1f62a3", size = 374945, upload-time = "2025-10-14T06:45:13.99Z" }, - { url = "https://files.pythonhosted.org/packages/db/8f/9431bf5fe0eedeb2aadb4fe81fb18945cf8d49adad98e7988fb3cdac76c2/blake3-1.0.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97c076d58ee37eb5b2d8d91bb9db59c5a008fd59c71845dc57fe438aeeabaf10", size = 507107, upload-time = "2025-10-14T06:45:17.055Z" }, - { url = "https://files.pythonhosted.org/packages/ac/55/3712cdaebaefa8d5acec46f8df7861ba1832e1e188bc1333dd5acd31f760/blake3-1.0.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78731ce7fca46f776ae45fb5271a2a76c4a92c9687dd4337e84b2ae9a174b28f", size = 393955, upload-time = "2025-10-14T06:45:18.718Z" }, { url = "https://files.pythonhosted.org/packages/1f/d0/add0441e7aaa6b358cac0ddc9246f0799b60d25f06bd542b554afe19fd85/blake3-1.0.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c65e373c8b47174b969ee61a89ee56922f722972eb650192845c8546df8d9db9", size = 387577, upload-time = "2025-10-14T06:45:20.332Z" }, { url = "https://files.pythonhosted.org/packages/28/c7/90c01091465628acff96534e82d4b3bc16ca22c515f69916d2715273c0e3/blake3-1.0.8-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:67d9c42c42eb1c7aedcf901591c743266009fcf48babf6d6f8450f567cb94a84", size = 554650, upload-time = "2025-10-14T06:45:23.047Z" }, { url = "https://files.pythonhosted.org/packages/3c/7e/ab9b5c4b650ff397d347451bfb1ad7e6e53dc06c945e2fd091f27a76422e/blake3-1.0.8-cp310-cp310-win_amd64.whl", hash = "sha256:725c52c4d393c7bd1a10682df322d480734002a1389b320366c660568708846b", size = 215660, upload-time = "2025-10-14T06:45:25.381Z" }, - { url = "https://files.pythonhosted.org/packages/a0/33/9d342a2bf5817f006bbe947335e5d387327541ea47590854947befd01251/blake3-1.0.8-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:58ce8d45a5bb5326482de72ea1969a378634236186a970fef63058a5b7b8b435", size = 374859, upload-time = "2025-10-14T06:45:35.262Z" }, - { url = "https://files.pythonhosted.org/packages/a5/67/167a65a4c431715407d07b1b8b1367698a3ad88e7260edb85f0c5293f08a/blake3-1.0.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b5573b052777142b2cecc453d022c3f21aa4aba75011258410bb98f41c1a727", size = 507519, upload-time = "2025-10-14T06:45:37.814Z" }, - { url = "https://files.pythonhosted.org/packages/32/e2/0886e192d634b264c613b0fbf380745b39992b424a0effc00ef08783644e/blake3-1.0.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe1b02ab49bfd969ef50b9f17482a2011c77536654af21807ba5c2674e0bb2a0", size = 393645, upload-time = "2025-10-14T06:45:39.146Z" }, { url = "https://files.pythonhosted.org/packages/fc/3b/7fb2fe615448caaa5f6632b2c7551117b38ccac747a3a5769181e9751641/blake3-1.0.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7780666dc6be809b49442d6d5ce06fdbe33024a87560b58471103ec17644682", size = 387640, upload-time = "2025-10-14T06:45:40.546Z" }, { url = "https://files.pythonhosted.org/packages/7e/75/0252be37620699b79dbaa799c9b402d63142a131d16731df4ef09d135dd7/blake3-1.0.8-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c63ece266a43014cf29e772a82857cd8e90315ae3ed53e3c5204851596edd5f2", size = 554463, upload-time = "2025-10-14T06:45:43.22Z" }, { url = "https://files.pythonhosted.org/packages/34/d7/33b01e27dc3542dc9ec44132684506f880cd0257b04da0bf7f4b2afa41c8/blake3-1.0.8-cp311-cp311-win_amd64.whl", hash = "sha256:8f2ef8527a7a8afd99b16997d015851ccc0fe2a409082cebb980af2554e5c74c", size = 215733, upload-time = "2025-10-14T06:45:46.049Z" }, - { url = "https://files.pythonhosted.org/packages/e3/20/488475254976ed93fab57c67aa80d3b40df77f7d9db6528c9274bff53e08/blake3-1.0.8-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66ca28a673025c40db3eba21a9cac52f559f83637efa675b3f6bd8683f0415f3", size = 374516, upload-time = "2025-10-14T06:45:51.23Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7d/db0626df16029713e7e61b67314c4835e85c296d82bd907c21c6ea271da2/blake3-1.0.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e5b5da177d62cc4b7edf0cea08fe4dec960c9ac27f916131efa890a01f747b93", size = 505420, upload-time = "2025-10-14T06:45:54.445Z" }, - { url = "https://files.pythonhosted.org/packages/5b/55/6e737850c2d58a6d9de8a76dad2ae0f75b852a23eb4ecb07a0b165e6e436/blake3-1.0.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:38209b10482c97e151681ea3e91cc7141f56adbbf4820a7d701a923124b41e6a", size = 394189, upload-time = "2025-10-14T06:45:55.719Z" }, { url = "https://files.pythonhosted.org/packages/5b/94/eafaa5cdddadc0c9c603a6a6d8339433475e1a9f60c8bb9c2eed2d8736b6/blake3-1.0.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:504d1399b7fb91dfe5c25722d2807990493185faa1917456455480c36867adb5", size = 388001, upload-time = "2025-10-14T06:45:57.067Z" }, { url = "https://files.pythonhosted.org/packages/0e/c6/d1fe8bdea4a6088bd54b5a58bc40aed89a4e784cd796af7722a06f74bae7/blake3-1.0.8-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a25db3d36b55f5ed6a86470155cc749fc9c5b91c949b8d14f48658f9d960d9ec", size = 554211, upload-time = "2025-10-14T06:46:00.269Z" }, { url = "https://files.pythonhosted.org/packages/4d/42/bbd02647169e3fbed27558555653ac2578c6f17ccacf7d1956c58ef1d214/blake3-1.0.8-cp312-cp312-win_amd64.whl", hash = "sha256:6a6eafc29e4f478d365a87d2f25782a521870c8514bb43734ac85ae9be71caf7", size = 215704, upload-time = "2025-10-14T06:46:02.79Z" }, @@ -2031,38 +2022,18 @@ version = "0.8.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/69/e7/f89d54fb04104114dd0552836dc2b47914f416cc0e200b409dd04a33de5e/fastar-0.8.0.tar.gz", hash = "sha256:f4d4d68dbf1c4c2808f0e730fac5843493fc849f70fe3ad3af60dfbaf68b9a12", size = 68524, upload-time = "2025-11-26T02:36:00.72Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4d/5e/4608184aa57cb6a54f62c1eb3e5133ba8d461fc7f13193c0255effbec12a/fastar-0.8.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a90695a601a78bbca910fdf2efcdf3103c55d0de5a5c6e93556d707bf886250b", size = 765987, upload-time = "2025-11-26T02:32:59.701Z" }, - { url = "https://files.pythonhosted.org/packages/e0/53/6afd2b680dddfa10df9a16bbcf6cabfee0d92435d5c7e3f4cfe3b1712662/fastar-0.8.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d0bf655ff4c9320b0ca8a5b128063d5093c0c8c1645a2b5f7167143fd8531aa", size = 930900, upload-time = "2025-11-26T02:33:16.059Z" }, - { url = "https://files.pythonhosted.org/packages/ef/1e/b7a304bfcc1d06845cbfa4b464516f6fff9c8c6692f6ef80a3a86b04e199/fastar-0.8.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d8df22cdd8d58e7689aa89b2e4a07e8e5fa4f88d2d9c2621f0e88a49be97ccea", size = 821523, upload-time = "2025-11-26T02:33:30.897Z" }, { url = "https://files.pythonhosted.org/packages/1d/da/9ef8605c6d233cd6ca3a95f7f518ac22aa064903afe6afa57733bfb7c31b/fastar-0.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8a5e6ad722685128521c8fb44cf25bd38669650ba3a4b466b8903e5aa28e1a0", size = 821268, upload-time = "2025-11-26T02:34:04.003Z" }, - { url = "https://files.pythonhosted.org/packages/ca/a6/366b15f432d85d4089e6e4b52a09cc2a2bcf4d7a1f0771e3d3194deccb1e/fastar-0.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:175db2a98d67ced106468e8987975484f8bbbd5ad99201da823b38bafb565ed5", size = 1041921, upload-time = "2025-11-26T02:35:07.292Z" }, { url = "https://files.pythonhosted.org/packages/c2/e2/a587796111a3cd4b78cd61ec3fc1252d8517d81f763f4164ed5680f84810/fastar-0.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:01084cb75f13ca6a8e80bd41584322523189f8e81b472053743d6e6c3062b5a6", size = 995141, upload-time = "2025-11-26T02:35:42.449Z" }, { url = "https://files.pythonhosted.org/packages/be/a9/8da4deb840121c59deabd939ce2dca3d6beec85576f3743d1144441938b5/fastar-0.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:fbc0f2ed0f4add7fb58034c576584d44d7eaaf93dee721dfb26dbed6e222dbac", size = 490701, upload-time = "2025-11-26T02:36:09.625Z" }, - { url = "https://files.pythonhosted.org/packages/d6/45/3eb0ee945a0b5d5f9df7e7c25c037ce7fa441cd0b4d44f76d286e2f4396a/fastar-0.8.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3719541a12bb09ab1eae91d2c987a9b2b7d7149c52e7109ba6e15b74aabc49b1", size = 765587, upload-time = "2025-11-26T02:33:01.174Z" }, - { url = "https://files.pythonhosted.org/packages/51/bb/7defd6ec0d9570b1987d8ebde52d07d97f3f26e10b592fb3e12738eba39a/fastar-0.8.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7a9b0fff8079b18acdface7ef1b7f522fd9a589f65ca4a1a0dd7c92a0886c2a2", size = 931150, upload-time = "2025-11-26T02:33:17.374Z" }, - { url = "https://files.pythonhosted.org/packages/28/54/62e51e684dab347c61878afbf09e177029c1a91eb1e39ef244e6b3ef9efa/fastar-0.8.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ac073576c1931959191cb20df38bab21dd152f66c940aa3ca8b22e39f753b2f3", size = 821354, upload-time = "2025-11-26T02:33:32.083Z" }, { url = "https://files.pythonhosted.org/packages/53/a8/12708ea4d21e3cf9f485b2a67d44ce84d949a6eddcc9aa5b3d324585ab43/fastar-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:003b59a7c3e405b6a7bff8fab17d31e0ccbc7f06730a8f8ca1694eeea75f3c76", size = 821626, upload-time = "2025-11-26T02:34:05.685Z" }, - { url = "https://files.pythonhosted.org/packages/dc/59/2dbe0dc2570764475e60030403738faa261a9d3bff16b08629c378ab939a/fastar-0.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:90957a30e64418b02df5b4d525bea50403d98a4b1f29143ce5914ddfa7e54ee4", size = 1041536, upload-time = "2025-11-26T02:35:08.926Z" }, { url = "https://files.pythonhosted.org/packages/cb/e7/23e3a19e06d261d1894f98eca9458f98c090c505a0c712dafc0ff1fc2965/fastar-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a03eaf287bbc93064688a1220580ce261e7557c8898f687f4d0b281c85b28d3c", size = 994992, upload-time = "2025-11-26T02:35:44.009Z" }, { url = "https://files.pythonhosted.org/packages/cb/3c/0142bee993c431ee91cf5535e6e4b079ad491f620c215fcd79b7e5ffeb2b/fastar-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:b48abd6056fef7bc3d414aafb453c5b07fdf06d2df5a2841d650288a3aa1e9d3", size = 490863, upload-time = "2025-11-26T02:36:11.114Z" }, - { url = "https://files.pythonhosted.org/packages/d0/00/c3155171b976003af3281f5258189f1935b15d1221bfc7467b478c631216/fastar-0.8.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:83c391e5b789a720e4d0029b9559f5d6dee3226693c5b39c0eab8eaece997e0f", size = 764717, upload-time = "2025-11-26T02:33:02.453Z" }, - { url = "https://files.pythonhosted.org/packages/b7/43/405b7ad76207b2c11b7b59335b70eac19e4a2653977f5588a1ac8fed54f4/fastar-0.8.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3258d7a78a72793cdd081545da61cabe85b1f37634a1d0b97ffee0ff11d105ef", size = 931502, upload-time = "2025-11-26T02:33:18.619Z" }, - { url = "https://files.pythonhosted.org/packages/da/8a/a3dde6d37cc3da4453f2845cdf16675b5686b73b164f37e2cc579b057c2c/fastar-0.8.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6eab95dd985cdb6a50666cbeb9e4814676e59cfe52039c880b69d67cfd44767", size = 821454, upload-time = "2025-11-26T02:33:33.427Z" }, { url = "https://files.pythonhosted.org/packages/da/c1/904fe2468609c8990dce9fe654df3fbc7324a8d8e80d8240ae2c89757064/fastar-0.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:829b1854166141860887273c116c94e31357213fa8e9fe8baeb18bd6c38aa8d9", size = 821647, upload-time = "2025-11-26T02:34:07Z" }, - { url = "https://files.pythonhosted.org/packages/af/af/60c1bfa6edab72366461a95f053d0f5f7ab1825fe65ca2ca367432cd8629/fastar-0.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b864a95229a7db0814cd9ef7987cb713fd43dce1b0d809dd17d9cd6f02fdde3e", size = 1040207, upload-time = "2025-11-26T02:35:10.65Z" }, { url = "https://files.pythonhosted.org/packages/a7/74/cf663af53c4706ba88e6b4af44a6b0c3bd7d7ca09f079dc40647a8f06585/fastar-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7f41c51ee96f338662ee3c3df4840511ba3f9969606840f1b10b7cb633a3c716", size = 994877, upload-time = "2025-11-26T02:35:45.797Z" }, { url = "https://files.pythonhosted.org/packages/dc/34/fc3b5e56d71a17b1904800003d9251716e8fd65f662e1b10a26881698a74/fastar-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:fc645994d5b927d769121094e8a649b09923b3c13a8b0b98696d8f853f23c532", size = 490429, upload-time = "2025-11-26T02:36:12.707Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/043b263c4126bf6557c942d099503989af9c5c7ee5cca9a04e00f754816f/fastar-0.8.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a78e5221b94a80800930b7fd0d0e797ae73aadf7044c05ed46cb9bdf870f022", size = 766755, upload-time = "2025-11-26T02:33:11.595Z" }, - { url = "https://files.pythonhosted.org/packages/57/ff/29a5dc06f2940439ebf98661ecc98d48d3f22fed8d6a2d5dc985d1e8da24/fastar-0.8.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:997092d31ff451de8d0568f6773f3517cb87dcd0bc76184edb65d7154390a6f8", size = 932732, upload-time = "2025-11-26T02:33:27.122Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e8/2218830f422b37aad52c24b53cb84b5d88bd6fd6ad411bd6689b1a32500d/fastar-0.8.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:558e8fcf8fe574541df5db14a46cd98bfbed14a811b7014a54f2b714c0cfac42", size = 822571, upload-time = "2025-11-26T02:33:42.986Z" }, { url = "https://files.pythonhosted.org/packages/6e/fd/ba6dfeff77cddfe58d85c490b1735c002b81c0d6f826916a8b6c4f8818bc/fastar-0.8.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1d2a54f87e2908cc19e1a6ee249620174fbefc54a219aba1eaa6f31657683c3", size = 822440, upload-time = "2025-11-26T02:34:15.439Z" }, - { url = "https://files.pythonhosted.org/packages/ee/c7/18115927f16deb1ddffdbd4ae992e7e33064bc6defa2b92a147948f8bc0c/fastar-0.8.0-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:0afbb92f78bf29d5e9db76fb46cbabc429e49015cddf72ab9e761afbe88ac100", size = 1042675, upload-time = "2025-11-26T02:35:20.252Z" }, { url = "https://files.pythonhosted.org/packages/44/ee/25cd645db749b206bb95e1512e57e75d56ccbbb8ec3536f52a7979deab6b/fastar-0.8.0-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:e6c4d6329da568ec36b1347b0c09c4d27f9dfdeddf9f438ddb16799ecf170098", size = 997397, upload-time = "2025-11-26T02:35:56.215Z" }, - { url = "https://files.pythonhosted.org/packages/0b/90/23a3f6c252f11b10c70f854bce09abc61f71b5a0e6a4b0eac2bcb9a2c583/fastar-0.8.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef0bcf4385bbdd3c1acecce2d9ea7dab7cc9b8ee0581bbccb7ab11908a7ce288", size = 766861, upload-time = "2025-11-26T02:33:12.824Z" }, - { url = "https://files.pythonhosted.org/packages/76/bb/beeb9078380acd4484db5c957d066171695d9340e3526398eb230127b0c2/fastar-0.8.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f10ef62b6eda6cb6fd9ba8e1fe08a07d7b2bdcc8eaa00eb91566143b92ed7eee", size = 932667, upload-time = "2025-11-26T02:33:28.405Z" }, - { url = "https://files.pythonhosted.org/packages/f4/6d/b034cc637bd0ee638d5a85d08e941b0b8ffd44cf391fb751ba98233734f7/fastar-0.8.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c4f6c82a8ee98c17aa48585ee73b51c89c1b010e5c951af83e07c3436180e3fc", size = 822712, upload-time = "2025-11-26T02:33:44.27Z" }, { url = "https://files.pythonhosted.org/packages/e2/2b/7d183c63f59227c4689792042d6647f2586a5e7273b55e81745063088d81/fastar-0.8.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6129067fcb86276635b5857010f4e9b9c7d5d15dd571bb03c6c1ed73c40fd92", size = 822659, upload-time = "2025-11-26T02:34:16.815Z" }, - { url = "https://files.pythonhosted.org/packages/a4/b9/9a8c3fd59958c1c8027bc075af11722cdc62c4968bb277e841d131232289/fastar-0.8.0-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:382bfe82c026086487cb17fee12f4c1e2b4e67ce230f2e04487d3e7ddfd69031", size = 1042911, upload-time = "2025-11-26T02:35:21.857Z" }, { url = "https://files.pythonhosted.org/packages/9e/8a/218ab6d9a2bab3b07718e6cd8405529600edc1e9c266320e8524c8f63251/fastar-0.8.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:1aa7dbde2d2d73eb5b6203d0f74875cb66350f0f1b4325b4839fc8fbbf5d074e", size = 997309, upload-time = "2025-11-26T02:35:57.722Z" }, ] @@ -2495,7 +2466,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/6a/33d1702184d94106d3cdd7bfb788e19723206fce152e303473ca3b946c7b/greenlet-3.3.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f8496d434d5cb2dce025773ba5597f71f5410ae499d5dd9533e0653258cdb3d", size = 273658, upload-time = "2025-12-04T14:23:37.494Z" }, { url = "https://files.pythonhosted.org/packages/d6/b7/2b5805bbf1907c26e434f4e448cd8b696a0b71725204fa21a211ff0c04a7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b96dc7eef78fd404e022e165ec55327f935b9b52ff355b067eb4a0267fc1cffb", size = 574810, upload-time = "2025-12-04T14:50:04.154Z" }, { url = "https://files.pythonhosted.org/packages/94/38/343242ec12eddf3d8458c73f555c084359883d4ddc674240d9e61ec51fd6/greenlet-3.3.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73631cd5cccbcfe63e3f9492aaa664d278fda0ce5c3d43aeda8e77317e38efbd", size = 586248, upload-time = "2025-12-04T14:57:39.35Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d0/0ae86792fb212e4384041e0ef8e7bc66f59a54912ce407d26a966ed2914d/greenlet-3.3.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b299a0cb979f5d7197442dccc3aee67fce53500cd88951b7e6c35575701c980b", size = 597403, upload-time = "2025-12-04T15:07:10.831Z" }, { url = "https://files.pythonhosted.org/packages/b6/a8/15d0aa26c0036a15d2659175af00954aaaa5d0d66ba538345bd88013b4d7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dee147740789a4632cace364816046e43310b59ff8fb79833ab043aefa72fd5", size = 586910, upload-time = "2025-12-04T14:25:59.705Z" }, { url = "https://files.pythonhosted.org/packages/e1/9b/68d5e3b7ccaba3907e5532cf8b9bf16f9ef5056a008f195a367db0ff32db/greenlet-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:39b28e339fc3c348427560494e28d8a6f3561c8d2bcf7d706e1c624ed8d822b9", size = 1547206, upload-time = "2025-12-04T15:04:21.027Z" }, { url = "https://files.pythonhosted.org/packages/66/bd/e3086ccedc61e49f91e2cfb5ffad9d8d62e5dc85e512a6200f096875b60c/greenlet-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b3c374782c2935cc63b2a27ba8708471de4ad1abaa862ffdb1ef45a643ddbb7d", size = 1613359, upload-time = "2025-12-04T14:27:26.548Z" }, @@ -2503,7 +2473,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/cb/48e964c452ca2b92175a9b2dca037a553036cb053ba69e284650ce755f13/greenlet-3.3.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e29f3018580e8412d6aaf5641bb7745d38c85228dacf51a73bd4e26ddf2a6a8e", size = 274908, upload-time = "2025-12-04T14:23:26.435Z" }, { url = "https://files.pythonhosted.org/packages/28/da/38d7bff4d0277b594ec557f479d65272a893f1f2a716cad91efeb8680953/greenlet-3.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a687205fb22794e838f947e2194c0566d3812966b41c78709554aa883183fb62", size = 577113, upload-time = "2025-12-04T14:50:05.493Z" }, { url = "https://files.pythonhosted.org/packages/3c/f2/89c5eb0faddc3ff014f1c04467d67dee0d1d334ab81fadbf3744847f8a8a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4243050a88ba61842186cb9e63c7dfa677ec146160b0efd73b855a3d9c7fcf32", size = 590338, upload-time = "2025-12-04T14:57:41.136Z" }, - { url = "https://files.pythonhosted.org/packages/80/d7/db0a5085035d05134f8c089643da2b44cc9b80647c39e93129c5ef170d8f/greenlet-3.3.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:670d0f94cd302d81796e37299bcd04b95d62403883b24225c6b5271466612f45", size = 601098, upload-time = "2025-12-04T15:07:11.898Z" }, { url = "https://files.pythonhosted.org/packages/dc/a6/e959a127b630a58e23529972dbc868c107f9d583b5a9f878fb858c46bc1a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb3a8ec3db4a3b0eb8a3c25436c2d49e3505821802074969db017b87bc6a948", size = 590206, upload-time = "2025-12-04T14:26:01.254Z" }, { url = "https://files.pythonhosted.org/packages/48/60/29035719feb91798693023608447283b266b12efc576ed013dd9442364bb/greenlet-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de5a0b09eab81fc6a382791b995b1ccf2b172a9fec934747a7a23d2ff291794", size = 1550668, upload-time = "2025-12-04T15:04:22.439Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/783a23754b691bfa86bd72c3033aa107490deac9b2ef190837b860996c9f/greenlet-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4449a736606bd30f27f8e1ff4678ee193bc47f6ca810d705981cfffd6ce0d8c5", size = 1615483, upload-time = "2025-12-04T14:27:28.083Z" }, @@ -2511,7 +2480,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, - { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -4617,6 +4585,48 @@ image-cuda12 = [ { name = "torchvision", version = "0.24.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'x86_64' or sys_platform == 'darwin'" }, { name = "torchvision", version = "0.24.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" }, ] +math-cpu = [ + { name = "beautifulsoup4" }, + { name = "fasttext" }, + { name = "ftfy" }, + { name = "justext" }, + { name = "lxml" }, + { name = "mwparserfromhell" }, + { name = "peft" }, + { name = "pycld2" }, + { name = "resiliparse" }, + { name = "s5cmd" }, + { name = "sentence-transformers" }, + { name = "sentencepiece" }, + { name = "trafilatura" }, + { name = "warcio" }, +] +math-cuda12 = [ + { name = "beautifulsoup4" }, + { name = "cudf-cu12" }, + { name = "cuml-cu12" }, + { name = "fasttext" }, + { name = "ftfy" }, + { name = "gpustat" }, + { name = "justext" }, + { name = "lxml" }, + { name = "mwparserfromhell" }, + { name = "nvidia-ml-py" }, + { name = "peft" }, + { name = "pycld2" }, + { name = "pylibcugraph-cu12" }, + { name = "pylibraft-cu12" }, + { name = "raft-dask-cu12" }, + { name = "rapidsmpf-cu12" }, + { name = "resiliparse" }, + { name = "s5cmd" }, + { name = "scikit-learn" }, + { name = "sentence-transformers" }, + { name = "sentencepiece" }, + { name = "trafilatura" }, + { name = "vllm", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" }, + { name = "warcio" }, +] sdg-cpu = [ { name = "data-designer" }, ] @@ -4747,13 +4757,18 @@ requires-dist = [ { name = "nemo-curator", extras = ["audio-cuda12"], marker = "extra == 'all'" }, { name = "nemo-curator", extras = ["cuda12"], marker = "extra == 'audio-cuda12'" }, { name = "nemo-curator", extras = ["cuda12"], marker = "extra == 'image-cuda12'" }, + { name = "nemo-curator", extras = ["cuda12"], marker = "extra == 'math-cuda12'" }, { name = "nemo-curator", extras = ["cuda12"], marker = "extra == 'text-cuda12'" }, { name = "nemo-curator", extras = ["cuda12"], marker = "extra == 'video-cuda12'" }, { name = "nemo-curator", extras = ["deduplication-cuda12"], marker = "extra == 'image-cuda12'" }, + { name = "nemo-curator", extras = ["deduplication-cuda12"], marker = "extra == 'math-cuda12'" }, { name = "nemo-curator", extras = ["deduplication-cuda12"], marker = "extra == 'text-cuda12'" }, { name = "nemo-curator", extras = ["image-cpu"], marker = "extra == 'image-cuda12'" }, { name = "nemo-curator", extras = ["image-cuda12"], marker = "extra == 'all'" }, + { name = "nemo-curator", extras = ["math-cpu"], marker = "extra == 'math-cuda12'" }, + { name = "nemo-curator", extras = ["math-cuda12"], marker = "extra == 'all'" }, { name = "nemo-curator", extras = ["sdg-cpu"], marker = "extra == 'all'" }, + { name = "nemo-curator", extras = ["text-cpu"], marker = "extra == 'math-cpu'" }, { name = "nemo-curator", extras = ["text-cpu"], marker = "extra == 'text-cuda12'" }, { name = "nemo-curator", extras = ["text-cuda12"], marker = "extra == 'all'" }, { name = "nemo-curator", extras = ["video-cpu"], marker = "extra == 'video-cuda12'" }, @@ -4794,10 +4809,11 @@ requires-dist = [ { name = "torchvision", marker = "(platform_machine != 'x86_64' and extra == 'video-cpu') or (sys_platform == 'darwin' and extra == 'video-cpu')", index = "https://pypi.org/simple" }, { name = "trafilatura", marker = "extra == 'text-cpu'", specifier = "==2.0.0" }, { name = "transformers" }, + { name = "vllm", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin' and extra == 'math-cuda12'", specifier = ">=0.13" }, { name = "vllm", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin' and extra == 'vllm'", specifier = ">=0.14.1" }, { name = "warcio", marker = "extra == 'text-cpu'" }, ] -provides-extras = ["cuda12", "vllm", "deduplication-cuda12", "audio-cpu", "audio-cuda12", "image-cpu", "image-cuda12", "text-cpu", "text-cuda12", "video-cpu", "video-cuda12", "sdg-cpu", "all"] +provides-extras = ["cuda12", "vllm", "deduplication-cuda12", "audio-cpu", "audio-cuda12", "image-cpu", "image-cuda12", "text-cpu", "text-cuda12", "video-cpu", "video-cuda12", "math-cpu", "math-cuda12", "sdg-cpu", "all"] [package.metadata.requires-dev] build = [ @@ -4959,15 +4975,7 @@ version = "1.13.0" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, - { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, - { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, - { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, - { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, - { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, - { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, { url = "https://files.pythonhosted.org/packages/29/45/c0adfbfb0b5895aa18cec400c535b4f7ff3e52536e0403602fc1a23f7de9/ninja-1.13.0-py3-none-win_amd64.whl", hash = "sha256:fb8ee8719f8af47fed145cced4a85f0755dd55d45b2bddaf7431fa89803c5f3e", size = 309975, upload-time = "2025-08-11T15:10:16.697Z" }, ] @@ -5489,10 +5497,7 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, - { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, - { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, { url = "https://files.pythonhosted.org/packages/40/1f/c83cf5a206c263ee70448a5ae4264682555f4d0b5bed0d2cc6ca1108103d/openai_harmony-0.0.8-cp38-abi3-win_amd64.whl", hash = "sha256:39d44f0d8f466bd56698e7ead708bead3141e27b9b87e3ab7d5a6d0e4a869ee5", size = 2438369, upload-time = "2025-11-05T19:07:08.1Z" }, ] @@ -6179,36 +6184,12 @@ source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/aa/b8/4ed5c7ad5ec15b08d35cc79ace6145d5c1ae426e46435f4987379439dfea/pybase64-1.4.3.tar.gz", hash = "sha256:c2ed274c9e0ba9c8f9c4083cfe265e66dd679126cd9c2027965d807352f3f053", size = 137272, upload-time = "2025-12-06T13:27:04.013Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/1c/22/e89739d8bc9b96c68ead44b4eec42fe555683d9997e4ba65216d384920fc/pybase64-1.4.3-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6ec7e53dd09b0a8116ccf5c3265c7c7fce13c980747525be76902aef36a514a", size = 68903, upload-time = "2025-12-06T13:22:31.29Z" }, - { url = "https://files.pythonhosted.org/packages/42/ad/f47dc7e6fe32022b176868b88b671a32dab389718c8ca905cab79280aaaf/pybase64-1.4.3-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:4ec645f32b50593879031e09158f8681a1db9f5df0f72af86b3969a1c5d1fa2b", size = 54533, upload-time = "2025-12-06T13:22:33.457Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/7ab312b5a324833953b00e47b23eb4f83d45bd5c5c854b4b4e51b2a0cf5b/pybase64-1.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:634a000c5b3485ccc18bb9b244e0124f74b6fbc7f43eade815170237a7b34c64", size = 57187, upload-time = "2025-12-06T13:22:34.566Z" }, - { url = "https://files.pythonhosted.org/packages/2c/84/80acab1fcbaaae103e6b862ef5019192c8f2cd8758433595a202179a0d1d/pybase64-1.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:309ea32ad07639a485580af1be0ad447a434deb1924e76adced63ac2319cfe15", size = 57730, upload-time = "2025-12-06T13:22:35.581Z" }, - { url = "https://files.pythonhosted.org/packages/1f/24/84256d472400ea3163d7d69c44bb7e2e1027f0f1d4d20c47629a7dc4578e/pybase64-1.4.3-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:d10d517566b748d3f25f6ac7162af779360c1c6426ad5f962927ee205990d27c", size = 53036, upload-time = "2025-12-06T13:22:36.621Z" }, - { url = "https://files.pythonhosted.org/packages/dc/1c/a341b050746658cbec8cab3c733aeb3ef52ce8f11e60d0d47adbdf729ebf/pybase64-1.4.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b591d774ac09d5eb73c156a03277cb271438fbd8042bae4109ff3a827cd218c", size = 50114, upload-time = "2025-12-06T13:22:38.752Z" }, - { url = "https://files.pythonhosted.org/packages/4c/71/774748eecc7fe23869b7e5df028e3c4c2efa16b506b83ea3fa035ea95dc2/pybase64-1.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:df8b122d5be2c96962231cc4831d9c2e1eae6736fb12850cec4356d8b06fe6f8", size = 55700, upload-time = "2025-12-06T13:22:41.289Z" }, - { url = "https://files.pythonhosted.org/packages/b3/91/dd15075bb2fe0086193e1cd4bad80a43652c38d8a572f9218d46ba721802/pybase64-1.4.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:31b7a85c661fc591bbcce82fb8adaebe2941e6a83b08444b0957b77380452a4b", size = 52491, upload-time = "2025-12-06T13:22:42.628Z" }, - { url = "https://files.pythonhosted.org/packages/7b/27/f357d63ea3774c937fc47160e040419ed528827aa3d4306d5ec9826259c0/pybase64-1.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e6d7beaae65979fef250e25e66cf81c68a8f81910bcda1a2f43297ab486a7e4e", size = 53957, upload-time = "2025-12-06T13:22:44.615Z" }, { url = "https://files.pythonhosted.org/packages/b3/c3/243693771701a54e67ff5ccbf4c038344f429613f5643169a7befc51f007/pybase64-1.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4a6276bc3a3962d172a2b5aba544d89881c4037ea954517b86b00892c703d007", size = 68422, upload-time = "2025-12-06T13:22:45.641Z" }, { url = "https://files.pythonhosted.org/packages/79/28/c169a769fe90128f16d394aad87b2096dd4bf2f035ae0927108a46b617df/pybase64-1.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:5db0b6bbda15110db2740c61970a8fda3bf9c93c3166a3f57f87c7865ed1125c", size = 35799, upload-time = "2025-12-06T13:22:48.731Z" }, { url = "https://files.pythonhosted.org/packages/64/15/8d60b9ec5e658185fc2ee3333e01a6e30d717cf677b24f47cbb3a859d13c/pybase64-1.4.3-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:95a57cccf106352a72ed8bc8198f6820b16cc7d55aa3867a16dea7011ae7c218", size = 71370, upload-time = "2025-12-06T13:22:55.517Z" }, - { url = "https://files.pythonhosted.org/packages/a9/00/8ffcf9810bd23f3984698be161cf7edba656fd639b818039a7be1d6405d4/pybase64-1.4.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:9fe9922698f3e2f72874b26890d53a051c431d942701bb3a37aae94da0b12107", size = 56652, upload-time = "2025-12-06T13:22:57.724Z" }, - { url = "https://files.pythonhosted.org/packages/81/62/379e347797cdea4ab686375945bc77ad8d039c688c0d4d0cfb09d247beb9/pybase64-1.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:af5f4bd29c86b59bb4375e0491d16ec8a67548fa99c54763aaedaf0b4b5a6632", size = 59382, upload-time = "2025-12-06T13:22:58.758Z" }, - { url = "https://files.pythonhosted.org/packages/c6/f2/9338ffe2f487086f26a2c8ca175acb3baa86fce0a756ff5670a0822bb877/pybase64-1.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c302f6ca7465262908131411226e02100f488f531bb5e64cb901aa3f439bccd9", size = 59990, upload-time = "2025-12-06T13:23:01.007Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a4/85a6142b65b4df8625b337727aa81dc199642de3d09677804141df6ee312/pybase64-1.4.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2f3f439fa4d7fde164ebbbb41968db7d66b064450ab6017c6c95cef0afa2b349", size = 54923, upload-time = "2025-12-06T13:23:02.369Z" }, - { url = "https://files.pythonhosted.org/packages/b0/73/d7e19a63e795c13837f2356268d95dc79d1180e756f57ced742a1e52fdeb/pybase64-1.4.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:56e6526f8565642abc5f84338cc131ce298a8ccab696b19bdf76fa6d7dc592ef", size = 52338, upload-time = "2025-12-06T13:23:04.458Z" }, - { url = "https://files.pythonhosted.org/packages/5d/b3/63cec68f9d6f6e4c0b438d14e5f1ef536a5fe63ce14b70733ac5e31d7ab8/pybase64-1.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:62ad29a5026bb22cfcd1ca484ec34b0a5ced56ddba38ceecd9359b2818c9c4f9", size = 58055, upload-time = "2025-12-06T13:23:06.931Z" }, - { url = "https://files.pythonhosted.org/packages/d5/cb/7acf7c3c06f9692093c07f109668725dc37fb9a3df0fa912b50add645195/pybase64-1.4.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11b9d1d2d32ec358c02214363b8fc3651f6be7dd84d880ecd597a6206a80e121", size = 54430, upload-time = "2025-12-06T13:23:07.936Z" }, - { url = "https://files.pythonhosted.org/packages/33/39/4eb33ff35d173bfff4002e184ce8907f5d0a42d958d61cd9058ef3570179/pybase64-1.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0aebaa7f238caa0a0d373616016e2040c6c879ebce3ba7ab3c59029920f13640", size = 56272, upload-time = "2025-12-06T13:23:09.253Z" }, { url = "https://files.pythonhosted.org/packages/19/97/a76d65c375a254e65b730c6f56bf528feca91305da32eceab8bcc08591e6/pybase64-1.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e504682b20c63c2b0c000e5f98a80ea867f8d97642e042a5a39818e44ba4d599", size = 70904, upload-time = "2025-12-06T13:23:10.336Z" }, { url = "https://files.pythonhosted.org/packages/39/dc/32efdf2f5927e5449cc341c266a1bbc5fecd5319a8807d9c5405f76e6d02/pybase64-1.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:a90a8fa16a901fabf20de824d7acce07586e6127dc2333f1de05f73b1f848319", size = 35797, upload-time = "2025-12-06T13:23:13.174Z" }, { url = "https://files.pythonhosted.org/packages/62/f7/965b79ff391ad208b50e412b5d3205ccce372a2d27b7218ae86d5295b105/pybase64-1.4.3-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb632edfd132b3eaf90c39c89aa314beec4e946e210099b57d40311f704e11d4", size = 71599, upload-time = "2025-12-06T13:23:20.195Z" }, - { url = "https://files.pythonhosted.org/packages/da/5d/c38d1572027fc601b62d7a407721688b04b4d065d60ca489912d6893e6cf/pybase64-1.4.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:c48361f90db32bacaa5518419d4eb9066ba558013aaf0c7781620279ecddaeb9", size = 56712, upload-time = "2025-12-06T13:23:22.77Z" }, - { url = "https://files.pythonhosted.org/packages/e7/d4/4e04472fef485caa8f561d904d4d69210a8f8fc1608ea15ebd9012b92655/pybase64-1.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:702bcaa16ae02139d881aeaef5b1c8ffb4a3fae062fe601d1e3835e10310a517", size = 59300, upload-time = "2025-12-06T13:23:24.543Z" }, - { url = "https://files.pythonhosted.org/packages/86/e7/16e29721b86734b881d09b7e23dfd7c8408ad01a4f4c7525f3b1088e25ec/pybase64-1.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:53d0ffe1847b16b647c6413d34d1de08942b7724273dd57e67dcbdb10c574045", size = 60278, upload-time = "2025-12-06T13:23:25.608Z" }, - { url = "https://files.pythonhosted.org/packages/b1/02/18515f211d7c046be32070709a8efeeef8a0203de4fd7521e6b56404731b/pybase64-1.4.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9a1792e8b830a92736dae58f0c386062eb038dfe8004fb03ba33b6083d89cd43", size = 54817, upload-time = "2025-12-06T13:23:26.633Z" }, - { url = "https://files.pythonhosted.org/packages/b4/8a/a2588dfe24e1bbd742a554553778ab0d65fdf3d1c9a06d10b77047d142aa/pybase64-1.4.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e97b7bdbd62e71898cd542a6a9e320d9da754ff3ebd02cb802d69087ee94d468", size = 52404, upload-time = "2025-12-06T13:23:28.714Z" }, - { url = "https://files.pythonhosted.org/packages/d3/3a/87c3201e555ed71f73e961a787241a2438c2bbb2ca8809c29ddf938a3157/pybase64-1.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c0efcf78f11cf866bed49caa7b97552bc4855a892f9cc2372abcd3ed0056f0d", size = 57854, upload-time = "2025-12-06T13:23:31.17Z" }, - { url = "https://files.pythonhosted.org/packages/fd/7d/931c2539b31a7b375e7d595b88401eeb5bd6c5ce1059c9123f9b608aaa14/pybase64-1.4.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:66e3791f2ed725a46593f8bd2761ff37d01e2cdad065b1dceb89066f476e50c6", size = 54333, upload-time = "2025-12-06T13:23:32.422Z" }, - { url = "https://files.pythonhosted.org/packages/de/5e/537601e02cc01f27e9d75f440f1a6095b8df44fc28b1eef2cd739aea8cec/pybase64-1.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:72bb0b6bddadab26e1b069bb78e83092711a111a80a0d6b9edcb08199ad7299b", size = 56492, upload-time = "2025-12-06T13:23:33.515Z" }, { url = "https://files.pythonhosted.org/packages/96/97/2a2e57acf8f5c9258d22aba52e71f8050e167b29ed2ee1113677c1b600c1/pybase64-1.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5b3365dbcbcdb0a294f0f50af0c0a16b27a232eddeeb0bceeefd844ef30d2a23", size = 70974, upload-time = "2025-12-06T13:23:36.27Z" }, { url = "https://files.pythonhosted.org/packages/83/e3/507ab649d8c3512c258819c51d25c45d6e29d9ca33992593059e7b646a33/pybase64-1.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:9f2da8f56d9b891b18b4daf463a0640eae45a80af548ce435be86aa6eff3603b", size = 35833, upload-time = "2025-12-06T13:23:38.877Z" }, { url = "https://files.pythonhosted.org/packages/bf/44/d4b7adc7bf4fd5b52d8d099121760c450a52c390223806b873f0b6a2d551/pybase64-1.4.3-graalpy311-graalpy242_311_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a492518f3078a4e3faaef310697d21df9c6bc71908cebc8c2f6fbfa16d7d6b1f", size = 43227, upload-time = "2025-12-06T13:26:21.845Z" }, @@ -7170,38 +7151,18 @@ version = "0.7.6" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/e5/f5/8bed2310abe4ae04b67a38374a4d311dd85220f5d8da56f47ae9361be0b0/rignore-0.7.6.tar.gz", hash = "sha256:00d3546cd793c30cb17921ce674d2c8f3a4b00501cb0e3dd0e82217dbeba2671", size = 57140, upload-time = "2025-11-05T21:41:21.968Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/35/71518847e10bdbf359badad8800e4681757a01f4777b3c5e03dbde8a42d8/rignore-0.7.6-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:626c3d4ba03af266694d25101bc1d8d16eda49c5feb86cedfec31c614fceca7d", size = 873813, upload-time = "2025-11-05T20:41:04.71Z" }, - { url = "https://files.pythonhosted.org/packages/f6/c8/32ae405d3e7fd4d9f9b7838f2fcca0a5005bb87fa514b83f83fd81c0df22/rignore-0.7.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0a43841e651e7a05a4274b9026cc408d1912e64016ede8cd4c145dae5d0635be", size = 1168019, upload-time = "2025-11-05T20:41:20.723Z" }, - { url = "https://files.pythonhosted.org/packages/25/98/013c955982bc5b4719bf9a5bea58be317eea28aa12bfd004025e3cd7c000/rignore-0.7.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7978c498dbf7f74d30cdb8859fe612167d8247f0acd377ae85180e34490725da", size = 942822, upload-time = "2025-11-05T20:41:36.99Z" }, { url = "https://files.pythonhosted.org/packages/90/fb/9a3f3156c6ed30bcd597e63690353edac1fcffe9d382ad517722b56ac195/rignore-0.7.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d22f72ab695c07d2d96d2a645208daff17084441b5d58c07378c9dd6f9c4c87", size = 959820, upload-time = "2025-11-05T20:42:06.364Z" }, - { url = "https://files.pythonhosted.org/packages/df/26/4b635f4ea5baf4baa8ba8eee06163f6af6e76dfbe72deb57da34bb24b19d/rignore-0.7.6-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ce2617fe28c51367fd8abfd4eeea9e61664af63c17d4ea00353d8ef56dfb95fa", size = 1139028, upload-time = "2025-11-05T21:40:27.977Z" }, { url = "https://files.pythonhosted.org/packages/fb/f4/27475db769a57cff18fe7e7267b36e6cdb5b1281caa185ba544171106cba/rignore-0.7.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:02cd240bfd59ecc3907766f4839cbba20530a2e470abca09eaa82225e4d946fb", size = 1128531, upload-time = "2025-11-05T21:41:02.734Z" }, { url = "https://files.pythonhosted.org/packages/c0/8a/53185c69abb3bb362e8a46b8089999f820bf15655629ff8395107633c8ab/rignore-0.7.6-cp310-cp310-win_amd64.whl", hash = "sha256:d80afd6071c78baf3765ec698841071b19e41c326f994cfa69b5a1df676f5d39", size = 727001, upload-time = "2025-11-05T21:41:32.778Z" }, - { url = "https://files.pythonhosted.org/packages/fb/c9/390a8fdfabb76d71416be773bd9f162977bd483084f68daf19da1dec88a6/rignore-0.7.6-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ba5524f5178deca4d7695e936604ebc742acb8958f9395776e1fcb8133f8257a", size = 873633, upload-time = "2025-11-05T20:41:06.193Z" }, - { url = "https://files.pythonhosted.org/packages/df/c9/79404fcb0faa76edfbc9df0901f8ef18568d1104919ebbbad6d608c888d1/rignore-0.7.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:62020dbb89a1dd4b84ab3d60547b3b2eb2723641d5fb198463643f71eaaed57d", size = 1167633, upload-time = "2025-11-05T20:41:22.491Z" }, - { url = "https://files.pythonhosted.org/packages/6e/8d/b3466d32d445d158a0aceb80919085baaae495b1f540fb942f91d93b5e5b/rignore-0.7.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b34acd532769d5a6f153a52a98dcb81615c949ab11697ce26b2eb776af2e174d", size = 941434, upload-time = "2025-11-05T20:41:38.151Z" }, { url = "https://files.pythonhosted.org/packages/e8/40/9cd949761a7af5bc27022a939c91ff622d29c7a0b66d0c13a863097dde2d/rignore-0.7.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c5e53b752f9de44dff7b3be3c98455ce3bf88e69d6dc0cf4f213346c5e3416c", size = 959461, upload-time = "2025-11-05T20:42:08.476Z" }, - { url = "https://files.pythonhosted.org/packages/17/18/162eedadb4c2282fa4c521700dbf93c9b14b8842e8354f7d72b445b8d593/rignore-0.7.6-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5991e46ab9b4868334c9e372ab0892b0150f3f586ff2b1e314272caeb38aaedb", size = 1139012, upload-time = "2025-11-05T21:40:29.399Z" }, { url = "https://files.pythonhosted.org/packages/9f/22/1c1a65047df864def9a047dbb40bc0b580b8289a4280e62779cd61ae21f2/rignore-0.7.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aaf938530dcc0b47c4cfa52807aa2e5bfd5ca6d57a621125fe293098692f6345", size = 1128182, upload-time = "2025-11-05T21:41:04.239Z" }, { url = "https://files.pythonhosted.org/packages/7c/c8/dda0983e1845706beb5826459781549a840fe5a7eb934abc523e8cd17814/rignore-0.7.6-cp311-cp311-win_amd64.whl", hash = "sha256:44f35ee844b1a8cea50d056e6a595190ce9d42d3cccf9f19d280ae5f3058973a", size = 727139, upload-time = "2025-11-05T21:41:34.367Z" }, - { url = "https://files.pythonhosted.org/packages/b3/2b/ee96db17ac1835e024c5d0742eefb7e46de60020385ac883dd3d1cde2c1f/rignore-0.7.6-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5fd5ab3840b8c16851d327ed06e9b8be6459702a53e5ab1fc4073b684b3789e", size = 873963, upload-time = "2025-11-05T20:41:07.49Z" }, - { url = "https://files.pythonhosted.org/packages/a5/8c/ad5a57bbb9d14d5c7e5960f712a8a0b902472ea3f4a2138cbf70d1777b75/rignore-0.7.6-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ced2a248352636a5c77504cb755dc02c2eef9a820a44d3f33061ce1bb8a7f2d2", size = 1169216, upload-time = "2025-11-05T20:41:23.73Z" }, - { url = "https://files.pythonhosted.org/packages/80/e6/5b00bc2a6bc1701e6878fca798cf5d9125eb3113193e33078b6fc0d99123/rignore-0.7.6-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a04a3b73b75ddc12c9c9b21efcdaab33ca3832941d6f1d67bffd860941cd448a", size = 942942, upload-time = "2025-11-05T20:41:39.393Z" }, { url = "https://files.pythonhosted.org/packages/85/e5/7f99bd0cc9818a91d0e8b9acc65b792e35750e3bdccd15a7ee75e64efca4/rignore-0.7.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24321efac92140b7ec910ac7c53ab0f0c86a41133d2bb4b0e6a7c94967f44dd", size = 959787, upload-time = "2025-11-05T20:42:09.765Z" }, - { url = "https://files.pythonhosted.org/packages/d4/cf/2c64f0b6725149f7c6e7e5a909d14354889b4beaadddaa5fff023ec71084/rignore-0.7.6-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5719ea14ea2b652c0c0894be5dfde954e1853a80dea27dd2fbaa749618d837f5", size = 1139186, upload-time = "2025-11-05T21:40:31.27Z" }, { url = "https://files.pythonhosted.org/packages/7f/5e/13b249613fd5d18d58662490ab910a9f0be758981d1797789913adb4e918/rignore-0.7.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3efdcf1dd84d45f3e2bd2f93303d9be103888f56dfa7c3349b5bf4f0657ec696", size = 1127725, upload-time = "2025-11-05T21:41:05.804Z" }, { url = "https://files.pythonhosted.org/packages/26/87/69387fb5dd81a0f771936381431780b8cf66fcd2cfe9495e1aaf41548931/rignore-0.7.6-cp312-cp312-win_amd64.whl", hash = "sha256:c96a285e4a8bfec0652e0bfcf42b1aabcdda1e7625f5006d188e3b1c87fdb543", size = 726090, upload-time = "2025-11-05T21:41:36.485Z" }, - { url = "https://files.pythonhosted.org/packages/09/ba/e5ea89fbde8e37a90ce456e31c5e9d85512cef5ae38e0f4d2426eb776a19/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d1a6671b2082c13bfd9a5cf4ce64670f832a6d41470556112c4ab0b6519b2fc4", size = 876987, upload-time = "2025-11-05T20:41:16.219Z" }, - { url = "https://files.pythonhosted.org/packages/d0/fb/93d14193f0ec0c3d35b763f0a000e9780f63b2031f3d3756442c2152622d/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2468729b4c5295c199d084ab88a40afcb7c8b974276805105239c07855bbacee", size = 1171110, upload-time = "2025-11-05T20:41:32.631Z" }, - { url = "https://files.pythonhosted.org/packages/9e/46/08436312ff96ffa29cfa4e1a987efc37e094531db46ba5e9fda9bb792afd/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:775710777fd71e5fdf54df69cdc249996a1d6f447a2b5bfb86dbf033fddd9cf9", size = 943339, upload-time = "2025-11-05T20:41:47.128Z" }, { url = "https://files.pythonhosted.org/packages/34/28/3b3c51328f505cfaf7e53f408f78a1e955d561135d02f9cb0341ea99f69a/rignore-0.7.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4565407f4a77f72cf9d91469e75d15d375f755f0a01236bb8aaa176278cc7085", size = 961680, upload-time = "2025-11-05T20:42:18.061Z" }, - { url = "https://files.pythonhosted.org/packages/43/f0/250b785c2e473b1ab763eaf2be820934c2a5409a722e94b279dddac21c7d/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:1b63a3dd76225ea35b01dd6596aa90b275b5d0f71d6dc28fce6dd295d98614aa", size = 1140998, upload-time = "2025-11-05T21:40:40.603Z" }, { url = "https://files.pythonhosted.org/packages/97/f4/aeb548374129dce3dc191a4bb598c944d9ed663f467b9af830315d86059c/rignore-0.7.6-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a0c6792406ae36f4e7664dc772da909451d46432ff8485774526232d4885063", size = 1130190, upload-time = "2025-11-05T21:41:16.403Z" }, - { url = "https://files.pythonhosted.org/packages/55/e4/b3c5dfdd8d8a10741dfe7199ef45d19a0e42d0c13aa377c83bd6caf65d90/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53fb28882d2538cb2d231972146c4927a9d9455e62b209f85d634408c4103538", size = 874843, upload-time = "2025-11-05T20:41:17.687Z" }, - { url = "https://files.pythonhosted.org/packages/cc/10/d6f3750233881a2a154cefc9a6a0a9b19da526b19f7f08221b552c6f827d/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87409f7eeb1103d6b77f3472a3a0d9a5953e3ae804a55080bdcb0120ee43995b", size = 1170348, upload-time = "2025-11-05T20:41:34.21Z" }, - { url = "https://files.pythonhosted.org/packages/6e/10/ad98ca05c9771c15af734cee18114a3c280914b6e34fde9ffea2e61e88aa/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684014e42e4341ab3ea23a203551857fcc03a7f8ae96ca3aefb824663f55db32", size = 942315, upload-time = "2025-11-05T20:41:48.508Z" }, { url = "https://files.pythonhosted.org/packages/de/00/ab5c0f872acb60d534e687e629c17e0896c62da9b389c66d3aa16b817aa8/rignore-0.7.6-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77356ebb01ba13f8a425c3d30fcad40e57719c0e37670d022d560884a30e4767", size = 961047, upload-time = "2025-11-05T20:42:19.403Z" }, - { url = "https://files.pythonhosted.org/packages/67/56/36d5d34210e5e7dfcd134eed8335b19e80ae940ee758f493e4f2b344dd70/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:c081f17290d8a2b96052b79207622aa635686ea39d502b976836384ede3d303c", size = 1139789, upload-time = "2025-11-05T21:40:42.119Z" }, { url = "https://files.pythonhosted.org/packages/ce/8b/a1299085b28a2f6135e30370b126e3c5055b61908622f2488ade67641479/rignore-0.7.6-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:d8955b57e42f2a5434670d5aa7b75eaf6e74602ccd8955dddf7045379cd762fb", size = 1129444, upload-time = "2025-11-05T21:41:17.906Z" }, ] @@ -7659,18 +7620,12 @@ source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/0e/f8/17bda581c517678260e6541b600eeb67745f53596dc077174141ba2f6702/setproctitle-1.3.7-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:00afa6fc507967d8c9d592a887cdc6c1f5742ceac6a4354d111ca0214847732c", size = 31793, upload-time = "2025-09-05T12:49:10.297Z" }, - { url = "https://files.pythonhosted.org/packages/59/27/1a07c38121967061564f5e0884414a5ab11a783260450172d4fc68c15621/setproctitle-1.3.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:83fcd271567d133eb9532d3b067c8a75be175b2b3b271e2812921a05303a693f", size = 34578, upload-time = "2025-09-05T12:49:13.393Z" }, - { url = "https://files.pythonhosted.org/packages/67/24/e4677ae8e1cb0d549ab558b12db10c175a889be0974c589c428fece5433e/setproctitle-1.3.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a05509cfb2059e5d2ddff701d38e474169e9ce2a298cf1b6fd5f3a213a553fe5", size = 33363, upload-time = "2025-09-05T12:49:16.829Z" }, { url = "https://files.pythonhosted.org/packages/55/d4/69ce66e4373a48fdbb37489f3ded476bb393e27f514968c3a69a67343ae0/setproctitle-1.3.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6da835e76ae18574859224a75db6e15c4c2aaa66d300a57efeaa4c97ca4c7381", size = 31508, upload-time = "2025-09-05T12:49:18.032Z" }, { url = "https://files.pythonhosted.org/packages/dc/fe/dd206cc19a25561921456f6cb12b405635319299b6f366e0bebe872abc18/setproctitle-1.3.7-cp310-cp310-win_amd64.whl", hash = "sha256:a97200acc6b64ec4cada52c2ecaf1fba1ef9429ce9c542f8a7db5bcaa9dcbd95", size = 13245, upload-time = "2025-09-05T12:49:21.023Z" }, { url = "https://files.pythonhosted.org/packages/18/2e/bd03ff02432a181c1787f6fc2a678f53b7dacdd5ded69c318fe1619556e8/setproctitle-1.3.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1607b963e7b53e24ec8a2cb4e0ab3ae591d7c6bf0a160feef0551da63452b37f", size = 32191, upload-time = "2025-09-05T12:49:24.567Z" }, - { url = "https://files.pythonhosted.org/packages/a0/3c/65edc65db3fa3df400cf13b05e9d41a3c77517b4839ce873aa6b4043184f/setproctitle-1.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8d961bba676e07d77665204f36cffaa260f526e7b32d07ab3df6a2c1dfb44ba", size = 34963, upload-time = "2025-09-05T12:49:27.044Z" }, - { url = "https://files.pythonhosted.org/packages/4a/18/77a765a339ddf046844cb4513353d8e9dcd8183da9cdba6e078713e6b0b2/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:db116850fcf7cca19492030f8d3b4b6e231278e8fe097a043957d22ce1bdf3ee", size = 33657, upload-time = "2025-09-05T12:49:30.323Z" }, { url = "https://files.pythonhosted.org/packages/6b/63/f0b6205c64d74d2a24a58644a38ec77bdbaa6afc13747e75973bf8904932/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316664d8b24a5c91ee244460bdaf7a74a707adaa9e14fbe0dc0a53168bb9aba1", size = 31836, upload-time = "2025-09-05T12:49:32.309Z" }, { url = "https://files.pythonhosted.org/packages/b6/7b/822a23f17e9003dfdee92cd72758441ca2a3680388da813a371b716fb07f/setproctitle-1.3.7-cp311-cp311-win_amd64.whl", hash = "sha256:acb9097213a8dd3410ed9f0dc147840e45ca9797785272928d4be3f0e69e3be4", size = 13243, upload-time = "2025-09-05T12:49:34.553Z" }, { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, - { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, - { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, { url = "https://files.pythonhosted.org/packages/e2/5b/a9fe517912cd6e28cf43a212b80cb679ff179a91b623138a99796d7d18a0/setproctitle-1.3.7-cp312-cp312-win_amd64.whl", hash = "sha256:9888ceb4faea3116cf02a920ff00bfbc8cc899743e4b4ac914b03625bdc3c300", size = 13247, upload-time = "2025-09-05T12:49:49.16Z" }, { url = "https://files.pythonhosted.org/packages/41/89/5b6f2faedd6ced3d3c085a5efbd91380fb1f61f4c12bc42acad37932f4e9/setproctitle-1.3.7-pp310-pypy310_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:502b902a0e4c69031b87870ff4986c290ebbb12d6038a70639f09c331b18efb2", size = 14284, upload-time = "2025-09-05T12:51:18.393Z" }, @@ -8816,21 +8771,12 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/9c/eda4615863cd8621e89aed4df680d8c3ec3da6a4cf1da113c17decd87c7f/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f0ab1c1af0cb38e3f598244c17919fb1a84d1629cc08355b0074b6d7f53138ac", size = 459065, upload-time = "2025-10-14T15:04:22.795Z" }, - { url = "https://files.pythonhosted.org/packages/86/93/cfa597fa9389e122488f7ffdbd6db505b3b915ca7435ecd7542e855898c2/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e84087b432b6ac94778de547e08611266f1f8ffad28c0ee4c82e028b0fc5966d", size = 595837, upload-time = "2025-10-14T15:04:25.057Z" }, - { url = "https://files.pythonhosted.org/packages/57/1e/68c1ed5652b48d89fc24d6af905d88ee4f82fa8bc491e2666004e307ded1/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:620bae625f4cb18427b1bb1a2d9426dc0dd5a5ba74c7c2cdb9de405f7b129863", size = 473456, upload-time = "2025-10-14T15:04:26.497Z" }, { url = "https://files.pythonhosted.org/packages/d5/dc/1a680b7458ffa3b14bb64878112aefc8f2e4f73c5af763cbf0bd43100658/watchfiles-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:544364b2b51a9b0c7000a4b4b02f90e9423d97fbbf7e06689236443ebcad81ab", size = 455614, upload-time = "2025-10-14T15:04:27.539Z" }, { url = "https://files.pythonhosted.org/packages/9b/73/bb5f38590e34687b2a9c47a244aa4dd50c56a825969c92c9c5fc7387cea1/watchfiles-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1a0bb430adb19ef49389e1ad368450193a90038b5b752f4ac089ec6942c4dff4", size = 622459, upload-time = "2025-10-14T15:04:29.491Z" }, { url = "https://files.pythonhosted.org/packages/11/a0/a60c5a7c2ec59fa062d9a9c61d02e3b6abd94d32aac2d8344c4bdd033326/watchfiles-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a36d8efe0f290835fd0f33da35042a1bb5dc0e83cbc092dcf69bce442579e88e", size = 287453, upload-time = "2025-10-14T15:04:31.53Z" }, - { url = "https://files.pythonhosted.org/packages/4a/24/33e71113b320030011c8e4316ccca04194bf0cbbaeee207f00cbc7d6b9f5/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f537afb3276d12814082a2e9b242bdcf416c2e8fd9f799a737990a1dbe906e5b", size = 460521, upload-time = "2025-10-14T15:04:35.963Z" }, - { url = "https://files.pythonhosted.org/packages/49/36/506447b73eb46c120169dc1717fe2eff07c234bb3232a7200b5f5bd816e9/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f3f58818dc0b07f7d9aa7fe9eb1037aecb9700e63e1f6acfed13e9fef648f5d", size = 596088, upload-time = "2025-10-14T15:04:38.39Z" }, - { url = "https://files.pythonhosted.org/packages/82/ab/5f39e752a9838ec4d52e9b87c1e80f1ee3ccdbe92e183c15b6577ab9de16/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bb9f66367023ae783551042d31b1d7fd422e8289eedd91f26754a66f44d5cff", size = 472923, upload-time = "2025-10-14T15:04:39.666Z" }, { url = "https://files.pythonhosted.org/packages/af/b9/a419292f05e302dea372fa7e6fda5178a92998411f8581b9830d28fb9edb/watchfiles-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aebfd0861a83e6c3d1110b78ad54704486555246e542be3e2bb94195eabb2606", size = 456080, upload-time = "2025-10-14T15:04:40.643Z" }, { url = "https://files.pythonhosted.org/packages/f7/77/16bddd9779fafb795f1a94319dc965209c5641db5bf1edbbccace6d1b3c0/watchfiles-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:399600947b170270e80134ac854e21b3ccdefa11a9529a3decc1327088180f10", size = 623046, upload-time = "2025-10-14T15:04:42.718Z" }, { url = "https://files.pythonhosted.org/packages/94/bc/f42d71125f19731ea435c3948cad148d31a64fccde3867e5ba4edee901f9/watchfiles-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:35c53bd62a0b885bf653ebf6b700d1bf05debb78ad9292cf2a942b23513dc4c4", size = 287598, upload-time = "2025-10-14T15:04:44.516Z" }, - { url = "https://files.pythonhosted.org/packages/b9/44/5769cb62d4ed055cb17417c0a109a92f007114a4e07f30812a73a4efdb11/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2edc3553362b1c38d9f06242416a5d8e9fe235c204a4072e988ce2e5bb1f69f6", size = 459485, upload-time = "2025-10-14T15:04:50.155Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2b/8530ed41112dd4a22f4dcfdb5ccf6a1baad1ff6eed8dc5a5f09e7e8c41c7/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8979280bdafff686ba5e4d8f97840f929a87ed9cdf133cbbd42f7766774d2aa", size = 594816, upload-time = "2025-10-14T15:04:52.031Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d2/f5f9fb49489f184f18470d4f99f4e862a4b3e9ac2865688eb2099e3d837a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dcc5c24523771db3a294c77d94771abcfcb82a0e0ee8efd910c37c59ec1b31bb", size = 475186, upload-time = "2025-10-14T15:04:53.064Z" }, { url = "https://files.pythonhosted.org/packages/cf/68/5707da262a119fb06fbe214d82dd1fe4a6f4af32d2d14de368d0349eb52a/watchfiles-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db5d7ae38ff20153d542460752ff397fcf5c96090c1230803713cf3147a6803", size = 456812, upload-time = "2025-10-14T15:04:55.174Z" }, { url = "https://files.pythonhosted.org/packages/78/46/7152ec29b8335f80167928944a94955015a345440f524d2dfe63fc2f437b/watchfiles-1.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:36193ed342f5b9842edd3532729a2ad55c4160ffcfa3700e0d54be496b70dd43", size = 622657, upload-time = "2025-10-14T15:04:57.521Z" }, { url = "https://files.pythonhosted.org/packages/87/0a/90eb755f568de2688cb220171c4191df932232c20946966c27a59c400850/watchfiles-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:91d4c9a823a8c987cce8fa2690923b069966dabb196dd8d137ea2cede885fde9", size = 288410, upload-time = "2025-10-14T15:05:00.081Z" },