diff --git a/benchmarking/Dockerfile b/benchmarking/Dockerfile index 64507f152a..fa960c28a9 100644 --- a/benchmarking/Dockerfile +++ b/benchmarking/Dockerfile @@ -19,6 +19,7 @@ FROM ${CURATOR_IMAGE} AS nemo_curator_benchmarking RUN apt-get update \ && apt-get install -y --no-install-recommends \ less \ + lynx \ openssh-client \ vim \ wget \ diff --git a/benchmarking/data_prep/__init__.py b/benchmarking/data_prep/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmarking/data_prep/prepare_math_benchmark_data.py b/benchmarking/data_prep/prepare_math_benchmark_data.py new file mode 100644 index 0000000000..78feeb30e9 --- /dev/null +++ b/benchmarking/data_prep/prepare_math_benchmark_data.py @@ -0,0 +1,583 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline data preparation for math pipeline benchmarks. + +Two subcommands prepare data for different benchmark stages: + + enrichment + Downloads FINEMATH_4PLUS from HuggingFace, then fetches raw binary + content from Common Crawl using WARC metadata. The resulting enriched + parquet files contain a ``binary_content`` column that the nightly math + benchmark can consume without network I/O. + + cc-index + Downloads a URL-only math dataset (e.g., MEGAMATH_WEB) from HuggingFace + and a subset of CC Index partition files from S3. + +This script is NOT part of the nightly benchmark YAML -- it is run once +(or whenever the dataset needs refreshing). + +Example usage: + + # Enrichment: download FINEMATH_4PLUS and fetch binary_content + python prepare_math_benchmark_data.py enrichment \\ + --output-path /datasets/finemath4plus_enriched \\ + --max-files 5 --workers 8 + + # CC Index: download OPENWEBMATH + CC Index partitions + python prepare_math_benchmark_data.py cc-index \\ + --output-path /datasets/cc_index_benchmark \\ + --dataset-name MEGAMATH_WEB --max-files 3 \\ + --crawl CC-MAIN-2024-10 --num-partitions 50 +""" + +from __future__ import annotations + +import argparse +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError +from huggingface_hub import hf_hub_download, list_repo_files +from huggingface_hub.utils import HfHubHTTPError, RepositoryNotFoundError +from loguru import logger + +from benchmarking.scripts.utils import setup_executor +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.download.common_crawl.download import CommonCrawlWARCReader +from nemo_curator.stages.text.io.reader import ParquetReader +from nemo_curator.stages.text.io.writer import ParquetWriter + +if TYPE_CHECKING: + import botocore.client + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +DEFAULT_S3_BUCKET = "cc-index" +DEFAULT_CC_INDEX_PREFIX = "table/cc-main/warc" +_MIN_HF_PATH_PARTS = 2 + + +@dataclass +class DownloadConfig: + """Configuration for dataset download.""" + + output_dir: Path + max_files: int | None = None + force: bool = False + workers: int = 1 + + +def load_datasets_config(config_path: Path) -> dict: + """Load dataset configurations from JSON file.""" + with open(config_path) as f: + config = json.load(f) + return {k: v for k, v in config.items() if not k.startswith("_")} + + +def _parse_huggingface_path(hf_path: str) -> tuple[str, str | None]: + """Parse HuggingFace path into (repo_id, optional subset). + + Accepts ``org/repo`` (no subset) or ``org/repo/sub/path/...`` + where everything after the second ``/`` is joined as the subset. + """ + parts = hf_path.split("/") + if len(parts) < _MIN_HF_PATH_PARTS: + msg = f"Invalid HuggingFace path (need at least org/repo): {hf_path}" + raise ValueError(msg) + + repo_id = f"{parts[0]}/{parts[1]}" + subset = "/".join(parts[2:]) or None + return repo_id, subset + + +def _get_parquet_files(repo_id: str, subset: str | None = None) -> list[str]: + """Get list of parquet files from a HuggingFace repository.""" + try: + all_files = list_repo_files(repo_id, repo_type="dataset") + except (HfHubHTTPError, RepositoryNotFoundError) as e: + logger.error(f"Failed to list files in {repo_id}: {e}") + raise + + parquet_files = [f for f in all_files if f.endswith(".parquet")] + + if subset: + subset_patterns = [f"{subset}/", f"data/{subset}/", f"train/{subset}/"] + subset_files = [] + for pattern in subset_patterns: + subset_files.extend([f for f in parquet_files if f.startswith(pattern)]) + parquet_files = subset_files or [f for f in parquet_files if subset in f] + + if not parquet_files: + logger.warning(f"No parquet files found in {repo_id}" + (f" for subset {subset}" if subset else "")) + + return sorted(parquet_files) + + +def _download_single_file( + repo_id: str, + file_path: str, + dataset_dir: Path, + force: bool = False, +) -> tuple[str, Path | None, str | None]: + """Download a single file from HuggingFace Hub.""" + try: + downloaded = hf_hub_download( + repo_id=repo_id, + filename=file_path, + repo_type="dataset", + local_dir=dataset_dir, + force_download=force, + ) + return (file_path, Path(downloaded), None) + except (HfHubHTTPError, RepositoryNotFoundError, OSError) as e: + return (file_path, None, str(e)) + + +def download_dataset( + dataset_name: str, + config: dict, + download_config: DownloadConfig, +) -> Path: + """Download a dataset from HuggingFace Hub.""" + hf_path = config["huggingface"] + repo_id, subset = _parse_huggingface_path(hf_path) + + dataset_dir_name = dataset_name.lower() + dataset_dir = download_config.output_dir / dataset_dir_name + dataset_dir.mkdir(parents=True, exist_ok=True) + + logger.info(f"Downloading {dataset_name} from {repo_id}" + (f" (subset: {subset})" if subset else "")) + logger.info(f"Output directory: {dataset_dir}") + + parquet_files = _get_parquet_files(repo_id, subset) + + if download_config.max_files: + parquet_files = parquet_files[: download_config.max_files] + logger.info(f"Limiting download to {download_config.max_files} files") + + total_files = len(parquet_files) + logger.info(f"Found {total_files} parquet files to download (workers: {download_config.workers})") + + downloaded_files = [] + failed_files = [] + + with ThreadPoolExecutor(max_workers=download_config.workers) as executor: + futures = { + executor.submit(_download_single_file, repo_id, fp, dataset_dir, download_config.force): fp + for fp in parquet_files + } + + for completed, future in enumerate(as_completed(futures), 1): + file_path = futures[future] + filename = Path(file_path).name + + try: + _, local_path, error = future.result() + if error: + logger.error(f"[{completed}/{total_files}] Failed {filename}: {error}") + failed_files.append((file_path, error)) + elif local_path: + logger.info(f"[{completed}/{total_files}] Downloaded {filename}") + downloaded_files.append(local_path) + except (RuntimeError, OSError) as e: + logger.error(f"[{completed}/{total_files}] Failed {filename}: {e}") + failed_files.append((file_path, str(e))) + + logger.info(f"Successfully downloaded {len(downloaded_files)} files to {dataset_dir}") + if failed_files: + logger.warning(f"Failed to download {len(failed_files)} files:") + for fp, err in failed_files: + logger.warning(f" - {fp}: {err}") + + return dataset_dir + + +def build_enrichment_pipeline(input_path: str, output_path: str) -> Pipeline: + """Build a pipeline that reads parquet, fetches CC binary content, and writes parquet.""" + pipeline = Pipeline( + name="math_benchmark_data_prep", + description="Fetch binary_content from Common Crawl for FINEMATH_4PLUS benchmark data", + ) + + pipeline.add_stage( + ParquetReader(file_paths=input_path).with_( + { + "file_partitioning": {"resources": Resources(cpus=1.0)}, + "parquet_reader": {"resources": Resources(cpus=1.0)}, + } + ) + ) + + pipeline.add_stage( + CommonCrawlWARCReader( + warc_filename_col="warc_filename", + warc_record_offset_col="warc_record_offset", + warc_record_length_col="warc_record_length", + ).with_(resources=Resources(cpus=0.5)) + ) + + pipeline.add_stage(ParquetWriter(path=output_path).with_(resources=Resources(cpus=1.0))) + + return pipeline + + +def _create_s3_client() -> botocore.client.BaseClient: + """Create an S3 client using boto3's standard credential chain.""" + return boto3.client("s3", config=BotoConfig(signature_version="s3v4")) + + +def list_cc_index_partitions( + s3_client: botocore.client.BaseClient, + bucket: str, + crawl: str, + prefix: str = DEFAULT_CC_INDEX_PREFIX, + limit: int | None = None, +) -> list[dict]: + """List CC Index partition files for a crawl. + + Returns a sorted list of dicts with 'key' and 'size' (bytes). + When *limit* is set, stops paginating once enough files are collected. + + Raises ``RuntimeError`` on S3 access failures. + """ + full_prefix = f"{prefix}/crawl={crawl}/subset=warc/" + paginator = s3_client.get_paginator("list_objects_v2") + + try: + files: list[dict] = [] + for page in paginator.paginate(Bucket=bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + if obj["Key"].endswith(".parquet"): + files.append({"key": obj["Key"], "size": obj["Size"]}) + if limit and len(files) >= limit: + break + if limit and len(files) >= limit: + break + except (BotoCoreError, ClientError) as e: + msg = f"Failed to list CC Index partitions at s3://{bucket}/{full_prefix}: {e}" + raise RuntimeError(msg) from e + + return sorted(files, key=lambda f: f["key"]) + + +def download_cc_index_partitions( + s3_client: botocore.client.BaseClient, + bucket: str, + partitions: list[dict], + warc_dir: Path, + *, + force: bool = False, +) -> list[Path]: + """Download CC Index partition files from S3. + + *warc_dir* is the final directory to write files into (caller builds + the hive-partitioned path). Returns list of local paths. + + Raises ``RuntimeError`` on download failure or post-download size mismatch. + """ + warc_dir.mkdir(parents=True, exist_ok=True) + + downloaded = [] + total = len(partitions) + + for i, part in enumerate(partitions, 1): + filename = Path(part["key"]).name + local_path = warc_dir / filename + expected_size = part["size"] + + if not force and local_path.exists() and local_path.stat().st_size == expected_size: + logger.info(f"[{i}/{total}] Already exists: {filename} ({expected_size / (1024**3):.2f} GB)") + downloaded.append(local_path) + continue + + logger.info(f"[{i}/{total}] Downloading {filename} ({expected_size / (1024**3):.2f} GB)...") + try: + s3_client.download_file(bucket, part["key"], str(local_path)) + except (BotoCoreError, ClientError) as e: + msg = f"Failed to download s3://{bucket}/{part['key']}: {e}" + raise RuntimeError(msg) from e + + actual_size = local_path.stat().st_size + if actual_size != expected_size: + local_path.unlink(missing_ok=True) + msg = f"Size mismatch for {filename}: expected {expected_size} bytes, got {actual_size} bytes" + raise RuntimeError(msg) + + logger.info(f"[{i}/{total}] Done: {actual_size / (1024**3):.2f} GB") + downloaded.append(local_path) + + return downloaded + + +def _scan_parquet_dir(path: Path) -> tuple[list[Path], float]: + """Return (parquet files, total size in bytes) under *path*.""" + files = sorted(path.rglob("*.parquet")) + total_bytes = sum(f.stat().st_size for f in files) + return files, total_bytes + + +def _download_hf_dataset(args: argparse.Namespace, output_subdir: str = "_raw_download") -> Path: + """Shared HF download logic for both subcommands. Returns dataset directory. + + Raises ``ValueError`` on configuration errors (missing args, unknown dataset). + """ + if args.skip_download: + if not args.raw_path: + msg = "--raw-path is required when using --skip-download" + raise ValueError(msg) + dataset_dir = args.raw_path.resolve() + logger.info(f"Skipping download, using existing data at: {dataset_dir}") + return dataset_dir + + config = load_datasets_config(args.datasets_config) + if args.dataset_name not in config: + available = ", ".join(config.keys()) + msg = f"Unknown dataset: {args.dataset_name}\nAvailable: {available}" + raise ValueError(msg) + + dataset_config = config[args.dataset_name] + output_path = args.output_path.resolve() + + logger.info(f"Downloading {args.dataset_name} from HuggingFace (max_files={args.max_files})...") + dataset_dir = download_dataset( + dataset_name=args.dataset_name, + config=dataset_config, + download_config=DownloadConfig( + output_dir=output_path / output_subdir, + max_files=args.max_files, + force=args.force, + workers=args.workers, + ), + ) + logger.info(f"Raw data downloaded to: {dataset_dir}") + return dataset_dir + + +def run_enrichment(args: argparse.Namespace) -> int: + """Download HF dataset and run WARC enrichment pipeline.""" + output_path = args.output_path.resolve() + output_path.mkdir(parents=True, exist_ok=True) + + try: + raw_data_dir = _download_hf_dataset(args) + except ValueError as e: + logger.error(str(e)) + return 1 + + logger.info("Building enrichment pipeline (ParquetReader -> CommonCrawlWARCReader -> ParquetWriter)...") + enriched_dir = str(output_path / "enriched") + pipeline = build_enrichment_pipeline( + input_path=str(raw_data_dir), + output_path=enriched_dir, + ) + + executor = setup_executor(args.executor) + logger.info(f"Pipeline description:\n{pipeline.describe()}") + logger.info("Starting enrichment pipeline...") + + try: + results = pipeline.run(executor, initial_tasks=None) + total_docs = sum(task.num_items for task in results) if results else 0 + logger.success(f"Enrichment complete: {total_docs} documents with binary_content") + logger.success(f"Enriched parquet written to: {enriched_dir}") + except Exception: + logger.exception("Enrichment pipeline failed") + return 1 + else: + return 0 + + +def _prepare_cc_index(args: argparse.Namespace, output_path: Path) -> None: + """Core logic for cc-index subcommand. + + Raises ``ValueError`` or ``RuntimeError`` on any failure so that + ``run_cc_index`` can map them to a non-zero exit code. + """ + dataset_dir = _download_hf_dataset(args, output_subdir="dataset") + + dataset_files, dataset_size = _scan_parquet_dir(dataset_dir) + if not dataset_files: + msg = f"No parquet files found under {dataset_dir}" + raise RuntimeError(msg) + dataset_size_mb = dataset_size / (1024**2) + logger.info(f"Dataset: {len(dataset_files)} parquet files, {dataset_size_mb:.1f} MB") + + if args.skip_cc_download: + if not args.cc_index_path: + msg = "--cc-index-path is required when using --skip-cc-download" + raise ValueError(msg) + cc_index_dir = args.cc_index_path.resolve() + logger.info(f"Skipping CC Index download, using existing data at: {cc_index_dir}") + else: + s3_client = _create_s3_client() + logger.info(f"Listing CC Index partitions for crawl={args.crawl} in s3://{args.s3_bucket}/...") + all_partitions = list_cc_index_partitions(s3_client, args.s3_bucket, args.crawl, limit=args.num_partitions) + + if not all_partitions: + msg = f"No CC Index partitions found for crawl={args.crawl}" + raise RuntimeError(msg) + + total_size_gb = sum(p["size"] for p in all_partitions) / (1024**3) + logger.info(f"Selected {len(all_partitions)} partitions ({total_size_gb:.2f} GB) for {args.crawl}") + + cc_index_dir = output_path / "cc_index" + warc_dir = cc_index_dir / f"crawl={args.crawl}" / "subset=warc" + download_cc_index_partitions(s3_client, args.s3_bucket, all_partitions, warc_dir, force=args.force) + + cc_files, cc_size = _scan_parquet_dir(cc_index_dir) + if not cc_files: + msg = f"No CC Index parquet files found under {cc_index_dir}" + raise RuntimeError(msg) + total_cc_size_gb = cc_size / (1024**3) + + logger.success("=" * 60) + logger.success("CC Index Lookup benchmark data prepared:") + logger.success(f" Dataset: {len(dataset_files)} files, {dataset_size_mb:.1f} MB ({dataset_dir})") + logger.success(f" CC Index: {len(cc_files)} files, {total_cc_size_gb:.2f} GB ({cc_index_dir})") + logger.success(f" Output root: {output_path}") + logger.success("=" * 60) + + +def run_cc_index(args: argparse.Namespace) -> int: + """Download HF dataset and CC Index partitions from S3.""" + output_path = args.output_path.resolve() + output_path.mkdir(parents=True, exist_ok=True) + + try: + _prepare_cc_index(args, output_path) + except (ValueError, RuntimeError) as e: + logger.error(str(e)) + return 1 + + return 0 + + +def _positive_int(value: str) -> int: + """argparse type that rejects zero and negative integers.""" + try: + n = int(value) + except ValueError: + msg = f"invalid int value: {value!r}" + raise argparse.ArgumentTypeError(msg) from None + if n < 1: + msg = f"must be >= 1, got {n}" + raise argparse.ArgumentTypeError(msg) + return n + + +def _add_common_args(parser: argparse.ArgumentParser) -> None: + """Add arguments shared by both subcommands.""" + parser.add_argument( + "--output-path", + type=Path, + required=True, + help="Root output directory", + ) + parser.add_argument( + "--datasets-config", + type=Path, + default=REPO_ROOT / "tutorials" / "math" / "datasets.json", + help="Path to datasets.json configuration file", + ) + parser.add_argument( + "--dataset-name", + help="Dataset key from datasets.json", + ) + parser.add_argument( + "--max-files", + type=_positive_int, + help="Maximum number of parquet files to download from HuggingFace", + ) + parser.add_argument( + "--workers", + type=_positive_int, + default=4, + help="Number of parallel HuggingFace download workers", + ) + parser.add_argument( + "--force", + action="store_true", + help="Force re-download even if files already exist", + ) + parser.add_argument( + "--skip-download", + action="store_true", + help="Skip HuggingFace download (use existing parquet at --raw-path)", + ) + parser.add_argument( + "--raw-path", + type=Path, + default=None, + help="Path to existing raw parquet files (used with --skip-download)", + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Prepare data for math pipeline benchmarks", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + enrich_parser = subparsers.add_parser( + "enrichment", + help="Download HF dataset and fetch binary_content via WARC", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + _add_common_args(enrich_parser) + enrich_parser.set_defaults(dataset_name="FINEMATH_4PLUS", max_files=5) + enrich_parser.add_argument( + "--executor", + default="xenna", + choices=["xenna", "ray_data"], + help="Executor to use for the enrichment pipeline", + ) + + cc_parser = subparsers.add_parser( + "cc-index", + help="Download HF dataset + CC Index partitions from S3", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + _add_common_args(cc_parser) + cc_parser.set_defaults(dataset_name="MEGAMATH_WEB", max_files=3) + cc_group = cc_parser.add_argument_group("CC Index Partitions") + cc_group.add_argument("--crawl", default="CC-MAIN-2024-10", help="Crawl ID (e.g. CC-MAIN-2024-10)") + cc_group.add_argument( + "--num-partitions", type=_positive_int, default=50, help="Number of CC Index partition files" + ) + cc_group.add_argument("--s3-bucket", default=DEFAULT_S3_BUCKET, help="S3 bucket for CC Index") + cc_group.add_argument("--skip-cc-download", action="store_true", help="Skip CC Index download") + cc_group.add_argument("--cc-index-path", type=Path, default=None, help="Existing local CC Index path") + + args = parser.parse_args() + + if args.command == "enrichment": + return run_enrichment(args) + if args.command == "cc-index": + return run_cc_index(args) + + parser.print_help() + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarking/nightly-benchmark.yaml b/benchmarking/nightly-benchmark.yaml index 7c10151695..1b7fba2e5e 100644 --- a/benchmarking/nightly-benchmark.yaml +++ b/benchmarking/nightly-benchmark.yaml @@ -77,6 +77,18 @@ datasets: formats: - type: "wds_tar_dir" path: "{datasets_path}/multimodal/mint1t" + - name: "finemath4plus" + formats: + - type: "parquet" + path: "{datasets_path}/finemath4plus/enriched" + - name: "cc_index_query_dataset" + formats: + - type: "parquet" + path: "{datasets_path}/cc_index_benchmark/dataset/megamath_web/megamath-web" + - name: "cc_index_partitions" + formats: + - type: "parquet" + path: "{datasets_path}/cc_index_benchmark/cc_index" default_timeout_s: 7200 # Optional sinks @@ -980,3 +992,123 @@ entries: min_value: 1 - metric: total_filtered_windows min_value: 1 + + - name: math_preprocess + enabled: true + script: math_pipeline_benchmark.py + args: >- + --benchmark-results-path={session_entry_dir} + --input-path={dataset:finemath4plus,parquet} + --output-path={session_entry_dir}/scratch/output + --executor=xenna + timeout_s: 600 + sink_data: + - name: slack + additional_metrics: + - throughput_docs_per_sec + - type_html_count + - extraction_success_rate + ping_on_failure: + - U07AZ1HPL1X # Ranjit Rajan + - U0849HBGCLD # Sukrit Rao + ray: + num_cpus: 64 + num_gpus: 0 + enable_object_spilling: false + requirements: + - metric: num_input_documents + exact_value: 1300 + - metric: extraction_success_rate + min_value: 1.0 + + - name: math_preprocess_classifier + enabled: true + script: math_pipeline_benchmark.py + args: >- + --benchmark-results-path={session_entry_dir} + --input-path={dataset:finemath4plus,parquet} + --output-path={session_entry_dir}/scratch/output + --executor=xenna + --run-classifier + timeout_s: 900 + sink_data: + - name: slack + additional_metrics: + - throughput_docs_per_sec + - mean_finemath_score + - classifier_percent_time + ping_on_failure: + - U07AZ1HPL1X # Ranjit Rajan + - U0849HBGCLD # Sukrit Rao + ray: + num_cpus: 64 + num_gpus: 4 + enable_object_spilling: false + requirements: + - metric: num_input_documents + exact_value: 1300 + - metric: docs_score_ge_3 + exact_value: 1057 + + - name: math_preprocess_llm_cleanup + enabled: true + script: math_pipeline_benchmark.py + args: >- + --benchmark-results-path={session_entry_dir} + --input-path={dataset:finemath4plus,parquet} + --output-path={session_entry_dir}/scratch/output + --executor=xenna + --run-llm-cleanup + --model=microsoft/phi-4 + --prompt=HTML_TO_TEXT_PROMPT + --chunk-data + --chunk-length=4096 + timeout_s: 1200 + sink_data: + - name: slack + additional_metrics: + - throughput_docs_per_sec + - num_output_documents_post_merge + - llm_cleanup_percent_time + ping_on_failure: + - U07AZ1HPL1X # Ranjit Rajan + - U0849HBGCLD # Sukrit Rao + ray: + num_cpus: 64 + num_gpus: 4 + enable_object_spilling: false + requirements: + - metric: num_input_documents + exact_value: 1300 + - metric: num_output_documents_post_merge + exact_value: 1300 + + - name: math_cc_index_lookup + enabled: true + script: cc_index_benchmark.py + args: >- + --benchmark-results-path={session_entry_dir} + --query-dataset-path={dataset:cc_index_query_dataset,parquet} + --cc-index-path={dataset:cc_index_partitions,parquet} + --output-path={session_entry_dir}/scratch/output + --executor=xenna + timeout_s: 1200 + sink_data: + - name: slack + additional_metrics: + - throughput_gb_per_sec + - total_cc_index_rows_scanned + - total_matched_rows + - match_rate + ping_on_failure: + - U07AZ1HPL1X # Ranjit Rajan + - U0849HBGCLD # Sukrit Rao + ray: + num_cpus: 64 + num_gpus: 8 + enable_object_spilling: false + requirements: + - metric: total_matched_rows + exact_value: 2384 + - metric: num_cc_index_files + exact_value: 50 diff --git a/benchmarking/scripts/cc_index_benchmark.py b/benchmarking/scripts/cc_index_benchmark.py new file mode 100644 index 0000000000..9eca0db8f7 --- /dev/null +++ b/benchmarking/scripts/cc_index_benchmark.py @@ -0,0 +1,265 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CC Index Lookup benchmark for nightly benchmarking. + +Imports stages and helpers from tutorials/math/1_cc_index_lookup.py +(CCIndexLookupStage, collect_unique_urls, get_cc_index_files) and +orchestrates them with timing and metrics collection. + +Example usage:: + + python cc_index_benchmark.py \\ + --benchmark-results-path=/tmp/results \\ + --query-dataset-path=/data/cc_index_benchmark/dataset/openwebmath/data \\ + --cc-index-path=/data/cc_index_benchmark/cc_index \\ + --output-path=/tmp/output \\ + --executor=xenna +""" + +import argparse +import importlib.util +import os +import time +import traceback +from pathlib import Path + +import ray +from loguru import logger +from utils import setup_executor, write_benchmark_results + +from nemo_curator.pipeline import Pipeline +from nemo_curator.stages.file_partitioning import FilePartitioningStage +from nemo_curator.tasks.utils import TaskPerfUtils +from nemo_curator.utils.file_utils import get_all_file_paths_under + +_TUTORIAL_PATH = Path(__file__).resolve().parent.parent.parent / "tutorials" / "math" / "1_cc_index_lookup.py" +_spec = importlib.util.spec_from_file_location("cc_index_lookup", _TUTORIAL_PATH) +if _spec is None or _spec.loader is None: + msg = ( + f"Could not load tutorial module from {_TUTORIAL_PATH}. " + "Ensure the tutorials directory is present relative to the benchmarking scripts." + ) + raise FileNotFoundError(msg) +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +CCIndexLookupStage = _mod.CCIndexLookupStage +collect_unique_urls = _mod.collect_unique_urls +get_cc_index_files = _mod.get_cc_index_files + + +def compute_output_metrics(output_path: str) -> dict: + """Compute post-hoc metrics from output enriched parquet files.""" + metrics: dict = {} + try: + output_files = get_all_file_paths_under(output_path, keep_extensions=[".parquet"]) + metrics["num_output_files"] = len(output_files) + total_bytes = sum(os.path.getsize(f) for f in output_files) + metrics["output_total_mb"] = total_bytes / (1024 * 1024) + except Exception as e: + error_traceback = traceback.format_exc() + logger.warning(f"Could not compute output metrics: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + return metrics + + +def run_benchmark(args: argparse.Namespace) -> dict: + """Run the CC Index lookup benchmark following the tutorial pattern.""" + query_path = Path(args.query_dataset_path).absolute() + cc_index_path = Path(args.cc_index_path).absolute() + output_path = Path(args.output_path).absolute() + output_path.mkdir(parents=True, exist_ok=True) + + # Ray must be initialized before ray.put(); replicate the executor's + # runtime_env so GPU visibility is configured for Xenna workers. + os.environ.setdefault("RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", "1") + ray.init( + address="auto", + ignore_reinit_error=True, + runtime_env={ + "env_vars": { + "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1", + }, + }, + ) + + # Step 1: Collect query URLs + logger.info(f"Collecting unique URLs from {query_path}...") + url_start = time.perf_counter() + query_urls = collect_unique_urls(str(query_path), args.url_col) + url_elapsed = time.perf_counter() - url_start + num_query_urls = len(query_urls) + logger.info(f"Collected {num_query_urls:,} unique URLs in {url_elapsed:.2f}s") + + # Step 2: Broadcast URLs via Ray object store + broadcast_start = time.perf_counter() + query_urls_ref = ray.put(query_urls) + broadcast_elapsed = time.perf_counter() - broadcast_start + logger.info(f"Broadcast URLs to Ray object store in {broadcast_elapsed:.2f}s") + + # Step 3: Collect CC Index files + cc_files = get_cc_index_files(str(cc_index_path), args.crawls) + num_cc_files = len(cc_files) + cc_total_bytes = sum(os.path.getsize(f) for f in cc_files) + logger.info(f"CC Index: {num_cc_files} files, {cc_total_bytes / (1024**3):.2f} GB") + + # Step 4: Build pipeline + pipeline = Pipeline( + name="cc_index_lookup_benchmark", + stages=[ + FilePartitioningStage(file_paths=cc_files, blocksize=args.blocksize), + CCIndexLookupStage( + query_urls_ref=query_urls_ref, + output_path=str(output_path), + url_col=args.url_col, + ), + ], + ) + + logger.info(f"Pipeline description:\n{pipeline.describe()}") + logger.info("Starting CC Index lookup pipeline...") + + # Step 5: Run pipeline + executor = setup_executor(args.executor) + pipeline_start = time.perf_counter() + try: + results = pipeline.run(executor) + success = True + except Exception as e: + error_traceback = traceback.format_exc() + logger.error(f"CC Index lookup pipeline failed: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + results = [] + success = False + pipeline_elapsed = time.perf_counter() - pipeline_start + + return _build_results( + results=results, + success=success, + timings={"url": url_elapsed, "broadcast": broadcast_elapsed, "pipeline": pipeline_elapsed}, + num_query_urls=num_query_urls, + num_cc_files=num_cc_files, + cc_total_bytes=cc_total_bytes, + output_path=str(output_path), + args=args, + ) + + +def _build_results( # noqa: PLR0913 + *, + results: list, + success: bool, + timings: dict[str, float], + num_query_urls: int, + num_cc_files: int, + cc_total_bytes: int, + output_path: str, + args: argparse.Namespace, +) -> dict: + """Aggregate task results into benchmark metrics.""" + total_elapsed = sum(timings.values()) + pipeline_elapsed = timings["pipeline"] + + total_input_rows = 0 + total_matched_rows = 0 + for task in results or []: + meta = getattr(task, "_metadata", {}) or {} + total_input_rows += meta.get("input_rows", 0) + total_matched_rows += meta.get("matched_rows", 0) + + metrics = { + "is_success": success, + "time_taken_s": total_elapsed, + "url_collection_time_s": timings["url"], + "broadcast_time_s": timings["broadcast"], + "pipeline_time_s": pipeline_elapsed, + "num_query_urls": num_query_urls, + "num_cc_index_files": num_cc_files, + "cc_index_total_gb": cc_total_bytes / (1024**3), + "num_output_tasks": len(results) if results else 0, + "total_cc_index_rows_scanned": total_input_rows, + "total_matched_rows": total_matched_rows, + "match_rate": total_matched_rows / total_input_rows if total_input_rows > 0 else 0, + "throughput_cc_rows_per_sec": total_input_rows / pipeline_elapsed if pipeline_elapsed > 0 else 0, + "throughput_gb_per_sec": (cc_total_bytes / (1024**3)) / pipeline_elapsed if pipeline_elapsed > 0 else 0, + } + + task_metrics = TaskPerfUtils.aggregate_task_metrics(results, prefix="task") + metrics.update(task_metrics) + metrics.update(compute_output_metrics(output_path)) + + logger.success(f"Benchmark completed in {total_elapsed:.2f}s (pipeline: {pipeline_elapsed:.2f}s)") + logger.success(f"CC Index rows scanned: {total_input_rows:,}") + logger.success(f"Matched rows: {total_matched_rows:,} ({metrics['match_rate']:.4%})") + logger.success(f"Throughput: {metrics['throughput_gb_per_sec']:.2f} GB/s") + + return { + "params": { + "args": vars(args), + "num_query_urls": num_query_urls, + "num_cc_index_files": num_cc_files, + }, + "metrics": metrics, + "tasks": results or [], + } + + +def main() -> int: + p = argparse.ArgumentParser( + description="CC Index Lookup benchmark", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--benchmark-results-path", required=True, help="Directory to write benchmark results") + p.add_argument("--query-dataset-path", required=True, help="Path to query dataset parquet (e.g. OPENWEBMATH)") + p.add_argument("--cc-index-path", required=True, help="Path to CC Index partitions (hive layout)") + p.add_argument("--output-path", required=True, help="Output directory for enriched parquet") + p.add_argument("--url-col", default="url", help="URL column name in query dataset") + p.add_argument("--blocksize", default="512MiB", help="File block size for partitioning CC Index files") + p.add_argument( + "--crawls", + nargs="+", + default=None, + help="Crawl IDs to include (default: auto-detect all)", + ) + p.add_argument("--executor", type=str, default="xenna", choices=["xenna"]) + + args = p.parse_args() + + logger.info("=== CC Index Lookup Benchmark Starting ===") + logger.info(f"Arguments: {vars(args)}") + + results = { + "params": {"args": vars(args)}, + "metrics": { + "is_success": False, + "time_taken_s": 0, + "pipeline_time_s": 0, + "num_query_urls": 0, + "num_cc_index_files": 0, + "total_cc_index_rows_scanned": 0, + "total_matched_rows": 0, + "throughput_cc_rows_per_sec": 0, + "throughput_gb_per_sec": 0, + }, + "tasks": [], + } + try: + results = run_benchmark(args) + finally: + write_benchmark_results(results, args.benchmark_results_path) + return 0 if results["metrics"]["is_success"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarking/scripts/math_pipeline_benchmark.py b/benchmarking/scripts/math_pipeline_benchmark.py new file mode 100644 index 0000000000..14d6b23037 --- /dev/null +++ b/benchmarking/scripts/math_pipeline_benchmark.py @@ -0,0 +1,403 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Math pipeline benchmark for nightly benchmarking. + +Runs a composable math curation pipeline with optional stages: + - Always: ParquetReader -> MathExtractStage -> JsonlWriter + - --run-classifier: adds FineMathClassifier after extraction + - --run-llm-cleanup: adds TokenSplitter -> LLMCleanup -> ChunkMerge after extraction + +Different YAML entries activate different stage combinations. + +Example usage: + # Preprocess only (extraction) + python math_pipeline_benchmark.py --benchmark-results-path=/tmp/results \\ + --input-path=/datasets/finemath4plus/enriched --output-path=/tmp/output + + # Preprocess + classifier + python math_pipeline_benchmark.py --benchmark-results-path=/tmp/results \\ + --input-path=/datasets/finemath4plus/enriched --output-path=/tmp/output \\ + --run-classifier + + # Preprocess + LLM cleanup + python math_pipeline_benchmark.py --benchmark-results-path=/tmp/results \\ + --input-path=/datasets/finemath4plus/enriched --output-path=/tmp/output \\ + --run-llm-cleanup --model=microsoft/phi-4 --prompt=HTML_TO_TEXT_PROMPT \\ + --chunk-data --chunk-length=4096 +""" + +import argparse +import time +import traceback +from pathlib import Path + +import pandas as pd +import ray.data +from loguru import logger +from utils import load_dataset_files, setup_executor, write_benchmark_results + +from nemo_curator.pipeline.pipeline import Pipeline +from nemo_curator.stages.math.classifiers.finemath import FineMathClassifier +from nemo_curator.stages.math.download.extract import MathContentExtractor, MathExtractStage +from nemo_curator.stages.math.modifiers.chunking import TokenSplitterStage +from nemo_curator.stages.math.modifiers.llm_cleanup import LLMCleanupStage +from nemo_curator.stages.math.modifiers.merge_chunks import ChunkMergeStage +from nemo_curator.stages.resources import Resources +from nemo_curator.stages.text.io.reader import ParquetReader +from nemo_curator.stages.text.io.writer import JsonlWriter +from nemo_curator.stages.text.modifiers import Modify +from nemo_curator.tasks.utils import TaskPerfUtils +from nemo_curator.utils import prompts +from nemo_curator.utils.file_utils import get_all_file_paths_under + +MIN_HIGH_QUALITY_SCORE = 3 + + +def _fill_null_text(text: str | None) -> str: + if pd.isna(text) or text is None: + return "" + return str(text) + + +def create_math_pipeline(args: argparse.Namespace, input_files: list[str]) -> Pipeline: + """Build the math benchmark pipeline with stages selected by CLI flags.""" + pipeline = Pipeline( + name="math_pipeline_benchmark", + description="Math curation benchmark: extract + optional classifier/LLM cleanup", + ) + + pipeline.add_stage(ParquetReader(file_paths=input_files, blocksize=args.input_blocksize)) + + pipeline.add_stage( + MathExtractStage( + extractor=MathContentExtractor( + binary_column="binary_content", + url_column="url", + mime_type_column="content_mime_type", + ), + add_filename_column=False, + ) + ) + + if args.run_classifier: + pipeline.add_stage( + FineMathClassifier(text_field="text").with_( + {"finemath_classifier_model": {"resources": Resources(cpus=1.0, gpus=1.0)}} + ) + ) + + if args.run_llm_cleanup: + pipeline.add_stage(Modify(modifier_fn=_fill_null_text, input_fields="text", output_fields="text")) + + if args.chunk_data and args.chunk_length: + pipeline.add_stage( + TokenSplitterStage( + model_name=args.model, + text_field="text", + max_length_tokens=args.chunk_length, + ) + ) + + try: + system_prompt = getattr(prompts, args.prompt) + except AttributeError: + logger.warning(f"Prompt '{args.prompt}' not found in prompts module, using as literal string.") + system_prompt = args.prompt + + pipeline.add_stage( + LLMCleanupStage( + model=args.model, + system_prompt=system_prompt, + text_field="text", + output_field="cleaned_text", + max_model_len=args.max_model_len, + ).with_(resources=Resources(cpus=1.0, gpus=1.0)) + ) + + if args.chunk_data and args.chunk_length: + pipeline.add_stage( + ChunkMergeStage( + text_field="cleaned_text", + raw_text_field="text", + chunk_id_field="chunk_id", + groupby_columns=["url"], + ) + ) + + pipeline.add_stage(JsonlWriter(path=str(args.output_path))) + + return pipeline + + +def compute_extraction_metrics(output_dir: str) -> dict: + """Compute post-hoc extraction metrics from output JSONL files.""" + metrics = {} + try: + jsonl_files = get_all_file_paths_under(output_dir, keep_extensions=[".jsonl"]) + if not jsonl_files: + return metrics + + ds = ray.data.read_json(jsonl_files).select_columns(["type", "text"]) + + def _count_types(batch: dict) -> dict: + types = batch.get("type", []) + texts = batch.get("text", []) + return { + "html": [sum(1 for t in types if t == "html")], + "text": [sum(1 for t in types if t == "text")], + "notebook": [sum(1 for t in types if t == "notebook")], + "html_empty": [ + sum(1 for t, tx in zip(types, texts, strict=False) if t == "html" and not str(tx or "").strip()) + ], + } + + counts = ds.map_batches(_count_types, batch_format="numpy") + totals = dict.fromkeys(("html", "text", "notebook", "html_empty"), 0) + for row in counts.iter_rows(): + for k in totals: + totals[k] += int(row[k]) + + metrics["type_html_count"] = totals["html"] + metrics["type_text_count"] = totals["text"] + metrics["type_notebook_count"] = totals["notebook"] + metrics["html_empty_text_count"] = totals["html_empty"] + + except Exception as e: + error_traceback = traceback.format_exc() + logger.warning(f"Could not compute extraction metrics: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + + return metrics + + +def compute_classifier_metrics(output_dir: str) -> dict: + """Compute post-hoc classifier metrics from output JSONL files.""" + metrics = {} + try: + jsonl_files = get_all_file_paths_under(output_dir, keep_extensions=[".jsonl"]) + if not jsonl_files: + return metrics + + ds = ray.data.read_json(jsonl_files).select_columns(["finemath_int_scores"]) + + score_counts = [0] * 6 + score_sum = 0 + total = 0 + for batch in ds.iter_batches(batch_format="numpy"): + for s in batch["finemath_int_scores"]: + score_int = int(s) + score_counts[min(score_int, 5)] += 1 + score_sum += score_int + total += 1 + + if total > 0: + metrics["mean_finemath_score"] = score_sum / total + + for i in range(6): + metrics[f"score_distribution_{i}"] = score_counts[i] + + metrics["docs_score_ge_3"] = sum(score_counts[MIN_HIGH_QUALITY_SCORE:]) + + except Exception as e: + error_traceback = traceback.format_exc() + logger.warning(f"Could not compute classifier metrics: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + + return metrics + + +def compute_llm_cleanup_metrics(output_dir: str) -> dict: + """Compute post-hoc LLM cleanup metrics from output JSONL files.""" + metrics = {} + try: + jsonl_files = get_all_file_paths_under(output_dir, keep_extensions=[".jsonl"]) + if not jsonl_files: + return metrics + + ds = ray.data.read_json(jsonl_files).select_columns(["cleaned_text"]) + + def _text_stats(batch: dict) -> dict: + lengths = [len(str(t or "")) for t in batch.get("cleaned_text", [])] + return { + "count": [len(lengths)], + "total_length": [sum(lengths)], + "no_content": [sum(1 for ln in lengths if ln == 0)], + } + + agg = ds.map_batches(_text_stats, batch_format="numpy") + totals = {"count": 0, "total_length": 0, "no_content": 0} + for row in agg.iter_rows(): + for k in totals: + totals[k] += int(row[k]) + + metrics["num_output_documents_post_merge"] = totals["count"] + if totals["count"] > 0: + metrics["avg_output_text_length"] = totals["total_length"] / totals["count"] + metrics["no_useful_content_count"] = totals["no_content"] + + except Exception as e: + error_traceback = traceback.format_exc() + logger.warning(f"Could not compute LLM cleanup metrics: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + + return metrics + + +def run_benchmark(args: argparse.Namespace) -> dict: + """Run the math pipeline benchmark and collect metrics.""" + input_path = Path(args.input_path).resolve() + output_path = Path(args.output_path).resolve() + output_path.mkdir(parents=True, exist_ok=True) + + # Load input files with optional size limiting + if args.dataset_size_gb: + input_files = load_dataset_files(input_path, dataset_size_gb=args.dataset_size_gb) + else: + input_files = load_dataset_files(input_path, dataset_ratio=1.0) + + num_input_files = len(input_files) + logger.info(f"Input files: {num_input_files}") + logger.info(f"Output path: {output_path}") + stages = "extract" + if args.run_classifier: + stages += " + classifier" + if args.run_llm_cleanup: + stages += " + llm_cleanup" + logger.info(f"Stages: {stages}") + + pipeline = create_math_pipeline(args, input_files) + executor = setup_executor(args.executor) + + logger.info(f"Pipeline description:\n{pipeline.describe()}") + logger.info("Starting math pipeline execution...") + + start = time.perf_counter() + + try: + results = pipeline.run(executor, initial_tasks=None) + success = True + except Exception as e: + error_traceback = traceback.format_exc() + logger.error(f"Pipeline failed: {e}") + logger.debug(f"Full traceback:\n{error_traceback}") + results = [] + success = False + + elapsed = time.perf_counter() - start + + num_input_documents = TaskPerfUtils.get_aggregated_stage_stat(results, "extract_", "num_items_processed") + num_output_documents = TaskPerfUtils.get_aggregated_stage_stat(results, "jsonl_writer", "num_items_processed") + + metrics = { + "is_success": success, + "time_taken_s": elapsed, + "num_output_tasks": len(results) if results else 0, + "num_input_documents": int(num_input_documents), + "num_output_documents": int(num_output_documents), + "num_input_files": num_input_files, + "throughput_docs_per_sec": num_input_documents / elapsed if elapsed > 0 else 0, + } + + if num_input_documents > 0: + metrics["extraction_success_rate"] = num_output_documents / num_input_documents + + task_metrics = TaskPerfUtils.aggregate_task_metrics(results, prefix="task") + metrics.update(task_metrics) + + # Domain-specific post-hoc metrics from output files + output_dir = str(output_path) + metrics.update(compute_extraction_metrics(output_dir)) + + if args.run_classifier: + metrics.update(compute_classifier_metrics(output_dir)) + + if args.run_llm_cleanup: + metrics.update(compute_llm_cleanup_metrics(output_dir)) + + logger.success(f"Benchmark completed in {elapsed:.2f}s") + logger.success(f"Throughput: {metrics['throughput_docs_per_sec']:.1f} docs/sec") + + return { + "params": {"args": vars(args), "num_input_files": num_input_files}, + "metrics": metrics, + "tasks": results or [], + } + + +def main() -> int: + p = argparse.ArgumentParser( + description="Math pipeline benchmark", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--benchmark-results-path", required=True, help="Directory to write benchmark results") + + # Input/output + p.add_argument("--input-path", type=str, required=True, help="Path to enriched parquet dataset") + p.add_argument("--output-path", type=str, required=True, help="Output directory for JSONL results") + p.add_argument("--dataset-size-gb", type=float, default=None, help="Limit input dataset size (GB); None = use all") + p.add_argument( + "--input-blocksize", + type=str, + default=None, + help="Merge input files into tasks of this size (e.g. '50MB'); None = 1 file per task", + ) + + # Stage selection + stage_group = p.add_argument_group("Stage Selection") + stage_group.add_argument("--run-classifier", action="store_true", help="Add FineMath classifier after extraction") + stage_group.add_argument("--run-llm-cleanup", action="store_true", help="Add LLM cleanup after extraction") + + # LLM cleanup options + llm_group = p.add_argument_group("LLM Cleanup Options (requires --run-llm-cleanup)") + llm_group.add_argument("--model", type=str, default="microsoft/phi-4", help="LLM model identifier") + llm_group.add_argument( + "--prompt", type=str, default="HTML_TO_TEXT_PROMPT", help="Prompt name from prompts module or literal string" + ) + llm_group.add_argument("--chunk-data", action="store_true", help="Enable token-based chunking before LLM") + llm_group.add_argument("--chunk-length", type=int, default=4096, help="Max tokens per chunk") + llm_group.add_argument("--max-model-len", type=int, default=None, help="Max model context length for vLLM") + + # Executor + p.add_argument("--executor", type=str, default="xenna", choices=["xenna", "ray_data"]) + + args = p.parse_args() + + if args.run_llm_cleanup and not args.model: + p.error("--model is required when using --run-llm-cleanup") + + logger.info("=== Math Pipeline Benchmark Starting ===") + logger.info(f"Arguments: {vars(args)}") + + results = { + "params": {"args": vars(args), "num_input_files": 0}, + "metrics": { + "is_success": False, + "time_taken_s": 0, + "num_output_tasks": 0, + "num_input_documents": 0, + "num_output_documents": 0, + "throughput_docs_per_sec": 0, + }, + "tasks": [], + } + try: + results = run_benchmark(args) + finally: + write_benchmark_results(results, args.benchmark_results_path) + return 0 if results["metrics"]["is_success"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/nemo_curator/stages/math/download/extract.py b/nemo_curator/stages/math/download/extract.py index d8d0cb901e..3bcddef930 100644 --- a/nemo_curator/stages/math/download/extract.py +++ b/nemo_curator/stages/math/download/extract.py @@ -105,6 +105,19 @@ def __post_init__(self): self._magic = None self._lock = threading.Lock() + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + state["_lynx"] = None + state["_magic"] = None + state["_lock"] = None + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._lynx = None + self._magic = None + self._lock = threading.Lock() + def input_columns(self) -> list[str]: return [self.binary_column, self.url_column, self.mime_type_column] diff --git a/nemo_curator/stages/text/download/common_crawl/download.py b/nemo_curator/stages/text/download/common_crawl/download.py index 039333446e..c530740e05 100644 --- a/nemo_curator/stages/text/download/common_crawl/download.py +++ b/nemo_curator/stages/text/download/common_crawl/download.py @@ -17,6 +17,7 @@ import io import os import subprocess +import threading from urllib.parse import urljoin, urlparse import pandas as pd @@ -109,10 +110,18 @@ def _download_to_path(self, url: str, path: str) -> tuple[bool, str | None]: class CommonCrawlWARCReader(ProcessingStage[DocumentBatch, DocumentBatch]): """ - Reads WARC records directly from Common Crawl using HTTPS range requests. + Reads WARC records directly from Common Crawl using HTTPS or S3 range requests. This stage fetches raw HTML content from Common Crawl's public servers - using byte-range requests. No AWS credentials or s5cmd required. + using byte-range requests. + + Transport modes: + - **HTTPS** (default): Uses ``data.commoncrawl.org`` via ``requests``. + - **S3**: Uses ``boto3`` range requests against the ``commoncrawl`` bucket. + Activated by ``use_s3=True`` or by setting ``CC_USE_S3=1``. + Credentials, region, and endpoint are resolved + by boto3's standard credential chain (env vars, ``~/.aws/`` config, + instance profiles, etc.). """ def __init__( # noqa: PLR0913 @@ -125,6 +134,9 @@ def __init__( # noqa: PLR0913 max_workers: int = 16, timeout: int = 30, max_retries: int = 3, + use_s3: bool | None = None, + s3_bucket: str | None = None, + s3_key_prefix: str | None = None, ): """ Initialize the WARC reader. @@ -138,6 +150,17 @@ def __init__( # noqa: PLR0913 max_workers: Number of parallel threads for fetching. timeout: HTTP request timeout in seconds. max_retries: Number of retries for failed requests. + use_s3: If True, fetch via S3 (boto3) instead of HTTPS. + If None (default), reads ``CC_USE_S3`` env var. + Accepted truthy values: ``1``, ``true``, ``yes``. + s3_bucket: S3 bucket name. Falls back to ``CC_S3_BUCKET`` env var, + then ``"commoncrawl"``. + s3_key_prefix: Prefix to strip from ``warc_filename`` when + building the S3 object key. Falls back to + ``CC_S3_KEY_PREFIX`` env var. Default empty (key = + warc_filename as-is, correct for the AWS ``commoncrawl`` + bucket). Set when the bucket name overlaps with the + leading path segment in the dataset's warc filenames. """ self.warc_filename_col = warc_filename_col self.warc_record_offset_col = warc_record_offset_col @@ -149,6 +172,28 @@ def __init__( # noqa: PLR0913 self.max_retries = max_retries self.name = "CommonCrawlWARCReader" self._session = None + self._s3_client = None + self._lock = threading.Lock() + + if use_s3 is None: + self.use_s3 = os.environ.get("CC_USE_S3", "").lower() in ("1", "true", "yes") + else: + self.use_s3 = use_s3 + self.s3_bucket = s3_bucket or os.environ.get("CC_S3_BUCKET", "commoncrawl") + self.s3_key_prefix = s3_key_prefix if s3_key_prefix is not None else os.environ.get("CC_S3_KEY_PREFIX", "") + + def __getstate__(self) -> dict[str, object]: + state = self.__dict__.copy() + state["_session"] = None + state["_s3_client"] = None + state["_lock"] = None + return state + + def __setstate__(self, state: dict[str, object]) -> None: + self.__dict__.update(state) + self._session = None + self._s3_client = None + self._lock = threading.Lock() def inputs(self) -> tuple[list[str], list[str]]: return ( @@ -162,17 +207,101 @@ def outputs(self) -> tuple[list[str], list[str]]: def _get_session(self) -> requests.Session: """Get or create a requests session for connection pooling.""" if self._session is None: - self._session = requests.Session() - # Configure connection pooling for better performance - adapter = requests.adapters.HTTPAdapter( - pool_connections=self.max_workers, - pool_maxsize=self.max_workers * 2, - max_retries=self.max_retries, - ) - self._session.mount("https://", adapter) - self._session.mount("http://", adapter) + with self._lock: + if self._session is None: + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=self.max_workers, + pool_maxsize=self.max_workers * 2, + max_retries=self.max_retries, + ) + session.mount("https://", adapter) + session.mount("http://", adapter) + self._session = session return self._session + def _get_s3_client(self) -> object: + """Get or create a boto3 S3 client with double-checked locking. + + Credentials, region, and endpoint are resolved entirely by boto3's + standard chain (``AWS_*`` env vars, ``~/.aws/config``, instance + profiles). Only connection-pool and retry settings are overridden. + """ + if self._s3_client is None: + with self._lock: + if self._s3_client is None: + try: + import boto3 + except ModuleNotFoundError as exc: + msg = ( + "CommonCrawlWARCReader configured with use_s3=True but boto3 is not installed. " + "Install boto3 or set use_s3=False (or unset CC_USE_S3)." + ) + raise RuntimeError(msg) from exc + from botocore.config import Config as BotoConfig + + boto_cfg = BotoConfig( + max_pool_connections=self.max_workers * 2, + retries={"max_attempts": self.max_retries, "mode": "adaptive"}, + connect_timeout=self.timeout, + read_timeout=self.timeout, + ) + self._s3_client = boto3.client("s3", config=boto_cfg) + logger.info(f"S3 client initialized for bucket={self.s3_bucket}") + return self._s3_client + + def _s3_key_from_filename(self, filename: str) -> str: + """Derive S3 object key from the warc_filename column value. + + Strips ``s3_key_prefix`` from the front of *filename* when present. + E.g. prefix ``"crawl-data/"`` + filename ``"crawl-data/CC-MAIN-…"`` + → key ``"CC-MAIN-…"``. With an empty prefix the filename is used + as-is (the default for the AWS ``commoncrawl`` bucket). + """ + if self.s3_key_prefix and filename.startswith(self.s3_key_prefix): + return filename[len(self.s3_key_prefix) :] + return filename + + def _read_warc_record_s3(self, row: pd.Series) -> bytes | None: + """Fetch a single WARC record using S3 range request (boto3).""" + filename = None + try: + filename = row[self.warc_filename_col] + offset = int(row[self.warc_record_offset_col]) + length = int(row[self.warc_record_length_col]) + end_byte = offset + length - 1 + + resp = self._get_s3_client().get_object( + Bucket=self.s3_bucket, + Key=self._s3_key_from_filename(filename), + Range=f"bytes={offset}-{end_byte}", + ) + raw_bytes = resp["Body"].read() + + try: + decompressed = gzip.decompress(raw_bytes) + except gzip.BadGzipFile: + decompressed = raw_bytes + + try: + stream = io.BytesIO(decompressed) + archive_iterator = ArchiveIterator(stream) + for record in archive_iterator: + if record.rec_type == "response": + return record.content_stream().read() + except Exception as e: # noqa: BLE001 + logger.debug(f"Failed to parse WARC record {filename}: {e}, returning decompressed bytes") + return decompressed + else: + logger.debug(f"No response record found in WARC for {filename}, returning raw content") + return decompressed + + except RuntimeError: + raise # Propagate configuration errors (e.g. missing boto3) + except Exception as e: # noqa: BLE001 + logger.warning(f"S3 fetch failed for {filename}: {e}") + return None + def _read_warc_record(self, row: pd.Series) -> bytes | None: # noqa: C901, PLR0911 """Fetch a single WARC record using HTTPS range request. @@ -250,10 +379,11 @@ def _read_warc_records_batch(self, df_partition: pd.DataFrame) -> list[bytes | N """Fetch multiple records in parallel using ThreadPoolExecutor.""" results = [None] * len(df_partition) rows = list(df_partition.iterrows()) + fetch_fn = self._read_warc_record_s3 if self.use_s3 else self._read_warc_record def fetch_row(row_data: tuple[int, pd.Series]) -> tuple[int, bytes | None]: idx, row = row_data - return idx, self._read_warc_record(row) + return idx, fetch_fn(row) # Use a thread pool to parallelize the HTTP requests # Requests are IO bound, so threads work well here @@ -264,7 +394,12 @@ def fetch_row(row_data: tuple[int, pd.Series]) -> tuple[int, bytes | None]: try: i, result = future.result() results[i] = result - except Exception as e: # noqa: BLE001, PERF203 + except RuntimeError: # noqa: PERF203 + # Propagate configuration errors (e.g. missing boto3) + for f in futures: + f.cancel() + raise + except Exception as e: # noqa: BLE001 logger.warning(f"Error in thread pool: {e}") return results diff --git a/pyproject.toml b/pyproject.toml index e010874599..73c6a5d0d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -171,6 +171,7 @@ video_cuda12 = [ # Math Curation Dependencies math_cpu = [ "nemo_curator[text_cpu]", # Math examples use text processing utilities + "boto3>=1.35", # S3 range requests for Common Crawl WARC fetching ] math_cuda12 = [ diff --git a/tests/stages/math_stages/download/test_math_content_extractor.py b/tests/stages/math_stages/download/test_math_content_extractor.py index 3be868bbef..e069db6dbf 100644 --- a/tests/stages/math_stages/download/test_math_content_extractor.py +++ b/tests/stages/math_stages/download/test_math_content_extractor.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy from unittest import mock import pytest @@ -254,7 +255,7 @@ def test_lazy_initialization_lynx( assert result["text"] == extracted_text_responses["lynx"] assert result["type"] == "html" - def test_lazy_initialization_magic(self, sample_test_content: str, sample_urls: dict) -> None: + def test_lazy_initialization_magic(self, sample_text_content: str, sample_urls: dict) -> None: """Test lazy initialization of magic MIME detector.""" extractor = MathContentExtractor() @@ -267,7 +268,7 @@ def test_lazy_initialization_magic(self, sample_test_content: str, sample_urls: mock_magic_class.return_value = mock_magic_instance record = { - "binary_content": sample_test_content.encode("utf-8"), + "binary_content": sample_text_content.encode("utf-8"), "url": sample_urls["text"], "mime_type": "text/plain", } @@ -278,5 +279,13 @@ def test_lazy_initialization_magic(self, sample_test_content: str, sample_urls: mock_magic_class.assert_called_once_with(mime=True) assert extractor._magic is mock_magic_instance assert result["magic_mime_type"] == "text/plain" - assert result["text"] == sample_test_content + assert result["text"] == sample_text_content assert result["type"] == "text" + + def test_deepcopy_extractor_with_lock(self) -> None: + extractor = MathContentExtractor() + + cloned = copy.deepcopy(extractor) + + assert cloned is not extractor + assert cloned._lock is not None diff --git a/tests/stages/text/download/common_crawl/test_warc_reader.py b/tests/stages/text/download/common_crawl/test_warc_reader.py new file mode 100644 index 0000000000..3f44bb1daa --- /dev/null +++ b/tests/stages/text/download/common_crawl/test_warc_reader.py @@ -0,0 +1,199 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import sys +from unittest import mock + +import pandas as pd +import pytest + +from nemo_curator.stages.text.download.common_crawl.download import CommonCrawlWARCReader + + +class TestTransportSelection: + """Test transport selection defaults and env-var behavior.""" + + @pytest.mark.parametrize("env_value", ["1", "true", "yes", "TRUE"]) + def test_cc_use_s3_env_enables_s3(self, monkeypatch: pytest.MonkeyPatch, env_value: str) -> None: + monkeypatch.setenv("CC_USE_S3", env_value) + + reader = CommonCrawlWARCReader() + + assert reader.use_s3 is True + + def test_use_s3_without_boto3_raises_clear_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(sys.modules, "boto3", None) + reader = CommonCrawlWARCReader(use_s3=True) + reader._s3_client = None + + with pytest.raises(RuntimeError, match="boto3 is not installed"): + reader._get_s3_client() + + +class TestS3KeyFromFilename: + """Test _s3_key_from_filename prefix-stripping logic.""" + + @pytest.mark.parametrize( + ("prefix", "filename", "expected"), + [ + ("crawl-data/", "crawl-data/CC-MAIN-2024-10/seg/warc.gz", "CC-MAIN-2024-10/seg/warc.gz"), + ("crawl-data/", "other-path/file.gz", "other-path/file.gz"), + ("", "crawl-data/CC-MAIN-2024-10/seg/warc.gz", "crawl-data/CC-MAIN-2024-10/seg/warc.gz"), + ], + ids=["strip-prefix", "prefix-no-match", "empty-prefix-passthrough"], + ) + def test_s3_key_from_filename(self, prefix: str, filename: str, expected: str) -> None: + reader = CommonCrawlWARCReader(use_s3=False, s3_key_prefix=prefix) + assert reader._s3_key_from_filename(filename) == expected + + +class TestReadWarcRecordS3: + """Test S3-based WARC record fetching.""" + + @staticmethod + def _make_gzipped_warc() -> bytes: + """Build minimal gzip-compressed WARC response record.""" + warc_record = "WARC/1.0\r\nWARC-Type: response\r\nContent-Length: 11\r\n\r\nHello World" + return gzip.compress(warc_record.encode("utf-8")) + + def test_read_warc_record_s3_default_bucket(self) -> None: + """S3 fetch uses the default 'commoncrawl' bucket when none is specified.""" + reader = CommonCrawlWARCReader(use_s3=True) + assert reader.s3_bucket == "commoncrawl" + + raw_gz = self._make_gzipped_warc() + mock_body = mock.Mock() + mock_body.read.return_value = raw_gz + mock_client = mock.Mock() + mock_client.get_object.return_value = {"Body": mock_body} + reader._s3_client = mock_client + + warc_filename = "crawl-data/CC-MAIN-2024-10/seg/warc.gz" + row = pd.Series( + { + "warc_filename": warc_filename, + "warc_record_offset": 100, + "warc_record_length": len(raw_gz), + } + ) + + result = reader._read_warc_record_s3(row) + + assert result is not None + mock_client.get_object.assert_called_once_with( + Bucket="commoncrawl", + Key=warc_filename, + Range=f"bytes=100-{100 + len(raw_gz) - 1}", + ) + + def test_read_warc_record_s3_custom_bucket(self) -> None: + """S3 fetch uses the specified custom bucket.""" + reader = CommonCrawlWARCReader(use_s3=True, s3_bucket="my-bucket", s3_key_prefix="crawl-data/") + raw_gz = self._make_gzipped_warc() + + mock_body = mock.Mock() + mock_body.read.return_value = raw_gz + mock_client = mock.Mock() + mock_client.get_object.return_value = {"Body": mock_body} + reader._s3_client = mock_client + + row = pd.Series( + { + "warc_filename": "crawl-data/CC-MAIN-2024-10/seg/warc.gz", + "warc_record_offset": 0, + "warc_record_length": len(raw_gz), + } + ) + + result = reader._read_warc_record_s3(row) + + assert result is not None + mock_client.get_object.assert_called_once_with( + Bucket="my-bucket", + Key="CC-MAIN-2024-10/seg/warc.gz", + Range=f"bytes=0-{len(raw_gz) - 1}", + ) + + def test_batch_fetch_propagates_boto3_runtime_error(self, monkeypatch: pytest.MonkeyPatch) -> None: + """RuntimeError from missing boto3 propagates through the threadpool.""" + monkeypatch.setitem(sys.modules, "boto3", None) + reader = CommonCrawlWARCReader(use_s3=True) + reader._s3_client = None + + df = pd.DataFrame( + { + "warc_filename": ["file.warc.gz"], + "warc_record_offset": [0], + "warc_record_length": [100], + } + ) + + with pytest.raises(RuntimeError, match="boto3 is not installed"): + reader._read_warc_records_batch(df) + + def test_read_warc_record_s3_failure(self) -> None: + """S3 exception returns None and does not raise.""" + reader = CommonCrawlWARCReader(use_s3=True) + + mock_client = mock.Mock() + mock_client.get_object.side_effect = Exception("AccessDenied") + reader._s3_client = mock_client + + row = pd.Series( + { + "warc_filename": "CC-MAIN-2024-10/seg/warc.gz", + "warc_record_offset": 0, + "warc_record_length": 100, + } + ) + + result = reader._read_warc_record_s3(row) + assert result is None + + +class TestReadWarcRecordHTTPS: + """Test HTTPS-based WARC record fetching.""" + + def test_read_warc_record_https(self) -> None: + """Successful HTTPS range-request fetch returns content.""" + reader = CommonCrawlWARCReader(use_s3=False) + + warc_payload = "WARC/1.0\r\nWARC-Type: response\r\nContent-Length: 11\r\n\r\nHello World" + raw_gz = gzip.compress(warc_payload.encode("utf-8")) + + mock_response = mock.Mock() + mock_response.status_code = 206 + mock_response.content = raw_gz + + mock_session = mock.Mock() + mock_session.get.return_value = mock_response + reader._session = mock_session + + row = pd.Series( + { + "warc_filename": "crawl-data/CC-MAIN-2024-10/seg/warc.gz", + "warc_record_offset": 0, + "warc_record_length": len(raw_gz), + } + ) + + result = reader._read_warc_record(row) + + assert result is not None + mock_session.get.assert_called_once_with( + "https://data.commoncrawl.org/crawl-data/CC-MAIN-2024-10/seg/warc.gz", + headers={"Range": f"bytes=0-{len(raw_gz) - 1}"}, + timeout=30, + ) diff --git a/tutorials/math/1_cc_index_lookup.py b/tutorials/math/1_cc_index_lookup.py index 865cd178b3..35d76afbf4 100644 --- a/tutorials/math/1_cc_index_lookup.py +++ b/tutorials/math/1_cc_index_lookup.py @@ -19,6 +19,7 @@ from typing import Any import cudf +import pandas as pd import ray from loguru import logger @@ -131,25 +132,29 @@ def process(self, task: FileGroupTask) -> FileGroupTask: def collect_unique_urls(input_path: str, url_col: str = "url") -> cudf.DataFrame: - """Collect unique URLs from input dataset using cuDF.""" + """Collect unique URLs from input dataset. + + Uses pandas for reading and dedup, then converts the small deduplicated result to cuDF + for the downstream GPU merge. + """ logger.info(f"Collecting unique URLs from: {input_path}") - input_files = get_all_file_paths_under(input_path, keep_extensions=[".parquet"]) + input_files = get_all_file_paths_under(input_path, recurse_subdirectories=True, keep_extensions=[".parquet"]) if not input_files: msg = f"No parquet files found at {input_path}" raise FileNotFoundError(msg) logger.info(f"Found {len(input_files)} input files") - dfs = [cudf.read_parquet(f, columns=[url_col]) for f in input_files] - combined = cudf.concat(dfs, ignore_index=True) + dfs = [pd.read_parquet(f, columns=[url_col]) for f in input_files] + combined = pd.concat(dfs, ignore_index=True) unique_urls = combined.drop_duplicates(subset=[url_col]) if url_col != "url": unique_urls = unique_urls.rename(columns={url_col: "url"}) logger.info(f"Collected {len(unique_urls):,} unique URLs") - return unique_urls + return cudf.DataFrame(unique_urls) def get_available_crawls(cc_index_base: str) -> list[str]: @@ -187,6 +192,10 @@ def get_cc_index_files(cc_index_base: str, crawls: list[str] | None = None) -> l def run_cc_index_lookup(config: CCIndexLookupConfig) -> None: + # Initialize Ray before ray.put() so the GPU visibility env var is + # set for all workers. Without this, Xenna cannot detect GPUs. + os.environ.setdefault("RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES", "1") + ray_client = RayClient() ray_client.start() diff --git a/tutorials/math/README.md b/tutorials/math/README.md index 16e926a64e..6661fb0ecf 100644 --- a/tutorials/math/README.md +++ b/tutorials/math/README.md @@ -405,7 +405,7 @@ Extract and preprocess text from raw web data: ```bash # For datasets with WARC metadata (FineMath, or after CC Index lookup) -# Uses --fetch-cc to download content from Common Crawl S3 +# Uses --fetch-cc to download content from Common Crawl (HTTPS by default; set CC_USE_S3=1 to use S3) python tutorials/math/2_text_preprocess.py \ --input "$MATH_DATA_DIR/enriched/*.parquet" \ --output $MATH_DATA_DIR/preprocessed \ @@ -424,6 +424,12 @@ python tutorials/math/2_text_preprocess.py \ --report-stats ``` +**Common Crawl fetch env vars (used by `CommonCrawlWARCReader`):** + +- `CC_USE_S3`: Set to `1`/`true`/`yes` to use S3 range requests; default is HTTPS. +- `CC_S3_BUCKET`: Override bucket name (default: `commoncrawl`). +- `CC_S3_KEY_PREFIX`: Optional prefix to strip from `warc_filename` when building S3 object key. + **Input**: Parquet files with either: - `warc_filename`, `warc_record_offset`, `warc_record_length` columns → use `--fetch-cc` to download from Common Crawl - `binary_content` (bytes) column → content already present, no fetch needed diff --git a/uv.lock b/uv.lock index 9625481eec..97f562a896 100644 --- a/uv.lock +++ b/uv.lock @@ -2555,15 +2555,15 @@ wheels = [ [[package]] name = "google-auth-httplib2" -version = "0.3.0" +version = "0.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-auth" }, { name = "httplib2" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d5/ad/c1f2b1175096a8d04cf202ad5ea6065f108d26be6fc7215876bde4a7981d/google_auth_httplib2-0.3.0.tar.gz", hash = "sha256:177898a0175252480d5ed916aeea183c2df87c1f9c26705d74ae6b951c268b0b", size = 11134, upload-time = "2025-12-15T22:13:51.825Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/99/107612bef8d24b298bb5a7c8466f908ecda791d43f9466f5c3978f5b24c1/google_auth_httplib2-0.3.1.tar.gz", hash = "sha256:0af542e815784cb64159b4469aa5d71dd41069ba93effa006e1916b1dcd88e55", size = 11152, upload-time = "2026-03-30T22:50:26.766Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/d5/3c97526c8796d3caf5f4b3bed2b05e8a7102326f00a334e7a438237f3b22/google_auth_httplib2-0.3.0-py3-none-any.whl", hash = "sha256:426167e5df066e3f5a0fc7ea18768c08e7296046594ce4c8c409c2457dd1f776", size = 9529, upload-time = "2025-12-15T22:13:51.048Z" }, + { url = "https://files.pythonhosted.org/packages/97/e9/93afb14d23a949acaa3f4e7cc51a0024671174e116e35f42850764b99634/google_auth_httplib2-0.3.1-py3-none-any.whl", hash = "sha256:682356a90ef4ba3d06548c37e9112eea6fc00395a11b0303a644c1a86abc275c", size = 9534, upload-time = "2026-03-30T22:49:03.384Z" }, ] [[package]] @@ -2607,6 +2607,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/6a/33d1702184d94106d3cdd7bfb788e19723206fce152e303473ca3b946c7b/greenlet-3.3.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f8496d434d5cb2dce025773ba5597f71f5410ae499d5dd9533e0653258cdb3d", size = 273658, upload-time = "2025-12-04T14:23:37.494Z" }, { url = "https://files.pythonhosted.org/packages/d6/b7/2b5805bbf1907c26e434f4e448cd8b696a0b71725204fa21a211ff0c04a7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b96dc7eef78fd404e022e165ec55327f935b9b52ff355b067eb4a0267fc1cffb", size = 574810, upload-time = "2025-12-04T14:50:04.154Z" }, { url = "https://files.pythonhosted.org/packages/94/38/343242ec12eddf3d8458c73f555c084359883d4ddc674240d9e61ec51fd6/greenlet-3.3.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73631cd5cccbcfe63e3f9492aaa664d278fda0ce5c3d43aeda8e77317e38efbd", size = 586248, upload-time = "2025-12-04T14:57:39.35Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d0/0ae86792fb212e4384041e0ef8e7bc66f59a54912ce407d26a966ed2914d/greenlet-3.3.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b299a0cb979f5d7197442dccc3aee67fce53500cd88951b7e6c35575701c980b", size = 597403, upload-time = "2025-12-04T15:07:10.831Z" }, { url = "https://files.pythonhosted.org/packages/b6/a8/15d0aa26c0036a15d2659175af00954aaaa5d0d66ba538345bd88013b4d7/greenlet-3.3.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dee147740789a4632cace364816046e43310b59ff8fb79833ab043aefa72fd5", size = 586910, upload-time = "2025-12-04T14:25:59.705Z" }, { url = "https://files.pythonhosted.org/packages/e1/9b/68d5e3b7ccaba3907e5532cf8b9bf16f9ef5056a008f195a367db0ff32db/greenlet-3.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:39b28e339fc3c348427560494e28d8a6f3561c8d2bcf7d706e1c624ed8d822b9", size = 1547206, upload-time = "2025-12-04T15:04:21.027Z" }, { url = "https://files.pythonhosted.org/packages/66/bd/e3086ccedc61e49f91e2cfb5ffad9d8d62e5dc85e512a6200f096875b60c/greenlet-3.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b3c374782c2935cc63b2a27ba8708471de4ad1abaa862ffdb1ef45a643ddbb7d", size = 1613359, upload-time = "2025-12-04T14:27:26.548Z" }, @@ -2614,6 +2615,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/cb/48e964c452ca2b92175a9b2dca037a553036cb053ba69e284650ce755f13/greenlet-3.3.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e29f3018580e8412d6aaf5641bb7745d38c85228dacf51a73bd4e26ddf2a6a8e", size = 274908, upload-time = "2025-12-04T14:23:26.435Z" }, { url = "https://files.pythonhosted.org/packages/28/da/38d7bff4d0277b594ec557f479d65272a893f1f2a716cad91efeb8680953/greenlet-3.3.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a687205fb22794e838f947e2194c0566d3812966b41c78709554aa883183fb62", size = 577113, upload-time = "2025-12-04T14:50:05.493Z" }, { url = "https://files.pythonhosted.org/packages/3c/f2/89c5eb0faddc3ff014f1c04467d67dee0d1d334ab81fadbf3744847f8a8a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4243050a88ba61842186cb9e63c7dfa677ec146160b0efd73b855a3d9c7fcf32", size = 590338, upload-time = "2025-12-04T14:57:41.136Z" }, + { url = "https://files.pythonhosted.org/packages/80/d7/db0a5085035d05134f8c089643da2b44cc9b80647c39e93129c5ef170d8f/greenlet-3.3.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:670d0f94cd302d81796e37299bcd04b95d62403883b24225c6b5271466612f45", size = 601098, upload-time = "2025-12-04T15:07:11.898Z" }, { url = "https://files.pythonhosted.org/packages/dc/a6/e959a127b630a58e23529972dbc868c107f9d583b5a9f878fb858c46bc1a/greenlet-3.3.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb3a8ec3db4a3b0eb8a3c25436c2d49e3505821802074969db017b87bc6a948", size = 590206, upload-time = "2025-12-04T14:26:01.254Z" }, { url = "https://files.pythonhosted.org/packages/48/60/29035719feb91798693023608447283b266b12efc576ed013dd9442364bb/greenlet-3.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2de5a0b09eab81fc6a382791b995b1ccf2b172a9fec934747a7a23d2ff291794", size = 1550668, upload-time = "2025-12-04T15:04:22.439Z" }, { url = "https://files.pythonhosted.org/packages/0a/5f/783a23754b691bfa86bd72c3033aa107490deac9b2ef190837b860996c9f/greenlet-3.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4449a736606bd30f27f8e1ff4678ee193bc47f6ca810d705981cfffd6ce0d8c5", size = 1615483, upload-time = "2025-12-04T14:27:28.083Z" }, @@ -2621,6 +2623,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/0a/a3871375c7b9727edaeeea994bfff7c63ff7804c9829c19309ba2e058807/greenlet-3.3.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:b01548f6e0b9e9784a2c99c5651e5dc89ffcbe870bc5fb2e5ef864e9cc6b5dcb", size = 276379, upload-time = "2025-12-04T14:23:30.498Z" }, { url = "https://files.pythonhosted.org/packages/43/ab/7ebfe34dce8b87be0d11dae91acbf76f7b8246bf9d6b319c741f99fa59c6/greenlet-3.3.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:349345b770dc88f81506c6861d22a6ccd422207829d2c854ae2af8025af303e3", size = 597294, upload-time = "2025-12-04T14:50:06.847Z" }, { url = "https://files.pythonhosted.org/packages/a4/39/f1c8da50024feecd0793dbd5e08f526809b8ab5609224a2da40aad3a7641/greenlet-3.3.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e8e18ed6995e9e2c0b4ed264d2cf89260ab3ac7e13555b8032b25a74c6d18655", size = 607742, upload-time = "2025-12-04T14:57:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/77/cb/43692bcd5f7a0da6ec0ec6d58ee7cddb606d055ce94a62ac9b1aa481e969/greenlet-3.3.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c024b1e5696626890038e34f76140ed1daf858e37496d33f2af57f06189e70d7", size = 622297, upload-time = "2025-12-04T15:07:13.552Z" }, { url = "https://files.pythonhosted.org/packages/75/b0/6bde0b1011a60782108c01de5913c588cf51a839174538d266de15e4bf4d/greenlet-3.3.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:047ab3df20ede6a57c35c14bf5200fcf04039d50f908270d3f9a7a82064f543b", size = 609885, upload-time = "2025-12-04T14:26:02.368Z" }, { url = "https://files.pythonhosted.org/packages/49/0e/49b46ac39f931f59f987b7cd9f34bfec8ef81d2a1e6e00682f55be5de9f4/greenlet-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2d9ad37fc657b1102ec880e637cccf20191581f75c64087a549e66c57e1ceb53", size = 1567424, upload-time = "2025-12-04T15:04:23.757Z" }, { url = "https://files.pythonhosted.org/packages/05/f5/49a9ac2dff7f10091935def9165c90236d8f175afb27cbed38fb1d61ab6b/greenlet-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83cd0e36932e0e7f36a64b732a6f60c2fc2df28c351bae79fbaf4f8092fe7614", size = 1636017, upload-time = "2025-12-04T14:27:29.688Z" }, @@ -4416,11 +4419,11 @@ wheels = [ [[package]] name = "meson" -version = "1.10.1" +version = "1.10.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1b/d3/8c43e758cf456273c32652bb8b7a4ec2d74327d8849856b0b714ad671da7/meson-1.10.1.tar.gz", hash = "sha256:c42296f12db316a4515b9375a5df330f2e751ccdd4f608430d41d7d6210e4317", size = 2413969, upload-time = "2026-01-18T14:45:08.687Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/99/f2e4780865ebabda14ab03b98def0bf27c021efaf7ec7138c1dbbf1c72cb/meson-1.10.2.tar.gz", hash = "sha256:7890287d911dd4ee1ebd0efb61ed0321bfcd87c725df923a837cf90c6508f96b", size = 2422765, upload-time = "2026-03-15T13:39:52.424Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/d5/582789135863eec7c8c1fa31fbde401b3d5d82dbbb4a0973351a1698f738/meson-1.10.1-py3-none-any.whl", hash = "sha256:fe43d1cc2e6de146fbea78f3a062194bcc0e779efc8a0f0d7c35544dfb86731f", size = 1057724, upload-time = "2026-01-18T14:45:02.584Z" }, + { url = "https://files.pythonhosted.org/packages/0b/78/7d049e63e624d51d0065191dae101a1e36d5d3a2360633772d9ad8afb2d5/meson-1.10.2-py3-none-any.whl", hash = "sha256:5f84ef186e6e788d9154db63620fc61b3ece69f643b94b43c8b9203c43d89b36", size = 1060890, upload-time = "2026-03-15T13:39:48.197Z" }, ] [[package]] @@ -4876,6 +4879,7 @@ interleaved-cpu = [ ] math-cpu = [ { name = "beautifulsoup4" }, + { name = "boto3" }, { name = "fasttext" }, { name = "ftfy" }, { name = "justext" }, @@ -4892,6 +4896,7 @@ math-cpu = [ ] math-cuda12 = [ { name = "beautifulsoup4" }, + { name = "boto3" }, { name = "cudf-cu12" }, { name = "cuml-cu12" }, { name = "fasttext" }, @@ -5038,6 +5043,7 @@ requires-dist = [ { name = "av", marker = "extra == 'video-cpu'", specifier = "==13.1.0" }, { name = "beautifulsoup4", marker = "extra == 'text-cpu'" }, { name = "boto3", marker = "extra == 'inference-server'", specifier = ">=1.35" }, + { name = "boto3", marker = "extra == 'math-cpu'", specifier = ">=1.35" }, { name = "comment-parser" }, { name = "cosmos-xenna", specifier = "==0.2.0" }, { name = "cudf-cu12", marker = "extra == 'deduplication-cuda12'", specifier = "==25.10.*" }, @@ -5325,27 +5331,27 @@ wheels = [ [[package]] name = "nixl" -version = "0.10.1" +version = "1.0.0" source = { registry = "https://pypi.nvidia.com/" } dependencies = [ { name = "nixl-cu12", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://pypi.nvidia.com/nixl/nixl-0.10.1-py3-none-any.whl", hash = "sha256:616465673dae5180d296525a03237af4cd5f2c00c3228d185bc06dbe621509b7" }, + { url = "https://pypi.nvidia.com/nixl/nixl-1.0.0-py3-none-any.whl", hash = "sha256:d4f2944e592358e53607276cdb4cc9d0827e11d5d6a8737edf020a134c3dca60" }, ] [[package]] name = "nixl-cu12" -version = "0.10.1" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" }, { name = "torch", version = "2.9.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "platform_machine == 'x86_64' and sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/50/35/3f46dd00810df6aef402a7ee0aabf3257f86978d2df44b09248d810fe985/nixl_cu12-0.10.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0bb1b3532f95c2f376e21008e91e8ec5791304a29af19e75d29fd1bcc754c9bc", size = 51522014, upload-time = "2026-03-03T19:54:59.365Z" }, - { url = "https://files.pythonhosted.org/packages/d6/1c/f5e974ea562fe27194d1591a4187be55f07642df7d7fc6a0f0db334530a3/nixl_cu12-0.10.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:48d3d9cc882edaa0a323d0ddfed39e0864b873ef1fa56e774c5a793629bcf083", size = 51524522, upload-time = "2026-03-03T19:55:02.013Z" }, - { url = "https://files.pythonhosted.org/packages/71/32/3fe6bb57de847ab59fb9e36c5a64249fb5674959773faca0aeefb791fd8e/nixl_cu12-0.10.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:26e59f9841985cf5b547202865036f84ae6dc23184789446fe5833e7499e21a9", size = 51534091, upload-time = "2026-03-03T19:55:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/01/5b/781eebc8e4f28ffa72946f1b0ebf541e5fdc6a916253f2bedfb0fccd01cb/nixl_cu12-1.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:60938a5f85d6e51cb4c02452edab31d549854a1bd661be30999696a44875ff69", size = 51235696, upload-time = "2026-03-13T06:46:26.444Z" }, + { url = "https://files.pythonhosted.org/packages/ba/06/a2b1571d926115aa395f392d1b96148d6f34393ad5dbdd2ea91332c25668/nixl_cu12-1.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a1d2fca137cd0917ec6b0e3c981bef20c5e40ab400333c57546a13c4d51e984e", size = 51238599, upload-time = "2026-03-13T06:46:52.384Z" }, + { url = "https://files.pythonhosted.org/packages/48/68/f58b0b1aa8d2d03dd8354f6893fa858c77267ddef6cdd2d20868cf0ea88b/nixl_cu12-1.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d4347c489c930fc4c2f35693be14dd6c54f7d094639b29564f477396bad664ab", size = 51247001, upload-time = "2026-03-13T06:47:22.813Z" }, ] [[package]] @@ -5414,14 +5420,14 @@ wheels = [ [[package]] name = "numba-cuda" -version = "0.19.2" +version = "0.19.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numba" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/84/4d021a10b101fa4f353bb58746cc555b7747cfe0057b4471854375c86ab6/numba_cuda-0.19.2.tar.gz", hash = "sha256:283e205a18769280d591b1a206b266d24f0f17c2c5ac35337a7bbd7e5a933ab8", size = 607432, upload-time = "2026-01-20T13:55:23.026Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/26/30d3ac0f94ae29accb7bdce74445bd20c9f6ad3395b14e54c0b7f81eac00/numba_cuda-0.19.1.tar.gz", hash = "sha256:181600ca8cbdc5984c3b4198880f71029b116dfde0397bceef666ff830a72c90", size = 606267, upload-time = "2025-08-22T16:58:34.435Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/e9/210e32faa7f9dd42d835d7d36394eaa1d0e29807c56950b389793ee55c1b/numba_cuda-0.19.2-py3-none-any.whl", hash = "sha256:15284c0d6a3fdd842fc5671db8bbb9c34ab3c122f5a3a06b01b104e23f100e5c", size = 744557, upload-time = "2026-01-20T13:55:21.273Z" }, + { url = "https://files.pythonhosted.org/packages/c8/63/7b81d594c7a34a558e6e574d25f10f85dfb7a2b17e6c7219a41b565ae7e1/numba_cuda-0.19.1-py3-none-any.whl", hash = "sha256:2a0a144506993538615f697a888dd04de47eff9f521603cfa92a159161ab3941", size = 743274, upload-time = "2025-08-22T16:58:32.366Z" }, ] [package.optional-dependencies] @@ -5886,64 +5892,51 @@ wheels = [ [[package]] name = "obstore" -version = "0.8.2" +version = "0.9.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/8c/9ec984edd0f3b72226adfaa19b1c61b15823b35b52f311ca4af36d009d15/obstore-0.8.2.tar.gz", hash = "sha256:a467bc4e97169e2ba749981b4fd0936015428d9b8f3fb83a5528536b1b6f377f", size = 168852, upload-time = "2025-09-16T15:34:55.786Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/e9/0a1e340ef262f225ad71f556ccba257896f85ca197f02cd228fe5e20b45a/obstore-0.8.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:49104c0d72688c180af015b02c691fbb6cf6a45b03a9d71b84059ed92dbec704", size = 3622821, upload-time = "2025-09-16T15:32:53.79Z" }, - { url = "https://files.pythonhosted.org/packages/24/86/2b53e8b0a838dbbf89ef5dfddde888770bc1a993c691698dae411a407228/obstore-0.8.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c49776abd416e4d80d003213522d82ad48ed3517bee27a6cf8ce0f0cf4e6337e", size = 3356349, upload-time = "2025-09-16T15:32:55.715Z" }, - { url = "https://files.pythonhosted.org/packages/e8/79/1ba6dc854d7de7704a2c474d723ffeb01b6884f72eea7cbe128efc472f4a/obstore-0.8.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1636372b5e171a98369612d122ea20b955661daafa6519ed8322f4f0cb43ff74", size = 3454842, upload-time = "2025-09-16T15:32:57.072Z" }, - { url = "https://files.pythonhosted.org/packages/ca/03/ca67ccc9b9e63cfc0cd069b84437807fed4ef880be1e445b3f29d11518e0/obstore-0.8.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2efed0d86ad4ebffcbe3d0c4d84f26c2c6b20287484a0a748499c169a8e1f2c4", size = 3688363, upload-time = "2025-09-16T15:32:58.164Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2f/c78eb4352d8be64a072934fe3ff2af79a1d06f4571af7c70d96f9741766b/obstore-0.8.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00c5542616dc5608de82ab6f6820633c9dbab6ff048e770fb8a5fcd1d30cd656", size = 3960133, upload-time = "2025-09-16T15:32:59.614Z" }, - { url = "https://files.pythonhosted.org/packages/4f/34/9e828d19194e227fd9f1d2dd70710da99c2bd2cd728686d59ea80be10b7c/obstore-0.8.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4d9df46aaf25ce80fff48c53382572adc67b6410611660b798024450281a3129", size = 3925493, upload-time = "2025-09-16T15:33:00.923Z" }, - { url = "https://files.pythonhosted.org/packages/5f/7d/9ec5967f3e2915fbc441f72c3892a7f0fb3618e3ae5c8a44181ce4aa641c/obstore-0.8.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ccf0f03a7fe453fb8640611c922bce19f021c6aaeee6ee44d6d8fb57db6be48", size = 3769401, upload-time = "2025-09-16T15:33:02.373Z" }, - { url = "https://files.pythonhosted.org/packages/85/bf/00b65013068bde630a7369610a2dae4579315cd6ce82d30e3d23315cf308/obstore-0.8.2-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:ddfbfadc88c5e9740b687ef0833384329a56cea07b34f44e1c4b00a0e97d94a9", size = 3534383, upload-time = "2025-09-16T15:33:03.903Z" }, - { url = "https://files.pythonhosted.org/packages/52/39/1b684fd96c9a33974fc52f417c52b42c1d50df40b44e588853c4a14d9ab1/obstore-0.8.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:53ad53bb16e64102f39559ec470efd78a5272b5e3b84c53aa0423993ac5575c1", size = 3697939, upload-time = "2025-09-16T15:33:05.355Z" }, - { url = "https://files.pythonhosted.org/packages/85/58/93a2c78935f17fde7e22842598a6373e46a9c32d0243ec3b26b5da92df27/obstore-0.8.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:b0b905b46354db0961ab818cad762b9c1ac154333ae5d341934c90635a6bd7ab", size = 3681746, upload-time = "2025-09-16T15:33:09.344Z" }, - { url = "https://files.pythonhosted.org/packages/38/90/225c2972338d18f92e7a56f71e34df6935b0b1bd7458bb6a0d2bd4d48f92/obstore-0.8.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:fee235694406ebb2dc4178752cf5587f471d6662659b082e9786c716a0a9465c", size = 3765156, upload-time = "2025-09-16T15:33:10.457Z" }, - { url = "https://files.pythonhosted.org/packages/79/eb/aca27e895bfcbbcd2bf05ea6a2538a94b718e6f6d72986e16ab158b753ec/obstore-0.8.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6c36faf7ace17dd0832aa454118a63ea21862e3d34f71b9297d0c788d00f4985", size = 3941190, upload-time = "2025-09-16T15:33:11.59Z" }, - { url = "https://files.pythonhosted.org/packages/33/ce/c8251a397e7507521768f05bc355b132a0daaff3739e861e51fa6abd821e/obstore-0.8.2-cp310-cp310-win_amd64.whl", hash = "sha256:948a1db1d34f88cfc7ab7e0cccdcfd84cf3977365634599c95ba03b4ef80d1c4", size = 3970041, upload-time = "2025-09-16T15:33:13.035Z" }, - { url = "https://files.pythonhosted.org/packages/2f/c4/018f90701f1e5ea3fbd57f61463f42e1ef5218e548d3adcf12b6be021c34/obstore-0.8.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2edaa97687c191c5324bb939d72f6fe86a7aa8191c410f1648c14e8296d05c1c", size = 3622568, upload-time = "2025-09-16T15:33:14.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/62/72dd1e7d52fc554bb1fdb1a9499bda219cf3facea5865a1d97fdc00b3a1b/obstore-0.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c4fb7ef8108f08d14edc8bec9e9a6a2e5c4d14eddb8819f5d0da498aff6e8888", size = 3356109, upload-time = "2025-09-16T15:33:15.315Z" }, - { url = "https://files.pythonhosted.org/packages/e0/ae/089fe5b9207091252fe5ce352551214f04560f85eb8f2cc4f716a6a1a57e/obstore-0.8.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fda8f658c0edf799ab1e264f9b12c7c184cd09a5272dc645d42e987810ff2772", size = 3454588, upload-time = "2025-09-16T15:33:16.421Z" }, - { url = "https://files.pythonhosted.org/packages/ea/10/1865ae2d1ba45e8ae85fb0c1aada2dc9533baf60c4dfe74dab905348d74a/obstore-0.8.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87fe2bc15ce4051ecb56abd484feca323c2416628beb62c1c7b6712114564d6e", size = 3688627, upload-time = "2025-09-16T15:33:17.604Z" }, - { url = "https://files.pythonhosted.org/packages/a6/09/5d7ba6d0aeac563ea5f5586401c677bace4f782af83522b1fdf15430e152/obstore-0.8.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2482aa2562ab6a4ca40250b26bea33f8375b59898a9b5615fd412cab81098123", size = 3959896, upload-time = "2025-09-16T15:33:18.789Z" }, - { url = "https://files.pythonhosted.org/packages/16/15/2b3eda59914761a9ff4d840e2daec5697fd29b293bd18d3dc11c593aed06/obstore-0.8.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4153b928f5d2e9c6cb645e83668a53e0b42253d1e8bcb4e16571fc0a1434599a", size = 3933162, upload-time = "2025-09-16T15:33:19.935Z" }, - { url = "https://files.pythonhosted.org/packages/14/7a/5fc63b41526587067537fb1498c59a210884664c65ccf0d1f8f823b0875a/obstore-0.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbfa9c38620cc191be98c8b5558c62071e495dc6b1cc724f38293ee439aa9f92", size = 3769605, upload-time = "2025-09-16T15:33:21.389Z" }, - { url = "https://files.pythonhosted.org/packages/77/4e/2208ab6e1fc021bf8b7e117249a10ab75d0ed24e0f2de1a8d7cd67d885b5/obstore-0.8.2-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:0822836eae8d52499f10daef17f26855b4c123119c6eb984aa4f2d525ec2678d", size = 3534396, upload-time = "2025-09-16T15:33:22.574Z" }, - { url = "https://files.pythonhosted.org/packages/1d/8f/a0e2882edd6bd285c82b8a5851c4ecf386c93fe75b6e340d5d9d30e809fc/obstore-0.8.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ef6435dfd586d83b4f778e7927a5d5b0d8b771e9ba914bc809a13d7805410e6", size = 3697777, upload-time = "2025-09-16T15:33:23.723Z" }, - { url = "https://files.pythonhosted.org/packages/94/78/ebf0c33bed5c9a8eed3b00eefafbcc0a687eeb1e05451c76fcf199d29ff8/obstore-0.8.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:0f2cba91f4271ca95a932a51aa8dda1537160342b33f7836c75e1eb9d40621a2", size = 3681546, upload-time = "2025-09-16T15:33:24.935Z" }, - { url = "https://files.pythonhosted.org/packages/af/21/9bf4fb9e53fd5f01af580b6538de2eae857e31d24b0ebfc4d916c306a1e4/obstore-0.8.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:23c876d603af0627627808d19a58d43eb5d8bfd02eecd29460bc9a58030fed55", size = 3765336, upload-time = "2025-09-16T15:33:26.069Z" }, - { url = "https://files.pythonhosted.org/packages/dd/3c/7f6895c23719482d231b2d6ed328e3223fdf99785f6850fba8d2fc5a86ee/obstore-0.8.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ff3c4b5d07629b70b9dee494cd6b94fff8465c3864752181a1cb81a77190fe42", size = 3941142, upload-time = "2025-09-16T15:33:27.275Z" }, - { url = "https://files.pythonhosted.org/packages/93/a4/56ccdb756161595680a28f4b0def2c04f7048ffacf128029be8394367b26/obstore-0.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:aadb2cb72de7227d07f4570f82729625ffc77522fadca5cf13c3a37fbe8c8de9", size = 3970172, upload-time = "2025-09-16T15:33:28.393Z" }, - { url = "https://files.pythonhosted.org/packages/2b/dc/60fefbb5736e69eab56657bca04ca64dc07fdeccb3814164a31b62ad066b/obstore-0.8.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bb70ce297a47392b1d9a3e310f18d59cd5ebbb9453428210fef02ed60e4d75d1", size = 3612955, upload-time = "2025-09-16T15:33:29.527Z" }, - { url = "https://files.pythonhosted.org/packages/d2/8b/844e8f382e5a12b8a3796a05d76a03e12c7aedc13d6900419e39207d7868/obstore-0.8.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1619bf618428abf1f607e0b219b2e230a966dcf697b717deccfa0983dd91f646", size = 3346564, upload-time = "2025-09-16T15:33:30.698Z" }, - { url = "https://files.pythonhosted.org/packages/89/73/8537f99e09a38a54a6a15ede907aa25d4da089f767a808f0b2edd9c03cec/obstore-0.8.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a4605c3ed7c9515aeb4c619b5f7f2c9986ed4a79fe6045e536b5e59b804b1476", size = 3460809, upload-time = "2025-09-16T15:33:31.837Z" }, - { url = "https://files.pythonhosted.org/packages/b4/99/7714dec721e43f521d6325a82303a002cddad089437640f92542b84e9cc8/obstore-0.8.2-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce42670417876dd8668cbb8659e860e9725e5f26bbc86449fd259970e2dd9d18", size = 3692081, upload-time = "2025-09-16T15:33:33.028Z" }, - { url = "https://files.pythonhosted.org/packages/ec/bd/4ac4175fe95a24c220a96021c25c432bcc0c0212f618be0737184eebbaad/obstore-0.8.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4a3e893b2a06585f651c541c1972fe1e3bf999ae2a5fda052ee55eb7e6516f5", size = 3957466, upload-time = "2025-09-16T15:33:34.528Z" }, - { url = "https://files.pythonhosted.org/packages/4e/04/caa288fb735484fc5cb019bdf3d896eaccfae0ac4622e520d05692c46790/obstore-0.8.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08462b32f95a9948ed56ed63e88406e2e5a4cae1fde198f9682e0fb8487100ed", size = 3951293, upload-time = "2025-09-16T15:33:35.733Z" }, - { url = "https://files.pythonhosted.org/packages/44/2f/d380239da2d6a1fda82e17df5dae600a404e8a93a065784518ff8325d5f6/obstore-0.8.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a0bf7763292a8fc47d01cd66e6f19002c5c6ad4b3ed4e6b2729f5e190fa8a0d", size = 3766199, upload-time = "2025-09-16T15:33:36.904Z" }, - { url = "https://files.pythonhosted.org/packages/28/41/d391be069d3da82969b54266948b2582aeca5dd735abeda4d63dba36e07b/obstore-0.8.2-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:bcd47f8126cb192cbe86942b8f73b1c45a651ce7e14c9a82c5641dfbf8be7603", size = 3529678, upload-time = "2025-09-16T15:33:38.221Z" }, - { url = "https://files.pythonhosted.org/packages/b9/4c/4862fdd1a3abde459ee8eea699b1797df638a460af235b18ca82c8fffb72/obstore-0.8.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:57eda9fd8c757c3b4fe36cf3918d7e589cc1286591295cc10b34122fa36dd3fd", size = 3698079, upload-time = "2025-09-16T15:33:39.696Z" }, - { url = "https://files.pythonhosted.org/packages/68/ca/014e747bc53b570059c27e3565b2316fbe5c107d4134551f4cd3e24aa667/obstore-0.8.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ea44442aad8992166baa69f5069750979e4c5d9ffce772e61565945eea5774b9", size = 3687154, upload-time = "2025-09-16T15:33:40.92Z" }, - { url = "https://files.pythonhosted.org/packages/6f/89/6db5f8edd93028e5b8bfbeee15e6bd3e56f72106107d31cb208b57659de4/obstore-0.8.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:41496a3ab8527402db4142aaaf0d42df9d7d354b13ba10d9c33e0e48dd49dd96", size = 3773444, upload-time = "2025-09-16T15:33:42.123Z" }, - { url = "https://files.pythonhosted.org/packages/26/e5/c9e2cc540689c873beb61246e1615d6e38301e6a34dec424f5a5c63c1afd/obstore-0.8.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43da209803f052df96c7c3cbec512d310982efd2407e4a435632841a51143170", size = 3939315, upload-time = "2025-09-16T15:33:43.252Z" }, - { url = "https://files.pythonhosted.org/packages/4d/c9/bb53280ca50103c1ffda373cdc9b0f835431060039c2897cbc87ddd92e42/obstore-0.8.2-cp312-cp312-win_amd64.whl", hash = "sha256:1836f5dcd49f9f2950c75889ab5c51fb290d3ea93cdc39a514541e0be3af016e", size = 3978234, upload-time = "2025-09-16T15:33:44.393Z" }, - { url = "https://files.pythonhosted.org/packages/c3/37/14bae1f5bf4369027abc5315cdba2428ad4c16e2fd3bd5d35b7ee584aa0c/obstore-0.8.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:6ea04118980a9c22fc8581225ff4507b6a161baf8949d728d96e68326ebaab59", size = 3624857, upload-time = "2025-09-16T15:34:35.601Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c4/8cba91629aa20479ba86a57c2c2b3bc0a54fc6a31a4594014213603efae6/obstore-0.8.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5f33a7570b6001b54252260fbec18c3f6d21e25d3ec57e9b6c5e7330e8290eb2", size = 3355999, upload-time = "2025-09-16T15:34:36.954Z" }, - { url = "https://files.pythonhosted.org/packages/f2/10/3e40557d6d9c38c5a0f7bac1508209b9dbb8c4da918ddfa9326ba9a1de3f/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11fa78dfb749edcf5a041cd6db20eae95b3e8b09dfdd9b38d14939da40e7c115", size = 3457322, upload-time = "2025-09-16T15:34:38.143Z" }, - { url = "https://files.pythonhosted.org/packages/1d/01/dcf7988350c286683698cbdd8c15498aec43cbca72eaabad06fd77f0f34a/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:872bc0921ff88305884546ba05e258ccd95672a03d77db123f0d0563fd3c000b", size = 3689452, upload-time = "2025-09-16T15:34:39.638Z" }, - { url = "https://files.pythonhosted.org/packages/97/02/643eb2ede58933e47bdbc92786058c83d9aa569826d5bf6e83362d24a27a/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:72556a2fbf018edd921286283e5c7eec9f69a21c6d12516d8a44108eceaa526a", size = 3961171, upload-time = "2025-09-16T15:34:41.232Z" }, - { url = "https://files.pythonhosted.org/packages/d8/5d/c0b515df6089d0f54109de8031a6f6ed31271361948bee90ab8271d22f79/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75fa1abf21499dfcfb0328941a175f89a9aa58245bf00e3318fe928e4b10d297", size = 3935988, upload-time = "2025-09-16T15:34:42.501Z" }, - { url = "https://files.pythonhosted.org/packages/7b/97/114d7bc172bb846472181d6fa3e950172ee1b1ccd11291777303c499dbdd/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f54f72f30cd608c4399679781c884bf8a0e816c1977a2fac993bf5e1fb30609f", size = 3771781, upload-time = "2025-09-16T15:34:44.405Z" }, - { url = "https://files.pythonhosted.org/packages/c3/43/4aa6de6dc406ef5e109b21a5614c34999575de638254deb456703fae24aa/obstore-0.8.2-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:b044ebf1bf7b8f7b0ca309375c1cd9e140be79e072ae8c70bbd5d9b2ad1f7678", size = 3536689, upload-time = "2025-09-16T15:34:45.649Z" }, - { url = "https://files.pythonhosted.org/packages/06/a5/870ce541aa1a9ee1d9c3e99c2187049bf5a4d278ee9678cc449aae0a4e68/obstore-0.8.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:b1326cd2288b64d6fe8857cc22d3a8003b802585fc0741eff2640a8dc35e8449", size = 3700560, upload-time = "2025-09-16T15:34:47.252Z" }, - { url = "https://files.pythonhosted.org/packages/7d/93/76a5fc3833aaa833b4152950d9cdfd328493a48316c24e32ddefe9b8870f/obstore-0.8.2-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:ba6863230648a9b0e11502d2745d881cf74262720238bc0093c3eabd22a3b24c", size = 3683450, upload-time = "2025-09-16T15:34:49.589Z" }, - { url = "https://files.pythonhosted.org/packages/15/3c/4c389362c187630c42f61ef9214e67fc336e44b8aafc47cf49ba9ab8007d/obstore-0.8.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:887615da9eeefeb2df849d87c380e04877487aa29dbeb367efc3f17f667470d3", size = 3766628, upload-time = "2025-09-16T15:34:51.937Z" }, - { url = "https://files.pythonhosted.org/packages/03/12/08547e63edf2239ec6660af434602208ab6f394955ef660a6edda13a0bee/obstore-0.8.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:4eec1fb32ffa4fb9fe9ad584611ff031927a5c22732b56075ee7204f0e35ebdf", size = 3944069, upload-time = "2025-09-16T15:34:54.108Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/24/18/cab734edaeb495a861cfbdced9fecdc0866ed1a85aa5a9202ec77cf4723e/obstore-0.9.2.tar.gz", hash = "sha256:7ef94323127a971c9dea2484109d6c706eb2b2594a2df13c2dd0a6d21a9a69ae", size = 123731, upload-time = "2026-03-11T19:10:18.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/8d/567efc2f49b6f9212f36c35759cfe95deca22f2b0cf5a9fa98dca14975e6/obstore-0.9.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f710b981071fce35ba57b0a47e4b13c9be0f22ad7e345707f0e798ff4814851f", size = 4101752, upload-time = "2026-03-11T19:08:47.094Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/dc5a541aecf450c18b12f1ea2f0b6b0e9c0d28045a05e4ed4a708381b1af/obstore-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:242adf90709097d664bbf9f4eb207a7f6eda06b58fa1e9c2adc7774eead6fe59", size = 3882599, upload-time = "2026-03-11T19:08:49.109Z" }, + { url = "https://files.pythonhosted.org/packages/60/d0/7ee2bb0f25138beae9e37f5829fe6976995a3e51bf55551c338dfa8c8bd1/obstore-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44b05f531ddbf5b23c8673755070cf41bf07b38e25c0a48534a38b98d5ac7c43", size = 4043233, upload-time = "2026-03-11T19:08:50.493Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0f/c09cb1c1da5799de86c24cf4aa1b55afc79f924fdc550c39731805507c02/obstore-0.9.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8a7b26e5085647d2225323e86ee011414ce59ecfbf7f6def28cee371cf1d1e3", size = 4142247, upload-time = "2026-03-11T19:08:52.024Z" }, + { url = "https://files.pythonhosted.org/packages/96/0f/3fed38d308bb6c4e8976f37e78d087925b4eed2b391bee064d557d6f957b/obstore-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f4eea11208b4cbd3974ac8fc3c63a9d516d7e1e18eb94f6c396923b8cd024134", size = 4422649, upload-time = "2026-03-11T19:08:53.253Z" }, + { url = "https://files.pythonhosted.org/packages/d7/79/94d303fc52009417551728e835326af0f2b9e4edd63181e607db9a868130/obstore-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:badcc28413fd54bdff7101a33dc9bee73d70e006b77f1464c562986680ddaf4b", size = 4347110, upload-time = "2026-03-11T19:08:54.742Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b6/d2ffaf0ef4556f18e7a75c5aec797f34093c00dc3aedab05cb041eb27535/obstore-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a26065607c51b15b452f3344d85ea88857253724c4f3a520612f92111b2cd63e", size = 4227249, upload-time = "2026-03-11T19:08:56.33Z" }, + { url = "https://files.pythonhosted.org/packages/2f/b1/6b5db701e64569ef5906fb2121fafe71bf4cf48f3304aad7c1037bdae05b/obstore-0.9.2-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:9409e3b1520d27c53bb14f16b693fafe3f57194a967ff2b1b8dee3b7cf3dada1", size = 4109922, upload-time = "2026-03-11T19:08:57.591Z" }, + { url = "https://files.pythonhosted.org/packages/9b/08/ee270efd365b6c95fb29f7c45652a186569c8123ad90684dd7605a69a3f2/obstore-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0a337e59d8dab3be761a3f2d3648155bcd618b32cbfe2875d65a93c33b846cc", size = 4297577, upload-time = "2026-03-11T19:08:58.849Z" }, + { url = "https://files.pythonhosted.org/packages/1c/8c/5cceba7ad82bedafa93c9a14d7f26336dff8c9705c6f0382e395cb478d88/obstore-0.9.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1d58bd7529432c5d9dd96a660d9abe21d74394a1c2e51e86843a2e6059d1ad4f", size = 4275910, upload-time = "2026-03-11T19:09:00.151Z" }, + { url = "https://files.pythonhosted.org/packages/c2/40/2ac9cb4d7fa73b0680c0f36ef3da5b8952fafab038b7ab06a408ebc10af3/obstore-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:878a7672059e07d55b4cdb32b36bbd29803a30f98b3ddadea0cc43c25676e052", size = 4263175, upload-time = "2026-03-11T19:09:02.35Z" }, + { url = "https://files.pythonhosted.org/packages/3e/59/6a6edcfed719c17feb4603d1b16271ffed13dcf3b78538c6a437a08da231/obstore-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:38db958d9b9b5ede9a1d834254edda2182e3dce64ec5cbf668790457ff365e82", size = 4446061, upload-time = "2026-03-11T19:09:03.849Z" }, + { url = "https://files.pythonhosted.org/packages/26/90/495f0d257ddae8094e0aae58a59457cdf7933d5dcda6d3f466a1a9f98bb3/obstore-0.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:fb3f3843f1cbf3aab4de968c13c2e97c9d1d771b73575d2f56fd7385d3f79357", size = 4185108, upload-time = "2026-03-11T19:09:05.319Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d2/b98058a552849719df56d59a53f7d97e6507b37fca0399a866534800f9fa/obstore-0.9.2-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:50d9c9d6de601ad4805a5a76a1a3d731f7b899383f96ef57276f97bc35202f95", size = 4105494, upload-time = "2026-03-11T19:09:06.573Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/4386622b94fd028cb2298b4780d5a8e2d959fc4c71e599fb63be869aa83d/obstore-0.9.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:4c6dcd9b76b802a2278e1cd88ad7305caf3c3c16f800b2bf5f86a606e9e83d96", size = 3878429, upload-time = "2026-03-11T19:09:07.962Z" }, + { url = "https://files.pythonhosted.org/packages/91/8d/0bfad11f1ee5fb1fbdb7833607212ad2586dbd1824b30cf328af63fe92fc/obstore-0.9.2-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d46e629beb47565fa67b6ef05919434258d72ef848efa340f911af5de2536da", size = 4041157, upload-time = "2026-03-11T19:09:09.278Z" }, + { url = "https://files.pythonhosted.org/packages/eb/98/bfde825f61a8b2541be9185cd6a4ddbb820de94c79750edc32f9f9dfb795/obstore-0.9.2-cp311-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:350d8cc1cd9564369291396e160ebfa133d705ec349d8c0d444a39158d6ef3e7", size = 4144757, upload-time = "2026-03-11T19:09:10.938Z" }, + { url = "https://files.pythonhosted.org/packages/19/35/1c101f6660ef91e5280c824677d8b5ab11ee25ed52e59b075cd795a86e69/obstore-0.9.2-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dddd38c9f98fd8eaf11a9805464f0bec7e57d8e04a5e0b0cb17582ec58d2fe41", size = 4427897, upload-time = "2026-03-11T19:09:12.137Z" }, + { url = "https://files.pythonhosted.org/packages/fb/eb/a9bdb64474d4e0ab4e4c0105c959090d6bd7ce38d4a945cae3679ead8c52/obstore-0.9.2-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca872e88e5c719faf1581632e348a6b01331b4f838d7ac29aff226107088dc35", size = 4336227, upload-time = "2026-03-11T19:09:13.822Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ec/e6d39aa311afec2241adb6f2067d7d6ca2eb4e0aab5a95c47796edadd524/obstore-0.9.2-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ee61ac2af5c32c5282fc13b9eba7ffa332f268cb65bc29134ad8ac45e069871", size = 4229010, upload-time = "2026-03-11T19:09:15.503Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fb/a24fd972b66b2d83829e2e89ccf236a759a82f881f909bf4fbe0b6c398ae/obstore-0.9.2-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:2f430cf8af76985e7ebb8d5f20c8ccef858c608103af6ea95c870f5380cd62f7", size = 4103835, upload-time = "2026-03-11T19:09:16.729Z" }, + { url = "https://files.pythonhosted.org/packages/d0/d4/c8cc60c8afc597712bf6c5059d629e050de521d901dad0f554b268c2d77f/obstore-0.9.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1df403f80feef7ac483ed66a2a5a964a469f3756ded533935640c4baf986dd49", size = 4292174, upload-time = "2026-03-11T19:09:18.461Z" }, + { url = "https://files.pythonhosted.org/packages/a7/80/dcf8f31814f25c390aa5501a95b78b9f6456d30cd4625109c2a6a5105ad1/obstore-0.9.2-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:c20f62b7c2f57c6f449215c36af4a8d502082ced2185c0b28f07a5e7c9698181", size = 4276266, upload-time = "2026-03-11T19:09:19.787Z" }, + { url = "https://files.pythonhosted.org/packages/16/71/5f5369fba652c5f83b44381d9e7a3cfe00793301d01802059b52b8663f2c/obstore-0.9.2-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:c296e7d60ee132babb7fd01eab946396fa28eb0d88264b9e60320922174e6010", size = 4264118, upload-time = "2026-03-11T19:09:21.081Z" }, + { url = "https://files.pythonhosted.org/packages/c5/50/a5bd1948f2b2efb1039852542829a33a198be0586da7d4247996d3f15d26/obstore-0.9.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:76f274a170731a4461d0fe3eefde38f3bdaf346011ae020c94a0bd18bfd3c4bc", size = 4446876, upload-time = "2026-03-11T19:09:22.401Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d6/bcc266e391403163ed12dd8cab53012f4db8f5020fb49e3b0a505d7a1bba/obstore-0.9.2-cp311-abi3-win_amd64.whl", hash = "sha256:f644fef2a91973b6c055623692524baf830abb1f8bb3ad348611f0e25224e160", size = 4190639, upload-time = "2026-03-11T19:09:23.637Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b0/958df4459af032a589a96657eb896949c7077f1c6b97fbd12dcbb2b31163/obstore-0.9.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:034071b5d54fc5e945cd9a15e0a7c3128cd3bc1b97bfffe153e09d940bfbf390", size = 4101577, upload-time = "2026-03-11T19:10:01.51Z" }, + { url = "https://files.pythonhosted.org/packages/d7/78/cb10d54e988506c26bcf0a3d1e9d4c896fa42f40640cffde91f6c28c0e12/obstore-0.9.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:fc62c77dd29f2642b9ac0338de16b105701445c0baf04f9a665250aa490ecfaf", size = 3881592, upload-time = "2026-03-11T19:10:02.803Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d8/f87684bf91af83da63963170d4ec40afa72615f6af66bb9820a1d00c1ac4/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fd12afd030a3867879e62a20aa294e89bf8d744e958cf889b462004edbdeca6", size = 4038571, upload-time = "2026-03-11T19:10:04Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ad/d71cad9086935b7f9e9f7c8f6a6e36e8946f888b83c46f1bd4c45567f00e/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:490606e6a0d52c846b36c85a4cd772fe14776821e50a4bff1680e8643c4a9d73", size = 4141769, upload-time = "2026-03-11T19:10:05.422Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/c789b07c56b4e88fd904a7e684dc78b0b179a7fc84afd5a9039efe37cc19/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ca53037bcc6a061cc02652ff0652c4313b75e9f66a0a147701459c65192a72d", size = 4422333, upload-time = "2026-03-11T19:10:06.795Z" }, + { url = "https://files.pythonhosted.org/packages/5d/80/956254c0dfffe2b795849987b14afbc8ebfe79e6bf4993e3dd5969c1f9d8/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:56138a4546fb5b46cb576b2b5951582fbc613cba97f51aab4a03fc305f66758b", size = 4345863, upload-time = "2026-03-11T19:10:08.156Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6a/d07a8f5b63d6bb259eb349958882638268624af5da5eda4731a510aa3dbe/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08280bb90ccd19350cca352348eef4c44ea34be34893819cb35f29ac578c01f9", size = 4226798, upload-time = "2026-03-11T19:10:09.792Z" }, + { url = "https://files.pythonhosted.org/packages/71/a4/8a6db6db5a2218de47a7cdecdf1d37185280a4ed364e1c27bdce813372da/obstore-0.9.2-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:a98c5da06ee34c285b0bdc32be03ab453af1fbf51c9a02fce36b195f2e404c03", size = 4108402, upload-time = "2026-03-11T19:10:11.125Z" }, + { url = "https://files.pythonhosted.org/packages/55/a9/3af105f9e8769840b9d7fa16f13ee3413e4ae40d5ef4b233a01c20faabb8/obstore-0.9.2-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:845b77a65add7449efc74994f02c62df87390a28d4a07a9296a065c88206d4fd", size = 4296537, upload-time = "2026-03-11T19:10:12.519Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d7/ba263d6bdd7f7fe61143233fead0b0fdf4907092eb2cf8f1d1db1e8e4bb6/obstore-0.9.2-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:50bc1f25ad3881bcd74142f143ab41a26c1b7a7c0b14a72ecbf6fddc34f5a36e", size = 4273106, upload-time = "2026-03-11T19:10:14.184Z" }, + { url = "https://files.pythonhosted.org/packages/05/22/179117970abd49d8fe53e014c9326363b8ce20f4635a4389e7d1215529bb/obstore-0.9.2-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:cada36b244361a25df6a943f374110034cb75c4bd284375df53bc1826c4b3bb6", size = 4262228, upload-time = "2026-03-11T19:10:15.463Z" }, + { url = "https://files.pythonhosted.org/packages/24/80/2de1995c1c195f5ce7d54184a01868741445333c751153f9d979b77de9e5/obstore-0.9.2-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:66ad31b4630c2cd41b1369529b6e9a472c84a7a8df045dc62b0bfea6d922110c", size = 4445675, upload-time = "2026-03-11T19:10:16.805Z" }, ] [[package]] @@ -6726,11 +6719,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/b6/6e630dff89739fcd427e3f72b3d905ce0acb85a45d4ec3e2678718a3487f/pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b", size = 146586, upload-time = "2026-01-16T18:04:18.534Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/44/b5/a96872e5184f354da9c84ae119971a0a4c221fe9b27a4d94bd43f2596727/pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf", size = 83371, upload-time = "2026-01-16T18:04:17.174Z" }, ] [[package]] @@ -7029,7 +7022,7 @@ wheels = [ [[package]] name = "pydrive2" -version = "1.21.1" +version = "1.21.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-python-client" }, @@ -7037,9 +7030,9 @@ dependencies = [ { name = "pyopenssl" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a7/2c/34370cc28e031214afca20de117745a6c0f66f132805d42b11ac02b346f1/pydrive2-1.21.1.tar.gz", hash = "sha256:70da0244a29a6922e28620a32e251ac6ab018449f1bb0485e9a39114a069dde0", size = 63287, upload-time = "2024-11-02T18:55:57.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/e2/67534dd1e644f170c750546b899963f5fb5f40d9719273eb1ec42a29d44d/pydrive2-1.21.2.tar.gz", hash = "sha256:2a21c8319a225943c70e7566eb13a1524d1d7193621de1eb8e5f95e037641508", size = 63280, upload-time = "2024-11-28T18:57:10.28Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/52/be/f49b3941347b21bb2b1d0847d75e0605247182d88b3819b06e0973d92a88/PyDrive2-1.21.1-py3-none-any.whl", hash = "sha256:d24b3334bc5c242e5ec58ad6ee7efbd2216aa92098c3eed353ce644f27a7e97b", size = 47946, upload-time = "2024-11-02T18:55:54.213Z" }, + { url = "https://files.pythonhosted.org/packages/a9/32/8f16d52289785f0ee95263922814be0cacc5c20ff3dc58a38d809d04ac30/PyDrive2-1.21.2-py3-none-any.whl", hash = "sha256:a3b72e522b8f5ba4e93ab165bbf120544567583c61a6c7904ef1ff47afc005d6", size = 47959, upload-time = "2024-11-28T18:57:08.705Z" }, ] [[package]] @@ -7164,15 +7157,14 @@ wheels = [ [[package]] name = "pyopenssl" -version = "26.0.0" +version = "22.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, - { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8e/11/a62e1d33b373da2b2c2cd9eb508147871c80f12b1cacde3c5d314922afdd/pyopenssl-26.0.0.tar.gz", hash = "sha256:f293934e52936f2e3413b89c6ce36df66a0b34ae1ea3a053b8c5020ff2f513fc", size = 185534, upload-time = "2026-03-15T14:28:26.353Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/d3/d6a9610f19d943e198df502ae660c6b5acf84cc3bc421a2aa3c0fb6b21d1/pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf", size = 178438, upload-time = "2022-01-29T20:13:05.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/7d/d4f7d908fa8415571771b30669251d57c3cf313b36a856e6d7548ae01619/pyopenssl-26.0.0-py3-none-any.whl", hash = "sha256:df94d28498848b98cc1c0ffb8ef1e71e40210d3b0a8064c9d29571ed2904bf81", size = 57969, upload-time = "2026-03-15T14:28:24.864Z" }, + { url = "https://files.pythonhosted.org/packages/d5/9f/9c0e3288b85f907a008f9d31318b0e4de31b2f67724a8745e633741f609c/pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0", size = 55833, upload-time = "2022-01-29T20:13:02.874Z" }, ] [[package]] @@ -9882,7 +9874,7 @@ wheels = [ [[package]] name = "xgrammar" -version = "0.1.32" +version = "0.1.33" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, @@ -9893,23 +9885,23 @@ dependencies = [ { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/99/6a/d51b44fc0b43e2d4adae42b6a17fe9ee49e177d6d768be739ed7dec7b57e/xgrammar-0.1.32.tar.gz", hash = "sha256:5d424d52779ca2d3ccaf72f2289d6519efe308e933d0d3fc3c292c780825bb12", size = 2365047, upload-time = "2026-03-04T12:01:52.544Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8e/e9/df431398353e5a436c6477f2843f2a775dc22d0680e5702f1540590b1737/xgrammar-0.1.32-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:1d179511a1258b410d1660ed00032da24256072b44d6743f5bd3f49743d90801", size = 18425864, upload-time = "2026-03-04T12:00:08.727Z" }, - { url = "https://files.pythonhosted.org/packages/16/9a/7ba4bc7aeff03ddd9b1b9cfbb94f1aa88de02a063f50ba24eaedfa88e40d/xgrammar-0.1.32-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b33d2e8f02ca31b93f3cbfea0ed1df7e706a2bea52d7706793884f92a8c1523a", size = 20582805, upload-time = "2026-03-04T12:00:11.575Z" }, - { url = "https://files.pythonhosted.org/packages/4b/3c/e93e601b3b0e1f22301a6fea7b6c58d748fd6716b28ec2a40e9cbdfecda1/xgrammar-0.1.32-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7d4f0f2041cdc97bacc096b4c2d1c841e91794b6e7a54e7a4853fc0907956dc", size = 37681723, upload-time = "2026-03-04T12:00:14.891Z" }, - { url = "https://files.pythonhosted.org/packages/2f/5b/1ace639497c50dc3d4ba4426798acc06562a7b646a62e13212d5f2a840e2/xgrammar-0.1.32-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d4f3d18da40cad18e87881925f82d0aed1418cc2fb05886d516e3c9d548db57f", size = 37708550, upload-time = "2026-03-04T12:00:18.408Z" }, - { url = "https://files.pythonhosted.org/packages/ac/7b/3880da6295e9a6695ca34f750eea25505a6cf36aee74ef846267e9b48995/xgrammar-0.1.32-cp310-cp310-win_amd64.whl", hash = "sha256:e7baf71bba03a5e734df435b6378da4406b1c15f5511ab4f5d4af9f72775c756", size = 6632934, upload-time = "2026-03-04T12:00:21.461Z" }, - { url = "https://files.pythonhosted.org/packages/28/cd/4b5e67c8030b626a1a00b65b4d149b1b031c885eef86d4e5fa296f6ec72e/xgrammar-0.1.32-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:51b41c47785aa198d19f8d056b394f75b4421deab88c415568f9c588b1f7e238", size = 18425822, upload-time = "2026-03-04T12:00:23.356Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c0/94fbc45642e733a9ad4a9f3f7300a1a06b265f8657af4d6a56acd8cf00c4/xgrammar-0.1.32-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d7030192cb1d8579699f1f72fd14d31347a402611aab98a2da6a04c3de07e917", size = 20582669, upload-time = "2026-03-04T12:00:26.463Z" }, - { url = "https://files.pythonhosted.org/packages/90/ea/2f4c8616d8ed0b5a3eb4e417b4987ad5a8d9dd9336ed966a8d48ffd45907/xgrammar-0.1.32-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a332c0364f665b410a6cfc2ada155c3a6ede430e385ac431015e31735a64fec3", size = 37682948, upload-time = "2026-03-04T12:00:29.814Z" }, - { url = "https://files.pythonhosted.org/packages/ac/ae/b9108fadd354ae776c1e7ecd26890a13ac8a30367f9fe8110443aedc4e6a/xgrammar-0.1.32-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b8ad132d0fcf3a51dc054ecb0dc9808566b302122de6edaac7b4aca460adbec", size = 37709617, upload-time = "2026-03-04T12:00:33.068Z" }, - { url = "https://files.pythonhosted.org/packages/9d/48/0096bd1f3b460eac48faaecf79418ea3172269dccf37968e78dff5114faf/xgrammar-0.1.32-cp311-cp311-win_amd64.whl", hash = "sha256:b8b1ca6d3f3c2842660458660e494aaf0a6745f1b07ae74e4c2230ab4ff70c11", size = 6632722, upload-time = "2026-03-04T12:00:36.133Z" }, - { url = "https://files.pythonhosted.org/packages/9f/fd/5e771276fa090e35eaf1cbfdede24b9d93d6bbd2e99cd4f8d558f381fdee/xgrammar-0.1.32-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:9b78d32265f096e5567ab52c72b681855cf473481a48a1e7e6d97d414ba30b82", size = 18425090, upload-time = "2026-03-04T12:00:38.5Z" }, - { url = "https://files.pythonhosted.org/packages/31/66/f06745755ef0750f43955cf679b4bd8bd88ac8bfab760f020225c192884f/xgrammar-0.1.32-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:23eacaf826c3aeebca0d91fc271417d9d96e157af2bacf6f14277297af7917ef", size = 20582048, upload-time = "2026-03-04T12:00:42.369Z" }, - { url = "https://files.pythonhosted.org/packages/79/29/3b0306800ccabce8f565123a5b97432dee43822c30142085d9b13b43f166/xgrammar-0.1.32-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f9a637d4e0c541149e0d409c24f4ec79cd74d87508ee6a17a7e64a9b9c0cf56f", size = 37680849, upload-time = "2026-03-04T12:00:46.712Z" }, - { url = "https://files.pythonhosted.org/packages/69/62/65e664d861cdadf2d788c03dd8fe67f1faaa7bd4bd2317a2ab850aebee20/xgrammar-0.1.32-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f96c7a4fcbd68e18b13cb3b6ed5d24b5326b256933f476bdaf2cc8e609c228db", size = 37711100, upload-time = "2026-03-04T12:00:50.188Z" }, - { url = "https://files.pythonhosted.org/packages/80/43/05f27a1739209eb590772f867f3f48e6db0a36f376d85db4e68f49aee799/xgrammar-0.1.32-cp312-cp312-win_amd64.whl", hash = "sha256:ba6e08c385cce53eda8e9b3bbfba63f100ba3dcb76fa0692a65921a36b20ad0a", size = 6632259, upload-time = "2026-03-04T12:00:53.184Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/db/43/e5dfddb1d2a4fccf3e3a88f103e88698cdefc3182f4e169a359ffe1c1794/xgrammar-0.1.33.tar.gz", hash = "sha256:8dbe5fc3d76651ab1fac7a68fc2a118b885fa0ec7189927fb6e0dce0081aea99", size = 2398956, upload-time = "2026-03-27T10:16:36.582Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ad/ac47eaa00e10877a74171b2290e599bb7b3002ebe89dc58f8e400619c3be/xgrammar-0.1.33-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6cdb24f917b104cdd3f4f8ec9c8ef79223d2865622e9bf4da8835776c81409d8", size = 22766493, upload-time = "2026-03-27T10:14:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/d1/2a/7eb0a1a94f4634899c4e7d4cf56b1aa7cb37177b31b3d2d5c552baebf7b0/xgrammar-0.1.33-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a38aee7a91f1b062b045e2c3e57279aef9155a84fa898bf9a9ae7820090465ce", size = 22703593, upload-time = "2026-03-27T10:14:35.993Z" }, + { url = "https://files.pythonhosted.org/packages/a3/7f/61485241556663db7cabefd1ffc6039ec42145821c8c8008996b6d96c3aa/xgrammar-0.1.33-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bb61a21f44f0c3bd4884426b2d18479622202e2fd0928b63c10a0168972c502e", size = 42131816, upload-time = "2026-03-27T10:14:39.76Z" }, + { url = "https://files.pythonhosted.org/packages/62/a9/69fbbcde74ac0ccc914e0eb222ac0916688fe291f9f234852dc77912a1bf/xgrammar-0.1.33-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b8aad12a149e3b689b5c4aef8ff529e58e55aee1406bd372c850b504131a8b7", size = 42204408, upload-time = "2026-03-27T10:14:43.692Z" }, + { url = "https://files.pythonhosted.org/packages/63/2a/1ba21492fe956a2a284d037e2035e585c170107cd522e888a865659fb8a2/xgrammar-0.1.33-cp310-cp310-win_amd64.whl", hash = "sha256:19299330f8b04242eaefc37967edd317d044b024c2d509363ba7dd4d483532ab", size = 7222752, upload-time = "2026-03-27T10:14:46.553Z" }, + { url = "https://files.pythonhosted.org/packages/7d/df/695172c6e16e3145ebeffadf7045d1b43d874990da19c7519b01c49ef45a/xgrammar-0.1.33-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:b5f0bbabe128ff5c985b77b4315d25f19a3c7247a0847708a4a484ae6214041a", size = 22766437, upload-time = "2026-03-27T10:14:49.587Z" }, + { url = "https://files.pythonhosted.org/packages/a0/de/14ab62bfa6035d0ad276f10f0795fb957cfafb0e3ebc77e87ef36befc461/xgrammar-0.1.33-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:08c7befefb38c89bf368c26a8e75d00204f03fc303eed61ca570dd3f568d9ead", size = 22703373, upload-time = "2026-03-27T10:14:53.501Z" }, + { url = "https://files.pythonhosted.org/packages/4b/16/f8297e0e3b468636d8e0190002badfe4a6d8d1c2af295fea2d164e7b5a8a/xgrammar-0.1.33-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5f561e676df8c9e941c7a2f6df9612bbf645bf1fc714b4a9282cf75cff532f8", size = 42132308, upload-time = "2026-03-27T10:14:58.545Z" }, + { url = "https://files.pythonhosted.org/packages/12/e0/629b892a3810446097635dd1be7e4d977107c42232efb229d70e5c827227/xgrammar-0.1.33-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bc9151d9f0d05862c253998c533f04c000273f57180fb6a4e3623e321fd47db", size = 42204526, upload-time = "2026-03-27T10:15:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/29/f5/aee458b54919ef989ca21d4a39721a51b8a3cba37614148b863968ef5c8c/xgrammar-0.1.33-cp311-cp311-win_amd64.whl", hash = "sha256:27f0cf751b9130805c7db745a7abb86f05228d58523d8388b0b970cada6dee0a", size = 7222644, upload-time = "2026-03-27T10:15:06.366Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5f/9a1ebc9505392ff626b9ba8fca54d46bdba454af80551169676ee5cd27d4/xgrammar-0.1.33-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:16d05f8f9df9852f2055e112adf2ce22440062979bc3dc66869a0b7c2f93eb0a", size = 22765695, upload-time = "2026-03-27T10:15:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/82/82/7081feb505873238583a003f790b10ce84d66ab3a8e8e244e8c1c4729d70/xgrammar-0.1.33-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bb9e0ed6748b8e82d4569be4ebc8c9e60694369295c4fed0ebaa4bab4aa4eb4", size = 22702262, upload-time = "2026-03-27T10:15:12.9Z" }, + { url = "https://files.pythonhosted.org/packages/4e/04/43d4baca876f5ae1b45897ec30a59801a2da37f16da1fcd85f9555e4c125/xgrammar-0.1.33-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c803e60d791854c5d1f271ece7e1f34d73c82dd4a8b2a06b7af5331482a78ac", size = 42133168, upload-time = "2026-03-27T10:15:16.994Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a8/672833a3cff027253793aa999401d8364896ebf396967e475c7a878b895f/xgrammar-0.1.33-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:52b8eaa533282a0efb0835db6998ae72e7b3c7875d7a52e360ffebff9b78c30a", size = 42205803, upload-time = "2026-03-27T10:15:21.599Z" }, + { url = "https://files.pythonhosted.org/packages/04/38/3fd9f21b101871b4b7f86ee2e15fe6d0cb61a3753f18b391bdee22c74810/xgrammar-0.1.33-cp312-cp312-win_amd64.whl", hash = "sha256:94fea66b41feb28be7e91f95f078986cbc850f42f7adb2d8987634eadf1fb94b", size = 7222161, upload-time = "2026-03-27T10:15:24.636Z" }, ] [[package]]