diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1312002ee44..7613dc59da5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,54 +1,12 @@ -megatron/core/ @NVIDIA/core-adlr @NVIDIA/core-nemo - -megatron/core/models/gpt/ @NVIDIA/gpt - -megatron/core/models/multimodal/ @NVIDIA/multi-modal - -megatron/core/models/mamba/ @NVIDIA/hybrid-mamba - -megatron/core/datasets/ @NVIDIA/datasets - -megatron/core/distributed/fsdp/ @NVIDIA/megatron-fsdp - -megatron/core/transformer/fsdp_dtensor_checkpoint.py @NVIDIA/megatron-fsdp - -megatron/core/dist_checkpointing/ @NVIDIA/dist-checkpointing - -megatron/core/optimizer/distrib_optimizer/ @NVIDIA/dist-optimizer - -megatron/core/inference/modelopt_support @NVIDIA/quantization-and-inference - -megatron/core/datasets/ @NVIDIA/datasets - -megatron/core/pipeline_parallel/ @NVIDIA/pipeline-parallelism - -megatron/core/transformer/ @NVIDIA/core-adlr @NVIDIA/core-nemo - -megatron/core/transformer/moe/ @NVIDIA/core-adlr @NVIDIA/core-devtech - -megatron/core/inference/ @NVIDIA/inference - -megatron/core/parallel_state.py @NVIDIA/core-nemo - -megatron/core/post_training/ @NVIDIA/post-training - -megatron/post_training/ @NVIDIA/post-training +* @NVIDIA/core-nemo @NVIDIA/core-devtech .gitlab/ @NVIDIA/ci .github/ @NVIDIA/ci .gitlab-ci.yml @NVIDIA/ci docker/ @NVIDIA/ci tests/unit_tests/run_ci_test.sh @NVIDIA/ci -tests/test_utils/python_scripts/ +tests/test_utils/python_scripts/ tests/functional_tests/python_test_utils/ @NVIDIA/ci tests/functional_tests/shell_test_utils/ @NVIDIA/ci -megatron/core/transformer/transformer_block.py @NVIDIA/ci -megatron/core/transformer/transformer_layer.py @NVIDIA/ci -tests/functional_tests/test_cases/ @NVIDIA/ci -tests/functional_tests/recipes/ @NVIDIA/ci -tests/unit_tests/ @NVIDIA/ci - -megatron/rl/ @NVIDIA/reinforcement-learning -examples/rl/ @NVIDIA/reinforcement-learning -test/unit_tests/test_rl_utils.py @NVIDIA/reinforcement-learning -train_rl.py @NVIDIA/reinforcement-learning +pyproject.toml @NVIDIA/ci +uv.lock @NVIDIA/ci diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml index 91fc4570b00..21df8a5c986 100644 --- a/.github/workflows/build-test-publish-wheel.yml +++ b/.github/workflows/build-test-publish-wheel.yml @@ -17,6 +17,7 @@ name: Build, test, and publish a PyPi wheel (to testpypi). on: push: branches: + - dev - main - "pull-request/[0-9]+" - "deploy-release/*" diff --git a/.github/workflows/cherry-pick-release-commit.yml b/.github/workflows/cherry-pick-release-commit.yml index 882b3f5b268..58b447939a7 100644 --- a/.github/workflows/cherry-pick-release-commit.yml +++ b/.github/workflows/cherry-pick-release-commit.yml @@ -17,6 +17,7 @@ on: push: branches: - main + - dev jobs: cherry-pick: diff --git a/.github/workflows/cicd-approve-test-queue.yml b/.github/workflows/cicd-approve-test-queue.yml index f34657eb509..1c35031cb35 100644 --- a/.github/workflows/cicd-approve-test-queue.yml +++ b/.github/workflows/cicd-approve-test-queue.yml @@ -155,6 +155,8 @@ jobs: workflow_id = workflow["id"] workflow_name = workflow["display_title"] + pr_info = workflow.get("pull_requests", [{}])[0] + pr_number = pr_info.get("number", "unknown") print(f"Approving workflow {workflow_name} with Run Id: {workflow_id}") deployment_url = f"actions/runs/{workflow_id}/pending_deployments" diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index eadc35b9b82..4a4a1a2cad1 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -65,12 +65,13 @@ jobs: id: check-membership env: IS_MAIN_BRANCH: ${{ github.ref == 'refs/heads/main' }} + IS_DEV_BRANCH: ${{ github.ref == 'refs/heads/dev' }} IS_MERGE_GROUP: ${{ github.event_name == 'merge_group' }} SCHEDULED_JOB: ${{ github.event_name == 'schedule' }} run: | PR_AUTHOR=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').user.login }} - if [ "${{ env.SCHEDULED_JOB }}" == "true" ] || [ "${IS_MAIN_BRANCH}" == "true" ] || [ "${IS_MERGE_GROUP}" == "true" ]; then + if [ "${{ env.SCHEDULED_JOB }}" == "true" ] || [ "${IS_MAIN_BRANCH}" == "true" ] || [ "${IS_DEV_BRANCH}" == "true" ] || [ "${IS_MERGE_GROUP}" == "true" ]; then echo "is_maintainer=true" | tee -a $GITHUB_OUTPUT exit 0 fi diff --git a/.github/workflows/copyright-check.yml b/.github/workflows/copyright-check.yml index dd89d7ed7cc..a6806ea0da8 100644 --- a/.github/workflows/copyright-check.yml +++ b/.github/workflows/copyright-check.yml @@ -17,6 +17,8 @@ name: Copyright check on: push: branches: + - dev + - main - "pull-request/[0-9]+" - "deploy-release/*" merge_group: diff --git a/.github/workflows/mirror-to-main.yml b/.github/workflows/mirror-to-main.yml new file mode 100644 index 00000000000..cb77851942b --- /dev/null +++ b/.github/workflows/mirror-to-main.yml @@ -0,0 +1,129 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name: Mirror Dev to Main + +on: + push: + branches: + - "pull-request/[0-9]+" + +jobs: + cherry-pick-to-main: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.PAT }} + + - name: Get PR info + id: get-pr-info + uses: nv-gha-runners/get-pr-info@main + + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "GitHub Actions Bot" + + - name: Cherry-pick to main + env: + GH_TOKEN: ${{ secrets.PAT }} + run: | + set -x + + PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} + BASE_REF="${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }}" + HAS_MIRROR_MAIN_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "mirror-to-main")' || echo "false") + TARGET_BRANCH="cherry-pick-$PR_NUMBER-into-main" + + # Skip if not labeled with mirror-to-main + if [ "$HAS_MIRROR_MAIN_LABEL" != "true" ]; then + echo "PR is not labeled with mirror-to-main, will not mirror to main." + exit 0 + fi + + # Skip if not targeting dev + if [ "$BASE_REF" != "dev" ]; then + echo "PR is not targeting dev, will not mirror to main." + exit 0 + fi + + # Check if target branch already exists + if git ls-remote --heads origin "refs/heads/$TARGET_BRANCH" | grep -q .; then + echo "Target branch already exists, will not cherry-pick again." + exit 0 + fi + + # Get PR details + PR_AUTHOR="${{ fromJSON(steps.get-pr-info.outputs.pr-info).user.login }}" + PR_TITLE="${{ fromJSON(steps.get-pr-info.outputs.pr-info).title }}" + SOURCE_BRANCH="${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.ref }}" + SOURCE_REPO="${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.repo.full_name }}" + + # Fetch all branches + git fetch origin dev + + # Handle forks vs same repo + if [ "$SOURCE_REPO" = "${{ github.repository }}" ]; then + git fetch origin "$SOURCE_BRANCH" + git checkout "$SOURCE_BRANCH" + else + git fetch "https://github.com/$SOURCE_REPO.git" "$SOURCE_BRANCH" + git checkout FETCH_HEAD + fi + + # Find commit range to cherry-pick + START_COMMIT=$(git merge-base origin/dev HEAD) + END_COMMIT=$(git rev-parse HEAD) + + # Create cherry-pick branch from main + git fetch origin main + git checkout main + git checkout -b "$TARGET_BRANCH" + + # Cherry-pick commits + if ! git cherry-pick "$START_COMMIT..$END_COMMIT"; then + # Comment on the original PR about the failure + COMMENT_BODY=$(cat <<'EOF' + ❌ **Cherry-pick to main failed** + + The cherry-pick encountered conflicts and could not be completed automatically. + + **Next steps:** + 1. Manually create a PR with these changes to main + 2. Resolve any conflicts + EOF + ) + + gh pr comment $PR_NUMBER --body "$COMMENT_BODY" + exit 1 + fi + + # Push branch + git push -u origin "$TARGET_BRANCH" + + # Create PR to main + gh pr create \ + --base main \ + --head "$TARGET_BRANCH" \ + --title "cp: \`$PR_TITLE ($PR_NUMBER)\` into \`main\`" \ + --body "[🤖]: Hi @$PR_AUTHOR 👋

We've cherry-picked \`$PR_TITLE (#$PR_NUMBER)\` into \`main\` for you! 🚀

Please review and approve this cherry-pick at your convenience!" \ + --label "cherry-pick" \ + --reviewer "$PR_AUTHOR" + diff --git a/.github/workflows/multi-approval-bot.yml b/.github/workflows/multi-approval-bot.yml deleted file mode 100644 index e8507605aa7..00000000000 --- a/.github/workflows/multi-approval-bot.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Codeowners Approval Workflow" - -on: - push: - branches: - - "pull-request/[0-9]+" - merge_group: - types: [checks_requested] - -jobs: - pre-flight: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.65.5 - if: github.repository == 'NVIDIA/Megatron-LM' - - codeowners-approval: - needs: [pre-flight] - runs-on: ubuntu-latest - environment: nemo-ci - if: | - !(needs.pre-flight.outputs.docs_only == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - || needs.pre-flight.outputs.is_deployment_workflow == 'true') - steps: - - name: Get PR info - id: get-pr-info - if: startsWith(github.ref, 'refs/heads/pull-request/') - uses: nv-gha-runners/get-pr-info@main - - - name: Checkout action - uses: actions/checkout@v3 - with: - repository: noamelf/codeowner-multi-approval-action - ref: v0.1 - path: codeowner-multi-approval-action - - - name: Check Codeowners Approval - uses: ./codeowner-multi-approval-action - with: - pr-number: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }} - repo-name: ${{ github.repository }} - github-token: ${{ secrets.PAT }} - - multi-approval-bot-summary: - needs: [pre-flight, codeowners-approval] - if: | - ( - needs.pre-flight.outputs.docs_only == 'true' - || needs.pre-flight.outputs.is_merge_group == 'true' - || needs.pre-flight.outputs.is_deployment_workflow == 'true' - || always() - ) - && github.repository == 'NVIDIA/Megatron-LM' - && !cancelled() - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Result - env: - GH_TOKEN: ${{ github.token }} - GITHUB_RUN_ID: ${{ github.run_id }} - SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_merge_group == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }} - run: | - FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0 - - if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then - echo "✅ All previous jobs completed successfully" - exit 0 - else - echo "❌ Found $FAILED_JOBS failed job(s)" - # Show which jobs failed - gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name' - exit 1 - fi diff --git a/.gitlab/stages/00.pre.yml b/.gitlab/stages/00.pre.yml index a22c2cf3ea7..2210ddd7d02 100644 --- a/.gitlab/stages/00.pre.yml +++ b/.gitlab/stages/00.pre.yml @@ -68,6 +68,7 @@ pre:create_ci_branches_dev: - branch: ci-dev-rebuild-mcore-nemo-image - branch: ci-dev-mr - branch: ci-dev-nightly + - branch: ci-dev-weekly - branch: ci-dev-upgrade-dependencies tags: - arch/amd64 diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index 4c50e3158e3..cc18c34722e 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -53,7 +53,7 @@ functional:configure: - | RECORD_CHECKPOINTS=$([[ "$CI_MERGE_REQUEST_LABELS" == *"Record checkpoints"* || "$FUNCTIONAL_TEST_RECORD_CHECKPOINTS" == "yes" ]] && echo "true" || echo "false") - | - if [[ "$FUNCTIONAL_TEST_SCOPE" == "release" || "$FUNCTIONAL_TEST_SCOPE" == "pre-release" ]]; then + if [[ "$FUNCTIONAL_TEST_SCOPE" == "release" || "$FUNCTIONAL_TEST_SCOPE" == "weekly" ]]; then FUNCTIONAL_TEST_NAME=$(eval echo $FUNCTIONAL_TEST_NAME) RELEASE_ARGS=( "--run-name" @@ -172,15 +172,15 @@ functional:run_dev_dgx_h100: functional:run_nemo: extends: [.functional_tests_rules] trigger: - project: "dl/joc/nemo-ci" + project: 'dl/joc/nemo-ci' branch: main-mirror strategy: depend inherit: variables: true variables: - MCORE_COMMIT: $CI_COMMIT_SHA - TEST_NEMO2_MODULE: "True" - ALLOW_FAILURE_DEPENDENCY: "True" + MCORE_MR_COMMIT: $CI_COMMIT_SHA + TEST_NEMO2_MODULE: 'True' + ALLOW_FAILURE_DEPENDENCY: 'True' TESTS_TO_RUN_ON_THIS_COMMIT: nightly rules: - if: $FUNCTIONAL_TEST == "yes" @@ -217,7 +217,7 @@ functional:x_notify: - export RO_API_TOKEN=${PROJECT_ACCESS_TOKEN_MCORE} - export GITLAB_ENDPOINT - export CONTEXT=$FUNCTIONAL_TEST_SCOPE - - export TAG_TEAM=$([[ "$CI_COMMIT_BRANCH" == "main" ]] && echo "1" || "0") + - export TAG_TEAM=$([[ "$CI_COMMIT_BRANCH" == "main" || "$CI_COMMIT_BRANCH" == "dev" ]] && echo "1" || "0") - export TEAM_SLUG=$SLACK_ADMIN - | python tests/test_utils/python_scripts/notify.py \ diff --git a/README.md b/README.md index 4ad69e341d1..abb581c9b34 100644 --- a/README.md +++ b/README.md @@ -11,477 +11,66 @@ Megatron-LM & Megatron Core
-## ⚡ Quick Start +> ## 🚨 **DEVELOPMENT BRANCH** +> ⚠️ **EXPERIMENTAL FEATURES** - This is the **dev branch** with experimental features. +> +> **→ For releases and comprehensive documentation, visit the [main branch](https://github.com/NVIDIA/Megatron-LM)** -```bash -# 1. Install Megatron Core with required dependencies -pip install --no-build-isolation megatron-core[mlm,dev] +## ⚡ Quickstart -# 2. Clone repository for examples -git clone https://github.com/NVIDIA/Megatron-LM.git +```bash +# Clone the dev branch +git clone -b dev https://github.com/NVIDIA/Megatron-LM.git cd Megatron-LM -pip install --no-build-isolation .[mlm,dev] -``` - -**→ [Complete Installation Guide](#installation)** - Docker, pip variants (dev,lts,etc.), source installation, and system requirements - -# Latest News -- 📣 NEW! **[Megatron Dev Branch](https://github.com/NVIDIA/Megatron-LM/tree/dev)** - early access branch with experimental features. -- 🔄 **[Megatron Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge)** - Bidirectional converter for interoperability between Hugging Face and Megatron checkpoints, featuring production-ready recipes for popular models. -- **[2025/08]** **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive roadmap for MoE features including DeepSeek-V3, Qwen3, advanced parallelism strategies, FP8 optimizations, and Blackwell performance enhancements. -- **[2025/08]** **[GPT-OSS Model](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions are being integrated into Megatron Core. -- **[2025/06]** **[Megatron MoE Model Zoo](https://github.com/yanring/Megatron-MoE-ModelZoo)** - Best practices and optimized configurations for training DeepSeek-V3, Mixtral, and Qwen3 MoE models with performance benchmarking and checkpoint conversion tools. -- **[2025/05]** Megatron Core v0.11.0 brings new capabilities for multi-data center LLM training ([blog](https://developer.nvidia.com/blog/turbocharge-llm-training-across-long-haul-data-center-networks-with-nvidia-nemo-framework/)). - -
-Previous News - -- **[2024/07]** Megatron Core v0.7 improves scalability and training resiliency and adds support for multimodal training ([blog](https://developer.nvidia.com/blog/train-generative-ai-models-more-efficiently-with-new-nvidia-Megatron-Core-functionalities/)). -- **[2024/06]** Megatron Core added supports for Mamba-based models. Check out our paper [An Empirical Study of Mamba-based Language Models](https://arxiv.org/pdf/2406.07887) and [code example](https://github.com/NVIDIA/Megatron-LM/tree/ssm/examples/mamba). -- **[2024/01 Announcement]** NVIDIA has released the core capabilities in **Megatron-LM** into [**Megatron Core**](https://github.com/NVIDIA/Megatron-LM/tree/main/megatron/core) in this repository. Megatron Core expands upon Megatron-LM's GPU-optimized techniques with more cutting-edge innovations on system-level optimizations, featuring composable and modular APIs. Explore the [Megatron Core intro](#Megatron Core) for more details. - -
+# Install from source with dev dependencies (includes transformer_engine) +pip install -e .[mlm,dev] +```
Table of Contents **Getting Started** +- [⚡ Quick Start](#-quick-start) +- [🧠 Dev Branch Philosophy](#-dev-branch-philosophy) +- [📊 Performance & Benchmarking](#-performance--benchmarking) +- [👥 Community & Support](#-community--support) -- [Quick Start](#-quick-start) -- [Latest News](#latest-news) -- [Megatron Overview](#megatron-overview) - - [Project Structure](#project-structure) - - [Megatron-LM: Reference Implementation](#megatron-lm-reference-implementation) - - [Megatron Core: Production Library](#megatron-core-production-library) -- [Installation](#installation) - - [Docker (Recommended)](#-docker-recommended) - - [Pip Installation](#-pip-installation) - - [Source Installation](#-source-installation) - - [System Requirements](#system-requirements) - -**Core Features** - -- [Performance Benchmarking](#performance-benchmarking) - - [Weak Scaling Results](#weak-scaling-results) - - [Strong Scaling Results](#strong-scaling-results) -- [Ecosystem Libraries](#ecosystem-libraries) - -**Training** - -- [Training](#training) - - [Getting Started](#getting-started) - - [Data Preparation](#data-preparation) -- [Parallelism Strategies](#parallelism-strategies) - - [Data Parallelism (DP)](#data-parallelism-dp) - - [Tensor Parallelism (TP)](#tensor-parallelism-tp) - - [Pipeline Parallelism (PP)](#pipeline-parallelism-pp) - - [Context Parallelism (CP)](#context-parallelism-cp) - - [Expert Parallelism (EP)](#expert-parallelism-ep) - - [Parallelism Selection Guide](#parallelism-selection-guide) -- [Performance Optimizations](#performance-optimizations) - -**Resources** - -- [Examples](./examples/) - Training scripts and tutorials -- [Documentation](https://docs.nvidia.com/Megatron-Core/) - Official docs -- [Roadmaps](#roadmaps) - Development roadmaps and feature tracking -- [Community & Support](#-community--support) - Get help and contribute - - [Getting Help](#getting-help) - - [Contributing](#contributing) - - [Citation](#citation) +**For Complete Documentation** → [Main Branch](https://github.com/NVIDIA/Megatron-LM) | [Official Docs](https://docs.nvidia.com/Megatron-Core/)
-# Megatron Overview - -## Project Structure - -``` -Megatron-LM/ -├── megatron/ -│ ├── core/ # Megatron Core (kernels, parallelism, building blocks) -│ │ ├── models/ # Transformer models -│ │ ├── transformer/ # Transformer building blocks -│ │ ├── tensor_parallel/ # Tensor parallelism -│ │ ├── pipeline_parallel/ # Pipeline parallelism -│ │ ├── distributed/ # Distributed training (FSDP, DDP) -│ │ ├── optimizer/ # Optimizers -│ │ ├── datasets/ # Dataset loaders -│ │ ├── inference/ # Inference engines -│ │ └── export/ # Model export (e.g. TensorRT-LLM) -│ ├── training/ # Training scripts -│ ├── inference/ # Inference server -│ ├── legacy/ # Legacy components -│ └── post_training/ # Post-training (RLHF, etc.) -├── examples/ # Ready-to-use training examples -├── tools/ # Utility tools -├── tests/ # Comprehensive test suite -└── docs/ # Documentation -``` - -### Megatron-LM: Reference Implementation - -**Reference implementation** that includes Megatron Core plus everything needed to train models. - -**Best for:** - -- **Training state-of-the-art foundation models** at scale with cutting-edge performance on latest NVIDIA hardware -- **Research teams** exploring new architectures and training techniques -- **Learning distributed training** concepts and best practices -- **Quick experimentation** with proven model configurations - -**What you get:** - -- Pre-configured training scripts for GPT, LLama, DeepSeek, Qwen, and more. -- End-to-end examples from data prep to evaluation -- Research-focused tools and utilities - -### Megatron Core: Composable Library - -**Composable library** with GPU-optimized building blocks for custom training frameworks. - -**Best for:** - -- **Framework developers** building on top of modular and optimized components -- **Research teams** needing custom training loops, optimizers, or data pipelines -- **ML engineers** requiring fault-tolerant training pipelines - -**What you get:** - -- Composable transformer building blocks (attention, MLP, etc.) -- Advanced parallelism strategies (TP, PP, DP, EP, CP) -- Pipeline schedules and distributed optimizers -- Mixed precision support (FP16, BF16, FP8) -- GPU-optimized kernels and memory management -- High-performance dataloaders and dataset utilities -- Model architectures (LLaMA, Qwen, GPT, Mixtral, Mamba, etc.) - -## Ecosystem Libraries - -**Libraries used by Megatron Core:** - -- **[Megatron Energon](https://github.com/NVIDIA/Megatron-Energon)** 📣 **NEW!** - Multi-modal data loader (text, images, video, audio) with distributed loading and dataset blending -- **[Transformer Engine](https://github.com/NVIDIA/TransformerEngine)** - Optimized kernels and FP8 mixed precision support -- **[Resiliency Extension (NVRx)](https://github.com/NVIDIA/nvidia-resiliency-ext)** - Fault tolerant training with failure detection and recovery - -**Libraries using Megatron Core:** - -- **[Megatron Bridge](https://github.com/NVIDIA-NeMo/Megatron-Bridge)** - Training library with bidirectional Hugging Face ↔ Megatron checkpoint conversion, flexible training loops, and production-ready recipes -- **[NeMo RL](https://github.com/NVIDIA-NeMo/RL)** - Scalable toolkit for efficient reinforcement learning with RLHF, DPO, and other post-training methods -- **[NeMo Framework](https://docs.nvidia.com/nemo-framework/user-guide/latest/overview.html)** - Enterprise framework with cloud-native support and end-to-end examples -- **[TensorRT Model Optimizer (ModelOpt)](https://github.com/NVIDIA/TensorRT-Model-Optimizer)** - Model optimization toolkit for quantization, pruning, and distillation - -**Compatible with:** [Hugging Face Accelerate](https://github.com/huggingface/accelerate), [Colossal-AI](https://github.com/hpcaitech/ColossalAI), [DeepSpeed](https://github.com/microsoft/DeepSpeed) - -# Installation -## 🐳 Docker (Recommended) -We strongly recommend using the previous releases of [PyTorch NGC Container](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch) rather than the latest one for optimal compatibility with Megatron Core release and testing. Our releases are always based on the previous month's NGC container, so this ensures compatibility and stability. -**Note:** The NGC PyTorch container constraints the python environment globally via `PIP_CONSTRAINT`. In the following examples we will unset the variable. -This container comes with all dependencies pre-installed with compatible versions and optimized configurations for NVIDIA GPUs: -- PyTorch (latest stable version) -- CUDA, cuDNN, NCCL (latest stable versions) -- Support for FP8 on NVIDIA Hopper, Ada, and Blackwell GPUs -- For best performance, use NVIDIA Turing GPU architecture generations and later +## Dev Branch Philosophy -```bash -# Run container with mounted directories -docker run --runtime --nvidia --gpus all -it --rm \ - -v /path/to/megatron:/workspace/megatron \ - -v /path/to/dataset:/workspace/dataset \ - -v /path/to/checkpoints:/workspace/checkpoints \ - -e PIP_CONSTRAINT= \ - nvcr.io/nvidia/pytorch:25.04-py3 -``` - -## Pip Installation - -Megatron Core offers support for two NGC PyTorch containers: - -- `dev`: Moving head that supports the most recent upstream dependencies -- `lts`: Long-term support of NGC PyTorch 24.01 - -Both containers can be combined with `mlm` which adds package dependencies for Megatron-LM on top of Megatron Core. - -```bash -# Install the latest release dependencies -pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2" -pip install --no-build-isolation megatron-core[dev] -# For running an M-LM application: -pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2" -pip install --no-build-isolation megatron-core[mlm,dev] -``` - -```bash -# Install packages for LTS support NGC PyTorch 24.01 -pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2" -pip install --no-build-isolation megatron-core[lts] -# For running an M-LM application: -pip install "setuptools<80.0.0,>=77.0.0" "packaging>=24.2" -pip install --no-build-isolation megatron-core[mlm,lts] -``` - -For a version of Megatron Core with only torch, run: - -```bash -pip install megatron-core -``` - -## System Requirements - -### Hardware Requirements - -- **FP8 Support**: NVIDIA Hopper, Ada, Blackwell GPUs -- **Recommended**: NVIDIA Turing architecture or later - -### Software Requirements +### Fast Iteration +- **Streamlined Review**: 1 code owner + 1 dev approver (can delegate review) + CI/CD -- **CUDA/cuDNN/NCCL**: Latest stable versions -- **PyTorch**: Latest stable version -- **Transformer Engine**: Latest stable version -- **Python**: 3.12 recommended +### Feature Lifecycle (Coming Soon) +- **6-Month Timeline**: Experimental features must graduate to stable or be deprecated +- **Migration Support**: Assistance provided for feature transitions -# Performance Benchmarking +### Stability Expectations +- **Experimental Nature**: Features may change or be removed as development progresses +- **Testing**: All features will pass convergence and performance validation before inclusion +- **Support**: Dev branch issues should include `[DEV]` prefix -For our latest performance benchmarking results, please refer to [NVIDIA NeMo Framework Performance Summary](https://docs.nvidia.com/nemo-framework/user-guide/latest/performance/performance_summary.html). +## Performance & Benchmarking -Our codebase efficiently trains models from 2B to 462B parameters across thousands of GPUs, achieving up to **47% Model FLOP Utilization (MFU)** on H100 clusters. +- 🚀 [2025/11] [Optimizing DeepSeek-V3 Training Performance on NVIDIA GB200 NVL72](docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-optimization.md). +- ⚡ [2025/11] [A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200](docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md). -![Model table](images/model_table.png) - -**Benchmark Configuration:** - -- **Vocabulary size**: 131,072 tokens -- **Sequence length**: 4096 tokens -- **Model scaling**: Varied hidden size, attention heads, and layers to achieve target parameter counts -- **Communication optimizations**: Fine-grained overlapping with DP (`--overlap-grad-reduce`, `--overlap-param-gather`), TP (`--tp-comm-overlap`), and PP (enabled by default) - -**Key Results:** - -- **6144 H100 GPUs**: Successfully benchmarked 462B parameter model training -- **Superlinear scaling**: MFU increases from 41% to 47-48% with model size -- **End-to-end measurement**: Throughputs include all operations (data loading, optimizer steps, communication, logging) -- **Production ready**: Full training pipeline with checkpointing and fault tolerance -- *Note: Performance results measured without training to convergence* - -## Weak Scaling Results - -Our weak scaled results show superlinear scaling (MFU increases from 41% for the smallest model considered to 47-48% for the largest models); this is because larger GEMMs have higher arithmetic intensity and are consequently more efficient to execute. - -![Weak scaling](images/weak_scaling.png) - -## Strong Scaling Results - -We also strong scaled the standard GPT-3 model (our version has slightly more than 175 billion parameters due to larger vocabulary size) from 96 H100 GPUs to 4608 GPUs, using the same batch size of 1152 sequences throughout. Communication becomes more exposed at larger scale, leading to a reduction in MFU from 47% to 42%. - -![Strong scaling](images/strong_scaling.png) - -# Training - -## Getting Started - -### Simple Training Example - -```bash -# Distributed training example (2 GPUs, mock data) -torchrun --nproc_per_node=2 examples/run_simple_mcore_train_loop.py -``` - -### LLama-3 Training Example - -```bash -# 8 GPUs, FP8 precision, mock data -./examples/llama/train_llama3_8b_fp8.sh -``` - -## Data Preparation - -### JSONL Data Format - -```json -{"text": "Your training text here..."} -{"text": "Another training sample..."} -``` - -### Basic Preprocessing - -```bash -python tools/preprocess_data.py \ - --input data.jsonl \ - --output-prefix processed_data \ - --tokenizer-type HuggingFaceTokenizer \ - --tokenizer-model /path/to/tokenizer.model \ - --workers 8 \ - --append-eod -``` - -### Key Arguments - -- `--input`: Path to input JSON/JSONL file -- `--output-prefix`: Prefix for output binary files (.bin and .idx) -- `--tokenizer-type`: Tokenizer type (`HuggingFaceTokenizer`, `GPT2BPETokenizer`, etc.) -- `--tokenizer-model`: Path to tokenizer model file -- `--workers`: Number of parallel workers for processing -- `--append-eod`: Add end-of-document token - - - -# Parallelism Strategies - -## Data Parallelism (DP) - -### Standard Data Parallel - -```bash -# Standard DDP - replicate model on each GPU -torchrun --nproc_per_node=8 pretrain_gpt.py \ - --data-parallel-sharding-strategy no_shard -``` - -### Fully Sharded Data Parallel (FSDP) - -```bash -# Megatron's optimized FSDP (~15% faster than PyTorch FSDP2) ---use-custom-fsdp - -# PyTorch FSDP2 ---use-torch-fsdp2 - -# Sharding strategies ---data-parallel-sharding-strategy optim # Shard optimizer states (ZeRO-1) ---data-parallel-sharding-strategy optim_grads # Shard gradients + optimizer (ZeRO-2) ---data-parallel-sharding-strategy optim_grads_params # Shard parameters + gradients + optimizer (ZeRO-3) -``` - -## Tensor Parallelism (TP) - -Split individual model layers across GPUs: - -```bash ---tensor-model-parallel-size 4 # 4-way tensor parallelism ---sequence-parallel # Enable sequence parallelism (recommended with TP) -``` - -## Pipeline Parallelism (PP) - -Split model depth across GPUs: - -```bash ---pipeline-model-parallel-size 8 # 8 pipeline stages ---virtual-pipeline-model-parallel-size 4 # Virtual pipeline for better load balancing -``` - -## Context Parallelism (CP) - -Split long sequences across GPUs for handling long contexts: - -```bash ---context-parallel-size 2 # 2-way context parallelism ---cp-comm-type p2p # Communication: p2p, a2a, allgather, a2a+p2p ---hierarchical-context-parallel-sizes 2 4 # Hierarchical context parallelism -``` - -## Expert Parallelism (EP) - -For Mixture of Experts (MoE) models: - -```bash ---expert-model-parallel-size 4 # 4-way expert parallelism ---num-experts 8 # 8 experts per MoE layer ---moe-grouped-gemm # Optimize expert computation -``` - -## Combining Parallelism Strategies - -### Parallelism Selection Guide - -Based on [NVIDIA NeMo production configurations](https://github.com/NVIDIA/NeMo/tree/main/scripts/performance/recommended_model_configs): - -| Model | Size | GPUs | TP | PP | CP | EP | Notes | -|-------|------|------|----|----|----|----|-------| -| **LLama-3** | 8B | 8 | 1 | 1 | 2 | 1 | CP for long seqlen (8K) | -| **LLama-3** | 70B | 64 | 4 | 4 | 2 | 1 | TP+PP | -| **LLama-3.1** | 405B | 1024 | 8 | 8 | 2 | 1 | 3D parallelism for scale | -| **GPT-3** | 175B | 128-512 | 4 | 8 | 1 | 1 | Large model config | -| **Mixtral** | 8x7B | 64 | 1 | 4 | 1 | 8 | EP for MoE | -| **Mixtral** | 8x22B | 256 | 4 | 4 | 8 | 8 | Combined TP+EP for large MoE | -| **DeepSeek-V3** | 671B | 1024 | 2 | 16 | 1 | 64 | Large MoE config | - -### MoE-Specific Requirements - -**Important**: When combining Expert Parallelism (EP) with Tensor Parallelism (TP), **Sequence Parallelism (SP) must be enabled**. - -## Performance Optimizations - -| Feature | Flag | Benefit | -|---------|------|---------| -| **FlashAttention** | `--attention-backend` | Faster attention and lower memory usage | -| **FP8 Training** | `--fp8-hybrid` | Faster training | -| **Activation Checkpointing** | `--recompute-activations` | Reduced memory usage | -| **Data Parallelism Communication Overlap** | `--overlap-grad-reduce` | Faster distributed training | -| **Distributed Optimizer** | `--use-distributed-optimizer` | Reduced checkpointing time | - -**→ [NVIDIA NeMo Framework Performance Tuning Guide](https://docs.nvidia.com/nemo-framework/user-guide/latest/performance/performance-guide.html#performance-tuning-guide)** - Comprehensive performance optimization guide covering advanced tuning techniques, communication overlaps, memory optimizations, and profiling options. - -### FlashAttention - -[FlashAttention](https://github.com/Dao-AILab/flash-attention) is a fast and memory-efficient attention algorithm. We recommend the default usage, which uses cuDNN for attention via Transformer Engine and provides up to 50% speedups on forward and 84% on backward propagation with FP8 kernels. The `flash-attn` package is also supported via `--use-flash-attn`. - -### Mixed Precision Training - -```bash ---fp16 # Standard FP16 ---bf16 # BFloat16 (recommended for large models) ---fp8-hybrid # FP8 training (Hopper, Ada, and Blackwell GPUs) -``` - -### Activation Checkpointing and Recomputation - -```bash -# For limited memory ---recompute-activations - -# For extreme memory constraints ---recompute-granularity full \ ---recompute-method uniform -``` - -### Data Parallelism Communication Overlap - -```bash ---overlap-grad-reduce ---overlap-param-gather -``` - -### Distributed Optimizer - -```bash ---use-distributed-optimizer -``` - -# Roadmaps - -Stay up-to-date with our development roadmaps and planned features: - -- **[MoE Q3-Q4 2025 Roadmap](https://github.com/NVIDIA/Megatron-LM/issues/1729)** - Comprehensive MoE feature development including DeepSeek-V3, Qwen3, advanced parallelism, FP8 optimizations, and Blackwell enhancements -- **[GPT-OSS Implementation Tracker](https://github.com/NVIDIA/Megatron-LM/issues/1739)** - Advanced features including YaRN RoPE scaling, attention sinks, and custom activation functions - -*More roadmap trackers will be added soon.* - -# Community & Support - -## Getting Help +## Community & Support +### Getting Help - 📖 **[Documentation](https://docs.nvidia.com/Megatron-Core/)** - Official documentation - 🐛 **[Issues](https://github.com/NVIDIA/Megatron-LM/issues)** - Bug reports and feature requests -## Contributing - +### Contributing We ❤️ contributions! Ways to contribute: - 🐛 **Report bugs** - Help us improve reliability @@ -491,8 +80,7 @@ We ❤️ contributions! Ways to contribute: **→ [Contributing Guide](./CONTRIBUTING.md)** -## Citation - +### Citation ```bibtex @article{megatron-lm, title={Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism}, diff --git a/docker/Dockerfile.ci.dev b/docker/Dockerfile.ci.dev index 709583d61fc..95fecdb3f9b 100644 --- a/docker/Dockerfile.ci.dev +++ b/docker/Dockerfile.ci.dev @@ -70,6 +70,7 @@ EOF COPY assets/ /opt/data/ ENV UV_PYTHON=$UV_PROJECT_ENVIRONMENT/bin/python +COPY . /opt/megatron-lm/ ##### For NVIDIANS only ##### FROM main as jet diff --git a/docs/discussions/README.md b/docs/discussions/README.md new file mode 100644 index 00000000000..26a2a8e1648 --- /dev/null +++ b/docs/discussions/README.md @@ -0,0 +1,26 @@ +# Megatron Discussions + +This directory contains in-depth guides, tutorials, and discussions about optimizing and using Megatron for various use cases. + +## Available Guides + +### Performance Optimization + +- **[Optimizing DeepSeek-V3 Training Performance on NVIDIA GB200 NVL72](deepseek-v3-gb200-optimization/deepseek-v3-gb200-optimization.md)** + + A comprehensive guide on optimizing DeepSeek-V3 model training on NVIDIA GB200 NVL72 systems, covering profiling techniques, performance bottlenecks, and optimization strategies. + +- **[A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200](deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md)** + + A detailed guide on how to reproduce the DeepSeek-V3 pre-training performance on GB200, incluing the dockerfile, package requirements and training scripts. + +## Contributing + +If you'd like to contribute a guide or tutorial, please follow this structure: + +1. Create a new directory: `docs/discussions/your-guide-name/` +2. Add your main guide: `docs/discussions/your-guide-name/your-guide-name.md` +3. Create an images directory: `docs/discussions/your-guide-name/images/` +4. Update this README.md with a link to your guide + +Each guide should be self-contained with its own images and supporting files. diff --git a/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md b/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md new file mode 100644 index 00000000000..61bd8289c66 --- /dev/null +++ b/docs/discussions/deepseek-v3-gb200-optimization/deepseek-v3-gb200-reproduce-guide.md @@ -0,0 +1,354 @@ +# A Guide to Reproduce DeepSeek-V3 Pre-training Performance on GB200 + +## 1. Dockerfile + +Requirements: +- Transformer Engine: We recommend using commit [d2945c6](https://github.com/NVIDIA/TransformerEngine/commit/d2945c6a571e3978677614d1fe08779966a5a4ef) with PR [2146](https://github.com/NVIDIA/TransformerEngine/pull/2146) and [2150](https://github.com/NVIDIA/TransformerEngine/pull/2150). You could prepare the branch by yourself, or use this [branch](https://github.com/hxbai/TransformerEngine/commits/dev_20251024/) based on TE v2.9 plus the above three commits/PRs. +- cuDNN: v9.14 is required. +- HybridEP: Install it from [here](https://github.com/deepseek-ai/DeepEP/commits/3f601f7ac1c062c46502646ff04c535013bfca00). + +Dockerfile for reference. + +```dockerfile +FROM nvcr.io/nvidia/pytorch:25.09-py3 AS base + +ENV SHELL=/bin/bash + +# ========================= +# Install system packages +# ========================= +RUN rm -rf /opt/megatron-lm && \ + apt-get update && \ + apt-get install -y sudo gdb bash-builtins git zsh autojump tmux curl gettext libfabric-dev && \ + wget https://github.com/mikefarah/yq/releases/download/v4.27.5/yq_linux_arm64 -O /usr/bin/yq && \ + chmod +x /usr/bin/yq + +# ========================= +# Install Python packages +# ========================= +# NOTE: `unset PIP_CONSTRAINT` to install packages that do not meet the default constraint in the base image. +# Some package requirements and related versions are from +# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/Dockerfile.linting. +# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/requirements_mlm.txt. +# https://github.com/NVIDIA/Megatron-LM/blob/core_v0.12.0/requirements_ci.txt. +RUN unset PIP_CONSTRAINT && pip install --no-cache-dir debugpy dm-tree torch_tb_profiler einops wandb \ + sentencepiece tokenizers transformers torchvision ftfy modelcards datasets tqdm pydantic \ + nvidia-pytriton py-spy yapf darker \ + tiktoken flask-restful \ + nltk wrapt pytest pytest_asyncio pytest-cov pytest_mock pytest-random-order \ + black==24.4.2 isort==5.13.2 flake8==7.1.0 pylint==3.2.6 coverage mypy \ + setuptools==69.5.1 + +# ========================= +# Install cudnn 9.14.0.64 for correct mxfp8 quantization and layernorm fusion +# ========================= +RUN apt-get update && \ + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb && \ + apt-get update && \ + apt-get -y install libcudnn9-cuda-13 + +# ========================= +# Install latest TE +# Use a specific commit instead of main to make it more stable. +# This is based on release_v2.9 branch and contains some CPU and quantization optimizations. +# ========================= +ARG COMMIT="7dd3914726abb79bc99ff5a5db1449458ed64151" +ARG TE="git+https://github.com/hxbai/TransformerEngine.git@${COMMIT}" +RUN pip install nvidia-mathdx==25.1.1 && \ + unset PIP_CONSTRAINT && \ + NVTE_CUDA_ARCHS="100" NVTE_BUILD_THREADS_PER_JOB=8 NVTE_FRAMEWORK=pytorch pip install --no-build-isolation --no-cache-dir $TE + +# ========================= +# Install HybridEP +# ========================= +WORKDIR /home/ +RUN git clone --branch hybrid-ep https://github.com/deepseek-ai/DeepEP.git && \ + cd DeepEP && git checkout 3f601f7ac1c062c46502646ff04c535013bfca00 && \ + TORCH_CUDA_ARCH_LIST="10.0" pip install --no-build-isolation . + +# ========================= +# Clean cache +# ========================= +RUN rm -rf /root/.cache /tmp/* +``` + +> [!Tip] +> +> If you prefer to use CUDA 12.9, please change the base container to `nvcr.io/nvidia/pytorch:25.06-py3` and the cuDNN to be installed to `libcudnn9-cuda-12`. + +## 2. Megatron-Core + +We recommend using the [dev branch](https://github.com/NVIDIA/Megatron-LM/tree/dev) after PR [1917](https://github.com/NVIDIA/Megatron-LM/pull/1917). + +```bash +git clone https://github.com/NVIDIA/Megatron-LM.git && \ +cd Megatron-LM && +git checkout effebd81f410bc6566fffee6c320b6f8f762e06d +``` + +## 3. Cluster Configuration + +Since we're using EP 32 on NVL72, it's important to make sure + +> [!Important] +> **Every 32 GB200 GPUs (8 nodes) are in the same NVL domain (or rack)**. + +Usually you can make it via your cluster workload manager. Taking Slurm as an example, you could pass `--segment 8` to the sbatch command to ensure that every segment of 8 nodes will be scheduled to a rack. + +## 4. Training scripts + +### Environment variables + +```bash +CUDA_DEVICE_MAX_CONNECTIONS=1 +NVTE_FWD_LAYERNORM_SM_MARGIN=0 +NVTE_BWD_LAYERNORM_SM_MARGIN=0 +NVLINK_DOMAIN_SIZE=72 +NVTE_ALLOW_NONDETERMINISTIC_ALGO=1 +PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +NCCL_NVLS_ENABLE=0 +NVTE_FUSED_ATTN=1 +NVTE_NORM_FWD_USE_CUDNN=1 +NVTE_NORM_BWD_USE_CUDNN=1 +PYTHONWARNINGS=ignore +NCCL_DEBUG=VERSION +NCCL_GRAPH_REGISTER=0 +``` + +### bindpcie + +Download [bindpcie](https://github.com/NVIDIA/mlperf-common/blob/main/client/bindpcie) to your workdir, make it executable, + +```bash +wget https://raw.githubusercontent.com/NVIDIA/mlperf-common/refs/heads/main/client/bindpcie && +chmod 755 bindpcie +``` + +and then + +> [!Important] +> **Place it at the beginning of your launch command in every process.** + +Taking Slurm as an example, your script should look like + +```bash +#!/bin/bash + +#SBATCH [... sbatch args] + +srun [... srun args] /path/to/bindpcie /path/to/pretrain_gpt.py [... mcore arguments] +``` + +This is a very important step on GB200. + +### Launch script + +```bash +/path/to/bindpcie \ +/path/to/megatron-lm/pretrain_gpt.py \ +--distributed-timeout-minutes 60 \ +--tensor-model-parallel-size 1 \ +--pipeline-model-parallel-size 8 \ +--expert-model-parallel-size 32 \ +--context-parallel-size 1 \ +--expert-tensor-parallel-size 1 \ +--use-distributed-optimizer \ +--overlap-grad-reduce \ +--overlap-param-gather \ +--use-mcore-models \ +--sequence-parallel \ +--use-flash-attn \ +--disable-bias-linear \ +--micro-batch-size 1 \ +--global-batch-size 2048 \ +--train-samples 585937500 \ +--exit-duration-in-mins 220 \ +--no-save-optim \ +--no-check-for-nan-in-loss-and-grad \ +--cross-entropy-loss-fusion \ +--cross-entropy-fusion-impl te \ +--manual-gc \ +--manual-gc-interval 10 \ +--enable-experimental \ +--transformer-impl transformer_engine \ +--seq-length 4096 \ +--data-cache-path /path/to/data_cache \ +--tokenizer-type HuggingFaceTokenizer \ +--tokenizer-model unsloth/DeepSeek-V3 \ +--data-path /path/to/data \ +--split 99,1,0 \ +--no-mmap-bin-files \ +--no-create-attention-mask-in-dataloader \ +--num-workers 6 \ +--num-layers 61 \ +--hidden-size 7168 \ +--ffn-hidden-size 18432 \ +--num-attention-heads 128 \ +--kv-channels 128 \ +--max-position-embeddings 4096 \ +--position-embedding-type rope \ +--rotary-base 10000 \ +--make-vocab-size-divisible-by 3232 \ +--normalization RMSNorm \ +--norm-epsilon 1e-6 \ +--swiglu \ +--untie-embeddings-and-output-weights \ +--multi-latent-attention \ +--attention-dropout 0.0 \ +--hidden-dropout 0.0 \ +--clip-grad 1.0 \ +--weight-decay 0.1 \ +--qk-layernorm \ +--lr-decay-samples 584765624 \ +--lr-warmup-samples 1536000 \ +--lr-warmup-init 3.9e-7 \ +--lr 3.9e-6 \ +--min-lr 3.9e-7 \ +--lr-decay-style cosine \ +--adam-beta1 0.9 \ +--adam-beta2 0.95 \ +--num-experts 256 \ +--moe-layer-freq ([0]*3+[1]*58) \ +--moe-ffn-hidden-size 2048 \ +--moe-shared-expert-intermediate-size 2048 \ +--moe-router-load-balancing-type seq_aux_loss \ +--moe-router-topk 8 \ +--moe-grouped-gemm \ +--moe-aux-loss-coeff 1e-4 \ +--moe-router-group-topk 4 \ +--moe-router-num-groups 8 \ +--moe-router-pre-softmax \ +--moe-router-padding-for-quantization \ +--moe-router-topk-scaling-factor 2.5 \ +--moe-router-score-function sigmoid \ +--moe-router-enable-expert-bias \ +--moe-router-bias-update-rate 1e-3 \ +--moe-router-dtype fp32 \ +--moe-permute-fusion \ +--moe-router-fusion \ +--q-lora-rank 1536 \ +--kv-lora-rank 512 \ +--qk-head-dim 128 \ +--qk-pos-emb-head-dim 64 \ +--v-head-dim 128 \ +--rotary-scaling-factor 40 \ +--mscale 1.0 \ +--mscale-all-dim 1.0 \ +--eval-iters 32 \ +--eval-interval 200 \ +--no-load-optim \ +--no-load-rng \ +--auto-detect-ckpt-format \ +--load None \ +--save /path/to/checkpoints \ +--save-interval 500 \ +--dist-ckpt-strictness log_all \ +--init-method-std 0.02 \ +--log-timers-to-tensorboard \ +--log-memory-to-tensorboard \ +--log-validation-ppl-to-tensorboard \ +--log-throughput \ +--log-interval 1 \ +--logging-level 40 \ +--tensorboard-dir /path/to/tensorboard \ +--wandb-project deepseek-v3-benchmarking-v0.15 \ +--wandb-exp-name DeepSeek-V3-TP1PP8EP32CP1VPP4-MBS1GBS2048-v0.15 \ +--bf16 \ +--enable-experimental \ +--recompute-granularity selective \ +--recompute-modules moe_act mlp \ +--cuda-graph-impl transformer_engine \ +--cuda-graph-scope attn moe_router moe_preprocess \ +--te-rng-tracker \ +--pipeline-model-parallel-layout "Et|(tt|)*30L" \ +--moe-router-force-load-balancing \ +--moe-token-dispatcher-type flex \ +--moe-flex-dispatcher-backend hybridep \ +--moe-hybridep-num-sms 32 \ +--fp8-recipe mxfp8 \ +--fp8-format e4m3 \ +--fp8-param-gather \ +--reuse-grad-buf-for-mxfp8-param-ag \ +--use-precision-aware-optimizer \ +--main-grads-dtype fp32 \ +--main-params-dtype fp32 \ +--exp-avg-dtype bf16 \ +--exp-avg-sq-dtype bf16 \ +``` + +### Explanation of arguments + +The following arguments indicate key optimizations. + +- Pipeline parallel layout + +```bash +--pipeline-model-parallel-layout "Et|(tt|)*30L" +``` + +`E` stands for embedding, `t` for transformer layer, `L` for Loss. So it's interpreted as a total of 32 stages, where the first stage is Embedding + 1 transformer layer, the last stage is Loss, and the middle 30 stages are 2 transformer layers. + +- Fine-grained recompute + +```bash +--recompute-granularity selective \ +--recompute-modules moe_act mlp \ +``` + +- Partial CUDA Graphs + +```bash +--cuda-graph-impl transformer_engine \ +--cuda-graph-scope attn moe_router moe_preprocess \ +--te-rng-tracker \ +``` + +- Force load balancing for performance benchmark + +```bash +--moe-router-force-load-balancing \ +``` + +- HybridEP + +```bash +--moe-token-dispatcher-type flex \ +--moe-flex-dispatcher-backend hybridep \ +--moe-hybridep-num-sms 32 \ +``` + +- MXFP8 recipe + +```bash +--fp8-recipe mxfp8 \ +--fp8-format e4m3 \ +--fp8-param-gather \ +--reuse-grad-buf-for-mxfp8-param-ag \ +``` + +- BF16 optimizer states + +```bash +--use-precision-aware-optimizer \ +--main-grads-dtype fp32 \ +--main-params-dtype fp32 \ +--exp-avg-dtype bf16 \ +--exp-avg-sq-dtype bf16 \ +``` + +- Kernel fusions + +```bash +--cross-entropy-loss-fusion \ +--cross-entropy-fusion-impl te \ +--moe-permute-fusion \ +--moe-router-fusion \ +``` + +- Manual GC to make ranks better synchronized + +```bash +--manual-gc \ +--manual-gc-interval 10 \ +``` diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png new file mode 100644 index 00000000000..6e4dad685c4 Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image1.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png new file mode 100644 index 00000000000..920e3c57f94 Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image2.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png new file mode 100644 index 00000000000..f606dbfb744 Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image3.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png new file mode 100644 index 00000000000..04239401edd Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image4.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png new file mode 100644 index 00000000000..0128fc7ae45 Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image5.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png new file mode 100644 index 00000000000..cb2ed2eb9ad Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image6.png differ diff --git a/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png b/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png new file mode 100644 index 00000000000..325d0fd4f52 Binary files /dev/null and b/docs/discussions/deepseek-v3-gb200-optimization/images/image7.png differ diff --git a/docs/source/api-guide/fine_grained_activation_offloading.md b/docs/source/api-guide/fine_grained_activation_offloading.md new file mode 100644 index 00000000000..969098263fc --- /dev/null +++ b/docs/source/api-guide/fine_grained_activation_offloading.md @@ -0,0 +1,31 @@ +# Fine-grained Activation Offloading (collaborated with rednote) + +Memory capacity is more and more important with the rising of extreme sparse MoE models like DeepSeek-V3 and Qwen3-235B. Fine-grained recomputing reduces the memory footprint at the cost of extra recomputation, while offloading could utilize the host-device bandwidth to achieve nearly zero-overhead. Fine-grained Activation Offloading targets at offloading the activation at the granularity of specific modules, so that we can calibrate the amount of offloading activation to maximize the training throughput. + +Currently, the supported offloading modules are `"attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act"`, which could work with fine-grained recomputation to release almost all activations of a transformer layer. + +**Features** +* Support PP=1/PP/Interleaved PP +* Compatible with fine-grained recomputation +* Support FP8 +* Support MTP +* Support mixed dense & moe layer +* Support A2A Overlap +* Support CUDA Graph + * (Temporary) cuda graph scope cannot contains the offloading modules + +**Usage** +```bash +# Enable fine-grained activation offloading +--fine-grained-activation-offloading + +# Specify which modules are going to offload its input +# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". +--offload-modules expert_fc1 +``` +**Compatible with Fine-grained Recomputation** +- For modules with minor perf overhead like layernorm or moe_act, use recomputing to reduce memory footprint; +- For other modules, use offloading to reduce memory footprint; +- Make sure the offloading/reloading could be overlapped with computing; + +![Fine-grained Activation Offloading and Fine-grained Recomputation](../images/fine_grained_activation_offloading/offloading_and_recomputing.png) diff --git a/docs/source/api-guide/index.rst b/docs/source/api-guide/index.rst index 710a7caf4de..ac6d7cb0b2d 100644 --- a/docs/source/api-guide/index.rst +++ b/docs/source/api-guide/index.rst @@ -22,3 +22,4 @@ API Guide optimizer_cpu_offload multi_token_prediction tokenizers + fine_grained_activation_offloading diff --git a/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png b/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png new file mode 100644 index 00000000000..6c8afa78bb1 Binary files /dev/null and b/docs/source/images/fine_grained_activation_offloading/offloading_and_recomputing.png differ diff --git a/examples/inference/gpt/gpt_static_inference.py b/examples/inference/gpt/gpt_static_inference.py index 2dcae5549a6..32a49191b19 100644 --- a/examples/inference/gpt/gpt_static_inference.py +++ b/examples/inference/gpt/gpt_static_inference.py @@ -104,7 +104,7 @@ def get_inference_engine(args: Namespace, model: MegatronModule) -> StaticInfere text_generation_controller = TextGenerationController( inference_wrapped_model=inference_wrapped_model, tokenizer=tokenizer ) - return StaticInferenceEngine(text_generation_controller=text_generation_controller, legacy=args.use_legacy_static_engine) + return StaticInferenceEngine(text_generation_controller=text_generation_controller, legacy=True) async def generate( diff --git a/gpt_builders.py b/gpt_builders.py index 89b228815ff..9fa1aff72c7 100644 --- a/gpt_builders.py +++ b/gpt_builders.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from megatron.core.models.gpt import GPTModel from megatron.core.models.gpt.gpt_layer_specs import ( @@ -6,6 +6,7 @@ get_gpt_layer_local_spec, get_gpt_layer_with_transformer_engine_spec, get_gpt_mtp_block_spec, + get_gpt_decoder_layer_specs, ) from megatron.core.models.gpt.heterogeneous.heterogeneous_layer_specs import ( get_gpt_heterogeneous_layer_spec, @@ -41,7 +42,7 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None): else: use_te = args.transformer_impl == "transformer_engine" - if args.num_experts: + if args.num_experts or (args.linear_attention_type is not None): # Define the decoder block spec transformer_layer_spec = get_gpt_decoder_block_spec( config, @@ -57,18 +58,19 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None): transformer_layer_spec = _get_transformer_layer_spec(use_te, config) mtp_block_spec = None if args.mtp_num_layers is not None: - if ( - hasattr(transformer_layer_spec, 'layer_specs') - and len(transformer_layer_spec.layer_specs) == 0 - ): - # Get the decoder layer spec explicitly if no decoder layer in the last stage, - # Only happens with block spec (TransformerBlockSubmodules) when using MoE. - transformer_layer_spec_for_mtp = _get_transformer_layer_spec(use_te, config) + # Get GPT decoder layer specs for the model. + if args.spec is not None: + mtp_transformer_layer_spec = import_module(args.spec) else: - transformer_layer_spec_for_mtp = transformer_layer_spec + # Define the decoder block spec + decoder_layer_specs = get_gpt_decoder_layer_specs( + config, use_transformer_engine=use_te, normalization=args.normalization, qk_l2_norm=args.qk_l2_norm, vp_stage=vp_stage + ) + mtp_transformer_layer_spec = decoder_layer_specs[-1] + # Use spec of the last layer in decoder block as spec of the transformer layer in MTP mtp_block_spec = get_gpt_mtp_block_spec( config, - transformer_layer_spec_for_mtp, + mtp_transformer_layer_spec, use_transformer_engine=use_te, vp_stage=vp_stage, ) @@ -96,12 +98,12 @@ def gpt_builder(args, pre_process, post_process, vp_stage=None, config=None): def _get_transformer_layer_spec(use_te, config): """Get transformer layer specification based on configuration. - + Args: use_te (bool): Whether to use Transformer Engine args: Training arguments config: Model configuration - + Returns: transformer_layer_spec: The transformer layer specification """ @@ -112,9 +114,11 @@ def _get_transformer_layer_spec(use_te, config): args.moe_grouped_gemm, args.qk_layernorm, args.multi_latent_attention, + args.linear_attention_type, moe_use_legacy_grouped_gemm=args.moe_use_legacy_grouped_gemm, qk_l2_norm=args.qk_l2_norm, use_kitchen=config.use_kitchen, + fallback_to_eager_attn=config.fallback_to_eager_attn, ) else: return get_gpt_layer_local_spec( @@ -122,6 +126,7 @@ def _get_transformer_layer_spec(use_te, config): args.moe_grouped_gemm, args.qk_layernorm, args.multi_latent_attention, + args.linear_attention_type, moe_use_legacy_grouped_gemm=args.moe_use_legacy_grouped_gemm, normalization=args.normalization, use_kitchen=config.use_kitchen, diff --git a/hello_world b/hello_world new file mode 100644 index 00000000000..e69de29bb2d diff --git a/megatron/core/datasets/blended_megatron_dataset_builder.py b/megatron/core/datasets/blended_megatron_dataset_builder.py index b1a000df167..3a663843f9b 100644 --- a/megatron/core/datasets/blended_megatron_dataset_builder.py +++ b/megatron/core/datasets/blended_megatron_dataset_builder.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import math @@ -47,11 +47,13 @@ def __init__( sizes: List[int], is_built_on_rank: Callable, config: BlendedMegatronDatasetConfig, + vp_stage: Optional[int] = None, ): self.cls = cls self.sizes = sizes self.is_built_on_rank = is_built_on_rank self.config = config + self.vp_stage = vp_stage log_single_rank( logger, diff --git a/megatron/core/dist_checkpointing/state_dict_utils.py b/megatron/core/dist_checkpointing/state_dict_utils.py index cfb2379a9d3..59ec18e70ed 100644 --- a/megatron/core/dist_checkpointing/state_dict_utils.py +++ b/megatron/core/dist_checkpointing/state_dict_utils.py @@ -13,7 +13,7 @@ StateDict, apply_factories, ) -from .utils import extract_nonpersistent, extract_sharded_base +from .utils import _clean_metadata_for_serialization, extract_nonpersistent, extract_sharded_base from .validation import determine_global_metadata, validate_sharding_integrity @@ -43,6 +43,11 @@ def save_preprocess( sharded_part = filter_out_empty_flatten_tensor(sharded_part) if validate_access_integrity: preprocessed_common_state_dict = common_state_dict + if "content_metadata" in preprocessed_common_state_dict: + preprocessed_common_state_dict["content_metadata"] = _clean_metadata_for_serialization( + preprocessed_common_state_dict["content_metadata"] + ) + if preprocess_common_before_consistancy_check: preprocessed_common_state_dict = preprocess_common_before_consistancy_check( common_state_dict diff --git a/megatron/core/dist_checkpointing/utils.py b/megatron/core/dist_checkpointing/utils.py index 6dcab0c0dda..161a3477725 100644 --- a/megatron/core/dist_checkpointing/utils.py +++ b/megatron/core/dist_checkpointing/utils.py @@ -330,3 +330,20 @@ def debug_msg(msg: str): """ with logger_stack(None, None) as (stacked_name, last_logger): last_logger.debug(f"{stacked_name} {msg}") + + +def _clean_metadata_for_serialization(metadata: dict) -> dict: + """Create a clean copy of metadata for serialization by removing non-serializable objects. + + Args: + metadata: Original metadata dict + + Returns: + Clean metadata dict suitable for serialization + """ + if metadata is None: + return None + clean_metadata = metadata.copy() + # Remove dp_cp_group as it's not serializable + clean_metadata.pop('dp_cp_group', None) + return clean_metadata diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py index df5bccd71ca..df1d7ae94db 100644 --- a/megatron/core/distributed/distributed_data_parallel.py +++ b/megatron/core/distributed/distributed_data_parallel.py @@ -8,7 +8,7 @@ from .. import parallel_state from ..config_logger import has_config_logger_enabled, log_config_to_disk -from ..fp8_utils import is_float8tensor +from ..fp8_utils import is_float8tensor, post_all_gather_processing from ..process_groups_config import ProcessGroupCollection from ..transformer.cuda_graphs import is_graph_capturing from ..transformer.transformer_config import TransformerConfig @@ -500,26 +500,34 @@ def start_param_sync(self, *unused, force_sync: bool = False, force_dispatch: bo for bucket_group in self.bucket_groups + self.expert_parallel_bucket_groups: bucket_group.start_param_sync(force_sync=force_sync) - # For MXFP8 params, we need to copy the all-gathered param data from the buffer to - # the param.data, since param buffer is not mapped to model params for MXFP8 case. - # The paramaters are cast from bf16 to MXFP8 during copy. - # In the case of "overlap_param_gather=True", the param copy is done - # in "finish_param_sync" stage after zeroing the shared gardient buffers. - if ( - self.ddp_config.reuse_grad_buf_for_mxfp8_param_ag - and not self.ddp_config.overlap_param_gather - ): - for bucket in bucket_group.buckets: - for param in bucket.params: - param_start, param_end = bucket.param_to_index[param] - param_slice = bucket.param_data.view(-1)[param_start:param_end] - param.data.copy_(param_slice.view(param.data.shape)) - # All-gathered params are not needed after being copied to param.data. - # Zero out the param buffer (shared with grad buffer) for gradient accumulation. - # We cannot zero out the entire grad buffer because one grad buffer may - # correspond to multiple param buffers. If we zero out the entire grad buffer, - # it would clear the data of those param buffers that have not yet completed AG. - bucket.param_data.zero_() + + if not self.ddp_config.overlap_param_gather: + # For MXFP8 params, we need to copy the all-gathered param data from the buffer to + # the param.data, since param buffer is not mapped to model params for MXFP8 case. + # The paramaters are cast from bf16 to MXFP8 during copy. + # In the case of "overlap_param_gather=True", the param copy is done + # in "finish_param_sync" stage after zeroing the shared gardient buffers. + if self.ddp_config.reuse_grad_buf_for_mxfp8_param_ag: + for bucket in bucket_group.buckets: + for param in bucket.params: + param_start, param_end = bucket.param_to_index[param] + param_slice = bucket.param_data.view(-1)[param_start:param_end] + param.data.copy_(param_slice.view(param.data.shape)) + # All-gathered params are not needed after being copied to param.data. + # Zero out the param buffer (shared with grad buffer) for gradient + # accumulation. We cannot zero out the entire grad buffer because one grad + # buffer may correspond to multiple param buffers. If we zero out the entire + # grad buffer, it would clear the data of those param buffers that have not + # yet completed AG. + bucket.param_data.zero_() + else: + fp8_params = [] + for bucket in bucket_group.buckets: + for param in bucket.params: + if is_float8tensor(param): + fp8_params.append(param) + if len(fp8_params) > 0: + post_all_gather_processing(fp8_params) def start_grad_sync(self, *unused): """ diff --git a/megatron/core/distributed/finalize_model_grads.py b/megatron/core/distributed/finalize_model_grads.py index 55663acdc10..ddaeb7e8d84 100644 --- a/megatron/core/distributed/finalize_model_grads.py +++ b/megatron/core/distributed/finalize_model_grads.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from functools import partial from typing import Callable, List, Optional, Union @@ -193,7 +193,11 @@ def _allreduce_word_embedding_grads( pp_group = parallel_state.get_pipeline_model_parallel_group() _allreduce_embedding_grad( - model, embd_group, pp_group, partial(_get_shared_word_embedding_weight, config=config) + model, + embd_group, + pp_group, + partial(_get_shared_word_embedding_weight, config=config), + config=config, ) @@ -203,6 +207,7 @@ def _allreduce_embedding_grad( pp_group: torch.distributed.ProcessGroup, weight_getter: Callable[[torch.nn.Module], Optional[torch.nn.Parameter]], skip_if_none: bool = True, + config: TransformerConfig = None, ): """Unified helper to all-reduce embedding parameters across pipeline stages. @@ -229,6 +234,9 @@ def _allreduce_embedding_grad( model_module = model[0] elif is_pp_last_stage(pp_group): model_module = model[-1] + elif getattr(config, 'mtp_num_layers', None) is not None and config.mtp_num_layers > 0: + # Embedding for MTP layers is in the last virtual pipeline model parallel stage. + model_module = model[-1] else: # We do not support an interleaved schedule for models with encoders yet. model_module = model[0] diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py index 806defa5b34..d34fdebaf75 100644 --- a/megatron/core/distributed/param_and_grad_buffer.py +++ b/megatron/core/distributed/param_and_grad_buffer.py @@ -17,7 +17,12 @@ from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.rerun_state_machine import get_rerun_state_machine -from ..fp8_utils import is_float8tensor, is_mxfp8tensor, modify_underlying_storage +from ..fp8_utils import ( + is_float8tensor, + is_mxfp8tensor, + modify_underlying_storage, + post_all_gather_processing, +) from ..utils import is_torch_min_version, log_on_each_pipeline_stage from .distributed_data_parallel_config import DistributedDataParallelConfig from .reduce_scatter_with_fp32_accumulation import reduce_scatter_with_fp32_accumulation @@ -311,10 +316,7 @@ def finish_param_sync(self, skip_next_bucket_dispatch: bool = False): # For the mxfp8_param with "reuse_grad_buf_for_mxfp8_param_ag=True", # we need to copy the param_data from the shared_param/grad_buffer to param.data # after the param all-gather. - if ( - self.ddp_config.reuse_grad_buf_for_mxfp8_param_ag - and self.ddp_config.overlap_param_gather - ): + if self.ddp_config.reuse_grad_buf_for_mxfp8_param_ag: for bucket in self.buckets: for param in bucket.params: param_start, param_end = bucket.param_to_index[param] @@ -326,6 +328,14 @@ def finish_param_sync(self, skip_next_bucket_dispatch: bool = False): # correspond to multiple param buffers. If we zero out the entire grad buffer, # it would clear the data of those param buffers that have not yet completed AG. bucket.param_data.zero_() + else: + fp8_params = [] + for bucket in self.buckets: + for param in bucket.params: + if is_float8tensor(param): + fp8_params.append(param) + if len(fp8_params) > 0: + post_all_gather_processing(fp8_params) def start_grad_sync(self): """ diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index e95409e08e9..85732c0f7ea 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import dataclasses import inspect @@ -42,6 +42,7 @@ from megatron.core.transformer.mlp import MLP from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, is_layer_window_attention, make_sharded_tensors_for_checkpoint, ) @@ -299,6 +300,7 @@ def __init__( extra_kwargs["delay_wgrad_compute"] = self.config.delay_wgrad_compute else: raise RuntimeError("Only TE with version >=2.3.0 supports delay_wgrad_compute now.") + if ( self.config.tp_comm_overlap and tp_comm_buffer_name @@ -419,6 +421,9 @@ def __init__( # duplicated across TP ranks setattr(param, "sequence_parallel", self.config.sequence_parallel) + tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self._tp_group = tp_group + def forward(self, x): """Forward.""" _is_first_microbatch = ( @@ -443,7 +448,14 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): self.parallel_mode is None ), "TELinear sharded_state_dict can only be used with duplicated parallel mode" state_dict = self.state_dict(prefix="", keep_vars=True) - return make_sharded_tensors_for_checkpoint(state_dict, prefix, None, sharded_offsets) + return make_sharded_tensors_for_checkpoint( + state_dict, + prefix, + None, + sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], + ) def backward_dw(self): """Compute weight gradients during the backward pass if delay_wgrad_compute is enabled.""" @@ -491,6 +503,7 @@ def __init__( # TODO: For backward compatibility, remove in v0.15. tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self._tp_group = tp_group # TE returns a zero length Tensor when bias=False and # return_bias=True, but we prefer None. So in that case we @@ -624,9 +637,15 @@ def forward(self, x): def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 0, bias sharded""" + metadata = ensure_metadata_has_dp_cp_group(metadata) state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 0, "bias": 0}, sharded_offsets + state_dict, + prefix, + {"weight": 0, "bias": 0}, + sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], ) def __repr__(self): @@ -669,6 +688,7 @@ def __init__( if gather_output: raise ValueError("Transformer Engine linear layers do not support gather_output = True") tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self._tp_group = tp_group world_size = get_pg_size(tp_group) rank = get_pg_rank(tp_group) @@ -719,7 +739,12 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 0, bias sharded""" state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 0, "bias": 0}, sharded_offsets + state_dict, + prefix, + {"weight": 0, "bias": 0}, + sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], ) def __repr__(self): @@ -763,6 +788,7 @@ def __init__( "Transformer Engine linear layers do not support input_is_parallel = False" ) tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self._tp_group = tp_group super().__init__( input_size=input_size, @@ -813,7 +839,12 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 1, bias not sharded""" state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 1}, sharded_offsets + state_dict, + prefix, + {"weight": 1}, + sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], ) def __repr__(self): @@ -900,6 +931,7 @@ def __init__( assert hasattr( pg_collection, "hcp" ), "TEDotProductAttention pg_collection must have hierarchical cp pg" + self._tp_group = pg_collection.tp if is_te_min_version("0.10.0"): extra_kwargs["attention_type"] = attention_type @@ -989,6 +1021,14 @@ def __init__( self.kept_packed_seq_params.discard("cu_seqlens_q_padded") self.kept_packed_seq_params.discard("cu_seqlens_kv_padded") + if config.qk_clip or config.log_max_attention_logit: + # qk-clip is only supported in TE 2.9.0 and later + assert is_te_min_version("2.9.0"), "qk-clip is only supported in TE 2.9.0 and later" + + # TE 2.9.0 introduces return_max_logit for qk-clip getting the max attention logits + extra_kwargs["return_max_logit"] = True + self.current_max_attn_logits = None + super().__init__( num_attention_heads=self.config.num_attention_heads, kv_channels=kv_channels, @@ -1058,6 +1098,22 @@ def forward( **attention_bias_kwargs, **packed_seq_kwargs, ) + + if self.config.qk_clip or self.config.log_max_attention_logit: + # qk-clip is only supported in TE 2.9.0 and later + assert is_te_min_version("2.9.0"), "qk-clip is only supported in TE 2.9.0 and later" + + # Update Q K outside of TE Attention API + core_attn_out, batch_max_attention_logits = core_attn_out + + # Update QK_Clip balancing eta + if self.current_max_attn_logits is None: + self.current_max_attn_logits = batch_max_attention_logits + else: + self.current_max_attn_logits = torch.max( + self.current_max_attn_logits, batch_max_attention_logits + ) + else: core_attn_out = super().forward( query, key, value, attention_mask, **attention_bias_kwargs, **packed_seq_kwargs @@ -1077,7 +1133,12 @@ def sharded_state_dict( else: state_dict = {} return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {'softmax_offset': 0}, sharded_offsets + state_dict, + prefix, + {'softmax_offset': 0}, + sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], ) @@ -1137,6 +1198,7 @@ def __init__( # The comms between TP and EP group is explicitly handled by MoE token dispatcher. # So we disable comms by making TE agnostic of model parallel. tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self._tp_group = tp_group tp_size = get_pg_size(tp_group) self.explicit_expert_comm = is_expert and (tp_size > 1 or self.expert_parallel) @@ -1371,7 +1433,12 @@ def _sharded_state_dict_grouped( (ep_axis, global_expert_idx, num_global_experts), ) sub_sd = make_sharded_tensors_for_checkpoint( - state_dict, '', tp_axis_map, new_sharded_offsets + state_dict, + '', + tp_axis_map, + new_sharded_offsets, + tp_group=self._tp_group, + dp_cp_group=metadata["dp_cp_group"], ) # Remove expert layers indexing from sharded keys replace_prefix_for_sharding(sub_sd, f"{gemm_idx}.", expert_prefix) @@ -2116,3 +2183,12 @@ def set_save_original_input(module): "set_save_original_input is only needed on transformer-engine modules that save " "quantized tensors by default. It needs transformer-engine>=2.6.0dev0." ) + + +try: + # pylint: disable=unused-import + from transformer_engine.pytorch import cpu_offload + from transformer_engine.pytorch.float8_tensor import Float8Tensor +except ImportError: + Float8Tensor = None + cpu_offload = None diff --git a/megatron/core/extensions/transformer_engine_spec_provider.py b/megatron/core/extensions/transformer_engine_spec_provider.py index c630671ad07..6f8947078b9 100644 --- a/megatron/core/extensions/transformer_engine_spec_provider.py +++ b/megatron/core/extensions/transformer_engine_spec_provider.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from typing import Optional, Tuple @@ -17,6 +17,7 @@ from megatron.core.fusions.fused_layer_norm import FusedLayerNorm from megatron.core.models.backends import BackendSpecProvider from megatron.core.tensor_parallel.layers import ColumnParallelLinear, RowParallelLinear +from megatron.core.transformer.dot_product_attention import DotProductAttention from megatron.core.transformer.mlp import MLPSubmodules from megatron.core.transformer.moe.experts import GroupedMLP, SequentialMLP, TEGroupedMLP from megatron.core.utils import get_te_version, is_te_min_version @@ -25,6 +26,10 @@ class TESpecProvider(BackendSpecProvider): """A protocol for providing the submodules used in Spec building.""" + def __init__(self, fallback_to_eager_attn: bool = False): + super().__init__() + self.fallback_to_eager_attn = fallback_to_eager_attn + def linear(self) -> type: """Which linear module TE backend uses""" return TELinear @@ -56,6 +61,8 @@ def layer_norm(self, rms_norm: bool = False, for_qk: bool = False) -> type: def core_attention(self) -> type: """Which module to use for attention""" + if self.fallback_to_eager_attn: + return DotProductAttention return TEDotProductAttention def grouped_mlp_modules( diff --git a/megatron/core/fp4_utils.py b/megatron/core/fp4_utils.py index 9aebf31c5da..4f997f6be15 100644 --- a/megatron/core/fp4_utils.py +++ b/megatron/core/fp4_utils.py @@ -48,6 +48,20 @@ def is_nvfp4tensor(tensor: torch.Tensor) -> bool: return HAVE_TE_FP4_TENSOR_CLASS and isinstance(tensor, FP4_TENSOR_CLASS) +def get_fp4_align_size(fp4_recipe: Fp4Recipe) -> int: + """ + Get the alignment size required for FP4 GEMM. + FP4 GEMM requires Blackwell and later architectures. + + The value 32 is a hardware requirement: TMA (Tensor Memory Accelerator) requires + a 16-byte aligned address for efficient memory access. Since FP4 uses 4 bits per value, + 16 bytes (128 bits) corresponds to 32 FP4 values. Therefore, the alignment size for FP4 + is 32. With this alignment, NVFP4 GEMM can be performed efficiently. + """ + # pylint: disable=unused-argument + return 32 + + def dequantize_fp4_tensor(fp4_tensor: torch.Tensor) -> torch.Tensor: """Dequantize a fp4 tensor to a higher precision tensor.""" if is_te_min_version("2.7.0.dev0"): diff --git a/megatron/core/fp8_utils.py b/megatron/core/fp8_utils.py index 9a697981738..1c52e965cd7 100644 --- a/megatron/core/fp8_utils.py +++ b/megatron/core/fp8_utils.py @@ -85,6 +85,13 @@ Fp8Padding = None Fp8Unpadding = None +try: + from transformer_engine.pytorch.tensor.utils import ( + post_all_gather_processing as te_post_all_gather_processing, + ) +except ImportError: + te_post_all_gather_processing = None + def is_float8tensor(tensor: torch.Tensor) -> bool: """Check if a tensor is a Transformer Engine Float8Tensor. @@ -247,7 +254,15 @@ def _quantize_param_shard_impl( raise NotImplementedError( f"FSDP with --fp8-param-gather is not supported in TE v{get_te_version()}" ) - cast_master_weights_to_fp8(*args) + + # For newer TE versions (i.e., have post_all_gather_processing function), we keep the + # columnwise data and manually call post_all_gather_processing after all-gather, this + # makes fp8 params compatible with CUDA graph. + kwargs = {} + if te_post_all_gather_processing is not None: + kwargs["manual_post_all_gather_processing"] = True + + cast_master_weights_to_fp8(*args, **kwargs) def _correct_amax_history_if_needed_impl(model: List[torch.nn.Module]) -> None: pass @@ -481,6 +496,20 @@ def correct_amax_history_if_needed(model: List[torch.nn.Module]): _correct_amax_history_if_needed_impl(model) +def post_all_gather_processing(model_params): + """ + Post-processing after all-gather for weights in distributed optimizer. + - tensorwise: may need to create a transposed view to match backend GEMM. + - blockwise: create column-wise storage. + """ + if te_post_all_gather_processing is not None: + te_post_all_gather_processing(model_params) + else: + # If the TE version is old and does not have post_all_gather_processing function, this is + # a no-op, and the transpose/columnwise data will be created in the next forward pass. + pass + + def is_first_last_bf16_layer(config: TransformerConfig, layer_no: int): """Check if the layer is in bf16.""" num_bf16_layers_at_start = ( diff --git a/megatron/core/fusions/fused_indices_converter.py b/megatron/core/fusions/fused_indices_converter.py index 4bba3307738..c7cff48d311 100644 --- a/megatron/core/fusions/fused_indices_converter.py +++ b/megatron/core/fusions/fused_indices_converter.py @@ -6,7 +6,7 @@ import torch from packaging import version -from megatron.core.utils import experimental_fn, null_decorator +from megatron.core.utils import null_decorator try: import triton @@ -279,7 +279,6 @@ def backward(ctx, grad_multihot_indices, grad_probs_in_multihot): return None, grad_probs_indices, None, None -@experimental_fn(introduced_with_version='0.11.0rc0') def fused_indices_to_multihot(indices, probs_indices, num_of_local_experts): """Convert moe topk indices to multihot representation. diff --git a/megatron/core/fusions/fused_mla_yarn_rope_apply.py b/megatron/core/fusions/fused_mla_yarn_rope_apply.py index a1bfea2d78e..1fd5dcfae37 100644 --- a/megatron/core/fusions/fused_mla_yarn_rope_apply.py +++ b/megatron/core/fusions/fused_mla_yarn_rope_apply.py @@ -6,7 +6,7 @@ import torch from packaging import version -from megatron.core.utils import experimental_fn, null_decorator +from megatron.core.utils import null_decorator try: import triton @@ -324,7 +324,6 @@ def backward(ctx, grad): return grad, None, None, None, None, None, None, None, None -@experimental_fn(introduced_with_version="0.13.0") def fused_apply_mla_rope_for_q( t: torch.Tensor, cos: torch.Tensor, @@ -733,7 +732,6 @@ def backward(ctx, dk, dv): return d_kv, d_emb, None, None, None, None, None, None, None, None, None -@experimental_fn(introduced_with_version="0.13.0") def fused_apply_mla_rope_for_kv( kv: torch.Tensor, k_pos_emb: torch.Tensor, diff --git a/megatron/core/fusions/fused_pad_routing_map.py b/megatron/core/fusions/fused_pad_routing_map.py index e7c3a7e48c9..c382178b6c9 100644 --- a/megatron/core/fusions/fused_pad_routing_map.py +++ b/megatron/core/fusions/fused_pad_routing_map.py @@ -1,10 +1,12 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + from unittest.mock import MagicMock import torch from packaging import version -from megatron.core.utils import experimental_fn, null_decorator +from megatron.core.jit import jit_fuser +from megatron.core.utils import null_decorator try: import triton @@ -68,7 +70,7 @@ def _pad_routing_map_kernel( tl.store(output_row_ptr + token_indices, output_row, mask=token_mask) -@experimental_fn(introduced_with_version="0.13.0") +@jit_fuser def fused_pad_routing_map(routing_map: torch.Tensor, pad_multiple: int) -> torch.Tensor: """Fused version of pad_routing_map. Args: diff --git a/megatron/core/inference/text_generation_controllers/text_generation_controller.py b/megatron/core/inference/text_generation_controllers/text_generation_controller.py index 002e72bf0b4..2bda1425710 100644 --- a/megatron/core/inference/text_generation_controllers/text_generation_controller.py +++ b/megatron/core/inference/text_generation_controllers/text_generation_controller.py @@ -851,7 +851,7 @@ def generate_all_output_tokens_static_batch( # Check whether CUDA graphs are enabled enable_cuda_graph = ( model_config.cuda_graph_impl == "local" - and model_config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in model_config.cuda_graph_scope ) # Pad batch tokens if necessary diff --git a/megatron/core/jit.py b/megatron/core/jit.py index b1aa3e0b611..b67810f2e34 100644 --- a/megatron/core/jit.py +++ b/megatron/core/jit.py @@ -7,12 +7,27 @@ jit_fuser = torch.jit.script # nvFuser is deprecated in PyTorch JIT starting from 2.2 -try: - if is_torch_min_version("2.2.0a0"): - jit_fuser = torch.compile -except ImportError: - def noop_decorator(func): - return func +def noop_decorator(func): + '''No-op decorator''' + return func + +def enable_jit_fuser(): + '''Enable the JIT fuser''' + global jit_fuser + try: + if is_torch_min_version("2.2.0a0"): + jit_fuser = torch.compile + except ImportError: + + jit_fuser = noop_decorator + + +def disable_jit_fuser(): + '''Disable the JIT fuser''' + global jit_fuser jit_fuser = noop_decorator + + +enable_jit_fuser() diff --git a/megatron/core/model_parallel_config.py b/megatron/core/model_parallel_config.py index 62d3d940481..e31fcd2577e 100644 --- a/megatron/core/model_parallel_config.py +++ b/megatron/core/model_parallel_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from dataclasses import dataclass @@ -316,6 +316,10 @@ class ModelParallelConfig: rank 1 | 0 1 2 0 1 2 3 4 3 4 """ + mtp_standalone: bool = False + """This will be set automatically according to the pipeline layout, + and will be set to True if MTP is in a separate vpp stage.""" + ################### # CPU Offloading ################### diff --git a/megatron/core/models/bert/bert_model.py b/megatron/core/models/bert/bert_model.py index b7b9bfc73f3..0655a1e6167 100644 --- a/megatron/core/models/bert/bert_model.py +++ b/megatron/core/models/bert/bert_model.py @@ -14,6 +14,7 @@ from megatron.core.models.common.embeddings.language_model_embedding import LanguageModelEmbedding from megatron.core.models.common.embeddings.rotary_pos_embedding import RotaryEmbedding from megatron.core.models.common.language_module.language_module import LanguageModule +from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.dot_product_attention import ( DotProductAttention as MCoreDotProductAttention, ) @@ -73,9 +74,10 @@ def __init__( seq_len_interpolation_factor: Optional[float] = None, add_binary_head=True, return_embeddings=False, + pg_collection: Optional[ProcessGroupCollection] = None, vp_stage: Optional[int] = None, ): - super(BertModel, self).__init__(config=config) + super(BertModel, self).__init__(config=config, pg_collection=pg_collection) if has_config_logger_enabled(config): log_config_to_disk(config, locals(), prefix=type(self).__name__) diff --git a/megatron/core/models/common/embeddings/rope_utils.py b/megatron/core/models/common/embeddings/rope_utils.py index 05cece2c275..e39540eb1d1 100644 --- a/megatron/core/models/common/embeddings/rope_utils.py +++ b/megatron/core/models/common/embeddings/rope_utils.py @@ -268,20 +268,21 @@ def apply_rotary_pos_emb( if config.apply_rope_fusion: if cu_seqlens is None: # NOTE: TE backends do not support mRoPE in bshd format when bs > 1. + use_unfused = False if config.mrope_section is not None and freqs.shape[1] > 1: # TODO: Add a check in TransformerConfig and remove this unfused implementation. warnings.warn( "apply_rope_fusion does not support mRoPE in bshd format when bs > 1. " "Please set apply_rope_fusion to false. This will become an error in v0.16." ) - return _apply_rotary_pos_emb_bshd( - t, - freqs, - rotary_interleaved=config.rotary_interleaved, - multi_latent_attention=config.multi_latent_attention, - mscale=mscale, + use_unfused = True + if mscale != 1.0: + warnings.warn( + f"mscale={mscale} is not supported by TE's fused RoPE. " + "Using unfused implementation." ) - else: + use_unfused = True + if not use_unfused: assert fused_apply_rotary_pos_emb is not None, "apply_rope_fusion is not available." return fused_apply_rotary_pos_emb(t, freqs, interleaved=config.rotary_interleaved) else: @@ -289,25 +290,25 @@ def apply_rotary_pos_emb( return fused_apply_rotary_pos_emb_thd( t, cu_seqlens, freqs, cp_size=cp_group.size(), cp_rank=cp_group.rank() ) + # use unfused implementation + if cu_seqlens is None: + return _apply_rotary_pos_emb_bshd( + t, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=config.multi_latent_attention, + mscale=mscale, + ) else: - if cu_seqlens is None: - return _apply_rotary_pos_emb_bshd( - t, - freqs, - rotary_interleaved=config.rotary_interleaved, - multi_latent_attention=config.multi_latent_attention, - mscale=mscale, - ) - else: - return _apply_rotary_pos_emb_thd( - t, - cu_seqlens, - freqs, - rotary_interleaved=config.rotary_interleaved, - multi_latent_attention=config.multi_latent_attention, - mscale=mscale, - cp_group=cp_group, - ) + return _apply_rotary_pos_emb_thd( + t, + cu_seqlens, + freqs, + rotary_interleaved=config.rotary_interleaved, + multi_latent_attention=config.multi_latent_attention, + mscale=mscale, + cp_group=cp_group, + ) def apply_rotary_pos_emb_with_cos_sin( diff --git a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py index 455a7757d28..bcbb74b0dff 100644 --- a/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py +++ b/megatron/core/models/common/embeddings/yarn_rotary_pos_embedding.py @@ -228,22 +228,25 @@ def _yarn_get_mscale(scale: float = 1, mscale: float = 1) -> float: @lru_cache(maxsize=8) def _yarn_get_concentration_factor( - scaling_factor: float, mscale: float, mscale_all_dim: float + scaling_factor: float, mscale: Optional[float], mscale_all_dim: Optional[float] ) -> float: """ Get the concentration factor (factor multiplied to the sine and cosine components of the embedding). This factor is also known as attention factor, and sometimes homonymously known as "mscale" """ + if mscale is None or mscale_all_dim is None: + return _yarn_get_mscale(scaling_factor) return float( _yarn_get_mscale(scaling_factor, mscale) / _yarn_get_mscale(scaling_factor, mscale_all_dim) ) def _yarn_get_concentration_factor_from_config(config: TransformerConfig) -> float: - fields = ["yarn_rotary_scaling_factor", "yarn_mscale", "yarn_mscale_all_dim"] - if all(hasattr(config, f) for f in fields): + if hasattr(config, "yarn_rotary_scaling_factor"): return _yarn_get_concentration_factor( - config.yarn_rotary_scaling_factor, config.yarn_mscale, config.yarn_mscale_all_dim + config.yarn_rotary_scaling_factor, + getattr(config, "yarn_mscale", None), + getattr(config, "yarn_mscale_all_dim", None), ) return 1.0 diff --git a/megatron/core/models/common/language_module/language_module.py b/megatron/core/models/common/language_module/language_module.py index d855322c2df..de2ecfb8011 100644 --- a/megatron/core/models/common/language_module/language_module.py +++ b/megatron/core/models/common/language_module/language_module.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import os from typing import Optional, Tuple @@ -24,7 +24,12 @@ from megatron.core.transformer.enums import AttnBackend from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.transformer_config import TransformerConfig -from megatron.core.utils import is_te_min_version, make_tp_sharded_tensor_for_checkpoint +from megatron.core.transformer.utils import ensure_metadata_has_dp_cp_group +from megatron.core.utils import ( + get_tensor_model_parallel_group_if_none, + is_te_min_version, + make_tp_sharded_tensor_for_checkpoint, +) class LanguageModule(MegatronModule): @@ -44,6 +49,7 @@ def __init__( pg_collection = ProcessGroupCollection.use_mpu_process_groups() self.pg_collection = pg_collection self.cp_group = pg_collection.cp + self.tp_group = get_tensor_model_parallel_group_if_none(pg_collection.tp) self.pp_group = pg_collection.pp assert hasattr(self.pg_collection, 'embd'), ( "pg_collection must have a embd. In previous version, it used default " @@ -62,6 +68,8 @@ def _is_in_embd_group(self): if torch.distributed.get_rank() in torch.distributed.get_process_group_ranks( self.embd_group ): + if getattr(self, 'mtp_process', False): + return True if ( torch.distributed.get_rank() == torch.distributed.get_process_group_ranks(self.embd_group)[0] @@ -136,7 +144,8 @@ def compute_language_model_loss(self, labels: Tensor, logits: Tensor) -> Tensor: # Use is_cg_capturable=True for full iteration CUDA graphs to avoid torch.equal checks is_cg_capturable = ( hasattr(self.config, 'cuda_graph_scope') - and self.config.cuda_graph_scope == 'full_iteration' + and self.config.cuda_graph_scope + and 'full_iteration' in self.config.cuda_graph_scope ) if is_cg_capturable and not is_te_min_version("2.7.0"): from megatron.core.utils import get_te_version @@ -201,7 +210,10 @@ def setup_embeddings_and_output_layer(self) -> None: ): self.shared_embedding_or_output_weight().shared_embedding = True - if (self.post_process or getattr(self, 'mtp_process', False)) and not self.pre_process: + if ( + (self.post_process and self.share_embeddings_and_output_weights) + or getattr(self, 'mtp_process', False) + ) and not self.pre_process: assert not ( is_vp_first_stage(self.vp_stage, self.vp_size) and is_pp_first_stage(self.pp_group) ) @@ -272,6 +284,10 @@ def sharded_state_dict( ShardedStateDict: sharded state dict for the LanguageModel """ assert not sharded_offsets, "Unexpected sharded offsets" + + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) + sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) first_stage_word_emb_key = f'{prefix}embedding.word_embeddings.weight' @@ -280,7 +296,7 @@ def sharded_state_dict( if self.share_embeddings_and_output_weights: self.tie_embeddings_and_output_weights_state_dict( - sharded_state_dict, output_layer_weight_key, first_stage_word_emb_key + sharded_state_dict, output_layer_weight_key, first_stage_word_emb_key, metadata ) elif self.post_process: # Make sure the output layer follows the embeddings padding logic @@ -297,6 +313,7 @@ def tie_embeddings_and_output_weights_state_dict( sharded_state_dict: ShardedStateDict, output_layer_weight_key: str, first_stage_word_emb_key: str, + metadata: Optional[dict] = None, ) -> None: """Ties the embedding and output weights in a given sharded state dict. @@ -306,9 +323,11 @@ def tie_embeddings_and_output_weights_state_dict( This entry will be replaced with a tied version first_stage_word_emb_key (str): this must be the same as the ShardedTensor.key of the first stage word embeddings. + metadata (Optional[Dict]): metadata controlling sharded state dict creation. Returns: None, acts in-place """ + metadata = ensure_metadata_has_dp_cp_group(metadata) if not self.post_process: # No output layer assert output_layer_weight_key not in sharded_state_dict, sharded_state_dict.keys() @@ -341,4 +360,6 @@ def tie_embeddings_and_output_weights_state_dict( key=first_stage_word_emb_key, replica_id=last_stage_word_emb_replica_id, allow_shape_mismatch=True, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) diff --git a/megatron/core/models/common/model_chunk_schedule_plan.py b/megatron/core/models/common/model_chunk_schedule_plan.py index 6a411ccdcf6..401d9a81a97 100644 --- a/megatron/core/models/common/model_chunk_schedule_plan.py +++ b/megatron/core/models/common/model_chunk_schedule_plan.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from contextlib import nullcontext from typing import Optional @@ -8,13 +8,15 @@ from megatron.core.enums import Fp8Recipe from megatron.core.fp8_utils import get_fp8_context +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.pipeline_parallel.utils import ( AbstractSchedulePlan, NoopScheduleNode, get_comm_stream, get_comp_stream, ) -from megatron.core.transformer.multi_token_prediction import get_mtp_num_layers_to_build class ModelChunkState: @@ -107,7 +109,11 @@ def _build_callable_nodes(self, event, comp_stream, comm_stream, extra_args): if is_mtp else isinstance(self.layer.mlp, MoELayer) ) - enable_deepep = self.layer.config.moe_enable_deepep + + enable_deepep = ( + self.layer.config.moe_token_dispatcher_type == "flex" + and self.layer.config.moe_flex_dispatcher_backend == "deepep" + ) extra_args["enable_deepep"] = enable_deepep extra_args["is_moe"] = is_moe extra_args["delay_wgrad_compute"] = self.layer.config.delay_wgrad_compute @@ -312,37 +318,40 @@ def __init__( self._model_chunk_state.context_mask = None self._model_chunk_state.attention_bias = None - transformer_num_layers = model.decoder.num_layers_per_pipeline_rank - mtp_num_layers = get_mtp_num_layers_to_build(model.config, vp_stage=self.vp_stage) - # build preprocess self.pre_process = PreProcessNode(model, self._model_chunk_state, self._event, comp_stream) - # build layer schedule plan for each layer - for layer_idx in range(transformer_num_layers): - layer = model.decoder._get_layer(layer_idx) - layer_plan = TransformerLayerSchedulePlan( - layer, self._event, self._model_chunk_state, comp_stream, comm_stream + + # build layer schedule plan for each layer. + # The methods to obtain layers are different for MTP so we need the other build plan for + # MTP. Also, this can help annotate MTP layer so that it can know where MTP is. + self._build_layer_schedule_plan(model.decoder, comp_stream, comm_stream) + self._build_layer_schedule_plan(getattr(model, "mtp", None), comp_stream, comm_stream) + + # build post process + if model.post_process: + self.post_process = PostProcessNode( + model, self._model_chunk_state, self._event, comp_stream ) - self._transformer_layers.append(layer_plan) - # build mtp layers - for layer_idx in range(mtp_num_layers): + def _build_layer_schedule_plan(self, module, comp_stream, comm_stream): + if module is None: + return + num_layers = len(module.layers) + for layer_idx in range(num_layers): extra_args = { "is_first_layer": layer_idx == 0, - "is_last_layer": layer_idx == mtp_num_layers - 1, + "is_last_layer": layer_idx == num_layers - 1, } - layer = model.mtp.layers[layer_idx] layer_plan = TransformerLayerSchedulePlan( - layer, self.event, self.state, comp_stream, comm_stream, extra_args + module.layers[layer_idx], + self.event, + self.state, + comp_stream, + comm_stream, + extra_args, ) self._transformer_layers.append(layer_plan) - # build post process - if model.post_process: - self.post_process = PostProcessNode( - model, self._model_chunk_state, self._event, comp_stream - ) - @property def event(self): """Gets the CUDA event for synchronization.""" @@ -446,6 +455,8 @@ def run( f_layer = f_schedule_plan.get_layer(i) b_layer = b_schedule_plan.get_layer(b_num_layers - 1 - i) torch.cuda.nvtx.range_push(f"layer_{i}f-layer_{b_num_layers - 1 - i}b") + if f_layer.layer.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(i == f_num_layers - 1) f_input, b_grad = TransformerLayerSchedulePlan.run( f_layer, b_layer, @@ -468,6 +479,8 @@ def run( for i in range(overlapped_layers, f_num_layers): f_layer = f_schedule_plan.get_layer(i) torch.cuda.nvtx.range_push(f"layer_{i}f") + if f_layer.layer.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(i == f_num_layers - 1) f_input, _ = TransformerLayerSchedulePlan.run(f_layer, None, f_input=f_input) torch.cuda.nvtx.range_pop() diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index b125ee11255..952b83f95fb 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import weakref from contextlib import nullcontext @@ -8,6 +8,11 @@ import torch from megatron.core import tensor_parallel +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.pipeline_parallel.utils import ScheduleNode, make_viewless from megatron.core.transformer.module import float16_to_fp32 from megatron.core.transformer.moe.moe_layer import MoELayer @@ -153,26 +158,19 @@ def forward_impl(self, hidden_states): """Implements the forward pass for postprocessing. This method handles: - 1. Final layer normalization - 2. Output layer computation - 3. Loss computation if labels are provided + 1. Output layer computation + 2. Loss computation if labels are provided Args: hidden_states: The hidden states from the transformer layers. Returns: The logits or loss depending on whether labels are provided. - """ - # Final layer norm from Decoder - if self.gpt_model.decoder.final_layernorm and not self.gpt_model.mtp_process: - hidden_states = self.gpt_model.decoder.final_layernorm(hidden_states) - # TENorm produces a "viewed" tensor. This will result in schedule.py's - # deallocate_output_tensor() throwing an error, so a viewless tensor is - # created to prevent this. - hidden_states = make_viewless_tensor( - inp=hidden_states, requires_grad=True, keep_graph=True - ) + Note: + Final layernorm now has been moved from the post-process stage to the + last decoder layer, so we don't need to run the final layer norm here. + """ # Run GPTModel._postprocess loss = self.gpt_model._postprocess( hidden_states=hidden_states, @@ -246,6 +244,7 @@ def __init__( self.submodule = submodule self.detached = tuple() self.before_detached = tuple() + self.is_mtp = extra_args.get("is_mtp", False) # Create flags to indicate first and last layer self.is_first_layer = extra_args.get("is_first_layer", False) @@ -325,7 +324,10 @@ def build_transformer_layer_callables(layer: TransformerLayer): """ is_moe = isinstance(layer.mlp, MoELayer) - enable_deepep = layer.config.moe_enable_deepep + enable_deepep = ( + layer.config.moe_token_dispatcher_type == "flex" + and layer.config.moe_flex_dispatcher_backend == "deepep" + ) def submodule_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor): """ @@ -347,15 +349,20 @@ def submodule_post_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor) Run forward pass for computations between attention and dispatch: pre mlp layernorm->router->dispatch preprocess """ + if layer.offload_mlp_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") if layer.recompute_pre_mlp_layernorm: layer.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( - layer.pre_mlp_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(layer.offload_mlp_norm): + pre_mlp_layernorm_output = layer.pre_mlp_norm_checkpoint.checkpoint( + layer.pre_mlp_layernorm, hidden_states + ) else: - pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) + with get_fine_grained_offloading_context(layer.offload_mlp_norm): + pre_mlp_layernorm_output = layer.pre_mlp_layernorm(hidden_states) - local_tokens, probs, _ = layer.mlp.router_and_preprocess(pre_mlp_layernorm_output) + probs, routing_map = layer.mlp.route(pre_mlp_layernorm_output) + local_tokens, probs, _ = layer.mlp.preprocess(pre_mlp_layernorm_output, probs, routing_map) # Detach here for mlp_bda residual connection node.layer_state.residual = node.detach(hidden_states) @@ -434,6 +441,10 @@ def submodule_combine_forward( hidden_states = layer.mlp_bda(layer.training, layer.config.bias_dropout_fusion)( mlp_output_with_bias, residual, layer.hidden_dropout ) + if layer.offload_mlp_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="mlp_norm", forced_released_tensors=[residual] + ) output = make_viewless_tensor( inp=hidden_states, requires_grad=hidden_states.requires_grad, keep_graph=True ) @@ -443,6 +454,12 @@ def submodule_combine_forward( # release tensor reference after use node.layer_state.residual = None + + # final layer norm from decoder + final_layernorm = node.chunk_state.model.decoder.final_layernorm + if not node.is_mtp and final_layernorm and node.is_last_layer: + output = final_layernorm(output) + output = make_viewless_tensor(inp=output, requires_grad=True, keep_graph=True) return output def mlp_wrapper(node: ScheduleNode, *args, **kwargs): @@ -482,15 +499,7 @@ def build_mtp_layer_callables(layer): def submodule_mtp_attn_forward(node, hidden_states): # MTP Block Preprocess if node.is_first_layer: - # Final layer norm from Decoder - final_layernorm = node.chunk_state.model.decoder.final_layernorm - if final_layernorm: - hidden_states = final_layernorm(hidden_states) - hidden_states = make_viewless_tensor( - inp=hidden_states, requires_grad=True, keep_graph=True - ) - hidden_states = node.detach(hidden_states) - offset = get_mtp_layer_offset(layer.config) + offset = get_mtp_layer_offset(layer.config, node.chunk_state.model.vp_stage) node.chunk_state.mtp_hidden_states = list(torch.chunk(hidden_states, 1 + offset, dim=0)) hidden_states = node.chunk_state.mtp_hidden_states[offset] diff --git a/megatron/core/models/gpt/gpt_layer_specs.py b/megatron/core/models/gpt/gpt_layer_specs.py index 68c1eb8c953..c5c9caa3d67 100755 --- a/megatron/core/models/gpt/gpt_layer_specs.py +++ b/megatron/core/models/gpt/gpt_layer_specs.py @@ -1,12 +1,16 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from typing import Optional, Union from megatron.core.fusions.fused_bias_dropout import get_bias_dropout_add from megatron.core.models.backends import BackendSpecProvider, LocalSpecProvider +from megatron.core.models.gpt.linear_attention_module_specs import ( + get_linear_attention_module_spec_for_backend, +) from megatron.core.models.gpt.moe_module_specs import get_moe_module_spec_for_backend from megatron.core.transformer.attention import SelfAttention, SelfAttentionSubmodules +from megatron.core.transformer.dot_product_attention import DotProductAttention from megatron.core.transformer.enums import AttnMaskType, LayerType from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.mlp import MLP, MLPSubmodules @@ -74,12 +78,15 @@ def get_gpt_layer_with_transformer_engine_spec( moe_grouped_gemm: Optional[bool] = False, qk_layernorm: Optional[bool] = False, multi_latent_attention: Optional[bool] = False, + linear_attention_type: Optional[str] = None, fp8: Optional[str] = None, # pylint: disable=unused-argument moe_use_legacy_grouped_gemm: Optional[bool] = False, + normalization: Optional[str] = None, qk_l2_norm: Optional[bool] = False, use_te_op_fuser: Optional[bool] = False, use_kitchen: bool = False, use_te_activation_func: bool = False, + fallback_to_eager_attn: bool = False, ) -> ModuleSpec: """Use this spec to use lower-level Transformer Engine modules (required for fp8 training). @@ -88,10 +95,14 @@ def get_gpt_layer_with_transformer_engine_spec( num_experts (int, optional): Number of experts. Defaults to None. moe_grouped_gemm (bool, optional): To use Grouped GEMM. Defaults to False. qk_layernorm (bool, optional): To use layernorm for queries/keys. Defaults to False. + multi_latent_attention (bool, optional): To use multi-latent attention. Defaults to False. + linear_attention_type (str, optional): The type of linear attention. Defaults to None. fp8 (str, optional): Deprecated. For temporary Nemo compatibility. moe_use_legacy_grouped_gemm (bool, optional): Force use the legacy GroupedMLP. Defaults to False. + normalization (str, optional): The normalization to use. Defaults to None. qk_l2_norm (bool, optional): To use l2 norm for queries/keys. Defaults to False. + use_kitchen (bool, optional): To use KitchenSpecProvider. Defaults to False. use_te_op_fuser (bool, optional): Use Transformer Engine's operation-based API, which may enable certain operation fusions. Defaults to False. @@ -107,16 +118,33 @@ def get_gpt_layer_with_transformer_engine_spec( if use_kitchen: assert HAVE_KITCHEN - backend: BackendSpecProvider = KitchenSpecProvider(fallback=TESpecProvider()) + backend: BackendSpecProvider = KitchenSpecProvider( + fallback=TESpecProvider(fallback_to_eager_attn=fallback_to_eager_attn) + ) if use_te_op_fuser: raise AssertionError("use_te_op_fuser not compatible with using kitchen in mlp.") if use_te_activation_func: raise AssertionError("use_te_activation_func not compatible with using kitchen.") else: - backend = TESpecProvider() + backend = TESpecProvider(fallback_to_eager_attn=fallback_to_eager_attn) + + sharded_state_dict_keys_map = {} + + attention = get_attention_module_spec_for_backend( + backend=backend, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, + linear_attention_type=linear_attention_type, + qk_layernorm=qk_layernorm, + qk_l2_norm=qk_l2_norm, + multi_latent_attention=multi_latent_attention, + mla_down_proj_use_column_parallel=False, + normalization=normalization, + fallback_to_eager_attn=fallback_to_eager_attn, + ) mlp = get_mlp_module_spec_for_backend( backend=backend, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, num_experts=num_experts, moe_grouped_gemm=moe_grouped_gemm, moe_use_legacy_grouped_gemm=moe_use_legacy_grouped_gemm, @@ -124,77 +152,13 @@ def get_gpt_layer_with_transformer_engine_spec( use_te_activation_func=use_te_activation_func, ) - if multi_latent_attention: - assert qk_l2_norm is False, "qk_l2_norm is not supported with MLA." - linear_q_up_proj = ( - backend.column_parallel_layer_norm_linear() - if qk_layernorm - else backend.column_parallel_linear() - ) - linear_kv_up_proj = ( - backend.column_parallel_layer_norm_linear() - if qk_layernorm - else backend.column_parallel_linear() - ) - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - input_layernorm=backend.layer_norm(), - self_attention=ModuleSpec( - module=MLASelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=MLASelfAttentionSubmodules( - linear_q_proj=backend.column_parallel_linear(), - linear_q_down_proj=backend.linear(), - linear_q_up_proj=linear_q_up_proj, - linear_kv_down_proj=backend.linear(), - linear_kv_up_proj=linear_kv_up_proj, - core_attention=backend.core_attention(), - linear_proj=backend.row_parallel_linear(), - q_layernorm=IdentityOp, - kv_layernorm=IdentityOp, - ), - ), - self_attn_bda=get_bias_dropout_add, - pre_mlp_layernorm=backend.layer_norm() if num_experts else IdentityOp, - mlp=mlp, - mlp_bda=get_bias_dropout_add, - ), - ) - else: - qk_norm = backend.layer_norm(for_qk=True) - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=SelfAttentionSubmodules( - linear_qkv=backend.column_parallel_layer_norm_linear(), - core_attention=backend.core_attention(), - linear_proj=backend.row_parallel_linear(), - q_layernorm=( - L2Norm if qk_l2_norm else (qk_norm if qk_layernorm else IdentityOp) - ), - k_layernorm=( - L2Norm if qk_l2_norm else (qk_norm if qk_layernorm else IdentityOp) - ), - ), - ), - self_attn_bda=get_bias_dropout_add, - pre_mlp_layernorm=backend.layer_norm() if num_experts else IdentityOp, - mlp=mlp, - mlp_bda=get_bias_dropout_add, - sharded_state_dict_keys_map={ - "mlp.0.weight": "mlp.linear_fc1.layer_norm_weight", - "mlp.0.bias": "mlp.linear_fc1.layer_norm_bias", - "mlp.1.basic_ops.0.weight": "mlp.linear_fc1.weight", - "mlp.1.basic_ops.1.bias": "mlp.linear_fc1.bias", - "mlp.3.basic_ops.0.weight": "mlp.linear_fc2.weight", - "mlp.3.basic_ops.1.bias": "mlp.linear_fc2.bias", - }, - ), - ) + return get_transformer_layer_spec_for_backend( + backend=backend, + attention=attention, + mlp=mlp, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, + normalization=normalization, + ) def get_gpt_layer_local_spec( @@ -202,6 +166,7 @@ def get_gpt_layer_local_spec( moe_grouped_gemm: Optional[bool] = False, qk_layernorm: Optional[bool] = False, multi_latent_attention: Optional[bool] = False, + linear_attention_type: Optional[str] = None, fp8: Optional[str] = None, # pylint: disable=unused-argument moe_use_legacy_grouped_gemm: Optional[bool] = False, normalization: Optional[str] = None, @@ -215,10 +180,14 @@ def get_gpt_layer_local_spec( num_experts (int, optional): Number of experts. Defaults to None. moe_grouped_gemm (bool, optional): To use Grouped GEMM. Defaults to False. qk_layernorm (bool, optional): To use layernorm for queries/keys. Defaults to False. + multi_latent_attention (bool, optional): To use multi-latent attention. Defaults to False. + linear_attention_type (str, optional): The type of linear attention. Defaults to None. fp8 (str, optional): Deprecated. For temporary Nemo compatibility. moe_use_legacy_grouped_gemm (bool, optional): Force use the legacy GroupedMLP. Defaults to False. + normalization (str, optional): The normalization to use. Defaults to None. qk_l2_norm (bool, optional): To use l2 norm for queries/keys. Defaults to False. + use_kitchen (bool, optional): To use KitchenSpecProvider. Defaults to False. Returns: ModuleSpec: Module specification with Megatron-Core modules @@ -229,13 +198,6 @@ def get_gpt_layer_local_spec( backend = KitchenSpecProvider(fallback=LocalSpecProvider()) else: backend = LocalSpecProvider() - # Adjust for RMS norm. - if normalization == "RMSNorm": - layer_norm = backend.layer_norm(rms_norm=True, for_qk=False) - qk_norm = backend.layer_norm(rms_norm=True, for_qk=True) - else: - layer_norm = backend.layer_norm(rms_norm=False, for_qk=False) - qk_norm = backend.layer_norm(rms_norm=False, for_qk=True) if fp8 is not None: warnings.warn( @@ -243,6 +205,23 @@ def get_gpt_layer_local_spec( " and will be removed soon. Please update your code accordingly." ) + if linear_attention_type is not None: + raise NotImplementedError("Linear attention is not supported with local spec yet.") + + sharded_state_dict_keys_map = {} + + attention = get_attention_module_spec_for_backend( + backend=backend, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, + linear_attention_type=linear_attention_type, + qk_layernorm=qk_layernorm, + qk_l2_norm=qk_l2_norm, + multi_latent_attention=multi_latent_attention, + mla_down_proj_use_column_parallel=True, + normalization=normalization, + fallback_to_eager_attn=False, + ) + mlp = get_mlp_module_spec_for_backend( backend=backend, num_experts=num_experts, @@ -250,63 +229,164 @@ def get_gpt_layer_local_spec( moe_use_legacy_grouped_gemm=moe_use_legacy_grouped_gemm, ) + return get_transformer_layer_spec_for_backend( + backend=backend, + attention=attention, + mlp=mlp, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, + normalization=normalization, + ) + + +def get_transformer_layer_spec_for_backend( + backend: BackendSpecProvider, + attention: ModuleSpec, + mlp: ModuleSpec, + sharded_state_dict_keys_map: Optional[dict] = None, + normalization: Optional[str] = None, +) -> ModuleSpec: + """Helper function to get module spec for TransformerLayer""" + + rms_norm = normalization == "RMSNorm" + + input_layernorm = ( + IdentityOp + if attention.metainfo["fuse_input_layernorm"] + else backend.layer_norm(rms_norm=rms_norm, for_qk=False) + ) + pre_mlp_layernorm = ( + IdentityOp + if mlp.metainfo["fuse_pre_mlp_layernorm"] + else backend.layer_norm(rms_norm=rms_norm, for_qk=False) + ) + + transformer_layer = ModuleSpec( + module=TransformerLayer, + submodules=TransformerLayerSubmodules( + input_layernorm=input_layernorm, + self_attention=attention, + self_attn_bda=get_bias_dropout_add, + pre_mlp_layernorm=pre_mlp_layernorm, + mlp=mlp, + mlp_bda=get_bias_dropout_add, + sharded_state_dict_keys_map=sharded_state_dict_keys_map, + ), + ) + return transformer_layer + + +def get_attention_module_spec_for_backend( + backend: BackendSpecProvider, + sharded_state_dict_keys_map: dict, + linear_attention_type: Optional[str] = None, + qk_layernorm: Optional[bool] = False, + qk_l2_norm: Optional[bool] = False, + multi_latent_attention: Optional[bool] = False, + mla_down_proj_use_column_parallel: Optional[bool] = False, + normalization: Optional[str] = None, + fallback_to_eager_attn: Optional[bool] = False, +) -> ModuleSpec: + """Helper function to get module spec for Attention""" + + if linear_attention_type is not None: + return get_linear_attention_module_spec_for_backend( + backend=backend, + linear_attention_type=linear_attention_type, + normalization=normalization, + ) + + # Adjust for RMS norm. + rms_norm = normalization == "RMSNorm" + qk_norm = backend.layer_norm(rms_norm=rms_norm, for_qk=True) + + core_attention = backend.core_attention() if not fallback_to_eager_attn else DotProductAttention if multi_latent_attention: assert qk_l2_norm is False, "qk_l2_norm is not supported with MLA." - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - input_layernorm=layer_norm, - self_attention=ModuleSpec( - module=MLASelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=MLASelfAttentionSubmodules( - linear_q_proj=backend.column_parallel_linear(), - linear_q_down_proj=backend.column_parallel_linear(), - linear_q_up_proj=backend.column_parallel_linear(), - linear_kv_down_proj=backend.column_parallel_linear(), - linear_kv_up_proj=backend.column_parallel_linear(), - core_attention=backend.core_attention(), - linear_proj=backend.row_parallel_linear(), - q_layernorm=qk_norm if qk_layernorm else IdentityOp, - kv_layernorm=qk_norm if qk_layernorm else IdentityOp, - ), - ), - self_attn_bda=get_bias_dropout_add, - pre_mlp_layernorm=layer_norm, - mlp=mlp, - mlp_bda=get_bias_dropout_add, + linear_q_down_proj = ( + backend.column_parallel_linear() + if mla_down_proj_use_column_parallel + else backend.linear() + ) + linear_kv_down_proj = ( + backend.column_parallel_linear() + if mla_down_proj_use_column_parallel + else backend.linear() + ) + linear_q_up_proj = ( + backend.column_parallel_layer_norm_linear() + if qk_layernorm and backend.fuse_layernorm_and_linear() + else backend.column_parallel_linear() + ) + linear_kv_up_proj = ( + backend.column_parallel_layer_norm_linear() + if qk_layernorm and backend.fuse_layernorm_and_linear() + else backend.column_parallel_linear() + ) + qk_norm = ( + backend.layer_norm(rms_norm=rms_norm, for_qk=True) + if qk_layernorm and not backend.fuse_layernorm_and_linear() + else IdentityOp + ) + attention = ModuleSpec( + module=MLASelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=MLASelfAttentionSubmodules( + linear_q_proj=backend.column_parallel_linear(), + linear_q_down_proj=linear_q_down_proj, + linear_q_up_proj=linear_q_up_proj, + linear_kv_down_proj=linear_kv_down_proj, + linear_kv_up_proj=linear_kv_up_proj, + core_attention=core_attention, + linear_proj=backend.row_parallel_linear(), + q_layernorm=qk_norm, + kv_layernorm=qk_norm, ), + metainfo={"fuse_input_layernorm": False}, ) else: - return ModuleSpec( - module=TransformerLayer, - submodules=TransformerLayerSubmodules( - input_layernorm=layer_norm, - self_attention=ModuleSpec( - module=SelfAttention, - params={"attn_mask_type": AttnMaskType.causal}, - submodules=SelfAttentionSubmodules( - linear_qkv=backend.column_parallel_linear(), - core_attention=backend.core_attention(), - linear_proj=backend.row_parallel_linear(), - q_layernorm=( - L2Norm if qk_l2_norm else (qk_norm if qk_layernorm else IdentityOp) - ), - k_layernorm=( - L2Norm if qk_l2_norm else (qk_norm if qk_layernorm else IdentityOp) - ), - ), - ), - self_attn_bda=get_bias_dropout_add, - pre_mlp_layernorm=layer_norm, - mlp=mlp, - mlp_bda=get_bias_dropout_add, - sharded_state_dict_keys_map={ - "input_layernorm.": "self_attention.linear_qkv.layer_norm_", - "pre_mlp_layernorm.": "mlp.linear_fc1.layer_norm_", - }, + linear_qkv = ( + backend.column_parallel_layer_norm_linear() + if backend.fuse_layernorm_and_linear() + else backend.column_parallel_linear() + ) + if qk_l2_norm: + qk_norm = L2Norm + elif qk_layernorm: + qk_norm = backend.layer_norm(rms_norm=rms_norm, for_qk=True) + else: + qk_norm = IdentityOp + attention = ModuleSpec( + module=SelfAttention, + params={"attn_mask_type": AttnMaskType.causal}, + submodules=SelfAttentionSubmodules( + linear_qkv=linear_qkv, + core_attention=core_attention, + linear_proj=backend.row_parallel_linear(), + q_layernorm=qk_norm, + k_layernorm=qk_norm, ), + metainfo={"fuse_input_layernorm": backend.fuse_layernorm_and_linear()}, ) + if backend.fuse_layernorm_and_linear(): + sharded_state_dict_keys_map.update( + { + "mlp.0.weight": "mlp.linear_fc1.layer_norm_weight", + "mlp.0.bias": "mlp.linear_fc1.layer_norm_bias", + "mlp.1.basic_ops.0.weight": "mlp.linear_fc1.weight", + "mlp.1.basic_ops.1.bias": "mlp.linear_fc1.bias", + "mlp.3.basic_ops.0.weight": "mlp.linear_fc2.weight", + "mlp.3.basic_ops.1.bias": "mlp.linear_fc2.bias", + } + ) + else: + sharded_state_dict_keys_map.update( + { + "input_layernorm.": "self_attention.linear_qkv.layer_norm_", + "pre_mlp_layernorm.": "mlp.linear_fc1.layer_norm_", + } + ) + + return attention def _get_mlp_module_spec( @@ -365,6 +445,7 @@ def get_mlp_module_spec( def get_mlp_module_spec_for_backend( backend: BackendSpecProvider, + sharded_state_dict_keys_map: Optional[dict] = None, num_experts: Optional[int] = None, moe_grouped_gemm: Optional[bool] = False, moe_use_legacy_grouped_gemm: Optional[bool] = False, @@ -382,13 +463,16 @@ def get_mlp_module_spec_for_backend( if backend.fuse_layernorm_and_linear(): linear_fc1 = backend.column_parallel_layer_norm_linear() assert linear_fc1 is not None + fuse_pre_mlp_layernorm = True else: linear_fc1 = backend.column_parallel_linear() + fuse_pre_mlp_layernorm = False return ModuleSpec( module=module, submodules=MLPSubmodules( linear_fc1=linear_fc1, linear_fc2=linear_fc2, activation_func=activation_func ), + metainfo={"fuse_pre_mlp_layernorm": fuse_pre_mlp_layernorm}, ) else: # Mixture of experts with modules in megatron core. @@ -401,7 +485,7 @@ def get_mlp_module_spec_for_backend( ) -def get_gpt_decoder_block_spec( +def get_gpt_decoder_layer_specs( config: TransformerConfig, use_transformer_engine: bool, normalization: Optional[str] = None, @@ -409,57 +493,63 @@ def get_gpt_decoder_block_spec( vp_stage: Optional[int] = None, pp_rank: Optional[int] = None, ) -> TransformerBlockSubmodules: - """GPT block spec.""" + """Helper function to get GPT block spec. + + Return a list of transformer layer spec of the current pipeline stage.""" + + get_layer_spec_kwargs = { + "qk_layernorm": config.qk_layernorm, + "moe_use_legacy_grouped_gemm": config.moe_use_legacy_grouped_gemm, + "qk_l2_norm": qk_l2_norm, + "use_kitchen": config.use_kitchen, + "normalization": normalization, + } if use_transformer_engine: layer_norm_impl = TENorm - dense_layer_spec = get_gpt_layer_with_transformer_engine_spec( - num_experts=None, - moe_grouped_gemm=False, - qk_layernorm=config.qk_layernorm, - multi_latent_attention=config.multi_latent_attention, - moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, - qk_l2_norm=qk_l2_norm, - use_kitchen=config.use_kitchen, - use_te_activation_func=config.use_te_activation_func, - ) - moe_layer_spec = get_gpt_layer_with_transformer_engine_spec( - num_experts=config.num_moe_experts, - moe_grouped_gemm=config.moe_grouped_gemm, - qk_layernorm=config.qk_layernorm, - multi_latent_attention=config.multi_latent_attention, - moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, - qk_l2_norm=qk_l2_norm, - use_kitchen=config.use_kitchen, - use_te_activation_func=config.use_te_activation_func, - ) + get_layer_spec_kwargs["use_te_activation_func"] = config.use_te_activation_func + get_layer_spec_kwargs['fallback_to_eager_attn'] = config.fallback_to_eager_attn + get_layer_spec_fn = get_gpt_layer_with_transformer_engine_spec else: layer_norm_impl = LNImpl - dense_layer_spec = get_gpt_layer_local_spec( - num_experts=None, - moe_grouped_gemm=False, - qk_layernorm=config.qk_layernorm, - multi_latent_attention=config.multi_latent_attention, - moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, - normalization=normalization, - qk_l2_norm=qk_l2_norm, - use_kitchen=config.use_kitchen, - ) - moe_layer_spec = get_gpt_layer_local_spec( - num_experts=config.num_moe_experts, - moe_grouped_gemm=config.moe_grouped_gemm, - qk_layernorm=config.qk_layernorm, - multi_latent_attention=config.multi_latent_attention, - moe_use_legacy_grouped_gemm=config.moe_use_legacy_grouped_gemm, - normalization=normalization, - qk_l2_norm=qk_l2_norm, - use_kitchen=config.use_kitchen, - ) + get_layer_spec_fn = get_gpt_layer_local_spec + + layer_spec_dict = {} + for mlp_type in ["dense", "moe"]: + for attention_type in ["softmax_attention", "linear_attention"]: + if mlp_type == "moe": + if config.moe_layer_freq is None: + # Skip if there is no MoE layer in the model. + continue + num_experts = config.num_moe_experts + moe_grouped_gemm = config.moe_grouped_gemm + else: + num_experts = None + moe_grouped_gemm = None + if attention_type == "linear_attention": + if config.linear_attention_type is None: + # Skip if there is no linear attention layer in the model. + continue + linear_attention_type = config.linear_attention_type + multi_latent_attention = None + else: + linear_attention_type = None + multi_latent_attention = config.multi_latent_attention + + layer_spec_key = f"{mlp_type}_{attention_type}" + layer_spec_dict[layer_spec_key] = get_layer_spec_fn( + num_experts=num_experts, + moe_grouped_gemm=moe_grouped_gemm, + multi_latent_attention=multi_latent_attention, + linear_attention_type=linear_attention_type, + **get_layer_spec_kwargs, + ) # Parse config.moe_layer_freq to determine the pattern of expert/dense layers. # 0 stands for dense layers, 1 stands for expert layers. # For integer N: Creates a pattern with one expert layer every N layers. # For string pattern: Evaluates the str directly (e.g. "[1,0,1]" for alternating expert/dense). if isinstance(config.moe_layer_freq, int): + # [1,0,0,...,0,1,0,0,...,0,...] moe_layer_pattern = [ 1 if (i % config.moe_layer_freq == 0) else 0 for i in range(config.num_layers) ] @@ -475,16 +565,65 @@ def get_gpt_decoder_block_spec( f"Invalid moe_layer_freq: {type(config.moe_layer_freq)}, {config.moe_layer_freq}" ) + # Parse config.linear_attention_freq to determine the pattern of expert/dense layers. + # 0 stands for SDPA layers, 1 stands for LA layers. + # For integer N: Creates a pattern with (N-1) LA layers and 1 SDPA layer every N layers. + # For string pattern: Evaluates the str directly (e.g. "[1,0,1]" for alternating LA/SDPA). + if isinstance(config.linear_attention_freq, int): + linear_attention_pattern = [ + # [1,1,...,1,0,1,1,...,1,0,...] + 0 if ((i + 1) % config.linear_attention_freq == 0) else 1 + for i in range(config.num_layers) + ] + elif isinstance(config.linear_attention_freq, list): + linear_attention_pattern = config.linear_attention_freq + assert len(linear_attention_pattern) == config.num_layers, ( + f"Invalid length of linear_attention_pattern: {len(linear_attention_pattern)}, " + f"expected {config.num_layers}, " + f"current linear attention pattern: {config.linear_attention_freq}" + ) + elif config.linear_attention_freq is None: + if config.linear_attention_type is None: + linear_attention_pattern = [0] * config.num_layers + else: + linear_attention_pattern = [1] * config.num_layers + warnings.warn( + "Linear attention type is specified but linear_attention_freq is None. " + "Setting linear_attention_pattern to [1] * config.num_layers as default." + ) + else: + raise ValueError( + f"Invalid linear_attention_freq: {type(config.linear_attention_freq)}," + f" {config.linear_attention_freq}" + ) + # Create the layer specs for the model. layer_specs = [] for layer_number in range(config.num_layers): - if moe_layer_pattern[layer_number] == 1: - layer_specs.append(moe_layer_spec) - elif moe_layer_pattern[layer_number] == 0: - layer_specs.append(dense_layer_spec) - else: - raise ValueError(f"Invalid layer pattern: {moe_layer_pattern}") + mlp_type = "moe" if moe_layer_pattern[layer_number] else "dense" + attention_type = ( + "linear_attention" if linear_attention_pattern[layer_number] else "softmax_attention" + ) + layer_spec_key = f"{mlp_type}_{attention_type}" + if layer_spec_key not in layer_spec_dict: + raise ValueError(f"Invalid layer spec key: {layer_spec_key}") + layer_specs.append(layer_spec_dict[layer_spec_key]) + return layer_specs + + +def get_gpt_decoder_block_spec( + config: TransformerConfig, + use_transformer_engine: bool, + normalization: Optional[str] = None, + qk_l2_norm: Optional[bool] = False, + vp_stage: Optional[int] = None, + pp_rank: Optional[int] = None, +) -> TransformerBlockSubmodules: + """GPT block spec.""" + layer_specs = get_gpt_decoder_layer_specs( + config, use_transformer_engine, normalization, qk_l2_norm + ) # Slice the layer specs to only include the layers that are built in this pipeline stage. # Note: MCore layer_number starts at 1 num_layers_to_build = get_num_layers_to_build(config, vp_stage=vp_stage, pp_rank=pp_rank) @@ -500,6 +639,10 @@ def get_gpt_decoder_block_spec( offset = get_transformer_layer_offset(config, vp_stage=vp_stage, pp_rank=pp_rank) local_layer_specs = layer_specs[offset : offset + num_layers_to_build] + if use_transformer_engine: + layer_norm_impl = TENorm + else: + layer_norm_impl = LNImpl # Block spec. block_spec = TransformerBlockSubmodules( layer_specs=local_layer_specs, layer_norm=layer_norm_impl @@ -518,9 +661,11 @@ def get_gpt_mtp_block_spec( """GPT Multi-Token Prediction (MTP) block spec.""" if use_transformer_engine: backend: BackendSpecProvider = ( - KitchenSpecProvider(fallback=TESpecProvider()) + KitchenSpecProvider( + fallback=TESpecProvider(fallback_to_eager_attn=config.fallback_to_eager_attn) + ) if config.use_kitchen - else TESpecProvider() + else TESpecProvider(fallback_to_eager_attn=config.fallback_to_eager_attn) ) else: backend = ( @@ -559,13 +704,10 @@ def get_gpt_mtp_block_spec_for_backend( mtp_num_layers = config.mtp_num_layers if config.mtp_num_layers else 0 mtp_layer_specs = [mtp_layer_spec] * mtp_num_layers - offset = get_mtp_layer_offset(config) + offset = get_mtp_layer_offset(config, vp_stage=vp_stage) # split the mtp layer specs to only include the layers that are built in this pipeline stage. mtp_layer_specs = mtp_layer_specs[offset : offset + num_layers_to_build] if len(mtp_layer_specs) > 0: - assert ( - len(mtp_layer_specs) == config.mtp_num_layers - ), +f"currently all of the mtp layers must stage in the same pipeline stage." mtp_block_spec = MultiTokenPredictionBlockSubmodules(layer_specs=mtp_layer_specs) else: mtp_block_spec = None diff --git a/megatron/core/models/gpt/gpt_model.py b/megatron/core/models/gpt/gpt_model.py index 25546d36629..e840fca99b3 100644 --- a/megatron/core/models/gpt/gpt_model.py +++ b/megatron/core/models/gpt/gpt_model.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from collections import OrderedDict from typing import Dict, Literal, Optional @@ -18,6 +18,9 @@ ) from megatron.core.models.common.language_module.language_module import LanguageModule from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_init_chunk_handler, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.quantization.utils import get_quant_config_or_none from megatron.core.tensor_parallel import gather_from_sequence_parallel_region @@ -27,7 +30,6 @@ MTPLossLoggingHelper, MultiTokenPredictionBlock, roll_tensor, - tie_output_layer_state_dict, tie_word_embeddings_state_dict, ) from megatron.core.transformer.spec_utils import ModuleSpec @@ -117,6 +119,7 @@ def __init__( self.parallel_output = parallel_output self.share_embeddings_and_output_weights = share_embeddings_and_output_weights self.vp_stage = vp_stage + self.disable_param_offloading = True if hasattr(self.config, 'position_embedding_type'): self.position_embedding_type = self.config.position_embedding_type @@ -246,7 +249,7 @@ def __init__( tp_group=self.pg_collection.tp, ) - if self.pre_process or self.post_process: + if self.pre_process or self.post_process or self.mtp_process: self.setup_embeddings_and_output_layer() if has_config_logger_enabled(self.config): @@ -371,7 +374,7 @@ def _preprocess( and ( ( self.config.cuda_graph_impl == "local" - and self.config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in self.config.cuda_graph_scope ) or self.config.flash_decode ) @@ -409,6 +412,24 @@ def _preprocess( return preproc_output + def preprocess_for_fine_grained_offloading(self): + """Preprocess for fine-grained activation offloading.""" + fine_grained_offloading_init_chunk_handler( + vp_size=self.config.virtual_pipeline_model_parallel_size, + vp_stage=self.vp_stage, + min_offloaded_tensor_size=self.config.min_offloaded_tensor_size, + ) + if self.disable_param_offloading: + for param in self.decoder.parameters(): + param.offloading_activation = False + if self.mtp_process: + for param in self.mtp.parameters(): + param.offloading_activation = False + if self.post_process: + for param in self.output_layer.parameters(): + param.offloading_activation = False + self.disable_param_offloading = False + def forward( self, input_ids: Tensor, @@ -434,6 +455,8 @@ def forward( runtime_gather_output (bool): Gather output at runtime. Default None means `parallel_output` arg in the constructor will be used. """ + if self.config.fine_grained_activation_offloading: + self.preprocess_for_fine_grained_offloading() inference_context = deprecate_inference_params(inference_context, inference_params) @@ -518,7 +541,6 @@ def _postprocess( output_weight = None if self.share_embeddings_and_output_weights: output_weight = self.shared_embedding_or_output_weight() - if mtp_in_postprocess: hidden_states = self.mtp( input_ids=input_ids, @@ -538,7 +560,7 @@ def _postprocess( if not self.post_process: return hidden_states - if self.mtp_process: + if self.config.mtp_num_layers is not None: mtp_labels = labels.clone() hidden_states_list = torch.chunk(hidden_states, 1 + self.config.mtp_num_layers, dim=0) hidden_states = hidden_states_list[0] @@ -580,6 +602,7 @@ def _postprocess( hidden_states, mtp_loss_scale * mtp_loss / num_tokens ) sequence_parallel_override = False + if in_inference_mode and inference_context.materialize_only_last_token_logits: if inference_context.is_static_batching(): hidden_states = hidden_states[-1:, :, :] @@ -698,6 +721,9 @@ def build_schedule_plan( TransformerModelChunkSchedulePlan: The model chunk schedule plan. """ + if self.config.fine_grained_activation_offloading: + self.preprocess_for_fine_grained_offloading() + from ..common.model_chunk_schedule_plan import TransformerModelChunkSchedulePlan return TransformerModelChunkSchedulePlan( @@ -739,27 +765,20 @@ def sharded_state_dict( output_extra_state and output_extra_state.data ), f'Expected output layer extra state to be empty, got: {output_extra_state}' - # Multi-Token Prediction (MTP) need both embedding layer and output layer in - # mtp process stage. + # Multi-Token Prediction (MTP) need embedding layer in mtp process stage. # If MTP is not placed in the pre processing stage, we need to maintain a copy of # embedding layer in the mtp process stage and tie it to the embedding in the pre # processing stage. - # Also, if MTP is not placed in the post processing stage, we need to maintain a copy - # of output layer in the mtp process stage and tie it to the output layer in the post - # processing stage. + # Now MTP loss is computed in post processing stage, so the output_layer is not needed. if self.mtp_process and not self.pre_process: emb_weight_key = f'{prefix}embedding.word_embeddings.weight' emb_weight = self.embedding.word_embeddings.weight - tie_word_embeddings_state_dict(sharded_state_dict, emb_weight, emb_weight_key) - if self.mtp_process and not self.post_process: - # We only need to tie the output layer weight if share_embeddings_and_output_weights - # is False. Because if share_embeddings_and_output_weights is True, the shared weight - # will be stored in embedding layer, and output layer will not have any weight. - if not self.share_embeddings_and_output_weights: - output_layer_weight_key = f'{prefix}output_layer.weight' - output_layer_weight = self.output_layer.weight - tie_output_layer_state_dict( - sharded_state_dict, output_layer_weight, output_layer_weight_key - ) + tie_word_embeddings_state_dict( + sharded_state_dict, + emb_weight, + emb_weight_key, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], + ) return sharded_state_dict diff --git a/megatron/core/models/gpt/linear_attention_module_specs.py b/megatron/core/models/gpt/linear_attention_module_specs.py new file mode 100644 index 00000000000..af23b4b2c08 --- /dev/null +++ b/megatron/core/models/gpt/linear_attention_module_specs.py @@ -0,0 +1,39 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +from typing import Optional + +from megatron.core.models.backends import BackendSpecProvider +from megatron.core.ssm.gated_delta_net import GatedDeltaNet, GatedDeltaNetSubmodules +from megatron.core.ssm.mamba_mixer import MambaMixer, MambaMixerSubmodules +from megatron.core.transformer.spec_utils import ModuleSpec + + +def get_linear_attention_module_spec_for_backend( + backend: BackendSpecProvider, linear_attention_type: str, normalization: Optional[str] = None +) -> ModuleSpec: + """Helper function to get module spec for Linear Attention""" + rms_norm = normalization == "RMSNorm" + if linear_attention_type == "mamba": + attention = ( + ModuleSpec( + module=MambaMixer, + submodules=MambaMixerSubmodules( + in_proj=backend.column_parallel_layer_norm_linear(), + out_proj=backend.row_parallel_linear(), + ), + metainfo={"fuse_input_layernorm": True}, + ), + ) + elif linear_attention_type == "gated_delta_net": + attention = ModuleSpec( + module=GatedDeltaNet, + submodules=GatedDeltaNetSubmodules( + in_proj=backend.column_parallel_layer_norm_linear(), + out_norm=backend.layer_norm(rms_norm=rms_norm, for_qk=False), + out_proj=backend.row_parallel_linear(), + ), + metainfo={"fuse_input_layernorm": True}, + ) + else: + raise ValueError(f"Invalid linear attention type: {linear_attention_type}") + return attention diff --git a/megatron/core/models/gpt/moe_module_specs.py b/megatron/core/models/gpt/moe_module_specs.py index e1ea7c163e9..1de0f14efcd 100755 --- a/megatron/core/models/gpt/moe_module_specs.py +++ b/megatron/core/models/gpt/moe_module_specs.py @@ -65,10 +65,12 @@ def get_moe_module_spec_for_backend( experts = ModuleSpec(module=expert_module, submodules=expert_submodule) # shared experts spec - shared_experts = ModuleSpec(module=SharedExpertMLP, params={"gate": False}, submodules=mlp) + shared_experts = ModuleSpec(module=SharedExpertMLP, submodules=mlp) # MoE module spec moe_module_spec = ModuleSpec( - module=MoELayer, submodules=MoESubmodules(experts=experts, shared_experts=shared_experts) + module=MoELayer, + submodules=MoESubmodules(experts=experts, shared_experts=shared_experts), + metainfo={"fuse_pre_mlp_layernorm": False}, ) return moe_module_spec diff --git a/megatron/core/models/multimodal/llava_model.py b/megatron/core/models/multimodal/llava_model.py index 2ac2657c1cd..dae9a02b780 100644 --- a/megatron/core/models/multimodal/llava_model.py +++ b/megatron/core/models/multimodal/llava_model.py @@ -124,6 +124,7 @@ def __init__( max_num_tiles: int = 0, tokenizer_type: str = "", vp_stage: Optional[int] = None, + use_vision_backbone_fp8_arch: bool = False, ) -> None: super().__init__(config=language_transformer_config) @@ -295,7 +296,7 @@ def __init__( ln_post_impl = None use_mask_token = False - if vision_transformer_config.fp8: + if vision_transformer_config.fp8 or use_vision_backbone_fp8_arch: # FP8 padding for final sequence length to be a multiple of 16 or 32. class_token_len = 32 if vision_transformer_config.fp8_recipe == "mxfp8" else 16 diff --git a/megatron/core/optimizer/cpu_offloading/hybrid_optimizer.py b/megatron/core/optimizer/cpu_offloading/hybrid_optimizer.py index 6f9999f0803..28487c3b367 100644 --- a/megatron/core/optimizer/cpu_offloading/hybrid_optimizer.py +++ b/megatron/core/optimizer/cpu_offloading/hybrid_optimizer.py @@ -52,7 +52,7 @@ def __init__( pin_cpu_grads: bool = True, pin_cpu_params: bool = True, overlap_cpu_optimizer_d2h_h2d: bool = True, - **kwargs + **kwargs, ): super(HybridDeviceOptimizer, self).__init__( params, diff --git a/megatron/core/optimizer/distrib_optimizer.py b/megatron/core/optimizer/distrib_optimizer.py index 8b4740516e2..6e093f96f7e 100644 --- a/megatron/core/optimizer/distrib_optimizer.py +++ b/megatron/core/optimizer/distrib_optimizer.py @@ -1153,7 +1153,8 @@ def _param_name(self, param: torch.nn.Parameter) -> str: "Ensure that each model chunk has unique parameter names." ) name_to_param.update(_name_to_param) - name_to_param = handle_experts_in_state_dict(name_to_param) + num_experts = self.model_chunks[0].config.num_moe_experts if self.model_chunks else None + name_to_param = handle_experts_in_state_dict(name_to_param, num_experts) self.param_to_name = {param: name for name, param in name_to_param.items()} assert ( param in self.param_to_name diff --git a/megatron/core/optimizer/layer_wise_optimizer.py b/megatron/core/optimizer/layer_wise_optimizer.py new file mode 100644 index 00000000000..e43601d90fd --- /dev/null +++ b/megatron/core/optimizer/layer_wise_optimizer.py @@ -0,0 +1,281 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +from typing import Callable, List, Optional + +import torch +from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors + +from megatron.core.dist_checkpointing.dict_utils import nested_values +from megatron.core.dist_checkpointing.mapping import ShardedStateDict +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.utils import get_pg_rank, get_pg_size + +from .clip_grads import count_zeros_fp32, get_grad_norm_fp32 +from .optimizer import ( + ChainedOptimizer, + Float16OptimizerWithFloat16Params, + FP32Optimizer, + MegatronOptimizer, +) +from .optimizer_config import OptimizerConfig + + +class LayerWiseDistributedOptimizer(ChainedOptimizer): + """Layer-wise distributed optimizer for Megatron-core models. + + Experimental distributed optimizer wrapper that distributes weight to DP ranks by layer. + Implemented as ChainedOptimizer to support multiple optimizers (e.g. muon + adamW) + When using, keep all megatron distributed-optimizer related options OFF. + + How LayerWiseDistributedOptimizer work: + 1. weights are splited into lists and each rank only keep its shard in its optimizer + 2. Megatron DDP handle allreduce grad, note that each rank have full model and grad + 3. optimizer is already modified so only param belong to this DP rank is updated + 4. grad_norm and zero counting will reduce metrics globally in step function + 5. Do regular update with chained optimizers, modified optimizer only update shard + 6. allgather updated params to every rank + """ + + def __init__( + self, + optimizers: List[MegatronOptimizer], + config: OptimizerConfig, + pg_collection: Optional[ProcessGroupCollection] = None, + init_state_fn_list: Optional[List[Callable]] = None, + ) -> None: + """ + Initialize LayerWiseDistributedOptimizer. + + Args: + optimizers: List of MegatronOptimizers. + config: OptimizerConfig. + pg_collection: ProcessGroupCollection. + init_state_fn_list: List of init state functions. + """ + + self.pg_collection = pg_collection + self.shard_params(optimizers) + if init_state_fn_list: + assert len(init_state_fn_list) == len( + optimizers + ), "init_state_fn_list must be the same length as optimizers if provided" + + # wrap optimizer after sharding to avoid unnecessary master weight creation + # for higher precision, optimizers are wrapped with megatron already + if config.bf16: + # unwrap FP32 optimizer, possibly from reusing get_megatron_optimizer for adam + for i in range(len(optimizers)): + opt = optimizers[i] + if isinstance(opt, Float16OptimizerWithFloat16Params): + raise TypeError( + 'LayerWiseDistributedOptimizer received Float16 optimizer already.' + ) + # unwrap FP32 optimizer from reusing get_megatron_optimizer for adam + if isinstance(opt, FP32Optimizer): + opt = opt.optimizer + optimizers[i] = Float16OptimizerWithFloat16Params( + opt, config, None, init_state_fn_list[i] if init_state_fn_list else None + ) + + super().__init__(optimizers) + + # TODO(kunlun, deyuf): potential future perf optimization + # since allreduce is unchanged and handled by megatron DDP, they're already in + # contiguous gbuf. So instead of shard param by layer randomly, we can shard by + # buf range but keep some "extras" to keep boundary weight not sharded. + # This way each rank do some duplicated work but allgather_v is no longer needed + # All current distopt optimization can also be potentially applied + + def shard_params(self, optimizers): + """Shard all params into lists by rank.""" + # list of parameter are sorted by numel and assigned to ranks in ping-pong style + # example of 4 ranks and 10 parameters p0-p9 after sorting, then dp_cp_params_list will be + # [[p0, p7, p8], [p1, p6, p9], [p2, p5], [p3, p4]] + + # simplify when dp_cp group size is 1 + if get_pg_size(self.pg_collection.dp_cp) == 1: + self.dp_cp_params_list = None + self.expt_dp_params_list = None + return + + dp_cp_idx, expt_dp_idx = 0, 0 + dp_cp_size = get_pg_size(self.pg_collection.dp_cp) + expt_dp_size = get_pg_size(self.pg_collection.expt_dp) + # create ping-pong style loop so memory is more balanced + dp_cp_loop = list(range(dp_cp_size)) + list(range(dp_cp_size))[::-1] + expt_dp_loop = list(range(expt_dp_size)) + list(range(expt_dp_size))[::-1] + self.dp_cp_params_list = [[] for _ in range(dp_cp_size)] + self.expt_dp_params_list = [[] for _ in range(expt_dp_size)] + # get all param groups + param_groups = [] + for optimizer in optimizers: + param_groups += optimizer.param_groups + + # sort param in all groups by param numel and assign to each rank evenly + param_list = [] + for group_index, group in enumerate(param_groups): + for p in group["params"]: + param_list.append((p, group_index)) + param_list.sort(key=lambda x: x[0].numel()) + param_groups_this_rank = [[] for g in param_groups] + + # assign params to rank in ping-pong style loop + for p, group_index in param_list: + if param_groups[group_index].get("is_expert_parallel", False): + if expt_dp_loop[expt_dp_idx] == get_pg_rank(self.pg_collection.expt_dp): + param_groups_this_rank[group_index].append(p) + self.expt_dp_params_list[expt_dp_loop[expt_dp_idx]].append(p) + expt_dp_idx = (expt_dp_idx + 1) % len(expt_dp_loop) + else: + if dp_cp_loop[dp_cp_idx] == get_pg_rank(self.pg_collection.dp_cp): + param_groups_this_rank[group_index].append(p) + self.dp_cp_params_list[dp_cp_loop[dp_cp_idx]].append(p) + dp_cp_idx = (dp_cp_idx + 1) % len(dp_cp_loop) + + # now we modify the group to only handle local params + for groups, params in zip(param_groups, param_groups_this_rank): + groups["params"] = params + + # simplify when expt_dp group size is 1 or expert parallel is off + if expt_dp_size == 1 or len(self.expt_dp_params_list[0]) == 0: + self.expt_dp_params_list = None + + @torch.no_grad() + def allgather_params(self) -> None: + """All-gather updated params from all ranks.""" + + # helper function to flatten local params, allgather, unflatten and copy to model params + def _allgather_helper(params_list, group): + # flatten this rank's params and create empty tensor output list + device = params_list[0][0].device + dtype = params_list[0][0].dtype + rank = get_pg_rank(group) + # for rank without params create empty tensor and participate in allgather + src = ( + _flatten_dense_tensors(params_list[rank]) + if len(params_list[rank]) > 0 + else torch.empty(0, device=device, dtype=dtype) + ) + output_list = [ + torch.empty(sum([p.numel() for p in params]), device=device, dtype=dtype) + for params in params_list + ] + # single all_gather_v to collect all updated params + torch.distributed.all_gather(output_list, src, group=group) + # unflatten and copy gathered params for each rank i + for idx, (flat_params, params) in enumerate(zip(output_list, params_list)): + # skip local params and empty tensors + if len(params) == 0 or idx == rank: + continue + updated_params = _unflatten_dense_tensors(flat_params, params) + for updated_p, model_p in zip(updated_params, params): + model_p.data.copy_(updated_p) + + if self.pg_collection is None: + return + if self.dp_cp_params_list: + _allgather_helper(self.dp_cp_params_list, self.pg_collection.dp_cp) + if self.expt_dp_params_list: + _allgather_helper(self.expt_dp_params_list, self.pg_collection.expt_dp) + + @torch.no_grad() + def broadcast_params(self): + """All rank broadcast updated local params.""" + # Broadcast linear layer weights to all other ranks. Kept as reference test. + if self.dp_cp_params_list is None: + return + for i, params in enumerate(self.dp_cp_params_list): + src_global_rank = torch.distributed.get_global_rank(self.pg_collection.dp_cp, i) + for p in params: + torch.distributed.broadcast(p, src_global_rank, self.pg_collection.dp_cp) + if self.expt_dp_params_list is None: + return + for i, params in enumerate(self.expt_dp_params_list): + src_global_rank = torch.distributed.get_global_rank(self.pg_collection.expt_dp, i) + for p in params: + torch.distributed.broadcast(p, src_global_rank, self.pg_collection.expt_dp) + + @torch.no_grad() + def get_grad_norm(self): + # similar to dist opt, always aggregate globally + grads_for_norm = [] + for optimizer in self.chained_optimizers: + grads_for_norm += optimizer.get_main_grads_for_grad_norm() + grad_norm = get_grad_norm_fp32(grads_for_norm, grad_stats_parallel_group=None) + return grad_norm + + @torch.no_grad() + def count_zeros(self): + params = [] + for optimizer in self.chained_optimizers: + params += optimizer.get_parameters() + return count_zeros_fp32( + params, + grad_stats_parallel_group=None, + use_decoupled_grad=self.config.use_precision_aware_optimizer_no_fp8_or_ds_fp8, + ) + + @torch.no_grad() + def step(self): # type: ignore[no-untyped-def] + """step function for layer-wise optimizer.""" + update_successful, grad_norm, num_zeros_in_grad = super().step() + + # All gather updated params. + self.allgather_params() + + return update_successful, grad_norm, num_zeros_in_grad + + def sharded_state_dict( + self, model_sharded_state_dict: ShardedStateDict, is_loading: bool = False, **kwargs + ): + """ + Sharded state dict for torch_dist format checkpointing. + For fixed DP usage only, set replica_id to 0 for all ShardedTensor. + """ + sharded_state_dict = super().sharded_state_dict( + model_sharded_state_dict, is_loading, **kwargs + ) + + # for fixed DP usage only + for sh_base in nested_values(sharded_state_dict): + if hasattr(sh_base, 'replica_id'): + assert ( + isinstance(sh_base.replica_id, int) or len(sh_base.replica_id) == 3 + ), f'Expected replica_id as int or (PP, TP, DP), got: {sh_base}' + sh_base.replica_id = ( + 0 if isinstance(sh_base.replica_id, int) else (*sh_base.replica_id[:2], 0) + ) + + if len(self.chained_optimizers) == 1: + wrapped_sharded_state_dict = {1: sharded_state_dict} + else: + wrapped_sharded_state_dict = sharded_state_dict + # Adjust dict due to possible empty rank 0 which output common_dict + for sd in wrapped_sharded_state_dict.values(): + # Drop empty group state to avoid save in common dict (non-empty rank still save) + if 'fp32_from_fp16_params' in sd: + sd['fp32_from_fp16_params'][:] = [ + group for group in sd['fp32_from_fp16_params'] if group + ] + # TODO(deyuf): 'common_step' code path is broken and 'step' is saved in 'param_groups' + # Find next 'step' if present. note this still break if rank0 adam is fully empty + step = next( + (group['step'] for group in sd['optimizer']['param_groups'] if 'step' in group), + None, + ) + if step is not None: + for group in sd['optimizer']['param_groups']: + group['step'] = step + + return sharded_state_dict + + def save_state_dict_to_file(self, filename: str) -> None: + """Save the parameter state of the optimizer. For torch format only. + Args: + filename: The filename to save the parameter state. + """ + torch.save(super().state_dict(), filename) + + def load_state_dict_from_file(self, filename: str) -> None: + """Load the parameter state of the optimizer. For torch format only.""" + super().load_state_dict(torch.load(filename)) diff --git a/megatron/core/optimizer/muon.py b/megatron/core/optimizer/muon.py new file mode 100644 index 00000000000..a31c84a6e8a --- /dev/null +++ b/megatron/core/optimizer/muon.py @@ -0,0 +1,353 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +"""Megatron muon optimizer wrapper to handle tensor-parallel.""" + +import logging +from typing import Any, Callable, List, Literal, Optional + +import torch +from torch.optim.optimizer import ParamsT + +from megatron.core import parallel_state +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.module import MegatronModule +from megatron.core.utils import get_pg_size, log_single_rank + +from . import _get_param_groups, get_megatron_optimizer +from .layer_wise_optimizer import LayerWiseDistributedOptimizer +from .optimizer import ( + ChainedOptimizer, + Float16OptimizerWithFloat16Params, + FP32Optimizer, + MegatronOptimizer, +) +from .optimizer_config import OptimizerConfig + +try: + from emerging_optimizers.orthogonalized_optimizers import ( + OrthogonalizedOptimizer, + get_muon_scale_factor, + ) + from emerging_optimizers.orthogonalized_optimizers.muon_utils import newton_schulz_tp + + HAVE_EMERGING_OPTIMIZERS = True +except ImportError: + HAVE_EMERGING_OPTIMIZERS = False + OrthogonalizedOptimizer = object + + +logger = logging.getLogger(__name__) + + +class TensorParallelMuon(OrthogonalizedOptimizer): + """Tensor Parallel Muon optimizer.""" + + def __init__( + self, + params: ParamsT, + lr: float = 3e-4, + momentum_beta: float = 0.95, + use_nesterov: bool = True, + weight_decay: float = 0.01, + use_decoupled_weight_decay: bool = True, + split_qkv: bool = False, + is_qkv_fn: Callable[[torch.Tensor], bool] | None = None, + qkv_split_shapes: tuple[int, int, int] | None = None, + fp32_matmul_prec: str = "medium", + coefficient_type: str = "quintic", + num_ns_steps: int = 5, + scale_mode: str = "spectral", + extra_scale_factor: float = 1.0, + pg_collection: Optional[ProcessGroupCollection] = None, + mode: Literal["blockwise", "duplicated", "distributed"] = "duplicated", + ) -> None: + if num_ns_steps < 1: + raise ValueError(f"num_ns_steps must be at least 1, got {num_ns_steps}") + + def scaled_orthogonalize_fn( + grad: torch.Tensor, + tp_group: torch.distributed.ProcessGroup, + partition_dim: int | None = None, + ) -> torch.Tensor: + log_single_rank( + logger, + logging.DEBUG, + f'Orthogonalizing grad with {num_ns_steps} steps, {coefficient_type} coefficient, ' + f'{scale_mode} scale mode, extra_scale_factor={extra_scale_factor}', + ) + size = [grad.size(-2), grad.size(-1)] + if partition_dim: + size[partition_dim] *= get_pg_size(tp_group) + orth_grad = newton_schulz_tp( + grad, + steps=num_ns_steps, + coefficient_type=coefficient_type, + tp_group=tp_group, + partition_dim=partition_dim, + mode="duplicated" if mode == "blockwise" else mode, + ) + scale_factor = get_muon_scale_factor(size[0], size[1], mode=scale_mode) + return orth_grad * scale_factor * extra_scale_factor + + self.pg_collection = pg_collection + self.mode = mode + self.split_qkv = split_qkv + self.is_qkv_fn = is_qkv_fn + self.qkv_split_shapes = qkv_split_shapes + + super().__init__( + params, + lr, + momentum_beta, + use_nesterov, + weight_decay, + use_decoupled_weight_decay, + fp32_matmul_prec, + scaled_orthogonalize_fn, + ) + + def orthogonalize(self, p: torch.Tensor, grad: torch.Tensor, **kwargs: Any) -> torch.Tensor: + """Orthogonalize the momentum. + + Args: + p: The parameter tensor. i is necessary to pass param tensor in addition to momentum + because a lot of information is only available in the param tensor, + attributes for example. + grad: The momentum tensor. + + Returns: + The orthogonalized gradient tensor. + """ + # TODO(deyuf): switch to group + if self.pg_collection: + tp_group = ( + self.pg_collection.expt_tp + if getattr(p, 'expert_tp', False) + else self.pg_collection.tp + ) + else: + tp_group = None + partition_dim = None if self.mode == "blockwise" else getattr(p, "partition_dim", None) + if partition_dim == -1: + # llm-shower use different default value for partition_dim than TE. + # Because -1 is a valid index for ndarray, we decided to not overload it. + partition_dim = None + + if self.split_qkv and self.is_qkv_fn(p): # type: ignore[misc] + # split grouped attention parameters (e.g., QKV, GQA, etc.) + grad_shape = grad.shape + log_single_rank( + logger, + logging.DEBUG, + f'qkv split grad shape {grad_shape}, split shapes {self.qkv_split_shapes}', + ) + num_query_groups = grad_shape[0] // sum(self.qkv_split_shapes) + qkv_grads = torch.split( + grad.view(num_query_groups, sum(self.qkv_split_shapes), -1), + self.qkv_split_shapes, + dim=1, + ) + qkv_grads = [g.reshape(-1, grad_shape[-1]) for g in qkv_grads] + + # Apply Newton-Schulz and scales to each component, concat back + qkv_grads = [ + self.scaled_orthogonalize_fn(g, tp_group, partition_dim).view( + num_query_groups, -1, grad_shape[-1] + ) + for g in qkv_grads + ] + grad = torch.cat(qkv_grads, dim=1).view(grad_shape) + else: + grad = self.scaled_orthogonalize_fn(grad, tp_group, partition_dim) + return grad + + +def get_megatron_muon_optimizer( + config: OptimizerConfig, + model_chunks: List[MegatronModule], + no_weight_decay_cond: Optional[Callable] = None, + scale_lr_cond: Optional[Callable] = None, + lr_mult: float = 1.0, + use_gloo_process_groups: bool = True, + layer_wise_distributed_optimizer: bool = False, + pg_collection: Optional[ProcessGroupCollection] = None, +) -> MegatronOptimizer: + """This function is used to get the muon optimizer for the model chunks. + It is used to get the muon optimizer for the model chunks. + + Args: + config (OptimizerConfig): optimizer configuration object. + model_chunks (List[MegatronModule]): model chunks to get optimizer for. + no_weight_decay_cond (func, optional): function to determine whether a parameter + should not perform weight decay. Defaults to None. + scale_lr_cond (func, optional): function to determine whether a parameter + should have a scaled learning rate. Defaults to None. + lr_mult (float, optional): learning rate multiplier for parameters that + satisfy scale_lr_cond. Defaults to 1.0. + use_gloo_process_groups (bool): if false, disable use of Gloo process groups + in underlying Megatron optimizers. + layer_wise_distributed_optimizer (bool): if true, use layer-wise distributed optimizer. + Defaults to False. + """ + assert HAVE_EMERGING_OPTIMIZERS, "Emerging Optimizers is not installed." + + # dist-optim is not supported due to strong coupling with how DDP init grad buffer + # in thoery we can put some weight to use non-dist-muon and rest to dist-adam + # but there are strong dependency and assumption in DDP that prevent it + if config.use_distributed_optimizer: + raise Exception('muon with dist optimizer is not supported.') + + # before this function receive properly created collection + if pg_collection is None: + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True) + pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group() + + log_single_rank(logger, logging.INFO, f'Setting up emerging optimizer with config {config}') + + optimizers = [] + # record list of non/linear params + linear_params = [] + nonlinear_params = [] + + for model_chunk in model_chunks: + # use config to determine qkv split shapes. + # no need to check tp since tp splits by head and this is per head(group) dimension + num_attention_heads = model_chunk.config.num_attention_heads + num_query_groups = model_chunk.config.num_query_groups + kv_channels = model_chunk.config.kv_channels + qkv_split_shapes = [ + num_attention_heads // num_query_groups * kv_channels, + kv_channels, + kv_channels, + ] + for name, param in model_chunk.named_parameters(): + if not param.requires_grad: + continue + # add flag for expert weight so optimizer can figure which tp group it uses + # alternatively, create new param group and save tp_group. this require more + # change in optimizer + if 'experts' in name and 'shared' not in name: + param.expert_tp = True + # add flag for qkv parameter + # TODO(deyuf): support MLA + if 'linear_qkv.weight' in name and len(param.shape) == 2: + param.is_qkv = True + # TODO(deyuf): might not be sufficient for future algorithm. revisit this conditioning + if not getattr(param, 'is_embedding_or_output_parameter', False) and not ( + len(param.shape) == 1 + ): + linear_params.append(param) + else: + nonlinear_params.append(param) + + # freezing nonlinear params and get param groups for muon + for param in nonlinear_params: + param.requires_grad = False + + linear_param_groups = _get_param_groups( + model_chunks, + no_weight_decay_cond, + scale_lr_cond, + lr_mult, + lr=config.lr, + min_lr=config.min_lr, + decoupled_lr=config.decoupled_lr, + decoupled_min_lr=config.decoupled_min_lr, + ) + + optimizer = TensorParallelMuon( + linear_param_groups, + lr=config.lr, + momentum_beta=config.muon_momentum, + use_nesterov=config.muon_use_nesterov, + weight_decay=config.weight_decay, + fp32_matmul_prec=config.muon_fp32_matmul_prec, + num_ns_steps=config.muon_num_ns_steps, + scale_mode=config.muon_scale_mode, + split_qkv=config.muon_split_qkv, + is_qkv_fn=lambda p: getattr(p, 'is_qkv', False), + qkv_split_shapes=qkv_split_shapes, + extra_scale_factor=config.muon_extra_scale_factor, + pg_collection=pg_collection, + mode=config.muon_tp_mode, + ) + + # set config here to: + # 1. get adam for rest of layer + # 2. avoid ChainedOptimizer check fail that assert all optimizers are same kind + # side effect is muon optimizer will have wrong name str, i.e. config.optimizer == 'adam' + # TODO(deyuf): allow user to select optimizer mix and relax ChainedOptimizer design + config.optimizer = 'adam' + + # Needed for torch_dist ckpt_format, unlike torch ckpt_format + # For other emerging optimizers, need to implement init_state_fn as well + # TODO(boxiangw): Improve usability after optimizer refactor + # TODO(boxiangw): support precision aware optimizer + def muon_init_state_fn(opt, config=None): + for group in opt.param_groups: + for p in group['params']: + if len(opt.state[p]) == 0: + opt.state[p]['momentum_buffer'] = torch.zeros_like(p.data) + + def adam_init_state_fn(opt, config=None): + for group in opt.param_groups: + for p in group['params']: + if len(opt.state[p]) == 0: + if config is None or not config.use_precision_aware_optimizer: + opt.state[p]['exp_avg'] = torch.zeros_like(p.data) + opt.state[p]['exp_avg_sq'] = torch.zeros_like(p.data) + else: + opt.initialize_state(p) + + # need to wrap into megatron mix precision optimizer. (only support bf16 w/o loss scale now) + if config.fp16: + raise Exception('muon with fp16 is not supported.') + + reset_config_bf16 = False + if config.bf16: + if layer_wise_distributed_optimizer: + # creating master weight before layerwise sharding will lead to unnecessary master + # weight so here we delay master weight creation into layer_wise unset config.bf16 + # will also result in all optimizers below(adam) to also not be wrapped + config.bf16 = False + reset_config_bf16 = True + else: + # if not using layer_wise wrapper, just create master weight here is fine + optimizer = Float16OptimizerWithFloat16Params( + optimizer, config, None, muon_init_state_fn + ) + else: + optimizer = FP32Optimizer(optimizer, config, muon_init_state_fn) + + optimizers.append(optimizer) + + # done with muon, unfreeze nonlinear and freeze linear + for param in nonlinear_params: + param.requires_grad = True + for param in linear_params: + param.requires_grad = False + + # call original get. linear params will be skipped since they're freezed + chained_adam = get_megatron_optimizer( + config, model_chunks, no_weight_decay_cond, scale_lr_cond, lr_mult, use_gloo_process_groups + ) + + # unfreeze everything + for param in linear_params: + param.requires_grad = True + + # chain everything together + optimizers += chained_adam.chained_optimizers + + if layer_wise_distributed_optimizer: + log_single_rank(logger, logging.INFO, 'Using LayerWiseDistributedOptimizer for Muon') + if reset_config_bf16: + config.bf16 = True + return LayerWiseDistributedOptimizer( + optimizers, + config, + pg_collection, + init_state_fn_list=[muon_init_state_fn, adam_init_state_fn], + ) + return ChainedOptimizer(optimizers) diff --git a/megatron/core/optimizer/optimizer_config.py b/megatron/core/optimizer/optimizer_config.py index 8151d5e9de1..8692d1e9b52 100644 --- a/megatron/core/optimizer/optimizer_config.py +++ b/megatron/core/optimizer/optimizer_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from dataclasses import dataclass from typing import Callable, Optional @@ -16,7 +16,7 @@ class OptimizerConfig: # General ############## optimizer: str = 'adam' - """Optimizer to use (one of Adam or SGD).""" + """Optimizer to use (one of Adam, SGD, or Muon).""" lr: Optional[float] = None """Initial learning rate. Depending on decay style and initial warmup, the learning rate at each @@ -124,6 +124,31 @@ class OptimizerConfig: sgd_momentum: float = 0.9 """Momentum factor for SGD optimizer.""" + # Muon + muon_momentum: float = 0.95 + """The momentum used by the internal SGD.""" + + muon_split_qkv: bool = True + """Whether to split QKV parameters for Muon optimizer.""" + + muon_use_nesterov: bool = False + """Whether to use Nesterov-style momentum in the internal SGD.""" + + muon_scale_mode: str = "spectral" + """The mode to use for the scale factor. Defaults to "spectral".""" + + muon_fp32_matmul_prec: str = "medium" + """The precision to use for the fp32 matmul. Defaults to "medium".""" + + muon_num_ns_steps: int = 5 + """The number of iteration steps to use in the Newton-Schulz iteration.""" + + muon_tp_mode: str = "blockwise" + """How to perform NS calculation for tensor parallel weights. Defaults to "blockwise".""" + + muon_extra_scale_factor: float = 1.0 + """Additional scale factor for the muon update.""" + ####################### # Distributed optimizer ####################### diff --git a/megatron/core/optimizer/qk_clip.py b/megatron/core/optimizer/qk_clip.py new file mode 100644 index 00000000000..f5b34a8216b --- /dev/null +++ b/megatron/core/optimizer/qk_clip.py @@ -0,0 +1,39 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. + +import torch + +from megatron.core import mpu + + +def clip_qk(model, log_max_only=False) -> float: + """ + Clip the QK attention logits to the threshold, recommended for Muon optimizer. + + Args: + model: The model to clip the QK attention logits, a list of model chunks. + log_only: Whether to only log the max attention logit, without updating the weights. + + Returns: + The maximum attention logit, a float. + """ + + with torch.no_grad(): + log_max_attention_logit = 0 + for model_chunk in model: + for transformer_layer in model_chunk.module.module.decoder.layers: + if hasattr(transformer_layer.self_attention, 'clip_qk'): + torch.distributed.all_reduce( + transformer_layer.self_attention.core_attention.current_max_attn_logits, + op=torch.distributed.ReduceOp.MAX, + group=mpu.get_data_parallel_group(with_context_parallel=True), + ) + log_max_attention_logit = max( + log_max_attention_logit, + torch.max( + transformer_layer.self_attention.core_attention.current_max_attn_logits + ).item(), + ) + if not log_max_only: + transformer_layer.self_attention.clip_qk() + + return log_max_attention_logit diff --git a/megatron/core/pipeline_parallel/bridge_communicator.py b/megatron/core/pipeline_parallel/bridge_communicator.py index a67ded6bf08..f1e74a2f16d 100644 --- a/megatron/core/pipeline_parallel/bridge_communicator.py +++ b/megatron/core/pipeline_parallel/bridge_communicator.py @@ -628,9 +628,6 @@ def send_forward_recv_backward( dist.broadcast( shape_tensor, src=self.current_rank, group=self.src_grid_broadcast_pg ) - dist.broadcast( - shape_tensor, src=self.current_rank, group=self.src_grid_broadcast_pg - ) # Broadcast the tensors to all ranks in the group dist.broadcast( diff --git a/megatron/core/pipeline_parallel/fine_grained_activation_offload.py b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py new file mode 100644 index 00000000000..1e280a09d35 --- /dev/null +++ b/megatron/core/pipeline_parallel/fine_grained_activation_offload.py @@ -0,0 +1,609 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import warnings +from collections import deque +from contextlib import nullcontext +from typing import Any + +import torch + +# CPU offload implementation for pipeline parallelism +DEBUG = False +DEBUG_RANK = 0 + + +def debug_rank(message): + """Print debug message for a specific rank when DEBUG is enabled.""" + # pylint: disable=bad-builtin + if not DEBUG: + return + assert torch.distributed.is_initialized() + if torch.distributed.get_rank() == DEBUG_RANK: + print(message) + + +def set_ideal_affinity_for_current_gpu(): + """Set CPU affinity for the current GPU to optimize host-device transfers.""" + import uuid + + try: + import cuda.bindings.driver as cuda_driver + import cuda.bindings.runtime as cuda_runtime + except ImportError: + try: + import cuda.cuda as cuda_driver + import cuda.cudart as cuda_runtime + except ImportError: + # print("cuda-python may not be installed, skipping GPU affinity setting") + warnings.warn("cuda-python may not be installed, skipping GPU affinity setting") + return + try: + import pynvml + except ImportError: + warnings.warn("pynvml is not installed, skipping GPU affinity setting") + return + + # Get current CUDA device ID + err, device_id = cuda_runtime.cudaGetDevice() + assert err == cuda_runtime.cudaError_t.cudaSuccess + # Get device UUID + err, device_uuid = cuda_driver.cuDeviceGetUuid(device_id) + assert err == cuda_driver.CUresult.CUDA_SUCCESS + # Set CPU affinity based on GPU's NUMA node + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByUUID("GPU-" + str(uuid.UUID(bytes=device_uuid.bytes))) + pynvml.nvmlDeviceSetCpuAffinity(handle) + + +class PipelineOffloadManager: + """ + Singleton manager for coordinating activation offloading across pipeline stages. + Manages chunk handlers, synchronizes GPU-CPU transfers, + and handles virtual pipeline parallelism. + """ + + OFFLOAD_MGR = None + + @classmethod + def get_instance(cls): + """Get the singleton instance of PipelineOffloadManager.""" + if cls.OFFLOAD_MGR is None: + cls.OFFLOAD_MGR = PipelineOffloadManager() + return cls.OFFLOAD_MGR + + def __init__(self): + """Initialize the manager with queues and dedicated CUDA streams.""" + # Queue to store chunk handlers for backward pass + self._queue = deque() + # Cache chunk handlers for each virtual pipeline stage + self._stages = None + # allocate streams and events for synchronization + self._d2h_stream = torch.cuda.Stream() + self._h2d_stream = torch.cuda.Stream() + self.reset() + + @property + def d2h_stream(self): + """Get the device-to-host (GPU to CPU) transfer stream.""" + return self._d2h_stream + + @property + def h2d_stream(self): + """Get the host-to-device (CPU to GPU) transfer stream.""" + return self._h2d_stream + + def reset(self): + """Reset manager state for a new training iteration.""" + set_ideal_affinity_for_current_gpu() + self._inside_context = False + self._cur_forward_chunk = None + self._cur_backward_chunk = None + # Track the first microbatch of the last virtual pipeline stage + self._is_first_last_vpp_chunk = True + + def flush(self): + """Flush all staged chunks to the backward queue in reverse order.""" + # Ensure all virtual pipeline stages have the same number of chunks + if len(self._stages[0]) == len(self._stages[-1]): + lens = [len(e) for e in self._stages] + assert min(lens) == max(lens), "All stages must have same chunk count" + # Clear the last stage and push all chunks in reverse order for backward + self._stages[-1] = [] + for chunks in reversed(self._stages): + for chunk in chunks: + self.push(chunk) + # Clear all stages after flushing + for i in range(self._vpp): + self._stages[i] = [] + + def push(self, handler): + """Add a chunk handler to the backward queue.""" + debug_rank(f"pushing handler {handler}") + self._queue.append(handler) + + def pop(self): + """Remove and set the next non-empty chunk as the current backward chunk.""" + assert self.size(), "Cannot pop from empty queue" + while self._queue: + self._cur_backward_chunk = self._queue.popleft() + if not self._cur_backward_chunk.is_empty_chunk(): + break + debug_rank(f"popping handler {self._cur_backward_chunk}") + + def front(self): + """Get the first non-empty chunk handler without removing it from the queue.""" + if not self.size(): + return None + for chunk_handler in self._queue: + if not chunk_handler.is_empty_chunk(): + return chunk_handler + return None + + def size(self): + """Return the number of chunk handlers in the queue.""" + return len(self._queue) + + def init_model_chunk_offload_handler( + self, vp_size, vp_stage, min_offloaded_tensor_size=1024 * 1024 + ): + """ + Initialize a chunk offload handler for a model chunk (microbatch). + + Args: + vp_size: Virtual pipeline size + vp_stage: Virtual pipeline stage index (None means stage 0) + min_offloaded_tensor_size: Minimum tensor size (in elements) to offload + """ + if self._stages is None: + vp_size = 1 if vp_size is None else vp_size + self._vpp = vp_size + self._stages = [[] for _ in range(vp_size)] + + if vp_stage is None: + cur_vpp_rank = 0 + else: + cur_vpp_rank = vp_stage + + is_first_last_vpp_chunk = self._is_first_last_vpp_chunk + # Flush staged chunks when reaching the last virtual pipeline stage + if cur_vpp_rank == self._vpp - 1: + self.flush() + # Determine if this is the first microbatch of the last virtual pipeline stage + is_first_last_vpp_chunk = is_first_last_vpp_chunk and (cur_vpp_rank == self._vpp - 1) + + cur_chunk = ChunkOffloadHandler(is_first_last_vpp_chunk, min_offloaded_tensor_size) + self._stages[cur_vpp_rank].append(cur_chunk) + # For the last stage, push immediately and flush + if cur_vpp_rank == self._vpp - 1: + self._is_first_last_vpp_chunk = False + self.push(cur_chunk) + self.flush() + self._cur_forward_chunk = cur_chunk + cur_chunk.vpp_rank = cur_vpp_rank + + def set_last_layer(self, is_last_layer): + """Mark whether the current forward chunk is processing the last layer.""" + self._cur_forward_chunk.is_last_layer = is_last_layer + + def cur_forward_chunk(self): + """Get the current forward pass chunk handler.""" + return self._cur_forward_chunk + + def cur_backward_chunk(self): + """Get the current backward pass chunk handler.""" + return self._cur_backward_chunk + + def __enter__(self): + """Enter context manager to enable activation offloading hooks.""" + debug_rank("----__enter__") + from megatron.core.extensions.transformer_engine import cpu_offload + + if cpu_offload is not None: + cpu_offload.CPUOffloadEnabled = True + self.inside_context = True + + torch._C._autograd._push_saved_tensors_default_hooks( + self.on_save_for_backward, self.on_get_saved_tensor + ) + + def __exit__(self, *args: Any): + """Exit context manager and restore original tensor saving behavior.""" + debug_rank("----__exit__") + from megatron.core.extensions.transformer_engine import cpu_offload + + if cpu_offload is not None: + cpu_offload.CPUOffloadEnabled = False + self.inside_context = False + torch._C._autograd._pop_saved_tensors_default_hooks() + + def on_save_for_backward(self, tensor: torch.Tensor) -> Any: + """ + Hook called when autograd saves a tensor for backward pass. + Returns a tag to identify the tensor later. + """ + debug_rank(f"------on_save_for_backward {tensor.shape}") + assert self.inside_context, "Must be inside offload context" + return self.cur_forward_chunk().tensor_push(tensor) + + def on_get_saved_tensor(self, saved_state: Any) -> torch.Tensor: + """ + Hook called when autograd retrieves a saved tensor during backward pass. + Returns the actual tensor (potentially reloading from CPU). + """ + debug_rank(f"----on_get_saved_tensor {saved_state}") + return self.cur_backward_chunk().tensor_pop(saved_state) + + +class ChunkOffloadHandler: + """ + Handles activation offloading and reloading for a single pipeline chunk (microbatch). + Manages tensor groups, coordinates asynchronous GPU-CPU transfers, and handles synchronization. + """ + + @staticmethod + def offload(src_tensor, pin_memory=True): + """Offload.""" + debug_rank("--------offload") + from megatron.core.extensions.transformer_engine import Float8Tensor + + fp8_offload = isinstance(src_tensor, Float8Tensor) if Float8Tensor is not None else False + + if not src_tensor.is_contiguous(): + src_tensor = src_tensor.contiguous() + + cpu_backup = torch.empty( + src_tensor.size(), + dtype=torch.uint8 if fp8_offload else src_tensor.dtype, + layout=src_tensor.layout, + device="cpu", + pin_memory=pin_memory, + ) + + if fp8_offload: + cpu_backup = Float8Tensor.make_like(src_tensor, data=cpu_backup) + + cpu_backup.copy_(src_tensor, non_blocking=pin_memory) + state = (src_tensor.device, cpu_backup) + return state + + @staticmethod + def reload(state, non_blocking=None): + """Reload.""" + debug_rank("------reload") + dev, cpu_backup = state + if non_blocking is None: + non_blocking = cpu_backup.is_pinned() + return cpu_backup.to(dev, non_blocking=non_blocking) + + def __init__(self, is_first_last_vpp_chunk, min_offloaded_tensor_size): + # Data Structure to maintain reference to activation tensors + self._tensor_tag_to_state = {} + # Mark the first microbatch of the last virtual pipeline stage + self._is_first_last_vpp_chunk = is_first_last_vpp_chunk + + # Group management for batching offload/reload operations + self._offloaded_group_index = 0 + self._groups_to_offload = [] + self._groups_to_reload = [] + self._tensor_count_current_group = 0 + + # Counter for special torch tensor types (FakeTensor, FunctionalTensor) + self.torch_tensor_count = 0 + self.d2h_stream = PipelineOffloadManager.get_instance().d2h_stream + self.h2d_stream = PipelineOffloadManager.get_instance().h2d_stream + self._offload_events = {} + self._reload_events = {} + self.min_offloaded_tensor_size = min_offloaded_tensor_size + self.is_last_layer = False + + def is_empty_chunk(self): + """Check if this chunk has no tensors to manage.""" + return len(self._tensor_tag_to_state) == 0 + + def is_first_last_layer(self): + """ + Check if this is the last layer of the first microbatch of the last vp stage. + These tensors should not be offloaded to avoid unnecessary overhead. + """ + debug_rank( + f"------is_first_last_layer {self._is_first_last_vpp_chunk} {self.is_last_layer}" + ) + return self._is_first_last_vpp_chunk and self.is_last_layer + + def tensor_push(self, tensor): + """Push tensor to the offload handler.""" + torch_stray_tensor = isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ) + + if not torch_stray_tensor: + # Assign unique tag based on group index and position within group + tensor_tag = (self._offloaded_group_index, self._tensor_count_current_group) + self._tensor_count_current_group += 1 + assert tensor_tag not in self._tensor_tag_to_state, "Duplicate tensor tag" + self._tensor_tag_to_state[tensor_tag] = tensor + else: + # Use negative group ID for special tensor types + tensor_tag = (-1, self.torch_tensor_count) + self.torch_tensor_count += 1 + self._tensor_tag_to_state[tensor_tag] = tensor + debug_rank(f"--------tensor_push {tensor_tag}") + return tensor_tag + + def tensor_pop(self, tensor_tag): + """Pop tensor from the offload handler.""" + debug_rank(f"--------tensor_pop {tensor_tag}") + assert tensor_tag in self._tensor_tag_to_state, f"Tag {tensor_tag} not found" + tensor = self._tensor_tag_to_state.pop(tensor_tag) + # If tensor is offloaded (stored as tuple), reload it + if isinstance(tensor, tuple): + tensor = self.reload(tensor) + debug_rank(f"--------tensor_pop {tensor.shape}") + return tensor + + def tensor_need_offloading_checker(self, tensor): + """Check if the tensor needs to be offloaded.""" + if tensor.numel() < self.min_offloaded_tensor_size: + return False + # Respect tensor's offload preference if specified + if hasattr(tensor, "offloading_activation") and not tensor.offloading_activation: + return False + return True + + def bulk_offload_group(self, group_to_offload): + """offload a group of tensors recorded in tensor_push().""" + debug_rank("------bulk_offload_group") + assert not self.is_first_last_layer(), "Should not offload first-last layer" + group_id_to_offload, name = group_to_offload + torch.cuda.nvtx.range_push("activation offloading " + name) + with torch.cuda.stream(self.d2h_stream): + for tensor_tag, state in self._tensor_tag_to_state.items(): + group_id, _ = tensor_tag + if group_id == group_id_to_offload: + debug_rank(f"------tensor_tag {tensor_tag}") + debug_rank(f"------group_to_offload {group_to_offload}") + assert not isinstance(state, tuple), "Tensor already offloaded" + tensor_on_device = state + if self.tensor_need_offloading_checker(tensor_on_device): + state = self.offload(tensor_on_device) + event = torch.cuda.Event() + event.record(self.d2h_stream) + self._offload_events[name] = event + tensor_on_device.record_stream(self.d2h_stream) + self._tensor_tag_to_state[tensor_tag] = state + torch.cuda.nvtx.range_pop() + + def get_offload_event(self, name): + """Get the CUDA event for a named offload operation.""" + return self._offload_events.get(name, None) + + def get_reload_event(self, name): + """Get the CUDA event for a named reload operation.""" + return self._reload_events.get(name, None) + + def bulk_reload_group(self, group_to_reload): + """Bulk reload group.""" + debug_rank("----bulk_reload_group") + found_reload_group = False + group_id_to_reload, name = group_to_reload + torch.cuda.nvtx.range_push("activation reloading " + name) + with torch.cuda.stream(self.h2d_stream): + for tensor_label, state in self._tensor_tag_to_state.items(): + group_id, _ = tensor_label + if group_id == group_id_to_reload: + debug_rank(f"----tensor_label {tensor_label}") + found_reload_group = True + event = self.get_offload_event(name) + # Only reload if tensor was offloaded (stored as tuple) + if isinstance(state, tuple): + # Wait for offload to complete before reloading + torch.cuda.current_stream().wait_event(event) + recovered_tensor = self.reload(state) + event.record(self.h2d_stream) + self._reload_events[name] = event + debug_rank(f"----recovered_tensor {recovered_tensor.shape}") + self._tensor_tag_to_state[tensor_label] = recovered_tensor + torch.cuda.nvtx.range_pop() + return found_reload_group + + def pre_reload_last_layer(self): + """Pre-reload the last layer of this chunk to hide reload latency.""" + debug_rank("pre_reload_last_layer") + assert not self._is_first_last_vpp_chunk, "Should not pre-reload first chunk" + debug_rank(f"len(self._groups_to_reload) {len(self._groups_to_reload)}") + if len(self._groups_to_reload) > 0: + # Reload the last group (last layer) early + if self.bulk_reload_group(self._groups_to_reload[-1]): + self._groups_to_reload.pop() + + def should_bulk_offload(self): + """Determine if the current group should be offloaded.""" + # Don't offload the first backward chunk's last layer + if self.is_first_last_layer(): + return False + + # Check if next backward chunk is this chunk (for last pipeline stage) + next_backward_chunk = PipelineOffloadManager.get_instance().front() + if next_backward_chunk is not None and next_backward_chunk is self: + # Don't offload last layer if it's about to be used immediately + if self.is_last_layer: + return False + + return True + + def bulk_offload(self, forced_released_tensors): + """Offload a group of tensors and optionally release their GPU memory.""" + debug_rank("----bulk_offload") + if self.should_bulk_offload(): + group_to_offload = self._groups_to_offload.pop() + self._groups_to_reload.append(group_to_offload) + self.bulk_offload_group(group_to_offload) + # Manually release tensors not auto-freed by torch GC + if len(forced_released_tensors) > 0: + cur_stream = torch.cuda.current_stream() + for release_tensor in forced_released_tensors: + if self.tensor_need_offloading_checker(release_tensor): + # Ensure tensor is not in use before freeing + release_tensor.record_stream(cur_stream) + release_tensor.untyped_storage().resize_(0) + + def on_group_commit_forward(self, forced_released_tensors): + """Called at the end of a layer group's forward pass to trigger offloading.""" + debug_rank("--on_group_commit_forward") + # Wait for compute to finish before starting offload + self.d2h_stream.wait_stream(torch.cuda.current_stream()) + self.bulk_offload(forced_released_tensors) + + def bulk_reload(self): + """Reload the next group of tensors from CPU to GPU.""" + debug_rank("--bulk_reload") + if len(self._groups_to_reload) > 0: + # Reload the next layer group + if self.bulk_reload_group(self._groups_to_reload[-1]): + debug_rank(f"--bulk_reload_group {self._groups_to_reload}") + self._groups_to_reload.pop() + else: + # Pre-load the last layer of the next backward chunk to hide latency + next_backward_chunk = PipelineOffloadManager.get_instance().front() + if next_backward_chunk is not None: + next_backward_chunk.pre_reload_last_layer() + + def on_group_commit_backward(self, name): + """ + Called at the end of a layer group's backward pass. + Ensures correct chunk is active and synchronizes reloads. + """ + debug_rank("--on_group_commit_backward") + cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() + # Switch to this chunk if it's not already current + if cur_backward_chunk is not self: + PipelineOffloadManager.get_instance().pop() + cur_backward_chunk = PipelineOffloadManager.get_instance().cur_backward_chunk() + assert cur_backward_chunk is self, "Chunk mismatch" + # Wait for reload to complete before using tensors + event = self.get_reload_event(name) + if event is not None: + torch.cuda.current_stream().wait_event(event) + self._offloaded_group_index = self._offloaded_group_index - 1 + + def on_group_start_forward(self, name): + """ + Called at the start of a layer group's forward pass. + Increments group index and prepares for offloading. + """ + debug_rank(f"--on_group_start_forward") + self._offloaded_group_index = self._offloaded_group_index + 1 + self._tensor_count_current_group = 0 + self._groups_to_offload.append((self._offloaded_group_index, name)) + + def on_group_start_backward(self): + """ + Called at the start of a layer group's backward pass. + Triggers reloading of tensors from CPU. + """ + debug_rank("--on_group_start_backward") + # Wait for compute to finish before starting reload + self.h2d_stream.wait_stream(torch.cuda.current_stream()) + self.bulk_reload() + + +class FineGrainedOffloadingGroupCommitFunction(torch.autograd.Function): + """ + Identity operation that marks the end of a layer group for offload synchronization. + Triggers offload during forward and synchronizes reload during backward. + """ + + @staticmethod + def forward(ctx, *args): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupCommitFunction forward") + + forced_released_tensors = args[-1] + name = args[-2] + cpu_offload_handler = args[-3] + tensor = args[:-3] + cpu_offload_handler.on_group_commit_forward(forced_released_tensors) + ctx.cpu_offload_handler = cpu_offload_handler + ctx.name = name + + # return the identical tensor + return tensor + + @staticmethod + def backward(ctx, *grad_output): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupCommitFunction backward") + + cpu_offload_handler = ctx.cpu_offload_handler + cpu_offload_handler.on_group_commit_backward(ctx.name) + return grad_output + (None, None, None) + + +def fine_grained_offloading_group_commit(*tensor, name, forced_released_tensors=[]): + """ + Specify the tensors to be released after offloading. + forced_released_tensors is a list of tensors to be released after offloading. + The tensors will be untyped_storage().resize_(0) after offloading. + Note: specify the tensors only when they are not automatically released by torch gc. + """ + cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + return FineGrainedOffloadingGroupCommitFunction.apply( + *tensor, cur_forward_chunk, name, forced_released_tensors + ) + + +class FineGrainedOffloadingGroupStartFunction(torch.autograd.Function): + """ + Identity operation that marks the start of a layer group for offload/reload. + Prepares for offload during forward and triggers reload during backward. + """ + + @staticmethod + def forward(ctx, tensor, cpu_offload_handler, name): + # pylint: disable=missing-function-docstring + ctx.cpu_offload_handler = cpu_offload_handler + debug_rank("FineGrainedOffloadingGroupStartFunction forward") + + cpu_offload_handler.on_group_start_forward(name) + # return the identical tensor + return tensor + + @staticmethod + def backward(ctx, grad_output): + # pylint: disable=missing-function-docstring + debug_rank("FineGrainedOffloadingGroupStartFunction backward") + cpu_offload_handler = ctx.cpu_offload_handler + cpu_offload_handler.on_group_start_backward() + return grad_output, None, None + + +def fine_grained_offloading_group_start(tensor, name=None): + """Mark the start of a layer group and prepare for offload/reload.""" + cur_forward_chunk = PipelineOffloadManager.get_instance().cur_forward_chunk() + return FineGrainedOffloadingGroupStartFunction.apply(tensor, cur_forward_chunk, name) + + +def get_fine_grained_offloading_context(flag): + """Get the fine-grained offload context""" + return PipelineOffloadManager.get_instance() if flag else nullcontext() + + +def fine_grained_offloading_set_last_layer(is_last_layer): + """Set the last layer flag.""" + PipelineOffloadManager.get_instance().set_last_layer(is_last_layer) + + +def fine_grained_offloading_init_chunk_handler(vp_size, vp_stage, min_offloaded_tensor_size): + """Initialize the chunk handler, called at the start of a microbatch forward pass.""" + PipelineOffloadManager.get_instance().init_model_chunk_offload_handler( + vp_size, vp_stage, min_offloaded_tensor_size + ) + + +def fine_grained_offloading_reset(): + """Reset the chunk handler, called at the start of a training iteration.""" + PipelineOffloadManager.get_instance().reset() diff --git a/megatron/core/pipeline_parallel/multimodule_communicator.py b/megatron/core/pipeline_parallel/multimodule_communicator.py new file mode 100644 index 00000000000..dfda270ef76 --- /dev/null +++ b/megatron/core/pipeline_parallel/multimodule_communicator.py @@ -0,0 +1,523 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +import logging +from dataclasses import dataclass +from typing import Dict, List, Optional, Union + +import torch +import torch.distributed as dist + +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.model_parallel_config import ModelParallelConfig +from megatron.core.pipeline_parallel.bridge_communicator import BridgeCommunicator +from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator + +# Types +Shape = Union[List[int], torch.Size] + + +@dataclass +class RankModuleInfo: + """Information about a rank in a module.""" + + # the stage of the current rank in the current module's pipeline. + pp_rank: int # the stage of the current rank in the current module's pipeline + pp_size: int # the number of ranks in the current module's pipeline + p2p_communicator: Optional[P2PCommunicator] + # key is either the src or dst module name connected to the current module + # one module may have multiple bridge communicators if it has multiple + # incoming or outgoing connections. + bridge_comms_as_src_module: Optional[List[BridgeCommunicator]] + bridge_comms_as_dest_module: Optional[List[BridgeCommunicator]] + # the absolute first stage in the overall model + # no incoming connections + is_source_stage: Optional[bool] = True + # the absolute last stage in the overall model + # no outgoing connections + is_terminal_stage: Optional[bool] = True + + +class MultiModulePipelineCommunicator: + """Communicator for a multi-module pipeline.""" + + def __init__( + self, + module_to_grid_map: Dict[str, HyperCommGrid], + topology: Dict[str, List[str]], + config: ModelParallelConfig, + dim_mapping: Dict[str, List[int]] = None, + ): + """ + Initialize the MultiModulePipelineCommunicator. + + Args: + module_to_grid_map (dict): A dictionary mapping module names to HyperCommGrids. + Example: + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid + } + topology (dict): A dictionary mapping module names to lists of outgoing modules. + Example: + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [] + } + config (ModelParallelConfig): A ModelParallelConfig object. + dim_mapping (Dict[str, List[int]]): Dimension mapping for sequence, batch, hidden. + Example: + dim_mapping = {'s': 0, 'h': 2, 'b': 1} + Default: None + """ + self.module_to_grid_map = module_to_grid_map + self.topology = topology + self.config = config + self.dim_mapping = dim_mapping + self.current_rank = dist.get_rank() + + # Build bridge communicators for all modules + self.bridge_comms = [] + self._build_bridge_comms() + + self.rank_module_map = {} + self._build_rank_module_info_map() + + def _build_bridge_comms(self): + """Construct and store BridgeCommunicator objects that describe the outgoing + communication relationships for all of the modules. + """ + for src_module_name, src_grid in self.module_to_grid_map.items(): + for dest_module_name in self.topology[src_module_name]: + dest_grid = self.module_to_grid_map[dest_module_name] + bridge_comm = BridgeCommunicator( + src_grid=src_grid, + dest_grid=dest_grid, + dim_mapping=self.dim_mapping, + comm_dtype=self.config.pipeline_dtype, + src_module_name=src_module_name, + dest_module_name=dest_module_name, + ) + self.bridge_comms.append(bridge_comm) + + @property + def is_pp_first_stage(self): + """Return True if the current rank has the absolute first stage in the overall model. + + The absolute first stage is defined as: + 1. The current rank must be in the first PP stage (pp_rank == 0) of some module + 2. That module must be a source module (no incoming connections in topology) + """ + for module_name, rank_module_info in self.rank_module_map.items(): + # Check if this rank is at the first PP stage of this module + if rank_module_info.pp_rank == 0: + # Check if this module is a source module (no incoming connections) + if self._is_source_module(module_name): + return True + return False + + @property + def is_pp_last_stage(self): + """Return True if the current rank has the absolute last stage in the overall model. + + The absolute last stage is defined as: + 1. The current rank must be in the last PP stage of some module + 2. That module must be a sink module (no outgoing connections in topology) + """ + for module_name, rank_module_info in self.rank_module_map.items(): + # Check if this rank is at the last PP stage of this module + if rank_module_info.pp_rank == rank_module_info.pp_size - 1: + # Check if this module is a sink module (no outgoing connections) + if self._is_sink_module(module_name): + return True + return False + + def _is_source_module(self, module_name: str) -> bool: + """Check if a module is a source module (has no incoming connections).""" + # A module is a source if no other module lists it as a destination + for src_module, dest_modules in self.topology.items(): + if module_name in dest_modules: + return False + return True + + def _is_sink_module(self, module_name: str) -> bool: + """Check if a module is a sink module (has no outgoing connections).""" + return len(self.topology.get(module_name, [])) == 0 + + def is_current_rank_in_grid(self, grid: HyperCommGrid) -> bool: + """Check if the current rank is in the grid.""" + return grid.rank_offset <= self.current_rank < grid.rank_offset + grid.size + + @property + def num_warmup_microbatches(self): + """Calculate the number of warmup microbatches for the current rank. + + Uses the same simple logic as P2PCommunicator: + total_pipeline_stages - current_rank_stage - 1 + + Returns: + int: Number of warmup microbatches for this rank + """ + # Get total pipeline depth across all modules + total_stages = self.compute_total_pipeline_stages(self.topology, self.module_to_grid_map) + + # Get current rank's position in the overall pipeline (0-indexed) + # Use compute_total_pipeline_stages with current rank to get cumulative position + if self.rank_module_map: + # Take the first module this rank belongs to + # TODO: ykarnati - improve this logic. + module_name = next(iter(self.rank_module_map.keys())) + current_stage = ( + self.compute_total_pipeline_stages( + self.topology, + self.module_to_grid_map, + rank=self.current_rank, + module_name=module_name, + ) + - 1 + ) # Convert from 1-indexed to 0-indexed + else: + current_stage = 0 + + assert ( + current_stage <= total_stages + ), f"current_stage: {current_stage} is greater than total_stages: {total_stages}" + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"current_stage: {current_stage} total_stages: {total_stages} " + f"num_warmup_microbatches: {total_stages - current_stage - 1}" + ) + return total_stages - current_stage - 1 + + def _build_rank_module_info_map(self): + """For each module in the current rank, initialize the P2P communicator + and build the bridge communicator info for the module. + Each rank may hold multiple modules when colocated. + """ + for module_name, module_grid in self.module_to_grid_map.items(): + if self.is_current_rank_in_grid(module_grid): + # Initialize P2P communicator + pp_group = module_grid.get_pg('pp') + p2p_comm = P2PCommunicator(pp_group, self.config) + pp_size = dist.get_world_size(pp_group) + rank_in_pp_group = dist.get_group_rank(pp_group, self.current_rank) + pp_rank = rank_in_pp_group % pp_size + + bridge_comms_as_dest_module = [] + bridge_comms_as_src_module = [] + # If first stage, check if the module has any incoming modules + # If so, initialize bridge communicator + if pp_rank == 0: + for bridge_comm in self.bridge_comms: + if ( + bridge_comm.is_current_rank_in_grid(bridge_comm.dest_grid) + and bridge_comm.dest_module_name == module_name + ): + bridge_comms_as_dest_module.append(bridge_comm) + # If last stage, check if the module has any outgoing modules + # If so, initialize bridge communicator + if pp_rank == pp_size - 1: + for bridge_comm in self.bridge_comms: + if ( + bridge_comm.is_current_rank_in_grid(bridge_comm.src_grid) + and bridge_comm.src_module_name == module_name + ): + bridge_comms_as_src_module.append(bridge_comm) + # Build RankModuleInfo for the module + rank_module_info = RankModuleInfo( + pp_rank=pp_rank, + pp_size=pp_size, + p2p_communicator=p2p_comm, + bridge_comms_as_dest_module=bridge_comms_as_dest_module, + bridge_comms_as_src_module=bridge_comms_as_src_module, + ) + self.rank_module_map[module_name] = rank_module_info + + def recv_forward( + self, tensor_shape: Optional[Shape] = None, is_first_stage: bool = False + ) -> Dict[str, torch.Tensor]: + """Receive forward activation tensor. + + Args: + tensor_shape: Expected activation tensor shape + + Returns: + A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[receive_forward] tensors_shape: {tensor_shape}, is_first_stage: {is_first_stage}" + ) + input_dict = {} + for module_name, rank_module_info in self.rank_module_map.items(): + + if rank_module_info.pp_rank == 0: + # If first stage, and has incoming modules, receive forward activation + # from incoming modules. + for bridge_comm in rank_module_info.bridge_comms_as_dest_module: + input_dict[bridge_comm.src_module_name] = bridge_comm.recv_forward() + else: + # If not first stage, receive forward activation tensor from P2P communicator. + input_dict[module_name] = rank_module_info.p2p_communicator.recv_forward( + tensor_shapes=tensor_shape, is_first_stage=False + ) + return input_dict + + def send_forward(self, output_dict: Dict[str, torch.Tensor], is_last_stage: bool = False): + """Send forward activation tensor. + + Args: + output_dict: A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[send_forward] output_dict keys: {output_dict.keys()}, is_last_stage: {is_last_stage}" + ) + for module_name, rank_module_info in self.rank_module_map.items(): + if rank_module_info.pp_rank == rank_module_info.pp_size - 1: + # If last stage, and has outgoing modules, send forward activation + # by using bridge communicator. + for bridge_comm in rank_module_info.bridge_comms_as_src_module: + bridge_comm.send_forward(output_dict[module_name]) + else: + # If not last stage, send forward activation by using P2P communicator. + rank_module_info.p2p_communicator.send_forward( + output_dict[module_name], is_last_stage=False + ) + + def send_forward_recv_backward( + self, + output_dict: Dict[str, torch.Tensor], + tensor_shape: Optional[Shape] = None, + is_last_stage: bool = False, + ) -> Dict[str, torch.Tensor]: + """Send forward activation tensor and receive backward activation tensor. + + Args: + output_dict: A dictionary mapping module names to tensors. + tensor_shape: Expected gradient tensor shape + + Returns: + A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[send_forward_recv_backward] output_dict keys: {output_dict.keys()}, " + f"tensor_shape: {tensor_shape}, is_last_stage: {is_last_stage}" + ) + grad_dict = {} + for module_name, rank_module_info in self.rank_module_map.items(): + if rank_module_info.pp_rank == rank_module_info.pp_size - 1: + # If last stage, and has outgoing modules, send forward activation and + # receive backward gradient by using bridge communicator. + for bridge_comm in rank_module_info.bridge_comms_as_src_module: + grad_dict[bridge_comm.src_module_name] = bridge_comm.send_forward_recv_backward( + output_dict[module_name] + ) + else: + # If not last stage, send forward activation and receive backward gradient + # by using P2P communicator. + grad_dict[module_name] = ( + rank_module_info.p2p_communicator.send_forward_recv_backward( + output_dict[module_name], tensor_shapes=tensor_shape, is_last_stage=False + ) + ) + return grad_dict + + def send_backward_recv_forward( + self, + grad_dict: Dict[str, torch.Tensor], + tensor_shape: Optional[Shape] = None, + is_first_stage: bool = False, + ) -> Dict[str, torch.Tensor]: + """Send backward activation tensor and receive forward activation tensor. + + Args: + grad_dict: A dictionary mapping module names to tensors. + tensor_shape: Expected gradient tensor shape + + Returns: + A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[send_backward_recv_forward] grad_dict keys: {grad_dict.keys()}, " + f"tensor_shape: {tensor_shape}, is_first_stage: {is_first_stage}" + ) + input_dict = {} + for module_name, rank_module_info in self.rank_module_map.items(): + if rank_module_info.pp_rank == 0: + for bridge_comm in rank_module_info.bridge_comms_as_dest_module: + # If first stage, and has incoming modules, send backward gradient and + # receive forward activation by using bridge communicator. + input_dict[bridge_comm.src_module_name] = ( + bridge_comm.send_backward_recv_forward( + grad_dict[bridge_comm.src_module_name] + ) + ) + else: + # If not first stage, send backward gradient and receive forward activation + # by using P2P communicator. + input_dict[module_name] = ( + rank_module_info.p2p_communicator.send_backward_recv_forward( + grad_dict[module_name], tensor_shapes=tensor_shape, is_first_stage=False + ) + ) + return input_dict + + def recv_backward( + self, tensor_shape: Optional[Shape] = None, is_last_stage: bool = False + ) -> Dict[str, torch.Tensor]: + """Receive backward activation tensor. + + Args: + tensor_shape: Expected gradient tensor shape + + Returns: + A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[recv_backward] tensor_shape: {tensor_shape}, is_last_stage: {is_last_stage}" + ) + grad_dict = {} + for module_name, rank_module_info in self.rank_module_map.items(): + if rank_module_info.pp_rank == rank_module_info.pp_size - 1: + # If last stage, and has incoming modules, receive backward gradient + # by using bridge communicator. + for bridge_comm in rank_module_info.bridge_comms_as_src_module: + grad_dict[bridge_comm.src_module_name] = bridge_comm.recv_backward() + else: + # If not last stage, receive backward gradient by using P2P communicator. + grad_dict[module_name] = rank_module_info.p2p_communicator.recv_backward( + tensor_shapes=tensor_shape, is_last_stage=False + ) + return grad_dict + + def send_backward(self, grad_dict: Dict[str, torch.Tensor], is_first_stage: bool = False): + """Send backward activation tensor. + + Args: + grad_dict: A dictionary mapping module names to tensors. + """ + logging.debug( + f"[Rank {dist.get_rank()} ][MultiModulePipelineCommunicator] " + f"[send_backward] grad_dict keys: {grad_dict.keys()}, is_first_stage: {is_first_stage}" + ) + for module_name, rank_module_info in self.rank_module_map.items(): + if rank_module_info.pp_rank == 0: + # If first stage, and has incoming modules, send backward activation + # by using bridge communicator. + for bridge_comm in rank_module_info.bridge_comms_as_dest_module: + bridge_comm.send_backward(grad_dict[bridge_comm.src_module_name]) + else: + # If not first stage, send backward activation by using P2P communicator. + rank_module_info.p2p_communicator.send_backward( + grad_dict[module_name], is_first_stage=False + ) + + @staticmethod + def compute_total_pipeline_stages( + topology: Dict[str, List[str]], + module_to_grid_map: Dict[str, HyperCommGrid], + rank: Optional[int] = None, + module_name: Optional[str] = None, + ) -> int: + """Compute the total number of pipeline stages across a multi-module chain. + + Interprets ``topology`` as a directed acyclic graph (DAG) where nodes are modules + and edges indicate forward data flow from source to destination modules. Each node + is assigned a weight equal to its pipeline parallel size (number of PP stages). + + The total number of stages is defined as the length of the longest path in this DAG + under node weights. + + If ``rank`` is None (default), returns the maximum over all terminal (sink) modules of + the sum of PP sizes along a path ending at that terminal. For example, given: + + image_encoder ->\ + -> llm -> generator + audio_encoder ->/ + + the total is: max(pp(image_encoder), pp(audio_encoder)) + pp(llm) + pp(generator). + + If ``rank`` is provided, the result is the total number of pipeline stages up to (and + including) the PP stage that ``rank`` occupies inside its module. In this case, the + weight of the target module equals (pp_rank_index(rank) + 1) instead of the module's + full PP size; other modules still contribute their full PP sizes. If the rank belongs to + multiple modules (colocation), pass ``module_name`` to disambiguate; otherwise the + maximum across all candidate modules containing the rank is returned. + + Args: + topology: Mapping from a module to its list of outgoing modules. + module_to_grid_map: Mapping from module name to its ``HyperCommGrid``. + + Returns: + The total number of pipeline stages along the longest path given the constraints. + + Raises: + ValueError: If the topology contains cycles; or has no terminal nodes when + ``rank`` is None + """ + nodes = set(module_to_grid_map.keys()) + # Build adjacency and reverse-adjacency (predecessors). + adj: Dict[str, List[str]] = {node: list(topology.get(node, [])) for node in nodes} + preds: Dict[str, List[str]] = {node: [] for node in nodes} + for src, outs in adj.items(): + for dst in outs: + preds[dst].append(src) + + # Identify terminal nodes (no outgoing edges) for the rank=None case. + sinks = [node for node, outs in adj.items() if not outs] + if rank is None and not sinks: + raise ValueError( + "Topology must be a DAG with at least one terminal (no outgoing) module." + ) + + def pp_size(name: str) -> int: + grid = module_to_grid_map[name] + pp_dim_index = grid.dim_names.index('pp') + return grid.shape[pp_dim_index] + + def partial_weight_for_target(target: str) -> Optional[int]: + if rank is None: + return None + grid = module_to_grid_map.get(target) + rank_groups = grid._gen_rank_enum(['pp']) + stage_index: Optional[int] = None + for group in rank_groups: + if rank in group: + stage_index = group.index(rank) + break + return stage_index + 1 + + def longest_path_to(target: str) -> int: + visiting = set() + partial = partial_weight_for_target(target) + + def weight(name: str) -> int: + if partial is not None and name == target: + return partial + return pp_size(name) + + def dfs(node: str) -> int: + if node in visiting: + raise ValueError("Topology contains cycles; expected a DAG.") + visiting.add(node) + best = 0 + for p in preds.get(node, []): + val = dfs(p) + if val > best: + best = val + visiting.remove(node) + return weight(node) + best + + return dfs(target) + + if rank is None: + return max(longest_path_to(sink) for sink in sinks) + + return longest_path_to(module_name) diff --git a/megatron/core/pipeline_parallel/p2p_communication.py b/megatron/core/pipeline_parallel/p2p_communication.py index 63ee9d1f537..ac839c21f18 100644 --- a/megatron/core/pipeline_parallel/p2p_communication.py +++ b/megatron/core/pipeline_parallel/p2p_communication.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from typing import List, Optional, Tuple, Union @@ -214,22 +214,22 @@ def _communicate_shapes(self, tensor_send_next, tensor_send_prev, recv_prev, rec ops = [] if send_prev_shape_tensor is not None: send_prev_op = torch.distributed.P2POp( - torch.distributed.isend, send_prev_shape_tensor, self.prev_rank + torch.distributed.isend, send_prev_shape_tensor, self.prev_rank, self.pp_group ) ops.append(send_prev_op) if recv_prev_shape_tensor is not None: recv_prev_op = torch.distributed.P2POp( - torch.distributed.irecv, recv_prev_shape_tensor, self.prev_rank + torch.distributed.irecv, recv_prev_shape_tensor, self.prev_rank, self.pp_group ) ops.append(recv_prev_op) if send_next_shape_tensor is not None: send_next_op = torch.distributed.P2POp( - torch.distributed.isend, send_next_shape_tensor, self.next_rank + torch.distributed.isend, send_next_shape_tensor, self.next_rank, self.pp_group ) ops.append(send_next_op) if recv_next_shape_tensor is not None: recv_next_op = torch.distributed.P2POp( - torch.distributed.irecv, recv_next_shape_tensor, self.next_rank + torch.distributed.irecv, recv_next_shape_tensor, self.next_rank, self.pp_group ) ops.append(recv_next_op) if len(ops) > 0: @@ -298,13 +298,13 @@ def _communicate( tensor_recv_prev_func = None tensor_recv_next_func = None - if not config.variable_seq_lengths: - recv_prev_shape = tensor_shape - recv_next_shape = tensor_shape - else: + if config.variable_seq_lengths or config.mtp_standalone: recv_prev_shape, recv_next_shape = self._communicate_shapes( tensor_send_next, tensor_send_prev, recv_prev, recv_next ) + else: + recv_prev_shape = tensor_shape + recv_next_shape = tensor_shape def create_tensor_recv_prev(): return torch.empty( diff --git a/megatron/core/pipeline_parallel/schedules.py b/megatron/core/pipeline_parallel/schedules.py index e83f8d90635..11e54e0fa53 100644 --- a/megatron/core/pipeline_parallel/schedules.py +++ b/megatron/core/pipeline_parallel/schedules.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import contextlib from functools import partial @@ -9,6 +9,9 @@ from megatron.core import parallel_state from megatron.core.enums import ModelType +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_reset, +) from megatron.core.pipeline_parallel.p2p_communication import P2PCommunicator from megatron.core.pipeline_parallel.utils import ( is_pp_first_stage, @@ -562,6 +565,9 @@ def forward_backward_no_pipelining( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + no_sync_func = config.no_sync_func if no_sync_func is None: no_sync_func = contextlib.nullcontext @@ -648,7 +654,7 @@ def forward_backward_no_pipelining( if ( hasattr(config, 'cuda_graph_impl') and config.cuda_graph_impl == "local" - and config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in config.cuda_graph_scope ): create_cudagraphs() @@ -898,6 +904,9 @@ def forward_backward_pipelining_with_interleaving( adjust_tensor_shapes_fn is None ), "adjust_tensor_shapes_fn is not supported for interleaved pipeline parallelism" + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + if config.overlap_p2p_comm and config.batch_p2p_comm: raise ValueError("Can not use both overlap_p2p_comm and batch_p2p_comm") @@ -1912,7 +1921,7 @@ def pp_post_backward(input_tensor_grad, vp_stage=None): if ( hasattr(config, 'cuda_graph_impl') and config.cuda_graph_impl == "local" - and config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in config.cuda_graph_scope ): create_cudagraphs() nvtx_range_pop(suffix="misc") @@ -2043,6 +2052,9 @@ def forward_backward_pipelining_without_interleaving( if config.timers is not None: config.timers('forward-backward', log_level=1).start(barrier=config.barrier_with_L1_time) + if not forward_only and config.fine_grained_activation_offloading: + fine_grained_offloading_reset() + # Disable async grad reductions no_sync_func = config.no_sync_func if no_sync_func is None: @@ -2296,7 +2308,7 @@ def enable_grad_sync(): if ( hasattr(config, 'cuda_graph_impl') and config.cuda_graph_impl == "local" - and config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in config.cuda_graph_scope ): create_cudagraphs() diff --git a/megatron/core/post_training/modelopt/layers.py b/megatron/core/post_training/modelopt/layers.py index 0ca4a8e4070..e502b81ac2f 100644 --- a/megatron/core/post_training/modelopt/layers.py +++ b/megatron/core/post_training/modelopt/layers.py @@ -1,5 +1,6 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +import logging from typing import Callable, List, Optional import torch @@ -10,6 +11,8 @@ from megatron.core.transformer.transformer_layer import TransformerLayer from megatron.core.transformer.utils import make_sharded_tensors_for_checkpoint +logger = logging.getLogger(__name__) + try: import transformer_engine as te @@ -116,6 +119,7 @@ def __init__( tp_group: Optional[torch.distributed.ProcessGroup] = None, ): self.config = config + self.tp_group = tp_group self._return_bias = skip_bias_add and bias @@ -153,7 +157,11 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): if v.ndim == 0: state_dict[k] = v.view(1) sharded_state_dict = make_sharded_tensors_for_checkpoint( - state_dict, prefix, sharded_offsets=sharded_offsets + state_dict, + prefix, + sharded_offsets=sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) return sharded_state_dict @@ -229,7 +237,7 @@ def _report_quantize_tensor_info(self): if not isinstance(v, torch.Tensor): continue original_dtype, original_shape = self._original_tensor_info.get(k, ("-", "-")) - print( + logger.info( "{:<64} {:<16} {:<32} {:<16} {:<32}".format( k, original_dtype, original_shape, str(v.dtype), str(v.shape) ) diff --git a/megatron/core/ssm/gated_delta_net.py b/megatron/core/ssm/gated_delta_net.py new file mode 100644 index 00000000000..1bef6416ded --- /dev/null +++ b/megatron/core/ssm/gated_delta_net.py @@ -0,0 +1,562 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025, Songlin Yang, Jan Kautz, Ali Hatamizadeh. + +# Some of this code was adopted from https://github.com/huggingface/transformers +# This source code is licensed under the Apache license found in the +# LICENSE file in the root directory of this source tree. + +import logging +from dataclasses import dataclass, replace +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor + +from megatron.core.dist_checkpointing import ShardedTensor +from megatron.core.dist_checkpointing.mapping import ReplicaId, ShardedTensorFactory +from megatron.core.fp8_utils import get_fp8_align_size +from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.jit import jit_fuser +from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel import get_cuda_rng_tracker +from megatron.core.transformer import TransformerConfig +from megatron.core.transformer.identity_op import IdentityOp +from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.spec_utils import ModuleSpec, build_module +from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, + make_sharded_tensors_for_checkpoint, + sharded_state_dict_default, +) +from megatron.core.utils import deprecate_inference_params, nvtx_range_pop, nvtx_range_push + +# TODO: Implement GatedDeltaNetContextParallel +# from .gated_delta_net_context_parallel import GatedDeltaNetContextParallel + +try: + from fla.modules.l2norm import l2norm + from fla.ops.gated_delta_rule import chunk_gated_delta_rule + + HAVE_FLA = True +except ImportError: + chunk_gated_delta_rule = None + + HAVE_FLA = False + +try: + from causal_conv1d import causal_conv1d_fn +except ImportError: + causal_conv1d_fn = None + + +logger = logging.getLogger(__name__) + + +@dataclass +class GatedDeltaNetSubmodules: + """ + Contains the module specs for the input linear, output norm, and output linear layers. + """ + + in_proj: Union[ModuleSpec, type] = IdentityOp + out_norm: Union[ModuleSpec, type] = IdentityOp + out_proj: Union[ModuleSpec, type] = IdentityOp + + +class GatedDeltaNet(MegatronModule): + """Gated Delta Net (GDN) layer class + + GDN layer takes input with size [s, b, h] + and returns output of the same size. + """ + + def __init__( + self, + config: TransformerConfig, + submodules: GatedDeltaNetSubmodules, + layer_number: int = None, + bias: bool = False, + conv_bias: bool = False, + conv_init: Optional[float] = None, + use_qk_l2norm: bool = True, + A_init_range: Tuple[float, float] = (1, 16), + pg_collection: ProcessGroupCollection = None, + ): + """ + Args: + config: The config of the model. + submodules: Contains the module specs for the input and output linear layers. + layer_number: The layer number of this GDN layer. + bias: Whether to use bias in the linear layers. + conv_bias: Whether to use bias in the causal convolution. + conv_init: The initialization range for the causal convolution weights. + use_qk_l2norm: Whether to use L2 normalization in the kernel of the gated delta rule. + A_init_range: The initialization range for the attention weights. + pg_collection: The required process groups to use for tensor model parallel and context + parallel. + """ + + if not HAVE_FLA: + raise ImportError("FLA is not installed. Please install it with `pip install fla`.") + + super().__init__(config) + + # Attributes from arguments + self.layer_number = layer_number + self.bias = bias + self.conv_bias = conv_bias + self.conv_init = conv_init + assert A_init_range[0] >= 0 and A_init_range[1] >= A_init_range[0] + self.A_init_range = A_init_range + self.use_qk_l2norm = use_qk_l2norm + assert pg_collection is not None, "pg_collection must be provided for GatedDeltaNet" + self.pg_collection = pg_collection + self.tp_size = self.pg_collection.tp.size() + self.sp_size = self.tp_size if config.sequence_parallel else 1 + + # Attributes from config + self.config = config + self.hidden_size = config.hidden_size + self.act_fn = config.activation_func + self.activation = self.act_fn.__name__ + self.conv_kernel_dim = config.linear_conv_kernel_dim + self.key_head_dim = config.linear_key_head_dim + self.value_head_dim = config.linear_value_head_dim + self.num_key_heads = config.linear_num_key_heads + self.num_value_heads = config.linear_num_value_heads + self.qk_dim = self.key_head_dim * self.num_key_heads + self.v_dim = self.value_head_dim * self.num_value_heads + + # Input projection (hidden_states -> q, k, v, gate, beta, alpha) + # TODO: for now, output gate is forced for GDN. + # We may remove this restriction in the future. + self.in_proj_dim = self.qk_dim * 2 + self.v_dim * 2 + self.num_value_heads * 2 + if self.config.fp8: + fp8_align_size = get_fp8_align_size(self.config.fp8_recipe) + assert self.in_proj_dim % fp8_align_size == 0, ( + "For FP8, the innermost dimension of the GDN layer " + "input projection output tensor must be a multiple of 16." + ) + self.in_proj = build_module( + submodules.in_proj, + self.hidden_size, + self.in_proj_dim, + config=self.config, + init_method=self.config.init_method, + gather_output=False, + bias=bias, + skip_bias_add=False, + is_expert=False, + tp_comm_buffer_name="fc1", + tp_group=self.pg_collection.tp, + ) + + # Conv1d for QKV + self.conv_dim = self.qk_dim * 2 + self.v_dim + self.conv_dim_local_tp = self.conv_dim // self.tp_size + + # weight shape: [conv_dim, 1, d_conv] + # bias shape: [conv_dim] + self.conv1d = nn.Conv1d( + in_channels=self.conv_dim_local_tp, + out_channels=self.conv_dim_local_tp, + bias=conv_bias, + kernel_size=self.conv_kernel_dim, + groups=self.conv_dim_local_tp, + padding=self.conv_kernel_dim - 1, + device=torch.cuda.current_device(), + dtype=config.params_dtype, + ) + setattr(self.conv1d.weight, "tensor_model_parallel", True) + if conv_bias: + setattr(self.conv1d.bias, "tensor_model_parallel", True) + + # Time step projection (discretization) + self.num_v_heads_local_tp = self.num_value_heads // self.tp_size + # dt_bias parameter + self.dt_bias = nn.Parameter( + torch.empty( + self.num_v_heads_local_tp, + dtype=config.params_dtype, + device=torch.cuda.current_device(), + ) + ) + setattr(self.dt_bias, "tensor_model_parallel", True) + # A_log parameter + self.A_log = nn.Parameter( + torch.empty( + self.num_v_heads_local_tp, + dtype=config.params_dtype, + device=torch.cuda.current_device(), + ) + ) + setattr(self.A_log, "tensor_model_parallel", True) + + # Output layernorm before projection + self.out_norm = build_module( + submodules.out_norm, + config=self.config, + hidden_size=self.value_head_dim, + eps=self.config.layernorm_epsilon, + ) + + self.out_proj = build_module( + submodules.out_proj, + self.v_dim, + self.hidden_size, + config=self.config, + init_method=self.config.output_layer_init_method, + bias=bias, + input_is_parallel=True, + skip_bias_add=True, + is_expert=False, + tp_comm_buffer_name="fc2", + tp_group=self.pg_collection.tp, + ) + + # TODO: support CP + + self.reset_parameters() + + def reset_parameters(self): + """Reset the parameters.""" + if self.config.perform_initialization: + with get_cuda_rng_tracker().fork(): + # conv1d.weight + if self.conv_init is not None: + nn.init.uniform_(self.conv1d.weight, -self.conv_init, self.conv_init) + # dt_bias + torch.ones( + self.num_v_heads_local_tp, + out=self.dt_bias.data, + dtype=self.config.params_dtype, + device=torch.cuda.current_device(), + ) + # A_log + A = torch.empty( + self.num_v_heads_local_tp, + dtype=self.config.params_dtype, + device=torch.cuda.current_device(), + ).uniform_(*self.A_init_range) + self.A_log.data.copy_(A) + + def forward( + self, + hidden_states: Tensor, + attention_mask: Tensor, + key_value_states: Optional[Tensor] = None, + inference_context: Optional[BaseInferenceContext] = None, + rotary_pos_emb: Optional[Union[Tensor, Tuple[Tensor, Tensor]]] = None, + rotary_pos_cos: Optional[Tensor] = None, + rotary_pos_sin: Optional[Tensor] = None, + rotary_pos_cos_sin: Optional[Tensor] = None, + attention_bias: Optional[Tensor] = None, + packed_seq_params: Optional[PackedSeqParams] = None, + sequence_len_offset: Optional[int] = None, + *, + inference_params: Optional[BaseInferenceContext] = None, + ): + """ + Perform a forward pass through the GDN module. + + Args: + hidden_states (Tensor): Hidden states. + attention_mask (Tensor): Attention mask. + key_value_states (Optional[Tensor]): Key/value states (for cross attention). + inference_context (Optional[BaseInferenceContext]): Inference context that manages + KV cache. + rotary_pos_emb (Optional[Union[Tensor, Tuple[Tensor, Tensor]]]): Rotary + embedding tensor(s). + rotary_pos_cos (Optional[Tensor]): Rotary embedding cosine. + rotary_pos_sin (Optional[Tensor]): Rotary embedding sine. + rotary_pos_cos_sin (Optional[Tensor]): Combined rotary embedding cosine and sine. + attention_bias (Optional[Tensor]): Attention bias. + packed_seq_params (Optional[PackedSeqparams]): Parameters used for THD format. + sequence_len_offset (Optional[int]): Sequence length offset used for + inference CUDA graphs. + + Return: + (Tuple[Tensor, Tensor]) GDN output and bias. + + """ + # TODO: Deal with attention_mask + + inference_context = deprecate_inference_params(inference_context, inference_params) + + seq_len, batch, _ = hidden_states.shape + seq_len = seq_len * self.sp_size + + if inference_context is not None: + assert ( + inference_context.is_static_batching() + ), "GDN does not currently support dynamic inference batching." + assert not self.config.sequence_parallel + # TODO: support inference + raise NotImplementedError("GDN does not support inference for now.") + + if packed_seq_params is not None: + # TODO: support packed sequence + raise NotImplementedError("GDN does not support packed sequence for now.") + + # Input projection + nvtx_range_push(suffix="in_proj") + qkvzba, _ = self.in_proj(hidden_states) + nvtx_range_pop(suffix="in_proj") + + # Transpose: s b x --> b s x + # From sbhd to bshd format + qkvzba = qkvzba.transpose(0, 1) + + # Split, reorder, and reshape the tensor into q, k, v, gate, beta, alpha + qkv, gate, beta, alpha = torch.split( + qkvzba, + [ + (self.qk_dim * 2 + self.v_dim) // self.tp_size, + self.v_dim // self.tp_size, + self.num_value_heads // self.tp_size, + self.num_value_heads // self.tp_size, + ], + dim=-1, + ) + gate = gate.reshape(batch, seq_len, -1, self.value_head_dim) + beta = beta.reshape(batch, seq_len, -1) + alpha = alpha.reshape(batch, seq_len, -1) + + # Convolution on qkv + qkv = qkv.transpose(1, 2).contiguous() # b, s, d -> b, d, s + nvtx_range_push(suffix="conv1d") + if causal_conv1d_fn is None: + qkv = self.act_fn(self.conv1d(qkv)[..., :seq_len]) + else: + assert self.activation in ["silu", "swish"] + qkv = causal_conv1d_fn( + x=qkv, + weight=self.conv1d.weight.squeeze(1), # d, 1, w -> d, w + bias=self.conv1d.bias, + activation=self.activation, + ) + nvtx_range_pop(suffix="conv1d") + # Split qkv into query, key, and value + qkv = qkv.transpose(1, 2) # b, d, s -> b, s, d + query, key, value = torch.split( + qkv, + [self.qk_dim // self.tp_size, self.qk_dim // self.tp_size, self.v_dim // self.tp_size], + dim=-1, + ) + query = query.reshape(batch, seq_len, -1, self.key_head_dim) + key = key.reshape(batch, seq_len, -1, self.key_head_dim) + value = value.reshape(batch, seq_len, -1, self.value_head_dim) + # Apply L2 norm to query and key + if self.use_qk_l2norm: + query = l2norm(query.contiguous()) + key = l2norm(key.contiguous()) + if self.num_value_heads // self.num_key_heads > 1: + query = query.repeat_interleave(self.num_value_heads // self.num_key_heads, dim=2) + key = key.repeat_interleave(self.num_value_heads // self.num_key_heads, dim=2) + + # Make contiguous + query = query.contiguous() + key = key.contiguous() + value = value.contiguous() + gate = gate.contiguous() + beta = beta.contiguous() + alpha = alpha.contiguous() + + # Calculate g and beta + nvtx_range_push(suffix="g_and_beta") + g = -self.A_log.exp() * F.softplus(alpha.float() + self.dt_bias) # In fp32 + beta = beta.sigmoid() + nvtx_range_pop(suffix="g_and_beta") + + nvtx_range_push(suffix="gated_delta_rule") + core_attn_out, last_recurrent_state = chunk_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=None, + output_final_state=False, + use_qk_l2norm_in_kernel=False, + ) + nvtx_range_pop(suffix="gated_delta_rule") + + # RMSNorm + nvtx_range_push(suffix="gated_norm") + norm_out = self._apply_gated_norm(core_attn_out, gate) + nvtx_range_pop(suffix="gated_norm") + + # Transpose: b s x --> s b x + # From bshd back to sbhd format + norm_out = norm_out.reshape(batch, seq_len, -1) + norm_out = norm_out.transpose(0, 1).contiguous() + + # Output projection + nvtx_range_push(suffix="out_proj") + out, out_bias = self.out_proj(norm_out) + nvtx_range_pop(suffix="out_proj") + + return out, out_bias + + @jit_fuser + def _apply_gated_norm(self, x, gate): + # Output Norm + x_dtype = x.dtype + x = x.reshape(-1, x.shape[-1]) + y = self.out_norm(x) + # Output gate + gate = gate.reshape(-1, gate.shape[-1]) + y = y * self.act_fn(gate.float()) + y = y.to(x_dtype) + return y + + def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None, tp_group=None): + """Provide a sharded state dictionary for distributed checkpointing.""" + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) + + sharded_state_dict = {} + # Parameters + self._save_to_state_dict(sharded_state_dict, "", keep_vars=True) + sharded_state_dict = make_sharded_tensors_for_checkpoint( + sharded_state_dict, + prefix, + tensor_parallel_layers_axis_map={ + "A_log": 0, + "dt_bias": 0, + }, # parameters sharded across TP + sharded_offsets=sharded_offsets, + tp_group=(tp_group if tp_group is not None else self.pg_collection.tp), + dp_cp_group=metadata['dp_cp_group'], + ) + # Submodules + tp_group = tp_group if tp_group is not None else self.pg_collection.tp + for name, module in self.named_children(): + if name == "conv1d": + # Add TP sharding for Conv1d + module_sd = module.state_dict(prefix="", keep_vars=True) + tp_sharding_map = {f"weight": 0} + if self.conv_bias: + tp_sharding_map[f"bias"] = 0 + module_sharded_sd = make_sharded_tensors_for_checkpoint( + module_sd, + f"{prefix}{name}.", + tp_sharding_map, + sharded_offsets, + tp_group=tp_group, + dp_cp_group=metadata['dp_cp_group'], + ) + else: + module_sharded_sd = sharded_state_dict_default( + module, f"{prefix}{name}.", sharded_offsets, metadata, tp_group=tp_group + ) + + sharded_state_dict.update(module_sharded_sd) + + # At this point the TP sharding is correctly defined for each tensor, but some of the + # tensors must be additionally split into separate parts + in_proj_dim_local_tp = self.in_proj_dim // self.tp_size + assert sharded_state_dict[f"{prefix}in_proj.weight"].data.size(0) == in_proj_dim_local_tp, ( + in_proj_dim_local_tp, + sharded_state_dict[f"{prefix}in_proj.weight"], + ) + + sharded_state_dict[f"{prefix}in_proj.weight"] = _split_tensor_factory( + sharded_state_dict[f"{prefix}in_proj.weight"], + [ + self.qk_dim // self.tp_size, + self.qk_dim // self.tp_size, + self.v_dim // self.tp_size, + self.v_dim // self.tp_size, + self.num_value_heads // self.tp_size, + self.num_value_heads // self.tp_size, + ], + ["query", "key", "value", "z", "beta", "alpha"], + 0, + ) + + conv_layer_name_list = ["conv1d.weight"] + assert ( + sharded_state_dict[f"{prefix}conv1d.weight"].data.size(0) == self.conv_dim_local_tp + ), (self.conv_dim_local_tp, sharded_state_dict[f"{prefix}conv1d.weight"]) + if self.conv_bias: + conv_layer_name_list.append("conv1d.bias") + assert ( + sharded_state_dict[f"{prefix}conv1d.bias"].data.size(0) == self.conv_dim_local_tp + ), (self.conv_dim_local_tp, sharded_state_dict[f"{prefix}conv1d.bias"]) + for conv_layer_name in conv_layer_name_list: + sharded_state_dict[f"{prefix}{conv_layer_name}"] = _split_tensor_factory( + sharded_state_dict[f"{prefix}{conv_layer_name}"], + [ + self.qk_dim // self.tp_size, + self.qk_dim // self.tp_size, + self.v_dim // self.tp_size, + ], + ["query", "key", "value"], + 0, + ) + + return sharded_state_dict + + +def _split_tensor_factory( + orig_sh_ten: ShardedTensor, split_sections: List[int], split_names: List[str], split_dim: int +) -> ShardedTensorFactory: + """Builds a factory that splits a given ShardedTensor into several independent chunks.""" + assert isinstance(orig_sh_ten, ShardedTensor), type(orig_sh_ten) + orig_sh_ten_no_data = orig_sh_ten.without_data() # remove `data` reference + + if sum(split_sections) != orig_sh_ten_no_data.local_shape[split_dim]: + raise ValueError( + f"Split sections must cover the whole dimension size, " + f"got {split_sections=} vs dimensions size " + f"{orig_sh_ten_no_data.local_shape[split_dim]}" + ) + + assert not isinstance( + split_sections, int + ), "Splitting into predefined section sizes is supported (`split_sections` must be a list)" + assert len(split_sections) == len(split_names), (len(split_sections), len(split_names)) + + @torch.no_grad() + def sh_ten_build_fn( + key: str, t: torch.Tensor, replica_id: ReplicaId, flattened_range: Optional[slice] + ): + factory_sh_ten = replace( + orig_sh_ten_no_data, + key=key, + data=t, + dtype=t.dtype, + replica_id=replica_id, + flattened_range=flattened_range, + ) + + chunk_sh_tens = [] + split_start = 0 + for split_size, split_name in zip(split_sections, split_names): + split_chunks = factory_sh_ten.narrow(split_dim, split_start, split_size) + for sh_ten in split_chunks: + sh_ten.key = f"{sh_ten.key}.{split_name}" + chunk_sh_tens.extend(split_chunks) + split_start += split_size + + assert split_start == orig_sh_ten_no_data.local_shape[split_dim], ( + split_start, + orig_sh_ten_no_data.local_shape[split_dim], + ) + assert sum(sh_ten.data.numel() for sh_ten in chunk_sh_tens) == t.numel(), ( + chunk_sh_tens, + t.shape, + ) + return chunk_sh_tens + + @torch.no_grad() + def sh_ten_merge_fn(sub_state_dict): + return torch.cat(sub_state_dict) + + return ShardedTensorFactory( + orig_sh_ten.key, orig_sh_ten.data, sh_ten_build_fn, sh_ten_merge_fn, orig_sh_ten.replica_id + ) diff --git a/megatron/core/ssm/mamba_block.py b/megatron/core/ssm/mamba_block.py index 7d8ca74c8f2..1bcadd0af10 100644 --- a/megatron/core/ssm/mamba_block.py +++ b/megatron/core/ssm/mamba_block.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Copyright (c) 2024, Tri Dao, Albert Gu. # Some of this code was adopted from https://github.com/state-spaces/mamba/ @@ -139,6 +139,7 @@ def __init__( assert pg_collection is not None, "pg_collection must be provided for MambaStack" self.pp_group = pg_collection.pp + self.tp_group = pg_collection.tp # Required for pipeline parallel schedules self.input_tensor = None @@ -203,13 +204,14 @@ def __init__( eps=self.config.layernorm_epsilon, ) - self.apply( - partial( - _init_weights, - n_layer=self.config.num_layers, - initializer_range=self.config.init_method_std, + if self.config.perform_initialization: + self.apply( + partial( + _init_weights, + n_layer=self.config.num_layers, + initializer_range=self.config.init_method_std, + ) ) - ) def _select_layers_for_pipeline_parallel(self, layer_type_list): num_layers_per_pipeline_rank = self.config.num_layers // self.pp_group.size() @@ -292,7 +294,7 @@ def forward( ( ( self.config.cuda_graph_impl == "local" - and self.config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in self.config.cuda_graph_scope ) or self.config.flash_decode ) @@ -407,7 +409,11 @@ def sharded_state_dict( if not module is self.layers: sharded_state_dict.update( sharded_state_dict_default( - module, f'{prefix}{name}.', sharded_offsets, metadata + module, + f'{prefix}{name}.', + sharded_offsets, + metadata, + tp_group=self.tp_group, ) ) diff --git a/megatron/core/ssm/mamba_mixer.py b/megatron/core/ssm/mamba_mixer.py index 895792ff05e..b792f8a2f1f 100644 --- a/megatron/core/ssm/mamba_mixer.py +++ b/megatron/core/ssm/mamba_mixer.py @@ -15,6 +15,7 @@ import torch.nn as nn import torch.nn.functional as F +from megatron.core import parallel_state from megatron.core.dist_checkpointing import ShardedTensor from megatron.core.dist_checkpointing.mapping import ReplicaId, ShardedTensorFactory from megatron.core.inference.contexts import BaseInferenceContext, DynamicInferenceContext @@ -24,6 +25,7 @@ from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, make_sharded_tensors_for_checkpoint, sharded_state_dict_default, ) @@ -79,9 +81,16 @@ class ExtendedRMSNorm(RMSNormGated): def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 0, bias not sharded""" + if not hasattr(self, 'tp_group'): + self.tp_group = parallel_state.get_tensor_model_parallel_group() state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 0}, sharded_offsets + state_dict, + prefix, + {"weight": 0}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata["dp_cp_group"], ) @@ -273,60 +282,68 @@ def __init__( ) conv_dim = self.d_inner_local_tp + 2 * self.ngroups_local_tp * self.d_state # x B C - with get_cuda_rng_tracker().fork(): - # weight shape: [conv_dim, 1, d_conv] - # bias shape: [conv_dim] - self.conv1d = nn.Conv1d( - in_channels=conv_dim, - out_channels=conv_dim, - bias=conv_bias, - kernel_size=d_conv, - groups=conv_dim, - padding=d_conv - 1, - device=torch.cuda.current_device(), - dtype=config.params_dtype, - ) - setattr(self.conv1d.weight, "tensor_model_parallel", True) - setattr(self.conv1d.bias, "tensor_model_parallel", True) + # weight shape: [conv_dim, 1, d_conv] + # bias shape: [conv_dim] + self.conv1d = nn.Conv1d( + in_channels=conv_dim, + out_channels=conv_dim, + bias=conv_bias, + kernel_size=d_conv, + groups=conv_dim, + padding=d_conv - 1, + device=torch.cuda.current_device(), + dtype=config.params_dtype, + ) + setattr(self.conv1d.weight, "tensor_model_parallel", True) + setattr(self.conv1d.bias, "tensor_model_parallel", True) - if self.conv_init is not None: + if self.config.perform_initialization and self.conv_init is not None: + with get_cuda_rng_tracker().fork(): nn.init.uniform_(self.conv1d.weight, -self.conv_init, self.conv_init) self.activation = "silu" self.act = nn.SiLU() - with get_cuda_rng_tracker().fork(): - # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max - dt = torch.exp( - torch.rand( - self.nheads_local_tp, - device=torch.cuda.current_device(), - dtype=config.params_dtype, - ) - * (math.log(dt_max) - math.log(dt_min)) - + math.log(dt_min) - ).clamp(min=dt_init_floor) - # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 - inv_dt = dt + torch.log(-torch.expm1(-dt)) - self.dt_bias = nn.Parameter(inv_dt) - # Our initialization would set all Linear.bias to zero, - # need to mark this one as _no_reinit - self.dt_bias._no_reinit = True - # Just to be explicit. Without this we already don't - # put wd on dt_bias because of the check - # name.endswith("bias") in param_grouping.py - self.dt_bias._no_weight_decay = True - setattr(self.dt_bias, "tensor_model_parallel", True) - - # A parameter - assert A_init_range[0] > 0 and A_init_range[1] >= A_init_range[0] - A = torch.empty( - self.nheads_local_tp, dtype=torch.float32, device=torch.cuda.current_device() - ).uniform_(*A_init_range) - A_log = torch.log(A) # Keep A_log in fp32 - self.A_log = nn.Parameter(A_log) - self.A_log._no_weight_decay = True - setattr(self.A_log, "tensor_model_parallel", True) + if self.config.perform_initialization: + with get_cuda_rng_tracker().fork(): + # Initialize dt bias so that F.softplus(dt_bias) is between dt_min and dt_max + dt = torch.exp( + torch.rand( + self.nheads_local_tp, + device=torch.cuda.current_device(), + dtype=config.params_dtype, + ) + * (math.log(dt_max) - math.log(dt_min)) + + math.log(dt_min) + ).clamp(min=dt_init_floor) + # Inverse of softplus: https://github.com/pytorch/pytorch/issues/72759 + inv_dt = dt + torch.log(-torch.expm1(-dt)) + else: + inv_dt = torch.empty( + self.nheads_local_tp, device=torch.cuda.current_device(), dtype=config.params_dtype + ) + + self.dt_bias = nn.Parameter(inv_dt) + # Our initialization would set all Linear.bias to zero, + # need to mark this one as _no_reinit + self.dt_bias._no_reinit = True + # Just to be explicit. Without this we already don't + # put wd on dt_bias because of the check + # name.endswith("bias") in param_grouping.py + self.dt_bias._no_weight_decay = True + setattr(self.dt_bias, "tensor_model_parallel", True) + + # A parameter + assert A_init_range[0] > 0 and A_init_range[1] >= A_init_range[0] + A = torch.empty( + self.nheads_local_tp, dtype=torch.float32, device=torch.cuda.current_device() + ) + if self.config.perform_initialization: + A = A.uniform_(*A_init_range) + A_log = torch.log(A) # Keep A_log in fp32 + self.A_log = nn.Parameter(A_log) + self.A_log._no_weight_decay = True + setattr(self.A_log, "tensor_model_parallel", True) # D "skip" parameter self.D = nn.Parameter( @@ -382,6 +399,7 @@ def __init__( D_cp1=self.D, D_has_hdim=self.D_has_hdim, ) + self.tp_group = pg_collection.tp def forward( self, @@ -1032,6 +1050,9 @@ def _get_states_from_cache(self, inference_context, batch_size, *, inference_par def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Provide a sharded state dictionary for distributed checkpointing.""" + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) + sharded_state_dict = {} # Parameters self._save_to_state_dict(sharded_state_dict, "", keep_vars=True) @@ -1051,12 +1072,17 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): # Add TP sharding for Conv1d module_sd = module.state_dict(prefix="", keep_vars=True) module_sharded_sd = make_sharded_tensors_for_checkpoint( - module_sd, f"{prefix}{name}.", {f"weight": 0, f"bias": 0}, sharded_offsets + module_sd, + f"{prefix}{name}.", + {f"weight": 0, f"bias": 0}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) else: module_sharded_sd = sharded_state_dict_default( - module, f"{prefix}{name}.", sharded_offsets, metadata + module, f"{prefix}{name}.", sharded_offsets, metadata, tp_group=self.tp_group ) sharded_state_dict.update(module_sharded_sd) diff --git a/megatron/core/tensor_parallel/layers.py b/megatron/core/tensor_parallel/layers.py index e6e65425b23..221f3327e50 100644 --- a/megatron/core/tensor_parallel/layers.py +++ b/megatron/core/tensor_parallel/layers.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch @@ -56,6 +56,8 @@ HAVE_TE = False _MODEL_PARALLEL_ATTRIBUTE_DEFAULTS = { + "expert_tp": False, + "is_qkv": False, "tensor_model_parallel": False, "partition_dim": -1, "partition_stride": 1, @@ -247,6 +249,10 @@ def __init__( rank=get_pg_rank(self.tp_group), world_size=get_pg_size(self.tp_group), ) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=0, stride=1 + ) else: self.weight = Parameter( torch.empty( @@ -258,6 +264,10 @@ def __init__( ) if config.perform_initialization: _initialize_affine_weight_gpu(self.weight, init_method, partition_dim=0, stride=1) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=0, stride=1 + ) def forward(self, input_): """Forward. @@ -310,6 +320,8 @@ def sharded_state_dict( key=weight_prefix, allow_shape_mismatch=True, prepend_offsets=sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata["dp_cp_group"], ) } @@ -857,6 +869,10 @@ def __init__( rank=rank, world_size=world_size, ) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=0, stride=stride + ) else: self.weight = Parameter( torch.empty( @@ -874,6 +890,10 @@ def __init__( stride=stride, is_expert=self.is_expert, ) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=0, stride=stride + ) setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel)) else: @@ -1046,7 +1066,12 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 0, bias sharded""" state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 0, "bias": 0}, sharded_offsets + state_dict, + prefix, + {"weight": 0, "bias": 0}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) def set_extra_state(self, state: Any): @@ -1169,6 +1194,10 @@ def __init__( rank=rank, world_size=world_size, ) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=1, stride=stride + ) else: self.weight = Parameter( torch.empty( @@ -1186,6 +1215,10 @@ def __init__( stride=stride, is_expert=self.is_expert, ) + else: + set_tensor_model_parallel_attributes( + tensor=self.weight, is_parallel=True, dim=1, stride=stride + ) setattr(self.weight, "allreduce", not (self.is_expert and self.expert_parallel)) if bias: @@ -1284,7 +1317,12 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None): """Sharding along axis 1, bias not sharded""" state_dict = self.state_dict(prefix="", keep_vars=True) return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {"weight": 1}, sharded_offsets + state_dict, + prefix, + {"weight": 1}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) def set_extra_state(self, state: Any): diff --git a/megatron/core/tensor_parallel/random.py b/megatron/core/tensor_parallel/random.py index 54cac0e41e3..396e5c54a2d 100644 --- a/megatron/core/tensor_parallel/random.py +++ b/megatron/core/tensor_parallel/random.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch @@ -510,10 +510,14 @@ def forward(ctx, run_function, checkpoint_without_output_obj, *args): @staticmethod def backward(ctx, *args): """Backward pass.""" - inputs = ctx.saved_tensors + # Get the inputs from the context instead of the saved tensors + # because the saved tensors are already cached by the recomputation. + # This is to avoid double-reloading the inputs in CPU offloading scenario. + inputs = ctx.inputs outputs = ctx.outputs torch.autograd.backward(outputs, args) ctx.outputs = None + ctx.inputs = None grads = tuple(inp.grad if isinstance(inp, torch.Tensor) else inp for inp in inputs) return (None, None) + grads @@ -573,8 +577,19 @@ def _recompute(self, _): recompute_ctx = contextlib.nullcontext() fp8_ctx = contextlib.nullcontext() + # Store the inputs for backward pass + inputs = self.ctx.saved_tensors + + def detach(t): + if isinstance(t, torch.Tensor): + requires_grad = t.requires_grad + t = t.detach() + t.requires_grad_(requires_grad) + return t + + inputs = tuple(detach(t) for t in inputs) with torch.enable_grad(), fp8_ctx, recompute_ctx: - outputs = self.run_function(*self.ctx.saved_tensors) + outputs = self.run_function(*inputs) self.run_function = None self.rng_states = None @@ -590,6 +605,7 @@ def _recompute(self, _): output.untyped_storage().copy_(recomputation_output.untyped_storage()) self.ctx.outputs = outputs + self.ctx.inputs = inputs self.outputs = None self.ctx = None diff --git a/megatron/core/timers.py b/megatron/core/timers.py index 75387c33d3a..03d1f2bd25b 100644 --- a/megatron/core/timers.py +++ b/megatron/core/timers.py @@ -270,6 +270,9 @@ def _get_elapsed_time_all_ranks(self, names, reset, barrier): torch.tensor: Tensor of size [world_size, len(names)] with times in float. """ + if len(names) == 0: + return None + # First make sure all the callers are in sync. if barrier: torch.distributed.barrier() @@ -302,16 +305,19 @@ def _get_global_min_max_time(self, names, reset, barrier, normalizer): """Report only min and max times across all ranks.""" rank_name_to_time = self._get_elapsed_time_all_ranks(names, reset, barrier) + # Using Python built-in methods to avoid the overhead of PyTorch operations. + rank_name_to_time = ( + rank_name_to_time.permute(1, 0).tolist() if rank_name_to_time is not None else None + ) name_to_min_max_time = {} for i, name in enumerate(names): - rank_to_time = rank_name_to_time[:, i] # filter out the ones we did not have any timings for - rank_to_time = rank_to_time[rank_to_time > 0.0] + rank_to_time = list(filter(lambda x: x > 0.0, rank_name_to_time[i])) # If the timer exists: - if rank_to_time.numel() > 0: + if len(rank_to_time) > 0: name_to_min_max_time[name] = ( - rank_to_time.min().item() / normalizer, - rank_to_time.max().item() / normalizer, + min(rank_to_time) / normalizer, + max(rank_to_time) / normalizer, ) return name_to_min_max_time @@ -427,7 +433,7 @@ def log( if rank is None: rank = torch.distributed.get_world_size() - 1 if rank == torch.distributed.get_rank() and output_string is not None: - print(output_string, flush=True) + print(output_string, flush=True) # pylint: disable=W0141 def write( self, diff --git a/megatron/core/transformer/attention.py b/megatron/core/transformer/attention.py index 187222bef3d..606befa1066 100644 --- a/megatron/core/transformer/attention.py +++ b/megatron/core/transformer/attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from abc import ABC, abstractmethod from dataclasses import dataclass @@ -9,6 +9,7 @@ from megatron.core import tensor_parallel from megatron.core.inference.contexts import BaseInferenceContext +from megatron.core.jit import jit_fuser from megatron.core.models.common.embeddings.rope_utils import ( apply_rotary_pos_emb, apply_rotary_pos_emb_with_cos_sin, @@ -22,6 +23,11 @@ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.module import MegatronModule @@ -170,6 +176,7 @@ def __init__( pg_collection, 'cp' ), "Attention pg_collection must have cp process group" self.pg_collection = pg_collection + self.tp_group = pg_collection.tp # Per attention head and per partition values world_size = get_pg_size(self.pg_collection.tp) @@ -199,6 +206,21 @@ def __init__( and "core_attn" in self.config.recompute_modules ) + self.offload_qkv_linear = ( + self.config.fine_grained_activation_offloading + and "qkv_linear" in self.config.offload_modules + ) + + self.offload_core_attention = ( + self.config.fine_grained_activation_offloading + and "core_attn" in self.config.offload_modules + ) + + self.offload_attn_proj = ( + self.config.fine_grained_activation_offloading + and "attn_proj" in self.config.offload_modules + ) + # Output. self.linear_proj = build_module( submodules.linear_proj, @@ -445,7 +467,9 @@ def _adjust_key_value_for_inference( return query, key, value, rotary_pos_emb, attn_mask_type, block_table @abstractmethod - def get_query_key_value_tensors(self, hidden_states, key_value_states, split_qkv=True): + def get_query_key_value_tensors( + self, hidden_states, key_value_states, output_gate, split_qkv=True + ): """ This method needs to be implemented based on whether the derived class is "self-attn" or "cross-attn". @@ -729,19 +753,33 @@ def forward( self.k_layernorm is None or isinstance(self.k_layernorm, IdentityOp), ] ) + output_gate = self.config.attention_output_gate # Check if fused_single_qkv_rope is requested but either unavailable or not # supported for the current use case. if self.attention_type != "cross": assert not ( self.config.fused_single_qkv_rope and split_qkv ), "fused_single_qkv_rope requested but not available/supported for the config." + if output_gate: + assert split_qkv, "output_gate is not supported for unsplit mixed_qkv tensor." + + if self.offload_qkv_linear: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="qkv_linear") + with get_fine_grained_offloading_context(self.offload_qkv_linear): + qkv_output = self.get_query_key_value_tensors( + hidden_states, key_value_states, output_gate=output_gate, split_qkv=split_qkv + ) + if self.offload_qkv_linear: + (qkv_output,) = fine_grained_offloading_group_commit( + qkv_output, name="qkv_linear", forced_released_tensors=[] + ) - qkv_output = self.get_query_key_value_tensors( - hidden_states, key_value_states, split_qkv=split_qkv - ) attn_mask_type = self.attn_mask_type block_table = None - if split_qkv: + gate = None + if output_gate and split_qkv: + query, key, value, gate = qkv_output + elif split_qkv: query, key, value = qkv_output else: mixed_qkv, qkv_split_arg_list = qkv_output @@ -785,7 +823,7 @@ def forward( if ( in_decode_mode and self.config.cuda_graph_impl == "local" - and self.config.cuda_graph_scope != "full_iteration" + and "full_iteration" not in self.config.cuda_graph_scope and inference_context.is_static_batching() ): raise ValueError(f"CUDA graphs must use flash decode with static batching!") @@ -884,17 +922,20 @@ def forward( packed_seq_params=packed_seq_params, ) else: + if self.offload_core_attention and self.training: + query = fine_grained_offloading_group_start(query, name="core_attn") if inference_context is None or inference_context.is_static_batching(): # Static batching attention kernel. - core_attn_out = self.core_attention( - query, - key, - value, - attention_mask, - attn_mask_type=attn_mask_type, - attention_bias=attention_bias, - packed_seq_params=packed_seq_params, - ) + with get_fine_grained_offloading_context(self.offload_core_attention): + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + attn_mask_type=attn_mask_type, + attention_bias=attention_bias, + packed_seq_params=packed_seq_params, + ) else: # Dynamic batching attention kernel. @@ -914,6 +955,10 @@ def forward( block_table, ) core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') + if self.offload_core_attention and self.training: + (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] + ) if packed_seq_params is not None and packed_seq_params.qkv_format == 'thd': # reshape to same output shape as unpacked case @@ -923,20 +968,49 @@ def forward( core_attn_out = core_attn_out.reshape(core_attn_out.size(0), 1, -1) nvtx_range_pop(suffix="core_attention") + # Output gate + if gate is not None: + nvtx_range_push(suffix="output_gate") + core_attn_out = self._apply_output_gate(core_attn_out, gate) + nvtx_range_pop(suffix="output_gate") + # ================= # Output. [sq, b, h] # ================= nvtx_range_push(suffix="linear_proj") - output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") + with get_fine_grained_offloading_context(self.offload_attn_proj): + output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + output, bias = fine_grained_offloading_group_commit( + output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + ) nvtx_range_pop(suffix="linear_proj") return output, bias + @jit_fuser + def _apply_output_gate(self, x, gate): + x_dtype = x.dtype + gate = gate.contiguous() + gate = gate.view(*x.shape) + x = x * torch.sigmoid(gate.float()) + x = x.to(x_dtype) + return x + def set_for_recompute_input_layernorm(self): """Set the attention layer for recompute input_layernorm. Only needed for fp8.""" raise NotImplementedError("set_for_recompute_input_layernorm is not implemented.") + def clip_qk(self): + """ + QK Clipping is a technique to clip the query and key attention logits to prevent the + attention logits from exploding. + """ + raise NotImplementedError("clip_qk is not implemented.") + class SelfAttention(Attention): """Self-attention layer class @@ -964,10 +1038,13 @@ def __init__( pg_collection=pg_collection, ) + self.linear_qkv_out_dim = self.query_projection_size + 2 * self.kv_projection_size + if self.config.attention_output_gate: + self.linear_qkv_out_dim += self.config.kv_channels * self.config.num_attention_heads self.linear_qkv = build_module( submodules.linear_qkv, self.config.hidden_size, - self.query_projection_size + 2 * self.kv_projection_size, + self.linear_qkv_out_dim, config=self.config, init_method=self.config.init_method, gather_output=False, @@ -1069,50 +1146,69 @@ def _compare(srcs, tgts, names, parallelism): "TP", ) - def get_query_key_value_tensors(self, hidden_states, key_value_states=None, split_qkv=True): + def get_query_key_value_tensors( + self, hidden_states, key_value_states=None, output_gate=False, split_qkv=True + ): """ - Derives `query`, `key` and `value` tensors from `hidden_states`. If `split_qkv=False`, then - the unsplit mixed_qkv tensor is returned. + Derives `query`, `key`, `value` tensors from `hidden_states`. + If `output_gate` is True, then also derives `gate` tensor. + If `split_qkv=False`, then the unsplit mixed_qkv tensor is returned. """ - # Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)] + # If no output gate: Attention heads [sq, b, h] --> [sq, b, ng * (np/ng + 2) * hn)] + # If have output gate: Attention heads [sq, b, h] --> [sq, b, ng * (2 * np/ng + 2) * hn)] mixed_qkv, _ = self.linear_qkv(hidden_states) + num_query_heads_per_group = ( + self.num_attention_heads_per_partition // self.num_query_groups_per_partition + ) + num_qkv_heads_per_group = num_query_heads_per_group + 2 + if output_gate: + num_qkv_heads_per_group += num_query_heads_per_group - # [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn] + # If no output gate: [sq, b, hp] --> [sq, b, ng, (np/ng + 2) * hn] + # If have output gate: [sq, b, hp] --> [sq, b, ng, (2 * np/ng + 2) * hn] new_tensor_shape = mixed_qkv.size()[:-1] + ( self.num_query_groups_per_partition, - ( - (self.num_attention_heads_per_partition // self.num_query_groups_per_partition + 2) - * self.hidden_size_per_attention_head - ), + num_qkv_heads_per_group * self.hidden_size_per_attention_head, ) mixed_qkv = mixed_qkv.view(*new_tensor_shape) - split_arg_list = [ - ( - self.num_attention_heads_per_partition - // self.num_query_groups_per_partition - * self.hidden_size_per_attention_head - ), - self.hidden_size_per_attention_head, - self.hidden_size_per_attention_head, - ] - - # Return unsplit mixed_qkv and split_arg_list - if not split_qkv: - return mixed_qkv, split_arg_list - - if SplitAlongDim is not None: + # Split the tensor into query, gate, key, and value. + if output_gate: + if not split_qkv: + raise ValueError("split_qkv not supported for gated attention yet.") + # If have output gate: [sq, b, ng, (2 * np/ng + 2) * hn] + # --> [sq, b, ng, np/ng * hn], [sq, b, ng, np/ng * hn], + # [sq, b, ng, hn], [sq, b, ng, hn] + split_arg_list = [ + num_query_heads_per_group * self.hidden_size_per_attention_head, + num_query_heads_per_group * self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + ] - # [sq, b, ng, (np/ng + 2) * hn] - # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] - (query, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list) + if SplitAlongDim is not None: + (query, gate, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list) + else: + (query, gate, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3) else: + # If no output gate: [sq, b, ng, (np/ng + 2) * hn] + # --> [sq, b, ng, np/ng * hn], None, [sq, b, ng, hn], [sq, b, ng, hn] + split_arg_list = [ + num_query_heads_per_group * self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + ] - # [sq, b, ng, (np/ng + 2) * hn] - # --> [sq, b, ng, np/ng * hn], [sq, b, ng, hn], [sq, b, ng, hn] - (query, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3) + # Return unsplit mixed_qkv and split_arg_list + if not split_qkv: + return mixed_qkv, split_arg_list + + if SplitAlongDim is not None: + (query, key, value) = SplitAlongDim(mixed_qkv, 3, split_arg_list) + else: + (query, key, value) = torch.split(mixed_qkv, split_arg_list, dim=3) - # [sq, b, ng, np/ng * hn] -> [sq, b, np, hn] + # Query [sq, b, ng, np/ng * hn] -> [sq, b, np, hn] query = query.reshape(query.size(0), query.size(1), -1, self.hidden_size_per_attention_head) if self.q_layernorm is not None: @@ -1124,6 +1220,11 @@ def get_query_key_value_tensors(self, hidden_states, key_value_states=None, spli if self.config.test_mode: self.run_realtime_tests() + if output_gate: + # Gate [sq, b, ng, np/ng * hn] -> [sq, b, np, hn] + gate = gate.reshape(*gate.shape[:2], -1, self.hidden_size_per_attention_head) + return query, key, value, gate + return query, key, value def backward_dw(self) -> NoReturn: @@ -1145,6 +1246,103 @@ def set_for_recompute_input_layernorm(self): set_save_original_input(self.linear_qkv) + def clip_qk(self): + """ + QK Clipping is a technique to clip the query and key attention logits to prevent the + attention logits from exploding. This function is experimental on GQA. + """ + if not self.config.qk_clip: + raise ValueError("qk_clip option needs to be enabled") + + if self.core_attention.current_max_attn_logits is None: + raise ValueError("current_max_attn_logits is None") + + assert self.core_attention.current_max_attn_logits.shape == ( + self.num_attention_heads_per_partition, + ), f"current_max_attn_logits shape is not ({self.num_attention_heads_per_partition}, ) \ + but {self.core_attention.current_max_attn_logits.shape}" + + grouped_max_attn_logits = torch.max( + self.core_attention.current_max_attn_logits.view( + self.num_query_groups_per_partition, -1 + ), + dim=1, + ).values + + # only update the weight if any head has + # current_max_attn_logits > qk_clip_threshold + if torch.any(grouped_max_attn_logits > self.config.qk_clip_threshold): + # Use num_query_groups_per_partition for tensor parallel scenarios + + # qk_clip_balancing_eta (g, 1, 1) + assert grouped_max_attn_logits.shape == ( + self.num_query_groups_per_partition, + ), f"current_max_attn_logits shape is not ({self.num_query_groups_per_partition},) \ + but {grouped_max_attn_logits.shape}" + self.qk_clip_balancing_eta = torch.clamp( + self.config.qk_clip_threshold / grouped_max_attn_logits, max=1.0 + ).view(self.num_query_groups_per_partition, 1, 1) + assert torch.all(self.qk_clip_balancing_eta <= 1.0) + + # Handle different weight access patterns (main_param vs direct access) + if hasattr(self.linear_qkv.weight, 'main_param'): + self.linear_qkv.weight.main_param.data.copy_( + self._clip_linear_qkv(self.linear_qkv.weight.main_param.data) + ) + + self.linear_qkv.weight.data.copy_(self._clip_linear_qkv(self.linear_qkv.weight.data)) + + # reset current_max_attn_logits + self.core_attention.current_max_attn_logits = None + + def _clip_linear_qkv(self, weight): + """Apply qkclip to linear_qkv layer""" + # Reshape to (g, query_projection_size + 2 * kv_projection_size, -1) + weight_reshaped = weight.view( + self.num_query_groups_per_partition, + (self.query_projection_size + 2 * self.kv_projection_size) + // self.num_query_groups_per_partition, + -1, + ) + + # Split into query_projection_size and 2 * kv_projection_size parts: + # (n, a, -1) and (n, b, -1) + weight_q = weight_reshaped[ + :, : self.query_projection_size // self.num_query_groups_per_partition, : + ] + weight_k = weight_reshaped[ + :, + self.query_projection_size + // self.num_query_groups_per_partition : ( + self.query_projection_size + self.kv_projection_size + ) + // self.num_query_groups_per_partition, + :, + ] + weight_v = weight_reshaped[ + :, + (self.query_projection_size + self.kv_projection_size) + // self.num_query_groups_per_partition :, + :, + ] + + # extend the qk_clip_balancing_eta to the same shape as weight_q and weight_k + self.qk_clip_balancing_eta_extended = self.qk_clip_balancing_eta.repeat( + 1, weight_q.size(1), 1 + ) + + # Clipping + weight_q.mul_(torch.pow(self.qk_clip_balancing_eta_extended, self.config.qk_clip_alpha)) + weight_k.mul_(torch.pow(self.qk_clip_balancing_eta, 1 - self.config.qk_clip_alpha)) + + # Concatenate back and reshape to original shape + weight_updated = torch.cat([weight_q, weight_k, weight_v], dim=1) + weight_updated = weight_updated.view( + self.query_projection_size + 2 * self.kv_projection_size, -1 + ) + + return weight_updated + class CrossAttention(Attention): """Cross-attention layer class @@ -1200,11 +1398,15 @@ def __init__( is_expert=False, ) - def get_query_key_value_tensors(self, hidden_states, key_value_states, split_qkv=True): + def get_query_key_value_tensors( + self, hidden_states, key_value_states, output_gate=False, split_qkv=True + ): """ Derives `query` tensor from `hidden_states`, and `key`/`value` tensors from `key_value_states`. """ + assert not output_gate, "Output gate is not supported in cross attention for now." + assert split_qkv, "split_qkv must be True for CrossAttention" # Attention heads [sk, b, h] --> [sk, b, (np * 2 * hn)] mixed_kv, _ = self.linear_kv(key_value_states) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index 9d50e34a2cc..12f15ee980a 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import gc import inspect @@ -22,7 +22,7 @@ get_cuda_rng_tracker, ) from megatron.core.transformer.identity_op import IdentityOp -from megatron.core.transformer.module import MegatronModule +from megatron.core.transformer.module import GraphableMegatronModule, MegatronModule from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.utils import ( get_attr_wrapped_model, @@ -1070,9 +1070,12 @@ def __init__( ), "RNG tracker does not support cudagraphs!" assert config.cuda_graph_impl == "local", "Option cuda_graph_impl=local not enabled." - assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), ( - "expandable_segments:True may not be safe when using CUDA Graphs, and may result in" - "a crash due to illegal memory access or other undefined behaviour." + assert ( + "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "") + or os.getenv("NCCL_GRAPH_REGISTER", "") == "0" + ), ( + "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using " + "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True." ) self.cudagraph_runners = [] @@ -1326,23 +1329,40 @@ def _layer_is_graphable(layer, config): Check if a layer is graphable. """ + # Only GraphableMegatronModule can be graphed. + if not isinstance(layer, GraphableMegatronModule): + return False + + # If cuda_graph_scope is not set, every layer is graphed. + if not config.cuda_graph_scope: + return True + # import modules here to avoid a circular import from megatron.core.ssm.mamba_layer import MambaLayer from megatron.core.transformer.identity_op import IdentityOp + from megatron.core.transformer.mlp import MLP + from megatron.core.transformer.moe.moe_layer import MoELayer from megatron.core.transformer.transformer_layer import TransformerLayer - if isinstance(layer, MambaLayer) and config.cuda_graph_scope == "full": + if isinstance(layer, MambaLayer) and 'mamba' in config.cuda_graph_scope: # mamba layer. return True if isinstance(layer, TransformerLayer): - if config.cuda_graph_scope == 'attn': - if not ( - isinstance(layer.self_attention, IdentityOp) - and isinstance(layer.cross_attention, IdentityOp) - ): - # attn layer. - return True - else: + if 'attn' in config.cuda_graph_scope and not ( + isinstance(layer.self_attention, IdentityOp) + and isinstance(layer.cross_attention, IdentityOp) + ): + # attn layer. + return True + if ( + 'moe' in config.cuda_graph_scope + or 'moe_router' in config.cuda_graph_scope + or 'moe_preprocess' in config.cuda_graph_scope + ) and isinstance(layer.mlp, MoELayer): + # moe layer. + return True + if 'mlp' in config.cuda_graph_scope and isinstance(layer.mlp, MLP): + # mlp layer. return True return False @@ -1361,18 +1381,17 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]): assert ( config.cuda_graph_impl == "transformer_engine" ), "Option cuda_graph_impl=transformer_engine not enabled." - assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), ( - "expandable_segments:True may not be safe when using CUDA Graphs, and may result in" - "a crash due to illegal memory access or other undefined behaviour." + assert ( + "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "") + or os.getenv("NCCL_GRAPH_REGISTER", "") == "0" + ), ( + "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using " + "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True." ) - assert config.cuda_graph_scope != "full_iteration", ( + assert "full_iteration" not in config.cuda_graph_scope, ( "full_iteration cuda graph is not supported for cuda_graph_impl=transformer_engine. " "Please use cuda_graph_impl=local instead." ) - assert config.cuda_graph_scope in [ - 'full', - 'attn', - ], f"--cuda-graph-scope should be full or attn, got {config.cuda_graph_scope}." self.model = model self.config = config @@ -1455,6 +1474,16 @@ def __init__(self, model, config, seq_length, micro_batch_size, optimizers=[]): f'{len(self.flattened_callables)} graphable layers.', ) + # One helper object can only capture CUDA Graphs once. Use this flag to check if the graphs + # have been created. + self._graphs_created = False + + def graphs_created(self): + """ + Returns whether the CUDA Graphs have been created. + """ + return self._graphs_created + def _get_cuda_graph_input_data(self): """ Create the CUDA Graph capturing input data. @@ -1495,8 +1524,13 @@ def get_rotary_pos_emb(transformer_module, transformer_input): from megatron.core.transformer.identity_op import IdentityOp from megatron.core.transformer.transformer_layer import TransformerLayer - contains_self_attn = isinstance(layer, TransformerLayer) and not isinstance( - layer.self_attention, IdentityOp + contains_self_attn = ( + isinstance(layer, TransformerLayer) + and not isinstance(layer.self_attention, IdentityOp) + and ( + not self.config.cuda_graph_scope + or 'attn' in self.config.cuda_graph_scope + ) ) if is_te_min_version("1.10.0"): # te.make_graphed_callables() accepts keyword arguments since 1.10.0. @@ -1605,6 +1639,8 @@ def _start_capturing(self): """ Start capturing CUDA Graphs. """ + assert not self._graphs_created, "CUDA Graphs have already been created." + torch.distributed.barrier() gc.collect() torch.cuda.empty_cache() @@ -1638,6 +1674,8 @@ def _finish_capturing(self, start_time): gc.collect() torch.cuda.empty_cache() + self._graphs_created = True + def create_cudagraphs(self): """ Capture CUDA Graphs per TransformerLayer per microbatch. diff --git a/megatron/core/transformer/dot_product_attention.py b/megatron/core/transformer/dot_product_attention.py index 2a6ac65a685..7102440552a 100644 --- a/megatron/core/transformer/dot_product_attention.py +++ b/megatron/core/transformer/dot_product_attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import math @@ -12,6 +12,9 @@ from megatron.core.fusions.fused_softmax import FusedScaleMaskSoftmax from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.dot_product_attention_context_parallel import ( + AttentionFuncionWithContextParallel, +) from megatron.core.transformer.enums import AttnMaskType from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.transformer_config import TransformerConfig @@ -54,9 +57,12 @@ def __init__( self.config: TransformerConfig = config - assert ( - self.config.context_parallel_size == 1 - ), "Context parallelism is only supported by TEDotProductAttention!" + if self.config.context_parallel_size > 1: + assert attention_dropout is None and self.config.attention_dropout == 0.0, ( + f'DotProductAttention with context parallelism does not support attention dropout,' + f' but got {self.config.context_parallel_size=},' + f' {attention_dropout=}, and {self.config.attention_dropout=}.' + ) self.layer_number = max(1, layer_number) self.attn_mask_type = attn_mask_type @@ -71,6 +77,8 @@ def __init__( assert hasattr( pg_collection, 'tp' ), "DotProductAttention pg_collection must have tp process group" + self.pg_collection = pg_collection + self.tp_group = self.pg_collection.tp world_size = pg_collection.tp.size() self.hidden_size_per_partition = divide(projection_size, world_size) @@ -116,13 +124,19 @@ def __init__( if self.config.softmax_type == "vanilla": self.softmax_offset = None elif self.config.softmax_type == "off-by-one": - self.softmax_offset = torch.zeros(self.num_attention_heads_per_partition) + self.softmax_offset = torch.zeros( + self.num_attention_heads_per_partition, + device=torch.cuda.current_device(), + dtype=self.config.params_dtype, + ) elif self.config.softmax_type == "learnable": self.register_parameter( "softmax_offset", torch.nn.Parameter( torch.empty( - self.num_attention_heads_per_partition, dtype=self.config.params_dtype + self.num_attention_heads_per_partition, + device=torch.cuda.current_device(), + dtype=self.config.params_dtype, ) ), ) @@ -166,6 +180,19 @@ def forward( self.num_attention_heads_per_partition // self.num_query_groups_per_partition, dim=2 ) + if self.config.context_parallel_size > 1: + output = AttentionFuncionWithContextParallel.apply( + query, + key, + value, + attention_mask, + self.config.attention_dropout, + self.softmax_scale, + parallel_state.get_context_parallel_group(), + ) + output = output.view(query.shape[0], query.shape[1], self.hidden_size_per_partition) + return output + # [b, np, sq, sk] output_size = (query.size(1), query.size(2), query.size(0), key.size(0)) @@ -254,5 +281,10 @@ def sharded_state_dict( else: state_dict = {} return make_sharded_tensors_for_checkpoint( - state_dict, prefix, {'softmax_offset': 0}, sharded_offsets + state_dict, + prefix, + {'softmax_offset': 0}, + sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) diff --git a/megatron/core/transformer/dot_product_attention_context_parallel.py b/megatron/core/transformer/dot_product_attention_context_parallel.py new file mode 100644 index 00000000000..89659a1d743 --- /dev/null +++ b/megatron/core/transformer/dot_product_attention_context_parallel.py @@ -0,0 +1,342 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +# Some of this code was adopted from https://github.com/zhuzilin/ring-flash-attention/ +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import torch +from torch.nn import functional as F + +try: + import einops + + HAVE_EINOPS = True +except ImportError: + HAVE_EINOPS = False + + +@torch.no_grad +def eager_attn_fwd(q, k, v, attn_bias, sinks, scale, dropout): + """Forward pass for eager attention""" + + # Rearrange query, key, value to (b, h, s, d) + b, sq, h, d = q.shape + sk = k.shape[1] + _q = einops.rearrange(q, 'b s h d -> b h s d') + _k = einops.rearrange(k, 'b s h d -> b h d s') + _v = einops.rearrange(v, 'b s h d -> b h s d') + + # Compute attention weights + attn_w = torch.matmul(_q, _k) * scale + attn_w = attn_w + attn_bias + + # Add sinks to attention weights + if sinks is None: + logits = attn_w + else: + _sinks = sinks.reshape(1, h, 1, 1).expand(b, -1, sq, 1) + logits = torch.cat([attn_w, _sinks], dim=-1) + + # Compute attention scores + probs = F.softmax(logits, dim=-1, dtype=logits.dtype) + if sinks is None: + attn_w = probs + else: + attn_w = probs[..., :-1] # Drop the sink + + # Compute attention output + attn_output = torch.matmul(attn_w, _v) + attn_output = einops.rearrange(attn_output, 'b h s d -> b s h d') + attn_output = attn_output.contiguous() + + return attn_output, probs + + +@torch.no_grad +def eager_attn_bwd(q, k, v, attn_bias, sinks, scale, dropout, attn_output, probs, grad_output): + """Backward pass for eager attention""" + + # Rearrange query, key, value to (b, h, s, d) + b, sq, h, d = q.shape + sk = k.shape[1] + _q_T = einops.rearrange(q, 'b s h d -> b h d s') + _k_T = einops.rearrange(k, 'b s h d -> b h s d') + _v_T = einops.rearrange(v, ' b s h d -> b h d s') + + # Backward pass for score @ value + if sinks is None: + attn_w = probs + else: + attn_w = probs[..., :-1] # Drop the sink + grad_output = einops.rearrange(grad_output, 'b s h d -> b h s d') + attn_w_T = einops.rearrange(attn_w, ' b h sq sk -> b h sk sq') + grad__v = torch.matmul(attn_w_T, grad_output) + grad_attn_w = torch.matmul(grad_output, _v_T) + + # Backward pass for softmax + if sinks is None: + grad_probs = grad_attn_w + else: + dummy = torch.zeros((b, h, sq, 1), device=q.device, dtype=q.dtype) + grad_probs = torch.cat([grad_attn_w, dummy], dim=3) + del grad_attn_w + grad_logits = torch._softmax_backward_data( + grad_probs, probs, -1, probs.dtype + ) # [b, h, sq, sk+1] + + # Backward pass for adding sinks + if sinks is None: + grad_sinks = None + grad_attn_w = grad_logits + else: + grad__sinks = grad_logits[:, :, :, -1] # [b, h, sq] + grad_sinks = einops.rearrange(grad__sinks, 'b h s -> h (b s)').sum(-1) + grad_attn_w = grad_logits[:, :, :, :-1].contiguous() # [b, h, sq, sk] + + # Backward pass for q @ K^T + grad_attn_w *= scale + grad__q = torch.matmul(grad_attn_w, _k_T) + grad__k = torch.matmul(_q_T, grad_attn_w) + + # Rearrange grads to (b, s, h, d) + grad_v = einops.rearrange(grad__v, 'b h s d -> b s h d') + grad_k = einops.rearrange(grad__k, 'b h d s -> b s h d') + grad_q = einops.rearrange(grad__q, 'b h s d -> b s h d') + return grad_q, grad_k, grad_v, grad_sinks + + +class AllGatherComm: + """All gather communication with async operations""" + + def __init__(self, group=None) -> None: + self.group = group + self.handles = [] + + def all_gather(self, output_tensor: torch.Tensor, input_tensor: torch.Tensor): + '''All gather the input tensor to the output tensor''' + + if self.group is None: + output_tensor.copy_(input_tensor) + else: + handle = torch.distributed.all_gather_into_tensor( + output_tensor, input_tensor, group=self.group, async_op=True + ) + self.handles.append(handle) + + def wait(self): + '''Wait for all gather operations to complete''' + + if self.group is not None: + for handle in self.handles: + handle.wait() + self.handles = [] + + +def to_zz_mask_attn_bias(attention_mask, cp_size, nheads, nheads_k, heads_k_stride, device, dtype): + '''Convert the attention mask to the attention bias''' + + if cp_size == 1: + zz_mask = attention_mask + else: + chunked = attention_mask.chunk(dim=3, chunks=cp_size * 2) + zz_mask = [_x for _p in zip(chunked[:cp_size], reversed(chunked[cp_size:])) for _x in _p] + zz_mask = torch.cat(zz_mask, dim=3) + attn_bias = torch.zeros(zz_mask.shape, device=device, dtype=dtype) + attn_bias.masked_fill_(zz_mask, float('-inf')) + attn_bias = attn_bias.expand(-1, heads_k_stride * (nheads // nheads_k), -1, -1) + return attn_bias + + +class AttentionFuncionWithContextParallel(torch.autograd.Function): + """Native attention function with context parallelism.""" + + @staticmethod + def forward(ctx, q, k, v, attention_mask, attention_dropout, softmax_scale, pg): + '''Forward pass for the native attention function with context parallelism''' + + # Assert einops exists + if not HAVE_EINOPS: + raise ImportError("einops is required by the attention CP but cannot be imported.") + + # Initialize communication group and constants + cp_size = 1 + if pg is not None: + cp_size = torch.distributed.get_world_size(pg) + comm = AllGatherComm(group=pg) + nheads = q.shape[2] + nheads_k = k.shape[2] + heads_k_stride = 1 + assert nheads % nheads_k == 0 and nheads_k % heads_k_stride == 0 + outs = [] + probs = [] + + # Initialize KV buffers + kv_buffer = torch.empty( + (2, k.shape[0] * cp_size, k.shape[1], heads_k_stride, k.shape[3]), + dtype=k.dtype, + device=k.device, + ) + kv_buffer_copy = torch.empty_like(kv_buffer) + + # All-gather first chunk of KV buffers + k_0 = k[:, :, :heads_k_stride].contiguous() + v_0 = v[:, :, :heads_k_stride].contiguous() + comm.all_gather(kv_buffer_copy[0], k_0) + comm.all_gather(kv_buffer_copy[1], v_0) + + # Prepare attention bias + attn_bias = to_zz_mask_attn_bias( + attention_mask, cp_size, nheads, nheads_k, heads_k_stride, q.device, q.dtype + ) + + # Iterate over heads + for i in range(0, nheads_k, heads_k_stride): + # Wait for previous all-gather to complete + comm.wait() + kv_buffer, kv_buffer_copy = kv_buffer_copy, kv_buffer + # All-gather the next portion of KV buffers if not the last iteration + if i < nheads_k - heads_k_stride: + kvsl = i + heads_k_stride + kvsr = kvsl + heads_k_stride + send_k = k[:, :, kvsl:kvsr].contiguous() + send_v = v[:, :, kvsl:kvsr].contiguous() + comm.all_gather(kv_buffer_copy[0], send_k) + comm.all_gather(kv_buffer_copy[1], send_v) + + # Prepare query, key, value for attention + q_i = q[:, :, i * nheads // nheads_k : (i + heads_k_stride) * nheads // nheads_k] + k_i = kv_buffer[0] + v_i = kv_buffer[1] + + # Rearrange query, key, value to (b, s, h, d) + q_i = einops.rearrange(q_i, 's b h d -> b s h d') + k_i = einops.rearrange(k_i, 's b h d -> b s h d') + v_i = einops.rearrange(v_i, 's b h d -> b s h d') + + # Forward pass + out_i, probs_i = eager_attn_fwd( + q_i, k_i, v_i, attn_bias, None, softmax_scale, attention_dropout + ) + outs.append(out_i) + probs.append(probs_i) + + # Concatenate outputs and rearrange to (s, b, h, d) + out = torch.cat(outs, dim=2) + out = einops.rearrange(out, 'b s h d -> s b h d') + + # Save contexts for backward pass + ctx.save_for_backward(q, k, v, attention_mask, *outs, *probs) + ctx.dropout = attention_dropout + ctx.scale = softmax_scale + ctx.heads_k_stride = heads_k_stride # TODO make it configurable + ctx.pg = pg + + return out + + @staticmethod + def backward(ctx, dout): + '''Backward pass for the native attention function with context parallelism''' + + # Initialize or resume constants and communication group + q, k, v, attention_mask, *rest = ctx.saved_tensors + nheads = q.shape[2] + nheads_k = k.shape[2] + heads_k_stride = ctx.heads_k_stride + assert nheads_k % heads_k_stride == 0 + outs = rest[: nheads_k // heads_k_stride] + probs = rest[nheads_k // heads_k_stride :] + pg = ctx.pg + cp_size = 1 + if pg is not None: + cp_size = torch.distributed.get_world_size(pg) + comm = AllGatherComm(group=pg) + + # Initialize KV buffers + kv_buffer = torch.empty( + (2, k.shape[0] * cp_size, k.shape[1], heads_k_stride, k.shape[3]), + dtype=k.dtype, + device=k.device, + ) + kv_buffer_copy = torch.empty_like(kv_buffer) + + # All-gather first chunk of KV buffers + dq = [] + dk = [] + dv = [] + k_0 = k[:, :, :heads_k_stride].contiguous() + v_0 = v[:, :, :heads_k_stride].contiguous() + comm.all_gather(kv_buffer_copy[0], k_0) + comm.all_gather(kv_buffer_copy[1], v_0) + + # Prepare attention bias + attn_bias = to_zz_mask_attn_bias( + attention_mask, cp_size, nheads, nheads_k, heads_k_stride, q.device, q.dtype + ) + + # Iterate over heads + for i in range(0, nheads_k, heads_k_stride): + # Slice query and output for this iteration + q_slice = slice(i * nheads // nheads_k, (i + heads_k_stride) * nheads // nheads_k) + q_i = q[:, :, q_slice] + dout_i = dout[:, :, q_slice] + + # Wait for previous all-gather to complete + comm.wait() + kv_buffer, kv_buffer_copy = kv_buffer_copy, kv_buffer + + # All-gather the next portion of KV buffers if not the last iteration + if i < nheads_k - heads_k_stride: + kvsl = i + heads_k_stride + kvsr = kvsl + heads_k_stride + send_k = k[:, :, kvsl:kvsr].contiguous() + send_v = v[:, :, kvsl:kvsr].contiguous() + comm.all_gather(kv_buffer_copy[0], send_k) + comm.all_gather(kv_buffer_copy[1], send_v) + + # Prepare key, value for attention + k_i = kv_buffer[0] + v_i = kv_buffer[1] + + # Rearrange query, key, value to (b, s, h, d) + q_i = einops.rearrange(q_i, 's b h d -> b s h d') + k_i = einops.rearrange(k_i, 's b h d -> b s h d') + v_i = einops.rearrange(v_i, 's b h d -> b s h d') + dout_i = einops.rearrange(dout_i, 's b h d -> b s h d') + + # Backward pass + dq_i, _dk_i, _dv_i, _ = eager_attn_bwd( + q_i, k_i, v_i, attn_bias, None, ctx.scale, ctx.dropout, outs[i], probs[i], dout_i + ) + + # Rearrange gradients to (s, b, h, d) + dq_i = einops.rearrange(dq_i, 'b s h d -> s b h d') + _dk_i = einops.rearrange(_dk_i, 'b s h d -> s b h d') + _dv_i = einops.rearrange(_dv_i, 'b s h d -> s b h d') + if pg is None: + dk_i = _dk_i + dv_i = _dv_i + else: + # Reduce-scatter gradients if CP > 1 + dk_i = torch.zeros( + (k_i.shape[1] // cp_size, k_i.shape[0], k_i.shape[2], k_i.shape[3]), + device=k_i.device, + dtype=k_i.dtype, + ) + dv_i = torch.zeros( + (v_i.shape[1] // cp_size, v_i.shape[0], v_i.shape[2], v_i.shape[3]), + device=v_i.device, + dtype=v_i.dtype, + ) + torch.distributed.reduce_scatter_tensor(dk_i, _dk_i, group=pg) + torch.distributed.reduce_scatter_tensor(dv_i, _dv_i, group=pg) + + # Collect gradients + dq.append(dq_i) + dk.append(dk_i) + dv.append(dv_i) + + # Concatenate gradients and return + dq = torch.cat(dq, dim=2) + dk = torch.cat(dk, dim=2) + dv = torch.cat(dv, dim=2) + return dq, dk, dv, None, None, None, None diff --git a/megatron/core/transformer/fsdp_dtensor_checkpoint.py b/megatron/core/transformer/fsdp_dtensor_checkpoint.py index 9ef3f1f1b82..65e2f5f9dff 100644 --- a/megatron/core/transformer/fsdp_dtensor_checkpoint.py +++ b/megatron/core/transformer/fsdp_dtensor_checkpoint.py @@ -47,29 +47,35 @@ from megatron.core.transformer.transformer_layer import TransformerLayer -def get_ep_layer_offset(): +def get_ep_layer_offset(num_experts: int | None = None) -> int: """ Get the expert layer offset for the current model. - """ - from megatron.training.global_vars import get_args - args = get_args() + Args: + num_experts: Total number of experts in the model. If None, returns 0. + + Returns: + The expert layer offset for the current EP rank. + """ ep_size = parallel_state.get_expert_model_parallel_world_size() ep_rank = parallel_state.get_expert_model_parallel_rank() - num_local_experts = args.num_experts // ep_size if args.num_experts else 0 + num_local_experts = num_experts // ep_size if num_experts else 0 local_expert_offset = ep_rank * num_local_experts return local_expert_offset -def get_total_num_experts(): +def get_total_num_experts(num_experts: int | None = None) -> int: """ Get the total number of experts for the current model. - """ - from megatron.training.global_vars import get_args - args = get_args() - return args.num_experts if args.num_experts else 0 + Args: + num_experts: Total number of experts in the model. If None, returns 0. + + Returns: + The total number of experts. + """ + return num_experts if num_experts else 0 def get_expert_index_from_key(key): @@ -96,12 +102,19 @@ def get_expert_index_from_key(key): return None -def handle_experts_in_state_dict(state_dict): +def handle_experts_in_state_dict(state_dict, num_experts: int | None = None): """ Rewrite expert keys in state dict. + + Args: + state_dict: The state dictionary to process. + num_experts: Total number of experts in the model. If None, no expert processing occurs. + + Returns: + The processed state dictionary with rewritten expert keys. """ - local_expert_start = get_ep_layer_offset() - local_expert_end = get_total_num_experts() + local_expert_start = get_ep_layer_offset(num_experts) + local_expert_end = get_total_num_experts(num_experts) def should_keep_expert_key(expert_index): """Determine if this rank should keep this expert key based on expert index""" @@ -147,9 +160,17 @@ def replace_expert_index_in_key(key, expert_index, state_dict): return state_dict -def expert_param_local_key(key): - """Get the module parameter corresponding to the key.""" - local_expert_offset = get_ep_layer_offset() +def expert_param_local_key(key: str, num_experts: int | None = None) -> str: + """Get the module parameter corresponding to the key. + + Args: + key: The parameter key to process. + num_experts: Total number of experts in the model. If None, no expert processing occurs. + + Returns: + The local parameter key with adjusted expert indices. + """ + local_expert_offset = get_ep_layer_offset(num_experts) expert_index = get_expert_index_from_key(key) if expert_index is not None: new_expert_index = expert_index - local_expert_offset @@ -174,6 +195,9 @@ def handle_swiglu_in_state_dict(model, model_state_dict, optimizer_state_dict): """ assert HAVE_MEGATRON_FSDP, "This function requires Megatron-FSDP to be installed." + # Extract num_experts from model config for expert parameter processing + num_experts = model.config.num_moe_experts if hasattr(model, 'config') else None + def intersection(s1, s2): # Only works for step=1 start = max(s1.start, s2.start) @@ -297,7 +321,9 @@ def split_swiglu_linear_fc1(data, dist_param, swiglu_shard_axis, is_expert_param new_opt_state_dict[f"{key}_w"] = opt_state_dict[key].copy() new_opt_state_dict[f"{key}_v"] = opt_state_dict[key].copy() for subkey in ["exp_avg", "exp_avg_sq"]: - dist_param = model.get_parameter(expert_param_local_key(key[len("module.") :])) + dist_param = model.get_parameter( + expert_param_local_key(key[len("module.") :], num_experts) + ) weight_w, weight_v = split_swiglu_linear_fc1( opt_state_dict[key][subkey], dist_param, @@ -426,6 +452,13 @@ def validate_loaded_state_dict(state_dict, checkpoint_path): def get_global_unique_param_name(model_chunks, param): """ Get the global unique parameter name for a given model and parameter. + + Args: + model_chunks: List of model chunks to search for the parameter. + param: The parameter to find the name for. + + Returns: + The global unique parameter name. """ param_name = None for model in model_chunks: @@ -450,6 +483,7 @@ def get_global_unique_param_name(model_chunks, param): param_name = re.sub(r"layers\.(\d+)", f"layers.{tf_layer_number - 1}", param_name) # Get EP unique parameter name - param_name = list(handle_experts_in_state_dict({param_name: None}).keys())[0] + num_experts = model_chunks[0].config.num_moe_experts if model_chunks else None + param_name = list(handle_experts_in_state_dict({param_name: None}, num_experts).keys())[0] return param_name diff --git a/megatron/core/transformer/mlp.py b/megatron/core/transformer/mlp.py index 9602beb2f71..8dcf196da94 100644 --- a/megatron/core/transformer/mlp.py +++ b/megatron/core/transformer/mlp.py @@ -87,7 +87,7 @@ def __init__( self.input_size = input_size if input_size != None else self.config.hidden_size - tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) + self.tp_group = get_tensor_model_parallel_group_if_none(tp_group, is_expert=is_expert) if ffn_hidden_size is None: if is_expert: raise ValueError("MoE MLP requires `ffn_hidden_size`, but it was not provided.") diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index 4fdcacb791b..1058a207b12 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -11,6 +11,7 @@ from megatron.core.dist_checkpointing.mapping import ShardedStateDict from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, make_sharded_tensors_for_checkpoint, sharded_state_dict_default, ) @@ -77,13 +78,26 @@ def sharded_state_dict( sharded_state_dict = {} # Save parameters self._save_to_state_dict(sharded_state_dict, '', keep_vars=True) + if not hasattr(self, 'tp_group'): + # some model interface hasn't updated for m4, fallback needed + tp_group = parallel_state.get_tensor_model_parallel_group() + else: + tp_group = self.tp_group + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) sharded_state_dict = make_sharded_tensors_for_checkpoint( - sharded_state_dict, prefix, sharded_offsets=sharded_offsets + sharded_state_dict, + prefix, + sharded_offsets=sharded_offsets, + tp_group=tp_group, + dp_cp_group=metadata['dp_cp_group'], ) # Recurse into submodules for name, module in self.named_children(): sharded_state_dict.update( - sharded_state_dict_default(module, f'{prefix}{name}.', sharded_offsets, metadata) + sharded_state_dict_default( + module, f'{prefix}{name}.', sharded_offsets, metadata, tp_group=tp_group + ) ) return sharded_state_dict diff --git a/megatron/core/transformer/moe/README.md b/megatron/core/transformer/moe/README.md index 3a79cdc4cbb..a44daea38e2 100644 --- a/megatron/core/transformer/moe/README.md +++ b/megatron/core/transformer/moe/README.md @@ -13,6 +13,7 @@ Megatron-Core MoE provides comprehensive parallelism strategies, seamlessly inte - Support Multi-Token Prediction (MTP) - Batch-level overlapping to hide EP-A2A communication - **Support DeepSeek's DeepEP for efficient token dispatching and combining** +- Support HybridEP for efficient token dispatching and combining within intra-node and MNNVL scenarios. - Add fusion for token permutation and unpermutation - Support Uneven virtual pipeline parallel split - Support output-discarding checkpointing on some submodules @@ -172,7 +173,13 @@ Note: The MoE model structure is defined through script arguments. All MoE-relat ### Leverage DeepSeek's DeepEP for High-Performance Cross-Node Token Dispatching - [DeepSeek-DeepEP](https://github.com/deepseek-ai/deepep) provides a highly optimized implementation for MoE token dispatching and combining operations, specifically designed for large-scale MoE training scenarios. - DeepEP is particularly recommended for training large-scale, fine-grained MoE architectures such as DeepSeek-V3 and other advanced MoE models. -- To enable DeepEP in your training configuration, simply set `--moe-token-dispatcher-type=flex` and `--moe-enable-deepep` in your command line arguments. +- To enable DeepEP in your training configuration, simply set `--moe-token-dispatcher-type=flex` and `--moe-flex-dispatcher-backend=deepep` in your command line arguments. + +### Integrate HybridEP for High-Performance Intra-Node Token Dispatching +- [HybridEP](https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep) is developed by NVIDIA as an optimized solution for large-scale MoE (Mixture of Experts) all-to-all communication. It is designed to leverage NVIDIA GPU hardware capabilities, significantly reducing Streaming Multiprocessor (SM) resource usage. +- HybridEP currently supports intra-node and multi-node NVLink scenarios. +- To enable HybridEP, set `--moe-token-dispatcher-type=flex` and + `--moe-flex-dispatcher-backend=hybridep` in your command line arguments. ### CUDA Graph Support CUDA Graph functionality can be enabled through the `--cuda-graph-impl` option. There are two implementations: @@ -203,6 +210,20 @@ Enable A2A overlap across different batches inspired by the DSv3 DualPipe implme --delay-wgrad-compute ``` +### Fine-grained Activation Offloading (collaborated with rednote) +Offload the input activation at the granularity of modules + +**Usage** +```bash +# Enable fine-grained activation offloading +--fine-grained-activation-offloading + +# Specify which modules are going to offload its input +# Choices: "attn_norm", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act". +--offload-modules expert_fc1 +``` +For more details, please refer to the ```docs/source/api-guide/fine_grained_activation_offloading.md``` + ### MoE Related Arguments | Item | Description | | --- | --- | @@ -233,12 +254,12 @@ Enable A2A overlap across different batches inspired by the DSv3 DualPipe implme | --moe-router-fusion | Enable fusion for MoE TopK routing and aux-loss computation. This is only supported in TransformerEngine 2.7.0 and above. | | --moe-router-bias-update-rate | The expert bias is updated based on the number of assigned tokens to each expert in a global batch, where the bias is increased for experts with less assigned tokens and decreased for experts with more assigned tokens. Default is 1e-3 same as that used in DeepSeekV3. | | --moe-router-force-load-balancing | (Experimental) Force override routing to balance token distribution using random logits for MoE routers, supporting naive top-k and group-limited top-k. This experimental feature is for benchmarking purposes only! | -| --moe-router-padding-for-fp8 | Pad the routing_map to make sure the number of tokens each expert received is a multiple of 16/32 for FP8 precision. It is suggested to enable this for dropless training with FP8 precision when num_local_experts > 1. This is a more efficient way to pad for FP8 which eliminates the explicit padding in the GroupedMLP layer. | +| --moe-router-padding-for-quantization | Pad the routing_map to make sure the number of tokens each expert received is a multiple of 16/32 for FP8/FP4 precision. It is suggested to enable this for dropless training with FP8 precision when num_local_experts > 1. This is a more efficient way to pad for FP8 which eliminates the explicit padding in the GroupedMLP layer. | | --moe-aux-loss-coeff | Scaling coefficient for the aux loss: a starting value of 1e-2 is recommended. Default is 0.0. | | --moe-z-loss-coeff | Scaling coefficient for the z-loss: a starting value of 1e-3 is recommended. Default is None. | | --moe-input-jitter-eps | Add noise to the input tensor by applying jitter with a specified epsilon value. Default is None. | | --moe-token-dispatcher-type | Determines the token dispatcher type. Choices are "allgather", "alltoall". Default is "allgather". We recommend using 'alltoall' if expert parallelism is applied. We have upgraded the "alltoall" dispatcher in place during MCore v0.9, while the original implementation renamed as "alltoall_seq" is retained until MCore v0.13.| -| --moe-enable-deepep | (Experimental) Enable DeepSeek/DeepEP for efficient token dispatching and combine in MoE models. Only works with flex token dispatcher by setting --moe-token-dispatcher-type=flex. | +| --moe-flex-dispatcher-backend | (Experimental) Select the backend for the flex token dispatcher. Supported options: "deepep", "hybridep". Enables efficient token dispatching and combining for MoE models. | | --moe-per-layer-logging | Enable per-layer logging for MoE, currently supports auxiliary loss and z loss. | | --moe-expert-capacity-factor | The capacity factor for each expert, None means no token will be dropped. Default is None. | | --moe-pad-expert-input-to-capacity | Pads the input for each expert to match the expert capacity length, effective only after the --moe-expert-capacity-factor is set. | @@ -439,7 +460,7 @@ By setting `--expert-tensor-parallel-size`, we can set MoE-specific TP size. - Token Dispatcher sends tokens to the designated expert, involves tensor rearangement and communications. - Dispatcher `allgather` is the default option. It achieves better performance and efficiency when only tensor parallelism is used or when the Top-k value is very large. - Dispatcher `alltoall` is recommended if expert parallelism is applied. -- Dispatcher `flex` is a new dispatcher decouples communication group from model parallelism. Currently, only the DeepEP backend is supported for by setting `--moe-enable-deepep`. +- Dispatcher `flex` is a new dispatcher decouples communication group from model parallelism. It supports two backends(DeepEP and HybridEP) selectable via `--moe-flex-dispatcher-backend`. **Enable Communication Overlap** - Enable `--overlap-param-gather` and `--overlap-grad-reduce` with distributed optimizer. @@ -462,7 +483,7 @@ Therefore, there are two recommended ways during the first 200 steps to avoid th **FP8 Training Best Practice** - Using latest version of [TransformerEngine](https://github.com/NVIDIA/TransformerEngine). -- Enable router padding with `--moe-router-padding-for-fp8` to reduce padding overhead. +- Enable router padding with `--moe-router-padding-for-quantization` to reduce padding overhead. - Enable native FP8 weights with `--fp8-param-gather` to reduce weights memory cost. ### Reference Best Parallel Mapping diff --git a/megatron/core/transformer/moe/experts.py b/megatron/core/transformer/moe/experts.py index d8dd3d03f02..8bb5caddc4b 100644 --- a/megatron/core/transformer/moe/experts.py +++ b/megatron/core/transformer/moe/experts.py @@ -1,9 +1,9 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy import itertools from copy import deepcopy -from functools import partial, wraps +from functools import partial from math import ceil from typing import Optional, Tuple @@ -11,7 +11,7 @@ import torch.nn.functional as F from torch.nn.parameter import Parameter -from megatron.core import parallel_state, tensor_parallel +from megatron.core import tensor_parallel from megatron.core.activations import squared_relu from megatron.core.dist_checkpointing import ShardedTensor from megatron.core.dist_checkpointing.mapping import ( @@ -21,14 +21,21 @@ ShardedTensorFactory, ) from megatron.core.dist_checkpointing.utils import replace_prefix_for_sharding +from megatron.core.fp4_utils import get_fp4_align_size from megatron.core.fp8_utils import get_fp8_align_size from megatron.core.fusions.fused_bias_geglu import quick_gelu, weighted_bias_quick_geglu_impl from megatron.core.fusions.fused_bias_swiglu import weighted_bias_swiglu_impl from megatron.core.fusions.fused_weighted_squared_relu import weighted_squared_relu_impl from megatron.core.jit import jit_fuser +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.tensor_parallel.layers import ( _initialize_affine_weight_cpu, _initialize_affine_weight_gpu, + set_tensor_model_parallel_attributes, ) from megatron.core.tensor_parallel.utils import divide from megatron.core.transformer.mlp import MLP, MLPSubmodules, apply_swiglu_sharded_factory @@ -38,6 +45,7 @@ from megatron.core.transformer.spec_utils import build_module from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.transformer.utils import ( + ensure_metadata_has_dp_cp_group, make_sharded_object_for_checkpoint, sharded_state_dict_default, ) @@ -54,49 +62,6 @@ HAVE_TE = False -# TODO(Hepteract): delete the usage of the global parallel_state. -# Currently we still have to use the global parallel_state in expert_dist_ckpt_decorator(), -# in order to set sub-module's process group while getting sharded_state_dict. -# After sub-module's refactoring is done, we can pass pg_collection to sub-module -# and delete the function expert_dist_ckpt_decorator. -def expert_dist_ckpt_decorator(func): - """Decorator of shared_state_dict in expert layer for distributed checkpoint. - - Since !1940, the TP size for Expert layer can be different with Attention. - To make distributed checkpoint work in such cases, we use a decorator to - replace the default TP parallel states with expert-TP parallel states. - """ - - @wraps(func) - def wrapper(*args, **kwargs): - # Store original states - original_rank = parallel_state._MPU_TENSOR_MODEL_PARALLEL_RANK - original_size = parallel_state._MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE - original_group = parallel_state._TENSOR_MODEL_PARALLEL_GROUP - try: - # Set new states - parallel_state._MPU_TENSOR_MODEL_PARALLEL_RANK = ( - parallel_state.get_expert_tensor_parallel_rank() - ) - parallel_state._MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = ( - parallel_state.get_expert_tensor_parallel_world_size() - ) - parallel_state._TENSOR_MODEL_PARALLEL_GROUP = ( - parallel_state.get_expert_tensor_parallel_group() - ) - - # Execute the function - result = func(*args, **kwargs) - finally: - # Restore original states - parallel_state._MPU_TENSOR_MODEL_PARALLEL_RANK = original_rank - parallel_state._MPU_TENSOR_MODEL_PARALLEL_WORLD_SIZE = original_size - parallel_state._TENSOR_MODEL_PARALLEL_GROUP = original_group - return result - - return wrapper - - class GroupedMLP(MegatronModule): """An efficient implementation of the Experts layer using GroupedGEMM. @@ -134,8 +99,10 @@ def glu(x): self.config.recompute_granularity == 'selective' and "moe_act" in self.config.recompute_modules ) - if self.activation_recompute and self.config.fp8: - raise ValueError("moe_act recompute for fp8 cannot work with the legacy GroupedMLP.") + if self.activation_recompute and (self.config.fp8 or self.config.fp4): + raise ValueError( + "moe_act recompute for fp8 or fp4 cannot work with the legacy GroupedMLP." + ) @jit_fuser def activation_func_with_probs(x, probs): @@ -205,6 +172,14 @@ def activation_func_with_probs(x, probs): rank=tp_rank, world_size=tp_size, ) + else: + # Ensure TP attrs are set even when not initializing + set_tensor_model_parallel_attributes( + tensor=self.weight1, is_parallel=True, dim=1, stride=1 + ) + set_tensor_model_parallel_attributes( + tensor=self.weight2, is_parallel=True, dim=0, stride=1 + ) else: self.weight1 = Parameter( torch.empty( @@ -229,6 +204,14 @@ def activation_func_with_probs(x, probs): _initialize_affine_weight_gpu( self.weight2, config.output_layer_init_method, partition_dim=0, is_expert=True ) + else: + # Ensure TP attrs are set even when not initializing + set_tensor_model_parallel_attributes( + tensor=self.weight1, is_parallel=True, dim=1, stride=1 + ) + set_tensor_model_parallel_attributes( + tensor=self.weight2, is_parallel=True, dim=0, stride=1 + ) setattr(self.weight1, 'allreduce', not self.expert_parallel) setattr(self.weight2, 'allreduce', not self.expert_parallel) @@ -305,7 +288,6 @@ def forward( return fc2_output, None - @expert_dist_ckpt_decorator def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): """ Maps local expert to global experts. @@ -764,6 +746,7 @@ def __init__( ), "bias_dropout_fusion is not supported in TEGroupedMLP when add_bias_linear=True" self.ep_group = pg_collection.ep + self.tp_group = pg_collection.expt_tp # Double the output width with gated linear unit, see https://arxiv.org/pdf/2002.05202.pdf ffn_hidden_size = self.config.moe_ffn_hidden_size @@ -805,19 +788,35 @@ def __init__( tp_group=pg_collection.expt_tp, ) + self.offload_expert_fc1 = ( + self.config.fine_grained_activation_offloading + and "expert_fc1" in self.config.offload_modules + ) + + self.offload_moe_act = ( + self.config.fine_grained_activation_offloading + and "moe_act" in self.config.offload_modules + ) + self.activation_recompute = ( self.config.recompute_granularity == 'selective' and "moe_act" in self.config.recompute_modules ) - if self.activation_recompute and self.config.fp8: + if self.activation_recompute and (self.config.fp8 or self.config.fp4): from megatron.core.extensions.transformer_engine import set_save_original_input set_save_original_input(self.linear_fc2) - if self.config.fp8: - assert HAVE_TE, "FP8 requires TE." - self.fp8_padding = Fp8Padding(self.num_local_experts) - self.fp8_unpadding = Fp8Unpadding(self.num_local_experts) + # This is to avoid the CPU overhead of multiple d2h copies + if self.offload_expert_fc1 and not (self.config.fp8 or self.config.fp4): + from megatron.core.extensions.transformer_engine import set_save_original_input + + set_save_original_input(self.linear_fc1) + + if self.config.fp8 or self.config.fp4: + assert HAVE_TE, "FP8 and FP4 requires TE." + self.quantization_padding = Fp8Padding(self.num_local_experts) + self.quantization_unpadding = Fp8Unpadding(self.num_local_experts) @staticmethod def _apply_bias(intermediate_parallel, bias_parallel, tokens_per_expert, permuted_probs): @@ -857,12 +856,12 @@ def forward( output (torch.Tensor): The output of the local experts. """ tokens_per_expert = tokens_per_expert.tolist() - if self.config.fp8: + if self.config.fp8 or self.config.fp4: actual_tokens_per_expert = tokens_per_expert - permuted_local_hidden_states, tokens_per_expert = self.fp8_padding( + permuted_local_hidden_states, tokens_per_expert = self.quantization_padding( permuted_local_hidden_states, tokens_per_expert ) - permuted_probs, _ = self.fp8_padding( + permuted_probs, _ = self.quantization_padding( permuted_probs.unsqueeze(-1), actual_tokens_per_expert ) else: @@ -878,9 +877,21 @@ def forward( # Probs already applied, so reset to 1. permuted_probs = torch.ones_like(permuted_probs) - intermediate_parallel, bias_parallel = self.linear_fc1( - permuted_local_hidden_states, tokens_per_expert - ) + if self.offload_expert_fc1: + permuted_local_hidden_states = fine_grained_offloading_group_start( + permuted_local_hidden_states, name="expert_fc1" + ) + with get_fine_grained_offloading_context(self.offload_expert_fc1): + fc1_output, bias_parallel = self.linear_fc1( + permuted_local_hidden_states, tokens_per_expert + ) + if self.offload_expert_fc1: + fc1_output, bias_parallel = fine_grained_offloading_group_commit( + fc1_output, + bias_parallel, + name="expert_fc1", + forced_released_tensors=[permuted_local_hidden_states], + ) def bias_act_func(intermediate_parallel, bias_parallel, permuted_probs): if self.config.use_te_activation_func: @@ -940,29 +951,36 @@ def glu(x): intermediate_parallel = intermediate_parallel.to(original_dtype) return intermediate_parallel + if self.offload_moe_act: + fc1_output = fine_grained_offloading_group_start(fc1_output, name="moe_act") + if self.activation_recompute: self.activation_checkpoint = tensor_parallel.CheckpointWithoutOutput() - intermediate_parallel = self.activation_checkpoint.checkpoint( - bias_act_func, intermediate_parallel, bias_parallel, permuted_probs - ) - output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) - self.activation_checkpoint.discard_output_and_register_recompute(output) + with get_fine_grained_offloading_context(self.offload_moe_act): + bias_act_output = self.activation_checkpoint.checkpoint( + bias_act_func, fc1_output, bias_parallel, permuted_probs + ) else: - intermediate_parallel = bias_act_func( - intermediate_parallel, bias_parallel, permuted_probs + with get_fine_grained_offloading_context(self.offload_moe_act): + bias_act_output = bias_act_func(fc1_output, bias_parallel, permuted_probs) + + output, output_bias = self.linear_fc2(bias_act_output, tokens_per_expert) + if self.activation_recompute: + self.activation_checkpoint.discard_output_and_register_recompute(output) + if self.offload_moe_act: + (output,) = fine_grained_offloading_group_commit( + output, name="moe_act", forced_released_tensors=[fc1_output] ) - output, output_bias = self.linear_fc2(intermediate_parallel, tokens_per_expert) # upad and concat the output - if self.config.fp8: - output = self.fp8_unpadding(output, actual_tokens_per_expert) + if self.config.fp8 or self.config.fp4: + output = self.quantization_unpadding(output, actual_tokens_per_expert) output = self._apply_bias(output, output_bias, tokens_per_expert, permuted_probs) output_bias = None return output, output_bias - @expert_dist_ckpt_decorator def sharded_state_dict( self, prefix: str = '', sharded_offsets: tuple = (), metadata: Optional[dict] = None ) -> ShardedStateDict: @@ -970,10 +988,14 @@ def sharded_state_dict( Maps local expert to global experts. The sharded state dict is interchangable with SequentialMLP's. """ + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) singleton_local_shards = (metadata or {}).get('singleton_local_shards', False) sharded_state_dict = {} for name, module in self._modules.items(): - sub_sd = sharded_state_dict_default(module, f'{name}.', sharded_offsets, metadata) + sub_sd = sharded_state_dict_default( + module, f'{name}.', sharded_offsets, metadata, tp_group=self.tp_group + ) if name == 'linear_fc1' and self.config.gated_linear_unit: num_global_experts = self.ep_group.size() * self.num_local_experts local_expert_indices_offset = self.ep_group.rank() * self.num_local_experts @@ -1037,6 +1059,7 @@ def __init__( self.num_local_experts = num_local_experts self.local_experts = torch.nn.ModuleList() self.ep_group = pg_collection.ep + self.tp_group = pg_collection.expt_tp # use pg_collection.expt_dp_group as data parallel group in this module. # TODO (Hepteract): expt_dp wont be needed here once distributed checkpoint is refactored self.dp_group = pg_collection.expt_dp @@ -1051,10 +1074,18 @@ def __init__( ) self.local_experts.append(expert) - def _pad_tensor_for_fp8(self, hidden, probs): + def _get_align_size_for_quantization(self): + """Get the alignment size for quantization.""" + if self.config.fp8: + return get_fp8_align_size(self.config.fp8_recipe) + elif self.config.fp4: + return get_fp4_align_size(self.config.fp4_recipe) + return 16 + + def _pad_tensor_for_quantization(self, hidden, probs): """Padding tensor shape to multiples of 16/32.""" actual_num_tokens = hidden.shape[0] - divisor = get_fp8_align_size(self.config.fp8_recipe) + divisor = self._get_align_size_for_quantization() padded_num_tokens = ceil(actual_num_tokens / divisor) * divisor - actual_num_tokens if padded_num_tokens > 0: pad_tensor = torch.zeros( @@ -1086,8 +1117,8 @@ def forward( permuted_probs = torch.ones_like(permuted_probs) if self.num_local_experts == 1: - if self.config.fp8: - hidden, probs = self._pad_tensor_for_fp8( + if self.config.fp8 or self.config.fp4: + hidden, probs = self._pad_tensor_for_quantization( permuted_local_hidden_states, permuted_probs ) output, output_bias = self.local_experts[0](hidden, probs) @@ -1106,8 +1137,8 @@ def forward( output_local_list = [] for expert, tokens, probs in zip(self.local_experts, tokens_list, probs_list): - if self.config.fp8: - hidden, probs = self._pad_tensor_for_fp8(tokens, probs) + if self.config.fp8 or self.config.fp4: + hidden, probs = self._pad_tensor_for_quantization(tokens, probs) output, output_bias = expert(hidden, probs) output = output[: tokens.shape[0]] else: @@ -1124,9 +1155,11 @@ def backward_dw(self): for expert in self.local_experts: expert.backward_dw() - @expert_dist_ckpt_decorator def sharded_state_dict(self, prefix='', sharded_offsets=(), metadata=None): """Maps local expert to global experts.""" + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) + sharded_state_dict = {} num_global_experts = self.ep_group.size() * self.num_local_experts local_expert_indices_offset = self.ep_group.rank() * self.num_local_experts diff --git a/megatron/core/transformer/moe/fused_a2a.py b/megatron/core/transformer/moe/fused_a2a.py index 00a840f2b7f..60b0b11a32c 100644 --- a/megatron/core/transformer/moe/fused_a2a.py +++ b/megatron/core/transformer/moe/fused_a2a.py @@ -262,3 +262,273 @@ def set_deepep_num_sms(num_sms): fused_dispatch = None fused_combine = None set_deepep_num_sms = None + + +try: + from deep_ep import HybridEPBuffer + + HAVE_HYBRIDEP = True +except ImportError: + HAVE_HYBRIDEP = False + +_hybrid_ep_buffer = None + + +def init_hybrid_ep_buffer( + group: torch.distributed.ProcessGroup, + hidden_dim: int, + seq_len: int, + num_local_experts: int, + num_sms_dispatch_api: int, + num_sms_combine_api: int, + fp8_dispatch: bool, +) -> None: + ''' + Initialize the HybridEP buffer, including buffer allocation and metadata + initialization. + + If a runtime dispatch/combine requires a larger buffer than the one + initialized, the buffer will be reallocated at runtime, + incuring extra run-time overhead. + + Args: + group (torch.distributed.ProcessGroup): + Process group for HybridEP all-to-all communication. + hidden_dim (int): + Hidden dimension of the input tensor. + seq_len (int): + Maximum sequence length of the input tensor. + num_local_experts (int): + Number of local experts. + num_sms_dispatch_api (int): + Number of SMs used by the dispatch API. + num_sms_combine_api (int): + Number of SMs used by the combine API. + fp8_dispatch (bool): + Whether to use FP8 communication during the dispatch phase. + ''' + assert not fp8_dispatch, "HybridEP dispatcher does not support fp8 dispatch now" + global _hybrid_ep_buffer + _hybrid_ep_buffer = HybridEPBuffer( + group=group, + hidden_dim=hidden_dim, + max_num_of_tokens_per_rank=seq_len, + num_local_experts=num_local_experts, + use_fp8=fp8_dispatch, + num_sms_dispatch_api=num_sms_dispatch_api, + num_sms_combine_api=num_sms_combine_api, + ) + + +class HybridEPDispatch(torch.autograd.Function): + ''' + Fused dispatch operation for permute + dispatch a2a + permute using the HybridEP backend + ''' + + @staticmethod + def forward( + ctx, + x, + routing_map, + probs, + group, + num_local_experts, + num_sms_dispatch_api=24, + num_sms_combine_api=24, + num_dispatched_tokens=None, + num_permuted_tokens=None, + pad_multiple=None, + ): + ''' + Forward pass of fused dispatch of the HybridEP backend + ''' + if _hybrid_ep_buffer is None: + seq_len, hidden_dim = x.shape[-2:] + fp8_dispatch = False # Currently, we do not support fp8 dispatch + init_hybrid_ep_buffer( + group, + hidden_dim, + seq_len, + num_local_experts, + num_sms_dispatch_api, + num_sms_combine_api, + fp8_dispatch, + ) + # Defaultly, the output token_per_expert and num_dispatched_tokens_tensor + # will be put on the CPU to avoid the potential sync in combine/backward pass, + # but if we provide the num_dispatched_tokens and num_permuted_tokens on CPU, + # we do not need to the D2H here. + use_host_meta = num_dispatched_tokens is None or num_permuted_tokens is None + # Process the dispatch + ( + dispatched_hidden, + dispatched_probs, + dispatched_scaling_factor, + tokens_per_expert, + handle, + ) = _hybrid_ep_buffer.dispatch_with_permute( + hidden=x, + routing_map=routing_map, + probs=probs, + scaling_factor=None, + num_of_experts_per_rank=num_local_experts, + pad_multiple=pad_multiple, + num_dispatched_tokens=num_dispatched_tokens, + num_permuted_tokens=num_permuted_tokens, + use_host_meta=use_host_meta, + ) + + ctx.handle = handle + ctx.pad_multiple = pad_multiple + ctx.num_dispatched_tokens = num_dispatched_tokens + return ( + dispatched_hidden, + dispatched_probs, + dispatched_scaling_factor, + tokens_per_expert, + handle, + ) + + @staticmethod + def backward(ctx, grad_x, grad_probs, grad_scaling_factor, grad_tokens_per_expert, grad_handle): + ''' + Backward pass of fused dispatch of the HybridEP backend + ''' + handle = ctx.handle + combined_hidden, combined_probs = _hybrid_ep_buffer.combine_with_unpermute( + hidden=grad_x, + probs=grad_probs, + handle=handle, + pad_multiple=ctx.pad_multiple, + num_dispatched_tokens=ctx.num_dispatched_tokens, + ) + return combined_hidden, None, combined_probs, None, None, None, None, None, None, None + + +class HybridEPCombine(torch.autograd.Function): + ''' + Fused combine operation for permute + combine a2a + permute using the HybridEP backend + ''' + + @staticmethod + def forward( + ctx, x, handle, num_dispatched_tokens=None, num_permuted_tokens=None, pad_multiple=None + ): + ''' + Forward pass of fused combine of the HybridEP backend + ''' + combined_hidden, _ = _hybrid_ep_buffer.combine_with_unpermute( + hidden=x, + handle=handle, + pad_multiple=pad_multiple, + num_dispatched_tokens=num_dispatched_tokens, + ) + ctx.handle = handle + ctx.pad_multiple = pad_multiple + ctx.num_dispatched_tokens = num_dispatched_tokens + ctx.num_permuted_tokens = num_permuted_tokens + return combined_hidden + + @staticmethod + def backward(ctx, grad_x): + ''' + Backward pass of fused combine of the HybridEP backend + ''' + handle = ctx.handle + dispatched_hidden, _, _, _, _ = _hybrid_ep_buffer.dispatch_with_permute( + hidden=grad_x, + scaling_factor=None, + handle=handle, + pad_multiple=ctx.pad_multiple, + num_dispatched_tokens=ctx.num_dispatched_tokens, + num_permuted_tokens=ctx.num_permuted_tokens, + ) + return dispatched_hidden, None, None, None, None + + +if HAVE_HYBRIDEP: + + def hybrid_ep_dispatch( + x, + routing_map, + probs, + group, + num_local_experts, + num_sms_dispatch_api=24, + num_sms_combine_api=24, + num_dispatched_tokens=None, + num_permuted_tokens=None, + pad_multiple=None, + ): + ''' + Perform fused dispatch for "permute + dispatch a2a + permute" using the + HybridEP backend. + + Args: + x (torch.Tensor): + Input hidden states to dispatch. + routing_map (torch.Tensor): + Map indicating which expert each token is routed to. + probs (torch.Tensor): + Routing probabilities for each token-expert pair. + group (torch.distributed.ProcessGroup): + Process group used for communication. + num_local_experts (int): + Number of local experts. + num_sms_dispatch_api (int): + Number of SMs used by the dispatch API. + num_sms_combine_api (int): + Number of SMs used by the combine API. + num_dispatched_tokens (int): + Number of tokens after dispatch but before permute. HybridEP uses this + to allocate buffers. If not provided, HybridEP obtains the size from + a GPU tensor, which causes a D2H synchronization. + num_permuted_tokens (int): + Number of tokens after permute. HybridEP uses this to allocate buffers. + If not provided, HybridEP obtains the size from a GPU tensor, + which causes a D2H synchronization. + pad_multiple (int): + Alignment multiple required for FP8 GEMM. If not provided, no padding + is performed. + ''' + return HybridEPDispatch.apply( + x, + routing_map, + probs, + group, + num_local_experts, + num_sms_dispatch_api, + num_sms_combine_api, + num_dispatched_tokens, + num_permuted_tokens, + pad_multiple, + ) + + def hybrid_ep_combine(x, handle, num_dispatched_tokens, num_permuted_tokens, pad_multiple): + ''' + Perform fused combine operation for unpermute + combine a2a + unpermute + using the HybridEP backend + + args: + x (torch.Tensor): + Input hidden states to combine + handle (EventHandle): + Communication handle from dispatch operation + num_dispatched_tokens (int): + The number of tokens after unpermute but before combine. HybridEP uses this + to allocate buffers. If not provided, HybridEP obtains the size from a GPU tensor, + which causes a D2H synchronization. + num_permuted_tokens (int): The number of tokens before unpermute. HybridEP uses this + to allocate buffers. If not provided, HybridEP obtains the size from a GPU tensor, + which causes a D2H synchronization. + pad_multiple (int): + The alignment multiple required for FP8 GEMM. If not provided, no padding + is performed. + ''' + return HybridEPCombine.apply( + x, handle, num_dispatched_tokens, num_permuted_tokens, pad_multiple + ) + +else: + hybrid_ep_dispatch = None + hybrid_ep_combine = None diff --git a/megatron/core/transformer/moe/moe_layer.py b/megatron/core/transformer/moe/moe_layer.py index d5a6be9224c..095e6526934 100644 --- a/megatron/core/transformer/moe/moe_layer.py +++ b/megatron/core/transformer/moe/moe_layer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from abc import ABC, abstractmethod from dataclasses import dataclass @@ -9,7 +9,12 @@ from megatron.core import parallel_state, tensor_parallel, utils from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.module import MegatronModule -from megatron.core.transformer.moe.moe_utils import get_default_pg_collection +from megatron.core.transformer.moe.moe_utils import ( + MoECudaGraphPartialCaptureSignal, + MoECudaGraphTensorStore, + get_default_pg_collection, + maybe_skip_or_early_return_by_cudagraph, +) from megatron.core.transformer.moe.router import TopKRouter from megatron.core.transformer.moe.token_dispatcher import ( MoEAllGatherTokenDispatcher, @@ -122,7 +127,7 @@ def __init__( # Initialize router self.router = TopKRouter(config=self.config, pg_collection=pg_collection) - + self.tp_group = pg_collection.tp # Initialize token dispatcher if config.moe_token_dispatcher_type == "allgather": self.token_dispatcher = MoEAllGatherTokenDispatcher( @@ -161,21 +166,37 @@ def __init__( # Initialize shared experts if self.use_shared_expert: self.shared_experts = build_module( - self.submodules.shared_experts, config=self.config, pg_collection=pg_collection + self.submodules.shared_experts, + config=self.config, + pg_collection=pg_collection, + gate=self.config.moe_shared_expert_gate, ) if self.shared_expert_overlap: self.token_dispatcher.set_shared_experts(self.shared_experts) - def router_and_preprocess(self, hidden_states: torch.Tensor): - """Compute and preprocess token routing for dispatch. + # Cudagraph tensor store for resuming the forward pass from the end of the cudagraph. + self.cudagraph_tensor_store = MoECudaGraphTensorStore() + + @maybe_skip_or_early_return_by_cudagraph("route") + def route(self, hidden_states: torch.Tensor): + """Compute token routing for preprocessing. This method uses the router to determine which experts to send each token to, - producing routing probabilities and a mapping. It then preprocesses the - hidden states and probabilities for the token dispatcher. The original - hidden states are returned as a residual connection. + producing routing probabilities and a mapping. """ - residual = hidden_states probs, routing_map = self.router(hidden_states) + return probs, routing_map + + @maybe_skip_or_early_return_by_cudagraph("preprocess") + def preprocess( + self, hidden_states: torch.Tensor, probs: torch.Tensor, routing_map: torch.Tensor + ): + """Preprocess token routing for dispatch. + + This method preprocesses the hidden states and routing probabilities for the token + dispatcher. The original hidden states are returned as a residual connection. + """ + residual = hidden_states hidden_states, probs = self.token_dispatcher.dispatch_preprocess( hidden_states, routing_map, probs ) @@ -183,12 +204,14 @@ def router_and_preprocess(self, hidden_states: torch.Tensor): def dispatch(self, hidden_states: torch.Tensor, probs: torch.Tensor): """Dispatches tokens to assigned expert ranks via communication. + This method performs the actual communication (e.g., All-to-All) to distribute tokens and their associated probabilities to the devices hosting their assigned experts. """ return self.token_dispatcher.token_dispatch(hidden_states, probs) + @maybe_skip_or_early_return_by_cudagraph("shared_experts_compute") def shared_experts_compute(self, hidden_states: torch.Tensor): """Computes the output of the shared experts. @@ -270,8 +293,18 @@ def forward(self, hidden_states: torch.Tensor): # MoE forward: route -> dispatch -> compute -> combine def custom_forward(hidden_states): - shared_expert_output = self.shared_experts_compute(hidden_states) - hidden_states, probs, residual = self.router_and_preprocess(hidden_states) + try: + shared_expert_output = self.shared_experts_compute(hidden_states) + probs, routing_map = self.route(hidden_states) + hidden_states, probs, residual = self.preprocess(hidden_states, probs, routing_map) + except MoECudaGraphPartialCaptureSignal as e: + # This signal is raised from the maybe_skip_or_early_return_by_cudagraph decorator. + # It means we should early-return from the MoE layer forward pass. + # This happens when we are partially capturing the CUDA graph of the MoE layer, + # like cuda_graph_scope=["moe_router", "moe_preprocess"]. + # We need to return the intermediate tensors as CUDA graph outputs. + return e.get_early_return_outputs(hidden_states, shared_expert_output) + dispatched_input, probs = self.dispatch(hidden_states, probs) output, mlp_bias = self.routed_experts_compute(dispatched_input, probs, residual) output = self.combine(output, shared_expert_output) @@ -279,7 +312,7 @@ def custom_forward(hidden_states): if self.moe_layer_recompute: if self.config.fp8: - output, mlp_bias = te_checkpoint( + outputs = te_checkpoint( custom_forward, False, tensor_parallel.random.get_cuda_rng_tracker, @@ -287,11 +320,11 @@ def custom_forward(hidden_states): hidden_states, ) else: - output, mlp_bias = tensor_parallel.checkpoint(custom_forward, False, hidden_states) + outputs = tensor_parallel.checkpoint(custom_forward, False, hidden_states) else: - output, mlp_bias = custom_forward(hidden_states) + outputs = custom_forward(hidden_states) - return output, mlp_bias + return outputs def backward_dw(self): """Compute weight gradients for experts and shared experts.""" diff --git a/megatron/core/transformer/moe/moe_utils.py b/megatron/core/transformer/moe/moe_utils.py index dc857129834..e8c6a05340c 100644 --- a/megatron/core/transformer/moe/moe_utils.py +++ b/megatron/core/transformer/moe/moe_utils.py @@ -1,12 +1,14 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import math +from dataclasses import dataclass from typing import List, Optional, Union import torch from megatron.core import parallel_state from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.cuda_graphs import is_graph_capturing try: import transformer_engine as te # pylint: disable=unused-import @@ -768,12 +770,15 @@ def reduce_aux_losses_tracker_across_ranks(track_names: Optional[List[str]] = No torch.distributed.all_reduce( values, group=tracker[name]['avg_group'], op=torch.distributed.ReduceOp.AVG ) - # This ensures proper loss averaging across all ranks including CP ranks - torch.distributed.all_reduce( - values, - group=parallel_state.get_data_parallel_group(with_context_parallel=True), - op=torch.distributed.ReduceOp.AVG, - ) + # Average aux losses across data parallel ranks. + # The `global_load_balancing_loss` already uses `tp_dp_cp_group` in `reduce_group`, + # so we don't need to reduce it again. Others use `tp_cp_group` in `reduce_group`. + if name != "global_load_balancing_loss": + torch.distributed.all_reduce( + values, + group=parallel_state.get_data_parallel_group(with_context_parallel=False), + op=torch.distributed.ReduceOp.AVG, + ) def track_moe_metrics( @@ -905,12 +910,16 @@ class RandomSTE(torch.autograd.Function): """ generator = None + random_logits = None @staticmethod def forward(ctx, logits): """ Forward pass returns random logits with rank-specific seed. """ + if is_graph_capturing() and RandomSTE.random_logits is not None: + return RandomSTE.random_logits + if RandomSTE.generator is None: global_rank = torch.distributed.get_rank() base_seed = 42 @@ -918,8 +927,8 @@ def forward(ctx, logits): RandomSTE.generator = torch.Generator(device=logits.device) RandomSTE.generator.manual_seed(seed) - random_logits = logits.clone().normal_(generator=RandomSTE.generator) - return random_logits + RandomSTE.random_logits = logits.clone().normal_(generator=RandomSTE.generator) + return RandomSTE.random_logits @staticmethod def backward(ctx, grad_output): @@ -1028,3 +1037,242 @@ def get_default_pg_collection(): with_context_parallel=True ) return pg_collection + + +class MoECudaGraphPartialCaptureSignal(Exception): + """ + Used to early-return from a MoE layer forward pass in CUDA graph capture. + This signal is raised when we are partially capturing the CUDA graph of the MoE layer, + and the related intermediate tensors are recorded in self.kwargs. + Call self.get_early_return_outputs() to collect the CUDA graph outputs. + """ + + def __init__(self, moe_layer, return_step: str, **kwargs): + self.moe_layer = moe_layer + self.return_step = return_step + self.kwargs = kwargs + + def get_early_return_outputs( + self, hidden_states: torch.Tensor, shared_expert_output: torch.Tensor + ): + """ + Get the CUDA graph early return outputs for the MoE layer, including the intermediate + tensors and the intermediate attributes of the token dispatcher. + """ + if self.return_step == "route": + # Capturing the router step returns three intermediate tensors: + # hidden states, routing probabilities, and routing map. + outputs = [hidden_states, self.kwargs['probs'], self.kwargs['routing_map']] + elif self.return_step == "preprocess": + # Capturing the preprocess step returns three intermediate tensors: + # hidden states, routing probabilities, and residual connection. + # It also returns the intermediate attributes of the token dispatcher, recorded in + # "token_dispatcher.cudagraph_attrs". + outputs = [self.kwargs['hidden_states'], self.kwargs['probs'], self.kwargs['residual']] + valid_cudagraph_attrs = [] + for attr_name in self.moe_layer.token_dispatcher.cudagraph_attrs: + hier_attr_name = attr_name.split('.') + attr = self.moe_layer.token_dispatcher + for name in hier_attr_name: + attr = getattr(attr, name, None) + if attr is None: + break + if isinstance(attr, torch.Tensor): + outputs.append(attr) + valid_cudagraph_attrs.append(attr_name) + if self.moe_layer.token_dispatcher.valid_cudagraph_attrs is None: + self.moe_layer.token_dispatcher.valid_cudagraph_attrs = valid_cudagraph_attrs + else: + assert ( + self.moe_layer.token_dispatcher.valid_cudagraph_attrs == valid_cudagraph_attrs + ), ( + "valid_cudagraph_attrs mismatch: " + f"{self.moe_layer.token_dispatcher.valid_cudagraph_attrs} != " + f"{valid_cudagraph_attrs}" + ) + # Also return the shared expert output, if it is not None. + if shared_expert_output is not None: + outputs.append(shared_expert_output) + return outputs + + +@dataclass +class MoECudaGraphTensorStore: + """Storage for tensors used in CUDA graph replay for MoE layers. + + This dataclass stores intermediate tensors computed during CUDA graph replay + that need to be resumed from the end of the CUDA graph scope to skip redundant computations. + + Attributes: + hidden_states (Optional[torch.Tensor]): The hidden states output from the CUDA graph replay. + probs (Optional[torch.Tensor]): The routing probabilities for each token-expert pair. + routing_map (Optional[torch.Tensor]): The sparse mapping indicating which experts + were selected for each token. Used to skip the normal router step. + residual (Optional[torch.Tensor]): The residual connection tensor before routing. + Used to skip the normal preprocess step. + shared_expert_output (Optional[torch.Tensor]): The output from shared experts + computation. Used to skip the normal shared expert computation step. + """ + + hidden_states: Optional[torch.Tensor] = None + probs: Optional[torch.Tensor] = None + routing_map: Optional[torch.Tensor] = None + residual: Optional[torch.Tensor] = None + shared_expert_output: Optional[torch.Tensor] = None + + def is_empty(self) -> bool: + """Check if the store has any non-None tensors. + + Returns: + bool: True if all fields are None, False otherwise. + """ + return all( + getattr(self, field_name) is None + for field_name in [ + 'hidden_states', + 'probs', + 'routing_map', + 'residual', + 'shared_expert_output', + ] + ) + + def set(self, **kwargs): + """Set the tensors in the store from keyword arguments.""" + for field_name, value in kwargs.items(): + assert field_name in [ + 'hidden_states', + 'probs', + 'routing_map', + 'residual', + 'shared_expert_output', + ], f"Invalid field name: {field_name}" + if value is not None: + assert isinstance( + value, torch.Tensor + ), f"Value must be a torch.Tensor, got {type(value)} for field {field_name}" + setattr(self, field_name, value) + + def clear(self): + """Reset all stored tensors to None.""" + for field_name in [ + 'hidden_states', + 'probs', + 'routing_map', + 'residual', + 'shared_expert_output', + ]: + setattr(self, field_name, None) + + +def maybe_skip_or_early_return_by_cudagraph(step_condition): + """ + Decorator to skip certain codepaths in the MoE layer forward pass in CUDA graph replay, + or early return from the MoE layer forward pass in CUDA graph capture. + + Args: + step_condition: The step condition to check. Can be "shared_experts_compute", "route", + or "preprocess". If "shared_experts_compute", the shared experts computation will be + skipped in replay if it is in the CUDA graph scope. If "route" or "preprocess", the + router or preprocess will be skipped in replay if it is in the CUDA graph scope, or + early return from the MoE layer forward pass if it is in CUDA graph capturing mode. + + Returns: + A decorator function that wraps the MoE layer forward pass. + """ + + def maybe_raise_signal(moe_layer, **kwargs): + """ + Check if the MoE layer should early return for CUDA graph capture. + If so, raise a MoECudaGraphPartialCaptureSignal. + """ + if ( + moe_layer.config.cuda_graph_impl == "transformer_engine" + and moe_layer.training + and is_graph_capturing() + ): + if ( + step_condition == "route" + and 'moe_router' in moe_layer.config.cuda_graph_scope + and 'moe_preprocess' not in moe_layer.config.cuda_graph_scope + ): + raise MoECudaGraphPartialCaptureSignal(moe_layer, "route", **kwargs) + elif ( + step_condition == "preprocess" + and 'moe_preprocess' in moe_layer.config.cuda_graph_scope + ): + raise MoECudaGraphPartialCaptureSignal(moe_layer, "preprocess", **kwargs) + + def decorator(func): + def wrapped_func(moe_layer, *args, **kwargs): + """ + Check if we should skip executing the original function based on the current + step condition and the tensor store status. If the tensor can be found in the store, + it indicates that it is already computed by the CUDA graph replay, so we can skip it. + Otherwise, we execute the original function and check if we should raise a signal to + early return in CUDA graph capture. + """ + # The non-cudagraph codepath just calls the original function. + if not is_graph_capturing() and moe_layer.cudagraph_tensor_store.is_empty(): + return func(moe_layer, *args, **kwargs) + + assert ( + not is_graph_capturing() or moe_layer.cudagraph_tensor_store.is_empty() + ), "cudagraph_tensor_store cannot be used when it is capturing cuda graph." + if step_condition == "shared_experts_compute": + if moe_layer.cudagraph_tensor_store.shared_expert_output is None: + # Don't skip the shared expert computation. + shared_expert_output = func(moe_layer, *args, **kwargs) + else: + # Skip the shared expert computation and get value from store. + shared_expert_output = moe_layer.cudagraph_tensor_store.shared_expert_output + return shared_expert_output + elif step_condition == "route": + if moe_layer.cudagraph_tensor_store.probs is None: + # Don't skip the router. + assert ( + moe_layer.cudagraph_tensor_store.routing_map is None + and moe_layer.cudagraph_tensor_store.residual is None + ), "both routing_map and residual must be None if probs is None" + probs, routing_map = func(moe_layer, *args, **kwargs) + + # Maybe early return after the router. + maybe_raise_signal(moe_layer, probs=probs, routing_map=routing_map) + else: + # Skip the router and get value from store. + assert ( + moe_layer.cudagraph_tensor_store.routing_map is not None + or moe_layer.cudagraph_tensor_store.residual is not None + ), "either routing_map or residual must be given if probs is given" + probs, routing_map = ( + moe_layer.cudagraph_tensor_store.probs, + moe_layer.cudagraph_tensor_store.routing_map, + ) + return probs, routing_map + elif step_condition == "preprocess": + if moe_layer.cudagraph_tensor_store.residual is None: + # Don't skip the preprocess. + hidden_states, probs, residual = func(moe_layer, *args, **kwargs) + + # Maybe early return after the preprocess. + maybe_raise_signal( + moe_layer, hidden_states=hidden_states, probs=probs, residual=residual + ) + else: + # Skip the preprocess and get value from store. + assert ( + moe_layer.cudagraph_tensor_store.probs is not None + ), "probs must not be None if residual is not None" + assert ( + moe_layer.cudagraph_tensor_store.routing_map is None + ), "routing_map must be None if residual is not None" + hidden_states, probs, residual = ( + moe_layer.cudagraph_tensor_store.hidden_states, + moe_layer.cudagraph_tensor_store.probs, + moe_layer.cudagraph_tensor_store.residual, + ) + return hidden_states, probs, residual + + return wrapped_func + + return decorator diff --git a/megatron/core/transformer/moe/router.py b/megatron/core/transformer/moe/router.py index 7fa4692ef2f..16fc9d9af8f 100644 --- a/megatron/core/transformer/moe/router.py +++ b/megatron/core/transformer/moe/router.py @@ -1,10 +1,11 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from abc import ABC, abstractmethod from typing import Optional import torch +from megatron.core.jit import jit_fuser from megatron.core.tensor_parallel import reduce_from_tensor_model_parallel_region from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.moe.moe_utils import ( @@ -468,6 +469,16 @@ def apply_input_jitter(self, input: torch.Tensor): else: return input + @jit_fuser + def _apply_expert_bias(self, routing_map: torch.Tensor): + """ + Update expert bias and tokens_per_expert + Prevent extra local tokens accumulation on evaluation or activation recomputation + """ + if self.enable_expert_bias and torch.is_grad_enabled(): + with torch.no_grad(): + self.local_tokens_per_expert += routing_map.sum(dim=0) + def routing(self, logits: torch.Tensor): """Top-k routing function @@ -526,11 +537,8 @@ def routing(self, logits: torch.Tensor): probs, scores_for_aux_loss, routing_map_for_aux_loss ) - # Update expert bias and tokens_per_expert - # Prevent extra local tokens accumulation on evaluation or activation recomputation - if self.enable_expert_bias and torch.is_grad_enabled(): - with torch.no_grad(): - self.local_tokens_per_expert += routing_map.sum(dim=0) + # Optionally apply expert bias + self._apply_expert_bias(routing_map) return probs, routing_map diff --git a/megatron/core/transformer/moe/shared_experts.py b/megatron/core/transformer/moe/shared_experts.py index 93e6ad04531..c63e074e1b1 100644 --- a/megatron/core/transformer/moe/shared_experts.py +++ b/megatron/core/transformer/moe/shared_experts.py @@ -49,7 +49,7 @@ def __init__( config.ffn_hidden_size = config.moe_shared_expert_intermediate_size # TODO(Hepteract): pass pg_collection to MLP after refactoring MLP - super().__init__(config=config, submodules=submodules) + super().__init__(config=config, submodules=submodules, tp_group=pg_collection.tp) self.use_shared_expert_gate = gate if self.use_shared_expert_gate: @@ -137,7 +137,11 @@ def sharded_state_dict( state_dict = self.state_dict(prefix='', keep_vars=True) sub_sd = { f'{prefix}{name}': make_sharded_tensor_for_checkpoint( - state_dict[name], f'{prefix}{name}', prepend_offsets=sharded_offsets + state_dict[name], + f'{prefix}{name}', + prepend_offsets=sharded_offsets, + tp_group=self.tp_group, + dp_cp_group=metadata['dp_cp_group'], ) } sharded_state_dict.update(sub_sd) diff --git a/megatron/core/transformer/moe/token_dispatcher.py b/megatron/core/transformer/moe/token_dispatcher.py index 82fb7b00583..51575d423e2 100644 --- a/megatron/core/transformer/moe/token_dispatcher.py +++ b/megatron/core/transformer/moe/token_dispatcher.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging from abc import ABC, abstractmethod @@ -8,9 +8,11 @@ from megatron.core import utils from megatron.core.config import is_experimental_enabled +from megatron.core.fp4_utils import get_fp4_align_size from megatron.core.fp8_utils import get_fp8_align_size from megatron.core.fusions.fused_indices_converter import fused_indices_to_multihot from megatron.core.fusions.fused_pad_routing_map import fused_pad_routing_map +from megatron.core.jit import jit_fuser from megatron.core.tensor_parallel import ( all_to_all, gather_from_sequence_parallel_region, @@ -19,6 +21,8 @@ from megatron.core.transformer.moe.fused_a2a import ( fused_combine, fused_dispatch, + hybrid_ep_combine, + hybrid_ep_dispatch, set_deepep_num_sms, ) from megatron.core.transformer.moe.moe_utils import ( @@ -33,6 +37,8 @@ from megatron.core.transformer.moe.shared_experts import SharedExpertMLP from megatron.core.transformer.transformer_config import TransformerConfig +logger = logging.getLogger(__name__) + """ We use the following notation throughout this file: H: hidden size B: micro batch size @@ -71,6 +77,11 @@ def __init__( self.tp_rank = utils.get_pg_rank(self.tp_group) self.ep_size = utils.get_pg_size(self.ep_group) + # Attributes that need to be captured in cudagraph. These attributes are returned + # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess. + self.cudagraph_attrs = [] + self.valid_cudagraph_attrs = None + @abstractmethod def dispatch_preprocess( self, tokens: torch.Tensor, routing_map: torch.Tensor, probs: torch.Tensor @@ -193,6 +204,14 @@ def set_shared_experts(self, shared_experts): assert self.config.moe_shared_expert_overlap self.shared_experts = shared_experts + def get_align_size_for_quantization(self): + """Get the alignment size for quantization.""" + if self.config.fp8: + return get_fp8_align_size(self.config.fp8_recipe) + elif self.config.fp4: + return get_fp4_align_size(self.config.fp4_recipe) + return 16 + class MoEAllGatherTokenDispatcher(MoETokenDispatcher): """ @@ -228,6 +247,10 @@ def __init__( # device token permutation is enabled and **AllGahter** is performed. self.global_local_map = None + # Attributes that need to be captured in cudagraph. These attributes are returned + # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess. + self.cudagraph_attrs = ['routing_map'] + def dispatch_preprocess( self, hidden_states: torch.Tensor, routing_map: torch.Tensor, probs: torch.Tensor ): @@ -420,12 +443,38 @@ def __init__( "before_finish": 3, "no_sync": 4, } - self.cuda_dtoh_point = "before_permutation_1" + if ( + config.cuda_graph_impl == "transformer_engine" + and 'moe_preprocess' in config.cuda_graph_scope + ): + self.cuda_dtoh_point = "before_ep_alltoall" + else: + self.cuda_dtoh_point = "before_permutation_1" if MoEAlltoAllTokenDispatcher.cuda_dtoh_stream is None: MoEAlltoAllTokenDispatcher.cuda_dtoh_stream = torch.cuda.Stream() + # Attributes that need to be captured in cudagraph. These attributes are returned + # as cudagraph outputs when the cuda_graph_scope contains moe_preprocess. + self.cudagraph_attrs = [ + 'tokens_per_expert', + 'input_splits', + 'output_splits', + 'output_splits_tp', + 'num_out_tokens', + 'num_global_tokens_per_local_expert', + 'reversed_local_input_permutation_mapping', + 'routing_map', + ] + self.shared_experts = None + def set_shared_experts(self, shared_experts): + """Set shared expert to the dispatcher.""" + super().set_shared_experts(shared_experts) + if shared_experts.use_shared_expert_gate: + self.cudagraph_attrs.append('shared_experts.gate_score') + self.cudagraph_attrs.append('shared_experts.cached_fc1_input') + def preprocess(self, routing_map: torch.Tensor) -> torch.Tensor: """ Preprocesses the token routing map for All-to-All communication and token permutation. @@ -472,7 +521,7 @@ def preprocess(self, routing_map: torch.Tensor) -> torch.Tensor: if ( self.config.moe_expert_capacity_factor is not None - or self.config.moe_router_padding_for_fp8 + or self.config.moe_router_padding_for_quantization ): # When using token dropping or router padding, output size is dynamic. # Need to sync output size GPU->CPU before allocating output buffer @@ -574,8 +623,8 @@ def dispatch_preprocess( assert routing_map.dtype == torch.bool, "Expected bool tensor for mask" hidden_states = hidden_states.view(-1, self.hidden_shape[-1]) - if self.config.moe_router_padding_for_fp8: - pad_multiple = get_fp8_align_size(self.config.fp8_recipe) + if self.config.moe_router_padding_for_quantization: + pad_multiple = self.get_align_size_for_quantization() if is_experimental_enabled() and self.config.moe_permute_fusion: self.routing_map = fused_pad_routing_map(self.routing_map, pad_multiple) else: @@ -888,11 +937,6 @@ def combine(self, hidden_states: torch.Tensor) -> torch.Tensor: """Combine the hidden_states after expert processing.""" pass - @abstractmethod - def get_dispached_metadata(self) -> torch.Tensor: - """Get the metadata of the dispatched hidden_states.""" - pass - @abstractmethod def get_permuted_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> torch.Tensor: """Get the permuted hidden states by instances.""" @@ -904,6 +948,161 @@ def get_restored_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> pass +class _HybridEPManager(_DispatchManager): + """ + A manager class to handle fused all-to-all communication processes for MoE models using + HybridEP backend. See https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep for more details. + + The workflow of the HybridEP dispatcher is: + (1) setup_metadata(): Process routing map and probabilities to prepare dispatch metadata + (2) dispatch(): + - Permute tokens for communication, perform all-to-all communication, + and permute tokens for experts in single step + (3) combine(): + - Unpermute tokens for communication, perform all-to-all communication, + and unpermute tokens for attention in single step + """ + + def __init__( + self, + group: torch.distributed.ProcessGroup, + num_local_experts: int, + num_experts: int, + config: TransformerConfig, + ): + """ + Initialize the HybridEP dispatcher. + + Args: + group (torch.distributed.ProcessGroup): The process group to use for communication. + This should be the ETPxEP group. + num_local_experts (int): The number of local experts. + num_experts (int): The total number of experts in the group. + config (TransformerConfig): The configuration for the transformer model. + """ + self.group = group + self.num_local_experts = num_local_experts + self.num_experts = num_experts + self.config = config + self.permute_fusion = config.moe_permute_fusion + self.capacity_factor = config.moe_expert_capacity_factor + # Drop and pad the input to capacity. + self.drop_and_pad = self.config.moe_pad_expert_input_to_capacity + if self.drop_and_pad: + assert self.capacity_factor is not None + self.capacity = None + # The up-bound for the number of tokens after dispatch op, -1 means no up-bound, + # which will cause a CPU sync + self.num_dispatched_tokens = None + # Actually the sum of tokens_per_expert, the up-bound for the number of tokens + # after permute op, -1 means no up-bound, will cause a CPU sync + self.num_permuted_tokens = None + + # Metadata + self.token_probs: Optional[torch.Tensor] = None + # Handle used for combine operation + self.handle = None + # Used for padding the output for each expert + self.pad_multiple = None + + if hybrid_ep_dispatch is None: + raise ImportError( + "HybridEP is not installed. Please install HybridEP package from " + "https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep." + ) + + def setup_metadata(self, routing_map: torch.Tensor, probs: torch.Tensor): + num_tokens = routing_map.shape[0] + self.routing_map = routing_map.reshape(num_tokens, self.num_experts) + self.token_probs = probs.reshape(num_tokens, self.num_experts) + # Compute the capacity for each expert at the drop_and_pad mode + if self.drop_and_pad: + num_out_tokens = num_tokens * self.config.moe_router_topk + # Drop and pad the input to capacity. + self.capacity = get_capacity( + num_tokens=num_out_tokens, + num_experts=self.num_experts, + capacity_factor=self.capacity_factor, + ) + # We cannot predict the actual number of tokens after the dispatch op, + # so we set it to the worst case in drop_and_pad mode + self.num_dispatched_tokens = self.capacity * self.group.size() * self.num_local_experts + # In drop_and_pad mode, the number of tokens after the permute op + # can be computed on the CPU + self.num_permuted_tokens = self.num_dispatched_tokens + self.tokens_per_expert = torch.full( + (self.num_local_experts,), self.capacity * self.group.size(), dtype=torch.long + ) + + def dispatch( + self, + hidden_states: torch.Tensor, + async_finish: bool = True, + allocate_on_comm_stream: bool = True, + ) -> torch.Tensor: + # HybridEP only supports float32 probs + if self.token_probs.dtype != torch.float32: + if self.token_probs.dtype in [torch.bfloat16, torch.float16]: + logger.warning( + "HybridEP only supports float32 probs, please set --moe-router-dtype=fp32" + ) + self.token_probs = self.token_probs.float() # downcast or upcast + if self.config.fp8: + self.pad_multiple = get_fp8_align_size(self.config.fp8_recipe) + dispatched_hidden, self.dispatched_probs, _, tokens_per_expert, self.handle = ( + hybrid_ep_dispatch( + x=hidden_states, + routing_map=self.routing_map, + probs=self.token_probs, + group=self.group, + num_local_experts=self.num_local_experts, + num_sms_dispatch_api=self.config.moe_hybridep_num_sms, + num_sms_combine_api=self.config.moe_hybridep_num_sms, + num_dispatched_tokens=self.num_dispatched_tokens, + num_permuted_tokens=self.num_permuted_tokens, + pad_multiple=self.pad_multiple, + ) + ) + + if not self.drop_and_pad: + self.tokens_per_expert = tokens_per_expert + # self.num_permuted_tokens is necessary to allocate the output tensor for permute + self.num_permuted_tokens = self.tokens_per_expert.sum() + + return dispatched_hidden + + def combine( + self, + hidden_states: torch.Tensor, + async_finish: bool = True, + allocate_on_comm_stream: bool = True, + ) -> torch.Tensor: + hidden_states = hybrid_ep_combine( + x=hidden_states, + handle=self.handle, + num_dispatched_tokens=self.num_dispatched_tokens, + num_permuted_tokens=self.num_permuted_tokens, + pad_multiple=self.pad_multiple, + ) + # Release the used handle/num_permuted_tokens which could change in each iteration + self.handle = None + self.num_permuted_tokens = None + self.num_dispatched_tokens = None + return hidden_states + + def get_permuted_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states, self.dispatched_probs + + def get_restored_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states + + def get_number_of_tokens_per_expert(self) -> torch.Tensor: + ''' + Get the number of tokens per expert. + ''' + return self.tokens_per_expert + + class _DeepepManager(_DispatchManager): """ A manager class to handle fused all-to-all communication processes for MoE models using @@ -989,7 +1188,9 @@ def dispatch( # DeepEP only supports float32 probs if self.token_probs.dtype != torch.float32: if self.token_probs.dtype in [torch.bfloat16, torch.float16]: - print("DeepEP only supports float32 probs, please set --moe-router-dtype=fp32") + logger.warning( + "DeepEP only supports float32 probs, please set --moe-router-dtype=fp32" + ) self.token_probs = self.token_probs.float() # downcast or upcast hidden_states, dispatched_indices, dispatched_probs, num_tokens_per_expert, handle = ( fused_dispatch( @@ -1039,9 +1240,6 @@ def _indices_to_multihot(self, indices, probs): multihot_probs[row_indices, valid_indices] = probs[mask] return multihot_routing_map.bool(), multihot_probs - def get_dispached_metadata(self) -> torch.Tensor: - return self.dispatched_indices, self.dispatched_probs - def get_number_of_tokens_per_expert(self) -> torch.Tensor: """ Get the number of tokens per expert. @@ -1071,7 +1269,7 @@ def _pad_routing_map( """ Pad the routing map to the nearest multiple of the pad_multiple. """ - pad_multiple = get_fp8_align_size(self.config.fp8_recipe) + pad_multiple = self.get_align_size_for_quantization() num_input_tokens = routing_map.shape[0] target_tokens_per_expert = ( @@ -1106,7 +1304,7 @@ def get_permuted_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> self.dispatched_routing_map, self.dispatched_probs = self._indices_to_multihot( self.dispatched_indices, self.dispatched_probs ) - if self.config.moe_router_padding_for_fp8: + if self.config.moe_router_padding_for_quantization: self.dispatched_routing_map, self.tokens_per_expert = self._pad_routing_map( self.dispatched_routing_map, self.tokens_per_expert ) @@ -1134,6 +1332,14 @@ def get_restored_hidden_states_by_experts(self, hidden_states: torch.Tensor) -> ) return hidden_states + def get_align_size_for_quantization(self): + """Get the alignment size for quantization.""" + if self.config.fp8: + return get_fp8_align_size(self.config.fp8_recipe) + elif self.config.fp4: + return get_fp4_align_size(self.config.fp4_recipe) + return 16 + class MoEFlexTokenDispatcher(MoETokenDispatcher): """A flexible token dispatcher that abstracts the underlying tensor and expert @@ -1162,19 +1368,29 @@ def __init__( self.num_local_experts = num_local_experts self.local_expert_indices = local_expert_indices assert self.tp_size * self.ep_size > 1, "Flex token dispatcher requires TPxEP > 1" - assert ( - self.config.moe_enable_deepep - ), "DeepEP is not enabled. Please set --moe-enable-deepep to use DeepEP backend." - assert ( - self.config.moe_pad_expert_input_to_capacity is False - ), "Flex token dispatcher does not support --moe-pad-expert-input-to-capacity" - self._comm_manager = _DeepepManager( - group=self.tp_ep_group, - num_local_experts=self.num_local_experts, - router_topk=self.tp_size * self.config.moe_router_topk, - num_experts=self.tp_size * self.config.num_moe_experts, - config=self.config, - ) + if self.config.moe_flex_dispatcher_backend == "deepep": + self._comm_manager = _DeepepManager( + group=self.tp_ep_group, + num_local_experts=self.num_local_experts, + router_topk=self.tp_size * self.config.moe_router_topk, + num_experts=self.tp_size * self.config.num_moe_experts, + config=self.config, + ) + self.cudagraph_attrs = ['_comm_manager.token_probs', '_comm_manager.token_indices'] + elif self.config.moe_flex_dispatcher_backend == "hybridep": + self._comm_manager = _HybridEPManager( + group=self.tp_ep_group, + num_local_experts=self.num_local_experts, + num_experts=self.tp_size * self.config.num_moe_experts, + config=self.config, + ) + self.cudagraph_attrs = ['_comm_manager.token_probs', '_comm_manager.routing_map'] + else: + raise ValueError( + f"Invalid backend: {self.config.moe_flex_dispatcher_backend}" + "Please set --moe-flex-dispatcher-backend=deepep or " + "--moe-flex-dispatcher-backend=hybridep" + ) def set_shared_experts(self, shared_experts): raise NotImplementedError( @@ -1208,6 +1424,7 @@ def _initialize_metadata(self, routing_map: torch.Tensor, probs: torch.Tensor) - ).contiguous() return routing_map, probs + @jit_fuser def dispatch_preprocess( self, hidden_states: torch.Tensor, routing_map: torch.Tensor, probs: torch.Tensor ): diff --git a/megatron/core/transformer/multi_latent_attention.py b/megatron/core/transformer/multi_latent_attention.py index a8893ebec36..46e09daa873 100644 --- a/megatron/core/transformer/multi_latent_attention.py +++ b/megatron/core/transformer/multi_latent_attention.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import math @@ -22,6 +22,11 @@ _yarn_get_mscale, apply_rotary_pos_emb, ) +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.layers import ColumnParallelLinear from megatron.core.tensor_parallel.mappings import ( @@ -266,15 +271,19 @@ def forward( query, key, value, attention_mask, packed_seq_params=packed_seq_params ) else: + if self.offload_core_attention and self.training: + query = fine_grained_offloading_group_start(query, name="core_attn") + if inference_context is None or inference_context.is_static_batching(): - core_attn_out = self.core_attention( - query, - key, - value, - attention_mask, - packed_seq_params=packed_seq_params, - attn_mask_type=attn_mask_type, - ) + with get_fine_grained_offloading_context(self.offload_core_attention): + core_attn_out = self.core_attention( + query, + key, + value, + attention_mask, + packed_seq_params=packed_seq_params, + attn_mask_type=attn_mask_type, + ) elif self.cache_mla_latents: # Dynamic batching attention kernel. q, k, v = (query, key, value) @@ -295,6 +304,10 @@ def forward( # Only rearrange if not in absorption mode (Flash MLA handles format correctly) if not inference_context.is_decode_only(): core_attn_out = rearrange(core_attn_out, 's b h d -> s b (h d)') + if self.offload_core_attention and self.training: + (core_attn_out,) = fine_grained_offloading_group_commit( + core_attn_out, name="core_attn", forced_released_tensors=[query, key, value] + ) # We are doing absorption with cache mla latents and decode mode. if self.cache_mla_latents and inference_context.is_decode_only(): @@ -320,7 +333,14 @@ def forward( # ================= # Output. [sq, b, h] # ================= - output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + core_attn_out = fine_grained_offloading_group_start(core_attn_out, name="attn_proj") + with get_fine_grained_offloading_context(self.offload_attn_proj): + output, bias = self.linear_proj(core_attn_out) + if self.offload_attn_proj: + output, bias = fine_grained_offloading_group_commit( + output, bias, name="attn_proj", forced_released_tensors=[core_attn_out] + ) return output, bias @@ -917,3 +937,123 @@ def set_for_recompute_input_layernorm(self): if self.config.q_lora_rank is not None: set_save_original_input(self.linear_q_down_proj) set_save_original_input(self.linear_kv_down_proj) + + def clip_qk(self): + """ + QK Clipping is a technique to clip the query and key attention logits to prevent the + attention logits from exploding. Per MuonClip usage, we update the weight by calling this + function after Muon optimizer step. + """ + + if not self.config.qk_clip: + raise ValueError("qk_clip option needs to be enabled") + + if self.core_attention.current_max_attn_logits is None: + raise ValueError("current_max_attn_logits is None") + + # Check if we're in absorption mode + if self.cache_mla_latents and not hasattr(self, 'linear_kv_up_proj'): + raise ValueError( + "qk_clip is not supported when cache_mla_latents is enabled and absorption is " + "active. The linear_kv_up_proj layer has been deleted during absorption " + "preparation." + ) + + assert self.core_attention.current_max_attn_logits.shape == ( + self.num_attention_heads_per_partition, + ), f"current_max_attn_logits shape is not ({self.num_attention_heads_per_partition}, ) \ + but {self.core_attention.current_max_attn_logits.shape}" + + # only update the weight if any head has + # current_max_attn_logits > qk_clip_threshold + if torch.any(self.core_attention.current_max_attn_logits > self.config.qk_clip_threshold): + # Use num_attention_heads_per_partition for tensor parallel scenarios + + # qk_clip_balancing_eta (n, 1, 1) + assert self.core_attention.current_max_attn_logits.shape == ( + self.num_attention_heads_per_partition, + ), f"current_max_attn_logits shape is not ({self.num_attention_heads_per_partition},) \ + but {self.core_attention.current_max_attn_logits.shape}" + self.qk_clip_balancing_eta = torch.clamp( + self.config.qk_clip_threshold / self.core_attention.current_max_attn_logits, max=1.0 + ).view(self.num_attention_heads_per_partition, 1, 1) + assert torch.all(self.qk_clip_balancing_eta <= 1.0) + + # Update q side weight, keep qk_pos_emb_head_dim side weight unchanged + if self.config.q_lora_rank is None: + q_proj_weight = self.linear_q_proj.weight + else: + q_proj_weight = self.linear_q_up_proj.weight + + # Handle different weight access patterns (main_param vs direct access) + if hasattr(q_proj_weight, 'main_param'): + q_proj_weight.main_param.data.copy_( + self._clip_q_proj_weight(q_proj_weight.main_param.data) + ) + q_proj_weight.data.copy_(self._clip_q_proj_weight(q_proj_weight.data)) + + # Update k side weight, keep v side weight unchanged + kv_proj_weight = self.linear_kv_up_proj.weight + + # Handle different weight access patterns + if hasattr(kv_proj_weight, 'main_param'): + kv_proj_weight.main_param.data.copy_( + self._clip_kv_proj_weight(kv_proj_weight.main_param.data) + ) + kv_proj_weight.data.copy_(self._clip_kv_proj_weight(kv_proj_weight.data)) + + # reset current_max_attn_logits + self.core_attention.current_max_attn_logits = None + + def _clip_q_proj_weight(self, weight): + """Clip q_proj_weight""" + # Reshape to (n, a + b, -1) + weight_reshaped = weight.view( + self.num_attention_heads_per_partition, + self.config.qk_head_dim + self.config.qk_pos_emb_head_dim, + -1, + ) + + # Split into qk_head_dim and qk_pos_emb_head_dim parts: (n, a, -1) and (n, b, -1) + weight_q_nope = weight_reshaped[:, : self.config.qk_head_dim, :] + weight_q_pe = weight_reshaped[:, self.config.qk_head_dim :, :] + + # Clipping + weight_q_nope.mul_(torch.pow(self.qk_clip_balancing_eta, self.config.qk_clip_alpha)) + weight_q_pe.mul_(self.qk_clip_balancing_eta) + + # Concatenate back and reshape to original shape + weight_q_updated = torch.cat([weight_q_nope, weight_q_pe], dim=1) + weight_q_updated = weight_q_updated.view( + self.num_attention_heads_per_partition + * (self.config.qk_head_dim + self.config.qk_pos_emb_head_dim), + -1, + ) + + return weight_q_updated + + def _clip_kv_proj_weight(self, weight): + """Clip kv_proj_weight""" + # shape: (n, qk_head_dim + v_head_dim, kv_lora_rank) + weight_reshaped = weight.view( + self.num_attention_heads_per_partition, + self.config.qk_head_dim + self.config.v_head_dim, + -1, + ) + + # Split into qk_head_dim and v_head_dim parts: (n, a, -1) and (n, b, -1) + weight_k = weight_reshaped[:, : self.config.qk_head_dim, :] + weight_v = weight_reshaped[:, self.config.qk_head_dim :, :] + + # Clipping + weight_k.mul_(torch.pow(self.qk_clip_balancing_eta, 1 - self.config.qk_clip_alpha)) + + # Concatenate back and reshape to original shape + weight_kv_updated = torch.cat([weight_k, weight_v], dim=1) + weight_kv_updated = weight_kv_updated.view( + self.num_attention_heads_per_partition + * (self.config.qk_head_dim + self.config.v_head_dim), + -1, + ) + + return weight_kv_updated diff --git a/megatron/core/transformer/multi_token_prediction.py b/megatron/core/transformer/multi_token_prediction.py index bd3aa9c8c96..94fbfb23677 100755 --- a/megatron/core/transformer/multi_token_prediction.py +++ b/megatron/core/transformer/multi_token_prediction.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. from contextlib import nullcontext from dataclasses import dataclass @@ -13,18 +13,22 @@ from megatron.core.fp8_utils import get_fp8_context from megatron.core.models.backends import BackendSpecProvider, LocalSpecProvider from megatron.core.packed_seq_params import PackedSeqParams -from megatron.core.pipeline_parallel.utils import is_vp_last_stage +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel import ( gather_from_tensor_model_parallel_region, scatter_to_sequence_parallel_region, ) -from megatron.core.transformer.enums import AttnMaskType +from megatron.core.transformer.enums import AttnMaskType, LayerType from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.transformer_block import TransformerBlockSubmodules from megatron.core.transformer.transformer_config import TransformerConfig +from megatron.core.transformer.transformer_layer import get_transformer_layer_offset from megatron.core.utils import ( + get_pg_rank, is_torch_min_version, make_tp_sharded_tensor_for_checkpoint, make_viewless_tensor, @@ -53,7 +57,11 @@ def tie_word_embeddings_state_dict( - sharded_state_dict: ShardedStateDict, word_emb_weight: Tensor, word_emb_weight_key: str + sharded_state_dict: ShardedStateDict, + word_emb_weight: Tensor, + word_emb_weight_key: str, + tp_group: torch.distributed.ProcessGroup, + dp_cp_group: torch.distributed.ProcessGroup, ) -> None: """tie the embedding of the mtp processing stage in a given sharded state dict. @@ -61,13 +69,15 @@ def tie_word_embeddings_state_dict( sharded_state_dict (ShardedStateDict): state dict with the weight to tie. word_emb_weight (Tensor): weight of the word embedding. word_emb_weight_key (str): key of the word embedding in the sharded state dict. + tp_group (torch.distributed.ProcessGroup): The tensor parallel group + dp_cp_group (torch.distributed.ProcessGroup): The dp-cp comm group Returns: None, acts in-place """ mtp_word_emb_replica_id = ( 1, # copy of embedding in pre processing stage 0, - parallel_state.get_data_parallel_rank(with_context_parallel=True), + get_pg_rank(dp_cp_group), ) assert word_emb_weight_key in sharded_state_dict del sharded_state_dict[word_emb_weight_key] @@ -76,11 +86,17 @@ def tie_word_embeddings_state_dict( key=word_emb_weight_key, replica_id=mtp_word_emb_replica_id, allow_shape_mismatch=True, + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) def tie_output_layer_state_dict( - sharded_state_dict: ShardedStateDict, output_layer_weight: Tensor, output_layer_weight_key: str + sharded_state_dict: ShardedStateDict, + output_layer_weight: Tensor, + output_layer_weight_key: str, + tp_group: torch.distributed.ProcessGroup, + dp_cp_group: torch.distributed.ProcessGroup, ) -> None: """tie the output layer of the mtp processing stage in a given sharded state dict. @@ -88,13 +104,15 @@ def tie_output_layer_state_dict( sharded_state_dict (ShardedStateDict): state dict with the weight to tie. output_layer_weight (Tensor): weight of the output layer. output_layer_weight_key (str): key of the output layer in the sharded state dict. + tp_group (torch.distributed.ProcessGroup): The tensor parallel group + dp_cp_group (torch.distributed.ProcessGroup): The dp-cp comm group Returns: None, acts in-place """ mtp_output_layer_replica_id = ( 1, # copy of output layer in post processing stage 0, - parallel_state.get_data_parallel_rank(with_context_parallel=True), + get_pg_rank(dp_cp_group), ) assert output_layer_weight_key in sharded_state_dict del sharded_state_dict[output_layer_weight_key] @@ -103,6 +121,8 @@ def tie_output_layer_state_dict( key=output_layer_weight_key, replica_id=mtp_output_layer_replica_id, allow_shape_mismatch=True, + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) @@ -332,25 +352,103 @@ def get_mtp_layer_spec_for_backend( return mtp_layer_spec -def get_mtp_layer_offset(config: TransformerConfig) -> int: +def mtp_on_this_rank( + config: TransformerConfig, ignore_virtual: Optional[bool] = True, vp_stage: Optional[int] = None +) -> bool: + """ + Check if there is MTP on the current rank. + + Behavior: + - If a custom pipeline model parallel layout is provided in the config: + - If virtual pipeline parallelism is enabled (and `ignore_virtual` is False), checks + whether any MTP layers are present on this (pp_rank, vp_stage) pair. + - Otherwise, checks all virtual pipeline ranks of the current pipeline rank. Returns + True if any virtual sub-rank includes at least one MTP layer. + - If no custom layout is provided, assumes all MTP layers (if any) are placed on the last + pipeline stage. The function returns True only on the last pipeline stage. + """ + mtp_on_this_rank = False + pp_rank = parallel_state.get_pipeline_model_parallel_rank() + if config.pipeline_model_parallel_layout is not None: + # with custom PP layout, we support put MTP layers on any pipeline stage + layout = config.pipeline_model_parallel_layout.layout + if ( + not ignore_virtual + and parallel_state.get_virtual_pipeline_model_parallel_world_size() is not None + ): + assert vp_stage is not None, "vp_stage must be passed if virtual pipeline is enabled" + num_layers_to_build = layout[pp_rank][vp_stage].count(LayerType.mtp) + mtp_on_this_rank = num_layers_to_build > 0 + else: + for vpp_rank in range(len(layout[pp_rank])): + num_layers_to_build = layout[pp_rank][vpp_rank].count(LayerType.mtp) + if num_layers_to_build > 0: + mtp_on_this_rank = True + break + else: + # without custom PP layout, we only support put all of MTP layers on the last pipeline stage + if config.mtp_num_layers is not None: + mtp_on_this_rank = parallel_state.is_pipeline_last_stage( + ignore_virtual=ignore_virtual, vp_stage=vp_stage + ) + else: + mtp_on_this_rank = False + return mtp_on_this_rank + + +def get_mtp_ranks(pp_ranks: List[int], config: TransformerConfig) -> List[int]: + """Get the ranks of the MTP layers.""" + mtp_ranks = set() + if config.mtp_num_layers is None: + return [] + if config.pipeline_model_parallel_layout is None: + return [pp_ranks[-1]] + layout = config.pipeline_model_parallel_layout.layout + for pp_rank in range(len(layout)): + for vpp_rank in range(len(layout[pp_rank])): + num_layers_to_build = layout[pp_rank][vpp_rank].count(LayerType.mtp) + if num_layers_to_build: + mtp_ranks.add(pp_ranks[pp_rank]) + return list(mtp_ranks) + + +def get_mtp_layer_offset(config: TransformerConfig, vp_stage: Optional[int] = None) -> int: """Get the offset of the MTP layer.""" - # Currently, we only support put all of MTP layers on the last pipeline stage. - return 0 + # TODO(shifangx): Currently, we only support put all of MTP layers + # on the last pipeline stage, so the offset is always 0. + # We will support more flexible MTP placement in the future. + if config.pipeline_model_parallel_size > 1: + if config.pipeline_model_parallel_layout: + offset = config.pipeline_model_parallel_layout.get_layer_offset( + layer_type=LayerType.mtp, vp_stage=vp_stage + ) + else: + offset = 0 + else: + offset = 0 + return offset def get_mtp_num_layers_to_build( config: TransformerConfig, vp_stage: Optional[int] = None, pp_rank: Optional[int] = None ) -> int: """Get the number of MTP layers to build.""" - # Currently, we only support put all of MTP layers on the last pipeline stage. - vp_size = config.virtual_pipeline_model_parallel_size - if pp_rank is None: - pp_rank = parallel_state.get_pipeline_model_parallel_rank() - is_last_pp_stage = pp_rank == config.pipeline_model_parallel_size - 1 - if is_vp_last_stage(vp_stage=vp_stage, vp_size=vp_size) and is_last_pp_stage: - return config.mtp_num_layers if config.mtp_num_layers else 0 + if config.pipeline_model_parallel_layout is not None: + # If we have a custom PP layout, get the number of mtp layers in the layout array. + num_layers_to_build = config.pipeline_model_parallel_layout.get_num_layers_to_build( + layer_type=LayerType.mtp, vp_stage=vp_stage + ) + assert num_layers_to_build == config.mtp_num_layers or num_layers_to_build == 0, ( + f"Currently, we only support put all of MTP layers on the last pipeline stage, " + f"so the number of MTP layers to build ({num_layers_to_build}) must match " + f"mtp_num_layers ({config.mtp_num_layers}) or be 0." + ) else: - return 0 + if parallel_state.is_pipeline_last_stage(ignore_virtual=False, vp_stage=vp_stage): + num_layers_to_build = config.mtp_num_layers if config.mtp_num_layers else 0 + else: + num_layers_to_build = 0 + return num_layers_to_build class MTPLossAutoScaler(torch.autograd.Function): @@ -430,7 +528,7 @@ def __init__( super().__init__(config=config) self.sequence_parallel = config.sequence_parallel self.submodules = submodules - self.layer_number = layer_number + self.layer_number = layer_number + get_mtp_layer_offset(self.config, vp_stage) self.vp_stage = vp_stage self.cp_group = pg_collection.cp @@ -472,8 +570,15 @@ def __init__( skip_bias_add=False, is_expert=False, ) + + diff_transformer_layer_offset = self.config.num_layers - get_transformer_layer_offset( + self.config, vp_stage + ) self.transformer_layer = build_module( - self.submodules.transformer_layer, config=self.config, vp_stage=vp_stage + self.submodules.transformer_layer, + config=self.config, + vp_stage=vp_stage, + layer_number=self.layer_number + diff_transformer_layer_offset, ) self.final_layernorm = build_module( @@ -897,10 +1002,12 @@ def forward( (Tensor): The mtp loss tensor of shape [b, s]. """ # get hidden states from previous mtp stages - offset = get_mtp_layer_offset(self.config) + offset = get_mtp_layer_offset(self.config, self.vp_stage) hidden_states_list = list(torch.chunk(hidden_states, 1 + offset, dim=0)) hidden_states = hidden_states_list[offset] for layer_number in range(len(self.layers)): + if self.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer(layer_number == len(self.layers) - 1) (hidden_states, input_ids, position_ids) = self.layers[layer_number]( input_ids=input_ids, position_ids=position_ids, @@ -942,7 +1049,7 @@ def sharded_state_dict( sharded_state_dict = super().sharded_state_dict(prefix, sharded_offsets, metadata) layer_prefix = f'{prefix}layers.' for layer in self.layers: - offset = get_mtp_layer_offset(self.config) + offset = get_mtp_layer_offset(self.config, self.vp_stage) sharded_prefix = f'{layer_prefix}{layer.layer_number - 1 }.' state_dict_prefix = f'{layer_prefix}{layer.layer_number - 1 - offset}.' diff --git a/megatron/core/transformer/pipeline_parallel_layer_layout.py b/megatron/core/transformer/pipeline_parallel_layer_layout.py index 56467bf0e9d..3ff2d6d4464 100644 --- a/megatron/core/transformer/pipeline_parallel_layer_layout.py +++ b/megatron/core/transformer/pipeline_parallel_layer_layout.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy import logging @@ -127,15 +127,28 @@ def validate_layer_layout(self, num_layers: int, mtp_num_layers: int): if LayerType.mtp in self.layout[pp_rank][-1]: assert ( self.layout[pp_rank][-1].count(LayerType.mtp) == mtp_num_layers - ), "All of the MTP layers must be in the same stage" - assert ( - pp_rank == self.pipeline_model_parallel_size - 1 - and LayerType.loss in self.layout[pp_rank][-1] - ), "MTP layers must be in the last stage together with Loss stage." + ), "All of the MTP layers must be in the same one virtual pipeline stage" + for vpp_rank in range(self.virtual_pipeline_model_parallel_size - 1): + assert LayerType.mtp not in self.layout[0][vpp_rank], ( + f"Corrently we restrict that the MTP should not be in the first pp rank." + f"But got {self.layout[0]} for the first pp rank." + ) + ## Detect MTP standalone usage. + mtp_standalone = False + for pp_rank in range(self.pipeline_model_parallel_size): + if ( + LayerType.mtp in self.layout[pp_rank][-1] + and pp_rank != self.pipeline_model_parallel_size - 1 + ): + mtp_standalone = True + break + # TODO: remove them in the future once they are supported if self.flatten_layout.count(LayerType.encoder) > 0: raise NotImplementedError("Encoder layer is not supported for flexible pipeline layout") + return mtp_standalone + def get_num_layers_to_build( self, layer_type: LayerType = LayerType.decoder, diff --git a/megatron/core/transformer/spec_utils.py b/megatron/core/transformer/spec_utils.py index b3de8541734..24df1add0eb 100644 --- a/megatron/core/transformer/spec_utils.py +++ b/megatron/core/transformer/spec_utils.py @@ -1,9 +1,12 @@ # Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +import logging import types from dataclasses import dataclass, field from typing import Tuple, Union +logger = logging.getLogger(__name__) + @dataclass class ModuleSpec: @@ -25,6 +28,7 @@ class ModuleSpec: module: Union[Tuple, type] params: dict = field(default_factory=lambda: {}) submodules: type = None + metainfo: dict = field(default_factory=lambda: {}) def import_module(module_path: Tuple[str]): @@ -37,12 +41,15 @@ def import_module(module_path: Tuple[str]): try: module = __import__(base_path, globals(), locals(), [name]) except ImportError as e: - print(f"couldn't import module due to {e}") + logger.error(f"couldn't import module due to {e}") return None return vars(module)[name] def get_module(spec_or_module: Union[ModuleSpec, type], **additional_kwargs): + """Retrieve the module class or function specified by a ModuleSpec or + return it as is if already provided. + """ # If a module clas is already provided return it as is if isinstance(spec_or_module, (type, types.FunctionType)): return spec_or_module @@ -56,6 +63,7 @@ def get_module(spec_or_module: Union[ModuleSpec, type], **additional_kwargs): def build_module(spec_or_module: Union[ModuleSpec, type], *args, **kwargs): + """Build a module from a ModuleSpec or return it as is if already provided.""" # If the passed `spec_or_module` is # a `Function`, then return it as it is # NOTE: to support an already initialized module add the following condition diff --git a/megatron/core/transformer/transformer_block.py b/megatron/core/transformer/transformer_block.py index aead6133f22..6f69927e9e8 100755 --- a/megatron/core/transformer/transformer_block.py +++ b/megatron/core/transformer/transformer_block.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging from contextlib import nullcontext from dataclasses import dataclass @@ -16,6 +16,9 @@ from megatron.core.fusions.fused_layer_norm import FusedLayerNorm from megatron.core.inference.contexts import BaseInferenceContext from megatron.core.packed_seq_params import PackedSeqParams +from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_set_last_layer, +) from megatron.core.pipeline_parallel.utils import is_vp_first_stage, is_vp_last_stage from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.transformer.enums import LayerType @@ -281,6 +284,7 @@ def __init__( if pg_collection is None: pg_collection = ProcessGroupCollection.use_mpu_process_groups() self.pg_collection = pg_collection + self.tp_group = pg_collection.tp pp_group = self.pg_collection.pp if hasattr(self.pg_collection, 'pp') else None pp_rank = get_pg_rank(pp_group) @@ -373,7 +377,7 @@ def build_layer(layer_spec, layer_number): # @TODO: add back account_for_embedding_in_pipeline_split (see issue #293) # In pipeline parallelism, we want to add this LN only to the last stage of the pipeline # self.post_process and self.post_layer_norm guide this behavior - if self.submodules.layer_norm and self.post_process and self.post_layer_norm: + if self.has_final_layernorm_in_this_stage(): self.final_layernorm = build_module( self.submodules.layer_norm, config=self.config, @@ -383,6 +387,35 @@ def build_layer(layer_spec, layer_number): else: self.final_layernorm = None # Either this or nn.Identity + def has_final_layernorm_in_this_stage(self): + """ + Check if this vpp stage contains the final layernorm. + + Note: + Final layernorm now has been moved from the post-process stage to the last decoder + layer by using this function. + There will be a small numeric difference because of grad norm reduction when final + layernorm is placed in different pipeline stages in deterministic mode. It can still + be bitwise aligned by disabling grad norm clipping. + """ + if self.config.mtp_num_layers is None: + # for model without MTPLayer, the final layernorm is set in the stage which does + # post_process + return self.submodules.layer_norm and self.post_process and self.post_layer_norm + else: + # for model with MTPLayer, the final layernorm is set in the stage which has the + # last layer of the decoder + has_final_layernorm_in_this_stage = False + for layer in self.layers: + if layer.layer_number == self.config.num_layers: + has_final_layernorm_in_this_stage = True + break + return ( + self.submodules.layer_norm + and has_final_layernorm_in_this_stage + and self.post_layer_norm + ) + def _get_layer(self, layer_number: int): return self.layers[layer_number] @@ -522,7 +555,7 @@ def _should_call_local_cudagraph(self, *args, **kwargs): kwargs.get('inference_context') is not None or kwargs.get('inference_params') is not None ) - and self.config.cuda_graph_scope == 'full_iteration' + and 'full_iteration' in self.config.cuda_graph_scope ): if kwargs['inference_context'].is_static_batching(): using_cuda_graph = kwargs['inference_context'].is_decode_only() @@ -693,6 +726,11 @@ def forward( else: inner_quantization_context = nullcontext() + if self.config.fine_grained_activation_offloading: + fine_grained_offloading_set_last_layer( + l_no == self.num_layers_per_pipeline_rank - 1 + ) + with self.offload_context, inner_quantization_context: hidden_states, context = layer( hidden_states=hidden_states, @@ -764,6 +802,12 @@ def sharded_state_dict( elif isinstance(self.config.moe_layer_freq, list): non_homogeneous_layers = True + if isinstance(self.config.linear_attention_freq, int): + if self.config.linear_attention_freq > 1: + non_homogeneous_layers = True + elif isinstance(self.config.linear_attention_freq, list): + non_homogeneous_layers = True + if self.config.heterogeneous_block_specs: non_homogeneous_layers = True @@ -808,7 +852,11 @@ def sharded_state_dict( if not module is self.layers: sharded_state_dict.update( sharded_state_dict_default( - module, f'{prefix}{name}.', sharded_offsets, metadata + module, + f'{prefix}{name}.', + sharded_offsets, + metadata, + tp_group=self.tp_group, ) ) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 68bdfbcf021..91484f66503 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import warnings from dataclasses import dataclass @@ -192,6 +192,22 @@ class TransformerConfig(ModelParallelConfig): qk_layernorm: bool = False """Whether to apply `normalization` type of normalization to the query and key embeddings.""" + qk_clip: bool = False + """Whether to clip the query and key weights. Needed for Muon MLA Model training.""" + + qk_clip_alpha: float = 0.5 + """The balancing alpha for qk-clip. Q = Q * (eta ** alpha)""" + + qk_clip_threshold: float = 100 + """The balancing threshold for qk-clip. eta = min(threshold / max_attention_logits, 1.0)""" + + log_max_attention_logit: bool = False + """Whether to log the max attention logit across whole model. Decoupled from qk_clip, + defualts to False. Setting qk_clip will automatically log the max logit""" + + attention_output_gate: bool = False + """Whether to apply output gate to the attention layers.""" + test_mode: bool = False """Whether to run real-time tests.""" @@ -212,6 +228,38 @@ class TransformerConfig(ModelParallelConfig): moe_deepep_num_sms: int = 20 """Number of SMs to use for DeepEP.""" + moe_hybridep_num_sms: int = 16 + """Number of SMs to use for HybridEP. In pure NVL scenarios, + 16 SMs can generally achieve good bandwidth.""" + + #################### + # linear attention + #################### + linear_attention_type: Optional[str] = None + """Type of linear attention to use. Currently support gated_delta_net.""" + + linear_attention_freq: Optional[Union[int, List[int]]] = None + """Frequency between LA (linear attention) layers + and SDPA (scaled dot-product attention) layers. + Accepts either: + - An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer + - A list that defines a custom pattern, e.g.: [1,1,1,0,1,1,1,0,1,1,1,0]""" + + linear_conv_kernel_dim: Optional[int] = None + """Conv kernel dimension for the gated delta net.""" + + linear_key_head_dim: Optional[int] = None + """Query and key head dimension for the gated delta net.""" + + linear_value_head_dim: Optional[int] = None + """Value and gate head dimension for the gated delta net.""" + + linear_num_key_heads: Optional[int] = None + """Number of query and key heads for the gated delta net.""" + + linear_num_value_heads: Optional[int] = None + """Number of value and gate heads for the gated delta net.""" + #################### # initialization #################### @@ -443,6 +491,9 @@ class TransformerConfig(ModelParallelConfig): different orders to the hidden_states, causing minor numerical differences in the hidden_states gradient.""" + moe_shared_expert_gate: bool = False + """Enable gate for shared expert.""" + moe_shared_expert_overlap: bool = False """Enable overlapping between shared expert computations and dispatcher communications. Without this, the shared experts execute before the router.""" @@ -477,10 +528,14 @@ class TransformerConfig(ModelParallelConfig): DEPRECATED and replaced by moe_router_num_groups and moe_router_group_topk. """ + moe_router_padding_for_quantization: Optional[bool] = False + """Whether to pad the routing_map to make sure the number of tokens each expert receives + is a multiple of 16/32 for quantized precision (e.g., FP8, FP4). This can remove the explicit + padding in the GroupedMLP layer.""" + moe_router_padding_for_fp8: Optional[bool] = False - """Whether to pad the routing_map to make sure the number of tokens each expert received - is a multiple of 16/32 for FP8 precision. This can remove the explicit padding in the - GroupedMLP layer.""" + """[Compatibility alias for moe_router_padding_for_quantization] + Enabling this will also enable moe_router_padding_for_quantization.""" moe_router_num_groups: Optional[int] = None """Number of groups to divide experts into for group-limited routing. @@ -566,6 +621,11 @@ class TransformerConfig(ModelParallelConfig): moe_enable_deepep: bool = False """[Experimental] Enable DeepEP for efficient token dispatching and combine in MoE models.""" + moe_flex_dispatcher_backend: str = "deepep" + """[Experimental] The backend to use for flex token dispatcher. The default is "deepep". + Options are "deepep" and "hybridep". Currently only "hybridep" backend supports + the MNNVL case.""" + moe_per_layer_logging: bool = False """Enable per-layer logging for MoE, currently supports auxiliary loss and z loss.""" @@ -651,11 +711,10 @@ class TransformerConfig(ModelParallelConfig): excluding optimizer) is enabled. "transformer_engine": capture the CUDA graph using TE make_graphed_callables().""" - cuda_graph_scope: str = "full" + cuda_graph_scope: Optional[List[str]] = None """Determines the CUDA graphs capturing scope. - When cuda_graph_impl is set to "transformer_engine", valid values are "full" and "attn". - "Full" scope captures a whole Transformer layer. "Attn" scope only captures operations in - TransformerLayer._forward_attention(). + When cuda_graph_impl is set to "transformer_engine", valid values are "attn", "mlp", "moe", + "moe_router", "moe_preprocess", "mamba". None means the full layer. When cuda_graph_impl is set to "local", "full_iteration" can be specified as cuda_graph_scope to enable whole iteration CUDA graph. All other values enable layerwise CUDA graph.""" @@ -733,6 +792,33 @@ class TransformerConfig(ModelParallelConfig): """Transformer implementation to use. Options are 'transformer_engine' for Transformer Engine and 'local' for MCore.""" + fallback_to_eager_attn: bool = False + """Whether to fallback to eager attention in TE implementation. + Suggested for when desired features are not available in TE implementation.""" + + ##################################### + # Fine-grained Activation Offloading + ##################################### + fine_grained_activation_offloading: bool = False + """If True, offload the input of the specified modules to the CPU. + Fine-grained activation offloading is a module-level offloading method + instead of a layer-level offloading method like cpu_offloading.""" + + offload_modules: Optional[list[str]] = None + """The submodules to offload its input. + choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", + "mlp_norm", "expert_fc1", "moe_act". + "attn_norm": offload the input of the normalization in the attention part. + "qkv_linear": offload the input of the qkv linear part. + "core_attn": offload the input of the core attention part. + "attn_proj": offload the input of the attn linear projection part. + "mlp_norm": offload the input of the normalization in the mlp part. + "expert_fc1": offload the input of the expert fc1 part. + "moe_act": offload the input of the moe act part. + """ + min_offloaded_tensor_size: int = 1024 * 1024 + """The minimum size of the tensor to be offloaded.""" + def __post_init__(self): """Python dataclass method that is used to modify attributes after initialization. See https://docs.python.org/3/library/dataclasses.html#post-init-processing for more @@ -769,6 +855,54 @@ def __post_init__(self): f"tensor_model_parallel_size ({self.tensor_model_parallel_size})." ) + if self.linear_attention_type is not None: + supported_la_types = ["gated_delta_net", "mamba"] + assert self.linear_attention_type in supported_la_types, ( + f"linear_attention_type ({self.linear_attention_type}) only support" + f" one of {supported_la_types}." + ) + assert ( + self.linear_attention_freq is not None + ), f"linear_attention_freq must be set for linear attention." + + if self.linear_attention_type == "gated_delta_net": + # Check required parameters + assert ( + self.linear_conv_kernel_dim is not None + ), "linear_conv_kernel_dim must be set for gated delta net." + assert ( + self.linear_key_head_dim is not None + ), "linear_key_head_dim must be set for gated delta net." + assert ( + self.linear_value_head_dim is not None + ), "linear_value_head_dim must be set for gated delta net." + assert ( + self.linear_num_key_heads is not None + ), "linear_num_key_heads must be set for gated delta net." + assert ( + self.linear_num_value_heads is not None + ), "linear_num_value_heads must be set for gated delta net." + assert self.linear_num_value_heads % self.linear_num_key_heads == 0, ( + f"linear_num_value_heads ({self.linear_num_value_heads}) must be a multiple of " + f"linear_num_key_heads ({self.linear_num_key_heads})." + ) + + # Check tensor parallelism compatibility + assert ( + self.linear_num_key_heads % self.tensor_model_parallel_size == 0 + ), "linear_num_key_heads must be a multiple of tensor_model_parallel_size." + assert ( + self.linear_num_value_heads % self.tensor_model_parallel_size == 0 + ), "linear_num_value_heads must be a multiple of tensor_model_parallel_size." + + # Do not support yet, but coming soon. + assert self.context_parallel_size == 1, ( + f"Gated delta net does not support context parallel for now," + f" but got {self.context_parallel_size=}." + ) + elif self.linear_attention_type == "mamba": + raise NotImplementedError("Mamba is not supported yet.") + if self.fp8: # cannot support first last layer bf16 with delayed scaling if self.first_last_layers_bf16 and self.fp8_recipe == Fp8Recipe.delayed: @@ -847,11 +981,18 @@ def __post_init__(self): if self.moe_enable_deepep: if self.moe_token_dispatcher_type != "flex": raise ValueError("DeepEP backend is only supported with flex token dispatcher.") + warnings.warn( + "moe_enable_deepep is deprecated." + "Please use --moe-flex-dispatcher-backend=deepep instead." + ) if self.moe_token_dispatcher_type == "flex": - if self.moe_pad_expert_input_to_capacity: + if self.moe_pad_expert_input_to_capacity and ( + self.moe_enable_deepep or self.moe_flex_dispatcher_backend == "deepep" + ): raise ValueError( - "Flex token dispatcher does not support moe_pad_expert_input_to_capacity" + "Flex token dispatcher with deepep backend does not support " + "moe_pad_expert_input_to_capacity" ) if self.moe_shared_expert_intermediate_size is not None: @@ -1039,6 +1180,32 @@ def __post_init__(self): if "moe" not in self.recompute_modules: self.recompute_modules.append("moe") + if self.fine_grained_activation_offloading: + assert ( + not self.cpu_offloading + ), "fine_grained_activation_offloading cannot be enabled with cpu_offloading." + assert self.offload_modules is not None and len(self.offload_modules) > 0 + allowed_modules = { + "core_attn", + "attn_proj", + "expert_fc1", + "moe_act", + "attn_norm", + "mlp_norm", + "qkv_linear", + } + invalid_modules = set(self.offload_modules) - allowed_modules + assert not invalid_modules, ( + f'Invalid choices for offload_modules: {invalid_modules}. ' + f'Allowed modules are: {allowed_modules}' + ) + if "attn_proj" in self.offload_modules and "core_attn" not in self.offload_modules: + raise ValueError( + "attn_proj cannot be set to offload_modules alone without core_attn " + "because the input of attn_proj is the output of core_attn, " + "which is needed in core_attn.backward()." + ) + if ( self.num_layers_in_first_pipeline_stage is not None or self.num_layers_in_last_pipeline_stage is not None @@ -1100,7 +1267,7 @@ def __post_init__(self): self.virtual_pipeline_model_parallel_size = detected_vpp_size # Check whether the layout is valid. - self.pipeline_model_parallel_layout.validate_layer_layout( + self.mtp_standalone = self.pipeline_model_parallel_layout.validate_layer_layout( num_layers=self.num_layers, mtp_num_layers=self.mtp_num_layers ) @@ -1292,6 +1459,10 @@ def __post_init__(self): "apply_rope_fusion is not available. Please install TE >= 1.4." ) + if self.fused_single_qkv_rope: + if self.attention_output_gate: + raise ValueError("fused_single_qkv_rope does not support gated attention for now.") + if self.multi_latent_attention and self.rotary_interleaved: raise ValueError("rotary_interleaved does not work with multi_latent_attention.") @@ -1348,13 +1519,23 @@ def __post_init__(self): ) if self.moe_router_padding_for_fp8: - if self.fp8 is None: - raise ValueError("fp8 must be specified when moe_router_padding_for_fp8 is True.") + # enable moe_router_padding_for_quantization + warnings.warn( + "--moe-router-padding-for-fp8 is going to be deprecated. " + "Use --moe-router-padding-for-quantization instead." + ) + self.moe_router_padding_for_quantization = True + + if self.moe_router_padding_for_quantization: + if self.fp8 is None and self.fp4 is None: + raise ValueError( + "fp8/fp4 must be specified when moe_router_padding_for_quantization is True." + ) if self.moe_token_dispatcher_type in ["allgather", "alltoall_seq"]: raise ValueError( "allgather and alltoall_seq dispatcher does not support " - "moe_router_padding_for_fp8." + "moe_router_padding_for_quantization." ) if ( @@ -1411,6 +1592,8 @@ def __post_init__(self): 'use cuda_graph_impl=transformer_engine instead.' ) self.cuda_graph_impl = "transformer_engine" + if self.cuda_graph_scope is None: + self.cuda_graph_scope = [] if self.cuda_graph_impl != "none": assert self.cuda_graph_impl in [ "transformer_engine", @@ -1418,24 +1601,131 @@ def __post_init__(self): ], f"Invalid cuda graph implementation: {self.cuda_graph_impl}" if self.cpu_offloading: raise ValueError("CUDA graphs not supported with CPU offloading.") - if self.recompute_granularity: - if ( - self.recompute_granularity != "selective" - or self.cuda_graph_impl != "transformer_engine" - or self.cuda_graph_scope != "attn" - ): - raise ValueError("CUDA graphs not supported with activation recomputation.") + + elif not isinstance(self.cuda_graph_scope, list): + assert isinstance(self.cuda_graph_scope, str), ( + "cuda_graph_scope must be a string or a list of strings, " + f"got {self.cuda_graph_scope}." + ) + self.cuda_graph_scope = [self.cuda_graph_scope] + + if self.cuda_graph_impl == "local": + assert not self.cuda_graph_scope or self.cuda_graph_scope == ["full_iteration"], ( + "For local cuda graph implementation, the only valid value " + "for cuda_graph_scope is full_iteration. " + "To use other scopes, use cuda_graph_impl=transformer_engine." + ) + + if self.cuda_graph_impl == "transformer_engine": + assert "full_iteration" not in self.cuda_graph_scope, ( + "To use full iteration cuda graph, please use " + "cuda_graph_impl=transformer_engine instead of cuda_graph_impl=local." + ) + for scope in self.cuda_graph_scope: + assert scope in [ + 'attn', + 'mlp', + 'moe', + 'moe_router', + 'moe_preprocess', + 'mamba', + ], ( + "--cuda-graph-scope should be attn, mlp, moe, moe_router, moe_preprocess, " + f"or mamba, got {self.cuda_graph_scope}." + ) + + assert ( + 'moe' not in self.cuda_graph_scope or 'moe_router' not in self.cuda_graph_scope + ), 'cuda_graph_scope must not contain both moe and moe_router.' + if 'moe_preprocess' in self.cuda_graph_scope: + assert ( + 'moe_router' in self.cuda_graph_scope + ), 'moe_preprocess cuda graph is only supported with moe_router cuda graph.' + if self.num_moe_experts is None or self.num_moe_experts <= 1: + assert ( + 'moe' not in self.cuda_graph_scope + and 'moe_router' not in self.cuda_graph_scope + ), 'moe cuda graph is only supported for MoE.' else: - for module in self.recompute_modules: - if module in ['core_attn', 'mla_up_proj']: - raise ValueError( - f'attn cuda graph is not supported with {module} recompute.' + if self.moe_layer_freq == 1 or ( + isinstance(self.moe_layer_freq, list) and 0 not in self.moe_layer_freq + ): + assert 'mlp' not in self.cuda_graph_scope, ( + 'mlp cuda graph is only supported for dense layers, ' + 'but not found in the model.' + ) + if ( + self.moe_expert_capacity_factor is None + or not self.moe_pad_expert_input_to_capacity + ): + assert ( + 'moe' not in self.cuda_graph_scope + ), 'moe cuda graph is only supported with drop-padding MoE.' + if self.moe_token_dispatcher_type == 'alltoall' and ( + self.moe_expert_capacity_factor is not None + or self.moe_router_padding_for_fp8 + ): + assert 'moe_preprocess' not in self.cuda_graph_scope, ( + 'moe_preprocess cuda graph is not supported when there are ' + 'DtoH copies and synchronizations in the preprocess step.' ) + + if self.recompute_granularity: + if self.recompute_granularity != "selective" or not self.cuda_graph_scope: + raise ValueError( + "Full-layer CUDA graphs not supported with activation recomputation." + ) + elif self.cuda_graph_scope != ['full_iteration']: + # For scoped CUDA graphs, only the non-graphed parts of the layer can be + # recomputed. So check if there are overlaps between the recomputed parts + # and the graphed parts. + if "attn" in self.cuda_graph_scope: + for module in self.recompute_modules: + if module in ['core_attn', 'mla_up_proj']: + raise ValueError( + f'attn cuda graph is not supported with {module} recompute.' + ) + if "mlp" in self.cuda_graph_scope and "mlp" in self.recompute_modules: + raise ValueError(f'mlp cuda graph is not supported with mlp recompute.') + if "moe" in self.cuda_graph_scope: + for module in self.recompute_modules: + if module in ['moe_act', 'moe', 'shared_experts']: + raise ValueError( + f'moe cuda graph is not supported with {module} recompute.' + ) + if "moe_router" in self.cuda_graph_scope: + for module in self.recompute_modules: + if module in ['moe', 'shared_experts']: + raise ValueError( + f'moe_router cuda graph is not supported with {module} ' + 'recompute.' + ) if "layernorm" in self.recompute_modules: - warnings.warn( - "input_layernorm recompute is not supported with attention " - "cudagraph. Will only recompute the pre_mlp_layernorm." - ) + if ( + "attn" in self.cuda_graph_scope + and "mlp" in self.cuda_graph_scope + and ( + "moe" in self.cuda_graph_scope + or "moe_router" in self.cuda_graph_scope + ) + ): + raise ValueError( + 'cuda graph is not supported with layernorm recompute.' + ) + if "attn" in self.cuda_graph_scope: + warnings.warn( + "input_layernorm recompute is not supported with attention " + "cudagraph. Will only recompute the pre_mlp_layernorm." + ) + if ( + "mlp" in self.cuda_graph_scope + or "moe" in self.cuda_graph_scope + or "moe_router" in self.cuda_graph_scope + ): + warnings.warn( + "pre_mlp_layernorm recompute is not supported with mlp/moe " + "cudagraph. Will only recompute the input_layernorm." + ) if self.moe_token_dispatcher_type in ["allgather"]: if self.variable_seq_lengths is True: @@ -1564,6 +1854,25 @@ def __post_init__(self): f"the number of layers ({self.num_layers})" ) + if self.fallback_to_eager_attn: + assert self.transformer_impl == "transformer_engine", ( + f"fallback_to_eager_attn is only available with transformer_engine implementation," + f" but got {self.transformer_impl=}." + ) + + if self.fallback_to_eager_attn or self.transformer_impl == "local": + if self.context_parallel_size > 1 and self.cp_comm_type is not None: + all_cp_comm_types_are_all_gather = ( + all(item == "all_gather" for item in self.cp_comm_type) + if isinstance(self.cp_comm_type, list) + else self.cp_comm_type == "all_gather" + ) + if not all_cp_comm_types_are_all_gather: + raise ValueError( + f"fallback_to_eager_attn only supports all_gather communication type " + f"for context parallelism, but got {self.cp_comm_type=} instead." + ) + @dataclass class MLATransformerConfig(TransformerConfig): @@ -1631,6 +1940,9 @@ def __post_init__(self): if self.multi_latent_attention and self.apply_rope_fusion and self.rope_type != "yarn": raise ValueError("apply_rope_fusion for MLA only works with YARN RoPE.") + if self.attention_output_gate: + raise NotImplementedError("Output gate is not supported for MLA yet.") + if self.cache_mla_latents: assert ( self.apply_rope_fusion is False diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index a5babece9d0..c322788af2f 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import logging import warnings @@ -15,6 +15,7 @@ from megatron.core.dist_checkpointing.utils import apply_prefix_mapping from megatron.core.packed_seq_params import PackedSeqParams from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer.cuda_graphs import is_graph_capturing from megatron.core.transformer.enums import LayerType from megatron.core.transformer.identity_op import IdentityFuncOp, IdentityOp from megatron.core.transformer.mlp import MLP @@ -272,6 +273,7 @@ def __init__( if pg_collection is None: pg_collection = ProcessGroupCollection.use_mpu_process_groups() self.pg_collection = pg_collection + self.tp_group = pg_collection.tp self.submodules_config = submodules self.layer_number = layer_number + get_transformer_layer_offset( @@ -371,19 +373,29 @@ def __init__( # [Module 9: BiasDropoutFusion] self.mlp_bda = build_module(submodules.mlp_bda) + self.is_moe_layer = isinstance(self.mlp, MoELayer) + self.recompute_input_layernorm = False self.recompute_pre_mlp_layernorm = False self.recompute_mlp = False if self.config.recompute_granularity == 'selective': if "layernorm" in self.config.recompute_modules: - if ( - not isinstance(self.input_layernorm, IdentityOp) - and self.config.cuda_graph_impl == "none" + if not isinstance(self.input_layernorm, IdentityOp) and ( + self.config.cuda_graph_impl == "none" + or 'attn' not in self.config.cuda_graph_scope ): self.recompute_input_layernorm = True if self.config.fp8: self.self_attention.set_for_recompute_input_layernorm() - if not isinstance(self.pre_mlp_layernorm, IdentityOp): + if not isinstance(self.pre_mlp_layernorm, IdentityOp) and ( + self.config.cuda_graph_impl == "none" + or (not self.is_moe_layer and 'mlp' not in self.config.cuda_graph_scope) + or ( + self.is_moe_layer + and 'moe' not in self.config.cuda_graph_scope + and 'moe_router' not in self.config.cuda_graph_scope + ) + ): self.recompute_pre_mlp_layernorm = True if self.config.fp8: if isinstance(self.mlp, MoELayer): @@ -395,8 +407,18 @@ def __init__( set_save_original_input(self.mlp.linear_fc1) if "mlp" in self.config.recompute_modules: - if not isinstance(self.mlp, MoELayer): + if not self.is_moe_layer: self.recompute_mlp = True + self.offload_attn_norm = ( + self.config.fine_grained_activation_offloading + and "attn_norm" in self.config.offload_modules + and not isinstance(self.input_layernorm, IdentityOp) + ) + self.offload_mlp_norm = ( + self.config.fine_grained_activation_offloading + and "mlp_norm" in self.config.offload_modules + and not isinstance(self.pre_mlp_layernorm, IdentityOp) + ) # @jcasper how should we handle nvfuser? # Set bias+dropout+add fusion grad_enable execution handler. @@ -479,20 +501,29 @@ def _forward_attention( context (Tensor): Updated context tensor if cross-attention is used, otherwise None. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, + ) inference_context = deprecate_inference_params(inference_context, inference_params) # Residual connection. residual = hidden_states + if self.offload_attn_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="attn_norm") # Optional Input Layer norm if self.recompute_input_layernorm: self.input_layernorm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( - self.input_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(self.offload_attn_norm): + input_layernorm_output = self.input_layernorm_checkpoint.checkpoint( + self.input_layernorm, hidden_states + ) else: - input_layernorm_output = self.input_layernorm(hidden_states) + with get_fine_grained_offloading_context(self.offload_attn_norm): + input_layernorm_output = self.input_layernorm(hidden_states) # Self attention. nvtx_range_push(suffix="self_attention") @@ -526,6 +557,11 @@ def _forward_attention( ) nvtx_range_pop(suffix="self_attn_bda") + if self.offload_attn_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="attn_norm", forced_released_tensors=[residual] + ) + # Residual connection. residual = hidden_states @@ -563,17 +599,26 @@ def _forward_mlp(self, hidden_states, inference_context=None): output (Tensor): Transformed hidden states of shape [s, b, h]. """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_start, + get_fine_grained_offloading_context, + ) + # Residual connection. residual = hidden_states + if self.offload_mlp_norm: + hidden_states = fine_grained_offloading_group_start(hidden_states, name="mlp_norm") # Optional Layer norm post the cross-attention. if self.recompute_pre_mlp_layernorm: self.pre_mlp_norm_checkpoint = tensor_parallel.CheckpointWithoutOutput() - pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( - self.pre_mlp_layernorm, hidden_states - ) + with get_fine_grained_offloading_context(self.offload_mlp_norm): + pre_mlp_layernorm_output = self.pre_mlp_norm_checkpoint.checkpoint( + self.pre_mlp_layernorm, hidden_states + ) else: - pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) + with get_fine_grained_offloading_context(self.offload_mlp_norm): + pre_mlp_layernorm_output = self.pre_mlp_layernorm(hidden_states) nvtx_range_push(suffix="mlp") # Potentially chunk the MLP computation during prefill to minimize the peak activation size @@ -584,7 +629,19 @@ def _forward_mlp(self, hidden_states, inference_context=None): and not isinstance(self.mlp, IdentityOp) ) - if self.recompute_mlp: + if ( + self.is_moe_layer + and self.config.cuda_graph_impl == "transformer_engine" + and self.training + and is_graph_capturing() + and 'moe_router' in self.config.cuda_graph_scope + ): + assert ( + not self.recompute_pre_mlp_layernorm + ), "Recomputation is not supported for CUDA graph." + cudagraph_outputs = self.mlp(pre_mlp_layernorm_output) + return cudagraph_outputs + [residual] + elif self.recompute_mlp: if self.config.fp8: # import here to avoid circular import from megatron.core.extensions.transformer_engine import te_checkpoint @@ -613,7 +670,6 @@ def _forward_mlp(self, hidden_states, inference_context=None): bias_chunks = [bias for _, bias in outputs if bias is not None] bias_output = torch.stack(bias_chunks, dim=0).sum(dim=0) if bias_chunks else None mlp_output_with_bias = (mlp_output, bias_output) - else: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) @@ -625,6 +681,23 @@ def _forward_mlp(self, hidden_states, inference_context=None): ) nvtx_range_pop(suffix="mlp") + return self._forward_post_mlp(mlp_output_with_bias, residual) + + def _forward_post_mlp(self, mlp_output_with_bias, residual): + """ + Perform operations after the MLP computation. + + Args: + mlp_output_with_bias (Tensor): Output tensor of the MLP layer with bias. + residual (Tensor): Residual tensor. + + Returns: + output (Tensor): Transformed hidden states of shape [s, b, h]. + """ + from megatron.core.pipeline_parallel.fine_grained_activation_offload import ( + fine_grained_offloading_group_commit, + ) + # TODO: could we move `bias_dropout_add_exec_handler` itself # inside the module provided in the `bias_dropout_add_spec` module? nvtx_range_push(suffix="mlp_bda") @@ -633,6 +706,10 @@ def _forward_mlp(self, hidden_states, inference_context=None): mlp_output_with_bias, residual, self.hidden_dropout ) nvtx_range_pop(suffix="mlp_bda") + if self.offload_mlp_norm: + (hidden_states,) = fine_grained_offloading_group_commit( + hidden_states, name="mlp_norm", forced_released_tensors=[residual] + ) # Jit compiled function creates 'view' tensor. This tensor # potentially gets saved in the MPU checkpoint function context, @@ -679,7 +756,9 @@ def get_layer_static_inputs(self, seq_length, micro_batch_size): """ static_inputs = super().get_layer_static_inputs(seq_length, micro_batch_size) - if not isinstance(self.self_attention, IdentityOp): + if not isinstance(self.self_attention, IdentityOp) and ( + not self.config.cuda_graph_scope or 'attn' in self.config.cuda_graph_scope + ): slen_per_cp = seq_length // self.config.context_parallel_size static_inputs["attention_mask"] = ( ~(torch.tril(torch.ones((slen_per_cp, seq_length))).bool()) @@ -693,18 +772,28 @@ def _get_submodules_under_cudagraphs(self): """ Get the submodules that are covered by cudagraphs. """ - if self.config.cuda_graph_scope == 'full': - submodules = [self] - else: - assert ( - self.config.cuda_graph_scope == 'attn' - ), f"Invalid cuda_graph_scope {self.config.cuda_graph_scope}" - submodules = [ + if not self.config.cuda_graph_scope: + return super()._get_submodules_under_cudagraphs() + + submodules = [] + if 'attn' in self.config.cuda_graph_scope: + submodules += [ self.input_layernorm, self.self_attention, self.pre_cross_attn_layernorm, self.cross_attention, ] + if (not self.is_moe_layer and 'mlp' in self.config.cuda_graph_scope) or ( + self.is_moe_layer and 'moe' in self.config.cuda_graph_scope + ): + submodules += [self.pre_mlp_layernorm, self.mlp] + elif self.is_moe_layer and 'moe_router' in self.config.cuda_graph_scope: + submodules += [self.pre_mlp_layernorm, self.mlp.router] + if ( + self.config.moe_shared_expert_intermediate_size is not None + and not self.config.moe_shared_expert_overlap + ): + submodules += [self.mlp.shared_experts] return submodules def _te_cuda_graph_capture(self, *args, **kwargs): @@ -715,12 +804,31 @@ def _te_cuda_graph_capture(self, *args, **kwargs): attribute can be set to control the scope of the CUDA graph. 2. If context is None, it cannot be returned as output. """ - hidden_states, context = self._forward_attention(*args, **kwargs) - - if self.config.cuda_graph_scope == "full": + context = None + if not self.config.cuda_graph_scope or 'attn' in self.config.cuda_graph_scope: + hidden_states, context = self._forward_attention(*args, **kwargs) + else: + if len(args) > 0: + hidden_states = args[0] + else: + hidden_states = kwargs.pop("hidden_states") + + if ( + not self.config.cuda_graph_scope + or (not self.is_moe_layer and 'mlp' in self.config.cuda_graph_scope) + or ( + self.is_moe_layer + and ( + 'moe' in self.config.cuda_graph_scope + or 'moe_router' in self.config.cuda_graph_scope + ) + ) + ): hidden_states = self._forward_mlp(hidden_states) - cuda_graph_outputs = [hidden_states] - + if not isinstance(hidden_states, list) and not isinstance(hidden_states, tuple): + cuda_graph_outputs = [hidden_states] + else: + cuda_graph_outputs = list(hidden_states) if context is not None: cuda_graph_outputs.append(context) return tuple(cuda_graph_outputs) @@ -732,6 +840,11 @@ def _te_cuda_graph_replay(self, *args, **kwargs): However, CUDA graph accepts only Tensor inputs. Hence, `inference_context` and `packed_seq_params` are excluded from input list. """ + context = None + if self.config.cuda_graph_scope and 'attn' not in self.config.cuda_graph_scope: + hidden_states, context = self._forward_attention(*args, **kwargs) + args = (hidden_states,) + kwargs = {} assert (kwargs.get('inference_context') is None) and ( kwargs.get('packed_seq_params') is None @@ -741,19 +854,69 @@ def _te_cuda_graph_replay(self, *args, **kwargs): "For inference cuda graph, please use cuda_graph_impl=local instead." ) - cuda_graph_output = super()._te_cuda_graph_replay(*args, **kwargs) + cuda_graph_output = list(super()._te_cuda_graph_replay(*args, **kwargs)) if kwargs.get('context') is not None: - context = cuda_graph_output[-1] - cuda_graph_output = cuda_graph_output[:-1] + context = cuda_graph_output.pop() + + if ( + not self.config.cuda_graph_scope + or (not self.is_moe_layer and 'mlp' in self.config.cuda_graph_scope) + or (self.is_moe_layer and 'moe' in self.config.cuda_graph_scope) + ): + # CUDA Graph captures the whole MLP/MoE part. CUDA Graph output is the layer output. + assert len(cuda_graph_output) == 1, "CUDA Graph output should be the layer output." + output = cuda_graph_output.pop() + elif self.is_moe_layer and 'moe_router' in self.config.cuda_graph_scope: + # CUDA Graph partially captures the MoE. + # The rest of the layer should go to the normal pass. + shared_expert_output, routing_map, residual = None, None, None + mlp_residual = cuda_graph_output.pop() + if ( + self.config.moe_shared_expert_intermediate_size is not None + and not self.config.moe_shared_expert_overlap + ): + # The shared expert output is the fourth element in the CUDA graph output. + shared_expert_output = cuda_graph_output.pop() + + # Split cudagraph outputs into function outputs and attribute outputs, and + # process them separately. Function outputs should have three tensors. + func_output, attr_outputs = cuda_graph_output[:3], cuda_graph_output[3:] + if 'moe_preprocess' in self.config.cuda_graph_scope: + hidden_states, probs, residual = func_output + valid_cudagraph_attrs = self.mlp.token_dispatcher.valid_cudagraph_attrs + assert len(attr_outputs) == len( + valid_cudagraph_attrs + ), f"attr_outputs: {len(attr_outputs)} != {len(valid_cudagraph_attrs)}" + for i, attr_name in enumerate(valid_cudagraph_attrs): + hier_attr_name = attr_name.split('.') + attr = self.mlp.token_dispatcher + for name in hier_attr_name[:-1]: + attr = getattr(attr, name) + setattr(attr, hier_attr_name[-1], attr_outputs[i]) + else: + hidden_states, probs, routing_map = func_output + assert not attr_outputs, "cuda_graph_attr_outputs should be empty" + + # Resume the MoELayer forward pass from the end of the CUDA graph scope. + # The MoE layer will skip redundant computations when we pass in the calculated values + # through the keyword arguments. See MoELayer.forward docstring for more details. + nvtx_range_push(suffix="mlp") + self.mlp.cudagraph_tensor_store.set( + hidden_states=hidden_states, + probs=probs, + routing_map=routing_map, + residual=residual, + shared_expert_output=shared_expert_output, + ) + mlp_output_with_bias = self.mlp(hidden_states) + self.mlp.cudagraph_tensor_store.clear() + nvtx_range_pop(suffix="mlp") + + output = self._forward_post_mlp(mlp_output_with_bias, mlp_residual) else: - context = None - if self.config.cuda_graph_scope == "attn": - # CUDA Graph only covers the attention layer. Feed-forward - # layer still goes through the normal pass. + # CUDA Graph does not capture the MLP/MoE part at all. output = self._forward_mlp(*cuda_graph_output) - else: - output = cuda_graph_output[0] return output, context def _get_te_cuda_graph_replay_args(self, *args, **kwargs): @@ -826,7 +989,7 @@ def _should_call_local_cudagraph(self, *args, **kwargs): (kwargs.get('inference_context') is not None) or (kwargs.get('inference_params') is not None) ) - and self.config.cuda_graph_scope != 'full_iteration' + and 'full_iteration' not in self.config.cuda_graph_scope ): if kwargs['inference_context'].is_static_batching(): using_cuda_graph = kwargs['inference_context'].is_decode_only() diff --git a/megatron/core/transformer/utils.py b/megatron/core/transformer/utils.py index ac00e6557cf..880c5309933 100644 --- a/megatron/core/transformer/utils.py +++ b/megatron/core/transformer/utils.py @@ -1,7 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. """Utilities for transformer layers.""" -from functools import lru_cache from operator import itemgetter from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union @@ -11,6 +10,8 @@ from megatron.core.dist_checkpointing.mapping import ShardedObject, ShardedStateDict, StateDict from megatron.core.jit import jit_fuser from megatron.core.utils import ( + get_pg_rank, + get_tensor_model_parallel_group_if_none, make_sharded_tensor_for_checkpoint, make_tp_sharded_tensor_for_checkpoint, ) @@ -29,13 +30,11 @@ def get_linear_layer(rows, columns, init_method, perform_initialization=True): return layer -@lru_cache(maxsize=32) def get_default_causal_mask(sq: int) -> torch.Tensor: """Return the causal upper triangular mask for softmax input.""" return torch.triu(torch.ones(sq, sq, device="cuda"), diagonal=1).bool() -@lru_cache(maxsize=32) def get_sliding_window_causal_mask(sq, skv, window_size): """Create the equivalent attention mask for SWA in [sq, skv] shape""" m = torch.ones(sq, skv, dtype=torch.bool, device="cuda") @@ -79,6 +78,8 @@ def make_sharded_tensors_for_checkpoint( tensor_parallel_layers_axis_map: Optional[Dict[str, int]] = None, sharded_offsets: Iterable[Tuple[int, int, int]] = (), extra_state_suffix: str = '_extra_state', + tp_group: Optional[torch.distributed.ProcessGroup] = None, + dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, ): """Wraps tensors from transformer layers with ShardedTensor or ShardedObject. @@ -96,31 +97,52 @@ def make_sharded_tensors_for_checkpoint( applied (e.g. PP related), passed along to ShardedTensor extra_state_suffix (str, default = '_extra_state'): layers with this suffix will be wrapped with ShardedObject instead of ShardedTensor. + tp_group (Optional[torch.distributed.ProcessGroup], optional): tensor parallel group. + If None, defaults to parallel_state.get_tensor_model_parallel_group() + dp_cp_group (Optional[torch.distributed.ProcessGroup], optional): data parallel group + with context parallel. If None, defaults to + parallel_state.get_data_parallel_group(with_context_parallel=True) """ if tensor_parallel_layers_axis_map is None: tensor_parallel_layers_axis_map = {} + if tp_group is None and dp_cp_group is None: + tp_group = get_tensor_model_parallel_group_if_none(tp_group) + dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True) + sharded_state_dict = {} for layer_name in state_dict.keys(): tensor = state_dict[layer_name] layer_key = f'{prefix}{layer_name}' if layer_name.endswith(extra_state_suffix): + # Compute replica_id when groups are provided + replica_id = (0, get_pg_rank(tp_group), get_pg_rank(dp_cp_group)) + sharded_state_dict[layer_key] = make_sharded_object_for_checkpoint( - tensor, layer_key, sharded_offsets + tensor, layer_key, sharded_offsets, replica_id=replica_id ) elif layer_name in tensor_parallel_layers_axis_map: tp_axis = tensor_parallel_layers_axis_map[layer_name] sharded_state_dict[layer_key] = make_tp_sharded_tensor_for_checkpoint( - tensor, layer_key, tp_axis, prepend_offsets=sharded_offsets + tensor, + layer_key, + tp_axis, + prepend_offsets=sharded_offsets, + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) else: sharded_state_dict[layer_key] = make_sharded_tensor_for_checkpoint( - tensor, layer_key, prepend_offsets=sharded_offsets + tensor, + layer_key, + prepend_offsets=sharded_offsets, + tp_group=tp_group, + dp_cp_group=dp_cp_group, ) return sharded_state_dict @@ -169,11 +191,27 @@ def _get_extra_state_offsets( return extra_state_shape, extra_state_offset +def ensure_metadata_has_dp_cp_group(metadata: Optional[dict]) -> dict: + """Ensure `metadata` is a dict containing `dp_cp_group` entry. + + If `metadata` is None, a new dict is returned with `dp_cp_group` set. + If `metadata` is a dict and missing `dp_cp_group`, it is updated in-place. + Otherwise, asserts that `dp_cp_group` exists. + """ + if metadata is None: + return {'dp_cp_group': parallel_state.get_data_parallel_group(with_context_parallel=True)} + assert isinstance(metadata, dict), "metadata must be a dict with dp_cp_group as key" + if 'dp_cp_group' not in metadata: + metadata['dp_cp_group'] = parallel_state.get_data_parallel_group(with_context_parallel=True) + return metadata + + def sharded_state_dict_default( module: torch.nn.Module, prefix: str = '', sharded_offsets: Tuple[Tuple[int, int, int]] = (), metadata: Optional[dict] = None, + tp_group: Optional[torch.distributed.ProcessGroup] = None, ) -> ShardedStateDict: """Provides implementation for sharded_state_dict method for non-MegatronModules. @@ -189,11 +227,16 @@ def sharded_state_dict_default( sharded_offsets (Tuple[Tuple[int, int, int]], optional): sharding already applied (e.g. PP related) by sup-modules. Passed along to ShardedTensor metadata (dict, optional): metadata passed to module sharded_state_dict method + tp_group (Optional[torch.distributed.ProcessGroup], optional): tensor parallel group. + If None, defaults to parallel_state.get_tensor_model_parallel_group() Returns: dict: dictionary of state dict keys mapped to ShardedTensors """ + # Guard for cases metadata is not provided + metadata = ensure_metadata_has_dp_cp_group(metadata) + if hasattr(module, 'sharded_state_dict'): module_sharded_sd = module.sharded_state_dict( prefix=prefix, sharded_offsets=sharded_offsets, metadata=metadata @@ -201,7 +244,12 @@ def sharded_state_dict_default( else: module_sd = module.state_dict(prefix='', keep_vars=True) module_sharded_sd = make_sharded_tensors_for_checkpoint( - module_sd, prefix, {}, sharded_offsets + module_sd, + prefix, + {}, + sharded_offsets, + tp_group=tp_group, + dp_cp_group=metadata['dp_cp_group'], ) return module_sharded_sd diff --git a/megatron/core/utils.py b/megatron/core/utils.py index f8954feede8..8d6269e23a4 100644 --- a/megatron/core/utils.py +++ b/megatron/core/utils.py @@ -870,15 +870,42 @@ def make_tp_sharded_tensor_for_checkpoint( is sharded across TP group. Optionally, can provide offsets which prepend new dimensions to the tensor. + + Args: + tensor: Tensor to shard + key: Key for the sharded tensor + tp_axis: Axis to shard across tensor parallel group (default: 0) + replica_id: Replica ID for the tensor (default: None) + prepend_offsets: Offsets to prepend to tensor dimensions (default: ()) + **kwargs: Additional arguments. May include: + - tp_group: Tensor parallel group + - dp_cp_group: Data parallel + context parallel group """ + # Pop group parameters from kwargs + tp_group = kwargs.pop('tp_group', None) + dp_cp_group = kwargs.pop('dp_cp_group', None) + # If there are any additional kwargs left, surface them for visibility + # (these will be forwarded to ShardedTensor.from_rank_offsets). + if kwargs: + logger.warning( + "make_tp_sharded_tensor_for_checkpoint received extra kwargs: %s", list(kwargs.keys()) + ) + prepend_axis_num = len(prepend_offsets) new_offsets = [] - tp_rank = parallel_state.get_tensor_model_parallel_rank() - dp_rank = parallel_state.get_data_parallel_rank(with_context_parallel=True) - tp_size = parallel_state.get_tensor_model_parallel_world_size() - dp_size = parallel_state.get_data_parallel_world_size(with_context_parallel=True) - dp_replica_id = parallel_state.get_data_parallel_rank(with_context_parallel=True) + + # Get groups with fallback to parallel_state + if tp_group is None and dp_cp_group is None: + tp_group = parallel_state.get_tensor_model_parallel_group() + dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True) + + # Use local get_pg_rank and get_pg_size functions + tp_rank = get_pg_rank(tp_group) + dp_rank = get_pg_rank(dp_cp_group) + tp_size = get_pg_size(tp_group) + dp_size = get_pg_size(dp_cp_group) + dp_replica_id = get_pg_rank(dp_cp_group) new_offsets.append((tp_axis + prepend_axis_num, tp_rank, tp_size)) @@ -914,14 +941,39 @@ def make_sharded_tensor_for_checkpoint(tensor, key, prepend_offsets=(), replica_ """Helper for instantiating a non-sharded ShardedTensor (replicated across TP and DP group). Optionally, can provide offsets which prepend new dimensions to the tensor. + + Keyword Args: + tensor: Tensor to create sharded tensor for + key: Key for the sharded tensor + prepend_offsets: Offsets to prepend to tensor dimensions (default: ()) + replica_id: Replica ID for the tensor (default: None) + **kwargs: Additional arguments. May include: + - tp_group: Tensor parallel group + - dp_cp_group: Data parallel + context parallel group """ + # Pop group parameters from kwargs + tp_group = kwargs.pop('tp_group', None) + dp_cp_group = kwargs.pop('dp_cp_group', None) + # If there are any additional kwargs left, surface them for visibility + # (these will be forwarded to ShardedTensor.from_rank_offsets). + if kwargs: + logger.warning( + "make_sharded_tensor_for_checkpoint received extra kwargs: %s", list(kwargs.keys()) + ) prepend_axis_num = len(prepend_offsets) new_offsets = [] - dp_rank = parallel_state.get_data_parallel_rank(with_context_parallel=True) - dp_size = parallel_state.get_data_parallel_world_size(with_context_parallel=True) - dp_replica_id = parallel_state.get_data_parallel_rank(with_context_parallel=True) + + # Get groups with fallback to parallel_state + if tp_group is None and dp_cp_group is None: + tp_group = parallel_state.get_tensor_model_parallel_group() + dp_cp_group = parallel_state.get_data_parallel_group(with_context_parallel=True) + + # Use local get_pg_rank and get_pg_size functions + dp_rank = get_pg_rank(dp_cp_group) + dp_size = get_pg_size(dp_cp_group) + dp_replica_id = get_pg_rank(dp_cp_group) if HAVE_DTENSOR and isinstance(tensor, DTensor): # FSDP2 sharding @@ -930,7 +982,7 @@ def make_sharded_tensor_for_checkpoint(tensor, key, prepend_offsets=(), replica_ new_offsets.append((prepend_axis_num, dp_rank, dp_size)) if replica_id is None: - replica_id = (0, parallel_state.get_tensor_model_parallel_rank(), dp_replica_id) + replica_id = (0, get_pg_rank(tp_group), dp_replica_id) return ShardedTensor.from_rank_offsets( key, diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 3eb23763625..43b276ae181 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """Megatron arguments.""" @@ -69,6 +69,7 @@ def add_megatron_arguments(parser: argparse.ArgumentParser): parser = _add_vision_args(parser) parser = _add_moe_args(parser) parser = _add_mla_args(parser) + parser = _add_linear_attention_args(parser) parser = _add_heterogeneous_args(parser) parser = _add_logging_args(parser) parser = _add_straggler_detector_args(parser) @@ -319,7 +320,7 @@ def moe_freq_type(x): This allows defining arbitrary patterns of expert and dense layers. The pattern length must match the total number of transformer layers. Examples: - "([0]+[1]*23)": 1 dense layer followed by 23 experts layers + "([0]+[1]*23)": 1 dense layer followed by 23 expert layers "([1]*3+[0]*2)*2": Three expert layers followed by two dense layers, repeated twice. """ if isinstance(x, int): @@ -332,6 +333,31 @@ def moe_freq_type(x): # it's a single int but in str return int(x) +def la_freq_type(x): + """Frequency between LA (linear attention) layers and SDPA (scaled dot-product attention) layers. + + Accepts either: + - An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer + - A string "N": Same as above, but provided as a string + - A string containing a Python list expression that defines a custom pattern, e.g.: + "([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0] + where 1 indicates an LA layer and 0 indicates a SDPA layer. + This allows defining arbitrary patterns of LA and SDPA layers. + The pattern length must match the total number of transformer layers. + Examples: + "([0]+[1]*23)": 1 SDPA layer followed by 23 LA layers + "([1]*3+[0]*2)*2": Three LA layers followed by two SDPA layers, repeated twice. + """ + if x is None or isinstance(x, int): + return x + assert isinstance(x, str) + if '[' in x: + # it's a custom pattern + return _eval_pattern(x) + else: + # it's a single int but in str + return int(x) + def tuple_type(x): """ Convert a string to a tuple of integers. @@ -746,7 +772,7 @@ def validate_args(args, defaults={}): if args.rank == 0: print('accumulate and all-reduce gradients in fp32 for ' 'bfloat16 data type.', flush=True) - if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration": + if args.cuda_graph_impl == "local" and "full_iteration" in args.cuda_graph_scope: if not args.inference_dynamic_batching: assert not args.check_for_nan_in_loss_and_grad, \ "--no-check-for-nan-in-loss-and-grad should be set with full_iteration CUDA graph" @@ -977,6 +1003,19 @@ def validate_args(args, defaults={}): if args.add_bias_linear: args.add_qkv_bias = True + if args.qk_clip: + assert is_te_min_version("2.9.0"), \ + '--qk-clip is only supported with TE >= 2.9.0.' + assert 0.0 < args.qk_clip_alpha < 1.0, \ + '--qk-clip-alpha must be between 0.0 and 1.0 when using --qk-clip.' + assert args.qk_clip_threshold > 0, \ + '--qk-clip-threshold must be greater than 0 when using --qk-clip.' + + # decoupled log max attention logit check + if args.log_max_attention_logit: + assert is_te_min_version("2.9.0"), \ + '--log-max-attention-logit is only supported with TE >= 2.9.0.' + # Retro checks. if args.retro_add_retriever: @@ -1150,6 +1189,13 @@ def validate_args(args, defaults={}): args.no_load_rng = True print('Warning: disabling --no-load-rng for upcycling.') + # Muon optimizercheck + if 'muon' in args.optimizer: + assert not args.use_distributed_optimizer, "Muon optimizer does not support distributed optimizer for now." + assert not args.use_torch_fsdp2, "Muon optimizer does not support Torch-FSDP2 for now." + assert not args.use_megatron_fsdp, "Muon optimizer does not support Megatron-FSDP for now." + assert args.ckpt_format in ["torch", "torch_dist"], "Muon optimizer supports torch and torch_dist checkpoint format." + # Optimizer CPU offload check if args.optimizer_cpu_offload: assert args.use_precision_aware_optimizer, ( @@ -1183,6 +1229,10 @@ def validate_args(args, defaults={}): "when enabling delay_wgrad_compute" ) + if args.fine_grained_activation_offloading: + assert args.transformer_impl == 'transformer_engine', \ + "Fine-grained activation offloading is only supported with transformer_engine implementation" + if args.mtp_num_layers: assert not args.use_legacy_models, "The legacy Megatron models does not support Multi-Token Prediction (MTP)." assert args.position_embedding_type == "rope" or args.position_embedding_type == "none", ( @@ -1198,13 +1248,19 @@ def validate_args(args, defaults={}): if args.transformer_impl == 'transformer_engine' and not args.te_rng_tracker: args.te_rng_tracker = True warn_rank_0("te_rng_tracker is not enabled, enabling it for CUDA graphs.", args.rank) - assert "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", ""), ( - "expandable_segments:True may not be safe when using CUDA Graphs with some specific parallel settings. " - "The training may crash with illegal memory access." + assert ( + "expandable_segments:True" not in os.getenv("PYTORCH_CUDA_ALLOC_CONF", "") + or os.getenv("NCCL_GRAPH_REGISTER", "") == "0" + ), ( + "Setting NCCL_GRAPH_REGISTER=0 to avoid illegal memory access when using " + "CUDA Graph with PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True." ) assert ( args.recompute_granularity != 'full' ), 'recompute_granularity must not be full when CUDA Graphs are enabled.' + + if args.multi_latent_attention: + assert not args.group_query_attention, "Group query attention is mutually exclusive with multi latent attention." # Print arguments. _print_args("arguments", args) @@ -1346,6 +1402,9 @@ def _add_transformer_engine_args(parser): group.add_argument('--transformer-impl', default='transformer_engine', choices=['local', 'transformer_engine'], help='Which Transformer implementation to use.') + group.add_argument('--fallback-to-eager-attn', action='store_true', + help='Fallback to eager attention in TE implementation. ' + 'Suggested for when desired features are not available in TE implementation.') group.add_argument('--fp8-param-gather', action='store_true', help='Keep the compute param in fp8 (do not use any other intermediate ' 'dtype) and perform the param all-gather in fp8.') @@ -1412,22 +1471,27 @@ def _add_inference_args(parser): help="Number of CUDA graph warmup steps") group.add_argument('--external-cuda-graph', action='store_true', help='Deprecated. Use --cuda-graph-impl=transformer_engine instead. ' - 'Use TE make_graphed_callables() to capture the CUDA graph.') + 'Use TE make_graphed_callables() to capture the CUDA graph. ' + 'Use --cuda-graph-scope=\"attn\", \"mlp\", \"moe\", \"moe_router\", \"moe_preprocess\", \"mamba\" for partial capture. ') group.add_argument('--cuda-graph-impl', type=str, default='none', choices=['none', 'local', 'transformer_engine'], help='Determines the CUDA graph capture implementation. ' '"none": no CUDA graph. ' '"local": capture the CUDA graph using MCore local implementation. --cuda-graph-scope=\"full_iteration\" enables whole iteration CUDA graph. ' '"transformer_engine": capture the CUDA graph using TE make_graphed_callables().') - group.add_argument('--cuda-graph-scope', type=str, default='full', - choices=['full', 'attn', 'full_iteration'], - help='Determines the CUDA graphs capturing scope. Valid values are ' - '\"full\", \"attn\" and \"full_iteration\". \"Full\" scope captures a whole ' - 'Transformer layer. \"Attn\" scope only captures operations in ' - 'TransformerLayer._forward_attention(). \"ful_iteration\" scope captures a ' - 'whole iteration. ' - 'full_iteration scope is only supported with --cuda-graph-impl=local, ' - 'attn scope is only supported with --cuda-graph-impl=transformer_engine.') + group.add_argument('--cuda-graph-scope', nargs='+', type=str, default=[], + help='Determines the CUDA graphs capturing scope. ' + 'choices: "attn", "mlp", "moe", "moe_router", "moe_preprocess", "mamba", "full_iteration". ' + '"attn": captures operations in TransformerLayer._forward_attention(). ' + '"mlp": captures operations in TransformerLayer._forward_mlp() for a dense layer. ' + '"moe": captures operations in TransformerLayer._forward_mlp() for a MoE layer. ' + '"moe_router": captures operations in TransformerLayer._forward_mlp() up to MoELayer.router(), ' + 'including the shared experts if they are not overlapped with EP comm. ' + '"moe_preprocess": captures operations in MoELayer.preprocess(). Must be used together with "moe_router". ' + '"mamba": captures the mamba layer. ' + '"full_iteration": captures a whole iteration. ' + 'full_iteration scope is only supported with --cuda-graph-impl=local, other scopes are only supported with --cuda-graph-impl=transformer_engine. ' + 'If not specified, the default scope is to capture the whole Transformer layer.') group.add_argument('--use-legacy-static-engine', action='store_true', default=False, help='Use legacy static engine. (Current static engine uses dynamic engine under the hood)', dest='use_legacy_static_engine') @@ -1586,6 +1650,8 @@ def _add_network_size_args(parser): group.add_argument('--group-query-attention', action='store_true', help='Use group-query attention.') group.add_argument('--num-query-groups', type=int, default=1) + group.add_argument('--attention-output-gate', action='store_true', + help='Whether to apply output gate to the attention.') group.add_argument('--softmax-type', type=str, default='vanilla', choices=['learnable', 'vanilla', 'off-by-one'], help='Type of softmax to use for the attention. Supports both a fixed offset and ' @@ -1872,6 +1938,8 @@ def _add_logging_args(parser): group.add_argument('--log-world-size-to-tensorboard', action='store_true', help='Enable world size logging to tensorboard.') + group.add_argument('--log-max-attention-logit', action='store_true', + help='Enable max attention logit logging to tensorboard.') group.add_argument('--wandb-project', type=str, default='', help='The wandb project name. Ignore wandb by default.') group.add_argument('--wandb-entity', type=str, default='', @@ -1904,6 +1972,12 @@ def _add_regularization_args(parser): group.add_argument('--weight-decay-incr-style', type=str, default='constant', choices=['constant', 'linear', 'cosine'], help='Weight decay increment function.') + group.add_argument('--no-weight-decay-cond-type', type=str, choices=['qwen3_next'], + help='Type of no weight decay condition. Choices: ' + 'None (default): param no weight decay if and only if it is 1D; or it is bias; ' + 'or it is embedding and embedding_init_method_std is not None. ' + '"qwen3_next": In addition to the default rules, ' + 'apply weight decay to qk layernorm as a special case.') group.add_argument('--clip-grad', type=float, default=1.0, help='Gradient clipping based on global L2 norm.') group.add_argument('--adam-beta1', type=float, default=0.9, @@ -1917,6 +1991,27 @@ def _add_regularization_args(parser): 'numerical stability') group.add_argument('--sgd-momentum', type=float, default=0.9, help='Momentum factor for sgd') + group.add_argument('--muon-momentum', type=float, default=0.9, + help='Momentum factor for Muon optimizer') + group.add_argument('--muon-no-split-qkv', action='store_false', default=True, + dest='muon_split_qkv', + help='Whether to split QKV parameters for Muon optimizer') + group.add_argument('--muon-use-nesterov', action='store_true', + help='Whether to use Nesterov-style momentum in the internal SGD') + group.add_argument('--muon-scale-mode', type=str, default='spectral', + choices=['spectral', 'unit_rms_norm', 'shape_scaling'], + help='Scale mode for Muon optimizer') + group.add_argument('--muon-fp32-matmul-prec', type=str, default='medium', + choices=['low', 'medium', 'high'], + help='FP32 matmul precision for Newton-Schulz iteration') + group.add_argument('--muon-num-ns-steps', type=int, default=5, + help='Number of Newton-Schulz steps for Muon optimizer') + group.add_argument('--muon-tp-mode', type=str, default='blockwise', + choices=['blockwise', 'duplicated', 'distributed'], + help='How to perform NS calculation for tensor model parallel weights') + group.add_argument('--muon-extra-scale-factor', type=float, default=1.0, + help='Additional scale factor for the muon update') + return parser @@ -2214,8 +2309,14 @@ def _add_training_args(parser): group.add_argument('--add-qkv-bias', action='store_true', help='Enable bias only in the QKV linear layers', dest='add_qkv_bias') + group.add_argument('--qk-clip', action='store_true', + help='Whether to use qk-clip for training stabilization, strongly recommended for Muon.') + group.add_argument('--qk-clip-alpha', type=float, default=0.5, + help='The balancing alpha for qk-clip.') + group.add_argument('--qk-clip-threshold', type=float, default=100, + help='The balancing threshold for qk-clip.') group.add_argument('--optimizer', type=str, default='adam', - choices=['adam', 'sgd'], + choices=['adam', 'sgd', 'muon', 'dist_muon'], help='Optimizer function') group.add_argument('--optimizer-cpu-offload', action='store_true', help='Offload optimizer state to CPU') @@ -2289,7 +2390,14 @@ def _add_training_args(parser): help='The communicator group names to use high priority streams.') group.add_argument('--use-te-activation-func', action='store_true', help='Use activation function kernel from Transformer Engine in MLP module.') - + group.add_argument('--fine-grained-activation-offloading', action='store_true', + help='Enable fine-grained activation offloading.') + group.add_argument('--offload-modules', nargs='*', type=str, default=[], + help='The submodules to offload its input. Choices: "attn_norm", "qkv_linear", "core_attn", "attn_proj", "mlp_norm", "expert_fc1", "moe_act".') + group.add_argument('--min-offloaded-tensor-size', type=int, default=1024*1024, + help='The minimum size of the tensor to be offloaded.') + group.add_argument('--disable-jit-fuser', action='store_true', + help='Disable the JIT fuser.') return parser @@ -3084,7 +3192,7 @@ def _add_moe_args(parser): '- A string containing a Python list expression that defines a custom pattern, e.g.: ' '"([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0] ' 'where 1 indicates an expert layer and 0 indicates a dense layer. ' - 'Examples: "([0]+[1]*23)": 1 dense layer followed by 23 experts layers, ' + 'Examples: "([0]+[1]*23)": 1 dense layer followed by 23 expert layers, ' '"([1]*3+[0]*2)*2": Three expert layers followed by two dense layers, repeated twice.') group.add_argument('--moe-ffn-hidden-size', type=int, default=None, help='The hidden size of each expert\'s feed-forward network (ffn). ' @@ -3092,13 +3200,9 @@ def _add_moe_args(parser): group.add_argument('--moe-shared-expert-intermediate-size', type=int, default=None, help='Shared expert total ffn hidden size. ' 'It should be equal to "num_shared_experts * ffn_size_of_each_shared_expert" if there are multiple shared experts. ' - 'None means no shared expert. ' - 'By default, the shared experts execute before the router. However, when ' - '--moe-shared-expert-overlap or --overlap-moe-expert-parallel-comm is set, ' - 'the shared experts execute after the router, before the routed experts. ' - 'This makes the gradients from the router and the shared experts added in ' - 'different orders to the hidden_states, causing minor numerical differences ' - 'in the hidden_states gradient.') + 'None means no shared expert.') + group.add_argument('--moe-shared-expert-gate', action='store_true', + help='Enable gate for shared expert. Only effective when moe-shared-expert-intermediate-size is set.') group.add_argument('--moe-shared-expert-overlap', action='store_true', help='Enable overlapping between shared expert computations and dispatcher communications. ' 'Without this, the shared experts execute before the router. ' @@ -3155,12 +3259,15 @@ def _add_moe_args(parser): 'The default value 1e-3 is same as that used in DeepSeekV3.') group.add_argument('--moe-router-force-load-balancing', action='store_true', help='[Experimental] Force override routing to balance token distribution using random logits for MoE routers, supporting naive top-k and group-limited top-k. This experimental feature is for benchmarking purposes only!') - group.add_argument('--moe-router-padding-for-fp8', action='store_true', + group.add_argument('--moe-router-padding-for-quantization', action='store_true', help='Pad the routing_map to make sure the number of tokens each expert received ' - 'is a multiple of 16/32 for FP8 precision. It is suggested to enable this for ' - 'dropless training with FP8 precision when num_local_experts > 1. This is a more ' - 'efficient way to pad for FP8 which eliminates the explicit padding in the ' + 'is a multiple of 16/32 for FP8/FP4 precision. It is suggested to enable this for ' + 'dropless training with FP8/FP4 precision when num_local_experts > 1. This is a more ' + 'efficient way to pad for FP8/FP4 which eliminates the explicit padding in the ' 'GroupedMLP layer.') + group.add_argument('--moe-router-padding-for-fp8', action='store_true', + help='[Compatibility alias for --moe-router-padding-for-quantization] ' + 'Enabling this will also enable --moe-router-padding-for-quantization.') group.add_argument('--moe-aux-loss-coeff', type=float, nargs='+', default=0.0, help='Scaling coefficient for the aux loss: a starting value of 1e-2 is recommended.') group.add_argument('--moe-z-loss-coeff', type=float, default=None, @@ -3175,9 +3282,15 @@ def _add_moe_args(parser): default='allgather', help="The type of token dispatcher to use. The default is 'allgather'. Options are 'allgather', 'alltoall'. We recommend using 'alltoall' when applying expert parallelism. For more information, please refer to the documentation in core/moe/README.") group.add_argument('--moe-enable-deepep', action='store_true', - help='[Experimental] Enable DeepSeek/DeepEP for efficient token dispatching and combine in MoE models. Only works with flex token dispatcher by setting --moe-token-dispatcher-type=flex.') + help='DEPRECATED: Please use --moe-flex-dispatcher-backend=deepep instead.') + group.add_argument('--moe-flex-dispatcher-backend', type=str, + choices=['deepep', 'hybridep'], + default='deepep', + help='The backend to use for flex token dispatcher. The default is "deepep". Options are "deepep" and "hybridep".') group.add_argument('--moe-deepep-num-sms', type=int, default=20, help='Number of SMs to use for DeepEP.') + group.add_argument('--moe-hybridep-num-sms', type=int, default=16, + help='Number of SMs to use for HybridEP.') group.add_argument('--moe-permute-fusion', action='store_true', help='Fuse token rearrangement ops during token dispatching.') # Token dropping arguments @@ -3227,6 +3340,31 @@ def _add_mla_args(parser): return parser +def _add_linear_attention_args(parser): + group = parser.add_argument_group(title="la") + group.add_argument('--linear-attention-type', default=None, choices=['gated_delta_net', 'mamba'], type=str, + help='Type of linear attention to use. Currently support gated_delta_net and mamba.') + group.add_argument('--linear-attention-freq', type=la_freq_type, default=None, + help='Frequency between LA (linear attention) layers and' + ' SDPA (scaled dot-product attention) layers. Accepts either: ' + '- An integer N: Represents a (N-1):N ratio, meaning (N-1) LA layers for every 1 SDPA layer ' + '- A string containing a Python list expression that defines a custom pattern, e.g.: ' + '"([1]*3+[0]*1)*3" evaluates to [1,1,1,0,1,1,1,0,1,1,1,0] ' + 'where 1 indicates an LA layer and 0 indicates a SDPA layer. ' + 'Examples: "([0]+[1]*23)": 1 SDPA layer followed by 23 LA layers, ' + '"([1]*3+[0]*2)*2": Three LA layers followed by two SDPA layers, repeated twice.') + group.add_argument('--linear-conv-kernel-dim', default=4, type=int, + help='Conv kernel dimension for the gated delta net.') + group.add_argument('--linear-key-head-dim', default=128, type=int, + help='Query and key head dimension for the gated delta net.') + group.add_argument('--linear-value-head-dim', default=128, type=int, + help='Value and gate head dimension for the gated delta net.') + group.add_argument('--linear-num-key-heads', default=16, type=int, + help='Number of query and key heads for the gated delta net.') + group.add_argument('--linear-num-value-heads', default=32, type=int, + help='Number of value and gate heads for the gated delta net.') + return parser + def _add_heterogeneous_args(parser): """ Heterogeneous models refer to transformer architectures where individual layers can differ diff --git a/megatron/training/checkpointing.py b/megatron/training/checkpointing.py index f912be84651..feacccba162 100644 --- a/megatron/training/checkpointing.py +++ b/megatron/training/checkpointing.py @@ -17,6 +17,7 @@ import numpy as np import torch +from typing import Optional, Union, List, Dict, Any from torch.distributed.checkpoint import FileSystemReader, default_planner from megatron.core import dist_checkpointing, mpu, tensor_parallel @@ -28,11 +29,13 @@ ) from megatron.core.msc_utils import MultiStorageClientFeature, open_file from megatron.core.num_microbatches_calculator import update_num_microbatches +from megatron.core.utils import get_pg_rank, get_pg_size from megatron.core.optimizer import DistributedOptimizer from megatron.core.rerun_state_machine import get_rerun_state_machine from megatron.core.utils import get_torch_version, is_torch_min_version from ..core.dist_checkpointing.serialization import get_default_save_sharded_strategy +from ..core.dist_checkpointing.utils import _clean_metadata_for_serialization from . import ft_integration, wandb_utils from .async_utils import is_empty_async_queue, schedule_async_save from .global_vars import get_args @@ -306,7 +309,7 @@ def read_metadata(tracker_filename): return max_iter, release -def get_rng_state(ckpt_format: str): +def get_rng_state(ckpt_format: str, tp_group: torch.distributed.ProcessGroup, pp_group: torch.distributed.ProcessGroup) -> Union[List[Dict[str, Any]], ShardedObject]: """Collect rng state across data parallel ranks.""" args = get_args() rng_state = { @@ -329,10 +332,10 @@ def get_rng_state(ckpt_format: str): rng_state_list = [rng_state] if ckpt_format == "torch_dist": - pp_rank = mpu.get_pipeline_model_parallel_rank() - pp_size = mpu.get_pipeline_model_parallel_world_size() - tp_rank = mpu.get_tensor_model_parallel_rank() - tp_size = mpu.get_tensor_model_parallel_world_size() + pp_rank = get_pg_rank(pp_group) + pp_size = get_pg_size(pp_group) + tp_rank = get_pg_rank(tp_group) + tp_size = get_pg_size(tp_group) rng_state_list = ShardedObject('rng_state', rng_state_list, (pp_size, tp_size), (pp_rank, tp_rank), replica_id=mpu.get_data_parallel_rank(with_context_parallel=True)) elif ckpt_format == "fsdp_dtensor": @@ -351,7 +354,8 @@ class CheckpointType(Enum): TORCH_DCP = auto() FSDP_DTENSOR = auto() -def _build_sharded_state_dict_metadata(args: Namespace) -> dict: + +def _build_sharded_state_dict_metadata(args: Namespace, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None) -> dict: """Builds metadata used for sharded_state_dict versioning. The whole content metadata is passed to ``shared_state_dict`` model and optimizer methods @@ -361,6 +365,10 @@ def _build_sharded_state_dict_metadata(args: Namespace) -> dict: In particular, a simple integer (or SemVer) versioning flag (e.g. `metadata['version'] = 3.4`) is discouraged, because the metadata serves for all models and optimizers and it's practically impossible to enforce a linearly increasing versioning for this whole space. + + Args: + args: Arguments namespace + dp_cp_group: Data parallel + context parallel group (default: None, falls back to mpu API) """ metadata = {} @@ -389,11 +397,15 @@ def _build_sharded_state_dict_metadata(args: Namespace) -> dict: metadata['singleton_local_shards'] = False metadata['chained_optim_avoid_prefix'] = True + # Add dp_cp_group to metadata. If not provided, fallback to global parallel state. + if dp_cp_group is None: + dp_cp_group = mpu.get_data_parallel_group(with_context_parallel=True) + metadata['dp_cp_group'] = dp_cp_group return metadata def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floating_point_operations_so_far, checkpointing_context=None, pipeline_rank=None, expert_rank=None, tensor_rank=None, pipeline_parallel=None, expert_parallel=None, non_persistent_ckpt=False, - train_data_iterator=None, preprocess_common_state_dict_fn = None, release=False): + train_data_iterator=None, preprocess_common_state_dict_fn = None, release=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None): """Save a model, optimizer and optionally dataloader checkpoint. Checkpointing context is used to persist some checkpointing state @@ -407,6 +419,9 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati Dataloader checkpoint is only saved if the dataloader supports it. Currently this applies only to the Megatron Energon dataloader (multimodal) and not the built-in Megatron dataloader (text-only). + + Args: + dp_cp_group: Data parallel + context parallel group (default: None, falls back to mpu API) """ start_ckpt = time() args = get_args() @@ -450,7 +465,10 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati iteration, save_dir, ckpt_format)) # Collect rng state across data parallel ranks. - rng_state = get_rng_state(args.ckpt_format) + if tp_group is None and pp_group is None: + tp_group = mpu.get_tensor_model_parallel_group() + pp_group = mpu.get_pipeline_model_parallel_group() + rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group) # Collect rerun state across all ranks rerun_state_machine = get_rerun_state_machine() @@ -478,6 +496,14 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati ensure_directory_exists(optim_checkpoint_name) if not optimizer.is_stub_optimizer: optimizer.save_parameter_state(optim_checkpoint_name) + + # LayerWiseDistributedOptimizer save optimizer state to file on different ranks + if getattr(args, "optimizer", "adam").startswith("dist_") and args.ckpt_format == 'torch': + dp_rank = mpu.get_data_parallel_rank() + optim_checkpoint_name = os.path.join(os.path.dirname(checkpoint_name), f"layer_wise_optimizer_{dp_rank}.pt") + ensure_directory_exists(optim_checkpoint_name) + if not optimizer.is_stub_optimizer: + optimizer.save_state_dict_to_file(optim_checkpoint_name) async_save_request = None if args.async_save: @@ -493,7 +519,7 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati or mpu.get_expert_data_parallel_rank() == 0 \ or ckpt_type != CheckpointType.LEGACY: if ckpt_type != CheckpointType.LEGACY: - sharded_sd_metadata = _build_sharded_state_dict_metadata(args) + sharded_sd_metadata = _build_sharded_state_dict_metadata(args, dp_cp_group=dp_cp_group) if args.use_distributed_optimizer: print_rank_0(f'Storing distributed optimizer sharded state of type' f' {sharded_sd_metadata["distrib_optim_sharding_type"]}') @@ -545,7 +571,7 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati async_sharded_save=args.async_save, validate_access_integrity=validate_sharding_integrity, preprocess_common_before_consistancy_check=preprocess_common_state_dict_fn, - content_metadata=sharded_sd_metadata) + content_metadata=_clean_metadata_for_serialization(sharded_sd_metadata)) # [ModelOpt]: save sharded modelopt_state if has_nvidia_modelopt: save_sharded_modelopt_state(model, checkpoint_name, (args.ckpt_format, 1)) @@ -806,7 +832,13 @@ def generate_state_dict( key = f"model{i}" if args.ckpt_format == "torch_dist": - model_sd = model[i].sharded_state_dict(**(model_sd_kwargs or {})) + model_sd = model[i].sharded_state_dict( + **(model_sd_kwargs or { + "metadata": { + "dp_cp_group": mpu.get_data_parallel_group(with_context_parallel=True) + } + }) + ) else: # torch, torch_dcp, fsdp_dtensor model_sd = model[i].state_dict_for_save_checkpoint() @@ -815,10 +847,16 @@ def generate_state_dict( # Optimizer stuff. if not args.no_save_optim: if optimizer is not None and not optimizer.is_stub_optimizer: - optimizer_sd = None if args.ckpt_format == "torch_dist": - optimizer_sd = optimizer.sharded_state_dict(state_dict, **(optim_sd_kwargs or {})) + optimizer_sd = optimizer.sharded_state_dict( + state_dict, + **(optim_sd_kwargs or { + "metadata": { + "dp_cp_group": mpu.get_data_parallel_group(with_context_parallel=True) + } + }) + ) elif args.ckpt_format == "fsdp_dtensor": if optim_sd_kwargs is None: optim_sd_kwargs = {} @@ -862,7 +900,7 @@ def preprocess_fsdp_dtensor_state_dict(args, raw_state_dict, model): ) state_dict["model"] = model_state_dict if args.num_experts: - state_dict["model"] = handle_experts_in_state_dict(state_dict["model"]) + state_dict["model"] = handle_experts_in_state_dict(state_dict["model"], args.num_experts) preprocess_state_dict_for_uneven_dtensor(state_dict) return state_dict @@ -1361,7 +1399,7 @@ def _set_arg(arg_name, old_arg_name=None, force=False): def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', strict=True, - checkpointing_context=None, skip_load_to_model_and_opt=False): + checkpointing_context=None, skip_load_to_model_and_opt=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None): """Load a model checkpoint and return the iteration. strict (bool): whether to strictly enforce that the keys in :attr:`state_dict` of the checkpoint match the names of @@ -1369,6 +1407,7 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', skip_load_to_model_and_opt (bool): whether to call `load_state_dict` for :attr:`model` and :attr:`optimizer`. In case of running FSDP2 with mcore distributed checkpointing, the tensors are already loaded in-place by `_load_base_checkpoint`. + dp_cp_group: Data parallel + context parallel group (default: None, falls back to mpu API) """ args = get_args() load_dir = getattr(args, load_arg) @@ -1442,7 +1481,10 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', # Determine if RNG state will be loaded if (ckpt_tp_pp == run_tp_pp and not release and not args.finetune and not args.no_load_rng and not getattr(ckpt_args, 'no_save_rng', False)): - gen_sd_rng_state = get_rng_state(args.ckpt_format) # we can load the rng state + if tp_group is None and pp_group is None: + tp_group = mpu.get_tensor_model_parallel_group() + pp_group = mpu.get_pipeline_model_parallel_group() + gen_sd_rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group) # we can load the rng state else: ignore_rng_state = True gen_sd_rng_state = None @@ -1454,6 +1496,7 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', else: sharded_sd_metadata = dist_checkpointing.load_content_metadata(preloaded_state_dict=state_dict) print_rank_0(f'sharded_state_dict metadata loaded from the checkpoint: {sharded_sd_metadata}') + # Determine if optimizer state will be loaded if (not release and not args.finetune and not args.no_load_optim and not getattr(ckpt_args, 'no_save_optim', False)): @@ -1487,6 +1530,15 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', gen_sd_optim = None gen_sd_opt_param_scheduler = None + if dp_cp_group is None: + dp_cp_group = mpu.get_data_parallel_group(with_context_parallel=True) + + # dist_checkpointing.load_content_metadata(...) may return None. + # Ensure we have a dict before updating to avoid NoneType AttributeError. + if sharded_sd_metadata is None: + sharded_sd_metadata = {} + sharded_sd_metadata["dp_cp_group"] = dp_cp_group + optim_sd_kwargs = dict(metadata=sharded_sd_metadata, is_loading=True) model_sd_kwargs = dict(metadata=sharded_sd_metadata) @@ -1528,12 +1580,15 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', elif args.ckpt_format == "torch_dcp": model_sd = model[0].state_dict() optimizer_sd = optimizer.state_dict(is_loading=True) + if tp_group is None and pp_group is None: + tp_group = mpu.get_tensor_model_parallel_group() + pp_group = mpu.get_pipeline_model_parallel_group() sharded_state_dict = { "model": model_sd, "optimizer": optimizer_sd, "args": None, "iteration": 1, - "rng_state": get_rng_state(args.ckpt_format), + "rng_state": get_rng_state(args.ckpt_format, tp_group, pp_group), "checkpoint_version": None, "opt_param_scheduler": opt_param_scheduler.state_dict(), "num_floating_point_operations_so_far": 0, @@ -1556,7 +1611,7 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', data_iterator=None, ckpt_format=ckpt_format, force=True, ) if not args.no_load_rng: - gen_sd_rng_state = get_rng_state(args.ckpt_format) + gen_sd_rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group) if not args.no_load_optim: gen_sd_optim = optimizer gen_sd_opt_param_scheduler = opt_param_scheduler @@ -1653,7 +1708,12 @@ def load_model_state_dict(module, state_dict, strict: bool): if not release and not args.finetune and not args.no_load_optim: try: # Load state dict. - if not skip_load_to_model_and_opt and optimizer is not None and not optimizer.is_stub_optimizer: + if getattr(args, "optimizer", "adam").startswith("dist_") and args.ckpt_format == 'torch': + # LayerWiseDistributedOptimizer load optimizer state from file on different ranks + dp_rank = mpu.get_data_parallel_rank() + optim_checkpoint_name = os.path.join(os.path.dirname(checkpoint_name), f"layer_wise_optimizer_{dp_rank}.pt") + optimizer.load_state_dict_from_file(optim_checkpoint_name) + elif not skip_load_to_model_and_opt and optimizer is not None and not optimizer.is_stub_optimizer: optimizer.load_state_dict(state_dict['optimizer']) # Load distributed optimizer's custom parameter state. diff --git a/megatron/training/global_vars.py b/megatron/training/global_vars.py index 17bda9cbac5..ec402263d29 100644 --- a/megatron/training/global_vars.py +++ b/megatron/training/global_vars.py @@ -9,6 +9,7 @@ from megatron.core import Timers from megatron.core.config import set_experimental_flag from megatron.core.energy_monitor import EnergyMonitor +from megatron.core.jit import disable_jit_fuser from megatron.core.num_microbatches_calculator import init_num_microbatches_calculator, unset_num_microbatches_calculator from megatron.training import dist_signal_handler from megatron.training.tokenizer import build_tokenizer @@ -111,6 +112,9 @@ def set_global_variables(args, build_tokenizer=True): if args.exit_signal_handler: _set_signal_handler() + if args.disable_jit_fuser: + disable_jit_fuser() + def unset_global_variables(): """Unset global vars. diff --git a/megatron/training/tokenizer/tokenizer.py b/megatron/training/tokenizer/tokenizer.py index b1aad6819b4..13b7526ca07 100644 --- a/megatron/training/tokenizer/tokenizer.py +++ b/megatron/training/tokenizer/tokenizer.py @@ -48,7 +48,7 @@ def build_tokenizer(args, **kwargs): tokenizer = _GPTSentencePieceTokenizer(args.tokenizer_model) elif args.tokenizer_type == 'HuggingFaceTokenizer': tokenizer = _HuggingFaceTokenizer( - args.tokenizer_model, trust_remote_code = args.trust_remote_code, **kwargs, + args.tokenizer_model, trust_remote_code=args.trust_remote_code, **kwargs ) elif args.tokenizer_type == 'Llama2Tokenizer': assert args.tokenizer_model is not None @@ -78,11 +78,7 @@ def build_tokenizer(args, **kwargs): kwargs = dict() if args.tokenizer_prompt_format == "nvlm-yi-34b": - kwargs = { - "from_slow": True, - "legacy": False, - "add_bos_token": True, - } + kwargs = {"from_slow": True, "legacy": False, "add_bos_token": True} # Currently, only HuggingFace tokenizers are supported. underlying_tokenizer = transformers.AutoTokenizer.from_pretrained( @@ -97,10 +93,7 @@ def build_tokenizer(args, **kwargs): args.force_system_message, ) elif args.tokenizer_type == "SFTTokenizer": - tokenizer = SFTTokenizer( - args.tokenizer_model, - args.sft_tokenizer_prompt_format, - ) + tokenizer = SFTTokenizer(args.tokenizer_model, args.sft_tokenizer_prompt_format) elif args.tokenizer_type == 'NullMultimodalTokenizer': assert args.vocab_size is not None tokenizer = _NullMultimodalTokenizer(args.vocab_size) @@ -144,7 +137,7 @@ def __init__(self, pretrained_model_name_or_path, trust_remote_code=False, **kwa self._tokenizer = transformers.AutoTokenizer.from_pretrained( pretrained_model_name_or_path=pretrained_model_name_or_path, trust_remote_code=trust_remote_code, - **kwargs + **kwargs, ) self._vocab = self._tokenizer.get_vocab() self._inv_vocab = {token_id: token for token, token_id in self._vocab.items()} @@ -367,6 +360,10 @@ def detokenize(self, token_ids): def eod(self): return self.eod_id + @property + def eos(self): + return self.eod_id + class _SentencePieceTokenizer(MegatronLegacyTokenizer): """SentencePieceTokenizer-Megatron wrapper""" @@ -573,6 +570,10 @@ def mask(self): def eod(self): return self._eos_id + @property + def eos(self): + return self._eos_id + @property def additional_special_tokens_ids(self): return None @@ -623,6 +624,10 @@ def mask(self): def eod(self): return self.eos_id + @property + def eos(self): + return self.eos_id + @property def additional_special_tokens_ids(self): return None @@ -747,7 +752,7 @@ def bos(self) -> int: @property def eos(self) -> int: return self._eos_id - + @property def pad(self) -> int: return self._pad_id @@ -858,19 +863,30 @@ def mask(self): def eod(self): return self._eod_id + @property + def eos(self): + return self._eod_id + @property def additional_special_tokens_ids(self): return None + class _NullMultimodalTokenizer(MegatronLegacyTokenizer): def __init__(self, vocab_size, image_token=None, image_token_id=None): super().__init__(None, vocab_size=vocab_size) self._vocab_size_without_eod = int(vocab_size) self._eod_id = self._vocab_size_without_eod - from megatron.core.models.multimodal.llava_model import DEFAULT_IMAGE_TOKEN_INDEX, IMAGE_TOKEN + from megatron.core.models.multimodal.llava_model import ( + DEFAULT_IMAGE_TOKEN_INDEX, + IMAGE_TOKEN, + ) + self._image_token = image_token if image_token is not None else IMAGE_TOKEN - self._image_token_id = image_token_id if image_token_id is not None else DEFAULT_IMAGE_TOKEN_INDEX + self._image_token_id = ( + image_token_id if image_token_id is not None else DEFAULT_IMAGE_TOKEN_INDEX + ) def tokenize(self, text): return [int(x) for x in text.split(' ')] @@ -887,7 +903,9 @@ def offsets(self, ids: list[int], text: str) -> list[int]: return offsets def convert_tokens_to_ids(self, tokens): - ids = [(int(t) if t != self._image_token else self._image_token_id) for t in tokens.split(' ')] + ids = [ + (int(t) if t != self._image_token else self._image_token_id) for t in tokens.split(' ') + ] return ids if len(ids) > 1 else ids[0] @property @@ -918,6 +936,10 @@ def mask(self): def eod(self): return self._eod_id + @property + def eos(self): + return self._eod_id + @property def additional_special_tokens_ids(self): return None diff --git a/megatron/training/training.py b/megatron/training/training.py index d8b57326f67..32599aa0889 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """Pretrain utilities.""" @@ -64,6 +64,7 @@ from megatron.core.distributed import DistributedDataParallel as DDP from megatron.core.distributed.fsdp.mcore_fsdp_adapter import FullyShardedDataParallel as megatron_FSDP from megatron.core.optimizer.optimizer import param_group_identifier_keys +from megatron.core.optimizer.qk_clip import clip_qk try: from megatron.core.distributed import TorchFullyShardedDataParallel as torch_FSDP @@ -75,6 +76,7 @@ from megatron.core.distributed import finalize_model_grads from megatron.core.enums import ModelType from megatron.core.optimizer import get_megatron_optimizer, OptimizerConfig +from megatron.core.optimizer.muon import get_megatron_muon_optimizer from megatron.core.rerun_state_machine import ( get_rerun_state_machine, destroy_rerun_state_machine, @@ -236,9 +238,6 @@ def hybrid_flops(batch_size, seq_len, hidden_size, def transformer_flops(): """Calculate FLOPs for a standard Transformer model.""" # TODO(helenn/dnarayanan): Refactor this to reuse the helper methods. - # Attention projection size. - query_projection_size = args.kv_channels * args.num_attention_heads - query_projection_to_hidden_size_ratio = query_projection_size / args.hidden_size # Group Query Attention. if not args.group_query_attention: args.num_query_groups = args.num_attention_heads @@ -329,10 +328,9 @@ def transformer_flops(): + args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim) + 1 ) - self_attn_term = ( + standard_self_attn_term = ( 3 * 2 # fwd(1) + bwd(2) *FMA - * num_layers * ( ## q lora + rope + q norm q_term @@ -349,29 +347,98 @@ def transformer_flops(): ## core attn + args.seq_length * (args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim)) - / 2 + / 2 # causal mask (only half of the mask is non-zero) + args.seq_length * args.num_attention_heads * args.v_head_dim / 2 ) ) else: ## MHA or GQA - self_attn_term = ( - expansion_factor - * num_layers - * args.hidden_size - * args.hidden_size + query_projection_size = args.kv_channels * args.num_attention_heads + key_projection_size = args.kv_channels * args.num_query_groups + value_projection_size = args.kv_channels * args.num_query_groups + standard_self_attn_term = ( + 3 + * 2 # fwd(1) + bwd(2) *FMA * ( - ( - 1 - + (args.num_query_groups / args.num_attention_heads) - # # Only half of the attention matrix is non-zero and needs to be multiplied with V. - + (args.seq_length / args.hidden_size / 2) - ) - * query_projection_to_hidden_size_ratio + ## qkv proj + args.hidden_size + * (query_projection_size + key_projection_size + value_projection_size) + ## core attention + + query_projection_size + * args.seq_length + / 2 # causal mask (only half of the mask is non-zero) + * 2 # QK^T and (QK^T)V + ## out proj + + query_projection_size + * args.hidden_size ) ) + if args.linear_attention_type is not None: + # Calculate number of dense and MoE Transformer MLPs. + if isinstance(args.linear_attention_freq, int): + linear_attention_pattern = [ + # [1,1,...,1,0,1,1,...,1,0,...] + 0 if ((i + 1) % args.linear_attention_freq == 0) + else 1 for i in range(num_layers) + ] + elif isinstance(args.linear_attention_freq, list): + linear_attention_pattern = args.linear_attention_freq + assert len(linear_attention_pattern) == num_layers, ( + f"Invalid length of linear_attention_pattern: {len(linear_attention_pattern)}, " + f"expected {num_layers}, " + f"current linear attention pattern: {args.linear_attention_freq}" + ) + elif args.linear_attention_freq is None: + linear_attention_pattern = [1] * num_layers + else: + raise ValueError( + f"Invalid linear_attention_freq: {type(args.linear_attention_freq)}," + f" {args.linear_attention_freq}" + ) + num_linear_attention_layers = sum(linear_attention_pattern) + num_standard_attention_layers = num_layers - num_linear_attention_layers + + if args.linear_attention_type == "gated_delta_net": + # Calculate the FLOPs for the gated delta net attention. + qk_head_dim = args.linear_key_head_dim + v_head_dim = args.linear_value_head_dim + num_qk_heads = args.linear_num_key_heads + num_v_heads = args.linear_num_value_heads + qk_dim = qk_head_dim * num_qk_heads + v_dim = v_head_dim * num_v_heads + linear_self_attn_term = ( + 3 + * 2 # fwd(1) + bwd(2) *FMA + * ( + ## in proj + args.hidden_size + * (2 * qk_dim + 2 * v_dim + 2 * num_v_heads) + ## conv1d + + args.linear_conv_kernel_dim + * (2 * qk_dim + v_dim) + ## gated delta rule + + num_v_heads + * (v_head_dim ** 2) + * 4 # KK^T, VK^T, S(a(I-bKK^T)), and SQ + ## out proj + + args.hidden_size + * v_dim + ) + ) + else: + raise ValueError(f"Invalid linear_attention_type: {args.linear_attention_type}") + else: + num_linear_attention_layers = 0 + linear_self_attn_term = 0 + num_standard_attention_layers = num_layers + + self_attn_term = ( + linear_self_attn_term * num_linear_attention_layers + + standard_self_attn_term * num_standard_attention_layers + ) + total_floating_point_operations = ( batch_size * args.seq_length @@ -527,6 +594,30 @@ def reorder_inner_param_groups(optimizer_state_dict): return preprocessed_common_state_dict +def get_no_wd_decay_cond(no_wd_decay_cond_type, default_skip_embedding_weight_decay): + """Get the no weight decay condition function.""" + + # Default case: no_wd_decay_cond_type is None + no_wd_decay_cond_fn = None + + if no_wd_decay_cond_type == 'qwen3_next': + # Qwen3-Next applies weight decay to qk layernorm as a special case + def qwen3_next_no_wd_decay_cond(name, param): + if "q_layernorm" in name or "k_layernorm" in name: + no_wd = False + else: + no_wd = ( + name.endswith(".bias") + or len(param.shape) == 1 + or (default_skip_embedding_weight_decay and "embedding" in name) + ) + return no_wd + no_wd_decay_cond_fn = qwen3_next_no_wd_decay_cond + elif no_wd_decay_cond_type is not None: + raise ValueError(f"Invalid no_wd_decay_cond_type: {no_wd_decay_cond_type}") + + return no_wd_decay_cond_fn + def pretrain( train_valid_test_dataset_provider, model_provider, @@ -663,8 +754,15 @@ def pretrain( # Model, optimizer, and learning rate. timers('model-and-optimizer-setup', log_level=0).start(barrier=True) + no_wd_decay_cond = get_no_wd_decay_cond( + args.no_weight_decay_cond_type, + default_skip_embedding_weight_decay=args.embedding_init_method_std is not None, + ) model, optimizer, opt_param_scheduler = setup_model_and_optimizer( - model_provider, model_type, checkpointing_context=checkpointing_context + model_provider, + model_type, + checkpointing_context=checkpointing_context, + no_wd_decay_cond=no_wd_decay_cond, ) timers('model-and-optimizer-setup').stop() @@ -1103,18 +1201,31 @@ def setup_model_and_optimizer( kwargs[f.name] = getattr(args, f.name) config = OptimizerConfig(**kwargs) config.timers = timers - optimizer = get_megatron_optimizer( - config, - model, - no_wd_decay_cond, - scale_lr_cond, - lr_mult, - use_gloo_process_groups=args.enable_gloo_process_groups, - # If the user is asking for a non-zero embedding init std, skip weight decay for embeddings - # to avoid embeddings from shrinking to zero as recommended in https://arxiv.org/abs/2312.16903 - default_skip_embedding_weight_decay=args.embedding_init_method_std is not None, - dump_param_to_param_group_map=args.dump_param_to_param_group_map, - ) + + if 'muon' not in config.optimizer: + optimizer = get_megatron_optimizer( + config, + model, + no_wd_decay_cond, + scale_lr_cond, + lr_mult, + use_gloo_process_groups=args.enable_gloo_process_groups, + # If the user is asking for a non-zero embedding init std, skip weight decay for embeddings + # to avoid embeddings from shrinking to zero as recommended in https://arxiv.org/abs/2312.16903 + default_skip_embedding_weight_decay=args.embedding_init_method_std is not None, + dump_param_to_param_group_map=args.dump_param_to_param_group_map, + ) + else: + optimizer = get_megatron_muon_optimizer( + config, + model, + no_wd_decay_cond, + scale_lr_cond, + lr_mult, + use_gloo_process_groups=args.enable_gloo_process_groups, + layer_wise_distributed_optimizer='dist' in config.optimizer, + ) + opt_param_scheduler = get_optimizer_param_scheduler(optimizer) one_logger and one_logger.log_metrics({"app_build_optimzer_finish_time": one_logger_utils.get_timestamp_in_ms()}) @@ -1134,7 +1245,7 @@ def setup_model_and_optimizer( # set dense model related args in to global args before getting dense model args.num_experts = None args.expert_model_parallel_size = 1 - args.ffn_hidden_size = moe_ffn_hidden_size * args.moe_upcycling_granularity + args.ffn_hidden_size = moe_ffn_hidden_size * args.moe_upcycling_granularity # get dense model dense_model_for_upcycling = get_model(model_provider_func, model_type) @@ -1281,7 +1392,7 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch ) should_checkpoint, should_exit, exit_code = rerun_state_machine.should_checkpoint_and_exit() if should_exit: - return {}, True, should_checkpoint, should_exit, exit_code, None, None + return {}, True, should_checkpoint, should_exit, exit_code, None, None, 0 # Empty unused memory. if args.empty_unused_memory_level >= 1: @@ -1296,6 +1407,13 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch timers('optimizer', log_level=1).start(barrier=args.barrier_with_L1_time) update_successful, grad_norm, num_zeros_in_grad = optimizer.step() + + # get max attention logit for logging and run clip_qk() + # Part of MuonClip Optimizer step + log_max_attention_logit = 0 + if args.qk_clip or args.log_max_attention_logit: + log_max_attention_logit = clip_qk(model, log_max_only=not args.qk_clip) + timers('optimizer').stop() # when freezing sub-models we may have a mixture of successful and unsucessful ranks, @@ -1367,8 +1485,9 @@ def train_step(forward_step_func, data_iterator, model, optimizer, opt_param_sch exit_code, grad_norm, num_zeros_in_grad, + log_max_attention_logit, ) - return {}, skipped_iter, should_checkpoint, should_exit, exit_code, grad_norm, num_zeros_in_grad + return {}, skipped_iter, should_checkpoint, should_exit, exit_code, grad_norm, num_zeros_in_grad, log_max_attention_logit def training_log( @@ -1383,6 +1502,7 @@ def training_log( grad_norm, params_norm, num_zeros_in_grad, + max_attention_logit, ): """Log training information such as losses, timing, ....""" args = get_args() @@ -1419,32 +1539,37 @@ def training_log( total_loss_dict[nan_iters_key] = total_loss_dict.get(nan_iters_key, 0) + int(got_nan) # Logging. - timers_to_log = [ - 'forward-backward', - 'forward-compute', - 'backward-compute', - 'batch-generator', - 'forward-recv', - 'forward-send', - 'backward-recv', - 'backward-send', - 'forward-send-forward-recv', - 'forward-send-backward-recv', - 'backward-send-forward-recv', - 'backward-send-backward-recv', - 'forward-backward-send-forward-backward-recv', - 'layernorm-grads-all-reduce', - 'embedding-grads-all-reduce', - 'all-grads-sync', - 'params-all-gather', - 'optimizer-copy-to-main-grad', - 'optimizer-unscale-and-check-inf', - 'optimizer-clip-main-grad', - 'optimizer-count-zeros', - 'optimizer-inner-step', - 'optimizer-copy-main-to-model-params', - 'optimizer', - ] + timers_to_log = [] + if args.timing_log_level >= 1: + timers_to_log.extend([ + 'forward-backward', + 'layernorm-grads-all-reduce', + 'embedding-grads-all-reduce', + 'all-grads-sync', + 'params-all-gather', + 'optimizer-copy-to-main-grad', + 'optimizer-unscale-and-check-inf', + 'optimizer-clip-main-grad', + 'optimizer-count-zeros', + 'optimizer-inner-step', + 'optimizer-copy-main-to-model-params', + 'optimizer', + ]) + if args.timing_log_level >= 2: + timers_to_log.extend([ + 'batch-generator', + 'forward-compute', + 'backward-compute', + 'forward-recv', + 'forward-send', + 'backward-recv', + 'backward-send', + 'forward-send-forward-recv', + 'forward-send-backward-recv', + 'backward-send-forward-recv', + 'backward-send-backward-recv', + 'forward-backward-send-forward-backward-recv', + ]) # Add timers from RL loop if needed. if getattr(args, 'perform_rl_step', False): timers_to_log.extend(['rollout-collection', 'inference-setup', 'collect-rollouts', 'postrollout-gc-collect', @@ -1540,6 +1665,10 @@ def training_log( "mem-max-allocated-bytes", mem_stats["allocated_bytes.all.peak"], iteration ) writer.add_scalar("mem-allocated-count", mem_stats["allocation.all.current"], iteration) + if args.log_max_attention_logit: + writer.add_scalar('max_attention_logit', max_attention_logit, iteration) + if wandb_writer: + wandb_writer.log({'max_attention_logit': max_attention_logit}, iteration) if args.num_experts is not None: moe_loss_scale = 1 / get_num_microbatches() track_names = [] @@ -1731,7 +1860,8 @@ def save_checkpoint_and_time( # Stop timer to get accurate train interval time and exclude checkpointing duration timers('interval-time').stop() - energy_monitor.pause() + if args.log_energy: + energy_monitor.pause() # Extra barrier is added to make sure all ranks report the max time. timer_key = 'save-checkpoint-non-persistent' if non_persistent_ckpt else 'save-checkpoint' @@ -1773,7 +1903,9 @@ def save_checkpoint_and_time( ) # Recover timing - energy_monitor.resume() + if args.log_energy: + energy_monitor.resume() + timers('interval-time', log_level=0).start(barrier=True) @@ -2127,7 +2259,7 @@ def train( eval_iterations = 0 # Wrap forward_backward_func for Full iteration CUDA graph forward_backward_func = get_forward_backward_func() - if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration": + if args.cuda_graph_impl == "local" and "full_iteration" in args.cuda_graph_scope: forward_backward_func = FullCudaGraphWrapper(forward_backward_func, cuda_graph_warmup_steps=args.cuda_graph_warmup_steps) def get_e2e_base_metrics(): @@ -2260,12 +2392,13 @@ def get_e2e_base_metrics(): # Capture CUDA Graphs. if ( args.cuda_graph_impl == "transformer_engine" - and iteration == args.cuda_graph_warmup_steps + and not cuda_graph_helper.graphs_created() + and iteration - start_iteration == args.cuda_graph_warmup_steps ): - if iteration > start_iteration and should_disable_forward_pre_hook(args): + if args.cuda_graph_warmup_steps > 0 and should_disable_forward_pre_hook(args): disable_forward_pre_hook(model, param_sync=False) cuda_graph_helper.create_cudagraphs() - if iteration > start_iteration and should_disable_forward_pre_hook(args): + if args.cuda_graph_warmup_steps > 0 and should_disable_forward_pre_hook(args): enable_forward_pre_hook(model) cuda_graph_helper.cuda_graph_set_manual_hooks() @@ -2303,6 +2436,7 @@ def get_e2e_base_metrics(): exit_code, grad_norm, num_zeros_in_grad, + max_attention_logit, ) = train_step( forward_step_func, train_data_iterator, model, optimizer, opt_param_scheduler, config, forward_backward_func ) @@ -2340,8 +2474,11 @@ def get_e2e_base_metrics(): # Set the manual hooks here since it's not set right after the capturing. if ( args.cuda_graph_impl == "transformer_engine" - and iteration == args.cuda_graph_warmup_steps + and args.cuda_graph_warmup_steps == 0 ): + assert ( + cuda_graph_helper.graphs_created() + ), "CUDA Graphs should have been created." cuda_graph_helper.cuda_graph_set_manual_hooks() iteration += 1 @@ -2404,6 +2541,7 @@ def get_e2e_base_metrics(): grad_norm, params_norm, num_zeros_in_grad, + max_attention_logit, ) # Evaluation. @@ -2542,7 +2680,7 @@ def evaluate( eval_batch_size = args.global_batch_size eval_num_microbatches = eval_batch_size // (args.micro_batch_size * args.data_parallel_size) forward_backward_func = get_forward_backward_func() - if args.cuda_graph_impl == "local" and args.cuda_graph_scope=="full_iteration": + if args.cuda_graph_impl == "local" and "full_iteration" in args.cuda_graph_scope: forward_backward_func = FullCudaGraphWrapper(forward_backward_func, cuda_graph_warmup_steps=args.cuda_graph_warmup_steps) if eval_iters is None: @@ -2690,7 +2828,7 @@ def evaluate_and_print_results( eval_iters = [args.eval_iters] else: eval_iters = args.eval_iters - + if args.full_validation: assert len(eval_iters) == len(data_iterators) @@ -2706,7 +2844,7 @@ def evaluate_and_print_results( eval_iters = [args.eval_iters] else: eval_iters = args.eval_iters - + for index, (iterator, iterations) in enumerate(zip(data_iterators, eval_iters)): suffix = "" if args.multiple_validation_sets: @@ -2781,18 +2919,20 @@ def get_train_valid_test_num_samples(): return (train_samples, eval_samples, test_iters * args.global_batch_size) -def build_train_valid_test_datasets(build_train_valid_test_datasets_provider, train_valid_test_num_samples=None): +def build_train_valid_test_datasets(build_train_valid_test_datasets_provider, train_valid_test_num_samples=None, vp_stage=None): """Build pretraining datasets.""" if train_valid_test_num_samples is None: train_valid_test_num_samples = get_train_valid_test_num_samples() - print_rank_0(' > datasets target sizes (minimum size):') print_rank_0(' train: {}'.format(train_valid_test_num_samples[0])) print_rank_0(' validation: {}'.format(train_valid_test_num_samples[1])) print_rank_0(' test: {}'.format(train_valid_test_num_samples[2])) - return build_train_valid_test_datasets_provider(train_valid_test_num_samples) + if vp_stage is not None: + return build_train_valid_test_datasets_provider(train_valid_test_num_samples, vp_stage=vp_stage) + else: + return build_train_valid_test_datasets_provider(train_valid_test_num_samples) -def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider): +def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider, vp_stage=None): """Build pretraining data loaders.""" args = get_args() @@ -2821,10 +2961,11 @@ def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider # Build datasets. train_ds, valid_ds, test_ds = build_train_valid_test_datasets( - build_train_valid_test_datasets_provider, (1, 1, 1) if getattr(args, 'perform_rl_step', False) else None + build_train_valid_test_datasets_provider, (1, 1, 1) if getattr(args, 'perform_rl_step', False) else None, + vp_stage=vp_stage, ) valid_ds = [valid_ds] if not isinstance(valid_ds, list) else valid_ds - + # Build dataloders. train_dataloader = build_pretraining_data_loader(train_ds, args.consumed_train_samples) @@ -2863,14 +3004,15 @@ def build_train_valid_test_data_loaders(build_train_valid_test_datasets_provider return train_dataloader, valid_dataloaders, test_dataloader -def build_train_valid_test_data_iterators(build_train_valid_test_datasets_provider): +def build_train_valid_test_data_iterators(build_train_valid_test_datasets_provider, vp_stage=None): """Build pretraining data iterators.""" args = get_args() # Build loaders. train_dataloader, valid_dataloaders, test_dataloader = build_train_valid_test_data_loaders( - build_train_valid_test_datasets_provider + build_train_valid_test_datasets_provider, + vp_stage=vp_stage ) # Build iterators. @@ -2899,7 +3041,7 @@ def _get_iterator(dataloader_type, dataloader): if valid_dataloaders is not None: # when using full validation, we need to override eval iters with the correct - # number of iterations on tp rank 0 so that it can be distributed to the other + # number of iterations on tp rank 0 so that it can be distributed to the other # ranks later if args.full_validation: if args.multiple_validation_sets: diff --git a/megatron/training/utils.py b/megatron/training/utils.py index cef71160791..3be7b6c8914 100644 --- a/megatron/training/utils.py +++ b/megatron/training/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """General utilities.""" import json @@ -38,6 +38,7 @@ from megatron.core.utils import ( get_batch_on_this_cp_rank, get_data_parallel_group_if_dtensor, + is_torch_min_version, to_local_if_dtensor, unwrap_model, ) @@ -271,6 +272,9 @@ def report_memory(name): string += ' | max allocated: {}'.format(torch.cuda.max_memory_allocated() / mega_bytes) string += ' | reserved: {}'.format(torch.cuda.memory_reserved() / mega_bytes) string += ' | max reserved: {}'.format(torch.cuda.max_memory_reserved() / mega_bytes) + if is_torch_min_version("2.6.0"): + # device usage is not supported in torch < 2.6.0 + string += ' | device usage: {}'.format(torch.cuda.device_memory_used() / mega_bytes) if mpu.get_data_parallel_rank() == 0: print("[Rank {}] {}".format(torch.distributed.get_rank(), string), flush=True) @@ -500,7 +504,7 @@ def get_blend_and_blend_per_split(args): return blend, blend_per_split -def get_batch_on_this_tp_rank(data_iterator): +def get_batch_on_this_tp_rank(data_iterator, mtp_on_this_rank: bool = False): args = get_args() @@ -528,7 +532,7 @@ def _broadcast(item): 'position_ids': data["position_ids"].cuda(non_blocking=True), } - if args.pipeline_model_parallel_size == 1: + if args.pipeline_model_parallel_size == 1 or mtp_on_this_rank: _broadcast(batch['tokens']) _broadcast(batch['labels']) _broadcast(batch['loss_mask']) @@ -544,9 +548,6 @@ def _broadcast(item): # Multi-Token Prediction (MTP) layers need tokens and position_ids to calculate embedding. # Currently the Multi-Token Prediction (MTP) layers is fixed on the last stage, so we need # to broadcast tokens and position_ids to all of the tensor parallel ranks on the last stage. - if args.mtp_num_layers is not None: - _broadcast(batch['tokens']) - _broadcast(batch['position_ids']) _broadcast(batch['labels']) _broadcast(batch['loss_mask']) _broadcast(batch['attention_mask']) @@ -582,7 +583,7 @@ def _broadcast(item): device=torch.cuda.current_device(), ) - if args.pipeline_model_parallel_size == 1: + if args.pipeline_model_parallel_size == 1 or mtp_on_this_rank: _broadcast(tokens) _broadcast(labels) _broadcast(loss_mask) @@ -601,12 +602,8 @@ def _broadcast(item): # Multi-Token Prediction (MTP) layers need tokens and position_ids to calculate embedding. # Currently the Multi-Token Prediction (MTP) layers is fixed on the last stage, so we need # to broadcast tokens and position_ids to all of the tensor parallel ranks on the last stage. - if args.mtp_num_layers is not None: - _broadcast(tokens) - _broadcast(position_ids) - else: - tokens = None - position_ids = None + tokens = None + position_ids = None _broadcast(labels) _broadcast(loss_mask) diff --git a/pretrain_gpt.py b/pretrain_gpt.py index 69f26f3271a..ecb7163ff70 100644 --- a/pretrain_gpt.py +++ b/pretrain_gpt.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. """Pretrain and SFT GPT.""" @@ -15,7 +15,9 @@ from megatron.core.models.gpt import GPTModel from megatron.core.rerun_state_machine import get_rerun_state_machine from megatron.core.tokenizers.text.utils.build_tokenizer import build_tokenizer +from megatron.core.transformer.multi_token_prediction import mtp_on_this_rank, get_mtp_ranks from megatron.core.utils import StragglerDetector, get_attr_wrapped_model +from megatron.training.arguments import core_transformer_config_from_args from megatron.training import get_args, get_timers, get_tokenizer, inprocess_restart, pretrain, print_rank_0 from megatron.training.datasets.sft_dataset import SFTDataset from megatron.training.utils import ( @@ -37,14 +39,20 @@ stimer = StragglerDetector() -def get_batch(data_iterator, vp_stage=None): +def get_batch(data_iterator, vp_stage: Optional[int] = None): """Generate a batch.""" + args = get_args() + config = core_transformer_config_from_args(args) # TODO: this is pretty hacky, find a better way - if not is_first_or_last_pipeline_stage(vp_stage): + if not is_first_or_last_pipeline_stage(vp_stage) and ( + (not mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage))): return None, None, None, None, None # get batches based on the TP rank you are on - batch = get_batch_on_this_tp_rank(data_iterator) + batch = get_batch_on_this_tp_rank( + data_iterator, + mtp_on_this_rank=mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage) + ) # slice batch along sequence dimension for context parallelism batch = get_batch_on_this_cp_rank(batch) @@ -158,7 +166,12 @@ def forward_step(data_iterator, model: GPTModel, return_schedule_plan: bool = Fa def is_dataset_built_on_rank(vp_stage=None): - return is_first_or_last_pipeline_stage(vp_stage) and parallel_state.get_tensor_model_parallel_rank() == 0 + args = get_args() + config = core_transformer_config_from_args(args) + return ( + is_first_or_last_pipeline_stage(vp_stage) + or mtp_on_this_rank(config, ignore_virtual=False, vp_stage=vp_stage) + ) and parallel_state.get_tensor_model_parallel_rank() == 0 def core_gpt_dataset_config_from_args(args): @@ -214,6 +227,7 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None print_rank_0("> building train, validation, and test datasets for GPT ...") + is_dataset_built = partial(is_dataset_built_on_rank, vp_stage=vp_stage) train_ds, valid_ds, test_ds = BlendedMegatronDatasetBuilder( dataset_type, train_val_test_num_samples, partial(is_dataset_built_on_rank, vp_stage=vp_stage), config ).build() @@ -223,6 +237,21 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None return train_ds, valid_ds, test_ds +def get_embedding_ranks(pp_ranks: List[int]): + """Get the embedding ranks.""" + embedding_ranks = [pp_ranks[0]] + if len(pp_ranks) > 1: + args = get_args() + if not args.untie_embeddings_and_output_weights: + embedding_ranks.append(pp_ranks[-1]) + config = core_transformer_config_from_args(args) + mtp_ranks = get_mtp_ranks(pp_ranks, config) + embedding_ranks.extend(mtp_ranks) + embedding_ranks = list(set(embedding_ranks)) + embedding_ranks = sorted(embedding_ranks) + return embedding_ranks + + if __name__ == "__main__": # Temporary for transition to core datasets @@ -239,4 +268,5 @@ def train_valid_test_datasets_provider(train_val_test_num_samples, vp_stage=None args_defaults={'tokenizer_type': 'GPT2BPETokenizer'}, extra_args_provider=add_modelopt_args if has_nvidia_modelopt else None, store=store, + get_embedding_ranks=get_embedding_ranks, ) diff --git a/pyproject.toml b/pyproject.toml index 9dfa88e541e..3e4267889e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,9 +67,9 @@ Homepage = "https://github.com/NVIDIA/Megatron-LM/megatron/core" mlm = ["flask-restful", "sentencepiece", "tiktoken", "wandb", "transformers"] dev = [ - "nvidia-modelopt[torch]; sys_platform != 'darwin'", + "nvidia-modelopt[torch]>=0.33.0a0,<0.34.0; sys_platform != 'darwin'", "transformer-engine[pytorch]>=2.9.0a0,<2.10.0", - "nvidia-resiliency-ext", + "nvidia-resiliency-ext>=0.4.0a0,<0.5.0", "tqdm", "einops~=0.8", "tensorstore~=0.1,!=0.1.46,!=0.1.72", @@ -85,24 +85,19 @@ dev = [ "flashinfer-python", "wget", "onnxscript", + "flash-linear-attention~=0.3.2", + "emerging_optimizers", ] lts = [ "tqdm", - "einops~=0.8", - "tensorstore~=0.1,!=0.1.46,!=0.1.72", - "nvtx~=0.2", - "multi-storage-client~=0.27", - "opentelemetry-api~=1.33.1", + "einops", + "tensorstore!=0.1.46,!=0.1.72", + "nvtx", + "transformers", + "zarr", "setuptools<80.0.0", - "mamba-ssm~=2.2", - "causal-conv1d~=1.5", - "nv-grouped-gemm~=1.1", - "megatron-energon[av_decode]~=6.0", - "av<16.0.0", # At the time, av 16.0.0 is not compatible with Python 3.12 - "flashinfer-python", "wget", - "onnxscript", ] [dependency-groups] @@ -175,7 +170,7 @@ flash_mla = [ ] transformer-engine = { git = "https://github.com/NVIDIA/TransformerEngine.git", rev = "release_v2.9" } # on `release_v2.9` nemo-run = { git = "https://github.com/NVIDIA-NeMo/Run.git", rev = "01a9a8ba360f7b2908728ad0516e0ad9d936966d" } -emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "fb1add873e7851ec34b48581ea1b15761b73d189" } +emerging_optimizers = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git", rev = "cf9909b777ffac18e05b67a6708282cadc000942" } [tool.isort] profile = "black" # black-compatible diff --git a/tests/functional_tests/python_test_utils/test_pretraining_resume_checkpoint_pipeline.py b/tests/functional_tests/python_test_utils/test_pretraining_resume_checkpoint_pipeline.py index 6aeb412a8f5..d5da2dea30c 100644 --- a/tests/functional_tests/python_test_utils/test_pretraining_resume_checkpoint_pipeline.py +++ b/tests/functional_tests/python_test_utils/test_pretraining_resume_checkpoint_pipeline.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import logging from typing import Dict @@ -20,9 +22,7 @@ def test_resume_checkpoint_pipeline( model_config = yaml.safe_load(f) checks_types = ( - model_config["METRICS"] - if "METRICS" in model_config - else ["lm loss", "num-zeros"] + model_config["METRICS"] if "METRICS" in model_config else ["lm loss", "num-zeros"] ) checks = { metric: test_pretraining_regular_pipeline.CHECK_THRESHOLDS[metric] diff --git a/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml b/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml index ab5558fa7d2..278ad6c17a8 100644 --- a/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml +++ b/tests/functional_tests/test_cases/bert/bert_release/model_config.yaml @@ -27,7 +27,7 @@ MODEL_ARGS: --pipeline-model-parallel-size: 8 # Data args --data-path: ${DATA_BLEND} - --vocab-file: ${DATA_PATH}/text/the_pile/bert_shard00/vocab.txt + --vocab-file: ${DATA_PATH}/vocab.txt --split: 949,50,1 --data-cache-path: ${DATA_CACHE_PATH} # EVAL_AND_LOGGING_ARGS diff --git a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml index 8b437ba75e7..32386558710 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt3_15b_8t_release_sm/model_config.yaml @@ -23,7 +23,7 @@ MODEL_ARGS: --micro-batch-size: 4 --rampup-batch-size: "[384 384 97656250]" --global-batch-size: 1152 - --train-samples: 4882812 + --train-samples: 19531250 --manual-gc: true # Transformer Engine args --transformer-impl: transformer_engine @@ -68,9 +68,10 @@ MODEL_ARGS: --eval-iters: 32 --eval-interval: 2000 # Add checkpointing args - --load: ${CHECKPOINT_LOAD_PATH} --save: ${CHECKPOINT_SAVE_PATH} - --save-interval: 1000 + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 5000 + --save-retain-interval: 10000 # Add initialization args --init-method-std: 0.0134 # Add logging args @@ -86,7 +87,7 @@ MODEL_ARGS: --wandb-exp-name: ${WANDB_EXPERIMENT} # Add mixed precision args --bf16: true - --exit-interval: 13000 + --exit-interval: 10200 --wandb-save-dir: ${WANDB_SAVE_PATH} METRICS: - "iteration-time" diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json index 1ba701443ce..7fa302274bf 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_resume_torch_dist_tunable_overlap/golden_values_lts_dgxa100_dracooci.json @@ -534,4 +534,4 @@ "100": 0.16898 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json index d8ec5426bd1..363e94d8f52 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json +++ b/tests/functional_tests/test_cases/gpt/gpt3_mcore_te_tp1_pp4_vp1_tunable_overlap/golden_values_lts_dgxa100_dracooci.json @@ -284,4 +284,4 @@ "50": 0.16165 } } -} \ No newline at end of file +} diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml index f6892ae5c24..048256c3504 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_b200_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml @@ -41,7 +41,7 @@ MODEL_ARGS: --pipeline-model-parallel-size: 2 --sequence-parallel: true --tp-comm-overlap: true - --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_b200_1N8G_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml + --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml --deterministic-mode: true --no-gradient-accumulation-fusion: true --fp8-format: hybrid diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..f56b5fa6f77 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/golden_values_dev_dgx_h100.json @@ -0,0 +1,10037 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 10.85954, + "2": 10.88017, + "3": 10.87732, + "4": 10.8999, + "5": 10.88699, + "6": 10.87335, + "7": 10.88219, + "8": 10.87225, + "9": 10.87277, + "10": 10.87494, + "11": 10.85221, + "12": 10.84405, + "13": 10.84222, + "14": 10.86461, + "15": 10.78656, + "16": 10.81059, + "17": 10.77436, + "18": 10.81246, + "19": 10.72203, + "20": 10.69596, + "21": 10.64272, + "22": 10.64956, + "23": 10.65288, + "24": 10.54233, + "25": 10.55491, + "26": 10.63818, + "27": 10.44117, + "28": 10.46928, + "29": 10.34986, + "30": 10.24645, + "31": 10.42625, + "32": 10.33791, + "33": 10.19559, + "34": 10.14074, + "35": 10.22182, + "36": 10.13202, + "37": 10.07533, + "38": 10.01538, + "39": 10.02986, + "40": 10.05768, + "41": 9.93219, + "42": 9.93962, + "43": 9.8498, + "44": 9.97902, + "45": 9.99946, + "46": 9.83276, + "47": 9.99696, + "48": 9.80958, + "49": 9.94884, + "50": 9.94537, + "51": 9.58197, + "52": 9.79331, + "53": 9.62548, + "54": 9.88686, + "55": 9.73482, + "56": 9.84492, + "57": 9.85708, + "58": 9.87627, + "59": 9.54205, + "60": 9.64489, + "61": 9.88334, + "62": 9.75928, + "63": 9.68107, + "64": 9.82461, + "65": 9.59476, + "66": 9.62868, + "67": 9.74002, + "68": 9.60205, + "69": 9.29216, + "70": 9.42139, + "71": 9.78753, + "72": 9.7124, + "73": 9.61815, + "74": 9.44773, + "75": 9.23898, + "76": 9.50824, + "77": 9.5795, + "78": 9.56058, + "79": 9.30801, + "80": 9.35768, + "81": 9.45813, + "82": 9.55358, + "83": 9.53407, + "84": 9.35442, + "85": 9.3992, + "86": 9.65282, + "87": 9.23449, + "88": 9.48753, + "89": 9.22214, + "90": 9.41067, + "91": 9.38753, + "92": 9.37682, + "93": 9.36024, + "94": 9.51507, + "95": 9.42125, + "96": 9.33616, + "97": 9.20399, + "98": 9.4954, + "99": 9.29284, + "100": 9.35905, + "101": 9.24757, + "102": 9.24676, + "103": 9.07735, + "104": 9.16669, + "105": 9.37858, + "106": 9.1496, + "107": 9.1756, + "108": 9.316, + "109": 9.29109, + "110": 9.36426, + "111": 9.17995, + "112": 9.23471, + "113": 9.35297, + "114": 9.35265, + "115": 9.32672, + "116": 9.00223, + "117": 9.06476, + "118": 9.06643, + "119": 9.22418, + "120": 9.08485, + "121": 9.19671, + "122": 9.14164, + "123": 9.25933, + "124": 9.45506, + "125": 9.21512, + "126": 9.06416, + "127": 9.01814, + "128": 9.22131, + "129": 8.98184, + "130": 9.13972, + "131": 9.15856, + "132": 9.03559, + "133": 8.85977, + "134": 9.18539, + "135": 8.88999, + "136": 9.16801, + "137": 9.15771, + "138": 9.23511, + "139": 9.09197, + "140": 8.87218, + "141": 9.29906, + "142": 9.19961, + "143": 9.1169, + "144": 9.24305, + "145": 9.10446, + "146": 8.98709, + "147": 8.98617, + "148": 9.13261, + "149": 9.06335, + "150": 9.01504, + "151": 8.92787, + "152": 8.8739, + "153": 9.06335, + "154": 9.17913, + "155": 9.13381, + "156": 9.04889, + "157": 9.15064, + "158": 9.04955, + "159": 9.03261, + "160": 8.88987, + "161": 9.04543, + "162": 8.89584, + "163": 8.84272, + "164": 8.97534, + "165": 8.93132, + "166": 8.65959, + "167": 8.83243, + "168": 8.81953, + "169": 8.6566, + "170": 9.04622, + "171": 8.72286, + "172": 8.82159, + "173": 8.91163, + "174": 8.84751, + "175": 8.70611, + "176": 8.75439, + "177": 8.7626, + "178": 8.7201, + "179": 8.64046, + "180": 8.74053, + "181": 8.69404, + "182": 8.72193, + "183": 9.08364, + "184": 8.6088, + "185": 8.88346, + "186": 8.74191, + "187": 8.56949, + "188": 8.67975, + "189": 8.86478, + "190": 8.53542, + "191": 8.66632, + "192": 8.61266, + "193": 8.57469, + "194": 8.75195, + "195": 8.59279, + "196": 8.77393, + "197": 8.74234, + "198": 8.62722, + "199": 8.77454, + "200": 8.73803, + "201": 8.66979, + "202": 8.54593, + "203": 8.54185, + "204": 8.71307, + "205": 8.2228, + "206": 8.8603, + "207": 8.68157, + "208": 8.70896, + "209": 8.75303, + "210": 8.57807, + "211": 8.84258, + "212": 8.49127, + "213": 8.57327, + "214": 8.51199, + "215": 8.5645, + "216": 8.50863, + "217": 8.53183, + "218": 8.52998, + "219": 8.64367, + "220": 8.54746, + "221": 8.39991, + "222": 8.50528, + "223": 8.43775, + "224": 8.53014, + "225": 8.57091, + "226": 8.4394, + "227": 8.67918, + "228": 8.38473, + "229": 8.45045, + "230": 8.49717, + "231": 8.49832, + "232": 8.49783, + "233": 8.49539, + "234": 8.63795, + "235": 8.55875, + "236": 8.39461, + "237": 8.48826, + "238": 8.30522, + "239": 8.562, + "240": 8.66952, + "241": 8.44144, + "242": 8.47219, + "243": 8.51768, + "244": 8.36825, + "245": 8.59274, + "246": 8.59497, + "247": 8.44008, + "248": 8.51279, + "249": 8.52035, + "250": 8.42183, + "251": 8.37751, + "252": 8.54393, + "253": 8.31454, + "254": 8.351, + "255": 8.29005, + "256": 8.20261, + "257": 8.394, + "258": 8.45386, + "259": 8.23708, + "260": 8.2437, + "261": 8.23617, + "262": 8.34919, + "263": 8.30683, + "264": 8.18831, + "265": 8.33481, + "266": 8.23369, + "267": 7.89923, + "268": 8.38063, + "269": 8.40466, + "270": 8.26271, + "271": 8.279, + "272": 8.32109, + "273": 8.13747, + "274": 8.09677, + "275": 8.01372, + "276": 7.92611, + "277": 8.24041, + "278": 8.05017, + "279": 7.96688, + "280": 7.75652, + "281": 8.10713, + "282": 8.15049, + "283": 8.15621, + "284": 8.10354, + "285": 8.07234, + "286": 7.90454, + "287": 7.9963, + "288": 8.24862, + "289": 8.17575, + "290": 8.13093, + "291": 8.25763, + "292": 8.08131, + "293": 8.12059, + "294": 7.98178, + "295": 7.97108, + "296": 8.24114, + "297": 7.79647, + "298": 8.04847, + "299": 7.94257, + "300": 7.85748, + "301": 8.01649, + "302": 7.95112, + "303": 7.99606, + "304": 7.96394, + "305": 8.00301, + "306": 7.98312, + "307": 7.99372, + "308": 8.00491, + "309": 8.01362, + "310": 7.97824, + "311": 7.9323, + "312": 7.89419, + "313": 7.84054, + "314": 7.83, + "315": 7.8335, + "316": 7.75122, + "317": 7.934, + "318": 7.98841, + "319": 7.83343, + "320": 7.57896, + "321": 7.75427, + "322": 7.83781, + "323": 7.7769, + "324": 7.91623, + "325": 7.80539, + "326": 7.65641, + "327": 7.86989, + "328": 7.79369, + "329": 7.89137, + "330": 7.7586, + "331": 7.52885, + "332": 7.81946, + "333": 7.84359, + "334": 7.68375, + "335": 7.69975, + "336": 7.91931, + "337": 7.65356, + "338": 7.90277, + "339": 7.7307, + "340": 7.7606, + "341": 7.70898, + "342": 7.82827, + "343": 7.61824, + "344": 7.58818, + "345": 7.61602, + "346": 7.46415, + "347": 7.5612, + "348": 7.68737, + "349": 7.58361, + "350": 7.65762, + "351": 7.75424, + "352": 7.711, + "353": 7.50477, + "354": 7.74925, + "355": 7.77011, + "356": 7.78305, + "357": 7.81855, + "358": 7.60031, + "359": 7.55187, + "360": 7.63213, + "361": 7.55298, + "362": 7.76875, + "363": 7.59465, + "364": 7.57928, + "365": 7.62839, + "366": 7.31096, + "367": 7.55919, + "368": 7.44577, + "369": 7.3551, + "370": 7.46985, + "371": 7.46609, + "372": 7.65475, + "373": 7.52989, + "374": 7.44843, + "375": 7.53627, + "376": 7.35288, + "377": 7.24313, + "378": 7.54312, + "379": 7.4994, + "380": 7.38859, + "381": 7.47577, + "382": 7.29951, + "383": 7.28478, + "384": 7.4126, + "385": 7.39829, + "386": 7.23652, + "387": 7.42535, + "388": 7.28487, + "389": 7.44425, + "390": 7.24578, + "391": 7.6482, + "392": 7.34245, + "393": 7.42463, + "394": 7.48248, + "395": 7.44483, + "396": 7.29231, + "397": 7.23386, + "398": 7.42507, + "399": 7.16173, + "400": 7.30149, + "401": 7.3585, + "402": 7.39832, + "403": 7.28806, + "404": 7.30832, + "405": 7.27202, + "406": 7.22485, + "407": 7.36688, + "408": 7.18877, + "409": 7.17334, + "410": 7.31999, + "411": 7.2223, + "412": 7.20595, + "413": 7.24047, + "414": 6.9176, + "415": 7.3341, + "416": 7.43139, + "417": 7.0298, + "418": 7.28201, + "419": 7.04286, + "420": 7.41864, + "421": 7.18456, + "422": 7.24003, + "423": 7.09785, + "424": 7.24581, + "425": 7.32182, + "426": 7.29342, + "427": 7.1359, + "428": 7.09617, + "429": 6.87976, + "430": 7.20691, + "431": 7.00662, + "432": 7.23762, + "433": 6.97996, + "434": 6.96131, + "435": 7.02219, + "436": 7.01484, + "437": 6.9921, + "438": 7.00514, + "439": 6.94235, + "440": 7.06367, + "441": 7.04936, + "442": 7.10187, + "443": 7.0941, + "444": 6.71175, + "445": 6.99825, + "446": 7.14631, + "447": 7.12745, + "448": 6.98621, + "449": 7.0508, + "450": 7.01761, + "451": 6.83255, + "452": 6.9157, + "453": 7.02056, + "454": 6.97019, + "455": 7.03145, + "456": 6.99451, + "457": 6.97283, + "458": 6.9066, + "459": 6.69482, + "460": 7.06773, + "461": 7.09857, + "462": 6.87116, + "463": 7.05522, + "464": 6.64922, + "465": 7.02852, + "466": 7.00594, + "467": 6.99935, + "468": 6.95215, + "469": 6.8291, + "470": 7.04615, + "471": 6.88316, + "472": 6.96104, + "473": 6.82398, + "474": 6.97228, + "475": 7.16917, + "476": 6.76379, + "477": 6.89771, + "478": 6.91142, + "479": 6.70396, + "480": 7.03025, + "481": 6.99763, + "482": 6.73608, + "483": 6.78502, + "484": 6.75413, + "485": 6.93205, + "486": 7.06796, + "487": 6.63653, + "488": 6.88737, + "489": 6.77108, + "490": 6.82685, + "491": 6.71122, + "492": 6.69849, + "493": 6.77155, + "494": 6.67651, + "495": 6.63733, + "496": 6.59006, + "497": 6.84564, + "498": 6.65256, + "499": 6.85952, + "500": 6.65795, + "501": 6.73562, + "502": 6.84527, + "503": 6.71173, + "504": 6.62075, + "505": 6.62291, + "506": 6.75234, + "507": 6.86844, + "508": 6.86157, + "509": 6.6555, + "510": 6.82834, + "511": 6.74132, + "512": 6.74051, + "513": 6.66032, + "514": 6.71273, + "515": 6.45045, + "516": 6.74436, + "517": 6.71073, + "518": 6.53817, + "519": 6.63527, + "520": 6.85868, + "521": 6.66571, + "522": 6.70871, + "523": 6.74553, + "524": 6.73396, + "525": 6.6762, + "526": 6.4139, + "527": 6.79901, + "528": 6.66011, + "529": 6.63182, + "530": 6.62611, + "531": 6.64289, + "532": 6.63292, + "533": 6.76391, + "534": 6.61301, + "535": 6.74754, + "536": 6.62605, + "537": 6.63867, + "538": 6.53166, + "539": 6.5542, + "540": 6.5862, + "541": 6.45207, + "542": 6.66957, + "543": 6.68064, + "544": 6.67601, + "545": 6.81307, + "546": 6.63333, + "547": 6.41838, + "548": 6.72367, + "549": 6.69982, + "550": 6.52974, + "551": 6.7478, + "552": 6.63991, + "553": 6.48451, + "554": 6.63407, + "555": 6.4629, + "556": 6.61792, + "557": 6.63496, + "558": 6.3874, + "559": 6.37379, + "560": 6.58293, + "561": 6.73352, + "562": 6.6356, + "563": 6.7444, + "564": 6.35291, + "565": 6.51482, + "566": 6.70247, + "567": 6.56973, + "568": 6.51145, + "569": 6.45578, + "570": 6.36768, + "571": 6.63597, + "572": 6.31359, + "573": 6.58668, + "574": 6.47613, + "575": 6.64961, + "576": 6.5168, + "577": 6.53078, + "578": 6.4847, + "579": 6.46709, + "580": 6.56793, + "581": 6.60857, + "582": 6.48362, + "583": 6.51541, + "584": 6.52831, + "585": 6.42713, + "586": 6.4178, + "587": 6.46113, + "588": 6.56878, + "589": 6.62653, + "590": 6.29114, + "591": 6.67541, + "592": 6.26902, + "593": 6.4773, + "594": 6.38719, + "595": 6.3632, + "596": 6.26099, + "597": 6.18986, + "598": 6.45726, + "599": 6.3998, + "600": 6.45709, + "601": 6.26132, + "602": 6.5338, + "603": 6.52288, + "604": 6.38993, + "605": 6.49993, + "606": 6.31475, + "607": 6.53507, + "608": 6.67525, + "609": 6.17714, + "610": 6.57295, + "611": 6.40188, + "612": 6.57929, + "613": 6.42667, + "614": 6.20672, + "615": 6.40081, + "616": 6.36019, + "617": 6.37969, + "618": 6.4512, + "619": 6.14244, + "620": 6.41233, + "621": 6.46338, + "622": 6.40096, + "623": 6.58352, + "624": 6.36078, + "625": 6.28553, + "626": 6.30525, + "627": 6.44574, + "628": 6.2557, + "629": 6.58813, + "630": 6.36641, + "631": 6.3498, + "632": 6.30972, + "633": 6.25733, + "634": 6.30887, + "635": 6.54592, + "636": 6.24834, + "637": 6.63634, + "638": 6.02046, + "639": 6.2798, + "640": 6.29548, + "641": 6.20953, + "642": 6.28471, + "643": 6.461, + "644": 6.25863, + "645": 6.25115, + "646": 6.40601, + "647": 6.33707, + "648": 6.35671, + "649": 6.3488, + "650": 6.48415, + "651": 6.33395, + "652": 6.25233, + "653": 6.3826, + "654": 6.45063, + "655": 6.52494, + "656": 6.32781, + "657": 6.43503, + "658": 6.24353, + "659": 6.1554, + "660": 6.39397, + "661": 6.17184, + "662": 6.27494, + "663": 6.37237, + "664": 6.33376, + "665": 6.40442, + "666": 6.16399, + "667": 6.1965, + "668": 6.2366, + "669": 6.21813, + "670": 6.24601, + "671": 6.24468, + "672": 6.49032, + "673": 6.34071, + "674": 6.2969, + "675": 6.38396, + "676": 6.39021, + "677": 6.30588, + "678": 6.27751, + "679": 6.23892, + "680": 6.2942, + "681": 6.20621, + "682": 6.08719, + "683": 6.27464, + "684": 6.32896, + "685": 6.30248, + "686": 6.15397, + "687": 6.2862, + "688": 6.20754, + "689": 6.6215, + "690": 6.17931, + "691": 6.18188, + "692": 6.2745, + "693": 6.14405, + "694": 6.23487, + "695": 6.32617, + "696": 6.11842, + "697": 6.15483, + "698": 6.23128, + "699": 6.46051, + "700": 6.0454, + "701": 6.06467, + "702": 6.25219, + "703": 6.18603, + "704": 6.21704, + "705": 6.13155, + "706": 6.07593, + "707": 6.25376, + "708": 6.31553, + "709": 6.01087, + "710": 6.16305, + "711": 6.26062, + "712": 6.18307, + "713": 5.89806, + "714": 6.10759, + "715": 6.11617, + "716": 6.41405, + "717": 6.19202, + "718": 6.2345, + "719": 6.27471, + "720": 6.26372, + "721": 6.26277, + "722": 6.23442, + "723": 6.0814, + "724": 6.22797, + "725": 6.04057, + "726": 6.30046, + "727": 6.01682, + "728": 6.04617, + "729": 6.09111, + "730": 6.18359, + "731": 6.10398, + "732": 6.08898, + "733": 6.12312, + "734": 6.38423, + "735": 6.27849, + "736": 6.18184, + "737": 6.36645, + "738": 6.13411, + "739": 6.14591, + "740": 5.87975, + "741": 6.00667, + "742": 5.98459, + "743": 6.17495, + "744": 6.02962, + "745": 6.15497, + "746": 6.03272, + "747": 6.09789, + "748": 6.23436, + "749": 5.94191, + "750": 6.16819, + "751": 5.9596, + "752": 6.01941, + "753": 6.02989, + "754": 6.28798, + "755": 6.13521, + "756": 6.25357, + "757": 6.02098, + "758": 6.20422, + "759": 6.23062, + "760": 6.02316, + "761": 6.19655, + "762": 6.22713, + "763": 6.03754, + "764": 5.9636, + "765": 5.93413, + "766": 5.97155, + "767": 5.81277, + "768": 6.18725, + "769": 6.27646, + "770": 6.29561, + "771": 5.78767, + "772": 6.03281, + "773": 6.18558, + "774": 5.88583, + "775": 6.03167, + "776": 6.13086, + "777": 5.88612, + "778": 6.05891, + "779": 5.87414, + "780": 6.14047, + "781": 5.85641, + "782": 6.04961, + "783": 5.95687, + "784": 5.91852, + "785": 6.09816, + "786": 6.10929, + "787": 5.66006, + "788": 5.99915, + "789": 6.21789, + "790": 6.26737, + "791": 5.79122, + "792": 5.99828, + "793": 6.18387, + "794": 6.02746, + "795": 6.0051, + "796": 6.17065, + "797": 6.05376, + "798": 6.06076, + "799": 6.11682, + "800": 6.02167, + "801": 6.15011, + "802": 5.98473, + "803": 6.15363, + "804": 6.00859, + "805": 5.83055, + "806": 6.08757, + "807": 6.04997, + "808": 5.92717, + "809": 5.77802, + "810": 6.01973, + "811": 5.93299, + "812": 5.91169, + "813": 5.96567, + "814": 6.0369, + "815": 5.8146, + "816": 6.12034, + "817": 5.94337, + "818": 6.0674, + "819": 6.01476, + "820": 5.7319, + "821": 5.95027, + "822": 6.20452, + "823": 5.83139, + "824": 5.98275, + "825": 6.18795, + "826": 6.20019, + "827": 6.05802, + "828": 6.06976, + "829": 5.89149, + "830": 5.94221, + "831": 5.89773, + "832": 5.97341, + "833": 6.06501, + "834": 5.99675, + "835": 6.00654, + "836": 5.79277, + "837": 6.11496, + "838": 5.86966, + "839": 5.83554, + "840": 6.18614, + "841": 5.78491, + "842": 5.89169, + "843": 5.95102, + "844": 6.00954, + "845": 6.09153, + "846": 5.68733, + "847": 5.75715, + "848": 5.96838, + "849": 6.09512, + "850": 5.84886, + "851": 6.01693, + "852": 5.75188, + "853": 5.99355, + "854": 6.01844, + "855": 5.81656, + "856": 5.99593, + "857": 6.00207, + "858": 6.05507, + "859": 5.95295, + "860": 6.09632, + "861": 6.07189, + "862": 6.00434, + "863": 5.83757, + "864": 5.84474, + "865": 5.93791, + "866": 5.89404, + "867": 5.87803, + "868": 6.06515, + "869": 6.08564, + "870": 5.97153, + "871": 6.04317, + "872": 5.89525, + "873": 5.84383, + "874": 6.02742, + "875": 5.9144, + "876": 5.96905, + "877": 5.92979, + "878": 6.09819, + "879": 5.76783, + "880": 6.01501, + "881": 5.99647, + "882": 5.9097, + "883": 5.67626, + "884": 5.96521, + "885": 5.74544, + "886": 5.99268, + "887": 5.90979, + "888": 5.83897, + "889": 6.01033, + "890": 6.02378, + "891": 5.95247, + "892": 5.70829, + "893": 6.0922, + "894": 5.73134, + "895": 5.84057, + "896": 5.84075, + "897": 5.8564, + "898": 5.9238, + "899": 5.93486, + "900": 5.89946, + "901": 5.95293, + "902": 5.83295, + "903": 6.05665, + "904": 5.93153, + "905": 5.90441, + "906": 5.6172, + "907": 5.91178, + "908": 5.73853, + "909": 5.99118, + "910": 5.86603, + "911": 5.70397, + "912": 5.70712, + "913": 5.76497, + "914": 5.83944, + "915": 5.80032, + "916": 5.8904, + "917": 5.86913, + "918": 5.82415, + "919": 5.81575, + "920": 5.89552, + "921": 5.84163, + "922": 5.62427, + "923": 6.03657, + "924": 5.60536, + "925": 5.62335, + "926": 5.86148, + "927": 5.96071, + "928": 5.84005, + "929": 5.82702, + "930": 5.95816, + "931": 5.765, + "932": 5.59211, + "933": 5.6351, + "934": 5.80541, + "935": 5.63715, + "936": 5.83772, + "937": 5.96629, + "938": 5.59109, + "939": 5.7899, + "940": 5.96726, + "941": 5.7264, + "942": 5.83547, + "943": 5.86622, + "944": 5.95478, + "945": 5.70263, + "946": 5.55832, + "947": 5.74831, + "948": 5.79312, + "949": 5.8268, + "950": 5.84353, + "951": 5.72242, + "952": 5.69295, + "953": 5.67852, + "954": 5.72473, + "955": 5.53107, + "956": 5.62074, + "957": 5.84076, + "958": 5.79676, + "959": 5.57317, + "960": 5.80125, + "961": 5.82952, + "962": 5.76695, + "963": 5.76461, + "964": 5.70677, + "965": 5.64012, + "966": 5.59617, + "967": 5.72434, + "968": 5.74036, + "969": 5.82392, + "970": 5.64422, + "971": 5.7065, + "972": 5.85308, + "973": 5.66884, + "974": 5.71841, + "975": 5.86273, + "976": 5.70493, + "977": 5.77104, + "978": 5.6858, + "979": 5.58655, + "980": 5.75924, + "981": 5.8969, + "982": 5.47038, + "983": 5.61817, + "984": 5.54504, + "985": 5.59032, + "986": 5.64132, + "987": 5.56966, + "988": 5.70939, + "989": 5.69379, + "990": 5.62195, + "991": 5.84899, + "992": 5.77877, + "993": 5.87022, + "994": 5.69735, + "995": 5.73242, + "996": 5.73704, + "997": 5.81329, + "998": 5.83634, + "999": 5.83399, + "1000": 5.68342, + "1001": 5.86668, + "1002": 5.76052, + "1003": 5.64259, + "1004": 5.79811, + "1005": 5.53617, + "1006": 5.326, + "1007": 5.76701, + "1008": 5.79136, + "1009": 5.65046, + "1010": 5.77942, + "1011": 5.89493, + "1012": 5.62303, + "1013": 5.61569, + "1014": 5.68111, + "1015": 5.55747, + "1016": 5.87327, + "1017": 5.83312, + "1018": 5.61865, + "1019": 5.73414, + "1020": 5.61755, + "1021": 5.848, + "1022": 5.50045, + "1023": 5.65182, + "1024": 5.74493, + "1025": 5.5692, + "1026": 5.41415, + "1027": 5.60696, + "1028": 5.6928, + "1029": 5.68764, + "1030": 5.68746, + "1031": 5.40696, + "1032": 5.78748, + "1033": 5.58136, + "1034": 5.61937, + "1035": 5.71368, + "1036": 5.62818, + "1037": 5.3679, + "1038": 5.66452, + "1039": 5.64347, + "1040": 5.57004, + "1041": 5.59722, + "1042": 5.81329, + "1043": 5.566, + "1044": 5.46906, + "1045": 5.9659, + "1046": 5.4866, + "1047": 5.38954, + "1048": 5.50027, + "1049": 5.67182, + "1050": 5.6991, + "1051": 5.57928, + "1052": 5.68227, + "1053": 5.62737, + "1054": 5.45766, + "1055": 5.60313, + "1056": 5.67386, + "1057": 5.75895, + "1058": 5.56782, + "1059": 5.74888, + "1060": 5.82022, + "1061": 5.47624, + "1062": 5.64897, + "1063": 5.50121, + "1064": 5.59136, + "1065": 5.55347, + "1066": 5.74367, + "1067": 5.67235, + "1068": 5.44068, + "1069": 5.60636, + "1070": 5.81264, + "1071": 5.51129, + "1072": 5.61871, + "1073": 5.62147, + "1074": 5.524, + "1075": 5.70529, + "1076": 5.5934, + "1077": 5.71153, + "1078": 5.56524, + "1079": 5.61728, + "1080": 5.64251, + "1081": 5.62319, + "1082": 5.49648, + "1083": 5.64086, + "1084": 5.55389, + "1085": 5.40631, + "1086": 5.62008, + "1087": 5.44148, + "1088": 5.51218, + "1089": 5.7676, + "1090": 5.53165, + "1091": 5.51388, + "1092": 5.41011, + "1093": 5.70025, + "1094": 5.57364, + "1095": 5.57735, + "1096": 5.61585, + "1097": 5.64586, + "1098": 5.64877, + "1099": 5.51631, + "1100": 5.63778, + "1101": 5.67335, + "1102": 5.54037, + "1103": 5.54969, + "1104": 5.53882, + "1105": 5.54754, + "1106": 5.68315, + "1107": 5.68556, + "1108": 5.78611, + "1109": 5.53666, + "1110": 5.66598, + "1111": 5.58973, + "1112": 5.58039, + "1113": 5.62611, + "1114": 5.61279, + "1115": 5.59718, + "1116": 5.65925, + "1117": 5.64676, + "1118": 5.65036, + "1119": 5.70919, + "1120": 5.62738, + "1121": 5.37352, + "1122": 5.22976, + "1123": 5.47237, + "1124": 5.64939, + "1125": 5.67974, + "1126": 5.679, + "1127": 5.56811, + "1128": 5.61992, + "1129": 5.29637, + "1130": 5.54359, + "1131": 5.63153, + "1132": 5.72427, + "1133": 5.51914, + "1134": 5.56063, + "1135": 5.52056, + "1136": 5.42646, + "1137": 5.45971, + "1138": 5.56927, + "1139": 5.41452, + "1140": 5.2656, + "1141": 5.58265, + "1142": 5.64152, + "1143": 5.38298, + "1144": 5.38584, + "1145": 5.36231, + "1146": 5.63508, + "1147": 5.49183, + "1148": 5.50524, + "1149": 5.52352, + "1150": 5.39801, + "1151": 5.5563, + "1152": 5.41525, + "1153": 5.44791, + "1154": 5.49757, + "1155": 5.43833, + "1156": 5.3488, + "1157": 5.66444, + "1158": 5.39487, + "1159": 5.33455, + "1160": 5.79503, + "1161": 5.53955, + "1162": 5.45818, + "1163": 5.52563, + "1164": 5.3837, + "1165": 5.52861, + "1166": 5.48753, + "1167": 5.36312, + "1168": 5.49491, + "1169": 5.39842, + "1170": 5.59202, + "1171": 5.48502, + "1172": 5.64238, + "1173": 5.62295, + "1174": 5.50843, + "1175": 5.34639, + "1176": 5.38504, + "1177": 5.55461, + "1178": 5.46852, + "1179": 5.49505, + "1180": 5.46014, + "1181": 5.56031, + "1182": 5.59593, + "1183": 5.77155, + "1184": 5.54926, + "1185": 5.29008, + "1186": 5.60451, + "1187": 5.55363, + "1188": 5.51655, + "1189": 5.39133, + "1190": 5.40482, + "1191": 5.39266, + "1192": 5.50142, + "1193": 5.46347, + "1194": 5.45607, + "1195": 5.32751, + "1196": 5.52219, + "1197": 5.4809, + "1198": 5.52789, + "1199": 5.3874, + "1200": 5.33059, + "1201": 5.48969, + "1202": 5.43584, + "1203": 5.49537, + "1204": 5.40861, + "1205": 5.48971, + "1206": 5.3371, + "1207": 5.58625, + "1208": 5.4312, + "1209": 5.29323, + "1210": 5.50765, + "1211": 5.51506, + "1212": 5.59777, + "1213": 5.42123, + "1214": 5.51018, + "1215": 5.23832, + "1216": 5.40989, + "1217": 5.38537, + "1218": 5.45232, + "1219": 5.48221, + "1220": 5.38594, + "1221": 5.44848, + "1222": 5.31032, + "1223": 5.47835, + "1224": 5.42017, + "1225": 5.43499, + "1226": 5.3238, + "1227": 5.47632, + "1228": 5.72418, + "1229": 5.32629, + "1230": 5.40556, + "1231": 5.06972, + "1232": 5.78794, + "1233": 5.28923, + "1234": 5.24535, + "1235": 5.37092, + "1236": 5.48471, + "1237": 5.20864, + "1238": 5.41643, + "1239": 5.40751, + "1240": 5.46767, + "1241": 5.57266, + "1242": 5.4536, + "1243": 5.43063, + "1244": 5.51812, + "1245": 5.19115, + "1246": 5.72042, + "1247": 5.43187, + "1248": 5.30004, + "1249": 5.40113, + "1250": 5.33798, + "1251": 5.42034, + "1252": 5.57217, + "1253": 5.48773, + "1254": 5.30628, + "1255": 5.51443, + "1256": 5.60755, + "1257": 5.4214, + "1258": 5.56457, + "1259": 5.48027, + "1260": 5.51461, + "1261": 5.63883, + "1262": 5.39531, + "1263": 5.32916, + "1264": 5.50671, + "1265": 5.30632, + "1266": 5.23819, + "1267": 5.37206, + "1268": 5.39267, + "1269": 5.15366, + "1270": 5.40418, + "1271": 5.27732, + "1272": 5.5252, + "1273": 5.30228, + "1274": 5.3516, + "1275": 5.38466, + "1276": 5.39786, + "1277": 5.46218, + "1278": 5.34689, + "1279": 5.44274, + "1280": 5.45919, + "1281": 5.40638, + "1282": 5.3824, + "1283": 5.42204, + "1284": 5.34841, + "1285": 5.50133, + "1286": 5.33557, + "1287": 5.58795, + "1288": 5.26493, + "1289": 5.429, + "1290": 5.50282, + "1291": 5.50335, + "1292": 5.44662, + "1293": 5.41955, + "1294": 5.49953, + "1295": 5.34675, + "1296": 5.19062, + "1297": 5.17238, + "1298": 5.11916, + "1299": 5.30339, + "1300": 5.21032, + "1301": 5.30157, + "1302": 5.27472, + "1303": 5.36107, + "1304": 5.43231, + "1305": 5.36999, + "1306": 5.25347, + "1307": 5.18829, + "1308": 5.27033, + "1309": 5.40736, + "1310": 5.26399, + "1311": 5.38109, + "1312": 5.35438, + "1313": 5.30056, + "1314": 5.2953, + "1315": 5.42245, + "1316": 5.26148, + "1317": 5.28065, + "1318": 5.2198, + "1319": 5.34619, + "1320": 5.42093, + "1321": 5.44976, + "1322": 5.46399, + "1323": 5.37327, + "1324": 5.25463, + "1325": 5.40657, + "1326": 5.54082, + "1327": 5.39378, + "1328": 5.21893, + "1329": 5.41851, + "1330": 5.40079, + "1331": 5.31685, + "1332": 5.31253, + "1333": 5.37243, + "1334": 5.44685, + "1335": 5.37136, + "1336": 5.43779, + "1337": 5.47852, + "1338": 5.30292, + "1339": 5.14181, + "1340": 5.41486, + "1341": 5.3443, + "1342": 5.36197, + "1343": 5.47816, + "1344": 5.37832, + "1345": 5.34294, + "1346": 5.08195, + "1347": 5.38558, + "1348": 5.4918, + "1349": 5.40832, + "1350": 5.02622, + "1351": 5.3151, + "1352": 5.1591, + "1353": 5.34674, + "1354": 5.35963, + "1355": 5.11092, + "1356": 5.2587, + "1357": 5.29209, + "1358": 5.15773, + "1359": 5.11035, + "1360": 5.17288, + "1361": 5.30521, + "1362": 5.06318, + "1363": 5.2947, + "1364": 5.40031, + "1365": 5.02241, + "1366": 5.11779, + "1367": 5.33051, + "1368": 5.18648, + "1369": 5.22984, + "1370": 5.19906, + "1371": 5.2839, + "1372": 5.26155, + "1373": 5.28402, + "1374": 5.28112, + "1375": 5.46052, + "1376": 5.2713, + "1377": 5.26467, + "1378": 5.31344, + "1379": 5.22741, + "1380": 5.26107, + "1381": 5.47871, + "1382": 5.08923, + "1383": 5.375, + "1384": 5.35914, + "1385": 5.38983, + "1386": 5.16417, + "1387": 5.16094, + "1388": 5.28017, + "1389": 5.30376, + "1390": 5.25514, + "1391": 5.26911, + "1392": 5.37008, + "1393": 5.38307, + "1394": 5.40394, + "1395": 5.32492, + "1396": 5.21356, + "1397": 5.28, + "1398": 5.37051, + "1399": 5.35873, + "1400": 5.26512, + "1401": 5.35924, + "1402": 5.42148, + "1403": 5.20238, + "1404": 5.28629, + "1405": 5.11984, + "1406": 4.99128, + "1407": 5.40442, + "1408": 5.19825, + "1409": 5.3964, + "1410": 5.37519, + "1411": 4.91758, + "1412": 5.35561, + "1413": 5.41314, + "1414": 5.21823, + "1415": 5.44159, + "1416": 5.32905, + "1417": 5.38859, + "1418": 5.29946, + "1419": 5.31787, + "1420": 5.43974, + "1421": 5.39414, + "1422": 5.41749, + "1423": 5.005, + "1424": 5.32995, + "1425": 5.58618, + "1426": 5.23059, + "1427": 5.31804, + "1428": 5.33277, + "1429": 5.07552, + "1430": 5.33075, + "1431": 5.32688, + "1432": 5.33826, + "1433": 5.19107, + "1434": 5.16341, + "1435": 5.19905, + "1436": 5.10851, + "1437": 5.229, + "1438": 5.31867, + "1439": 5.34731, + "1440": 5.34991, + "1441": 5.16484, + "1442": 5.22015, + "1443": 5.20933, + "1444": 5.13701, + "1445": 5.07414, + "1446": 5.26836, + "1447": 5.25895, + "1448": 5.2904, + "1449": 5.2498, + "1450": 5.34281, + "1451": 5.07084, + "1452": 5.27052, + "1453": 5.1668, + "1454": 5.01539, + "1455": 5.12292, + "1456": 5.2717, + "1457": 5.18713, + "1458": 5.00608, + "1459": 5.22304, + "1460": 5.23389, + "1461": 5.07142, + "1462": 4.96923, + "1463": 5.14383, + "1464": 5.21128, + "1465": 5.26911, + "1466": 5.34961, + "1467": 5.33438, + "1468": 5.22205, + "1469": 5.04373, + "1470": 5.11715, + "1471": 5.25199, + "1472": 5.12294, + "1473": 5.10395, + "1474": 5.21775, + "1475": 5.18567, + "1476": 5.15287, + "1477": 5.26203, + "1478": 5.30399, + "1479": 5.01175, + "1480": 5.1809, + "1481": 5.24516, + "1482": 5.34866, + "1483": 5.26395, + "1484": 4.92397, + "1485": 5.29179, + "1486": 5.04178, + "1487": 4.88296, + "1488": 5.18145, + "1489": 5.10246, + "1490": 5.04399, + "1491": 5.31709, + "1492": 5.22469, + "1493": 4.94051, + "1494": 5.10929, + "1495": 5.13424, + "1496": 5.05862, + "1497": 5.36633, + "1498": 5.30967, + "1499": 5.13834, + "1500": 5.09851, + "1501": 5.03466, + "1502": 5.15527, + "1503": 5.43143, + "1504": 5.31968, + "1505": 5.00114, + "1506": 5.14444, + "1507": 5.16068, + "1508": 5.16575, + "1509": 5.31451, + "1510": 5.0185, + "1511": 5.11697, + "1512": 4.98287, + "1513": 5.16993, + "1514": 5.33962, + "1515": 5.36563, + "1516": 5.27715, + "1517": 5.22687, + "1518": 5.02626, + "1519": 5.29861, + "1520": 5.1417, + "1521": 5.15866, + "1522": 5.32824, + "1523": 5.24625, + "1524": 5.06725, + "1525": 5.20424, + "1526": 5.27994, + "1527": 5.25677, + "1528": 5.23589, + "1529": 5.18688, + "1530": 5.24365, + "1531": 5.09964, + "1532": 5.15141, + "1533": 5.05087, + "1534": 5.21589, + "1535": 5.1635, + "1536": 5.09678, + "1537": 5.02713, + "1538": 4.91184, + "1539": 5.23801, + "1540": 5.11515, + "1541": 5.25246, + "1542": 5.23484, + "1543": 5.05152, + "1544": 5.07544, + "1545": 5.1161, + "1546": 5.33085, + "1547": 5.11115, + "1548": 5.23527, + "1549": 5.23735, + "1550": 4.97596, + "1551": 5.2566, + "1552": 5.02944, + "1553": 5.14849, + "1554": 5.11205, + "1555": 5.10901, + "1556": 5.19824, + "1557": 5.08883, + "1558": 5.23067, + "1559": 5.00402, + "1560": 5.11835, + "1561": 5.14529, + "1562": 5.17996, + "1563": 5.24454, + "1564": 5.26389, + "1565": 5.08902, + "1566": 5.29474, + "1567": 5.04166, + "1568": 5.09256, + "1569": 5.20014, + "1570": 5.17348, + "1571": 4.95353, + "1572": 5.04005, + "1573": 5.02897, + "1574": 4.99751, + "1575": 5.2314, + "1576": 5.21263, + "1577": 5.12799, + "1578": 5.36241, + "1579": 4.94367, + "1580": 5.12197, + "1581": 5.09638, + "1582": 5.28497, + "1583": 5.04918, + "1584": 5.05482, + "1585": 5.11977, + "1586": 5.30243, + "1587": 5.13447, + "1588": 5.2184, + "1589": 4.83833, + "1590": 5.09497, + "1591": 5.17411, + "1592": 5.13721, + "1593": 5.23457, + "1594": 5.11805, + "1595": 5.10775, + "1596": 5.18964, + "1597": 5.11486, + "1598": 5.15917, + "1599": 5.19102, + "1600": 4.86871, + "1601": 5.11732, + "1602": 5.23185, + "1603": 5.19543, + "1604": 5.05128, + "1605": 5.02692, + "1606": 4.98659, + "1607": 5.07391, + "1608": 4.97985, + "1609": 5.07337, + "1610": 5.04745, + "1611": 4.99848, + "1612": 4.75205, + "1613": 5.03316, + "1614": 4.88034, + "1615": 5.07442, + "1616": 5.23082, + "1617": 5.06132, + "1618": 4.98704, + "1619": 5.18333, + "1620": 5.14491, + "1621": 5.31452, + "1622": 5.05677, + "1623": 5.14346, + "1624": 5.1355, + "1625": 5.12006, + "1626": 5.10245, + "1627": 5.10987, + "1628": 5.06581, + "1629": 4.92971, + "1630": 5.06799, + "1631": 5.06088, + "1632": 5.10428, + "1633": 4.97515, + "1634": 4.9235, + "1635": 5.05833, + "1636": 4.92289, + "1637": 5.24051, + "1638": 5.15574, + "1639": 4.977, + "1640": 5.00918, + "1641": 5.12718, + "1642": 5.08305, + "1643": 5.04894, + "1644": 5.1181, + "1645": 4.96677, + "1646": 5.11931, + "1647": 5.03295, + "1648": 5.19969, + "1649": 4.92396, + "1650": 5.05963, + "1651": 4.92965, + "1652": 5.21121, + "1653": 5.15959, + "1654": 5.12828, + "1655": 5.16263, + "1656": 5.34595, + "1657": 5.20677, + "1658": 5.04112, + "1659": 4.9258, + "1660": 4.80954, + "1661": 5.03086, + "1662": 5.14123, + "1663": 5.15449, + "1664": 4.981, + "1665": 5.11714, + "1666": 5.10575, + "1667": 4.84897, + "1668": 5.11513, + "1669": 5.06995, + "1670": 5.11266, + "1671": 5.17201, + "1672": 4.77569, + "1673": 5.03851, + "1674": 4.91569, + "1675": 5.05176, + "1676": 5.00402, + "1677": 4.79944, + "1678": 5.02487, + "1679": 4.89421, + "1680": 5.03847, + "1681": 5.06815, + "1682": 5.03274, + "1683": 4.90688, + "1684": 5.06515, + "1685": 5.13579, + "1686": 5.0732, + "1687": 4.97656, + "1688": 5.16537, + "1689": 5.14707, + "1690": 4.99688, + "1691": 5.00011, + "1692": 4.91822, + "1693": 5.01472, + "1694": 4.94657, + "1695": 4.91341, + "1696": 5.08209, + "1697": 5.04294, + "1698": 4.9511, + "1699": 5.00187, + "1700": 4.95393, + "1701": 5.16563, + "1702": 5.07666, + "1703": 5.17125, + "1704": 5.14332, + "1705": 4.96247, + "1706": 4.98333, + "1707": 4.79005, + "1708": 5.03831, + "1709": 5.23334, + "1710": 5.02934, + "1711": 5.19037, + "1712": 5.1958, + "1713": 5.03582, + "1714": 5.04603, + "1715": 4.91495, + "1716": 4.9332, + "1717": 4.86109, + "1718": 5.0273, + "1719": 5.12334, + "1720": 5.02189, + "1721": 4.92752, + "1722": 5.05412, + "1723": 4.93537, + "1724": 5.0407, + "1725": 5.1914, + "1726": 5.06447, + "1727": 4.90742, + "1728": 5.02116, + "1729": 5.04574, + "1730": 4.90343, + "1731": 4.99945, + "1732": 4.92083, + "1733": 5.1311, + "1734": 4.82837, + "1735": 5.20905, + "1736": 4.91585, + "1737": 4.85859, + "1738": 4.97909, + "1739": 5.16688, + "1740": 4.83514, + "1741": 4.77896, + "1742": 4.90909, + "1743": 5.08523, + "1744": 4.9784, + "1745": 4.82327, + "1746": 4.94833, + "1747": 4.87022, + "1748": 5.06379, + "1749": 4.8705, + "1750": 5.01347, + "1751": 5.12189, + "1752": 4.90364, + "1753": 5.09398, + "1754": 5.05918, + "1755": 4.89649, + "1756": 5.02243, + "1757": 5.14389, + "1758": 4.8716, + "1759": 4.94237, + "1760": 4.83366, + "1761": 5.02233, + "1762": 4.81292, + "1763": 4.77382, + "1764": 4.93787, + "1765": 5.14977, + "1766": 5.33847, + "1767": 5.22339, + "1768": 4.95072, + "1769": 5.00607, + "1770": 4.98077, + "1771": 4.96436, + "1772": 4.98395, + "1773": 4.97312, + "1774": 4.86859, + "1775": 4.95207, + "1776": 4.99761, + "1777": 4.94332, + "1778": 4.99268, + "1779": 5.08376, + "1780": 4.83276, + "1781": 5.05321, + "1782": 4.9968, + "1783": 5.01268, + "1784": 4.93195, + "1785": 5.16736, + "1786": 4.81265, + "1787": 4.97081, + "1788": 4.82725, + "1789": 4.88846, + "1790": 4.79821, + "1791": 4.73741, + "1792": 4.87626, + "1793": 5.10356, + "1794": 4.98084, + "1795": 4.96551, + "1796": 4.99704, + "1797": 4.7903, + "1798": 4.76702, + "1799": 5.01884, + "1800": 4.91364, + "1801": 5.04679, + "1802": 4.82665, + "1803": 4.95171, + "1804": 4.88594, + "1805": 4.90346, + "1806": 4.87351, + "1807": 4.92406, + "1808": 4.92697, + "1809": 5.1451, + "1810": 5.09976, + "1811": 4.95906, + "1812": 4.80139, + "1813": 5.09748, + "1814": 4.77766, + "1815": 4.86134, + "1816": 5.05005, + "1817": 4.79012, + "1818": 4.80376, + "1819": 5.02382, + "1820": 4.68652, + "1821": 5.02661, + "1822": 4.66251, + "1823": 4.8659, + "1824": 4.78635, + "1825": 5.06537, + "1826": 4.81944, + "1827": 4.7895, + "1828": 4.94677, + "1829": 5.11262, + "1830": 4.91236, + "1831": 4.89818, + "1832": 4.83359, + "1833": 4.78363, + "1834": 4.9482, + "1835": 4.95795, + "1836": 4.90747, + "1837": 4.67243, + "1838": 4.80953, + "1839": 4.89546, + "1840": 4.90488, + "1841": 4.8292, + "1842": 4.94678, + "1843": 4.70293, + "1844": 4.61431, + "1845": 5.00086, + "1846": 4.74657, + "1847": 4.8645, + "1848": 4.89695, + "1849": 4.85358, + "1850": 4.8676, + "1851": 5.02236, + "1852": 4.97647, + "1853": 4.83325, + "1854": 4.86791, + "1855": 4.8219, + "1856": 4.75614, + "1857": 4.9619, + "1858": 4.96856, + "1859": 4.75323, + "1860": 4.86592, + "1861": 5.20685, + "1862": 4.61669, + "1863": 4.83385, + "1864": 4.7505, + "1865": 4.86441, + "1866": 4.79455, + "1867": 4.99688, + "1868": 4.71331, + "1869": 4.75634, + "1870": 4.93203, + "1871": 4.99184, + "1872": 4.68332, + "1873": 4.69823, + "1874": 4.85174, + "1875": 4.85999, + "1876": 4.7392, + "1877": 4.80362, + "1878": 4.81239, + "1879": 4.82084, + "1880": 4.89314, + "1881": 4.79389, + "1882": 4.79419, + "1883": 4.78157, + "1884": 4.97086, + "1885": 4.91799, + "1886": 4.82203, + "1887": 4.81334, + "1888": 4.97395, + "1889": 4.95922, + "1890": 4.70676, + "1891": 4.65282, + "1892": 4.84393, + "1893": 4.64594, + "1894": 4.90265, + "1895": 4.7886, + "1896": 4.66112, + "1897": 4.78966, + "1898": 4.9139, + "1899": 4.77532, + "1900": 4.91571, + "1901": 4.84525, + "1902": 4.78411, + "1903": 4.75997, + "1904": 4.65339, + "1905": 4.54188, + "1906": 4.81097, + "1907": 4.90225, + "1908": 5.03012, + "1909": 4.88434, + "1910": 4.78852, + "1911": 4.80477, + "1912": 4.64685, + "1913": 4.94065, + "1914": 4.87965, + "1915": 4.85906, + "1916": 4.92227, + "1917": 4.85425, + "1918": 4.87001, + "1919": 4.99304, + "1920": 4.76319, + "1921": 4.88494, + "1922": 4.81295, + "1923": 4.7592, + "1924": 4.82501, + "1925": 5.05793, + "1926": 4.92996, + "1927": 4.92587, + "1928": 4.92702, + "1929": 4.92705, + "1930": 4.91019, + "1931": 4.77616, + "1932": 4.85963, + "1933": 4.83545, + "1934": 4.84013, + "1935": 5.10729, + "1936": 4.88314, + "1937": 4.87654, + "1938": 4.79463, + "1939": 4.71148, + "1940": 4.82418, + "1941": 4.73372, + "1942": 4.87249, + "1943": 4.7353, + "1944": 4.74198, + "1945": 4.6818, + "1946": 4.91539, + "1947": 4.86756, + "1948": 4.59887, + "1949": 4.90387, + "1950": 4.78785, + "1951": 4.95942, + "1952": 4.73677, + "1953": 4.79496, + "1954": 4.73264, + "1955": 4.84308, + "1956": 4.88233, + "1957": 4.73496, + "1958": 4.70018, + "1959": 4.75966, + "1960": 4.76849, + "1961": 4.7146, + "1962": 4.83392, + "1963": 4.82321, + "1964": 4.84664, + "1965": 4.87523, + "1966": 4.78753, + "1967": 4.59211, + "1968": 4.82724, + "1969": 4.59184, + "1970": 4.56633, + "1971": 4.9072, + "1972": 4.90064, + "1973": 4.54642, + "1974": 4.82423, + "1975": 4.82778, + "1976": 4.71327, + "1977": 4.57967, + "1978": 5.0045, + "1979": 4.66094, + "1980": 4.74256, + "1981": 4.86301, + "1982": 4.72234, + "1983": 4.8786, + "1984": 4.64152, + "1985": 4.78, + "1986": 4.70167, + "1987": 4.81036, + "1988": 4.8871, + "1989": 4.63185, + "1990": 4.79636, + "1991": 4.69424, + "1992": 4.79439, + "1993": 4.74063, + "1994": 4.84977, + "1995": 4.5596, + "1996": 4.65161, + "1997": 4.80342, + "1998": 4.67403, + "1999": 4.72284, + "2000": 4.61765 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 80.0, + "2": 70.0, + "3": 78.0, + "4": 80.0, + "5": 75.0, + "6": 87.0, + "7": 63.0, + "8": 77.0, + "9": 62.0, + "10": 90.0, + "11": 74.0, + "12": 79.0, + "13": 77.0, + "14": 83.0, + "15": 78.0, + "16": 69.0, + "17": 64.0, + "18": 63.0, + "19": 87.0, + "20": 90.0, + "21": 75.0, + "22": 84.0, + "23": 81.0, + "24": 78.0, + "25": 87.0, + "26": 69.0, + "27": 86.0, + "28": 91.0, + "29": 94.0, + "30": 115.0, + "31": 99.0, + "32": 109.0, + "33": 92.0, + "34": 103.0, + "35": 118.0, + "36": 117.0, + "37": 105.0, + "38": 129.0, + "39": 89.0, + "40": 129.0, + "41": 114.0, + "42": 121.0, + "43": 135.0, + "44": 128.0, + "45": 126.0, + "46": 129.0, + "47": 133.0, + "48": 139.0, + "49": 135.0, + "50": 157.0, + "51": 122.0, + "52": 150.0, + "53": 108.0, + "54": 140.0, + "55": 133.0, + "56": 156.0, + "57": 150.0, + "58": 153.0, + "59": 135.0, + "60": 135.0, + "61": 165.0, + "62": 145.0, + "63": 199.0, + "64": 161.0, + "65": 162.0, + "66": 162.0, + "67": 195.0, + "68": 140.0, + "69": 158.0, + "70": 169.0, + "71": 188.0, + "72": 160.0, + "73": 151.0, + "74": 154.0, + "75": 172.0, + "76": 169.0, + "77": 165.0, + "78": 193.0, + "79": 144.0, + "80": 173.0, + "81": 150.0, + "82": 141.0, + "83": 186.0, + "84": 169.0, + "85": 183.0, + "86": 196.0, + "87": 197.0, + "88": 184.0, + "89": 169.0, + "90": 182.0, + "91": 200.0, + "92": 179.0, + "93": 165.0, + "94": 153.0, + "95": 176.0, + "96": 191.0, + "97": 183.0, + "98": 199.0, + "99": 163.0, + "100": 157.0, + "101": 144.0, + "102": 184.0, + "103": 206.0, + "104": 171.0, + "105": 215.0, + "106": 176.0, + "107": 172.0, + "108": 172.0, + "109": 172.0, + "110": 216.0, + "111": 182.0, + "112": 172.0, + "113": 167.0, + "114": 192.0, + "115": 175.0, + "116": 181.0, + "117": 177.0, + "118": 142.0, + "119": 212.0, + "120": 164.0, + "121": 193.0, + "122": 160.0, + "123": 169.0, + "124": 191.0, + "125": 214.0, + "126": 160.0, + "127": 192.0, + "128": 160.0, + "129": 180.0, + "130": 214.0, + "131": 219.0, + "132": 173.0, + "133": 166.0, + "134": 171.0, + "135": 182.0, + "136": 172.0, + "137": 176.0, + "138": 174.0, + "139": 161.0, + "140": 178.0, + "141": 164.0, + "142": 159.0, + "143": 192.0, + "144": 157.0, + "145": 144.0, + "146": 149.0, + "147": 148.0, + "148": 169.0, + "149": 143.0, + "150": 111.0, + "151": 159.0, + "152": 115.0, + "153": 147.0, + "154": 162.0, + "155": 185.0, + "156": 144.0, + "157": 147.0, + "158": 130.0, + "159": 165.0, + "160": 190.0, + "161": 141.0, + "162": 155.0, + "163": 140.0, + "164": 174.0, + "165": 168.0, + "166": 179.0, + "167": 147.0, + "168": 138.0, + "169": 161.0, + "170": 159.0, + "171": 125.0, + "172": 193.0, + "173": 172.0, + "174": 190.0, + "175": 192.0, + "176": 146.0, + "177": 168.0, + "178": 172.0, + "179": 177.0, + "180": 148.0, + "181": 161.0, + "182": 213.0, + "183": 215.0, + "184": 201.0, + "185": 154.0, + "186": 207.0, + "187": 175.0, + "188": 183.0, + "189": 169.0, + "190": 167.0, + "191": 163.0, + "192": 193.0, + "193": 169.0, + "194": 161.0, + "195": 141.0, + "196": 174.0, + "197": 188.0, + "198": 168.0, + "199": 150.0, + "200": 187.0, + "201": 173.0, + "202": 183.0, + "203": 142.0, + "204": 177.0, + "205": 153.0, + "206": 198.0, + "207": 168.0, + "208": 140.0, + "209": 179.0, + "210": 175.0, + "211": 167.0, + "212": 194.0, + "213": 192.0, + "214": 174.0, + "215": 188.0, + "216": 164.0, + "217": 170.0, + "218": 171.0, + "219": 211.0, + "220": 195.0, + "221": 181.0, + "222": 154.0, + "223": 176.0, + "224": 173.0, + "225": 166.0, + "226": 174.0, + "227": 211.0, + "228": 146.0, + "229": 193.0, + "230": 149.0, + "231": 177.0, + "232": 169.0, + "233": 193.0, + "234": 183.0, + "235": 215.0, + "236": 200.0, + "237": 218.0, + "238": 179.0, + "239": 139.0, + "240": 217.0, + "241": 174.0, + "242": 193.0, + "243": 192.0, + "244": 181.0, + "245": 206.0, + "246": 221.0, + "247": 219.0, + "248": 175.0, + "249": 189.0, + "250": 156.0, + "251": 205.0, + "252": 164.0, + "253": 172.0, + "254": 184.0, + "255": 218.0, + "256": 171.0, + "257": 208.0, + "258": 210.0, + "259": 174.0, + "260": 199.0, + "261": 178.0, + "262": 185.0, + "263": 181.0, + "264": 200.0, + "265": 171.0, + "266": 149.0, + "267": 141.0, + "268": 186.0, + "269": 198.0, + "270": 170.0, + "271": 168.0, + "272": 210.0, + "273": 151.0, + "274": 212.0, + "275": 182.0, + "276": 172.0, + "277": 159.0, + "278": 169.0, + "279": 185.0, + "280": 174.0, + "281": 160.0, + "282": 171.0, + "283": 174.0, + "284": 183.0, + "285": 169.0, + "286": 173.0, + "287": 203.0, + "288": 168.0, + "289": 202.0, + "290": 157.0, + "291": 241.0, + "292": 172.0, + "293": 209.0, + "294": 194.0, + "295": 207.0, + "296": 217.0, + "297": 160.0, + "298": 126.0, + "299": 170.0, + "300": 177.0, + "301": 189.0, + "302": 209.0, + "303": 170.0, + "304": 177.0, + "305": 148.0, + "306": 172.0, + "307": 213.0, + "308": 184.0, + "309": 193.0, + "310": 218.0, + "311": 159.0, + "312": 178.0, + "313": 177.0, + "314": 199.0, + "315": 165.0, + "316": 168.0, + "317": 185.0, + "318": 261.0, + "319": 181.0, + "320": 196.0, + "321": 200.0, + "322": 217.0, + "323": 198.0, + "324": 200.0, + "325": 184.0, + "326": 283.0, + "327": 211.0, + "328": 231.0, + "329": 189.0, + "330": 248.0, + "331": 205.0, + "332": 208.0, + "333": 199.0, + "334": 182.0, + "335": 202.0, + "336": 207.0, + "337": 216.0, + "338": 231.0, + "339": 213.0, + "340": 240.0, + "341": 207.0, + "342": 153.0, + "343": 264.0, + "344": 214.0, + "345": 202.0, + "346": 183.0, + "347": 194.0, + "348": 216.0, + "349": 206.0, + "350": 218.0, + "351": 218.0, + "352": 207.0, + "353": 225.0, + "354": 213.0, + "355": 201.0, + "356": 227.0, + "357": 217.0, + "358": 206.0, + "359": 186.0, + "360": 217.0, + "361": 187.0, + "362": 256.0, + "363": 226.0, + "364": 203.0, + "365": 200.0, + "366": 241.0, + "367": 205.0, + "368": 192.0, + "369": 160.0, + "370": 221.0, + "371": 212.0, + "372": 193.0, + "373": 218.0, + "374": 164.0, + "375": 249.0, + "376": 195.0, + "377": 197.0, + "378": 222.0, + "379": 254.0, + "380": 210.0, + "381": 199.0, + "382": 217.0, + "383": 208.0, + "384": 238.0, + "385": 183.0, + "386": 221.0, + "387": 185.0, + "388": 205.0, + "389": 185.0, + "390": 217.0, + "391": 241.0, + "392": 212.0, + "393": 247.0, + "394": 242.0, + "395": 247.0, + "396": 197.0, + "397": 202.0, + "398": 191.0, + "399": 231.0, + "400": 211.0, + "401": 200.0, + "402": 210.0, + "403": 261.0, + "404": 211.0, + "405": 171.0, + "406": 209.0, + "407": 200.0, + "408": 226.0, + "409": 200.0, + "410": 220.0, + "411": 196.0, + "412": 194.0, + "413": 168.0, + "414": 223.0, + "415": 204.0, + "416": 225.0, + "417": 213.0, + "418": 196.0, + "419": 203.0, + "420": 203.0, + "421": 217.0, + "422": 200.0, + "423": 213.0, + "424": 237.0, + "425": 239.0, + "426": 178.0, + "427": 213.0, + "428": 196.0, + "429": 174.0, + "430": 243.0, + "431": 169.0, + "432": 203.0, + "433": 211.0, + "434": 194.0, + "435": 188.0, + "436": 208.0, + "437": 170.0, + "438": 194.0, + "439": 156.0, + "440": 199.0, + "441": 190.0, + "442": 232.0, + "443": 225.0, + "444": 172.0, + "445": 194.0, + "446": 221.0, + "447": 209.0, + "448": 233.0, + "449": 257.0, + "450": 207.0, + "451": 199.0, + "452": 177.0, + "453": 200.0, + "454": 227.0, + "455": 263.0, + "456": 196.0, + "457": 204.0, + "458": 169.0, + "459": 131.0, + "460": 216.0, + "461": 223.0, + "462": 210.0, + "463": 203.0, + "464": 208.0, + "465": 187.0, + "466": 190.0, + "467": 192.0, + "468": 194.0, + "469": 188.0, + "470": 193.0, + "471": 221.0, + "472": 166.0, + "473": 191.0, + "474": 193.0, + "475": 196.0, + "476": 192.0, + "477": 168.0, + "478": 180.0, + "479": 176.0, + "480": 145.0, + "481": 197.0, + "482": 167.0, + "483": 198.0, + "484": 172.0, + "485": 175.0, + "486": 192.0, + "487": 143.0, + "488": 182.0, + "489": 172.0, + "490": 178.0, + "491": 175.0, + "492": 194.0, + "493": 211.0, + "494": 159.0, + "495": 165.0, + "496": 153.0, + "497": 145.0, + "498": 196.0, + "499": 195.0, + "500": 165.0, + "501": 183.0, + "502": 167.0, + "503": 175.0, + "504": 182.0, + "505": 212.0, + "506": 177.0, + "507": 159.0, + "508": 135.0, + "509": 195.0, + "510": 156.0, + "511": 186.0, + "512": 177.0, + "513": 186.0, + "514": 173.0, + "515": 190.0, + "516": 175.0, + "517": 143.0, + "518": 169.0, + "519": 186.0, + "520": 156.0, + "521": 146.0, + "522": 173.0, + "523": 175.0, + "524": 172.0, + "525": 202.0, + "526": 168.0, + "527": 178.0, + "528": 173.0, + "529": 183.0, + "530": 168.0, + "531": 161.0, + "532": 185.0, + "533": 172.0, + "534": 166.0, + "535": 140.0, + "536": 164.0, + "537": 150.0, + "538": 155.0, + "539": 125.0, + "540": 151.0, + "541": 130.0, + "542": 153.0, + "543": 149.0, + "544": 185.0, + "545": 132.0, + "546": 184.0, + "547": 150.0, + "548": 155.0, + "549": 162.0, + "550": 170.0, + "551": 144.0, + "552": 147.0, + "553": 213.0, + "554": 182.0, + "555": 150.0, + "556": 162.0, + "557": 154.0, + "558": 181.0, + "559": 144.0, + "560": 194.0, + "561": 174.0, + "562": 147.0, + "563": 125.0, + "564": 169.0, + "565": 143.0, + "566": 136.0, + "567": 144.0, + "568": 153.0, + "569": 167.0, + "570": 153.0, + "571": 131.0, + "572": 143.0, + "573": 128.0, + "574": 162.0, + "575": 133.0, + "576": 143.0, + "577": 171.0, + "578": 167.0, + "579": 140.0, + "580": 165.0, + "581": 164.0, + "582": 145.0, + "583": 151.0, + "584": 146.0, + "585": 148.0, + "586": 102.0, + "587": 147.0, + "588": 146.0, + "589": 123.0, + "590": 146.0, + "591": 149.0, + "592": 115.0, + "593": 166.0, + "594": 159.0, + "595": 127.0, + "596": 113.0, + "597": 135.0, + "598": 139.0, + "599": 157.0, + "600": 129.0, + "601": 144.0, + "602": 129.0, + "603": 125.0, + "604": 125.0, + "605": 139.0, + "606": 135.0, + "607": 144.0, + "608": 149.0, + "609": 139.0, + "610": 135.0, + "611": 148.0, + "612": 148.0, + "613": 115.0, + "614": 150.0, + "615": 132.0, + "616": 156.0, + "617": 120.0, + "618": 145.0, + "619": 136.0, + "620": 170.0, + "621": 147.0, + "622": 150.0, + "623": 119.0, + "624": 128.0, + "625": 141.0, + "626": 122.0, + "627": 121.0, + "628": 157.0, + "629": 126.0, + "630": 134.0, + "631": 147.0, + "632": 146.0, + "633": 131.0, + "634": 145.0, + "635": 174.0, + "636": 151.0, + "637": 169.0, + "638": 128.0, + "639": 164.0, + "640": 145.0, + "641": 136.0, + "642": 132.0, + "643": 134.0, + "644": 124.0, + "645": 145.0, + "646": 106.0, + "647": 123.0, + "648": 121.0, + "649": 134.0, + "650": 153.0, + "651": 117.0, + "652": 163.0, + "653": 155.0, + "654": 140.0, + "655": 154.0, + "656": 124.0, + "657": 116.0, + "658": 130.0, + "659": 114.0, + "660": 145.0, + "661": 121.0, + "662": 143.0, + "663": 124.0, + "664": 139.0, + "665": 138.0, + "666": 111.0, + "667": 127.0, + "668": 144.0, + "669": 116.0, + "670": 139.0, + "671": 132.0, + "672": 136.0, + "673": 139.0, + "674": 119.0, + "675": 165.0, + "676": 123.0, + "677": 127.0, + "678": 135.0, + "679": 83.0, + "680": 139.0, + "681": 120.0, + "682": 111.0, + "683": 119.0, + "684": 121.0, + "685": 145.0, + "686": 127.0, + "687": 145.0, + "688": 117.0, + "689": 119.0, + "690": 119.0, + "691": 124.0, + "692": 118.0, + "693": 112.0, + "694": 156.0, + "695": 114.0, + "696": 141.0, + "697": 123.0, + "698": 130.0, + "699": 147.0, + "700": 119.0, + "701": 139.0, + "702": 111.0, + "703": 113.0, + "704": 118.0, + "705": 115.0, + "706": 102.0, + "707": 121.0, + "708": 115.0, + "709": 116.0, + "710": 95.0, + "711": 101.0, + "712": 98.0, + "713": 117.0, + "714": 127.0, + "715": 135.0, + "716": 124.0, + "717": 88.0, + "718": 143.0, + "719": 114.0, + "720": 120.0, + "721": 106.0, + "722": 117.0, + "723": 101.0, + "724": 97.0, + "725": 106.0, + "726": 103.0, + "727": 95.0, + "728": 123.0, + "729": 104.0, + "730": 124.0, + "731": 111.0, + "732": 78.0, + "733": 96.0, + "734": 129.0, + "735": 142.0, + "736": 110.0, + "737": 132.0, + "738": 110.0, + "739": 136.0, + "740": 106.0, + "741": 102.0, + "742": 123.0, + "743": 133.0, + "744": 130.0, + "745": 109.0, + "746": 122.0, + "747": 125.0, + "748": 133.0, + "749": 114.0, + "750": 121.0, + "751": 113.0, + "752": 111.0, + "753": 96.0, + "754": 118.0, + "755": 87.0, + "756": 113.0, + "757": 91.0, + "758": 105.0, + "759": 99.0, + "760": 125.0, + "761": 106.0, + "762": 105.0, + "763": 101.0, + "764": 109.0, + "765": 118.0, + "766": 95.0, + "767": 133.0, + "768": 115.0, + "769": 122.0, + "770": 106.0, + "771": 123.0, + "772": 106.0, + "773": 136.0, + "774": 128.0, + "775": 116.0, + "776": 112.0, + "777": 95.0, + "778": 113.0, + "779": 119.0, + "780": 99.0, + "781": 107.0, + "782": 80.0, + "783": 108.0, + "784": 122.0, + "785": 111.0, + "786": 111.0, + "787": 115.0, + "788": 116.0, + "789": 108.0, + "790": 127.0, + "791": 83.0, + "792": 117.0, + "793": 102.0, + "794": 106.0, + "795": 123.0, + "796": 121.0, + "797": 124.0, + "798": 112.0, + "799": 136.0, + "800": 99.0, + "801": 117.0, + "802": 93.0, + "803": 166.0, + "804": 127.0, + "805": 124.0, + "806": 97.0, + "807": 134.0, + "808": 108.0, + "809": 121.0, + "810": 126.0, + "811": 107.0, + "812": 116.0, + "813": 126.0, + "814": 105.0, + "815": 98.0, + "816": 99.0, + "817": 97.0, + "818": 97.0, + "819": 109.0, + "820": 106.0, + "821": 88.0, + "822": 109.0, + "823": 108.0, + "824": 127.0, + "825": 108.0, + "826": 128.0, + "827": 134.0, + "828": 100.0, + "829": 125.0, + "830": 113.0, + "831": 114.0, + "832": 107.0, + "833": 113.0, + "834": 100.0, + "835": 98.0, + "836": 123.0, + "837": 95.0, + "838": 118.0, + "839": 96.0, + "840": 109.0, + "841": 98.0, + "842": 114.0, + "843": 113.0, + "844": 123.0, + "845": 108.0, + "846": 124.0, + "847": 112.0, + "848": 115.0, + "849": 118.0, + "850": 92.0, + "851": 145.0, + "852": 89.0, + "853": 106.0, + "854": 101.0, + "855": 113.0, + "856": 125.0, + "857": 105.0, + "858": 129.0, + "859": 107.0, + "860": 118.0, + "861": 85.0, + "862": 106.0, + "863": 95.0, + "864": 81.0, + "865": 104.0, + "866": 105.0, + "867": 104.0, + "868": 106.0, + "869": 109.0, + "870": 105.0, + "871": 122.0, + "872": 114.0, + "873": 100.0, + "874": 113.0, + "875": 108.0, + "876": 93.0, + "877": 130.0, + "878": 110.0, + "879": 122.0, + "880": 106.0, + "881": 103.0, + "882": 80.0, + "883": 107.0, + "884": 115.0, + "885": 113.0, + "886": 116.0, + "887": 131.0, + "888": 89.0, + "889": 120.0, + "890": 110.0, + "891": 103.0, + "892": 102.0, + "893": 106.0, + "894": 91.0, + "895": 118.0, + "896": 110.0, + "897": 103.0, + "898": 115.0, + "899": 119.0, + "900": 120.0, + "901": 99.0, + "902": 100.0, + "903": 102.0, + "904": 127.0, + "905": 105.0, + "906": 124.0, + "907": 104.0, + "908": 117.0, + "909": 124.0, + "910": 108.0, + "911": 102.0, + "912": 117.0, + "913": 122.0, + "914": 130.0, + "915": 98.0, + "916": 120.0, + "917": 113.0, + "918": 112.0, + "919": 85.0, + "920": 110.0, + "921": 108.0, + "922": 111.0, + "923": 116.0, + "924": 119.0, + "925": 105.0, + "926": 128.0, + "927": 120.0, + "928": 106.0, + "929": 94.0, + "930": 116.0, + "931": 102.0, + "932": 123.0, + "933": 114.0, + "934": 133.0, + "935": 86.0, + "936": 114.0, + "937": 96.0, + "938": 118.0, + "939": 111.0, + "940": 110.0, + "941": 102.0, + "942": 98.0, + "943": 119.0, + "944": 107.0, + "945": 106.0, + "946": 112.0, + "947": 93.0, + "948": 119.0, + "949": 116.0, + "950": 124.0, + "951": 112.0, + "952": 106.0, + "953": 97.0, + "954": 111.0, + "955": 112.0, + "956": 87.0, + "957": 117.0, + "958": 97.0, + "959": 91.0, + "960": 103.0, + "961": 102.0, + "962": 103.0, + "963": 127.0, + "964": 113.0, + "965": 120.0, + "966": 106.0, + "967": 104.0, + "968": 119.0, + "969": 89.0, + "970": 121.0, + "971": 115.0, + "972": 96.0, + "973": 90.0, + "974": 113.0, + "975": 109.0, + "976": 113.0, + "977": 85.0, + "978": 104.0, + "979": 109.0, + "980": 100.0, + "981": 94.0, + "982": 105.0, + "983": 84.0, + "984": 112.0, + "985": 108.0, + "986": 92.0, + "987": 88.0, + "988": 123.0, + "989": 106.0, + "990": 103.0, + "991": 128.0, + "992": 104.0, + "993": 109.0, + "994": 98.0, + "995": 104.0, + "996": 93.0, + "997": 128.0, + "998": 121.0, + "999": 89.0, + "1000": 118.0, + "1001": 104.0, + "1002": 96.0, + "1003": 107.0, + "1004": 88.0, + "1005": 103.0, + "1006": 105.0, + "1007": 102.0, + "1008": 83.0, + "1009": 117.0, + "1010": 104.0, + "1011": 127.0, + "1012": 117.0, + "1013": 106.0, + "1014": 111.0, + "1015": 110.0, + "1016": 91.0, + "1017": 76.0, + "1018": 115.0, + "1019": 123.0, + "1020": 111.0, + "1021": 106.0, + "1022": 108.0, + "1023": 137.0, + "1024": 122.0, + "1025": 104.0, + "1026": 109.0, + "1027": 92.0, + "1028": 96.0, + "1029": 116.0, + "1030": 96.0, + "1031": 122.0, + "1032": 103.0, + "1033": 108.0, + "1034": 111.0, + "1035": 86.0, + "1036": 74.0, + "1037": 123.0, + "1038": 85.0, + "1039": 128.0, + "1040": 95.0, + "1041": 116.0, + "1042": 107.0, + "1043": 96.0, + "1044": 116.0, + "1045": 115.0, + "1046": 92.0, + "1047": 106.0, + "1048": 88.0, + "1049": 121.0, + "1050": 117.0, + "1051": 105.0, + "1052": 96.0, + "1053": 98.0, + "1054": 85.0, + "1055": 110.0, + "1056": 91.0, + "1057": 109.0, + "1058": 95.0, + "1059": 106.0, + "1060": 109.0, + "1061": 97.0, + "1062": 105.0, + "1063": 91.0, + "1064": 103.0, + "1065": 108.0, + "1066": 112.0, + "1067": 108.0, + "1068": 108.0, + "1069": 123.0, + "1070": 100.0, + "1071": 95.0, + "1072": 111.0, + "1073": 118.0, + "1074": 101.0, + "1075": 95.0, + "1076": 111.0, + "1077": 89.0, + "1078": 94.0, + "1079": 113.0, + "1080": 82.0, + "1081": 114.0, + "1082": 87.0, + "1083": 116.0, + "1084": 105.0, + "1085": 97.0, + "1086": 119.0, + "1087": 86.0, + "1088": 93.0, + "1089": 114.0, + "1090": 87.0, + "1091": 109.0, + "1092": 90.0, + "1093": 109.0, + "1094": 101.0, + "1095": 90.0, + "1096": 106.0, + "1097": 100.0, + "1098": 105.0, + "1099": 96.0, + "1100": 92.0, + "1101": 108.0, + "1102": 94.0, + "1103": 86.0, + "1104": 103.0, + "1105": 109.0, + "1106": 87.0, + "1107": 87.0, + "1108": 96.0, + "1109": 102.0, + "1110": 89.0, + "1111": 76.0, + "1112": 110.0, + "1113": 104.0, + "1114": 89.0, + "1115": 114.0, + "1116": 97.0, + "1117": 108.0, + "1118": 107.0, + "1119": 118.0, + "1120": 112.0, + "1121": 96.0, + "1122": 103.0, + "1123": 112.0, + "1124": 98.0, + "1125": 97.0, + "1126": 121.0, + "1127": 80.0, + "1128": 91.0, + "1129": 106.0, + "1130": 96.0, + "1131": 82.0, + "1132": 103.0, + "1133": 86.0, + "1134": 92.0, + "1135": 98.0, + "1136": 90.0, + "1137": 120.0, + "1138": 102.0, + "1139": 109.0, + "1140": 88.0, + "1141": 90.0, + "1142": 95.0, + "1143": 88.0, + "1144": 77.0, + "1145": 92.0, + "1146": 85.0, + "1147": 108.0, + "1148": 77.0, + "1149": 93.0, + "1150": 101.0, + "1151": 116.0, + "1152": 72.0, + "1153": 90.0, + "1154": 103.0, + "1155": 106.0, + "1156": 91.0, + "1157": 100.0, + "1158": 101.0, + "1159": 111.0, + "1160": 114.0, + "1161": 90.0, + "1162": 92.0, + "1163": 90.0, + "1164": 96.0, + "1165": 100.0, + "1166": 114.0, + "1167": 82.0, + "1168": 96.0, + "1169": 77.0, + "1170": 91.0, + "1171": 94.0, + "1172": 99.0, + "1173": 124.0, + "1174": 106.0, + "1175": 97.0, + "1176": 102.0, + "1177": 78.0, + "1178": 108.0, + "1179": 103.0, + "1180": 84.0, + "1181": 76.0, + "1182": 115.0, + "1183": 104.0, + "1184": 122.0, + "1185": 104.0, + "1186": 104.0, + "1187": 91.0, + "1188": 112.0, + "1189": 101.0, + "1190": 106.0, + "1191": 97.0, + "1192": 90.0, + "1193": 105.0, + "1194": 99.0, + "1195": 118.0, + "1196": 120.0, + "1197": 93.0, + "1198": 101.0, + "1199": 103.0, + "1200": 90.0, + "1201": 108.0, + "1202": 120.0, + "1203": 90.0, + "1204": 98.0, + "1205": 113.0, + "1206": 102.0, + "1207": 116.0, + "1208": 104.0, + "1209": 85.0, + "1210": 101.0, + "1211": 87.0, + "1212": 100.0, + "1213": 109.0, + "1214": 92.0, + "1215": 103.0, + "1216": 117.0, + "1217": 102.0, + "1218": 135.0, + "1219": 95.0, + "1220": 122.0, + "1221": 121.0, + "1222": 109.0, + "1223": 103.0, + "1224": 93.0, + "1225": 107.0, + "1226": 82.0, + "1227": 108.0, + "1228": 106.0, + "1229": 87.0, + "1230": 97.0, + "1231": 109.0, + "1232": 95.0, + "1233": 99.0, + "1234": 107.0, + "1235": 105.0, + "1236": 101.0, + "1237": 110.0, + "1238": 102.0, + "1239": 118.0, + "1240": 114.0, + "1241": 119.0, + "1242": 90.0, + "1243": 104.0, + "1244": 102.0, + "1245": 105.0, + "1246": 104.0, + "1247": 121.0, + "1248": 104.0, + "1249": 129.0, + "1250": 111.0, + "1251": 91.0, + "1252": 120.0, + "1253": 121.0, + "1254": 110.0, + "1255": 113.0, + "1256": 97.0, + "1257": 114.0, + "1258": 110.0, + "1259": 106.0, + "1260": 93.0, + "1261": 104.0, + "1262": 109.0, + "1263": 104.0, + "1264": 101.0, + "1265": 85.0, + "1266": 106.0, + "1267": 104.0, + "1268": 90.0, + "1269": 102.0, + "1270": 106.0, + "1271": 107.0, + "1272": 79.0, + "1273": 85.0, + "1274": 99.0, + "1275": 127.0, + "1276": 89.0, + "1277": 144.0, + "1278": 109.0, + "1279": 110.0, + "1280": 123.0, + "1281": 98.0, + "1282": 94.0, + "1283": 110.0, + "1284": 88.0, + "1285": 112.0, + "1286": 106.0, + "1287": 86.0, + "1288": 100.0, + "1289": 118.0, + "1290": 109.0, + "1291": 82.0, + "1292": 106.0, + "1293": 97.0, + "1294": 96.0, + "1295": 91.0, + "1296": 110.0, + "1297": 120.0, + "1298": 105.0, + "1299": 114.0, + "1300": 113.0, + "1301": 106.0, + "1302": 112.0, + "1303": 102.0, + "1304": 94.0, + "1305": 109.0, + "1306": 83.0, + "1307": 97.0, + "1308": 120.0, + "1309": 126.0, + "1310": 103.0, + "1311": 126.0, + "1312": 100.0, + "1313": 101.0, + "1314": 107.0, + "1315": 117.0, + "1316": 101.0, + "1317": 107.0, + "1318": 103.0, + "1319": 98.0, + "1320": 103.0, + "1321": 112.0, + "1322": 86.0, + "1323": 117.0, + "1324": 94.0, + "1325": 94.0, + "1326": 139.0, + "1327": 82.0, + "1328": 124.0, + "1329": 103.0, + "1330": 91.0, + "1331": 94.0, + "1332": 106.0, + "1333": 86.0, + "1334": 86.0, + "1335": 96.0, + "1336": 113.0, + "1337": 114.0, + "1338": 126.0, + "1339": 104.0, + "1340": 101.0, + "1341": 83.0, + "1342": 106.0, + "1343": 122.0, + "1344": 99.0, + "1345": 93.0, + "1346": 110.0, + "1347": 105.0, + "1348": 104.0, + "1349": 103.0, + "1350": 111.0, + "1351": 121.0, + "1352": 106.0, + "1353": 108.0, + "1354": 108.0, + "1355": 92.0, + "1356": 89.0, + "1357": 103.0, + "1358": 120.0, + "1359": 110.0, + "1360": 125.0, + "1361": 116.0, + "1362": 133.0, + "1363": 103.0, + "1364": 109.0, + "1365": 101.0, + "1366": 100.0, + "1367": 93.0, + "1368": 108.0, + "1369": 127.0, + "1370": 99.0, + "1371": 121.0, + "1372": 116.0, + "1373": 110.0, + "1374": 94.0, + "1375": 107.0, + "1376": 104.0, + "1377": 115.0, + "1378": 100.0, + "1379": 106.0, + "1380": 88.0, + "1381": 103.0, + "1382": 101.0, + "1383": 118.0, + "1384": 120.0, + "1385": 117.0, + "1386": 123.0, + "1387": 93.0, + "1388": 86.0, + "1389": 119.0, + "1390": 116.0, + "1391": 103.0, + "1392": 84.0, + "1393": 100.0, + "1394": 112.0, + "1395": 77.0, + "1396": 101.0, + "1397": 124.0, + "1398": 104.0, + "1399": 120.0, + "1400": 103.0, + "1401": 100.0, + "1402": 105.0, + "1403": 82.0, + "1404": 104.0, + "1405": 93.0, + "1406": 102.0, + "1407": 118.0, + "1408": 100.0, + "1409": 114.0, + "1410": 85.0, + "1411": 101.0, + "1412": 99.0, + "1413": 117.0, + "1414": 116.0, + "1415": 115.0, + "1416": 90.0, + "1417": 99.0, + "1418": 97.0, + "1419": 96.0, + "1420": 119.0, + "1421": 108.0, + "1422": 113.0, + "1423": 91.0, + "1424": 123.0, + "1425": 101.0, + "1426": 110.0, + "1427": 107.0, + "1428": 116.0, + "1429": 128.0, + "1430": 87.0, + "1431": 96.0, + "1432": 113.0, + "1433": 92.0, + "1434": 101.0, + "1435": 101.0, + "1436": 111.0, + "1437": 122.0, + "1438": 105.0, + "1439": 99.0, + "1440": 101.0, + "1441": 104.0, + "1442": 89.0, + "1443": 109.0, + "1444": 86.0, + "1445": 100.0, + "1446": 87.0, + "1447": 105.0, + "1448": 102.0, + "1449": 88.0, + "1450": 100.0, + "1451": 94.0, + "1452": 95.0, + "1453": 116.0, + "1454": 98.0, + "1455": 92.0, + "1456": 91.0, + "1457": 132.0, + "1458": 121.0, + "1459": 109.0, + "1460": 111.0, + "1461": 111.0, + "1462": 89.0, + "1463": 99.0, + "1464": 108.0, + "1465": 97.0, + "1466": 87.0, + "1467": 99.0, + "1468": 127.0, + "1469": 88.0, + "1470": 103.0, + "1471": 101.0, + "1472": 106.0, + "1473": 120.0, + "1474": 96.0, + "1475": 123.0, + "1476": 85.0, + "1477": 122.0, + "1478": 107.0, + "1479": 113.0, + "1480": 109.0, + "1481": 107.0, + "1482": 118.0, + "1483": 86.0, + "1484": 98.0, + "1485": 91.0, + "1486": 96.0, + "1487": 119.0, + "1488": 106.0, + "1489": 93.0, + "1490": 113.0, + "1491": 107.0, + "1492": 100.0, + "1493": 123.0, + "1494": 105.0, + "1495": 121.0, + "1496": 105.0, + "1497": 99.0, + "1498": 112.0, + "1499": 106.0, + "1500": 104.0, + "1501": 129.0, + "1502": 109.0, + "1503": 91.0, + "1504": 111.0, + "1505": 97.0, + "1506": 116.0, + "1507": 122.0, + "1508": 103.0, + "1509": 141.0, + "1510": 86.0, + "1511": 120.0, + "1512": 120.0, + "1513": 128.0, + "1514": 100.0, + "1515": 108.0, + "1516": 99.0, + "1517": 109.0, + "1518": 106.0, + "1519": 88.0, + "1520": 89.0, + "1521": 101.0, + "1522": 112.0, + "1523": 88.0, + "1524": 113.0, + "1525": 94.0, + "1526": 110.0, + "1527": 112.0, + "1528": 84.0, + "1529": 91.0, + "1530": 114.0, + "1531": 113.0, + "1532": 119.0, + "1533": 95.0, + "1534": 112.0, + "1535": 112.0, + "1536": 109.0, + "1537": 97.0, + "1538": 111.0, + "1539": 115.0, + "1540": 114.0, + "1541": 88.0, + "1542": 126.0, + "1543": 97.0, + "1544": 84.0, + "1545": 105.0, + "1546": 82.0, + "1547": 93.0, + "1548": 90.0, + "1549": 99.0, + "1550": 93.0, + "1551": 98.0, + "1552": 86.0, + "1553": 120.0, + "1554": 109.0, + "1555": 111.0, + "1556": 98.0, + "1557": 90.0, + "1558": 120.0, + "1559": 84.0, + "1560": 107.0, + "1561": 103.0, + "1562": 121.0, + "1563": 116.0, + "1564": 113.0, + "1565": 114.0, + "1566": 113.0, + "1567": 102.0, + "1568": 91.0, + "1569": 122.0, + "1570": 95.0, + "1571": 115.0, + "1572": 102.0, + "1573": 100.0, + "1574": 121.0, + "1575": 108.0, + "1576": 88.0, + "1577": 116.0, + "1578": 101.0, + "1579": 98.0, + "1580": 114.0, + "1581": 102.0, + "1582": 108.0, + "1583": 115.0, + "1584": 70.0, + "1585": 112.0, + "1586": 120.0, + "1587": 101.0, + "1588": 118.0, + "1589": 99.0, + "1590": 103.0, + "1591": 108.0, + "1592": 106.0, + "1593": 121.0, + "1594": 110.0, + "1595": 103.0, + "1596": 117.0, + "1597": 115.0, + "1598": 105.0, + "1599": 76.0, + "1600": 90.0, + "1601": 108.0, + "1602": 105.0, + "1603": 122.0, + "1604": 113.0, + "1605": 122.0, + "1606": 117.0, + "1607": 92.0, + "1608": 118.0, + "1609": 115.0, + "1610": 103.0, + "1611": 117.0, + "1612": 106.0, + "1613": 106.0, + "1614": 104.0, + "1615": 114.0, + "1616": 88.0, + "1617": 97.0, + "1618": 111.0, + "1619": 107.0, + "1620": 112.0, + "1621": 91.0, + "1622": 130.0, + "1623": 109.0, + "1624": 102.0, + "1625": 121.0, + "1626": 100.0, + "1627": 119.0, + "1628": 99.0, + "1629": 119.0, + "1630": 117.0, + "1631": 105.0, + "1632": 116.0, + "1633": 112.0, + "1634": 120.0, + "1635": 99.0, + "1636": 105.0, + "1637": 94.0, + "1638": 107.0, + "1639": 97.0, + "1640": 106.0, + "1641": 120.0, + "1642": 101.0, + "1643": 135.0, + "1644": 117.0, + "1645": 110.0, + "1646": 106.0, + "1647": 127.0, + "1648": 82.0, + "1649": 114.0, + "1650": 121.0, + "1651": 107.0, + "1652": 100.0, + "1653": 108.0, + "1654": 114.0, + "1655": 92.0, + "1656": 80.0, + "1657": 110.0, + "1658": 114.0, + "1659": 105.0, + "1660": 104.0, + "1661": 102.0, + "1662": 124.0, + "1663": 96.0, + "1664": 127.0, + "1665": 89.0, + "1666": 115.0, + "1667": 114.0, + "1668": 122.0, + "1669": 94.0, + "1670": 114.0, + "1671": 102.0, + "1672": 99.0, + "1673": 109.0, + "1674": 117.0, + "1675": 105.0, + "1676": 116.0, + "1677": 101.0, + "1678": 110.0, + "1679": 112.0, + "1680": 96.0, + "1681": 93.0, + "1682": 97.0, + "1683": 106.0, + "1684": 103.0, + "1685": 101.0, + "1686": 109.0, + "1687": 104.0, + "1688": 127.0, + "1689": 88.0, + "1690": 98.0, + "1691": 90.0, + "1692": 107.0, + "1693": 111.0, + "1694": 125.0, + "1695": 129.0, + "1696": 112.0, + "1697": 126.0, + "1698": 104.0, + "1699": 124.0, + "1700": 112.0, + "1701": 120.0, + "1702": 89.0, + "1703": 103.0, + "1704": 103.0, + "1705": 111.0, + "1706": 124.0, + "1707": 93.0, + "1708": 96.0, + "1709": 116.0, + "1710": 133.0, + "1711": 107.0, + "1712": 100.0, + "1713": 91.0, + "1714": 122.0, + "1715": 108.0, + "1716": 110.0, + "1717": 121.0, + "1718": 101.0, + "1719": 110.0, + "1720": 121.0, + "1721": 109.0, + "1722": 96.0, + "1723": 125.0, + "1724": 118.0, + "1725": 122.0, + "1726": 113.0, + "1727": 99.0, + "1728": 98.0, + "1729": 115.0, + "1730": 106.0, + "1731": 96.0, + "1732": 95.0, + "1733": 115.0, + "1734": 106.0, + "1735": 102.0, + "1736": 104.0, + "1737": 122.0, + "1738": 94.0, + "1739": 92.0, + "1740": 105.0, + "1741": 113.0, + "1742": 129.0, + "1743": 113.0, + "1744": 110.0, + "1745": 113.0, + "1746": 127.0, + "1747": 108.0, + "1748": 120.0, + "1749": 115.0, + "1750": 104.0, + "1751": 114.0, + "1752": 122.0, + "1753": 113.0, + "1754": 123.0, + "1755": 114.0, + "1756": 115.0, + "1757": 126.0, + "1758": 105.0, + "1759": 109.0, + "1760": 136.0, + "1761": 111.0, + "1762": 104.0, + "1763": 104.0, + "1764": 105.0, + "1765": 133.0, + "1766": 118.0, + "1767": 108.0, + "1768": 114.0, + "1769": 105.0, + "1770": 98.0, + "1771": 112.0, + "1772": 92.0, + "1773": 77.0, + "1774": 130.0, + "1775": 104.0, + "1776": 85.0, + "1777": 106.0, + "1778": 84.0, + "1779": 111.0, + "1780": 109.0, + "1781": 124.0, + "1782": 109.0, + "1783": 128.0, + "1784": 117.0, + "1785": 118.0, + "1786": 111.0, + "1787": 112.0, + "1788": 104.0, + "1789": 135.0, + "1790": 105.0, + "1791": 115.0, + "1792": 130.0, + "1793": 119.0, + "1794": 128.0, + "1795": 110.0, + "1796": 130.0, + "1797": 97.0, + "1798": 139.0, + "1799": 104.0, + "1800": 103.0, + "1801": 94.0, + "1802": 134.0, + "1803": 117.0, + "1804": 139.0, + "1805": 124.0, + "1806": 127.0, + "1807": 128.0, + "1808": 99.0, + "1809": 92.0, + "1810": 116.0, + "1811": 104.0, + "1812": 103.0, + "1813": 122.0, + "1814": 129.0, + "1815": 94.0, + "1816": 104.0, + "1817": 98.0, + "1818": 128.0, + "1819": 112.0, + "1820": 99.0, + "1821": 126.0, + "1822": 83.0, + "1823": 117.0, + "1824": 96.0, + "1825": 95.0, + "1826": 127.0, + "1827": 124.0, + "1828": 120.0, + "1829": 110.0, + "1830": 123.0, + "1831": 110.0, + "1832": 92.0, + "1833": 100.0, + "1834": 113.0, + "1835": 120.0, + "1836": 113.0, + "1837": 114.0, + "1838": 99.0, + "1839": 123.0, + "1840": 109.0, + "1841": 95.0, + "1842": 101.0, + "1843": 122.0, + "1844": 113.0, + "1845": 127.0, + "1846": 100.0, + "1847": 117.0, + "1848": 133.0, + "1849": 87.0, + "1850": 103.0, + "1851": 89.0, + "1852": 99.0, + "1853": 93.0, + "1854": 99.0, + "1855": 107.0, + "1856": 111.0, + "1857": 121.0, + "1858": 92.0, + "1859": 105.0, + "1860": 115.0, + "1861": 92.0, + "1862": 91.0, + "1863": 112.0, + "1864": 109.0, + "1865": 125.0, + "1866": 124.0, + "1867": 110.0, + "1868": 113.0, + "1869": 119.0, + "1870": 137.0, + "1871": 126.0, + "1872": 95.0, + "1873": 119.0, + "1874": 105.0, + "1875": 128.0, + "1876": 104.0, + "1877": 120.0, + "1878": 95.0, + "1879": 99.0, + "1880": 123.0, + "1881": 99.0, + "1882": 97.0, + "1883": 101.0, + "1884": 115.0, + "1885": 106.0, + "1886": 123.0, + "1887": 121.0, + "1888": 121.0, + "1889": 114.0, + "1890": 100.0, + "1891": 110.0, + "1892": 107.0, + "1893": 113.0, + "1894": 134.0, + "1895": 114.0, + "1896": 111.0, + "1897": 122.0, + "1898": 108.0, + "1899": 94.0, + "1900": 123.0, + "1901": 125.0, + "1902": 115.0, + "1903": 112.0, + "1904": 113.0, + "1905": 109.0, + "1906": 115.0, + "1907": 95.0, + "1908": 113.0, + "1909": 79.0, + "1910": 97.0, + "1911": 135.0, + "1912": 122.0, + "1913": 105.0, + "1914": 112.0, + "1915": 129.0, + "1916": 117.0, + "1917": 115.0, + "1918": 113.0, + "1919": 117.0, + "1920": 122.0, + "1921": 105.0, + "1922": 86.0, + "1923": 113.0, + "1924": 111.0, + "1925": 110.0, + "1926": 112.0, + "1927": 103.0, + "1928": 108.0, + "1929": 113.0, + "1930": 121.0, + "1931": 111.0, + "1932": 106.0, + "1933": 114.0, + "1934": 117.0, + "1935": 93.0, + "1936": 109.0, + "1937": 121.0, + "1938": 108.0, + "1939": 132.0, + "1940": 127.0, + "1941": 126.0, + "1942": 101.0, + "1943": 120.0, + "1944": 87.0, + "1945": 114.0, + "1946": 105.0, + "1947": 109.0, + "1948": 109.0, + "1949": 106.0, + "1950": 111.0, + "1951": 120.0, + "1952": 104.0, + "1953": 113.0, + "1954": 116.0, + "1955": 131.0, + "1956": 91.0, + "1957": 118.0, + "1958": 139.0, + "1959": 114.0, + "1960": 96.0, + "1961": 109.0, + "1962": 113.0, + "1963": 125.0, + "1964": 112.0, + "1965": 108.0, + "1966": 130.0, + "1967": 120.0, + "1968": 110.0, + "1969": 96.0, + "1970": 110.0, + "1971": 121.0, + "1972": 104.0, + "1973": 103.0, + "1974": 110.0, + "1975": 101.0, + "1976": 144.0, + "1977": 122.0, + "1978": 118.0, + "1979": 121.0, + "1980": 115.0, + "1981": 114.0, + "1982": 136.0, + "1983": 123.0, + "1984": 112.0, + "1985": 116.0, + "1986": 104.0, + "1987": 133.0, + "1988": 107.0, + "1989": 100.0, + "1990": 112.0, + "1991": 119.0, + "1992": 103.0, + "1993": 133.0, + "1994": 123.0, + "1995": 118.0, + "1996": 109.0, + "1997": 119.0, + "1998": 107.0, + "1999": 119.0, + "2000": 134.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 442918400.0, + "2": 442918400.0, + "3": 442918400.0, + "4": 442918400.0, + "5": 442918400.0, + "6": 442918400.0, + "7": 442918400.0, + "8": 442918400.0, + "9": 442918400.0, + "10": 442918400.0, + "11": 442918400.0, + "12": 442918400.0, + "13": 442918400.0, + "14": 442918400.0, + "15": 442918400.0, + "16": 442918400.0, + "17": 442918400.0, + "18": 442918400.0, + "19": 442918400.0, + "20": 442918400.0, + "21": 442918400.0, + "22": 442918400.0, + "23": 442918400.0, + "24": 442918400.0, + "25": 442918400.0, + "26": 442918400.0, + "27": 442918400.0, + "28": 442918400.0, + "29": 442918400.0, + "30": 442918400.0, + "31": 442918400.0, + "32": 442918400.0, + "33": 442918400.0, + "34": 442918400.0, + "35": 442918400.0, + "36": 442918400.0, + "37": 442918400.0, + "38": 442918400.0, + "39": 442918400.0, + "40": 442918400.0, + "41": 442918400.0, + "42": 442918400.0, + "43": 442918400.0, + "44": 442918400.0, + "45": 442918400.0, + "46": 442918400.0, + "47": 442918400.0, + "48": 442918400.0, + "49": 442918400.0, + "50": 442918400.0, + "51": 442918400.0, + "52": 442918400.0, + "53": 442918400.0, + "54": 442918400.0, + "55": 442918400.0, + "56": 442918400.0, + "57": 442918400.0, + "58": 442918400.0, + "59": 442918400.0, + "60": 442918400.0, + "61": 442918400.0, + "62": 442918400.0, + "63": 442918400.0, + "64": 442918400.0, + "65": 442918400.0, + "66": 442918400.0, + "67": 442918400.0, + "68": 442918400.0, + "69": 442918400.0, + "70": 442918400.0, + "71": 442918400.0, + "72": 442918400.0, + "73": 442918400.0, + "74": 442918400.0, + "75": 442918400.0, + "76": 442918400.0, + "77": 442918400.0, + "78": 442918400.0, + "79": 442918400.0, + "80": 442918400.0, + "81": 442918400.0, + "82": 442918400.0, + "83": 442918400.0, + "84": 442918400.0, + "85": 442918400.0, + "86": 442918400.0, + "87": 442918400.0, + "88": 442918400.0, + "89": 442918400.0, + "90": 442918400.0, + "91": 442918400.0, + "92": 442918400.0, + "93": 442918400.0, + "94": 442918400.0, + "95": 442918400.0, + "96": 442918400.0, + "97": 442918400.0, + "98": 442918400.0, + "99": 442918400.0, + "100": 442918400.0, + "101": 442918400.0, + "102": 442918400.0, + "103": 442918400.0, + "104": 442918400.0, + "105": 442918400.0, + "106": 442918400.0, + "107": 442918400.0, + "108": 442918400.0, + "109": 442918400.0, + "110": 442918400.0, + "111": 442918400.0, + "112": 442918400.0, + "113": 442918400.0, + "114": 442918400.0, + "115": 442918400.0, + "116": 442918400.0, + "117": 442918400.0, + "118": 442918400.0, + "119": 442918400.0, + "120": 442918400.0, + "121": 442918400.0, + "122": 442918400.0, + "123": 442918400.0, + "124": 442918400.0, + "125": 442918400.0, + "126": 442918400.0, + "127": 442918400.0, + "128": 442918400.0, + "129": 442918400.0, + "130": 442918400.0, + "131": 442918400.0, + "132": 442918400.0, + "133": 442918400.0, + "134": 442918400.0, + "135": 442918400.0, + "136": 442918400.0, + "137": 442918400.0, + "138": 442918400.0, + "139": 442918400.0, + "140": 442918400.0, + "141": 442918400.0, + "142": 442918400.0, + "143": 442918400.0, + "144": 442918400.0, + "145": 442918400.0, + "146": 442918400.0, + "147": 442918400.0, + "148": 442918400.0, + "149": 442918400.0, + "150": 442918400.0, + "151": 442918400.0, + "152": 442918400.0, + "153": 442918400.0, + "154": 442918400.0, + "155": 442918400.0, + "156": 442918400.0, + "157": 442918400.0, + "158": 442918400.0, + "159": 442918400.0, + "160": 442918400.0, + "161": 442918400.0, + "162": 442918400.0, + "163": 442918400.0, + "164": 442918400.0, + "165": 442918400.0, + "166": 442918400.0, + "167": 442918400.0, + "168": 442918400.0, + "169": 442918400.0, + "170": 442918400.0, + "171": 442918400.0, + "172": 442918400.0, + "173": 442918400.0, + "174": 442918400.0, + "175": 442918400.0, + "176": 442918400.0, + "177": 442918400.0, + "178": 442918400.0, + "179": 442918400.0, + "180": 442918400.0, + "181": 442918400.0, + "182": 442918400.0, + "183": 442918400.0, + "184": 442918400.0, + "185": 442918400.0, + "186": 442918400.0, + "187": 442918400.0, + "188": 442918400.0, + "189": 442918400.0, + "190": 442918400.0, + "191": 442918400.0, + "192": 442918400.0, + "193": 442918400.0, + "194": 442918400.0, + "195": 442918400.0, + "196": 442918400.0, + "197": 442918400.0, + "198": 442918400.0, + "199": 442918400.0, + "200": 442918400.0, + "201": 442918400.0, + "202": 442918400.0, + "203": 442918400.0, + "204": 442918400.0, + "205": 442918400.0, + "206": 442918400.0, + "207": 442918400.0, + "208": 442918400.0, + "209": 442918400.0, + "210": 442918400.0, + "211": 442918400.0, + "212": 442918400.0, + "213": 442918400.0, + "214": 442918400.0, + "215": 442918400.0, + "216": 442918400.0, + "217": 442918400.0, + "218": 442918400.0, + "219": 442918400.0, + "220": 442918400.0, + "221": 442918400.0, + "222": 442918400.0, + "223": 442918400.0, + "224": 442918400.0, + "225": 442918400.0, + "226": 442918400.0, + "227": 442918400.0, + "228": 442918400.0, + "229": 442918400.0, + "230": 442918400.0, + "231": 442918400.0, + "232": 442918400.0, + "233": 442918400.0, + "234": 442918400.0, + "235": 442918400.0, + "236": 442918400.0, + "237": 442918400.0, + "238": 442918400.0, + "239": 442918400.0, + "240": 442918400.0, + "241": 442918400.0, + "242": 442918400.0, + "243": 442918400.0, + "244": 442918400.0, + "245": 442918400.0, + "246": 442918400.0, + "247": 442918400.0, + "248": 442918400.0, + "249": 442918400.0, + "250": 442918400.0, + "251": 442918400.0, + "252": 442918400.0, + "253": 442918400.0, + "254": 442918400.0, + "255": 442918400.0, + "256": 442918400.0, + "257": 442918400.0, + "258": 442918400.0, + "259": 442918400.0, + "260": 442918400.0, + "261": 442918400.0, + "262": 442918400.0, + "263": 442918400.0, + "264": 442918400.0, + "265": 442918400.0, + "266": 442918400.0, + "267": 442918400.0, + "268": 442918400.0, + "269": 442918400.0, + "270": 442918400.0, + "271": 442918400.0, + "272": 442918400.0, + "273": 442918400.0, + "274": 442918400.0, + "275": 442918400.0, + "276": 442918400.0, + "277": 442918400.0, + "278": 442918400.0, + "279": 442918400.0, + "280": 442918400.0, + "281": 442918400.0, + "282": 442918400.0, + "283": 442918400.0, + "284": 442918400.0, + "285": 442918400.0, + "286": 442918400.0, + "287": 442918400.0, + "288": 442918400.0, + "289": 442918400.0, + "290": 442918400.0, + "291": 442918400.0, + "292": 442918400.0, + "293": 442918400.0, + "294": 442918400.0, + "295": 442918400.0, + "296": 442918400.0, + "297": 442918400.0, + "298": 442918400.0, + "299": 442918400.0, + "300": 442918400.0, + "301": 442918400.0, + "302": 442918400.0, + "303": 442918400.0, + "304": 442918400.0, + "305": 442918400.0, + "306": 442918400.0, + "307": 442918400.0, + "308": 442918400.0, + "309": 442918400.0, + "310": 442918400.0, + "311": 442918400.0, + "312": 442918400.0, + "313": 442918400.0, + "314": 442918400.0, + "315": 442918400.0, + "316": 442918400.0, + "317": 442918400.0, + "318": 442918400.0, + "319": 442918400.0, + "320": 442918400.0, + "321": 442918400.0, + "322": 442918400.0, + "323": 442918400.0, + "324": 442918400.0, + "325": 442918400.0, + "326": 442918400.0, + "327": 442918400.0, + "328": 442918400.0, + "329": 442918400.0, + "330": 442918400.0, + "331": 442918400.0, + "332": 442918400.0, + "333": 442918400.0, + "334": 442918400.0, + "335": 442918400.0, + "336": 442918400.0, + "337": 442918400.0, + "338": 442918400.0, + "339": 442918400.0, + "340": 442918400.0, + "341": 442918400.0, + "342": 442918400.0, + "343": 442918400.0, + "344": 442918400.0, + "345": 442918400.0, + "346": 442918400.0, + "347": 442918400.0, + "348": 442918400.0, + "349": 442918400.0, + "350": 442918400.0, + "351": 442918400.0, + "352": 442918400.0, + "353": 442918400.0, + "354": 442918400.0, + "355": 442918400.0, + "356": 442918400.0, + "357": 442918400.0, + "358": 442918400.0, + "359": 442918400.0, + "360": 442918400.0, + "361": 442918400.0, + "362": 442918400.0, + "363": 442918400.0, + "364": 442918400.0, + "365": 442918400.0, + "366": 442918400.0, + "367": 442918400.0, + "368": 442918400.0, + "369": 442918400.0, + "370": 442918400.0, + "371": 442918400.0, + "372": 442918400.0, + "373": 442918400.0, + "374": 442918400.0, + "375": 442918400.0, + "376": 442918400.0, + "377": 442918400.0, + "378": 442918400.0, + "379": 442918400.0, + "380": 442918400.0, + "381": 442918400.0, + "382": 442918400.0, + "383": 442918400.0, + "384": 442918400.0, + "385": 442918400.0, + "386": 442918400.0, + "387": 442918400.0, + "388": 442918400.0, + "389": 442918400.0, + "390": 442918400.0, + "391": 442918400.0, + "392": 442918400.0, + "393": 442918400.0, + "394": 442918400.0, + "395": 442918400.0, + "396": 442918400.0, + "397": 442918400.0, + "398": 442918400.0, + "399": 442918400.0, + "400": 442918400.0, + "401": 442918400.0, + "402": 442918400.0, + "403": 442918400.0, + "404": 442918400.0, + "405": 442918400.0, + "406": 442918400.0, + "407": 442918400.0, + "408": 442918400.0, + "409": 442918400.0, + "410": 442918400.0, + "411": 442918400.0, + "412": 442918400.0, + "413": 442918400.0, + "414": 442918400.0, + "415": 442918400.0, + "416": 442918400.0, + "417": 442918400.0, + "418": 442918400.0, + "419": 442918400.0, + "420": 442918400.0, + "421": 442918400.0, + "422": 442918400.0, + "423": 442918400.0, + "424": 442918400.0, + "425": 442918400.0, + "426": 442918400.0, + "427": 442918400.0, + "428": 442918400.0, + "429": 442918400.0, + "430": 442918400.0, + "431": 442918400.0, + "432": 442918400.0, + "433": 442918400.0, + "434": 442918400.0, + "435": 442918400.0, + "436": 442918400.0, + "437": 442918400.0, + "438": 442918400.0, + "439": 442918400.0, + "440": 442918400.0, + "441": 442918400.0, + "442": 442918400.0, + "443": 442918400.0, + "444": 442918400.0, + "445": 442918400.0, + "446": 442918400.0, + "447": 442918400.0, + "448": 442918400.0, + "449": 442918400.0, + "450": 442918400.0, + "451": 442918400.0, + "452": 442918400.0, + "453": 442918400.0, + "454": 442918400.0, + "455": 442918400.0, + "456": 442918400.0, + "457": 442918400.0, + "458": 442918400.0, + "459": 442918400.0, + "460": 442918400.0, + "461": 442918400.0, + "462": 442918400.0, + "463": 442918400.0, + "464": 442918400.0, + "465": 442918400.0, + "466": 442918400.0, + "467": 442918400.0, + "468": 442918400.0, + "469": 442918400.0, + "470": 442918400.0, + "471": 442918400.0, + "472": 442918400.0, + "473": 442918400.0, + "474": 442918400.0, + "475": 442918400.0, + "476": 442918400.0, + "477": 442918400.0, + "478": 442918400.0, + "479": 442918400.0, + "480": 442918400.0, + "481": 442918400.0, + "482": 442918400.0, + "483": 442918400.0, + "484": 442918400.0, + "485": 442918400.0, + "486": 442918400.0, + "487": 442918400.0, + "488": 442918400.0, + "489": 442918400.0, + "490": 442918400.0, + "491": 442918400.0, + "492": 442918400.0, + "493": 442918400.0, + "494": 442918400.0, + "495": 442918400.0, + "496": 442918400.0, + "497": 442918400.0, + "498": 442918400.0, + "499": 442918400.0, + "500": 442918400.0, + "501": 442918400.0, + "502": 442918400.0, + "503": 442918400.0, + "504": 442918400.0, + "505": 442918400.0, + "506": 442918400.0, + "507": 442918400.0, + "508": 442918400.0, + "509": 442918400.0, + "510": 442918400.0, + "511": 442918400.0, + "512": 442918400.0, + "513": 442918400.0, + "514": 442918400.0, + "515": 442918400.0, + "516": 442918400.0, + "517": 442918400.0, + "518": 442918400.0, + "519": 442918400.0, + "520": 442918400.0, + "521": 442918400.0, + "522": 442918400.0, + "523": 442918400.0, + "524": 442918400.0, + "525": 442918400.0, + "526": 442918400.0, + "527": 442918400.0, + "528": 442918400.0, + "529": 442918400.0, + "530": 442918400.0, + "531": 442918400.0, + "532": 442918400.0, + "533": 442918400.0, + "534": 442918400.0, + "535": 442918400.0, + "536": 442918400.0, + "537": 442918400.0, + "538": 442918400.0, + "539": 442918400.0, + "540": 442918400.0, + "541": 442918400.0, + "542": 442918400.0, + "543": 442918400.0, + "544": 442918400.0, + "545": 442918400.0, + "546": 442918400.0, + "547": 442918400.0, + "548": 442918400.0, + "549": 442918400.0, + "550": 442918400.0, + "551": 442918400.0, + "552": 442918400.0, + "553": 442918400.0, + "554": 442918400.0, + "555": 442918400.0, + "556": 442918400.0, + "557": 442918400.0, + "558": 442918400.0, + "559": 442918400.0, + "560": 442918400.0, + "561": 442918400.0, + "562": 442918400.0, + "563": 442918400.0, + "564": 442918400.0, + "565": 442918400.0, + "566": 442918400.0, + "567": 442918400.0, + "568": 442918400.0, + "569": 442918400.0, + "570": 442918400.0, + "571": 442918400.0, + "572": 442918400.0, + "573": 442918400.0, + "574": 442918400.0, + "575": 442918400.0, + "576": 442918400.0, + "577": 442918400.0, + "578": 442918400.0, + "579": 442918400.0, + "580": 442918400.0, + "581": 442918400.0, + "582": 442918400.0, + "583": 442918400.0, + "584": 442918400.0, + "585": 442918400.0, + "586": 442918400.0, + "587": 442918400.0, + "588": 442918400.0, + "589": 442918400.0, + "590": 442918400.0, + "591": 442918400.0, + "592": 442918400.0, + "593": 442918400.0, + "594": 442918400.0, + "595": 442918400.0, + "596": 442918400.0, + "597": 442918400.0, + "598": 442918400.0, + "599": 442918400.0, + "600": 442918400.0, + "601": 442918400.0, + "602": 442918400.0, + "603": 442918400.0, + "604": 442918400.0, + "605": 442918400.0, + "606": 442918400.0, + "607": 442918400.0, + "608": 442918400.0, + "609": 442918400.0, + "610": 442918400.0, + "611": 442918400.0, + "612": 442918400.0, + "613": 442918400.0, + "614": 442918400.0, + "615": 442918400.0, + "616": 442918400.0, + "617": 442918400.0, + "618": 442918400.0, + "619": 442918400.0, + "620": 442918400.0, + "621": 442918400.0, + "622": 442918400.0, + "623": 442918400.0, + "624": 442918400.0, + "625": 442918400.0, + "626": 442918400.0, + "627": 442918400.0, + "628": 442918400.0, + "629": 442918400.0, + "630": 442918400.0, + "631": 442918400.0, + "632": 442918400.0, + "633": 442918400.0, + "634": 442918400.0, + "635": 442918400.0, + "636": 442918400.0, + "637": 442918400.0, + "638": 442918400.0, + "639": 442918400.0, + "640": 442918400.0, + "641": 442918400.0, + "642": 442918400.0, + "643": 442918400.0, + "644": 442918400.0, + "645": 442918400.0, + "646": 442918400.0, + "647": 442918400.0, + "648": 442918400.0, + "649": 442918400.0, + "650": 442918400.0, + "651": 442918400.0, + "652": 442918400.0, + "653": 442918400.0, + "654": 442918400.0, + "655": 442918400.0, + "656": 442918400.0, + "657": 442918400.0, + "658": 442918400.0, + "659": 442918400.0, + "660": 442918400.0, + "661": 442918400.0, + "662": 442918400.0, + "663": 442918400.0, + "664": 442918400.0, + "665": 442918400.0, + "666": 442918400.0, + "667": 442918400.0, + "668": 442918400.0, + "669": 442918400.0, + "670": 442918400.0, + "671": 442918400.0, + "672": 442918400.0, + "673": 442918400.0, + "674": 442918400.0, + "675": 442918400.0, + "676": 442918400.0, + "677": 442918400.0, + "678": 442918400.0, + "679": 442918400.0, + "680": 442918400.0, + "681": 442918400.0, + "682": 442918400.0, + "683": 442918400.0, + "684": 442918400.0, + "685": 442918400.0, + "686": 442918400.0, + "687": 442918400.0, + "688": 442918400.0, + "689": 442918400.0, + "690": 442918400.0, + "691": 442918400.0, + "692": 442918400.0, + "693": 442918400.0, + "694": 442918400.0, + "695": 442918400.0, + "696": 442918400.0, + "697": 442918400.0, + "698": 442918400.0, + "699": 442918400.0, + "700": 442918400.0, + "701": 442918400.0, + "702": 442918400.0, + "703": 442918400.0, + "704": 442918400.0, + "705": 442918400.0, + "706": 442918400.0, + "707": 442918400.0, + "708": 442918400.0, + "709": 442918400.0, + "710": 442918400.0, + "711": 442918400.0, + "712": 442918400.0, + "713": 442918400.0, + "714": 442918400.0, + "715": 442918400.0, + "716": 442918400.0, + "717": 442918400.0, + "718": 442918400.0, + "719": 442918400.0, + "720": 442918400.0, + "721": 442918400.0, + "722": 442918400.0, + "723": 442918400.0, + "724": 442918400.0, + "725": 442918400.0, + "726": 442918400.0, + "727": 442918400.0, + "728": 442918400.0, + "729": 442918400.0, + "730": 442918400.0, + "731": 442918400.0, + "732": 442918400.0, + "733": 442918400.0, + "734": 442918400.0, + "735": 442918400.0, + "736": 442918400.0, + "737": 442918400.0, + "738": 442918400.0, + "739": 442918400.0, + "740": 442918400.0, + "741": 442918400.0, + "742": 442918400.0, + "743": 442918400.0, + "744": 442918400.0, + "745": 442918400.0, + "746": 442918400.0, + "747": 442918400.0, + "748": 442918400.0, + "749": 442918400.0, + "750": 442918400.0, + "751": 442918400.0, + "752": 442918400.0, + "753": 442918400.0, + "754": 442918400.0, + "755": 442918400.0, + "756": 442918400.0, + "757": 442918400.0, + "758": 442918400.0, + "759": 442918400.0, + "760": 442918400.0, + "761": 442918400.0, + "762": 442918400.0, + "763": 442918400.0, + "764": 442918400.0, + "765": 442918400.0, + "766": 442918400.0, + "767": 442918400.0, + "768": 442918400.0, + "769": 442918400.0, + "770": 442918400.0, + "771": 442918400.0, + "772": 442918400.0, + "773": 442918400.0, + "774": 442918400.0, + "775": 442918400.0, + "776": 442918400.0, + "777": 442918400.0, + "778": 442918400.0, + "779": 442918400.0, + "780": 442918400.0, + "781": 442918400.0, + "782": 442918400.0, + "783": 442918400.0, + "784": 442918400.0, + "785": 442918400.0, + "786": 442918400.0, + "787": 442918400.0, + "788": 442918400.0, + "789": 442918400.0, + "790": 442918400.0, + "791": 442918400.0, + "792": 442918400.0, + "793": 442918400.0, + "794": 442918400.0, + "795": 442918400.0, + "796": 442918400.0, + "797": 442918400.0, + "798": 442918400.0, + "799": 442918400.0, + "800": 442918400.0, + "801": 442918400.0, + "802": 442918400.0, + "803": 442918400.0, + "804": 442918400.0, + "805": 442918400.0, + "806": 442918400.0, + "807": 442918400.0, + "808": 442918400.0, + "809": 442918400.0, + "810": 442918400.0, + "811": 442918400.0, + "812": 442918400.0, + "813": 442918400.0, + "814": 442918400.0, + "815": 442918400.0, + "816": 442918400.0, + "817": 442918400.0, + "818": 442918400.0, + "819": 442918400.0, + "820": 442918400.0, + "821": 442918400.0, + "822": 442918400.0, + "823": 442918400.0, + "824": 442918400.0, + "825": 442918400.0, + "826": 442918400.0, + "827": 442918400.0, + "828": 442918400.0, + "829": 442918400.0, + "830": 442918400.0, + "831": 442918400.0, + "832": 442918400.0, + "833": 442918400.0, + "834": 442918400.0, + "835": 442918400.0, + "836": 442918400.0, + "837": 442918400.0, + "838": 442918400.0, + "839": 442918400.0, + "840": 442918400.0, + "841": 442918400.0, + "842": 442918400.0, + "843": 442918400.0, + "844": 442918400.0, + "845": 442918400.0, + "846": 442918400.0, + "847": 442918400.0, + "848": 442918400.0, + "849": 442918400.0, + "850": 442918400.0, + "851": 442918400.0, + "852": 442918400.0, + "853": 442918400.0, + "854": 442918400.0, + "855": 442918400.0, + "856": 442918400.0, + "857": 442918400.0, + "858": 442918400.0, + "859": 442918400.0, + "860": 442918400.0, + "861": 442918400.0, + "862": 442918400.0, + "863": 442918400.0, + "864": 442918400.0, + "865": 442918400.0, + "866": 442918400.0, + "867": 442918400.0, + "868": 442918400.0, + "869": 442918400.0, + "870": 442918400.0, + "871": 442918400.0, + "872": 442918400.0, + "873": 442918400.0, + "874": 442918400.0, + "875": 442918400.0, + "876": 442918400.0, + "877": 442918400.0, + "878": 442918400.0, + "879": 442918400.0, + "880": 442918400.0, + "881": 442918400.0, + "882": 442918400.0, + "883": 442918400.0, + "884": 442918400.0, + "885": 442918400.0, + "886": 442918400.0, + "887": 442918400.0, + "888": 442918400.0, + "889": 442918400.0, + "890": 442918400.0, + "891": 442918400.0, + "892": 442918400.0, + "893": 442918400.0, + "894": 442918400.0, + "895": 442918400.0, + "896": 442918400.0, + "897": 442918400.0, + "898": 442918400.0, + "899": 442918400.0, + "900": 442918400.0, + "901": 442918400.0, + "902": 442918400.0, + "903": 442918400.0, + "904": 442918400.0, + "905": 442918400.0, + "906": 442918400.0, + "907": 442918400.0, + "908": 442918400.0, + "909": 442918400.0, + "910": 442918400.0, + "911": 442918400.0, + "912": 442918400.0, + "913": 442918400.0, + "914": 442918400.0, + "915": 442918400.0, + "916": 442918400.0, + "917": 442918400.0, + "918": 442918400.0, + "919": 442918400.0, + "920": 442918400.0, + "921": 442918400.0, + "922": 442918400.0, + "923": 442918400.0, + "924": 442918400.0, + "925": 442918400.0, + "926": 442918400.0, + "927": 442918400.0, + "928": 442918400.0, + "929": 442918400.0, + "930": 442918400.0, + "931": 442918400.0, + "932": 442918400.0, + "933": 442918400.0, + "934": 442918400.0, + "935": 442918400.0, + "936": 442918400.0, + "937": 442918400.0, + "938": 442918400.0, + "939": 442918400.0, + "940": 442918400.0, + "941": 442918400.0, + "942": 442918400.0, + "943": 442918400.0, + "944": 442918400.0, + "945": 442918400.0, + "946": 442918400.0, + "947": 442918400.0, + "948": 442918400.0, + "949": 442918400.0, + "950": 442918400.0, + "951": 442918400.0, + "952": 442918400.0, + "953": 442918400.0, + "954": 442918400.0, + "955": 442918400.0, + "956": 442918400.0, + "957": 442918400.0, + "958": 442918400.0, + "959": 442918400.0, + "960": 442918400.0, + "961": 442918400.0, + "962": 442918400.0, + "963": 442918400.0, + "964": 442918400.0, + "965": 442918400.0, + "966": 442918400.0, + "967": 442918400.0, + "968": 442918400.0, + "969": 442918400.0, + "970": 442918400.0, + "971": 442918400.0, + "972": 442918400.0, + "973": 442918400.0, + "974": 442918400.0, + "975": 442918400.0, + "976": 442918400.0, + "977": 442918400.0, + "978": 442918400.0, + "979": 442918400.0, + "980": 442918400.0, + "981": 442918400.0, + "982": 442918400.0, + "983": 442918400.0, + "984": 442918400.0, + "985": 442918400.0, + "986": 442918400.0, + "987": 442918400.0, + "988": 442918400.0, + "989": 442918400.0, + "990": 442918400.0, + "991": 442918400.0, + "992": 442918400.0, + "993": 442918400.0, + "994": 442918400.0, + "995": 442918400.0, + "996": 442918400.0, + "997": 442918400.0, + "998": 442918400.0, + "999": 442918400.0, + "1000": 442918400.0, + "1001": 442918400.0, + "1002": 442918400.0, + "1003": 442918400.0, + "1004": 442918400.0, + "1005": 442918400.0, + "1006": 442918400.0, + "1007": 442918400.0, + "1008": 442918400.0, + "1009": 442918400.0, + "1010": 442918400.0, + "1011": 442918400.0, + "1012": 442918400.0, + "1013": 442918400.0, + "1014": 442918400.0, + "1015": 442918400.0, + "1016": 442918400.0, + "1017": 442918400.0, + "1018": 442918400.0, + "1019": 442918400.0, + "1020": 442918400.0, + "1021": 442918400.0, + "1022": 442918400.0, + "1023": 442918400.0, + "1024": 442918400.0, + "1025": 442918400.0, + "1026": 442918400.0, + "1027": 442918400.0, + "1028": 442918400.0, + "1029": 442918400.0, + "1030": 442918400.0, + "1031": 442918400.0, + "1032": 442918400.0, + "1033": 442918400.0, + "1034": 442918400.0, + "1035": 442918400.0, + "1036": 442918400.0, + "1037": 442918400.0, + "1038": 442918400.0, + "1039": 442918400.0, + "1040": 442918400.0, + "1041": 442918400.0, + "1042": 442918400.0, + "1043": 442918400.0, + "1044": 442918400.0, + "1045": 442918400.0, + "1046": 442918400.0, + "1047": 442918400.0, + "1048": 442918400.0, + "1049": 442918400.0, + "1050": 442918400.0, + "1051": 442918400.0, + "1052": 442918400.0, + "1053": 442918400.0, + "1054": 442918400.0, + "1055": 442918400.0, + "1056": 442918400.0, + "1057": 442918400.0, + "1058": 442918400.0, + "1059": 442918400.0, + "1060": 442918400.0, + "1061": 442918400.0, + "1062": 442918400.0, + "1063": 442918400.0, + "1064": 442918400.0, + "1065": 442918400.0, + "1066": 442918400.0, + "1067": 442918400.0, + "1068": 442918400.0, + "1069": 442918400.0, + "1070": 442918400.0, + "1071": 442918400.0, + "1072": 442918400.0, + "1073": 442918400.0, + "1074": 442918400.0, + "1075": 442918400.0, + "1076": 442918400.0, + "1077": 442918400.0, + "1078": 442918400.0, + "1079": 442918400.0, + "1080": 442918400.0, + "1081": 442918400.0, + "1082": 442918400.0, + "1083": 442918400.0, + "1084": 442918400.0, + "1085": 442918400.0, + "1086": 442918400.0, + "1087": 442918400.0, + "1088": 442918400.0, + "1089": 442918400.0, + "1090": 442918400.0, + "1091": 442918400.0, + "1092": 442918400.0, + "1093": 442918400.0, + "1094": 442918400.0, + "1095": 442918400.0, + "1096": 442918400.0, + "1097": 442918400.0, + "1098": 442918400.0, + "1099": 442918400.0, + "1100": 442918400.0, + "1101": 442918400.0, + "1102": 442918400.0, + "1103": 442918400.0, + "1104": 442918400.0, + "1105": 442918400.0, + "1106": 442918400.0, + "1107": 442918400.0, + "1108": 442918400.0, + "1109": 442918400.0, + "1110": 442918400.0, + "1111": 442918400.0, + "1112": 442918400.0, + "1113": 442918400.0, + "1114": 442918400.0, + "1115": 442918400.0, + "1116": 442918400.0, + "1117": 442918400.0, + "1118": 442918400.0, + "1119": 442918400.0, + "1120": 442918400.0, + "1121": 442918400.0, + "1122": 442918400.0, + "1123": 442918400.0, + "1124": 442918400.0, + "1125": 442918400.0, + "1126": 442918400.0, + "1127": 442918400.0, + "1128": 442918400.0, + "1129": 442918400.0, + "1130": 442918400.0, + "1131": 442918400.0, + "1132": 442918400.0, + "1133": 442918400.0, + "1134": 442918400.0, + "1135": 442918400.0, + "1136": 442918400.0, + "1137": 442918400.0, + "1138": 442918400.0, + "1139": 442918400.0, + "1140": 442918400.0, + "1141": 442918400.0, + "1142": 442918400.0, + "1143": 442918400.0, + "1144": 442918400.0, + "1145": 442918400.0, + "1146": 442918400.0, + "1147": 442918400.0, + "1148": 442918400.0, + "1149": 442918400.0, + "1150": 442918400.0, + "1151": 442918400.0, + "1152": 442918400.0, + "1153": 442918400.0, + "1154": 442918400.0, + "1155": 442918400.0, + "1156": 442918400.0, + "1157": 442918400.0, + "1158": 442918400.0, + "1159": 442918400.0, + "1160": 442918400.0, + "1161": 442918400.0, + "1162": 442918400.0, + "1163": 442918400.0, + "1164": 442918400.0, + "1165": 442918400.0, + "1166": 442918400.0, + "1167": 442918400.0, + "1168": 442918400.0, + "1169": 442918400.0, + "1170": 442918400.0, + "1171": 442918400.0, + "1172": 442918400.0, + "1173": 442918400.0, + "1174": 442918400.0, + "1175": 442918400.0, + "1176": 442918400.0, + "1177": 442918400.0, + "1178": 442918400.0, + "1179": 442918400.0, + "1180": 442918400.0, + "1181": 442918400.0, + "1182": 442918400.0, + "1183": 442918400.0, + "1184": 442918400.0, + "1185": 442918400.0, + "1186": 442918400.0, + "1187": 442918400.0, + "1188": 442918400.0, + "1189": 442918400.0, + "1190": 442918400.0, + "1191": 442918400.0, + "1192": 442918400.0, + "1193": 442918400.0, + "1194": 442918400.0, + "1195": 442918400.0, + "1196": 442918400.0, + "1197": 442918400.0, + "1198": 442918400.0, + "1199": 442918400.0, + "1200": 442918400.0, + "1201": 442918400.0, + "1202": 442918400.0, + "1203": 442918400.0, + "1204": 442918400.0, + "1205": 442918400.0, + "1206": 442918400.0, + "1207": 442918400.0, + "1208": 442918400.0, + "1209": 442918400.0, + "1210": 442918400.0, + "1211": 442918400.0, + "1212": 442918400.0, + "1213": 442918400.0, + "1214": 442918400.0, + "1215": 442918400.0, + "1216": 442918400.0, + "1217": 442918400.0, + "1218": 442918400.0, + "1219": 442918400.0, + "1220": 442918400.0, + "1221": 442918400.0, + "1222": 442918400.0, + "1223": 442918400.0, + "1224": 442918400.0, + "1225": 442918400.0, + "1226": 442918400.0, + "1227": 442918400.0, + "1228": 442918400.0, + "1229": 442918400.0, + "1230": 442918400.0, + "1231": 442918400.0, + "1232": 442918400.0, + "1233": 442918400.0, + "1234": 442918400.0, + "1235": 442918400.0, + "1236": 442918400.0, + "1237": 442918400.0, + "1238": 442918400.0, + "1239": 442918400.0, + "1240": 442918400.0, + "1241": 442918400.0, + "1242": 442918400.0, + "1243": 442918400.0, + "1244": 442918400.0, + "1245": 442918400.0, + "1246": 442918400.0, + "1247": 442918400.0, + "1248": 442918400.0, + "1249": 442918400.0, + "1250": 442918400.0, + "1251": 442918400.0, + "1252": 442918400.0, + "1253": 442918400.0, + "1254": 442918400.0, + "1255": 442918400.0, + "1256": 442918400.0, + "1257": 442918400.0, + "1258": 442918400.0, + "1259": 442918400.0, + "1260": 442918400.0, + "1261": 442918400.0, + "1262": 442918400.0, + "1263": 442918400.0, + "1264": 442918400.0, + "1265": 442918400.0, + "1266": 442918400.0, + "1267": 442918400.0, + "1268": 442918400.0, + "1269": 442918400.0, + "1270": 442918400.0, + "1271": 442918400.0, + "1272": 442918400.0, + "1273": 442918400.0, + "1274": 442918400.0, + "1275": 442918400.0, + "1276": 442918400.0, + "1277": 442918400.0, + "1278": 442918400.0, + "1279": 442918400.0, + "1280": 442918400.0, + "1281": 442918400.0, + "1282": 442918400.0, + "1283": 442918400.0, + "1284": 442918400.0, + "1285": 442918400.0, + "1286": 442918400.0, + "1287": 442918400.0, + "1288": 442918400.0, + "1289": 442918400.0, + "1290": 442918400.0, + "1291": 442918400.0, + "1292": 442918400.0, + "1293": 442918400.0, + "1294": 442918400.0, + "1295": 442918400.0, + "1296": 442918400.0, + "1297": 442918400.0, + "1298": 442918400.0, + "1299": 442918400.0, + "1300": 442918400.0, + "1301": 442918400.0, + "1302": 442918400.0, + "1303": 442918400.0, + "1304": 442918400.0, + "1305": 442918400.0, + "1306": 442918400.0, + "1307": 442918400.0, + "1308": 442918400.0, + "1309": 442918400.0, + "1310": 442918400.0, + "1311": 442918400.0, + "1312": 442918400.0, + "1313": 442918400.0, + "1314": 442918400.0, + "1315": 442918400.0, + "1316": 442918400.0, + "1317": 442918400.0, + "1318": 442918400.0, + "1319": 442918400.0, + "1320": 442918400.0, + "1321": 442918400.0, + "1322": 442918400.0, + "1323": 442918400.0, + "1324": 442918400.0, + "1325": 442918400.0, + "1326": 442918400.0, + "1327": 442918400.0, + "1328": 442918400.0, + "1329": 442918400.0, + "1330": 442918400.0, + "1331": 442918400.0, + "1332": 442918400.0, + "1333": 442918400.0, + "1334": 442918400.0, + "1335": 442918400.0, + "1336": 442918400.0, + "1337": 442918400.0, + "1338": 442918400.0, + "1339": 442918400.0, + "1340": 442918400.0, + "1341": 442918400.0, + "1342": 442918400.0, + "1343": 442918400.0, + "1344": 442918400.0, + "1345": 442918400.0, + "1346": 442918400.0, + "1347": 442918400.0, + "1348": 442918400.0, + "1349": 442918400.0, + "1350": 442918400.0, + "1351": 442918400.0, + "1352": 442918400.0, + "1353": 442918400.0, + "1354": 442918400.0, + "1355": 442918400.0, + "1356": 442918400.0, + "1357": 442918400.0, + "1358": 442918400.0, + "1359": 442918400.0, + "1360": 442918400.0, + "1361": 442918400.0, + "1362": 442918400.0, + "1363": 442918400.0, + "1364": 442918400.0, + "1365": 442918400.0, + "1366": 442918400.0, + "1367": 442918400.0, + "1368": 442918400.0, + "1369": 442918400.0, + "1370": 442918400.0, + "1371": 442918400.0, + "1372": 442918400.0, + "1373": 442918400.0, + "1374": 442918400.0, + "1375": 442918400.0, + "1376": 442918400.0, + "1377": 442918400.0, + "1378": 442918400.0, + "1379": 442918400.0, + "1380": 442918400.0, + "1381": 442918400.0, + "1382": 442918400.0, + "1383": 442918400.0, + "1384": 442918400.0, + "1385": 442918400.0, + "1386": 442918400.0, + "1387": 442918400.0, + "1388": 442918400.0, + "1389": 442918400.0, + "1390": 442918400.0, + "1391": 442918400.0, + "1392": 442918400.0, + "1393": 442918400.0, + "1394": 442918400.0, + "1395": 442918400.0, + "1396": 442918400.0, + "1397": 442918400.0, + "1398": 442918400.0, + "1399": 442918400.0, + "1400": 442918400.0, + "1401": 442918400.0, + "1402": 442918400.0, + "1403": 442918400.0, + "1404": 442918400.0, + "1405": 442918400.0, + "1406": 442918400.0, + "1407": 442918400.0, + "1408": 442918400.0, + "1409": 442918400.0, + "1410": 442918400.0, + "1411": 442918400.0, + "1412": 442918400.0, + "1413": 442918400.0, + "1414": 442918400.0, + "1415": 442918400.0, + "1416": 442918400.0, + "1417": 442918400.0, + "1418": 442918400.0, + "1419": 442918400.0, + "1420": 442918400.0, + "1421": 442918400.0, + "1422": 442918400.0, + "1423": 442918400.0, + "1424": 442918400.0, + "1425": 442918400.0, + "1426": 442918400.0, + "1427": 442918400.0, + "1428": 442918400.0, + "1429": 442918400.0, + "1430": 442918400.0, + "1431": 442918400.0, + "1432": 442918400.0, + "1433": 442918400.0, + "1434": 442918400.0, + "1435": 442918400.0, + "1436": 442918400.0, + "1437": 442918400.0, + "1438": 442918400.0, + "1439": 442918400.0, + "1440": 442918400.0, + "1441": 442918400.0, + "1442": 442918400.0, + "1443": 442918400.0, + "1444": 442918400.0, + "1445": 442918400.0, + "1446": 442918400.0, + "1447": 442918400.0, + "1448": 442918400.0, + "1449": 442918400.0, + "1450": 442918400.0, + "1451": 442918400.0, + "1452": 442918400.0, + "1453": 442918400.0, + "1454": 442918400.0, + "1455": 442918400.0, + "1456": 442918400.0, + "1457": 442918400.0, + "1458": 442918400.0, + "1459": 442918400.0, + "1460": 442918400.0, + "1461": 442918400.0, + "1462": 442918400.0, + "1463": 442918400.0, + "1464": 442918400.0, + "1465": 442918400.0, + "1466": 442918400.0, + "1467": 442918400.0, + "1468": 442918400.0, + "1469": 442918400.0, + "1470": 442918400.0, + "1471": 442918400.0, + "1472": 442918400.0, + "1473": 442918400.0, + "1474": 442918400.0, + "1475": 442918400.0, + "1476": 442918400.0, + "1477": 442918400.0, + "1478": 442918400.0, + "1479": 442918400.0, + "1480": 442918400.0, + "1481": 442918400.0, + "1482": 442918400.0, + "1483": 442918400.0, + "1484": 442918400.0, + "1485": 442918400.0, + "1486": 442918400.0, + "1487": 442918400.0, + "1488": 442918400.0, + "1489": 442918400.0, + "1490": 442918400.0, + "1491": 442918400.0, + "1492": 442918400.0, + "1493": 442918400.0, + "1494": 442918400.0, + "1495": 442918400.0, + "1496": 442918400.0, + "1497": 442918400.0, + "1498": 442918400.0, + "1499": 442918400.0, + "1500": 442918400.0, + "1501": 442918400.0, + "1502": 442918400.0, + "1503": 442918400.0, + "1504": 442918400.0, + "1505": 442918400.0, + "1506": 442918400.0, + "1507": 442918400.0, + "1508": 442918400.0, + "1509": 442918400.0, + "1510": 442918400.0, + "1511": 442918400.0, + "1512": 442918400.0, + "1513": 442918400.0, + "1514": 442918400.0, + "1515": 442918400.0, + "1516": 442918400.0, + "1517": 442918400.0, + "1518": 442918400.0, + "1519": 442918400.0, + "1520": 442918400.0, + "1521": 442918400.0, + "1522": 442918400.0, + "1523": 442918400.0, + "1524": 442918400.0, + "1525": 442918400.0, + "1526": 442918400.0, + "1527": 442918400.0, + "1528": 442918400.0, + "1529": 442918400.0, + "1530": 442918400.0, + "1531": 442918400.0, + "1532": 442918400.0, + "1533": 442918400.0, + "1534": 442918400.0, + "1535": 442918400.0, + "1536": 442918400.0, + "1537": 442918400.0, + "1538": 442918400.0, + "1539": 442918400.0, + "1540": 442918400.0, + "1541": 442918400.0, + "1542": 442918400.0, + "1543": 442918400.0, + "1544": 442918400.0, + "1545": 442918400.0, + "1546": 442918400.0, + "1547": 442918400.0, + "1548": 442918400.0, + "1549": 442918400.0, + "1550": 442918400.0, + "1551": 442918400.0, + "1552": 442918400.0, + "1553": 442918400.0, + "1554": 442918400.0, + "1555": 442918400.0, + "1556": 442918400.0, + "1557": 442918400.0, + "1558": 442918400.0, + "1559": 442918400.0, + "1560": 442918400.0, + "1561": 442918400.0, + "1562": 442918400.0, + "1563": 442918400.0, + "1564": 442918400.0, + "1565": 442918400.0, + "1566": 442918400.0, + "1567": 442918400.0, + "1568": 442918400.0, + "1569": 442918400.0, + "1570": 442918400.0, + "1571": 442918400.0, + "1572": 442918400.0, + "1573": 442918400.0, + "1574": 442918400.0, + "1575": 442918400.0, + "1576": 442918400.0, + "1577": 442918400.0, + "1578": 442918400.0, + "1579": 442918400.0, + "1580": 442918400.0, + "1581": 442918400.0, + "1582": 442918400.0, + "1583": 442918400.0, + "1584": 442918400.0, + "1585": 442918400.0, + "1586": 442918400.0, + "1587": 442918400.0, + "1588": 442918400.0, + "1589": 442918400.0, + "1590": 442918400.0, + "1591": 442918400.0, + "1592": 442918400.0, + "1593": 442918400.0, + "1594": 442918400.0, + "1595": 442918400.0, + "1596": 442918400.0, + "1597": 442918400.0, + "1598": 442918400.0, + "1599": 442918400.0, + "1600": 442918400.0, + "1601": 442918400.0, + "1602": 442918400.0, + "1603": 442918400.0, + "1604": 442918400.0, + "1605": 442918400.0, + "1606": 442918400.0, + "1607": 442918400.0, + "1608": 442918400.0, + "1609": 442918400.0, + "1610": 442918400.0, + "1611": 442918400.0, + "1612": 442918400.0, + "1613": 442918400.0, + "1614": 442918400.0, + "1615": 442918400.0, + "1616": 442918400.0, + "1617": 442918400.0, + "1618": 442918400.0, + "1619": 442918400.0, + "1620": 442918400.0, + "1621": 442918400.0, + "1622": 442918400.0, + "1623": 442918400.0, + "1624": 442918400.0, + "1625": 442918400.0, + "1626": 442918400.0, + "1627": 442918400.0, + "1628": 442918400.0, + "1629": 442918400.0, + "1630": 442918400.0, + "1631": 442918400.0, + "1632": 442918400.0, + "1633": 442918400.0, + "1634": 442918400.0, + "1635": 442918400.0, + "1636": 442918400.0, + "1637": 442918400.0, + "1638": 442918400.0, + "1639": 442918400.0, + "1640": 442918400.0, + "1641": 442918400.0, + "1642": 442918400.0, + "1643": 442918400.0, + "1644": 442918400.0, + "1645": 442918400.0, + "1646": 442918400.0, + "1647": 442918400.0, + "1648": 442918400.0, + "1649": 442918400.0, + "1650": 442918400.0, + "1651": 442918400.0, + "1652": 442918400.0, + "1653": 442918400.0, + "1654": 442918400.0, + "1655": 442918400.0, + "1656": 442918400.0, + "1657": 442918400.0, + "1658": 442918400.0, + "1659": 442918400.0, + "1660": 442918400.0, + "1661": 442918400.0, + "1662": 442918400.0, + "1663": 442918400.0, + "1664": 442918400.0, + "1665": 442918400.0, + "1666": 442918400.0, + "1667": 442918400.0, + "1668": 442918400.0, + "1669": 442918400.0, + "1670": 442918400.0, + "1671": 442918400.0, + "1672": 442918400.0, + "1673": 442918400.0, + "1674": 442918400.0, + "1675": 442918400.0, + "1676": 442918400.0, + "1677": 442918400.0, + "1678": 442918400.0, + "1679": 442918400.0, + "1680": 442918400.0, + "1681": 442918400.0, + "1682": 442918400.0, + "1683": 442918400.0, + "1684": 442918400.0, + "1685": 442918400.0, + "1686": 442918400.0, + "1687": 442918400.0, + "1688": 442918400.0, + "1689": 442918400.0, + "1690": 442918400.0, + "1691": 442918400.0, + "1692": 442918400.0, + "1693": 442918400.0, + "1694": 442918400.0, + "1695": 442918400.0, + "1696": 442918400.0, + "1697": 442918400.0, + "1698": 442918400.0, + "1699": 442918400.0, + "1700": 442918400.0, + "1701": 442918400.0, + "1702": 442918400.0, + "1703": 442918400.0, + "1704": 442918400.0, + "1705": 442918400.0, + "1706": 442918400.0, + "1707": 442918400.0, + "1708": 442918400.0, + "1709": 442918400.0, + "1710": 442918400.0, + "1711": 442918400.0, + "1712": 442918400.0, + "1713": 442918400.0, + "1714": 442918400.0, + "1715": 442918400.0, + "1716": 442918400.0, + "1717": 442918400.0, + "1718": 442918400.0, + "1719": 442918400.0, + "1720": 442918400.0, + "1721": 442918400.0, + "1722": 442918400.0, + "1723": 442918400.0, + "1724": 442918400.0, + "1725": 442918400.0, + "1726": 442918400.0, + "1727": 442918400.0, + "1728": 442918400.0, + "1729": 442918400.0, + "1730": 442918400.0, + "1731": 442918400.0, + "1732": 442918400.0, + "1733": 442918400.0, + "1734": 442918400.0, + "1735": 442918400.0, + "1736": 442918400.0, + "1737": 442918400.0, + "1738": 442918400.0, + "1739": 442918400.0, + "1740": 442918400.0, + "1741": 442918400.0, + "1742": 442918400.0, + "1743": 442918400.0, + "1744": 442918400.0, + "1745": 442918400.0, + "1746": 442918400.0, + "1747": 442918400.0, + "1748": 442918400.0, + "1749": 442918400.0, + "1750": 442918400.0, + "1751": 442918400.0, + "1752": 442918400.0, + "1753": 442918400.0, + "1754": 442918400.0, + "1755": 442918400.0, + "1756": 442918400.0, + "1757": 442918400.0, + "1758": 442918400.0, + "1759": 442918400.0, + "1760": 442918400.0, + "1761": 442918400.0, + "1762": 442918400.0, + "1763": 442918400.0, + "1764": 442918400.0, + "1765": 442918400.0, + "1766": 442918400.0, + "1767": 442918400.0, + "1768": 442918400.0, + "1769": 442918400.0, + "1770": 442918400.0, + "1771": 442918400.0, + "1772": 442918400.0, + "1773": 442918400.0, + "1774": 442918400.0, + "1775": 442918400.0, + "1776": 442918400.0, + "1777": 442918400.0, + "1778": 442918400.0, + "1779": 442918400.0, + "1780": 442918400.0, + "1781": 442918400.0, + "1782": 442918400.0, + "1783": 442918400.0, + "1784": 442918400.0, + "1785": 442918400.0, + "1786": 442918400.0, + "1787": 442918400.0, + "1788": 442918400.0, + "1789": 442918400.0, + "1790": 442918400.0, + "1791": 442918400.0, + "1792": 442918400.0, + "1793": 442918400.0, + "1794": 442918400.0, + "1795": 442918400.0, + "1796": 442918400.0, + "1797": 442918400.0, + "1798": 442918400.0, + "1799": 442918400.0, + "1800": 442918400.0, + "1801": 442918400.0, + "1802": 442918400.0, + "1803": 442918400.0, + "1804": 442918400.0, + "1805": 442918400.0, + "1806": 442918400.0, + "1807": 442918400.0, + "1808": 442918400.0, + "1809": 442918400.0, + "1810": 442918400.0, + "1811": 442918400.0, + "1812": 442918400.0, + "1813": 442918400.0, + "1814": 442918400.0, + "1815": 442918400.0, + "1816": 442918400.0, + "1817": 442918400.0, + "1818": 442918400.0, + "1819": 442918400.0, + "1820": 442918400.0, + "1821": 442918400.0, + "1822": 442918400.0, + "1823": 442918400.0, + "1824": 442918400.0, + "1825": 442918400.0, + "1826": 442918400.0, + "1827": 442918400.0, + "1828": 442918400.0, + "1829": 442918400.0, + "1830": 442918400.0, + "1831": 442918400.0, + "1832": 442918400.0, + "1833": 442918400.0, + "1834": 442918400.0, + "1835": 442918400.0, + "1836": 442918400.0, + "1837": 442918400.0, + "1838": 442918400.0, + "1839": 442918400.0, + "1840": 442918400.0, + "1841": 442918400.0, + "1842": 442918400.0, + "1843": 442918400.0, + "1844": 442918400.0, + "1845": 442918400.0, + "1846": 442918400.0, + "1847": 442918400.0, + "1848": 442918400.0, + "1849": 442918400.0, + "1850": 442918400.0, + "1851": 442918400.0, + "1852": 442918400.0, + "1853": 442918400.0, + "1854": 442918400.0, + "1855": 442918400.0, + "1856": 442918400.0, + "1857": 442918400.0, + "1858": 442918400.0, + "1859": 442918400.0, + "1860": 442918400.0, + "1861": 442918400.0, + "1862": 442918400.0, + "1863": 442918400.0, + "1864": 442918400.0, + "1865": 442918400.0, + "1866": 442918400.0, + "1867": 442918400.0, + "1868": 442918400.0, + "1869": 442918400.0, + "1870": 442918400.0, + "1871": 442918400.0, + "1872": 442918400.0, + "1873": 442918400.0, + "1874": 442918400.0, + "1875": 442918400.0, + "1876": 442918400.0, + "1877": 442918400.0, + "1878": 442918400.0, + "1879": 442918400.0, + "1880": 442918400.0, + "1881": 442918400.0, + "1882": 442918400.0, + "1883": 442918400.0, + "1884": 442918400.0, + "1885": 442918400.0, + "1886": 442918400.0, + "1887": 442918400.0, + "1888": 442918400.0, + "1889": 442918400.0, + "1890": 442918400.0, + "1891": 442918400.0, + "1892": 442918400.0, + "1893": 442918400.0, + "1894": 442918400.0, + "1895": 442918400.0, + "1896": 442918400.0, + "1897": 442918400.0, + "1898": 442918400.0, + "1899": 442918400.0, + "1900": 442918400.0, + "1901": 442918400.0, + "1902": 442918400.0, + "1903": 442918400.0, + "1904": 442918400.0, + "1905": 442918400.0, + "1906": 442918400.0, + "1907": 442918400.0, + "1908": 442918400.0, + "1909": 442918400.0, + "1910": 442918400.0, + "1911": 442918400.0, + "1912": 442918400.0, + "1913": 442918400.0, + "1914": 442918400.0, + "1915": 442918400.0, + "1916": 442918400.0, + "1917": 442918400.0, + "1918": 442918400.0, + "1919": 442918400.0, + "1920": 442918400.0, + "1921": 442918400.0, + "1922": 442918400.0, + "1923": 442918400.0, + "1924": 442918400.0, + "1925": 442918400.0, + "1926": 442918400.0, + "1927": 442918400.0, + "1928": 442918400.0, + "1929": 442918400.0, + "1930": 442918400.0, + "1931": 442918400.0, + "1932": 442918400.0, + "1933": 442918400.0, + "1934": 442918400.0, + "1935": 442918400.0, + "1936": 442918400.0, + "1937": 442918400.0, + "1938": 442918400.0, + "1939": 442918400.0, + "1940": 442918400.0, + "1941": 442918400.0, + "1942": 442918400.0, + "1943": 442918400.0, + "1944": 442918400.0, + "1945": 442918400.0, + "1946": 442918400.0, + "1947": 442918400.0, + "1948": 442918400.0, + "1949": 442918400.0, + "1950": 442918400.0, + "1951": 442918400.0, + "1952": 442918400.0, + "1953": 442918400.0, + "1954": 442918400.0, + "1955": 442918400.0, + "1956": 442918400.0, + "1957": 442918400.0, + "1958": 442918400.0, + "1959": 442918400.0, + "1960": 442918400.0, + "1961": 442918400.0, + "1962": 442918400.0, + "1963": 442918400.0, + "1964": 442918400.0, + "1965": 442918400.0, + "1966": 442918400.0, + "1967": 442918400.0, + "1968": 442918400.0, + "1969": 442918400.0, + "1970": 442918400.0, + "1971": 442918400.0, + "1972": 442918400.0, + "1973": 442918400.0, + "1974": 442918400.0, + "1975": 442918400.0, + "1976": 442918400.0, + "1977": 442918400.0, + "1978": 442918400.0, + "1979": 442918400.0, + "1980": 442918400.0, + "1981": 442918400.0, + "1982": 442918400.0, + "1983": 442918400.0, + "1984": 442918400.0, + "1985": 442918400.0, + "1986": 442918400.0, + "1987": 442918400.0, + "1988": 442918400.0, + "1989": 442918400.0, + "1990": 442918400.0, + "1991": 442918400.0, + "1992": 442918400.0, + "1993": 442918400.0, + "1994": 442918400.0, + "1995": 442918400.0, + "1996": 442918400.0, + "1997": 442918400.0, + "1998": 442918400.0, + "1999": 442918400.0, + "2000": 442918400.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 761183744.0, + "2": 849621504.0, + "3": 849621504.0, + "4": 849621504.0, + "5": 849621504.0, + "6": 849621504.0, + "7": 849621504.0, + "8": 849621504.0, + "9": 849621504.0, + "10": 849621504.0, + "11": 849621504.0, + "12": 849621504.0, + "13": 849621504.0, + "14": 849621504.0, + "15": 849621504.0, + "16": 849621504.0, + "17": 849621504.0, + "18": 849621504.0, + "19": 849621504.0, + "20": 849621504.0, + "21": 849621504.0, + "22": 849621504.0, + "23": 849621504.0, + "24": 849621504.0, + "25": 849621504.0, + "26": 849621504.0, + "27": 849621504.0, + "28": 849621504.0, + "29": 849621504.0, + "30": 849621504.0, + "31": 849621504.0, + "32": 849621504.0, + "33": 849621504.0, + "34": 849621504.0, + "35": 849621504.0, + "36": 849621504.0, + "37": 849621504.0, + "38": 849621504.0, + "39": 849621504.0, + "40": 849621504.0, + "41": 849621504.0, + "42": 849621504.0, + "43": 849621504.0, + "44": 849621504.0, + "45": 849621504.0, + "46": 849621504.0, + "47": 849621504.0, + "48": 849621504.0, + "49": 849621504.0, + "50": 849621504.0, + "51": 849621504.0, + "52": 849621504.0, + "53": 849621504.0, + "54": 849621504.0, + "55": 849621504.0, + "56": 849621504.0, + "57": 849621504.0, + "58": 849621504.0, + "59": 849621504.0, + "60": 849621504.0, + "61": 849621504.0, + "62": 849621504.0, + "63": 849621504.0, + "64": 849621504.0, + "65": 849621504.0, + "66": 849621504.0, + "67": 849621504.0, + "68": 849621504.0, + "69": 849621504.0, + "70": 849621504.0, + "71": 849621504.0, + "72": 849621504.0, + "73": 849621504.0, + "74": 849621504.0, + "75": 849621504.0, + "76": 849621504.0, + "77": 849621504.0, + "78": 849621504.0, + "79": 849621504.0, + "80": 849621504.0, + "81": 849621504.0, + "82": 849621504.0, + "83": 849621504.0, + "84": 849621504.0, + "85": 849621504.0, + "86": 849621504.0, + "87": 849621504.0, + "88": 849621504.0, + "89": 849621504.0, + "90": 849621504.0, + "91": 849621504.0, + "92": 849621504.0, + "93": 849621504.0, + "94": 849621504.0, + "95": 849621504.0, + "96": 849621504.0, + "97": 849621504.0, + "98": 849621504.0, + "99": 849621504.0, + "100": 849621504.0, + "101": 849621504.0, + "102": 849621504.0, + "103": 849621504.0, + "104": 849621504.0, + "105": 849621504.0, + "106": 849621504.0, + "107": 849621504.0, + "108": 849621504.0, + "109": 849621504.0, + "110": 849621504.0, + "111": 849621504.0, + "112": 849621504.0, + "113": 849621504.0, + "114": 849621504.0, + "115": 849621504.0, + "116": 849621504.0, + "117": 849621504.0, + "118": 849621504.0, + "119": 849621504.0, + "120": 849621504.0, + "121": 849621504.0, + "122": 849621504.0, + "123": 849621504.0, + "124": 849621504.0, + "125": 849621504.0, + "126": 849621504.0, + "127": 849621504.0, + "128": 849621504.0, + "129": 849621504.0, + "130": 849621504.0, + "131": 849621504.0, + "132": 849621504.0, + "133": 849621504.0, + "134": 849621504.0, + "135": 849621504.0, + "136": 849621504.0, + "137": 849621504.0, + "138": 849621504.0, + "139": 849621504.0, + "140": 849621504.0, + "141": 849621504.0, + "142": 849621504.0, + "143": 849621504.0, + "144": 849621504.0, + "145": 849621504.0, + "146": 849621504.0, + "147": 849621504.0, + "148": 849621504.0, + "149": 849621504.0, + "150": 849621504.0, + "151": 849621504.0, + "152": 849621504.0, + "153": 849621504.0, + "154": 849621504.0, + "155": 849621504.0, + "156": 849621504.0, + "157": 849621504.0, + "158": 849621504.0, + "159": 849621504.0, + "160": 849621504.0, + "161": 849621504.0, + "162": 849621504.0, + "163": 849621504.0, + "164": 849621504.0, + "165": 849621504.0, + "166": 849621504.0, + "167": 849621504.0, + "168": 849621504.0, + "169": 849621504.0, + "170": 849621504.0, + "171": 849621504.0, + "172": 849621504.0, + "173": 849621504.0, + "174": 849621504.0, + "175": 849621504.0, + "176": 849621504.0, + "177": 849621504.0, + "178": 849621504.0, + "179": 849621504.0, + "180": 849621504.0, + "181": 849621504.0, + "182": 849621504.0, + "183": 849621504.0, + "184": 849621504.0, + "185": 849621504.0, + "186": 849621504.0, + "187": 849621504.0, + "188": 849621504.0, + "189": 849621504.0, + "190": 849621504.0, + "191": 849621504.0, + "192": 849621504.0, + "193": 849621504.0, + "194": 849621504.0, + "195": 849621504.0, + "196": 849621504.0, + "197": 849621504.0, + "198": 849621504.0, + "199": 849621504.0, + "200": 849621504.0, + "201": 849621504.0, + "202": 849621504.0, + "203": 849621504.0, + "204": 849621504.0, + "205": 849621504.0, + "206": 849621504.0, + "207": 849621504.0, + "208": 849621504.0, + "209": 849621504.0, + "210": 849621504.0, + "211": 849621504.0, + "212": 849621504.0, + "213": 849621504.0, + "214": 849621504.0, + "215": 849621504.0, + "216": 849621504.0, + "217": 849621504.0, + "218": 849621504.0, + "219": 849621504.0, + "220": 849621504.0, + "221": 849621504.0, + "222": 849621504.0, + "223": 849621504.0, + "224": 849621504.0, + "225": 849621504.0, + "226": 849621504.0, + "227": 849621504.0, + "228": 849621504.0, + "229": 849621504.0, + "230": 849621504.0, + "231": 849621504.0, + "232": 849621504.0, + "233": 849621504.0, + "234": 849621504.0, + "235": 849621504.0, + "236": 849621504.0, + "237": 849621504.0, + "238": 849621504.0, + "239": 849621504.0, + "240": 849621504.0, + "241": 849621504.0, + "242": 849621504.0, + "243": 849621504.0, + "244": 849621504.0, + "245": 849621504.0, + "246": 849621504.0, + "247": 849621504.0, + "248": 849621504.0, + "249": 849621504.0, + "250": 849621504.0, + "251": 849621504.0, + "252": 849621504.0, + "253": 849621504.0, + "254": 849621504.0, + "255": 849621504.0, + "256": 849621504.0, + "257": 849621504.0, + "258": 849621504.0, + "259": 849621504.0, + "260": 849621504.0, + "261": 849621504.0, + "262": 849621504.0, + "263": 849621504.0, + "264": 849621504.0, + "265": 849621504.0, + "266": 849621504.0, + "267": 849621504.0, + "268": 849621504.0, + "269": 849621504.0, + "270": 849621504.0, + "271": 849621504.0, + "272": 849621504.0, + "273": 849621504.0, + "274": 849621504.0, + "275": 849621504.0, + "276": 849621504.0, + "277": 849621504.0, + "278": 849621504.0, + "279": 849621504.0, + "280": 849621504.0, + "281": 849621504.0, + "282": 849621504.0, + "283": 849621504.0, + "284": 849621504.0, + "285": 849621504.0, + "286": 849621504.0, + "287": 849621504.0, + "288": 849621504.0, + "289": 849621504.0, + "290": 849621504.0, + "291": 849621504.0, + "292": 849621504.0, + "293": 849621504.0, + "294": 849621504.0, + "295": 849621504.0, + "296": 849621504.0, + "297": 849621504.0, + "298": 849621504.0, + "299": 849621504.0, + "300": 849621504.0, + "301": 849621504.0, + "302": 849621504.0, + "303": 849621504.0, + "304": 849621504.0, + "305": 849621504.0, + "306": 849621504.0, + "307": 849621504.0, + "308": 849621504.0, + "309": 849621504.0, + "310": 849621504.0, + "311": 849621504.0, + "312": 849621504.0, + "313": 849621504.0, + "314": 849621504.0, + "315": 849621504.0, + "316": 849621504.0, + "317": 849621504.0, + "318": 849621504.0, + "319": 849621504.0, + "320": 849621504.0, + "321": 849621504.0, + "322": 849621504.0, + "323": 849621504.0, + "324": 849621504.0, + "325": 849621504.0, + "326": 849621504.0, + "327": 849621504.0, + "328": 849621504.0, + "329": 849621504.0, + "330": 849621504.0, + "331": 849621504.0, + "332": 849621504.0, + "333": 849621504.0, + "334": 849621504.0, + "335": 849621504.0, + "336": 849621504.0, + "337": 849621504.0, + "338": 849621504.0, + "339": 849621504.0, + "340": 849621504.0, + "341": 849621504.0, + "342": 849621504.0, + "343": 849621504.0, + "344": 849621504.0, + "345": 849621504.0, + "346": 849621504.0, + "347": 849621504.0, + "348": 849621504.0, + "349": 849621504.0, + "350": 849621504.0, + "351": 849621504.0, + "352": 849621504.0, + "353": 849621504.0, + "354": 849621504.0, + "355": 849621504.0, + "356": 849621504.0, + "357": 849621504.0, + "358": 849621504.0, + "359": 849621504.0, + "360": 849621504.0, + "361": 849621504.0, + "362": 849621504.0, + "363": 849621504.0, + "364": 849621504.0, + "365": 849621504.0, + "366": 849621504.0, + "367": 849621504.0, + "368": 849621504.0, + "369": 849621504.0, + "370": 849621504.0, + "371": 849621504.0, + "372": 849621504.0, + "373": 849621504.0, + "374": 849621504.0, + "375": 849621504.0, + "376": 849621504.0, + "377": 849621504.0, + "378": 849621504.0, + "379": 849621504.0, + "380": 849621504.0, + "381": 849621504.0, + "382": 849621504.0, + "383": 849621504.0, + "384": 849621504.0, + "385": 849621504.0, + "386": 849621504.0, + "387": 849621504.0, + "388": 849621504.0, + "389": 849621504.0, + "390": 849621504.0, + "391": 849621504.0, + "392": 849621504.0, + "393": 849621504.0, + "394": 849621504.0, + "395": 849621504.0, + "396": 849621504.0, + "397": 849621504.0, + "398": 849621504.0, + "399": 849621504.0, + "400": 849621504.0, + "401": 849621504.0, + "402": 849621504.0, + "403": 849621504.0, + "404": 849621504.0, + "405": 849621504.0, + "406": 849621504.0, + "407": 849621504.0, + "408": 849621504.0, + "409": 849621504.0, + "410": 849621504.0, + "411": 849621504.0, + "412": 849621504.0, + "413": 849621504.0, + "414": 849621504.0, + "415": 849621504.0, + "416": 849621504.0, + "417": 849621504.0, + "418": 849621504.0, + "419": 849621504.0, + "420": 849621504.0, + "421": 849621504.0, + "422": 849621504.0, + "423": 849621504.0, + "424": 849621504.0, + "425": 849621504.0, + "426": 849621504.0, + "427": 849621504.0, + "428": 849621504.0, + "429": 849621504.0, + "430": 849621504.0, + "431": 849621504.0, + "432": 849621504.0, + "433": 849621504.0, + "434": 849621504.0, + "435": 849621504.0, + "436": 849621504.0, + "437": 849621504.0, + "438": 849621504.0, + "439": 849621504.0, + "440": 849621504.0, + "441": 849621504.0, + "442": 849621504.0, + "443": 849621504.0, + "444": 849621504.0, + "445": 849621504.0, + "446": 849621504.0, + "447": 849621504.0, + "448": 849621504.0, + "449": 849621504.0, + "450": 849621504.0, + "451": 849621504.0, + "452": 849621504.0, + "453": 849621504.0, + "454": 849621504.0, + "455": 849621504.0, + "456": 849621504.0, + "457": 849621504.0, + "458": 849621504.0, + "459": 849621504.0, + "460": 849621504.0, + "461": 849621504.0, + "462": 849621504.0, + "463": 849621504.0, + "464": 849621504.0, + "465": 849621504.0, + "466": 849621504.0, + "467": 849621504.0, + "468": 849621504.0, + "469": 849621504.0, + "470": 849621504.0, + "471": 849621504.0, + "472": 849621504.0, + "473": 849621504.0, + "474": 849621504.0, + "475": 849621504.0, + "476": 849621504.0, + "477": 849621504.0, + "478": 849621504.0, + "479": 849621504.0, + "480": 849621504.0, + "481": 849621504.0, + "482": 849621504.0, + "483": 849621504.0, + "484": 849621504.0, + "485": 849621504.0, + "486": 849621504.0, + "487": 849621504.0, + "488": 849621504.0, + "489": 849621504.0, + "490": 849621504.0, + "491": 849621504.0, + "492": 849621504.0, + "493": 849621504.0, + "494": 849621504.0, + "495": 849621504.0, + "496": 849621504.0, + "497": 849621504.0, + "498": 849621504.0, + "499": 849621504.0, + "500": 849621504.0, + "501": 849621504.0, + "502": 849621504.0, + "503": 849621504.0, + "504": 849621504.0, + "505": 849621504.0, + "506": 849621504.0, + "507": 849621504.0, + "508": 849621504.0, + "509": 849621504.0, + "510": 849621504.0, + "511": 849621504.0, + "512": 849621504.0, + "513": 849621504.0, + "514": 849621504.0, + "515": 849621504.0, + "516": 849621504.0, + "517": 849621504.0, + "518": 849621504.0, + "519": 849621504.0, + "520": 849621504.0, + "521": 849621504.0, + "522": 849621504.0, + "523": 849621504.0, + "524": 849621504.0, + "525": 849621504.0, + "526": 849621504.0, + "527": 849621504.0, + "528": 849621504.0, + "529": 849621504.0, + "530": 849621504.0, + "531": 849621504.0, + "532": 849621504.0, + "533": 849621504.0, + "534": 849621504.0, + "535": 849621504.0, + "536": 849621504.0, + "537": 849621504.0, + "538": 849621504.0, + "539": 849621504.0, + "540": 849621504.0, + "541": 849621504.0, + "542": 849621504.0, + "543": 849621504.0, + "544": 849621504.0, + "545": 849621504.0, + "546": 849621504.0, + "547": 849621504.0, + "548": 849621504.0, + "549": 849621504.0, + "550": 849621504.0, + "551": 849621504.0, + "552": 849621504.0, + "553": 849621504.0, + "554": 849621504.0, + "555": 849621504.0, + "556": 849621504.0, + "557": 849621504.0, + "558": 849621504.0, + "559": 849621504.0, + "560": 849621504.0, + "561": 849621504.0, + "562": 849621504.0, + "563": 849621504.0, + "564": 849621504.0, + "565": 849621504.0, + "566": 849621504.0, + "567": 849621504.0, + "568": 849621504.0, + "569": 849621504.0, + "570": 849621504.0, + "571": 849621504.0, + "572": 849621504.0, + "573": 849621504.0, + "574": 849621504.0, + "575": 849621504.0, + "576": 849621504.0, + "577": 849621504.0, + "578": 849621504.0, + "579": 849621504.0, + "580": 849621504.0, + "581": 849621504.0, + "582": 849621504.0, + "583": 849621504.0, + "584": 849621504.0, + "585": 849621504.0, + "586": 849621504.0, + "587": 849621504.0, + "588": 849621504.0, + "589": 849621504.0, + "590": 849621504.0, + "591": 849621504.0, + "592": 849621504.0, + "593": 849621504.0, + "594": 849621504.0, + "595": 849621504.0, + "596": 849621504.0, + "597": 849621504.0, + "598": 849621504.0, + "599": 849621504.0, + "600": 849621504.0, + "601": 849621504.0, + "602": 849621504.0, + "603": 849621504.0, + "604": 849621504.0, + "605": 849621504.0, + "606": 849621504.0, + "607": 849621504.0, + "608": 849621504.0, + "609": 849621504.0, + "610": 849621504.0, + "611": 849621504.0, + "612": 849621504.0, + "613": 849621504.0, + "614": 849621504.0, + "615": 849621504.0, + "616": 849621504.0, + "617": 849621504.0, + "618": 849621504.0, + "619": 849621504.0, + "620": 849621504.0, + "621": 849621504.0, + "622": 849621504.0, + "623": 849621504.0, + "624": 849621504.0, + "625": 849621504.0, + "626": 849621504.0, + "627": 849621504.0, + "628": 849621504.0, + "629": 849621504.0, + "630": 849621504.0, + "631": 849621504.0, + "632": 849621504.0, + "633": 849621504.0, + "634": 849621504.0, + "635": 849621504.0, + "636": 849621504.0, + "637": 849621504.0, + "638": 849621504.0, + "639": 849621504.0, + "640": 849621504.0, + "641": 849621504.0, + "642": 849621504.0, + "643": 849621504.0, + "644": 849621504.0, + "645": 849621504.0, + "646": 849621504.0, + "647": 849621504.0, + "648": 849621504.0, + "649": 849621504.0, + "650": 849621504.0, + "651": 849621504.0, + "652": 849621504.0, + "653": 849621504.0, + "654": 849621504.0, + "655": 849621504.0, + "656": 849621504.0, + "657": 849621504.0, + "658": 849621504.0, + "659": 849621504.0, + "660": 849621504.0, + "661": 849621504.0, + "662": 849621504.0, + "663": 849621504.0, + "664": 849621504.0, + "665": 849621504.0, + "666": 849621504.0, + "667": 849621504.0, + "668": 849621504.0, + "669": 849621504.0, + "670": 849621504.0, + "671": 849621504.0, + "672": 849621504.0, + "673": 849621504.0, + "674": 849621504.0, + "675": 849621504.0, + "676": 849621504.0, + "677": 849621504.0, + "678": 849621504.0, + "679": 849621504.0, + "680": 849621504.0, + "681": 849621504.0, + "682": 849621504.0, + "683": 849621504.0, + "684": 849621504.0, + "685": 849621504.0, + "686": 849621504.0, + "687": 849621504.0, + "688": 849621504.0, + "689": 849621504.0, + "690": 849621504.0, + "691": 849621504.0, + "692": 849621504.0, + "693": 849621504.0, + "694": 849621504.0, + "695": 849621504.0, + "696": 849621504.0, + "697": 849621504.0, + "698": 849621504.0, + "699": 849621504.0, + "700": 849621504.0, + "701": 849621504.0, + "702": 849621504.0, + "703": 849621504.0, + "704": 849621504.0, + "705": 849621504.0, + "706": 849621504.0, + "707": 849621504.0, + "708": 849621504.0, + "709": 849621504.0, + "710": 849621504.0, + "711": 849621504.0, + "712": 849621504.0, + "713": 849621504.0, + "714": 849621504.0, + "715": 849621504.0, + "716": 849621504.0, + "717": 849621504.0, + "718": 849621504.0, + "719": 849621504.0, + "720": 849621504.0, + "721": 849621504.0, + "722": 849621504.0, + "723": 849621504.0, + "724": 849621504.0, + "725": 849621504.0, + "726": 849621504.0, + "727": 849621504.0, + "728": 849621504.0, + "729": 849621504.0, + "730": 849621504.0, + "731": 849621504.0, + "732": 849621504.0, + "733": 849621504.0, + "734": 849621504.0, + "735": 849621504.0, + "736": 849621504.0, + "737": 849621504.0, + "738": 849621504.0, + "739": 849621504.0, + "740": 849621504.0, + "741": 849621504.0, + "742": 849621504.0, + "743": 849621504.0, + "744": 849621504.0, + "745": 849621504.0, + "746": 849621504.0, + "747": 849621504.0, + "748": 849621504.0, + "749": 849621504.0, + "750": 849621504.0, + "751": 849621504.0, + "752": 849621504.0, + "753": 849621504.0, + "754": 849621504.0, + "755": 849621504.0, + "756": 849621504.0, + "757": 849621504.0, + "758": 849621504.0, + "759": 849621504.0, + "760": 849621504.0, + "761": 849621504.0, + "762": 849621504.0, + "763": 849621504.0, + "764": 849621504.0, + "765": 849621504.0, + "766": 849621504.0, + "767": 849621504.0, + "768": 849621504.0, + "769": 849621504.0, + "770": 849621504.0, + "771": 849621504.0, + "772": 849621504.0, + "773": 849621504.0, + "774": 849621504.0, + "775": 849621504.0, + "776": 849621504.0, + "777": 849621504.0, + "778": 849621504.0, + "779": 849621504.0, + "780": 849621504.0, + "781": 849621504.0, + "782": 849621504.0, + "783": 849621504.0, + "784": 849621504.0, + "785": 849621504.0, + "786": 849621504.0, + "787": 849621504.0, + "788": 849621504.0, + "789": 849621504.0, + "790": 849621504.0, + "791": 849621504.0, + "792": 849621504.0, + "793": 849621504.0, + "794": 849621504.0, + "795": 849621504.0, + "796": 849621504.0, + "797": 849621504.0, + "798": 849621504.0, + "799": 849621504.0, + "800": 849621504.0, + "801": 849621504.0, + "802": 849621504.0, + "803": 849621504.0, + "804": 849621504.0, + "805": 849621504.0, + "806": 849621504.0, + "807": 849621504.0, + "808": 849621504.0, + "809": 849621504.0, + "810": 849621504.0, + "811": 849621504.0, + "812": 849621504.0, + "813": 849621504.0, + "814": 849621504.0, + "815": 849621504.0, + "816": 849621504.0, + "817": 849621504.0, + "818": 849621504.0, + "819": 849621504.0, + "820": 849621504.0, + "821": 849621504.0, + "822": 849621504.0, + "823": 849621504.0, + "824": 849621504.0, + "825": 849621504.0, + "826": 849621504.0, + "827": 849621504.0, + "828": 849621504.0, + "829": 849621504.0, + "830": 849621504.0, + "831": 849621504.0, + "832": 849621504.0, + "833": 849621504.0, + "834": 849621504.0, + "835": 849621504.0, + "836": 849621504.0, + "837": 849621504.0, + "838": 849621504.0, + "839": 849621504.0, + "840": 849621504.0, + "841": 849621504.0, + "842": 849621504.0, + "843": 849621504.0, + "844": 849621504.0, + "845": 849621504.0, + "846": 849621504.0, + "847": 849621504.0, + "848": 849621504.0, + "849": 849621504.0, + "850": 849621504.0, + "851": 849621504.0, + "852": 849621504.0, + "853": 849621504.0, + "854": 849621504.0, + "855": 849621504.0, + "856": 849621504.0, + "857": 849621504.0, + "858": 849621504.0, + "859": 849621504.0, + "860": 849621504.0, + "861": 849621504.0, + "862": 849621504.0, + "863": 849621504.0, + "864": 849621504.0, + "865": 849621504.0, + "866": 849621504.0, + "867": 849621504.0, + "868": 849621504.0, + "869": 849621504.0, + "870": 849621504.0, + "871": 849621504.0, + "872": 849621504.0, + "873": 849621504.0, + "874": 849621504.0, + "875": 849621504.0, + "876": 849621504.0, + "877": 849621504.0, + "878": 849621504.0, + "879": 849621504.0, + "880": 849621504.0, + "881": 849621504.0, + "882": 849621504.0, + "883": 849621504.0, + "884": 849621504.0, + "885": 849621504.0, + "886": 849621504.0, + "887": 849621504.0, + "888": 849621504.0, + "889": 849621504.0, + "890": 849621504.0, + "891": 849621504.0, + "892": 849621504.0, + "893": 849621504.0, + "894": 849621504.0, + "895": 849621504.0, + "896": 849621504.0, + "897": 849621504.0, + "898": 849621504.0, + "899": 849621504.0, + "900": 849621504.0, + "901": 849621504.0, + "902": 849621504.0, + "903": 849621504.0, + "904": 849621504.0, + "905": 849621504.0, + "906": 849621504.0, + "907": 849621504.0, + "908": 849621504.0, + "909": 849621504.0, + "910": 849621504.0, + "911": 849621504.0, + "912": 849621504.0, + "913": 849621504.0, + "914": 849621504.0, + "915": 849621504.0, + "916": 849621504.0, + "917": 849621504.0, + "918": 849621504.0, + "919": 849621504.0, + "920": 849621504.0, + "921": 849621504.0, + "922": 849621504.0, + "923": 849621504.0, + "924": 849621504.0, + "925": 849621504.0, + "926": 849621504.0, + "927": 849621504.0, + "928": 849621504.0, + "929": 849621504.0, + "930": 849621504.0, + "931": 849621504.0, + "932": 849621504.0, + "933": 849621504.0, + "934": 849621504.0, + "935": 849621504.0, + "936": 849621504.0, + "937": 849621504.0, + "938": 849621504.0, + "939": 849621504.0, + "940": 849621504.0, + "941": 849621504.0, + "942": 849621504.0, + "943": 849621504.0, + "944": 849621504.0, + "945": 849621504.0, + "946": 849621504.0, + "947": 849621504.0, + "948": 849621504.0, + "949": 849621504.0, + "950": 849621504.0, + "951": 849621504.0, + "952": 849621504.0, + "953": 849621504.0, + "954": 849621504.0, + "955": 849621504.0, + "956": 849621504.0, + "957": 849621504.0, + "958": 849621504.0, + "959": 849621504.0, + "960": 849621504.0, + "961": 849621504.0, + "962": 849621504.0, + "963": 849621504.0, + "964": 849621504.0, + "965": 849621504.0, + "966": 849621504.0, + "967": 849621504.0, + "968": 849621504.0, + "969": 849621504.0, + "970": 849621504.0, + "971": 849621504.0, + "972": 849621504.0, + "973": 849621504.0, + "974": 849621504.0, + "975": 849621504.0, + "976": 849621504.0, + "977": 849621504.0, + "978": 849621504.0, + "979": 849621504.0, + "980": 849621504.0, + "981": 849621504.0, + "982": 849621504.0, + "983": 849621504.0, + "984": 849621504.0, + "985": 849621504.0, + "986": 849621504.0, + "987": 849621504.0, + "988": 849621504.0, + "989": 849621504.0, + "990": 849621504.0, + "991": 849621504.0, + "992": 849621504.0, + "993": 849621504.0, + "994": 849621504.0, + "995": 849621504.0, + "996": 849621504.0, + "997": 849621504.0, + "998": 849621504.0, + "999": 849621504.0, + "1000": 849621504.0, + "1001": 849621504.0, + "1002": 849621504.0, + "1003": 849621504.0, + "1004": 849621504.0, + "1005": 849621504.0, + "1006": 849621504.0, + "1007": 849621504.0, + "1008": 849621504.0, + "1009": 849621504.0, + "1010": 849621504.0, + "1011": 849621504.0, + "1012": 849621504.0, + "1013": 849621504.0, + "1014": 849621504.0, + "1015": 849621504.0, + "1016": 849621504.0, + "1017": 849621504.0, + "1018": 849621504.0, + "1019": 849621504.0, + "1020": 849621504.0, + "1021": 849621504.0, + "1022": 849621504.0, + "1023": 849621504.0, + "1024": 849621504.0, + "1025": 849621504.0, + "1026": 849621504.0, + "1027": 849621504.0, + "1028": 849621504.0, + "1029": 849621504.0, + "1030": 849621504.0, + "1031": 849621504.0, + "1032": 849621504.0, + "1033": 849621504.0, + "1034": 849621504.0, + "1035": 849621504.0, + "1036": 849621504.0, + "1037": 849621504.0, + "1038": 849621504.0, + "1039": 849621504.0, + "1040": 849621504.0, + "1041": 849621504.0, + "1042": 849621504.0, + "1043": 849621504.0, + "1044": 849621504.0, + "1045": 849621504.0, + "1046": 849621504.0, + "1047": 849621504.0, + "1048": 849621504.0, + "1049": 849621504.0, + "1050": 849621504.0, + "1051": 849621504.0, + "1052": 849621504.0, + "1053": 849621504.0, + "1054": 849621504.0, + "1055": 849621504.0, + "1056": 849621504.0, + "1057": 849621504.0, + "1058": 849621504.0, + "1059": 849621504.0, + "1060": 849621504.0, + "1061": 849621504.0, + "1062": 849621504.0, + "1063": 849621504.0, + "1064": 849621504.0, + "1065": 849621504.0, + "1066": 849621504.0, + "1067": 849621504.0, + "1068": 849621504.0, + "1069": 849621504.0, + "1070": 849621504.0, + "1071": 849621504.0, + "1072": 849621504.0, + "1073": 849621504.0, + "1074": 849621504.0, + "1075": 849621504.0, + "1076": 849621504.0, + "1077": 849621504.0, + "1078": 849621504.0, + "1079": 849621504.0, + "1080": 849621504.0, + "1081": 849621504.0, + "1082": 849621504.0, + "1083": 849621504.0, + "1084": 849621504.0, + "1085": 849621504.0, + "1086": 849621504.0, + "1087": 849621504.0, + "1088": 849621504.0, + "1089": 849621504.0, + "1090": 849621504.0, + "1091": 849621504.0, + "1092": 849621504.0, + "1093": 849621504.0, + "1094": 849621504.0, + "1095": 849621504.0, + "1096": 849621504.0, + "1097": 849621504.0, + "1098": 849621504.0, + "1099": 849621504.0, + "1100": 849621504.0, + "1101": 849621504.0, + "1102": 849621504.0, + "1103": 849621504.0, + "1104": 849621504.0, + "1105": 849621504.0, + "1106": 849621504.0, + "1107": 849621504.0, + "1108": 849621504.0, + "1109": 849621504.0, + "1110": 849621504.0, + "1111": 849621504.0, + "1112": 849621504.0, + "1113": 849621504.0, + "1114": 849621504.0, + "1115": 849621504.0, + "1116": 849621504.0, + "1117": 849621504.0, + "1118": 849621504.0, + "1119": 849621504.0, + "1120": 849621504.0, + "1121": 849621504.0, + "1122": 849621504.0, + "1123": 849621504.0, + "1124": 849621504.0, + "1125": 849621504.0, + "1126": 849621504.0, + "1127": 849621504.0, + "1128": 849621504.0, + "1129": 849621504.0, + "1130": 849621504.0, + "1131": 849621504.0, + "1132": 849621504.0, + "1133": 849621504.0, + "1134": 849621504.0, + "1135": 849621504.0, + "1136": 849621504.0, + "1137": 849621504.0, + "1138": 849621504.0, + "1139": 849621504.0, + "1140": 849621504.0, + "1141": 849621504.0, + "1142": 849621504.0, + "1143": 849621504.0, + "1144": 849621504.0, + "1145": 849621504.0, + "1146": 849621504.0, + "1147": 849621504.0, + "1148": 849621504.0, + "1149": 849621504.0, + "1150": 849621504.0, + "1151": 849621504.0, + "1152": 849621504.0, + "1153": 849621504.0, + "1154": 849621504.0, + "1155": 849621504.0, + "1156": 849621504.0, + "1157": 849621504.0, + "1158": 849621504.0, + "1159": 849621504.0, + "1160": 849621504.0, + "1161": 849621504.0, + "1162": 849621504.0, + "1163": 849621504.0, + "1164": 849621504.0, + "1165": 849621504.0, + "1166": 849621504.0, + "1167": 849621504.0, + "1168": 849621504.0, + "1169": 849621504.0, + "1170": 849621504.0, + "1171": 849621504.0, + "1172": 849621504.0, + "1173": 849621504.0, + "1174": 849621504.0, + "1175": 849621504.0, + "1176": 849621504.0, + "1177": 849621504.0, + "1178": 849621504.0, + "1179": 849621504.0, + "1180": 849621504.0, + "1181": 849621504.0, + "1182": 849621504.0, + "1183": 849621504.0, + "1184": 849621504.0, + "1185": 849621504.0, + "1186": 849621504.0, + "1187": 849621504.0, + "1188": 849621504.0, + "1189": 849621504.0, + "1190": 849621504.0, + "1191": 849621504.0, + "1192": 849621504.0, + "1193": 849621504.0, + "1194": 849621504.0, + "1195": 849621504.0, + "1196": 849621504.0, + "1197": 849621504.0, + "1198": 849621504.0, + "1199": 849621504.0, + "1200": 849621504.0, + "1201": 849621504.0, + "1202": 849621504.0, + "1203": 849621504.0, + "1204": 849621504.0, + "1205": 849621504.0, + "1206": 849621504.0, + "1207": 849621504.0, + "1208": 849621504.0, + "1209": 849621504.0, + "1210": 849621504.0, + "1211": 849621504.0, + "1212": 849621504.0, + "1213": 849621504.0, + "1214": 849621504.0, + "1215": 849621504.0, + "1216": 849621504.0, + "1217": 849621504.0, + "1218": 849621504.0, + "1219": 849621504.0, + "1220": 849621504.0, + "1221": 849621504.0, + "1222": 849621504.0, + "1223": 849621504.0, + "1224": 849621504.0, + "1225": 849621504.0, + "1226": 849621504.0, + "1227": 849621504.0, + "1228": 849621504.0, + "1229": 849621504.0, + "1230": 849621504.0, + "1231": 849621504.0, + "1232": 849621504.0, + "1233": 849621504.0, + "1234": 849621504.0, + "1235": 849621504.0, + "1236": 849621504.0, + "1237": 849621504.0, + "1238": 849621504.0, + "1239": 849621504.0, + "1240": 849621504.0, + "1241": 849621504.0, + "1242": 849621504.0, + "1243": 849621504.0, + "1244": 849621504.0, + "1245": 849621504.0, + "1246": 849621504.0, + "1247": 849621504.0, + "1248": 849621504.0, + "1249": 849621504.0, + "1250": 849621504.0, + "1251": 849621504.0, + "1252": 849621504.0, + "1253": 849621504.0, + "1254": 849621504.0, + "1255": 849621504.0, + "1256": 849621504.0, + "1257": 849621504.0, + "1258": 849621504.0, + "1259": 849621504.0, + "1260": 849621504.0, + "1261": 849621504.0, + "1262": 849621504.0, + "1263": 849621504.0, + "1264": 849621504.0, + "1265": 849621504.0, + "1266": 849621504.0, + "1267": 849621504.0, + "1268": 849621504.0, + "1269": 849621504.0, + "1270": 849621504.0, + "1271": 849621504.0, + "1272": 849621504.0, + "1273": 849621504.0, + "1274": 849621504.0, + "1275": 849621504.0, + "1276": 849621504.0, + "1277": 849621504.0, + "1278": 849621504.0, + "1279": 849621504.0, + "1280": 849621504.0, + "1281": 849621504.0, + "1282": 849621504.0, + "1283": 849621504.0, + "1284": 849621504.0, + "1285": 849621504.0, + "1286": 849621504.0, + "1287": 849621504.0, + "1288": 849621504.0, + "1289": 849621504.0, + "1290": 849621504.0, + "1291": 849621504.0, + "1292": 849621504.0, + "1293": 849621504.0, + "1294": 849621504.0, + "1295": 849621504.0, + "1296": 849621504.0, + "1297": 849621504.0, + "1298": 849621504.0, + "1299": 849621504.0, + "1300": 849621504.0, + "1301": 849621504.0, + "1302": 849621504.0, + "1303": 849621504.0, + "1304": 849621504.0, + "1305": 849621504.0, + "1306": 849621504.0, + "1307": 849621504.0, + "1308": 849621504.0, + "1309": 849621504.0, + "1310": 849621504.0, + "1311": 849621504.0, + "1312": 849621504.0, + "1313": 849621504.0, + "1314": 849621504.0, + "1315": 849621504.0, + "1316": 849621504.0, + "1317": 849621504.0, + "1318": 849621504.0, + "1319": 849621504.0, + "1320": 849621504.0, + "1321": 849621504.0, + "1322": 849621504.0, + "1323": 849621504.0, + "1324": 849621504.0, + "1325": 849621504.0, + "1326": 849621504.0, + "1327": 849621504.0, + "1328": 849621504.0, + "1329": 849621504.0, + "1330": 849621504.0, + "1331": 849621504.0, + "1332": 849621504.0, + "1333": 849621504.0, + "1334": 849621504.0, + "1335": 849621504.0, + "1336": 849621504.0, + "1337": 849621504.0, + "1338": 849621504.0, + "1339": 849621504.0, + "1340": 849621504.0, + "1341": 849621504.0, + "1342": 849621504.0, + "1343": 849621504.0, + "1344": 849621504.0, + "1345": 849621504.0, + "1346": 849621504.0, + "1347": 849621504.0, + "1348": 849621504.0, + "1349": 849621504.0, + "1350": 849621504.0, + "1351": 849621504.0, + "1352": 849621504.0, + "1353": 849621504.0, + "1354": 849621504.0, + "1355": 849621504.0, + "1356": 849621504.0, + "1357": 849621504.0, + "1358": 849621504.0, + "1359": 849621504.0, + "1360": 849621504.0, + "1361": 849621504.0, + "1362": 849621504.0, + "1363": 849621504.0, + "1364": 849621504.0, + "1365": 849621504.0, + "1366": 849621504.0, + "1367": 849621504.0, + "1368": 849621504.0, + "1369": 849621504.0, + "1370": 849621504.0, + "1371": 849621504.0, + "1372": 849621504.0, + "1373": 849621504.0, + "1374": 849621504.0, + "1375": 849621504.0, + "1376": 849621504.0, + "1377": 849621504.0, + "1378": 849621504.0, + "1379": 849621504.0, + "1380": 849621504.0, + "1381": 849621504.0, + "1382": 849621504.0, + "1383": 849621504.0, + "1384": 849621504.0, + "1385": 849621504.0, + "1386": 849621504.0, + "1387": 849621504.0, + "1388": 849621504.0, + "1389": 849621504.0, + "1390": 849621504.0, + "1391": 849621504.0, + "1392": 849621504.0, + "1393": 849621504.0, + "1394": 849621504.0, + "1395": 849621504.0, + "1396": 849621504.0, + "1397": 849621504.0, + "1398": 849621504.0, + "1399": 849621504.0, + "1400": 849621504.0, + "1401": 849621504.0, + "1402": 849621504.0, + "1403": 849621504.0, + "1404": 849621504.0, + "1405": 849621504.0, + "1406": 849621504.0, + "1407": 849621504.0, + "1408": 849621504.0, + "1409": 849621504.0, + "1410": 849621504.0, + "1411": 849621504.0, + "1412": 849621504.0, + "1413": 849621504.0, + "1414": 849621504.0, + "1415": 849621504.0, + "1416": 849621504.0, + "1417": 849621504.0, + "1418": 849621504.0, + "1419": 849621504.0, + "1420": 849621504.0, + "1421": 849621504.0, + "1422": 849621504.0, + "1423": 849621504.0, + "1424": 849621504.0, + "1425": 849621504.0, + "1426": 849621504.0, + "1427": 849621504.0, + "1428": 849621504.0, + "1429": 849621504.0, + "1430": 849621504.0, + "1431": 849621504.0, + "1432": 849621504.0, + "1433": 849621504.0, + "1434": 849621504.0, + "1435": 849621504.0, + "1436": 849621504.0, + "1437": 849621504.0, + "1438": 849621504.0, + "1439": 849621504.0, + "1440": 849621504.0, + "1441": 849621504.0, + "1442": 849621504.0, + "1443": 849621504.0, + "1444": 849621504.0, + "1445": 849621504.0, + "1446": 849621504.0, + "1447": 849621504.0, + "1448": 849621504.0, + "1449": 849621504.0, + "1450": 849621504.0, + "1451": 849621504.0, + "1452": 849621504.0, + "1453": 849621504.0, + "1454": 849621504.0, + "1455": 849621504.0, + "1456": 849621504.0, + "1457": 849621504.0, + "1458": 849621504.0, + "1459": 849621504.0, + "1460": 849621504.0, + "1461": 849621504.0, + "1462": 849621504.0, + "1463": 849621504.0, + "1464": 849621504.0, + "1465": 849621504.0, + "1466": 849621504.0, + "1467": 849621504.0, + "1468": 849621504.0, + "1469": 849621504.0, + "1470": 849621504.0, + "1471": 849621504.0, + "1472": 849621504.0, + "1473": 849621504.0, + "1474": 849621504.0, + "1475": 849621504.0, + "1476": 849621504.0, + "1477": 849621504.0, + "1478": 849621504.0, + "1479": 849621504.0, + "1480": 849621504.0, + "1481": 849621504.0, + "1482": 849621504.0, + "1483": 849621504.0, + "1484": 849621504.0, + "1485": 849621504.0, + "1486": 849621504.0, + "1487": 849621504.0, + "1488": 849621504.0, + "1489": 849621504.0, + "1490": 849621504.0, + "1491": 849621504.0, + "1492": 849621504.0, + "1493": 849621504.0, + "1494": 849621504.0, + "1495": 849621504.0, + "1496": 849621504.0, + "1497": 849621504.0, + "1498": 849621504.0, + "1499": 849621504.0, + "1500": 849621504.0, + "1501": 849621504.0, + "1502": 849621504.0, + "1503": 849621504.0, + "1504": 849621504.0, + "1505": 849621504.0, + "1506": 849621504.0, + "1507": 849621504.0, + "1508": 849621504.0, + "1509": 849621504.0, + "1510": 849621504.0, + "1511": 849621504.0, + "1512": 849621504.0, + "1513": 849621504.0, + "1514": 849621504.0, + "1515": 849621504.0, + "1516": 849621504.0, + "1517": 849621504.0, + "1518": 849621504.0, + "1519": 849621504.0, + "1520": 849621504.0, + "1521": 849621504.0, + "1522": 849621504.0, + "1523": 849621504.0, + "1524": 849621504.0, + "1525": 849621504.0, + "1526": 849621504.0, + "1527": 849621504.0, + "1528": 849621504.0, + "1529": 849621504.0, + "1530": 849621504.0, + "1531": 849621504.0, + "1532": 849621504.0, + "1533": 849621504.0, + "1534": 849621504.0, + "1535": 849621504.0, + "1536": 849621504.0, + "1537": 849621504.0, + "1538": 849621504.0, + "1539": 849621504.0, + "1540": 849621504.0, + "1541": 849621504.0, + "1542": 849621504.0, + "1543": 849621504.0, + "1544": 849621504.0, + "1545": 849621504.0, + "1546": 849621504.0, + "1547": 849621504.0, + "1548": 849621504.0, + "1549": 849621504.0, + "1550": 849621504.0, + "1551": 849621504.0, + "1552": 849621504.0, + "1553": 849621504.0, + "1554": 849621504.0, + "1555": 849621504.0, + "1556": 849621504.0, + "1557": 849621504.0, + "1558": 849621504.0, + "1559": 849621504.0, + "1560": 849621504.0, + "1561": 849621504.0, + "1562": 849621504.0, + "1563": 849621504.0, + "1564": 849621504.0, + "1565": 849621504.0, + "1566": 849621504.0, + "1567": 849621504.0, + "1568": 849621504.0, + "1569": 849621504.0, + "1570": 849621504.0, + "1571": 849621504.0, + "1572": 849621504.0, + "1573": 849621504.0, + "1574": 849621504.0, + "1575": 849621504.0, + "1576": 849621504.0, + "1577": 849621504.0, + "1578": 849621504.0, + "1579": 849621504.0, + "1580": 849621504.0, + "1581": 849621504.0, + "1582": 849621504.0, + "1583": 849621504.0, + "1584": 849621504.0, + "1585": 849621504.0, + "1586": 849621504.0, + "1587": 849621504.0, + "1588": 849621504.0, + "1589": 849621504.0, + "1590": 849621504.0, + "1591": 849621504.0, + "1592": 849621504.0, + "1593": 849621504.0, + "1594": 849621504.0, + "1595": 849621504.0, + "1596": 849621504.0, + "1597": 849621504.0, + "1598": 849621504.0, + "1599": 849621504.0, + "1600": 849621504.0, + "1601": 849621504.0, + "1602": 849621504.0, + "1603": 849621504.0, + "1604": 849621504.0, + "1605": 849621504.0, + "1606": 849621504.0, + "1607": 849621504.0, + "1608": 849621504.0, + "1609": 849621504.0, + "1610": 849621504.0, + "1611": 849621504.0, + "1612": 849621504.0, + "1613": 849621504.0, + "1614": 849621504.0, + "1615": 849621504.0, + "1616": 849621504.0, + "1617": 849621504.0, + "1618": 849621504.0, + "1619": 849621504.0, + "1620": 849621504.0, + "1621": 849621504.0, + "1622": 849621504.0, + "1623": 849621504.0, + "1624": 849621504.0, + "1625": 849621504.0, + "1626": 849621504.0, + "1627": 849621504.0, + "1628": 849621504.0, + "1629": 849621504.0, + "1630": 849621504.0, + "1631": 849621504.0, + "1632": 849621504.0, + "1633": 849621504.0, + "1634": 849621504.0, + "1635": 849621504.0, + "1636": 849621504.0, + "1637": 849621504.0, + "1638": 849621504.0, + "1639": 849621504.0, + "1640": 849621504.0, + "1641": 849621504.0, + "1642": 849621504.0, + "1643": 849621504.0, + "1644": 849621504.0, + "1645": 849621504.0, + "1646": 849621504.0, + "1647": 849621504.0, + "1648": 849621504.0, + "1649": 849621504.0, + "1650": 849621504.0, + "1651": 849621504.0, + "1652": 849621504.0, + "1653": 849621504.0, + "1654": 849621504.0, + "1655": 849621504.0, + "1656": 849621504.0, + "1657": 849621504.0, + "1658": 849621504.0, + "1659": 849621504.0, + "1660": 849621504.0, + "1661": 849621504.0, + "1662": 849621504.0, + "1663": 849621504.0, + "1664": 849621504.0, + "1665": 849621504.0, + "1666": 849621504.0, + "1667": 849621504.0, + "1668": 849621504.0, + "1669": 849621504.0, + "1670": 849621504.0, + "1671": 849621504.0, + "1672": 849621504.0, + "1673": 849621504.0, + "1674": 849621504.0, + "1675": 849621504.0, + "1676": 849621504.0, + "1677": 849621504.0, + "1678": 849621504.0, + "1679": 849621504.0, + "1680": 849621504.0, + "1681": 849621504.0, + "1682": 849621504.0, + "1683": 849621504.0, + "1684": 849621504.0, + "1685": 849621504.0, + "1686": 849621504.0, + "1687": 849621504.0, + "1688": 849621504.0, + "1689": 849621504.0, + "1690": 849621504.0, + "1691": 849621504.0, + "1692": 849621504.0, + "1693": 849621504.0, + "1694": 849621504.0, + "1695": 849621504.0, + "1696": 849621504.0, + "1697": 849621504.0, + "1698": 849621504.0, + "1699": 849621504.0, + "1700": 849621504.0, + "1701": 849621504.0, + "1702": 849621504.0, + "1703": 849621504.0, + "1704": 849621504.0, + "1705": 849621504.0, + "1706": 849621504.0, + "1707": 849621504.0, + "1708": 849621504.0, + "1709": 849621504.0, + "1710": 849621504.0, + "1711": 849621504.0, + "1712": 849621504.0, + "1713": 849621504.0, + "1714": 849621504.0, + "1715": 849621504.0, + "1716": 849621504.0, + "1717": 849621504.0, + "1718": 849621504.0, + "1719": 849621504.0, + "1720": 849621504.0, + "1721": 849621504.0, + "1722": 849621504.0, + "1723": 849621504.0, + "1724": 849621504.0, + "1725": 849621504.0, + "1726": 849621504.0, + "1727": 849621504.0, + "1728": 849621504.0, + "1729": 849621504.0, + "1730": 849621504.0, + "1731": 849621504.0, + "1732": 849621504.0, + "1733": 849621504.0, + "1734": 849621504.0, + "1735": 849621504.0, + "1736": 849621504.0, + "1737": 849621504.0, + "1738": 849621504.0, + "1739": 849621504.0, + "1740": 849621504.0, + "1741": 849621504.0, + "1742": 849621504.0, + "1743": 849621504.0, + "1744": 849621504.0, + "1745": 849621504.0, + "1746": 849621504.0, + "1747": 849621504.0, + "1748": 849621504.0, + "1749": 849621504.0, + "1750": 849621504.0, + "1751": 849621504.0, + "1752": 849621504.0, + "1753": 849621504.0, + "1754": 849621504.0, + "1755": 849621504.0, + "1756": 849621504.0, + "1757": 849621504.0, + "1758": 849621504.0, + "1759": 849621504.0, + "1760": 849621504.0, + "1761": 849621504.0, + "1762": 849621504.0, + "1763": 849621504.0, + "1764": 849621504.0, + "1765": 849621504.0, + "1766": 849621504.0, + "1767": 849621504.0, + "1768": 849621504.0, + "1769": 849621504.0, + "1770": 849621504.0, + "1771": 849621504.0, + "1772": 849621504.0, + "1773": 849621504.0, + "1774": 849621504.0, + "1775": 849621504.0, + "1776": 849621504.0, + "1777": 849621504.0, + "1778": 849621504.0, + "1779": 849621504.0, + "1780": 849621504.0, + "1781": 849621504.0, + "1782": 849621504.0, + "1783": 849621504.0, + "1784": 849621504.0, + "1785": 849621504.0, + "1786": 849621504.0, + "1787": 849621504.0, + "1788": 849621504.0, + "1789": 849621504.0, + "1790": 849621504.0, + "1791": 849621504.0, + "1792": 849621504.0, + "1793": 849621504.0, + "1794": 849621504.0, + "1795": 849621504.0, + "1796": 849621504.0, + "1797": 849621504.0, + "1798": 849621504.0, + "1799": 849621504.0, + "1800": 849621504.0, + "1801": 849621504.0, + "1802": 849621504.0, + "1803": 849621504.0, + "1804": 849621504.0, + "1805": 849621504.0, + "1806": 849621504.0, + "1807": 849621504.0, + "1808": 849621504.0, + "1809": 849621504.0, + "1810": 849621504.0, + "1811": 849621504.0, + "1812": 849621504.0, + "1813": 849621504.0, + "1814": 849621504.0, + "1815": 849621504.0, + "1816": 849621504.0, + "1817": 849621504.0, + "1818": 849621504.0, + "1819": 849621504.0, + "1820": 849621504.0, + "1821": 849621504.0, + "1822": 849621504.0, + "1823": 849621504.0, + "1824": 849621504.0, + "1825": 849621504.0, + "1826": 849621504.0, + "1827": 849621504.0, + "1828": 849621504.0, + "1829": 849621504.0, + "1830": 849621504.0, + "1831": 849621504.0, + "1832": 849621504.0, + "1833": 849621504.0, + "1834": 849621504.0, + "1835": 849621504.0, + "1836": 849621504.0, + "1837": 849621504.0, + "1838": 849621504.0, + "1839": 849621504.0, + "1840": 849621504.0, + "1841": 849621504.0, + "1842": 849621504.0, + "1843": 849621504.0, + "1844": 849621504.0, + "1845": 849621504.0, + "1846": 849621504.0, + "1847": 849621504.0, + "1848": 849621504.0, + "1849": 849621504.0, + "1850": 849621504.0, + "1851": 849621504.0, + "1852": 849621504.0, + "1853": 849621504.0, + "1854": 849621504.0, + "1855": 849621504.0, + "1856": 849621504.0, + "1857": 849621504.0, + "1858": 849621504.0, + "1859": 849621504.0, + "1860": 849621504.0, + "1861": 849621504.0, + "1862": 849621504.0, + "1863": 849621504.0, + "1864": 849621504.0, + "1865": 849621504.0, + "1866": 849621504.0, + "1867": 849621504.0, + "1868": 849621504.0, + "1869": 849621504.0, + "1870": 849621504.0, + "1871": 849621504.0, + "1872": 849621504.0, + "1873": 849621504.0, + "1874": 849621504.0, + "1875": 849621504.0, + "1876": 849621504.0, + "1877": 849621504.0, + "1878": 849621504.0, + "1879": 849621504.0, + "1880": 849621504.0, + "1881": 849621504.0, + "1882": 849621504.0, + "1883": 849621504.0, + "1884": 849621504.0, + "1885": 849621504.0, + "1886": 849621504.0, + "1887": 849621504.0, + "1888": 849621504.0, + "1889": 849621504.0, + "1890": 849621504.0, + "1891": 849621504.0, + "1892": 849621504.0, + "1893": 849621504.0, + "1894": 849621504.0, + "1895": 849621504.0, + "1896": 849621504.0, + "1897": 849621504.0, + "1898": 849621504.0, + "1899": 849621504.0, + "1900": 849621504.0, + "1901": 849621504.0, + "1902": 849621504.0, + "1903": 849621504.0, + "1904": 849621504.0, + "1905": 849621504.0, + "1906": 849621504.0, + "1907": 849621504.0, + "1908": 849621504.0, + "1909": 849621504.0, + "1910": 849621504.0, + "1911": 849621504.0, + "1912": 849621504.0, + "1913": 849621504.0, + "1914": 849621504.0, + "1915": 849621504.0, + "1916": 849621504.0, + "1917": 849621504.0, + "1918": 849621504.0, + "1919": 849621504.0, + "1920": 849621504.0, + "1921": 849621504.0, + "1922": 849621504.0, + "1923": 849621504.0, + "1924": 849621504.0, + "1925": 849621504.0, + "1926": 849621504.0, + "1927": 849621504.0, + "1928": 849621504.0, + "1929": 849621504.0, + "1930": 849621504.0, + "1931": 849621504.0, + "1932": 849621504.0, + "1933": 849621504.0, + "1934": 849621504.0, + "1935": 849621504.0, + "1936": 849621504.0, + "1937": 849621504.0, + "1938": 849621504.0, + "1939": 849621504.0, + "1940": 849621504.0, + "1941": 849621504.0, + "1942": 849621504.0, + "1943": 849621504.0, + "1944": 849621504.0, + "1945": 849621504.0, + "1946": 849621504.0, + "1947": 849621504.0, + "1948": 849621504.0, + "1949": 849621504.0, + "1950": 849621504.0, + "1951": 849621504.0, + "1952": 849621504.0, + "1953": 849621504.0, + "1954": 849621504.0, + "1955": 849621504.0, + "1956": 849621504.0, + "1957": 849621504.0, + "1958": 849621504.0, + "1959": 849621504.0, + "1960": 849621504.0, + "1961": 849621504.0, + "1962": 849621504.0, + "1963": 849621504.0, + "1964": 849621504.0, + "1965": 849621504.0, + "1966": 849621504.0, + "1967": 849621504.0, + "1968": 849621504.0, + "1969": 849621504.0, + "1970": 849621504.0, + "1971": 849621504.0, + "1972": 849621504.0, + "1973": 849621504.0, + "1974": 849621504.0, + "1975": 849621504.0, + "1976": 849621504.0, + "1977": 849621504.0, + "1978": 849621504.0, + "1979": 849621504.0, + "1980": 849621504.0, + "1981": 849621504.0, + "1982": 849621504.0, + "1983": 849621504.0, + "1984": 849621504.0, + "1985": 849621504.0, + "1986": 849621504.0, + "1987": 849621504.0, + "1988": 849621504.0, + "1989": 849621504.0, + "1990": 849621504.0, + "1991": 849621504.0, + "1992": 849621504.0, + "1993": 849621504.0, + "1994": 849621504.0, + "1995": 849621504.0, + "1996": 849621504.0, + "1997": 849621504.0, + "1998": 849621504.0, + "1999": 849621504.0, + "2000": 849621504.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 14.94115, + "2": 1.30868, + "3": 1.13391, + "4": 1.12792, + "5": 1.13103, + "6": 1.1383, + "7": 1.13573, + "8": 1.15789, + "9": 1.12704, + "10": 1.1241, + "11": 1.12786, + "12": 1.1288, + "13": 1.1399, + "14": 1.13165, + "15": 1.12333, + "16": 1.12398, + "17": 1.12493, + "18": 1.11586, + "19": 1.1123, + "20": 1.11192, + "21": 1.1266, + "22": 1.13629, + "23": 1.13171, + "24": 1.14969, + "25": 1.17022, + "26": 1.14634, + "27": 1.14242, + "28": 1.14353, + "29": 1.14554, + "30": 1.28826, + "31": 1.14265, + "32": 1.14023, + "33": 1.15286, + "34": 1.14975, + "35": 1.13988, + "36": 1.62757, + "37": 2.22703, + "38": 1.36074, + "39": 1.1325, + "40": 1.14106, + "41": 1.14114, + "42": 1.13305, + "43": 1.12375, + "44": 1.12631, + "45": 1.12358, + "46": 1.12334, + "47": 1.12398, + "48": 1.12749, + "49": 1.13897, + "50": 1.13563, + "51": 1.13628, + "52": 1.12935, + "53": 1.12779, + "54": 1.13147, + "55": 1.1279, + "56": 1.12777, + "57": 1.1269, + "58": 1.13989, + "59": 1.13378, + "60": 1.13552, + "61": 1.12879, + "62": 1.4796, + "63": 1.12843, + "64": 1.12488, + "65": 1.12888, + "66": 1.14028, + "67": 1.13532, + "68": 1.13278, + "69": 1.12779, + "70": 1.12468, + "71": 1.12483, + "72": 1.12423, + "73": 1.12335, + "74": 1.12699, + "75": 1.13379, + "76": 1.13001, + "77": 1.12994, + "78": 1.13166, + "79": 1.12415, + "80": 1.126, + "81": 1.16016, + "82": 1.13845, + "83": 1.13882, + "84": 1.14455, + "85": 1.46908, + "86": 1.1259, + "87": 1.12119, + "88": 1.12312, + "89": 1.12593, + "90": 1.51995, + "91": 1.16022, + "92": 1.1304, + "93": 1.13161, + "94": 1.13511, + "95": 1.13911, + "96": 1.80205, + "97": 1.13368, + "98": 1.13335, + "99": 1.13549, + "100": 1.13409, + "101": 1.13703, + "102": 1.14592, + "103": 1.13516, + "104": 1.13661, + "105": 1.13299, + "106": 1.13577, + "107": 1.13657, + "108": 1.13144, + "109": 1.14828, + "110": 1.15036, + "111": 1.1486, + "112": 1.14183, + "113": 1.14297, + "114": 1.1411, + "115": 1.14318, + "116": 1.14291, + "117": 1.14168, + "118": 1.15055, + "119": 1.1482, + "120": 1.15352, + "121": 1.13046, + "122": 1.145, + "123": 1.14278, + "124": 1.1428, + "125": 1.14189, + "126": 1.13609, + "127": 1.14025, + "128": 1.14097, + "129": 1.13489, + "130": 1.13417, + "131": 1.13581, + "132": 1.13708, + "133": 1.17896, + "134": 1.13176, + "135": 1.12984, + "136": 1.1435, + "137": 1.15088, + "138": 1.14391, + "139": 1.14409, + "140": 1.14238, + "141": 1.14313, + "142": 1.1493, + "143": 1.13518, + "144": 1.13229, + "145": 1.13749, + "146": 1.15049, + "147": 1.16077, + "148": 1.14254, + "149": 1.14071, + "150": 1.14075, + "151": 1.13943, + "152": 1.15276, + "153": 1.15369, + "154": 1.14618, + "155": 1.14225, + "156": 1.14285, + "157": 1.14106, + "158": 1.14415, + "159": 1.14445, + "160": 1.14934, + "161": 1.14229, + "162": 1.14167, + "163": 1.14058, + "164": 1.14064, + "165": 1.14012, + "166": 1.15198, + "167": 1.15221, + "168": 1.1471, + "169": 1.14122, + "170": 1.14769, + "171": 1.14073, + "172": 1.14205, + "173": 1.14583, + "174": 1.14217, + "175": 1.14015, + "176": 1.14319, + "177": 1.14097, + "178": 1.14115, + "179": 1.14122, + "180": 1.15137, + "181": 1.14856, + "182": 1.15203, + "183": 1.14535, + "184": 1.13997, + "185": 1.15174, + "186": 1.18192, + "187": 1.14929, + "188": 1.14842, + "189": 1.14724, + "190": 1.14922, + "191": 1.14932, + "192": 1.14856, + "193": 1.1562, + "194": 1.153, + "195": 1.16371, + "196": 1.14525, + "197": 1.1411, + "198": 1.14592, + "199": 1.14301, + "200": 1.15088, + "201": 1.14229, + "202": 1.14171, + "203": 1.14083, + "204": 1.13968, + "205": 1.13977, + "206": 1.14177, + "207": 1.15548, + "208": 1.15609, + "209": 1.14509, + "210": 1.1487, + "211": 1.14163, + "212": 1.13971, + "213": 1.15326, + "214": 1.14129, + "215": 1.14055, + "216": 1.13893, + "217": 1.14191, + "218": 1.1418, + "219": 1.14249, + "220": 1.14162, + "221": 1.14077, + "222": 1.15513, + "223": 1.15668, + "224": 1.14515, + "225": 1.14589, + "226": 1.14548, + "227": 1.14318, + "228": 1.14204, + "229": 1.14391, + "230": 1.14565, + "231": 1.1439, + "232": 1.14309, + "233": 1.14396, + "234": 1.14146, + "235": 1.14229, + "236": 1.14106, + "237": 1.14362, + "238": 1.15203, + "239": 1.1942, + "240": 1.18025, + "241": 1.15197, + "242": 1.15276, + "243": 1.15399, + "244": 1.15628, + "245": 1.14958, + "246": 1.14931, + "247": 1.14093, + "248": 1.13869, + "249": 1.1385, + "250": 1.13897, + "251": 1.13787, + "252": 1.13939, + "253": 1.17282, + "254": 1.13361, + "255": 1.13502, + "256": 1.13895, + "257": 1.16245, + "258": 1.1352, + "259": 1.15685, + "260": 1.14637, + "261": 1.2867, + "262": 1.13699, + "263": 1.13959, + "264": 1.15414, + "265": 1.14324, + "266": 1.14515, + "267": 1.14328, + "268": 1.14359, + "269": 1.144, + "270": 1.15446, + "271": 1.15182, + "272": 1.15575, + "273": 1.15561, + "274": 1.15762, + "275": 1.15307, + "276": 1.1516, + "277": 1.1569, + "278": 1.15789, + "279": 1.168, + "280": 1.16711, + "281": 1.16858, + "282": 1.16899, + "283": 1.15631, + "284": 1.15543, + "285": 1.15685, + "286": 1.15663, + "287": 1.15204, + "288": 1.15333, + "289": 1.15257, + "290": 1.14865, + "291": 1.15067, + "292": 1.15626, + "293": 1.15161, + "294": 1.15116, + "295": 1.15102, + "296": 1.15104, + "297": 1.17304, + "298": 1.17562, + "299": 1.17694, + "300": 1.15026, + "301": 1.15562, + "302": 1.15582, + "303": 1.15039, + "304": 1.14517, + "305": 1.14745, + "306": 1.15392, + "307": 1.15054, + "308": 1.14391, + "309": 1.1426, + "310": 1.1434, + "311": 1.14297, + "312": 1.14164, + "313": 1.15234, + "314": 1.14891, + "315": 1.14745, + "316": 1.15325, + "317": 1.15145, + "318": 1.51061, + "319": 1.13797, + "320": 1.13871, + "321": 1.20976, + "322": 1.19788, + "323": 1.14258, + "324": 1.14169, + "325": 1.14227, + "326": 1.1426, + "327": 1.14596, + "328": 1.14584, + "329": 1.14606, + "330": 1.13676, + "331": 1.14712, + "332": 1.14502, + "333": 1.14602, + "334": 1.14598, + "335": 1.15781, + "336": 1.15666, + "337": 1.1498, + "338": 1.15651, + "339": 1.15267, + "340": 1.14703, + "341": 1.14889, + "342": 1.14863, + "343": 1.14731, + "344": 1.1479, + "345": 1.20819, + "346": 1.15653, + "347": 1.15548, + "348": 1.15594, + "349": 1.15558, + "350": 1.15652, + "351": 1.15348, + "352": 1.15517, + "353": 1.15665, + "354": 1.15895, + "355": 1.15829, + "356": 1.16229, + "357": 1.17016, + "358": 1.16317, + "359": 1.18492, + "360": 1.20126, + "361": 1.19034, + "362": 1.18723, + "363": 1.16724, + "364": 1.14627, + "365": 1.14394, + "366": 1.14503, + "367": 1.14264, + "368": 1.14464, + "369": 1.14478, + "370": 1.14447, + "371": 1.15012, + "372": 1.14509, + "373": 1.14362, + "374": 1.14617, + "375": 1.14658, + "376": 1.13748, + "377": 1.15141, + "378": 1.14564, + "379": 1.14278, + "380": 1.14166, + "381": 1.14361, + "382": 1.14293, + "383": 1.14196, + "384": 1.14178, + "385": 1.14053, + "386": 1.14184, + "387": 1.14451, + "388": 1.14162, + "389": 1.1419, + "390": 1.14477, + "391": 1.15539, + "392": 1.16117, + "393": 1.16925, + "394": 1.16815, + "395": 1.1561, + "396": 1.15146, + "397": 1.15422, + "398": 1.14884, + "399": 1.14136, + "400": 1.14059, + "401": 1.14105, + "402": 1.14013, + "403": 1.15094, + "404": 1.13492, + "405": 1.1425, + "406": 1.14173, + "407": 1.14385, + "408": 1.14421, + "409": 1.14226, + "410": 1.1417, + "411": 1.1511, + "412": 1.15763, + "413": 1.15891, + "414": 1.15294, + "415": 1.15191, + "416": 1.15346, + "417": 1.15001, + "418": 1.15279, + "419": 1.14974, + "420": 1.14848, + "421": 1.14722, + "422": 1.15396, + "423": 1.1499, + "424": 1.15269, + "425": 1.15087, + "426": 1.14945, + "427": 1.15106, + "428": 1.15515, + "429": 1.14379, + "430": 1.16231, + "431": 1.18658, + "432": 1.17212, + "433": 1.16725, + "434": 1.17832, + "435": 1.16254, + "436": 1.16094, + "437": 1.15865, + "438": 1.16104, + "439": 1.1621, + "440": 1.13911, + "441": 1.13485, + "442": 1.13534, + "443": 1.13627, + "444": 1.13432, + "445": 1.13868, + "446": 1.13561, + "447": 1.13518, + "448": 1.1365, + "449": 1.13444, + "450": 1.13455, + "451": 1.14098, + "452": 1.15368, + "453": 1.1566, + "454": 1.15931, + "455": 1.18151, + "456": 1.16215, + "457": 1.16012, + "458": 1.15916, + "459": 1.15837, + "460": 1.16214, + "461": 1.1652, + "462": 1.16044, + "463": 1.16179, + "464": 1.163, + "465": 1.16332, + "466": 1.15968, + "467": 1.16196, + "468": 1.1592, + "469": 1.15988, + "470": 1.16081, + "471": 1.16128, + "472": 1.15868, + "473": 1.16004, + "474": 1.16125, + "475": 1.15956, + "476": 1.16733, + "477": 1.18857, + "478": 1.15838, + "479": 1.16068, + "480": 1.16004, + "481": 1.15956, + "482": 1.15757, + "483": 1.15802, + "484": 1.16061, + "485": 1.15848, + "486": 1.16058, + "487": 1.15819, + "488": 1.15991, + "489": 1.15831, + "490": 1.1589, + "491": 1.16144, + "492": 1.15934, + "493": 1.15973, + "494": 1.16104, + "495": 1.15933, + "496": 1.16173, + "497": 1.16203, + "498": 1.16059, + "499": 1.16461, + "500": 1.16533, + "501": 1.1723, + "502": 1.17075, + "503": 1.17256, + "504": 1.16176, + "505": 1.15972, + "506": 1.16185, + "507": 1.21311, + "508": 1.16326, + "509": 1.15384, + "510": 1.15071, + "511": 1.15307, + "512": 1.15748, + "513": 1.1518, + "514": 1.15181, + "515": 1.15338, + "516": 1.1524, + "517": 1.15481, + "518": 1.15358, + "519": 1.16302, + "520": 1.16218, + "521": 1.15461, + "522": 1.157, + "523": 1.15817, + "524": 1.15517, + "525": 1.15361, + "526": 1.15183, + "527": 1.15237, + "528": 1.15423, + "529": 1.15637, + "530": 1.15521, + "531": 1.15012, + "532": 1.15132, + "533": 1.1495, + "534": 1.14919, + "535": 1.1546, + "536": 1.15442, + "537": 1.1514, + "538": 1.15195, + "539": 1.15221, + "540": 1.15639, + "541": 1.1549, + "542": 1.15495, + "543": 1.15683, + "544": 1.16361, + "545": 1.16186, + "546": 1.15697, + "547": 1.15978, + "548": 1.16151, + "549": 1.15737, + "550": 1.15451, + "551": 1.16057, + "552": 1.20604, + "553": 1.15937, + "554": 1.21638, + "555": 1.16193, + "556": 1.16004, + "557": 1.15937, + "558": 1.15924, + "559": 1.15864, + "560": 1.16064, + "561": 1.15935, + "562": 1.43389, + "563": 1.16041, + "564": 1.16122, + "565": 1.49173, + "566": 1.15954, + "567": 1.17345, + "568": 1.16261, + "569": 1.15966, + "570": 1.1607, + "571": 1.15553, + "572": 1.1568, + "573": 1.15385, + "574": 1.15701, + "575": 1.15849, + "576": 1.15634, + "577": 1.15908, + "578": 1.15576, + "579": 1.15627, + "580": 1.14973, + "581": 1.16027, + "582": 1.16176, + "583": 1.15493, + "584": 1.15722, + "585": 1.15744, + "586": 1.15502, + "587": 1.1559, + "588": 1.15496, + "589": 1.16378, + "590": 1.16595, + "591": 1.16611, + "592": 1.16989, + "593": 1.16842, + "594": 1.17261, + "595": 1.15925, + "596": 1.16083, + "597": 1.16113, + "598": 1.16297, + "599": 1.16456, + "600": 1.15983, + "601": 1.16187, + "602": 1.15943, + "603": 1.15985, + "604": 1.1592, + "605": 1.15871, + "606": 1.16032, + "607": 1.15919, + "608": 1.17988, + "609": 1.16067, + "610": 1.18157, + "611": 1.15299, + "612": 1.15282, + "613": 1.15274, + "614": 1.15344, + "615": 1.15192, + "616": 1.15757, + "617": 1.15404, + "618": 1.16198, + "619": 1.12381, + "620": 1.11492, + "621": 1.14943, + "622": 1.16512, + "623": 1.16958, + "624": 1.16409, + "625": 1.15844, + "626": 1.14917, + "627": 1.15285, + "628": 1.15477, + "629": 1.15363, + "630": 1.15213, + "631": 1.14647, + "632": 1.14867, + "633": 1.15423, + "634": 1.15566, + "635": 1.15345, + "636": 1.15319, + "637": 1.1511, + "638": 1.15409, + "639": 1.15188, + "640": 1.15258, + "641": 1.15414, + "642": 1.15983, + "643": 1.15819, + "644": 1.15887, + "645": 1.15631, + "646": 1.15765, + "647": 1.16277, + "648": 1.16768, + "649": 1.17095, + "650": 1.16972, + "651": 1.16894, + "652": 1.16584, + "653": 1.1612, + "654": 1.17303, + "655": 1.16406, + "656": 1.1617, + "657": 1.16573, + "658": 1.16082, + "659": 1.16677, + "660": 1.16969, + "661": 1.16374, + "662": 1.16155, + "663": 1.16674, + "664": 1.16865, + "665": 1.16719, + "666": 1.16772, + "667": 1.16872, + "668": 1.16616, + "669": 1.16505, + "670": 1.16449, + "671": 1.16777, + "672": 1.16457, + "673": 1.16059, + "674": 1.16013, + "675": 1.1589, + "676": 1.1645, + "677": 1.16737, + "678": 1.16262, + "679": 1.44417, + "680": 1.16641, + "681": 1.16441, + "682": 1.16834, + "683": 1.17163, + "684": 1.16041, + "685": 1.16815, + "686": 1.16615, + "687": 1.1689, + "688": 1.16377, + "689": 1.16277, + "690": 1.15926, + "691": 1.15823, + "692": 1.15747, + "693": 1.15897, + "694": 1.15722, + "695": 1.15679, + "696": 1.15619, + "697": 1.15686, + "698": 1.15548, + "699": 1.15619, + "700": 1.15662, + "701": 1.15701, + "702": 1.15611, + "703": 1.1578, + "704": 1.15921, + "705": 1.15626, + "706": 1.15696, + "707": 1.15676, + "708": 1.15718, + "709": 1.15643, + "710": 1.16154, + "711": 1.15995, + "712": 1.159, + "713": 1.16786, + "714": 1.15799, + "715": 1.15749, + "716": 1.52131, + "717": 1.15676, + "718": 1.16066, + "719": 1.15878, + "720": 1.16243, + "721": 1.15801, + "722": 1.16032, + "723": 1.15929, + "724": 1.16338, + "725": 1.15949, + "726": 1.16444, + "727": 1.31697, + "728": 1.15571, + "729": 1.15513, + "730": 1.15845, + "731": 1.16172, + "732": 1.15814, + "733": 1.1597, + "734": 1.15388, + "735": 1.15282, + "736": 1.15589, + "737": 1.15547, + "738": 1.1547, + "739": 1.15614, + "740": 1.15546, + "741": 1.15558, + "742": 1.15607, + "743": 1.15425, + "744": 1.15442, + "745": 1.16502, + "746": 1.15566, + "747": 1.15865, + "748": 1.15828, + "749": 1.16418, + "750": 1.15709, + "751": 1.15988, + "752": 1.15915, + "753": 1.15069, + "754": 1.15176, + "755": 1.15161, + "756": 1.1502, + "757": 1.14643, + "758": 1.7155, + "759": 1.15471, + "760": 1.15638, + "761": 1.15684, + "762": 1.16005, + "763": 1.1585, + "764": 1.16197, + "765": 1.22988, + "766": 1.16563, + "767": 1.16594, + "768": 1.16751, + "769": 1.16167, + "770": 1.16736, + "771": 1.16232, + "772": 1.16021, + "773": 1.16138, + "774": 1.16446, + "775": 1.15216, + "776": 1.15086, + "777": 1.15506, + "778": 1.15465, + "779": 1.15872, + "780": 1.15533, + "781": 1.15836, + "782": 1.15778, + "783": 1.21735, + "784": 1.15535, + "785": 1.14905, + "786": 1.14868, + "787": 1.14899, + "788": 1.1521, + "789": 1.1498, + "790": 1.15389, + "791": 1.15198, + "792": 1.14834, + "793": 1.14935, + "794": 1.14986, + "795": 1.15066, + "796": 1.15229, + "797": 1.15036, + "798": 1.15026, + "799": 1.15231, + "800": 1.15717, + "801": 1.15355, + "802": 1.15502, + "803": 1.15201, + "804": 1.15023, + "805": 1.15209, + "806": 1.15072, + "807": 1.48449, + "808": 1.15218, + "809": 1.1522, + "810": 1.15111, + "811": 1.15134, + "812": 1.15187, + "813": 1.15379, + "814": 1.15585, + "815": 1.16392, + "816": 1.15452, + "817": 1.15487, + "818": 1.15245, + "819": 1.14836, + "820": 1.14547, + "821": 1.74382, + "822": 1.14655, + "823": 1.13629, + "824": 1.15244, + "825": 1.14064, + "826": 1.14002, + "827": 1.14234, + "828": 1.1401, + "829": 1.13945, + "830": 1.14243, + "831": 1.14339, + "832": 1.13963, + "833": 1.14165, + "834": 1.13931, + "835": 1.13828, + "836": 1.13924, + "837": 1.13918, + "838": 1.14038, + "839": 1.14023, + "840": 1.13827, + "841": 1.14334, + "842": 1.26736, + "843": 1.15235, + "844": 1.16327, + "845": 1.15615, + "846": 1.15656, + "847": 1.14563, + "848": 1.14836, + "849": 1.14901, + "850": 1.14852, + "851": 1.15019, + "852": 1.14893, + "853": 1.14907, + "854": 1.14895, + "855": 1.14997, + "856": 1.14951, + "857": 1.15014, + "858": 1.14881, + "859": 1.15072, + "860": 1.16126, + "861": 1.15807, + "862": 1.15716, + "863": 1.15555, + "864": 1.15038, + "865": 1.15177, + "866": 1.15177, + "867": 1.14884, + "868": 1.14782, + "869": 1.15086, + "870": 1.14982, + "871": 1.14833, + "872": 1.14875, + "873": 1.15147, + "874": 1.15225, + "875": 1.29099, + "876": 2.39847, + "877": 2.16612, + "878": 1.53276, + "879": 1.14604, + "880": 1.1515, + "881": 1.16208, + "882": 1.15925, + "883": 1.14916, + "884": 1.14927, + "885": 1.1758, + "886": 1.17545, + "887": 1.17369, + "888": 1.17655, + "889": 1.16376, + "890": 1.14874, + "891": 1.148, + "892": 1.14787, + "893": 1.15123, + "894": 1.15168, + "895": 1.15419, + "896": 1.15535, + "897": 1.15242, + "898": 1.15508, + "899": 1.15225, + "900": 1.15072, + "901": 1.1534, + "902": 1.15136, + "903": 1.15481, + "904": 1.15989, + "905": 1.16184, + "906": 1.14716, + "907": 1.15192, + "908": 1.15696, + "909": 1.15328, + "910": 1.14059, + "911": 1.1604, + "912": 1.14941, + "913": 1.14972, + "914": 1.14954, + "915": 1.15073, + "916": 1.14475, + "917": 1.15414, + "918": 1.1385, + "919": 1.14185, + "920": 1.14089, + "921": 1.13784, + "922": 1.13875, + "923": 1.13882, + "924": 1.14141, + "925": 1.13908, + "926": 1.13874, + "927": 1.13823, + "928": 1.13737, + "929": 1.13836, + "930": 1.13809, + "931": 1.14893, + "932": 1.13972, + "933": 1.1369, + "934": 1.1362, + "935": 1.13765, + "936": 1.14369, + "937": 1.1504, + "938": 1.14208, + "939": 1.14841, + "940": 1.14975, + "941": 1.14225, + "942": 1.14185, + "943": 1.13864, + "944": 1.13915, + "945": 1.14062, + "946": 1.15111, + "947": 1.14071, + "948": 1.13898, + "949": 1.1399, + "950": 1.15937, + "951": 1.16785, + "952": 1.16807, + "953": 1.1506, + "954": 1.15006, + "955": 1.15045, + "956": 1.17067, + "957": 1.14856, + "958": 1.14992, + "959": 1.15251, + "960": 1.15045, + "961": 1.15121, + "962": 1.14957, + "963": 1.15095, + "964": 1.15, + "965": 1.15089, + "966": 1.15156, + "967": 1.15423, + "968": 1.16332, + "969": 1.15359, + "970": 1.15613, + "971": 1.15232, + "972": 1.15652, + "973": 1.15399, + "974": 1.15065, + "975": 1.1485, + "976": 1.15243, + "977": 1.15368, + "978": 1.14828, + "979": 1.14969, + "980": 1.15374, + "981": 1.1505, + "982": 1.15031, + "983": 1.15033, + "984": 1.14921, + "985": 1.15504, + "986": 1.15572, + "987": 1.153, + "988": 1.15573, + "989": 1.14747, + "990": 1.14636, + "991": 1.14517, + "992": 1.1463, + "993": 1.14805, + "994": 1.14644, + "995": 1.14583, + "996": 1.14485, + "997": 1.14418, + "998": 1.14622, + "999": 1.14662, + "1000": 1.14312, + "1001": 1.15227, + "1002": 1.14681, + "1003": 1.14794, + "1004": 1.14889, + "1005": 1.15067, + "1006": 1.14757, + "1007": 1.14767, + "1008": 1.15061, + "1009": 1.15075, + "1010": 1.14894, + "1011": 1.14975, + "1012": 1.14667, + "1013": 1.14688, + "1014": 1.14788, + "1015": 1.167, + "1016": 1.44606, + "1017": 1.14923, + "1018": 1.15268, + "1019": 1.14981, + "1020": 1.15011, + "1021": 1.47391, + "1022": 1.15277, + "1023": 1.14774, + "1024": 1.146, + "1025": 1.15253, + "1026": 1.14633, + "1027": 1.14525, + "1028": 1.14728, + "1029": 1.14654, + "1030": 1.14663, + "1031": 1.14708, + "1032": 1.14715, + "1033": 1.1454, + "1034": 1.14763, + "1035": 1.14591, + "1036": 1.14493, + "1037": 1.14584, + "1038": 1.14665, + "1039": 1.14812, + "1040": 1.14495, + "1041": 1.15044, + "1042": 1.14701, + "1043": 1.14657, + "1044": 1.14631, + "1045": 1.14822, + "1046": 1.14789, + "1047": 1.14525, + "1048": 1.14815, + "1049": 1.14939, + "1050": 1.14592, + "1051": 1.14667, + "1052": 1.15232, + "1053": 1.14863, + "1054": 1.14908, + "1055": 1.14931, + "1056": 1.14644, + "1057": 1.149, + "1058": 1.14751, + "1059": 1.14668, + "1060": 1.14758, + "1061": 1.14789, + "1062": 1.43562, + "1063": 1.14875, + "1064": 1.14846, + "1065": 1.14888, + "1066": 1.15486, + "1067": 1.15212, + "1068": 1.14934, + "1069": 1.14526, + "1070": 1.14506, + "1071": 1.14599, + "1072": 1.14774, + "1073": 1.14651, + "1074": 1.14609, + "1075": 1.14817, + "1076": 1.14662, + "1077": 1.15159, + "1078": 1.14735, + "1079": 1.14525, + "1080": 1.1516, + "1081": 1.14601, + "1082": 1.13989, + "1083": 1.13569, + "1084": 1.1371, + "1085": 1.1366, + "1086": 1.13713, + "1087": 1.13756, + "1088": 1.13768, + "1089": 1.13917, + "1090": 1.13759, + "1091": 1.13884, + "1092": 1.13707, + "1093": 1.13679, + "1094": 1.13513, + "1095": 1.1351, + "1096": 1.13494, + "1097": 1.13589, + "1098": 1.14132, + "1099": 1.13697, + "1100": 1.14195, + "1101": 1.14189, + "1102": 1.13736, + "1103": 1.13781, + "1104": 1.14284, + "1105": 1.13518, + "1106": 1.13585, + "1107": 1.13621, + "1108": 1.13665, + "1109": 1.13792, + "1110": 1.13764, + "1111": 1.13778, + "1112": 1.13619, + "1113": 1.13651, + "1114": 1.13628, + "1115": 1.13802, + "1116": 1.13792, + "1117": 1.13642, + "1118": 1.13784, + "1119": 1.14898, + "1120": 1.15049, + "1121": 1.15028, + "1122": 1.14509, + "1123": 1.1445, + "1124": 1.14756, + "1125": 1.15117, + "1126": 1.14917, + "1127": 1.1475, + "1128": 1.1481, + "1129": 1.14683, + "1130": 1.14088, + "1131": 1.13493, + "1132": 1.13613, + "1133": 1.13537, + "1134": 1.13473, + "1135": 1.13657, + "1136": 1.13516, + "1137": 1.13606, + "1138": 1.13473, + "1139": 1.13442, + "1140": 1.13398, + "1141": 1.13591, + "1142": 1.13975, + "1143": 1.13478, + "1144": 1.13376, + "1145": 1.13428, + "1146": 1.1348, + "1147": 1.13462, + "1148": 1.1351, + "1149": 1.13494, + "1150": 1.13506, + "1151": 1.13487, + "1152": 1.14039, + "1153": 1.13991, + "1154": 1.13825, + "1155": 1.1373, + "1156": 1.13451, + "1157": 1.13683, + "1158": 1.13335, + "1159": 1.13548, + "1160": 1.1339, + "1161": 1.13613, + "1162": 1.13429, + "1163": 1.13448, + "1164": 1.13542, + "1165": 1.13453, + "1166": 1.13398, + "1167": 1.13549, + "1168": 1.1342, + "1169": 1.13502, + "1170": 1.13535, + "1171": 1.13581, + "1172": 1.13532, + "1173": 1.13552, + "1174": 1.13371, + "1175": 1.13456, + "1176": 1.13401, + "1177": 1.1335, + "1178": 1.13628, + "1179": 1.13907, + "1180": 1.13757, + "1181": 1.1538, + "1182": 1.15712, + "1183": 1.16123, + "1184": 1.15318, + "1185": 1.14801, + "1186": 1.14711, + "1187": 1.1471, + "1188": 1.15109, + "1189": 1.14707, + "1190": 1.14787, + "1191": 1.1451, + "1192": 1.14677, + "1193": 1.14621, + "1194": 1.14554, + "1195": 1.14738, + "1196": 1.14756, + "1197": 1.14799, + "1198": 1.1487, + "1199": 1.14616, + "1200": 1.14688, + "1201": 1.14531, + "1202": 1.14639, + "1203": 1.14696, + "1204": 1.1469, + "1205": 1.1472, + "1206": 1.14687, + "1207": 1.1494, + "1208": 1.14873, + "1209": 1.15175, + "1210": 1.14868, + "1211": 1.14793, + "1212": 1.14766, + "1213": 1.14823, + "1214": 1.15557, + "1215": 1.15986, + "1216": 1.14175, + "1217": 1.1392, + "1218": 1.13591, + "1219": 1.13796, + "1220": 1.14086, + "1221": 1.14081, + "1222": 1.13816, + "1223": 1.13977, + "1224": 1.14436, + "1225": 1.13986, + "1226": 1.13821, + "1227": 1.13854, + "1228": 1.13738, + "1229": 1.1384, + "1230": 1.13897, + "1231": 1.13732, + "1232": 1.13852, + "1233": 1.14144, + "1234": 1.13711, + "1235": 1.14105, + "1236": 1.13578, + "1237": 1.13838, + "1238": 1.13809, + "1239": 1.13782, + "1240": 1.13859, + "1241": 1.1381, + "1242": 1.13717, + "1243": 1.14814, + "1244": 1.16451, + "1245": 1.17765, + "1246": 1.17167, + "1247": 1.15708, + "1248": 1.15406, + "1249": 1.17391, + "1250": 1.14803, + "1251": 1.14601, + "1252": 1.14796, + "1253": 1.14706, + "1254": 1.14679, + "1255": 1.14306, + "1256": 1.14387, + "1257": 1.14608, + "1258": 1.14617, + "1259": 1.14999, + "1260": 1.1468, + "1261": 1.14332, + "1262": 1.15005, + "1263": 1.1449, + "1264": 1.14544, + "1265": 1.14292, + "1266": 1.14481, + "1267": 1.154, + "1268": 1.15455, + "1269": 1.15329, + "1270": 1.15008, + "1271": 1.15345, + "1272": 1.14616, + "1273": 1.15423, + "1274": 1.15349, + "1275": 1.14785, + "1276": 1.14536, + "1277": 1.14467, + "1278": 1.1456, + "1279": 1.14593, + "1280": 1.1462, + "1281": 1.14599, + "1282": 1.14837, + "1283": 1.14585, + "1284": 1.14656, + "1285": 1.14618, + "1286": 1.14615, + "1287": 1.14657, + "1288": 1.44686, + "1289": 1.14572, + "1290": 1.14398, + "1291": 1.1431, + "1292": 1.14524, + "1293": 1.14421, + "1294": 1.14593, + "1295": 1.16051, + "1296": 1.16214, + "1297": 1.15606, + "1298": 1.14439, + "1299": 1.14445, + "1300": 1.1445, + "1301": 1.1455, + "1302": 1.14117, + "1303": 1.14365, + "1304": 1.14474, + "1305": 1.14456, + "1306": 1.14522, + "1307": 1.144, + "1308": 1.14453, + "1309": 1.14471, + "1310": 1.1456, + "1311": 1.15495, + "1312": 1.15256, + "1313": 1.14805, + "1314": 1.14996, + "1315": 1.14425, + "1316": 1.14401, + "1317": 1.14262, + "1318": 1.14556, + "1319": 1.14661, + "1320": 1.14567, + "1321": 1.14648, + "1322": 1.14709, + "1323": 1.14522, + "1324": 1.14764, + "1325": 1.14331, + "1326": 1.14538, + "1327": 1.1453, + "1328": 1.14734, + "1329": 1.18619, + "1330": 1.48212, + "1331": 1.14651, + "1332": 1.15204, + "1333": 1.14629, + "1334": 1.14624, + "1335": 1.14927, + "1336": 1.14601, + "1337": 1.15642, + "1338": 1.14811, + "1339": 1.14508, + "1340": 1.15069, + "1341": 1.14629, + "1342": 1.14635, + "1343": 1.14657, + "1344": 1.14655, + "1345": 1.14564, + "1346": 1.14633, + "1347": 1.14523, + "1348": 1.14691, + "1349": 1.14575, + "1350": 1.14592, + "1351": 1.14631, + "1352": 1.14436, + "1353": 1.14573, + "1354": 1.14471, + "1355": 1.14554, + "1356": 1.14492, + "1357": 1.14301, + "1358": 1.141, + "1359": 1.14219, + "1360": 1.14228, + "1361": 1.14109, + "1362": 1.1413, + "1363": 1.14096, + "1364": 1.15355, + "1365": 1.14229, + "1366": 1.14615, + "1367": 1.14174, + "1368": 1.13953, + "1369": 1.14014, + "1370": 1.14132, + "1371": 1.14139, + "1372": 1.13849, + "1373": 1.14304, + "1374": 1.14028, + "1375": 1.13912, + "1376": 1.14082, + "1377": 1.1416, + "1378": 1.13936, + "1379": 1.13866, + "1380": 1.13826, + "1381": 1.14443, + "1382": 1.14029, + "1383": 1.13913, + "1384": 1.14177, + "1385": 1.14492, + "1386": 1.1415, + "1387": 1.1398, + "1388": 1.14017, + "1389": 1.14077, + "1390": 1.14782, + "1391": 1.15011, + "1392": 1.15174, + "1393": 1.14605, + "1394": 1.14761, + "1395": 1.14735, + "1396": 1.14827, + "1397": 1.14566, + "1398": 1.14659, + "1399": 1.14187, + "1400": 1.14737, + "1401": 1.14674, + "1402": 1.14468, + "1403": 1.14534, + "1404": 1.14726, + "1405": 1.14773, + "1406": 1.14711, + "1407": 1.14543, + "1408": 1.14568, + "1409": 1.14559, + "1410": 1.14443, + "1411": 1.14591, + "1412": 1.14444, + "1413": 1.14904, + "1414": 1.14806, + "1415": 1.14757, + "1416": 1.14307, + "1417": 1.14119, + "1418": 1.14392, + "1419": 1.14104, + "1420": 1.14278, + "1421": 1.13949, + "1422": 1.14028, + "1423": 1.14112, + "1424": 1.14151, + "1425": 1.14321, + "1426": 1.14894, + "1427": 1.14281, + "1428": 1.14881, + "1429": 1.14225, + "1430": 1.13905, + "1431": 1.14148, + "1432": 1.14895, + "1433": 1.15186, + "1434": 1.14773, + "1435": 1.14968, + "1436": 1.14689, + "1437": 1.1487, + "1438": 1.14731, + "1439": 1.14746, + "1440": 1.14835, + "1441": 1.15151, + "1442": 1.15182, + "1443": 1.15073, + "1444": 1.14751, + "1445": 1.15081, + "1446": 1.15106, + "1447": 1.14876, + "1448": 1.15178, + "1449": 1.15117, + "1450": 1.1479, + "1451": 1.14851, + "1452": 1.14502, + "1453": 1.1454, + "1454": 1.14722, + "1455": 1.14628, + "1456": 1.14413, + "1457": 1.14761, + "1458": 1.14681, + "1459": 1.14632, + "1460": 1.14804, + "1461": 1.14676, + "1462": 1.14566, + "1463": 1.14599, + "1464": 1.14679, + "1465": 1.14572, + "1466": 1.14995, + "1467": 1.14848, + "1468": 1.14679, + "1469": 1.15027, + "1470": 1.14636, + "1471": 1.14406, + "1472": 1.14039, + "1473": 1.13768, + "1474": 1.13897, + "1475": 1.14331, + "1476": 1.1403, + "1477": 1.14139, + "1478": 1.14985, + "1479": 1.14611, + "1480": 1.47655, + "1481": 1.45511, + "1482": 1.14381, + "1483": 1.13941, + "1484": 1.13782, + "1485": 1.13771, + "1486": 1.13796, + "1487": 1.13795, + "1488": 1.13829, + "1489": 1.13758, + "1490": 1.13822, + "1491": 1.13667, + "1492": 1.13847, + "1493": 1.13787, + "1494": 1.14072, + "1495": 1.14614, + "1496": 1.14436, + "1497": 1.14422, + "1498": 1.1393, + "1499": 1.13987, + "1500": 1.13991, + "1501": 1.14215, + "1502": 1.13842, + "1503": 1.13883, + "1504": 1.1496, + "1505": 1.14028, + "1506": 1.13931, + "1507": 1.13949, + "1508": 1.14063, + "1509": 1.13913, + "1510": 1.1402, + "1511": 1.13931, + "1512": 1.13839, + "1513": 1.13771, + "1514": 1.13848, + "1515": 1.13796, + "1516": 1.13782, + "1517": 1.13889, + "1518": 1.13716, + "1519": 1.13908, + "1520": 1.13972, + "1521": 1.13966, + "1522": 1.13875, + "1523": 1.15781, + "1524": 1.15885, + "1525": 1.15802, + "1526": 1.14191, + "1527": 1.14054, + "1528": 1.1385, + "1529": 1.13922, + "1530": 1.12994, + "1531": 1.12552, + "1532": 1.27166, + "1533": 1.12707, + "1534": 1.12638, + "1535": 1.12608, + "1536": 1.12654, + "1537": 1.12511, + "1538": 1.16008, + "1539": 1.13169, + "1540": 1.13294, + "1541": 1.13386, + "1542": 1.13461, + "1543": 1.13337, + "1544": 1.1331, + "1545": 1.13294, + "1546": 1.13283, + "1547": 1.13316, + "1548": 1.13651, + "1549": 1.13626, + "1550": 1.13638, + "1551": 1.13187, + "1552": 1.20522, + "1553": 1.15894, + "1554": 1.14738, + "1555": 1.14563, + "1556": 1.14409, + "1557": 1.15018, + "1558": 1.14323, + "1559": 1.14591, + "1560": 1.14645, + "1561": 1.14673, + "1562": 1.14543, + "1563": 1.14518, + "1564": 1.14589, + "1565": 1.14486, + "1566": 1.14436, + "1567": 1.14357, + "1568": 1.1454, + "1569": 1.14493, + "1570": 1.14347, + "1571": 1.14477, + "1572": 1.14203, + "1573": 1.14441, + "1574": 1.14468, + "1575": 1.14607, + "1576": 1.14532, + "1577": 1.14389, + "1578": 1.1433, + "1579": 1.14321, + "1580": 1.14391, + "1581": 1.1421, + "1582": 1.14368, + "1583": 1.1444, + "1584": 1.14356, + "1585": 1.14875, + "1586": 1.14497, + "1587": 1.14521, + "1588": 1.14708, + "1589": 1.14631, + "1590": 1.14662, + "1591": 1.14949, + "1592": 1.15354, + "1593": 1.14014, + "1594": 1.1408, + "1595": 1.14166, + "1596": 1.14151, + "1597": 1.14228, + "1598": 1.14126, + "1599": 1.14028, + "1600": 1.14528, + "1601": 1.14125, + "1602": 1.14085, + "1603": 1.13862, + "1604": 1.13487, + "1605": 1.13314, + "1606": 1.13467, + "1607": 1.13153, + "1608": 1.12971, + "1609": 1.13044, + "1610": 1.14013, + "1611": 1.13008, + "1612": 1.13161, + "1613": 1.13128, + "1614": 1.13059, + "1615": 1.13169, + "1616": 1.13043, + "1617": 1.13141, + "1618": 1.12976, + "1619": 1.13071, + "1620": 1.12907, + "1621": 1.13138, + "1622": 1.12994, + "1623": 1.12985, + "1624": 1.12999, + "1625": 1.13035, + "1626": 1.13761, + "1627": 1.13703, + "1628": 1.15487, + "1629": 1.13257, + "1630": 1.13549, + "1631": 1.13358, + "1632": 1.13488, + "1633": 1.13601, + "1634": 1.13282, + "1635": 1.13439, + "1636": 1.13078, + "1637": 1.13147, + "1638": 1.13065, + "1639": 1.13181, + "1640": 1.13227, + "1641": 1.13282, + "1642": 1.13305, + "1643": 1.19491, + "1644": 1.15821, + "1645": 1.15349, + "1646": 1.1437, + "1647": 1.1416, + "1648": 1.14282, + "1649": 1.1408, + "1650": 1.13388, + "1651": 1.13396, + "1652": 1.15414, + "1653": 1.13734, + "1654": 1.13143, + "1655": 1.13124, + "1656": 1.13417, + "1657": 1.13376, + "1658": 1.12932, + "1659": 1.13161, + "1660": 1.13178, + "1661": 1.1315, + "1662": 1.13209, + "1663": 1.13118, + "1664": 1.13332, + "1665": 1.12981, + "1666": 1.13001, + "1667": 1.12943, + "1668": 1.12938, + "1669": 1.12973, + "1670": 1.13031, + "1671": 1.14164, + "1672": 1.14108, + "1673": 1.14165, + "1674": 1.14189, + "1675": 1.14174, + "1676": 1.14802, + "1677": 1.14434, + "1678": 1.14543, + "1679": 1.14285, + "1680": 1.14529, + "1681": 1.14548, + "1682": 1.14333, + "1683": 1.14553, + "1684": 1.14327, + "1685": 1.1476, + "1686": 1.1406, + "1687": 1.13769, + "1688": 1.13364, + "1689": 1.13418, + "1690": 1.13026, + "1691": 1.13222, + "1692": 1.13195, + "1693": 1.13247, + "1694": 1.13264, + "1695": 1.13167, + "1696": 1.13234, + "1697": 1.13335, + "1698": 1.13463, + "1699": 1.1337, + "1700": 1.13362, + "1701": 1.13339, + "1702": 1.13335, + "1703": 1.13412, + "1704": 1.1332, + "1705": 1.13109, + "1706": 1.13306, + "1707": 1.42699, + "1708": 1.14258, + "1709": 1.13227, + "1710": 1.13333, + "1711": 1.13316, + "1712": 1.13147, + "1713": 1.1325, + "1714": 1.13279, + "1715": 1.13509, + "1716": 1.132, + "1717": 1.13183, + "1718": 1.13123, + "1719": 1.13209, + "1720": 1.13195, + "1721": 1.12891, + "1722": 1.12633, + "1723": 1.12872, + "1724": 1.1269, + "1725": 1.12641, + "1726": 1.12585, + "1727": 1.12446, + "1728": 1.12583, + "1729": 1.1336, + "1730": 1.1322, + "1731": 1.13153, + "1732": 1.132, + "1733": 1.13239, + "1734": 1.13216, + "1735": 1.13252, + "1736": 1.13132, + "1737": 1.13165, + "1738": 1.13359, + "1739": 1.126, + "1740": 1.124, + "1741": 1.12533, + "1742": 1.12379, + "1743": 1.12474, + "1744": 1.12432, + "1745": 1.13505, + "1746": 1.13795, + "1747": 1.13914, + "1748": 1.17805, + "1749": 1.13962, + "1750": 1.13602, + "1751": 1.13778, + "1752": 1.13639, + "1753": 1.14452, + "1754": 1.14424, + "1755": 1.14388, + "1756": 1.14572, + "1757": 1.17074, + "1758": 1.14596, + "1759": 1.14637, + "1760": 1.14576, + "1761": 1.1441, + "1762": 1.13385, + "1763": 1.13833, + "1764": 1.13995, + "1765": 1.14229, + "1766": 1.2706, + "1767": 1.15999, + "1768": 1.13873, + "1769": 1.1421, + "1770": 1.13078, + "1771": 1.13059, + "1772": 1.13076, + "1773": 1.13527, + "1774": 1.13153, + "1775": 1.1299, + "1776": 1.13144, + "1777": 1.13048, + "1778": 1.1312, + "1779": 1.13109, + "1780": 1.13227, + "1781": 1.1318, + "1782": 1.13195, + "1783": 1.13076, + "1784": 1.13371, + "1785": 1.13513, + "1786": 1.13544, + "1787": 1.13286, + "1788": 1.13114, + "1789": 1.12859, + "1790": 1.13136, + "1791": 1.13775, + "1792": 1.1401, + "1793": 1.13769, + "1794": 1.13564, + "1795": 1.13638, + "1796": 1.13621, + "1797": 1.13614, + "1798": 1.13707, + "1799": 1.13631, + "1800": 1.13547, + "1801": 1.13673, + "1802": 1.13706, + "1803": 1.13765, + "1804": 1.13506, + "1805": 1.13603, + "1806": 1.13717, + "1807": 1.13637, + "1808": 1.13841, + "1809": 1.13734, + "1810": 1.1379, + "1811": 1.13795, + "1812": 1.13826, + "1813": 1.13875, + "1814": 1.13885, + "1815": 1.13773, + "1816": 1.13726, + "1817": 1.14087, + "1818": 1.1378, + "1819": 1.13714, + "1820": 1.13737, + "1821": 1.13928, + "1822": 1.1371, + "1823": 1.13901, + "1824": 1.14485, + "1825": 1.12803, + "1826": 1.12264, + "1827": 1.12651, + "1828": 1.13421, + "1829": 1.13198, + "1830": 1.13242, + "1831": 1.13488, + "1832": 1.13287, + "1833": 1.13394, + "1834": 1.13403, + "1835": 1.13598, + "1836": 1.13357, + "1837": 1.13518, + "1838": 1.13404, + "1839": 1.13577, + "1840": 1.13254, + "1841": 1.13422, + "1842": 1.13496, + "1843": 1.135, + "1844": 1.13791, + "1845": 1.13082, + "1846": 1.13135, + "1847": 1.13026, + "1848": 1.13098, + "1849": 1.13032, + "1850": 1.13038, + "1851": 1.13107, + "1852": 1.13535, + "1853": 1.1311, + "1854": 1.13935, + "1855": 1.13148, + "1856": 1.13042, + "1857": 1.13238, + "1858": 1.13034, + "1859": 1.13083, + "1860": 1.13262, + "1861": 1.13117, + "1862": 1.13181, + "1863": 1.13237, + "1864": 1.13125, + "1865": 1.13519, + "1866": 1.14006, + "1867": 1.13476, + "1868": 1.13101, + "1869": 1.13227, + "1870": 1.13399, + "1871": 1.13455, + "1872": 1.13237, + "1873": 1.13088, + "1874": 1.13163, + "1875": 1.13336, + "1876": 1.13121, + "1877": 1.13209, + "1878": 1.13199, + "1879": 1.13177, + "1880": 1.13322, + "1881": 1.13141, + "1882": 1.13236, + "1883": 1.12859, + "1884": 1.12504, + "1885": 1.12493, + "1886": 1.12502, + "1887": 1.12484, + "1888": 1.1248, + "1889": 1.12719, + "1890": 1.13286, + "1891": 1.1293, + "1892": 1.13422, + "1893": 1.12646, + "1894": 1.12508, + "1895": 1.12422, + "1896": 1.12724, + "1897": 1.12903, + "1898": 1.13203, + "1899": 1.12741, + "1900": 1.12527, + "1901": 1.12359, + "1902": 1.12382, + "1903": 1.12536, + "1904": 1.12683, + "1905": 1.12606, + "1906": 1.12607, + "1907": 1.12626, + "1908": 1.44717, + "1909": 1.12543, + "1910": 1.12376, + "1911": 1.12429, + "1912": 1.12442, + "1913": 1.12355, + "1914": 1.12476, + "1915": 1.12331, + "1916": 1.12342, + "1917": 1.12442, + "1918": 1.12472, + "1919": 1.12536, + "1920": 1.12387, + "1921": 1.12347, + "1922": 1.12561, + "1923": 1.12391, + "1924": 1.12342, + "1925": 1.12607, + "1926": 1.12383, + "1927": 1.12305, + "1928": 1.125, + "1929": 1.12399, + "1930": 1.1237, + "1931": 1.12459, + "1932": 1.12475, + "1933": 1.12278, + "1934": 1.12413, + "1935": 1.12588, + "1936": 1.12473, + "1937": 1.12412, + "1938": 1.12444, + "1939": 1.12303, + "1940": 1.12421, + "1941": 1.12404, + "1942": 1.12568, + "1943": 1.12645, + "1944": 1.12388, + "1945": 1.44561, + "1946": 1.12748, + "1947": 1.44404, + "1948": 1.12309, + "1949": 1.12591, + "1950": 1.124, + "1951": 1.12953, + "1952": 1.12429, + "1953": 1.48105, + "1954": 1.12576, + "1955": 1.1274, + "1956": 1.12693, + "1957": 1.1261, + "1958": 1.1276, + "1959": 1.18913, + "1960": 1.12817, + "1961": 1.12615, + "1962": 1.12581, + "1963": 1.12682, + "1964": 1.12747, + "1965": 1.14301, + "1966": 1.14417, + "1967": 1.14427, + "1968": 1.14017, + "1969": 1.13872, + "1970": 1.13824, + "1971": 1.14731, + "1972": 1.13727, + "1973": 1.13816, + "1974": 1.13684, + "1975": 1.13985, + "1976": 1.13777, + "1977": 1.13833, + "1978": 1.14247, + "1979": 1.14554, + "1980": 1.14074, + "1981": 1.1396, + "1982": 1.13784, + "1983": 1.19896, + "1984": 1.13952, + "1985": 1.13865, + "1986": 1.13959, + "1987": 1.13909, + "1988": 1.13875, + "1989": 1.13947, + "1990": 1.13762, + "1991": 1.13799, + "1992": 1.13904, + "1993": 1.13674, + "1994": 1.13869, + "1995": 1.13884, + "1996": 1.13807, + "1997": 1.13986, + "1998": 1.14151, + "1999": 1.13582, + "2000": 1.16726 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml index 5668a7575e2..15ac9782df5 100644 --- a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml +++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/model_config.yaml @@ -42,7 +42,7 @@ MODEL_ARGS: --pipeline-model-parallel-size: 2 --sequence-parallel: true --tp-comm-overlap: true - --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_345m_weekly_dgx_h100_1N8G_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml + --tp-comm-overlap-cfg: tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp2_pp2_current_scaling_native_fp8_tp_pp_sp_tp_overlap/tp_comm_overlap_cfg.yaml --deterministic-mode: true --no-gradient-accumulation-fusion: true --fp8-format: hybrid diff --git a/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..b6e543e2cf8 --- /dev/null +++ b/tests/functional_tests/test_cases/gpt/gpt3_weekly_dgx_h100_mcore_tp4_cp2_native_fp8_tp_sp_cp_tp_overlap/golden_values_dev_dgx_h100.json @@ -0,0 +1,10037 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 10.85229, + "2": 10.85951, + "3": 10.85469, + "4": 10.86843, + "5": 10.85304, + "6": 10.85362, + "7": 10.8602, + "8": 10.85298, + "9": 10.84874, + "10": 10.84674, + "11": 10.83863, + "12": 10.83549, + "13": 10.82524, + "14": 10.84078, + "15": 10.78613, + "16": 10.79372, + "17": 10.76553, + "18": 10.78902, + "19": 10.73057, + "20": 10.69489, + "21": 10.64595, + "22": 10.64791, + "23": 10.65524, + "24": 10.55349, + "25": 10.56424, + "26": 10.63262, + "27": 10.47084, + "28": 10.471, + "29": 10.36495, + "30": 10.27406, + "31": 10.43126, + "32": 10.35361, + "33": 10.22439, + "34": 10.17135, + "35": 10.23744, + "36": 10.15766, + "37": 10.10704, + "38": 10.03631, + "39": 10.04895, + "40": 10.06978, + "41": 9.95276, + "42": 9.95577, + "43": 9.87217, + "44": 9.99154, + "45": 10.00766, + "46": 9.84803, + "47": 10.00018, + "48": 9.81816, + "49": 9.94941, + "50": 9.94449, + "51": 9.5964, + "52": 9.79483, + "53": 9.63207, + "54": 9.8854, + "55": 9.74063, + "56": 9.85006, + "57": 9.86123, + "58": 9.87737, + "59": 9.54716, + "60": 9.64756, + "61": 9.87994, + "62": 9.76465, + "63": 9.68066, + "64": 9.82801, + "65": 9.59733, + "66": 9.62928, + "67": 9.74212, + "68": 9.60593, + "69": 9.29694, + "70": 9.42495, + "71": 9.79013, + "72": 9.71358, + "73": 9.61909, + "74": 9.45334, + "75": 9.24289, + "76": 9.50821, + "77": 9.57857, + "78": 9.56035, + "79": 9.31048, + "80": 9.36161, + "81": 9.46136, + "82": 9.55628, + "83": 9.53353, + "84": 9.35526, + "85": 9.40111, + "86": 9.65137, + "87": 9.23621, + "88": 9.48942, + "89": 9.22457, + "90": 9.41443, + "91": 9.39014, + "92": 9.3793, + "93": 9.36366, + "94": 9.51552, + "95": 9.42012, + "96": 9.33698, + "97": 9.20729, + "98": 9.49265, + "99": 9.29333, + "100": 9.35883, + "101": 9.24766, + "102": 9.24259, + "103": 9.07796, + "104": 9.16832, + "105": 9.37671, + "106": 9.15179, + "107": 9.17832, + "108": 9.31483, + "109": 9.28984, + "110": 9.36705, + "111": 9.17605, + "112": 9.23281, + "113": 9.35413, + "114": 9.35742, + "115": 9.32337, + "116": 9.00364, + "117": 9.06445, + "118": 9.06523, + "119": 9.22504, + "120": 9.08324, + "121": 9.19428, + "122": 9.14006, + "123": 9.25894, + "124": 9.45689, + "125": 9.21857, + "126": 9.0614, + "127": 9.01413, + "128": 9.22025, + "129": 8.98394, + "130": 9.14098, + "131": 9.15643, + "132": 9.03479, + "133": 8.86261, + "134": 9.18468, + "135": 8.88922, + "136": 9.1645, + "137": 9.15944, + "138": 9.23186, + "139": 9.08834, + "140": 8.87267, + "141": 9.29752, + "142": 9.19877, + "143": 9.12079, + "144": 9.24324, + "145": 9.10527, + "146": 8.98338, + "147": 8.9881, + "148": 9.1361, + "149": 9.06877, + "150": 9.01122, + "151": 8.93192, + "152": 8.87852, + "153": 9.06711, + "154": 9.1802, + "155": 9.13786, + "156": 9.05095, + "157": 9.15163, + "158": 9.05301, + "159": 9.03638, + "160": 8.89244, + "161": 9.04764, + "162": 8.89639, + "163": 8.84472, + "164": 8.97496, + "165": 8.93105, + "166": 8.65677, + "167": 8.83411, + "168": 8.8203, + "169": 8.65961, + "170": 9.04726, + "171": 8.72167, + "172": 8.82105, + "173": 8.91105, + "174": 8.85007, + "175": 8.70985, + "176": 8.7611, + "177": 8.76567, + "178": 8.72394, + "179": 8.64132, + "180": 8.74357, + "181": 8.6941, + "182": 8.72315, + "183": 9.08667, + "184": 8.60959, + "185": 8.88334, + "186": 8.74346, + "187": 8.57546, + "188": 8.6841, + "189": 8.86656, + "190": 8.53754, + "191": 8.66593, + "192": 8.61152, + "193": 8.5763, + "194": 8.75183, + "195": 8.5938, + "196": 8.7761, + "197": 8.744, + "198": 8.63042, + "199": 8.77202, + "200": 8.73627, + "201": 8.67068, + "202": 8.55099, + "203": 8.54134, + "204": 8.71213, + "205": 8.22486, + "206": 8.85986, + "207": 8.67928, + "208": 8.70826, + "209": 8.75243, + "210": 8.58226, + "211": 8.84167, + "212": 8.4913, + "213": 8.57316, + "214": 8.51316, + "215": 8.56549, + "216": 8.50617, + "217": 8.53369, + "218": 8.53635, + "219": 8.64298, + "220": 8.54526, + "221": 8.39761, + "222": 8.50474, + "223": 8.44078, + "224": 8.52901, + "225": 8.5708, + "226": 8.44247, + "227": 8.67823, + "228": 8.3859, + "229": 8.4537, + "230": 8.4985, + "231": 8.50257, + "232": 8.49898, + "233": 8.49438, + "234": 8.64018, + "235": 8.5617, + "236": 8.39791, + "237": 8.49075, + "238": 8.30637, + "239": 8.56099, + "240": 8.67125, + "241": 8.447, + "242": 8.47179, + "243": 8.51685, + "244": 8.36975, + "245": 8.59641, + "246": 8.59557, + "247": 8.43962, + "248": 8.50986, + "249": 8.52277, + "250": 8.42301, + "251": 8.3783, + "252": 8.54698, + "253": 8.3164, + "254": 8.35246, + "255": 8.29609, + "256": 8.20858, + "257": 8.39462, + "258": 8.45148, + "259": 8.23213, + "260": 8.24039, + "261": 8.23733, + "262": 8.34866, + "263": 8.30632, + "264": 8.1907, + "265": 8.33202, + "266": 8.2336, + "267": 7.9013, + "268": 8.37861, + "269": 8.40384, + "270": 8.26475, + "271": 8.27885, + "272": 8.31844, + "273": 8.13253, + "274": 8.09818, + "275": 8.00901, + "276": 7.92522, + "277": 8.23699, + "278": 8.04701, + "279": 7.96356, + "280": 7.75515, + "281": 8.10016, + "282": 8.14722, + "283": 8.15666, + "284": 8.10022, + "285": 8.06894, + "286": 7.90037, + "287": 7.99127, + "288": 8.24359, + "289": 8.17176, + "290": 8.12684, + "291": 8.25357, + "292": 8.0756, + "293": 8.11914, + "294": 7.97501, + "295": 7.96533, + "296": 8.23576, + "297": 7.79081, + "298": 8.04236, + "299": 7.93831, + "300": 7.8498, + "301": 8.00964, + "302": 7.94515, + "303": 7.99053, + "304": 7.95899, + "305": 7.9946, + "306": 7.9738, + "307": 7.98707, + "308": 7.9953, + "309": 8.0059, + "310": 7.97168, + "311": 7.92562, + "312": 7.88182, + "313": 7.82955, + "314": 7.82035, + "315": 7.82475, + "316": 7.74495, + "317": 7.92567, + "318": 7.97631, + "319": 7.82443, + "320": 7.563, + "321": 7.74534, + "322": 7.82917, + "323": 7.76703, + "324": 7.90668, + "325": 7.79387, + "326": 7.64901, + "327": 7.86137, + "328": 7.7832, + "329": 7.87669, + "330": 7.74815, + "331": 7.52005, + "332": 7.81037, + "333": 7.8379, + "334": 7.67759, + "335": 7.69435, + "336": 7.90998, + "337": 7.64618, + "338": 7.89178, + "339": 7.7192, + "340": 7.75318, + "341": 7.70375, + "342": 7.81451, + "343": 7.61028, + "344": 7.58433, + "345": 7.60474, + "346": 7.45825, + "347": 7.55021, + "348": 7.67669, + "349": 7.57925, + "350": 7.65118, + "351": 7.74172, + "352": 7.69877, + "353": 7.4955, + "354": 7.73645, + "355": 7.75823, + "356": 7.76871, + "357": 7.8083, + "358": 7.59223, + "359": 7.54129, + "360": 7.62161, + "361": 7.53913, + "362": 7.75707, + "363": 7.58184, + "364": 7.57393, + "365": 7.61381, + "366": 7.30007, + "367": 7.55433, + "368": 7.4381, + "369": 7.34072, + "370": 7.45786, + "371": 7.45479, + "372": 7.64528, + "373": 7.51803, + "374": 7.43579, + "375": 7.52279, + "376": 7.33856, + "377": 7.23275, + "378": 7.53208, + "379": 7.48549, + "380": 7.37893, + "381": 7.46259, + "382": 7.28593, + "383": 7.26774, + "384": 7.4035, + "385": 7.38617, + "386": 7.2246, + "387": 7.41197, + "388": 7.27354, + "389": 7.42884, + "390": 7.23295, + "391": 7.63854, + "392": 7.32743, + "393": 7.41119, + "394": 7.46811, + "395": 7.43164, + "396": 7.27624, + "397": 7.22237, + "398": 7.41314, + "399": 7.14965, + "400": 7.28882, + "401": 7.34645, + "402": 7.38389, + "403": 7.27445, + "404": 7.29549, + "405": 7.25441, + "406": 7.20955, + "407": 7.35305, + "408": 7.17476, + "409": 7.15738, + "410": 7.30843, + "411": 7.21046, + "412": 7.19143, + "413": 7.22421, + "414": 6.90584, + "415": 7.32329, + "416": 7.41955, + "417": 7.01436, + "418": 7.26656, + "419": 7.03251, + "420": 7.40294, + "421": 7.17304, + "422": 7.22884, + "423": 7.08611, + "424": 7.2354, + "425": 7.3087, + "426": 7.28003, + "427": 7.12262, + "428": 7.08425, + "429": 6.87125, + "430": 7.19779, + "431": 6.99763, + "432": 7.22298, + "433": 6.96906, + "434": 6.95232, + "435": 7.01097, + "436": 7.00141, + "437": 6.9848, + "438": 6.99447, + "439": 6.93128, + "440": 7.05472, + "441": 7.03406, + "442": 7.09324, + "443": 7.0854, + "444": 6.69941, + "445": 6.98741, + "446": 7.13474, + "447": 7.11726, + "448": 6.97509, + "449": 7.04203, + "450": 7.00855, + "451": 6.82317, + "452": 6.90281, + "453": 7.00796, + "454": 6.96028, + "455": 7.02393, + "456": 6.98781, + "457": 6.96156, + "458": 6.89735, + "459": 6.68323, + "460": 7.05439, + "461": 7.088, + "462": 6.86315, + "463": 7.04576, + "464": 6.64275, + "465": 7.02272, + "466": 6.99895, + "467": 6.99097, + "468": 6.94728, + "469": 6.82004, + "470": 7.0355, + "471": 6.87321, + "472": 6.95214, + "473": 6.81396, + "474": 6.96547, + "475": 7.1584, + "476": 6.75391, + "477": 6.88861, + "478": 6.89832, + "479": 6.69636, + "480": 7.01803, + "481": 6.98503, + "482": 6.72248, + "483": 6.77484, + "484": 6.74297, + "485": 6.92045, + "486": 7.05544, + "487": 6.62222, + "488": 6.87375, + "489": 6.76024, + "490": 6.81377, + "491": 6.69837, + "492": 6.68149, + "493": 6.75646, + "494": 6.66282, + "495": 6.62263, + "496": 6.57706, + "497": 6.8292, + "498": 6.63548, + "499": 6.84385, + "500": 6.64283, + "501": 6.71966, + "502": 6.82988, + "503": 6.69833, + "504": 6.60751, + "505": 6.6112, + "506": 6.73586, + "507": 6.85391, + "508": 6.84629, + "509": 6.6384, + "510": 6.81034, + "511": 6.72977, + "512": 6.72804, + "513": 6.64821, + "514": 6.70064, + "515": 6.43824, + "516": 6.73421, + "517": 6.69542, + "518": 6.52993, + "519": 6.62474, + "520": 6.84935, + "521": 6.65329, + "522": 6.6979, + "523": 6.73262, + "524": 6.72634, + "525": 6.6655, + "526": 6.40663, + "527": 6.79088, + "528": 6.65206, + "529": 6.62295, + "530": 6.61639, + "531": 6.63503, + "532": 6.62382, + "533": 6.75435, + "534": 6.60296, + "535": 6.74138, + "536": 6.61812, + "537": 6.63086, + "538": 6.52418, + "539": 6.54299, + "540": 6.57593, + "541": 6.44382, + "542": 6.66189, + "543": 6.67325, + "544": 6.66927, + "545": 6.80511, + "546": 6.6246, + "547": 6.40979, + "548": 6.71663, + "549": 6.68986, + "550": 6.51987, + "551": 6.74092, + "552": 6.63227, + "553": 6.47534, + "554": 6.62778, + "555": 6.45222, + "556": 6.60749, + "557": 6.62431, + "558": 6.37676, + "559": 6.36118, + "560": 6.5756, + "561": 6.72381, + "562": 6.62768, + "563": 6.73287, + "564": 6.34176, + "565": 6.50706, + "566": 6.6902, + "567": 6.55838, + "568": 6.50084, + "569": 6.44415, + "570": 6.35619, + "571": 6.62259, + "572": 6.30471, + "573": 6.5721, + "574": 6.46259, + "575": 6.63541, + "576": 6.50701, + "577": 6.51656, + "578": 6.47574, + "579": 6.45618, + "580": 6.5583, + "581": 6.59714, + "582": 6.46959, + "583": 6.50413, + "584": 6.51087, + "585": 6.41424, + "586": 6.40258, + "587": 6.4501, + "588": 6.55622, + "589": 6.61456, + "590": 6.27891, + "591": 6.66415, + "592": 6.2545, + "593": 6.46521, + "594": 6.37467, + "595": 6.34819, + "596": 6.25003, + "597": 6.18054, + "598": 6.44279, + "599": 6.38602, + "600": 6.44414, + "601": 6.25051, + "602": 6.51804, + "603": 6.50819, + "604": 6.37382, + "605": 6.48026, + "606": 6.3013, + "607": 6.51999, + "608": 6.66049, + "609": 6.16075, + "610": 6.55805, + "611": 6.38737, + "612": 6.56702, + "613": 6.41056, + "614": 6.18827, + "615": 6.38286, + "616": 6.34421, + "617": 6.36273, + "618": 6.43626, + "619": 6.12502, + "620": 6.3943, + "621": 6.44427, + "622": 6.38402, + "623": 6.56769, + "624": 6.34417, + "625": 6.26521, + "626": 6.28634, + "627": 6.4276, + "628": 6.24043, + "629": 6.57298, + "630": 6.3523, + "631": 6.33431, + "632": 6.29554, + "633": 6.24213, + "634": 6.29476, + "635": 6.53142, + "636": 6.23005, + "637": 6.62121, + "638": 6.00686, + "639": 6.26506, + "640": 6.2796, + "641": 6.19435, + "642": 6.27007, + "643": 6.44413, + "644": 6.2445, + "645": 6.23092, + "646": 6.38932, + "647": 6.3209, + "648": 6.34188, + "649": 6.33297, + "650": 6.47025, + "651": 6.31782, + "652": 6.23993, + "653": 6.36817, + "654": 6.43495, + "655": 6.5135, + "656": 6.31371, + "657": 6.4163, + "658": 6.22993, + "659": 6.1432, + "660": 6.3808, + "661": 6.15725, + "662": 6.2613, + "663": 6.36151, + "664": 6.32043, + "665": 6.39194, + "666": 6.15182, + "667": 6.18562, + "668": 6.22741, + "669": 6.20408, + "670": 6.23602, + "671": 6.22904, + "672": 6.47492, + "673": 6.32812, + "674": 6.28343, + "675": 6.37362, + "676": 6.38018, + "677": 6.29511, + "678": 6.26804, + "679": 6.22803, + "680": 6.28357, + "681": 6.19077, + "682": 6.07906, + "683": 6.26403, + "684": 6.31575, + "685": 6.2874, + "686": 6.14011, + "687": 6.27685, + "688": 6.19835, + "689": 6.61075, + "690": 6.16856, + "691": 6.17286, + "692": 6.2649, + "693": 6.13689, + "694": 6.22553, + "695": 6.31786, + "696": 6.1061, + "697": 6.14556, + "698": 6.21959, + "699": 6.45326, + "700": 6.03519, + "701": 6.05302, + "702": 6.23703, + "703": 6.17441, + "704": 6.20621, + "705": 6.11844, + "706": 6.06567, + "707": 6.24456, + "708": 6.30245, + "709": 5.99551, + "710": 6.15229, + "711": 6.2479, + "712": 6.17146, + "713": 5.88608, + "714": 6.09975, + "715": 6.10497, + "716": 6.40586, + "717": 6.18363, + "718": 6.23537, + "719": 6.26862, + "720": 6.25804, + "721": 6.25605, + "722": 6.22472, + "723": 6.07187, + "724": 6.22017, + "725": 6.0314, + "726": 6.29244, + "727": 6.00644, + "728": 6.03616, + "729": 6.0826, + "730": 6.17412, + "731": 6.09163, + "732": 6.07888, + "733": 6.11348, + "734": 6.37763, + "735": 6.26791, + "736": 6.17709, + "737": 6.36077, + "738": 6.13247, + "739": 6.14636, + "740": 5.87836, + "741": 6.00499, + "742": 5.98594, + "743": 6.17515, + "744": 6.02317, + "745": 6.14565, + "746": 6.03122, + "747": 6.09452, + "748": 6.22864, + "749": 5.93308, + "750": 6.16381, + "751": 5.95292, + "752": 6.01389, + "753": 6.02392, + "754": 6.28379, + "755": 6.12598, + "756": 6.2443, + "757": 6.01404, + "758": 6.19738, + "759": 6.22084, + "760": 6.02115, + "761": 6.1856, + "762": 6.21798, + "763": 6.02971, + "764": 5.95856, + "765": 5.92315, + "766": 5.96127, + "767": 5.81063, + "768": 6.18012, + "769": 6.27004, + "770": 6.28915, + "771": 5.78425, + "772": 6.0231, + "773": 6.17908, + "774": 5.87868, + "775": 6.02111, + "776": 6.12258, + "777": 5.875, + "778": 6.04901, + "779": 5.86583, + "780": 6.13275, + "781": 5.8451, + "782": 6.03644, + "783": 5.94982, + "784": 5.91239, + "785": 6.08718, + "786": 6.0949, + "787": 5.6498, + "788": 5.99117, + "789": 6.20208, + "790": 6.25533, + "791": 5.78584, + "792": 5.98398, + "793": 6.17232, + "794": 6.02303, + "795": 5.99758, + "796": 6.15575, + "797": 6.04799, + "798": 6.04773, + "799": 6.10394, + "800": 6.00523, + "801": 6.13976, + "802": 5.97143, + "803": 6.14303, + "804": 5.99897, + "805": 5.8162, + "806": 6.08016, + "807": 6.03933, + "808": 5.91779, + "809": 5.76774, + "810": 6.00748, + "811": 5.92407, + "812": 5.89853, + "813": 5.95603, + "814": 6.0199, + "815": 5.80113, + "816": 6.10732, + "817": 5.92704, + "818": 6.05349, + "819": 5.99954, + "820": 5.71925, + "821": 5.93871, + "822": 6.18742, + "823": 5.82051, + "824": 5.97479, + "825": 6.17898, + "826": 6.18992, + "827": 6.04811, + "828": 6.0618, + "829": 5.8808, + "830": 5.9338, + "831": 5.89066, + "832": 5.95946, + "833": 6.05775, + "834": 5.98694, + "835": 5.99225, + "836": 5.78808, + "837": 6.1001, + "838": 5.85774, + "839": 5.82603, + "840": 6.17451, + "841": 5.77389, + "842": 5.88244, + "843": 5.93827, + "844": 6.0037, + "845": 6.08214, + "846": 5.68388, + "847": 5.75348, + "848": 5.96075, + "849": 6.0909, + "850": 5.83839, + "851": 6.01221, + "852": 5.74277, + "853": 5.9819, + "854": 6.00994, + "855": 5.81104, + "856": 5.99027, + "857": 5.99462, + "858": 6.04349, + "859": 5.94378, + "860": 6.08776, + "861": 6.05806, + "862": 5.99259, + "863": 5.83184, + "864": 5.83727, + "865": 5.93014, + "866": 5.88373, + "867": 5.87071, + "868": 6.0603, + "869": 6.08011, + "870": 5.96321, + "871": 6.03762, + "872": 5.89053, + "873": 5.83933, + "874": 6.02181, + "875": 5.90658, + "876": 5.96303, + "877": 5.92074, + "878": 6.09702, + "879": 5.76213, + "880": 6.0073, + "881": 5.98795, + "882": 5.90217, + "883": 5.67039, + "884": 5.95748, + "885": 5.74054, + "886": 5.98445, + "887": 5.90648, + "888": 5.8314, + "889": 6.00733, + "890": 6.01123, + "891": 5.94286, + "892": 5.70277, + "893": 6.08459, + "894": 5.72165, + "895": 5.83588, + "896": 5.83978, + "897": 5.84943, + "898": 5.92347, + "899": 5.93201, + "900": 5.8958, + "901": 5.94689, + "902": 5.82987, + "903": 6.04738, + "904": 5.92586, + "905": 5.89894, + "906": 5.61575, + "907": 5.90522, + "908": 5.73333, + "909": 5.98526, + "910": 5.85686, + "911": 5.69844, + "912": 5.69856, + "913": 5.76407, + "914": 5.82436, + "915": 5.79681, + "916": 5.88608, + "917": 5.867, + "918": 5.8166, + "919": 5.80848, + "920": 5.88971, + "921": 5.8407, + "922": 5.62064, + "923": 6.03383, + "924": 5.60482, + "925": 5.61823, + "926": 5.85786, + "927": 5.95554, + "928": 5.83872, + "929": 5.82237, + "930": 5.95411, + "931": 5.75622, + "932": 5.59098, + "933": 5.63134, + "934": 5.80496, + "935": 5.63538, + "936": 5.8317, + "937": 5.96485, + "938": 5.58943, + "939": 5.79158, + "940": 5.96089, + "941": 5.72676, + "942": 5.83595, + "943": 5.87091, + "944": 5.95881, + "945": 5.70173, + "946": 5.55832, + "947": 5.74676, + "948": 5.79172, + "949": 5.82702, + "950": 5.84636, + "951": 5.72232, + "952": 5.6926, + "953": 5.67846, + "954": 5.72814, + "955": 5.52701, + "956": 5.6247, + "957": 5.84082, + "958": 5.79725, + "959": 5.57236, + "960": 5.8033, + "961": 5.83318, + "962": 5.76931, + "963": 5.768, + "964": 5.70825, + "965": 5.63755, + "966": 5.60344, + "967": 5.72795, + "968": 5.74037, + "969": 5.82565, + "970": 5.64868, + "971": 5.70857, + "972": 5.85255, + "973": 5.67308, + "974": 5.7177, + "975": 5.86027, + "976": 5.71074, + "977": 5.77363, + "978": 5.68598, + "979": 5.5901, + "980": 5.76431, + "981": 5.89808, + "982": 5.47164, + "983": 5.61909, + "984": 5.54693, + "985": 5.58914, + "986": 5.6395, + "987": 5.57215, + "988": 5.71212, + "989": 5.69568, + "990": 5.62713, + "991": 5.85071, + "992": 5.77178, + "993": 5.87182, + "994": 5.69827, + "995": 5.7311, + "996": 5.73947, + "997": 5.81776, + "998": 5.83946, + "999": 5.83213, + "1000": 5.68618, + "1001": 5.86902, + "1002": 5.75759, + "1003": 5.64206, + "1004": 5.80056, + "1005": 5.53357, + "1006": 5.3287, + "1007": 5.7697, + "1008": 5.79391, + "1009": 5.65438, + "1010": 5.78459, + "1011": 5.89696, + "1012": 5.62269, + "1013": 5.61367, + "1014": 5.67992, + "1015": 5.56146, + "1016": 5.87263, + "1017": 5.83169, + "1018": 5.62357, + "1019": 5.73336, + "1020": 5.61404, + "1021": 5.85353, + "1022": 5.49696, + "1023": 5.65062, + "1024": 5.74334, + "1025": 5.57222, + "1026": 5.40994, + "1027": 5.59905, + "1028": 5.68935, + "1029": 5.68346, + "1030": 5.68799, + "1031": 5.40526, + "1032": 5.78443, + "1033": 5.57561, + "1034": 5.6274, + "1035": 5.71529, + "1036": 5.62368, + "1037": 5.36621, + "1038": 5.66561, + "1039": 5.6477, + "1040": 5.57324, + "1041": 5.59731, + "1042": 5.81493, + "1043": 5.56271, + "1044": 5.46406, + "1045": 5.9683, + "1046": 5.48617, + "1047": 5.39181, + "1048": 5.49562, + "1049": 5.67791, + "1050": 5.69881, + "1051": 5.5776, + "1052": 5.68149, + "1053": 5.63114, + "1054": 5.45857, + "1055": 5.59887, + "1056": 5.67508, + "1057": 5.75628, + "1058": 5.56524, + "1059": 5.74843, + "1060": 5.82162, + "1061": 5.47233, + "1062": 5.65043, + "1063": 5.50248, + "1064": 5.59125, + "1065": 5.55564, + "1066": 5.74466, + "1067": 5.67043, + "1068": 5.44061, + "1069": 5.61122, + "1070": 5.81207, + "1071": 5.51069, + "1072": 5.62291, + "1073": 5.6192, + "1074": 5.52379, + "1075": 5.70748, + "1076": 5.5951, + "1077": 5.70681, + "1078": 5.56223, + "1079": 5.61677, + "1080": 5.64259, + "1081": 5.62201, + "1082": 5.50149, + "1083": 5.64213, + "1084": 5.55087, + "1085": 5.40393, + "1086": 5.62042, + "1087": 5.44171, + "1088": 5.51111, + "1089": 5.76887, + "1090": 5.52736, + "1091": 5.51307, + "1092": 5.40781, + "1093": 5.69672, + "1094": 5.56925, + "1095": 5.5731, + "1096": 5.61367, + "1097": 5.6454, + "1098": 5.65292, + "1099": 5.51436, + "1100": 5.63973, + "1101": 5.67989, + "1102": 5.53567, + "1103": 5.54943, + "1104": 5.53818, + "1105": 5.55271, + "1106": 5.68243, + "1107": 5.68309, + "1108": 5.78112, + "1109": 5.54014, + "1110": 5.6617, + "1111": 5.59215, + "1112": 5.58702, + "1113": 5.62687, + "1114": 5.61504, + "1115": 5.59863, + "1116": 5.66461, + "1117": 5.64732, + "1118": 5.65418, + "1119": 5.70846, + "1120": 5.63501, + "1121": 5.37809, + "1122": 5.23308, + "1123": 5.47298, + "1124": 5.65454, + "1125": 5.68419, + "1126": 5.68674, + "1127": 5.56954, + "1128": 5.62438, + "1129": 5.29406, + "1130": 5.54548, + "1131": 5.6238, + "1132": 5.72077, + "1133": 5.51615, + "1134": 5.55302, + "1135": 5.51992, + "1136": 5.42021, + "1137": 5.46757, + "1138": 5.5657, + "1139": 5.41524, + "1140": 5.26144, + "1141": 5.58424, + "1142": 5.64054, + "1143": 5.385, + "1144": 5.3823, + "1145": 5.36615, + "1146": 5.62886, + "1147": 5.49181, + "1148": 5.50478, + "1149": 5.51839, + "1150": 5.39997, + "1151": 5.5553, + "1152": 5.42174, + "1153": 5.4602, + "1154": 5.50372, + "1155": 5.44072, + "1156": 5.34868, + "1157": 5.66217, + "1158": 5.39889, + "1159": 5.33332, + "1160": 5.79511, + "1161": 5.53597, + "1162": 5.45589, + "1163": 5.52529, + "1164": 5.38319, + "1165": 5.52473, + "1166": 5.48721, + "1167": 5.36058, + "1168": 5.49334, + "1169": 5.40387, + "1170": 5.58667, + "1171": 5.48535, + "1172": 5.64049, + "1173": 5.62012, + "1174": 5.51308, + "1175": 5.34473, + "1176": 5.38256, + "1177": 5.55838, + "1178": 5.46714, + "1179": 5.49373, + "1180": 5.46571, + "1181": 5.55314, + "1182": 5.59825, + "1183": 5.76884, + "1184": 5.54748, + "1185": 5.28691, + "1186": 5.60427, + "1187": 5.55401, + "1188": 5.51546, + "1189": 5.38634, + "1190": 5.40233, + "1191": 5.38976, + "1192": 5.49689, + "1193": 5.46486, + "1194": 5.45443, + "1195": 5.32542, + "1196": 5.52268, + "1197": 5.47666, + "1198": 5.52589, + "1199": 5.38688, + "1200": 5.33164, + "1201": 5.49012, + "1202": 5.43748, + "1203": 5.49375, + "1204": 5.40666, + "1205": 5.48999, + "1206": 5.33478, + "1207": 5.58651, + "1208": 5.42414, + "1209": 5.2931, + "1210": 5.49969, + "1211": 5.5071, + "1212": 5.59732, + "1213": 5.41745, + "1214": 5.49785, + "1215": 5.23706, + "1216": 5.41194, + "1217": 5.38264, + "1218": 5.4506, + "1219": 5.48501, + "1220": 5.38351, + "1221": 5.4519, + "1222": 5.31254, + "1223": 5.47747, + "1224": 5.41418, + "1225": 5.42845, + "1226": 5.32249, + "1227": 5.47547, + "1228": 5.73249, + "1229": 5.32716, + "1230": 5.41211, + "1231": 5.07649, + "1232": 5.78792, + "1233": 5.28531, + "1234": 5.24399, + "1235": 5.36824, + "1236": 5.47881, + "1237": 5.20655, + "1238": 5.41404, + "1239": 5.40719, + "1240": 5.46621, + "1241": 5.57221, + "1242": 5.45465, + "1243": 5.43424, + "1244": 5.51633, + "1245": 5.19115, + "1246": 5.71566, + "1247": 5.43, + "1248": 5.29843, + "1249": 5.40246, + "1250": 5.34088, + "1251": 5.41904, + "1252": 5.57108, + "1253": 5.489, + "1254": 5.31099, + "1255": 5.51387, + "1256": 5.60708, + "1257": 5.42325, + "1258": 5.55956, + "1259": 5.47585, + "1260": 5.50779, + "1261": 5.63801, + "1262": 5.39496, + "1263": 5.32432, + "1264": 5.50348, + "1265": 5.30656, + "1266": 5.23675, + "1267": 5.37031, + "1268": 5.38615, + "1269": 5.14823, + "1270": 5.39882, + "1271": 5.27753, + "1272": 5.52297, + "1273": 5.29632, + "1274": 5.34638, + "1275": 5.37784, + "1276": 5.3975, + "1277": 5.4606, + "1278": 5.35501, + "1279": 5.43897, + "1280": 5.45708, + "1281": 5.4056, + "1282": 5.38482, + "1283": 5.42347, + "1284": 5.34377, + "1285": 5.50505, + "1286": 5.33544, + "1287": 5.58814, + "1288": 5.2615, + "1289": 5.42995, + "1290": 5.49991, + "1291": 5.49987, + "1292": 5.44631, + "1293": 5.4171, + "1294": 5.49492, + "1295": 5.34499, + "1296": 5.18358, + "1297": 5.16726, + "1298": 5.11761, + "1299": 5.30129, + "1300": 5.21142, + "1301": 5.30283, + "1302": 5.27612, + "1303": 5.35547, + "1304": 5.43158, + "1305": 5.36825, + "1306": 5.25293, + "1307": 5.19217, + "1308": 5.27071, + "1309": 5.40774, + "1310": 5.26053, + "1311": 5.37774, + "1312": 5.35324, + "1313": 5.29428, + "1314": 5.29224, + "1315": 5.41906, + "1316": 5.25856, + "1317": 5.27981, + "1318": 5.21136, + "1319": 5.34401, + "1320": 5.4177, + "1321": 5.44957, + "1322": 5.46219, + "1323": 5.37269, + "1324": 5.24973, + "1325": 5.40538, + "1326": 5.53891, + "1327": 5.38638, + "1328": 5.21164, + "1329": 5.41667, + "1330": 5.39695, + "1331": 5.30979, + "1332": 5.3112, + "1333": 5.36823, + "1334": 5.44451, + "1335": 5.36788, + "1336": 5.43552, + "1337": 5.46933, + "1338": 5.30246, + "1339": 5.1362, + "1340": 5.41205, + "1341": 5.34033, + "1342": 5.35625, + "1343": 5.47387, + "1344": 5.37842, + "1345": 5.34238, + "1346": 5.07927, + "1347": 5.38404, + "1348": 5.49312, + "1349": 5.40746, + "1350": 5.02698, + "1351": 5.31566, + "1352": 5.15947, + "1353": 5.3409, + "1354": 5.35878, + "1355": 5.11364, + "1356": 5.25842, + "1357": 5.28929, + "1358": 5.15831, + "1359": 5.10775, + "1360": 5.17385, + "1361": 5.30604, + "1362": 5.06672, + "1363": 5.29722, + "1364": 5.3953, + "1365": 5.01953, + "1366": 5.1147, + "1367": 5.33054, + "1368": 5.18248, + "1369": 5.22391, + "1370": 5.1961, + "1371": 5.27906, + "1372": 5.25988, + "1373": 5.28404, + "1374": 5.2779, + "1375": 5.46001, + "1376": 5.26713, + "1377": 5.26807, + "1378": 5.31427, + "1379": 5.22765, + "1380": 5.25807, + "1381": 5.47919, + "1382": 5.08739, + "1383": 5.37543, + "1384": 5.36108, + "1385": 5.39028, + "1386": 5.16582, + "1387": 5.16244, + "1388": 5.27616, + "1389": 5.30262, + "1390": 5.25131, + "1391": 5.26406, + "1392": 5.36794, + "1393": 5.37824, + "1394": 5.40104, + "1395": 5.32383, + "1396": 5.21137, + "1397": 5.2828, + "1398": 5.36587, + "1399": 5.35557, + "1400": 5.26522, + "1401": 5.35981, + "1402": 5.42507, + "1403": 5.19768, + "1404": 5.27957, + "1405": 5.11754, + "1406": 4.98933, + "1407": 5.39818, + "1408": 5.1921, + "1409": 5.39429, + "1410": 5.37153, + "1411": 4.91585, + "1412": 5.35244, + "1413": 5.41055, + "1414": 5.21699, + "1415": 5.44044, + "1416": 5.32598, + "1417": 5.39078, + "1418": 5.29894, + "1419": 5.31316, + "1420": 5.43638, + "1421": 5.39683, + "1422": 5.41859, + "1423": 4.99867, + "1424": 5.33177, + "1425": 5.58491, + "1426": 5.23068, + "1427": 5.31742, + "1428": 5.33463, + "1429": 5.07871, + "1430": 5.32748, + "1431": 5.32237, + "1432": 5.34216, + "1433": 5.18496, + "1434": 5.16175, + "1435": 5.20122, + "1436": 5.10715, + "1437": 5.22566, + "1438": 5.31423, + "1439": 5.34769, + "1440": 5.34295, + "1441": 5.16777, + "1442": 5.21935, + "1443": 5.20553, + "1444": 5.12984, + "1445": 5.07414, + "1446": 5.26456, + "1447": 5.25775, + "1448": 5.29302, + "1449": 5.24616, + "1450": 5.34316, + "1451": 5.07004, + "1452": 5.26796, + "1453": 5.1741, + "1454": 5.01458, + "1455": 5.12771, + "1456": 5.27213, + "1457": 5.1882, + "1458": 5.00695, + "1459": 5.2215, + "1460": 5.23955, + "1461": 5.08, + "1462": 4.97269, + "1463": 5.15114, + "1464": 5.22113, + "1465": 5.27344, + "1466": 5.36076, + "1467": 5.34631, + "1468": 5.2303, + "1469": 5.05117, + "1470": 5.12322, + "1471": 5.25302, + "1472": 5.12175, + "1473": 5.10167, + "1474": 5.21744, + "1475": 5.18613, + "1476": 5.15517, + "1477": 5.26215, + "1478": 5.30407, + "1479": 5.01063, + "1480": 5.182, + "1481": 5.25124, + "1482": 5.3494, + "1483": 5.27058, + "1484": 4.92644, + "1485": 5.29103, + "1486": 5.04435, + "1487": 4.88432, + "1488": 5.18325, + "1489": 5.10139, + "1490": 5.04545, + "1491": 5.3188, + "1492": 5.22283, + "1493": 4.94061, + "1494": 5.10891, + "1495": 5.13402, + "1496": 5.05779, + "1497": 5.36536, + "1498": 5.30609, + "1499": 5.143, + "1500": 5.09554, + "1501": 5.0349, + "1502": 5.15423, + "1503": 5.43131, + "1504": 5.32574, + "1505": 5.00836, + "1506": 5.14423, + "1507": 5.16501, + "1508": 5.16864, + "1509": 5.3204, + "1510": 5.02703, + "1511": 5.1198, + "1512": 4.98354, + "1513": 5.1699, + "1514": 5.33407, + "1515": 5.36306, + "1516": 5.27572, + "1517": 5.2256, + "1518": 5.02899, + "1519": 5.29833, + "1520": 5.13757, + "1521": 5.15715, + "1522": 5.33462, + "1523": 5.24144, + "1524": 5.06791, + "1525": 5.20708, + "1526": 5.27861, + "1527": 5.25864, + "1528": 5.2395, + "1529": 5.18253, + "1530": 5.23913, + "1531": 5.09996, + "1532": 5.15679, + "1533": 5.05231, + "1534": 5.21917, + "1535": 5.16769, + "1536": 5.102, + "1537": 5.0318, + "1538": 4.91991, + "1539": 5.2394, + "1540": 5.11391, + "1541": 5.25502, + "1542": 5.23775, + "1543": 5.05438, + "1544": 5.08156, + "1545": 5.11794, + "1546": 5.32713, + "1547": 5.10763, + "1548": 5.23418, + "1549": 5.23089, + "1550": 4.97536, + "1551": 5.25942, + "1552": 5.0226, + "1553": 5.14887, + "1554": 5.11051, + "1555": 5.11223, + "1556": 5.19882, + "1557": 5.08844, + "1558": 5.22982, + "1559": 5.00137, + "1560": 5.11269, + "1561": 5.14639, + "1562": 5.18443, + "1563": 5.24639, + "1564": 5.26429, + "1565": 5.08809, + "1566": 5.29393, + "1567": 5.04372, + "1568": 5.08304, + "1569": 5.2002, + "1570": 5.17168, + "1571": 4.95228, + "1572": 5.04524, + "1573": 5.02748, + "1574": 4.99831, + "1575": 5.23124, + "1576": 5.20891, + "1577": 5.12722, + "1578": 5.36355, + "1579": 4.94343, + "1580": 5.12556, + "1581": 5.09739, + "1582": 5.28014, + "1583": 5.04619, + "1584": 5.0566, + "1585": 5.11727, + "1586": 5.30646, + "1587": 5.13281, + "1588": 5.22351, + "1589": 4.83814, + "1590": 5.09825, + "1591": 5.18082, + "1592": 5.14078, + "1593": 5.23646, + "1594": 5.11532, + "1595": 5.10761, + "1596": 5.19194, + "1597": 5.11362, + "1598": 5.16252, + "1599": 5.18865, + "1600": 4.86676, + "1601": 5.11898, + "1602": 5.22827, + "1603": 5.19524, + "1604": 5.05797, + "1605": 5.03277, + "1606": 4.98991, + "1607": 5.06915, + "1608": 4.97927, + "1609": 5.07061, + "1610": 5.04561, + "1611": 4.9918, + "1612": 4.75806, + "1613": 5.03141, + "1614": 4.87811, + "1615": 5.07817, + "1616": 5.22549, + "1617": 5.06182, + "1618": 4.98945, + "1619": 5.18486, + "1620": 5.14429, + "1621": 5.31666, + "1622": 5.06737, + "1623": 5.15063, + "1624": 5.1305, + "1625": 5.12197, + "1626": 5.10206, + "1627": 5.1085, + "1628": 5.06234, + "1629": 4.93316, + "1630": 5.06616, + "1631": 5.05719, + "1632": 5.10145, + "1633": 4.97087, + "1634": 4.92194, + "1635": 5.05013, + "1636": 4.9202, + "1637": 5.22863, + "1638": 5.15783, + "1639": 4.9808, + "1640": 5.00716, + "1641": 5.12367, + "1642": 5.0869, + "1643": 5.05029, + "1644": 5.12283, + "1645": 4.96415, + "1646": 5.12257, + "1647": 5.03267, + "1648": 5.1903, + "1649": 4.92263, + "1650": 5.0596, + "1651": 4.93391, + "1652": 5.21143, + "1653": 5.1587, + "1654": 5.13384, + "1655": 5.16235, + "1656": 5.34793, + "1657": 5.21074, + "1658": 5.04155, + "1659": 4.92889, + "1660": 4.8117, + "1661": 5.02968, + "1662": 5.14515, + "1663": 5.15868, + "1664": 4.98471, + "1665": 5.11027, + "1666": 5.10315, + "1667": 4.84929, + "1668": 5.10956, + "1669": 5.07311, + "1670": 5.11152, + "1671": 5.16545, + "1672": 4.77709, + "1673": 5.03502, + "1674": 4.91572, + "1675": 5.04406, + "1676": 5.0023, + "1677": 4.80013, + "1678": 5.02745, + "1679": 4.88908, + "1680": 5.03791, + "1681": 5.06371, + "1682": 5.03586, + "1683": 4.90255, + "1684": 5.06133, + "1685": 5.13096, + "1686": 5.075, + "1687": 4.97679, + "1688": 5.17279, + "1689": 5.1507, + "1690": 4.99681, + "1691": 4.99961, + "1692": 4.91412, + "1693": 5.02305, + "1694": 4.94741, + "1695": 4.91895, + "1696": 5.0846, + "1697": 5.05067, + "1698": 4.95116, + "1699": 5.00638, + "1700": 4.94576, + "1701": 5.16681, + "1702": 5.07316, + "1703": 5.16582, + "1704": 5.14235, + "1705": 4.96408, + "1706": 4.98303, + "1707": 4.78833, + "1708": 5.03283, + "1709": 5.2281, + "1710": 5.02918, + "1711": 5.18873, + "1712": 5.19088, + "1713": 5.03631, + "1714": 5.04689, + "1715": 4.91662, + "1716": 4.93663, + "1717": 4.86445, + "1718": 5.02654, + "1719": 5.12575, + "1720": 5.02353, + "1721": 4.9343, + "1722": 5.06572, + "1723": 4.93302, + "1724": 5.03906, + "1725": 5.19169, + "1726": 5.06497, + "1727": 4.91076, + "1728": 5.01922, + "1729": 5.04885, + "1730": 4.91107, + "1731": 5.00108, + "1732": 4.91468, + "1733": 5.12873, + "1734": 4.83023, + "1735": 5.21293, + "1736": 4.91729, + "1737": 4.86164, + "1738": 4.97933, + "1739": 5.16149, + "1740": 4.84041, + "1741": 4.78298, + "1742": 4.91062, + "1743": 5.09353, + "1744": 4.98531, + "1745": 4.82544, + "1746": 4.94973, + "1747": 4.86843, + "1748": 5.06696, + "1749": 4.86793, + "1750": 5.01333, + "1751": 5.12023, + "1752": 4.90813, + "1753": 5.09204, + "1754": 5.05813, + "1755": 4.89777, + "1756": 5.02216, + "1757": 5.14157, + "1758": 4.87188, + "1759": 4.94434, + "1760": 4.83222, + "1761": 5.02427, + "1762": 4.81507, + "1763": 4.77391, + "1764": 4.93175, + "1765": 5.14727, + "1766": 5.33614, + "1767": 5.22331, + "1768": 4.94712, + "1769": 5.0043, + "1770": 4.98512, + "1771": 4.96473, + "1772": 4.98299, + "1773": 4.97266, + "1774": 4.87138, + "1775": 4.9493, + "1776": 4.9958, + "1777": 4.94665, + "1778": 4.99288, + "1779": 5.08212, + "1780": 4.83608, + "1781": 5.05478, + "1782": 4.99549, + "1783": 5.01236, + "1784": 4.93254, + "1785": 5.16842, + "1786": 4.80892, + "1787": 4.9699, + "1788": 4.82948, + "1789": 4.88554, + "1790": 4.80386, + "1791": 4.74542, + "1792": 4.87988, + "1793": 5.11081, + "1794": 4.98659, + "1795": 4.97147, + "1796": 5.00354, + "1797": 4.79101, + "1798": 4.77029, + "1799": 5.01913, + "1800": 4.91155, + "1801": 5.04891, + "1802": 4.82591, + "1803": 4.95313, + "1804": 4.88492, + "1805": 4.90634, + "1806": 4.88167, + "1807": 4.92894, + "1808": 4.92469, + "1809": 5.15028, + "1810": 5.09708, + "1811": 4.96325, + "1812": 4.8059, + "1813": 5.1023, + "1814": 4.7819, + "1815": 4.86518, + "1816": 5.05104, + "1817": 4.79238, + "1818": 4.80401, + "1819": 5.02672, + "1820": 4.68884, + "1821": 5.02319, + "1822": 4.66224, + "1823": 4.86936, + "1824": 4.7914, + "1825": 5.06607, + "1826": 4.81841, + "1827": 4.79544, + "1828": 4.9506, + "1829": 5.10848, + "1830": 4.9163, + "1831": 4.89965, + "1832": 4.83328, + "1833": 4.78854, + "1834": 4.94794, + "1835": 4.96175, + "1836": 4.91339, + "1837": 4.6762, + "1838": 4.80703, + "1839": 4.89949, + "1840": 4.91213, + "1841": 4.84083, + "1842": 4.9567, + "1843": 4.71182, + "1844": 4.6194, + "1845": 5.00584, + "1846": 4.75435, + "1847": 4.86491, + "1848": 4.9035, + "1849": 4.85124, + "1850": 4.87005, + "1851": 5.01617, + "1852": 4.97859, + "1853": 4.82821, + "1854": 4.86426, + "1855": 4.82455, + "1856": 4.75214, + "1857": 4.96641, + "1858": 4.96711, + "1859": 4.7484, + "1860": 4.86558, + "1861": 5.21257, + "1862": 4.61253, + "1863": 4.83567, + "1864": 4.74748, + "1865": 4.86472, + "1866": 4.78934, + "1867": 5.00307, + "1868": 4.72073, + "1869": 4.76301, + "1870": 4.93972, + "1871": 5.00163, + "1872": 4.68713, + "1873": 4.70038, + "1874": 4.85131, + "1875": 4.85367, + "1876": 4.74378, + "1877": 4.80696, + "1878": 4.8139, + "1879": 4.82462, + "1880": 4.89248, + "1881": 4.79379, + "1882": 4.79882, + "1883": 4.78556, + "1884": 4.97714, + "1885": 4.92363, + "1886": 4.82454, + "1887": 4.82091, + "1888": 4.97246, + "1889": 4.96553, + "1890": 4.71236, + "1891": 4.65764, + "1892": 4.85277, + "1893": 4.65022, + "1894": 4.90165, + "1895": 4.79, + "1896": 4.66068, + "1897": 4.79617, + "1898": 4.92161, + "1899": 4.77736, + "1900": 4.91325, + "1901": 4.84998, + "1902": 4.787, + "1903": 4.76372, + "1904": 4.65638, + "1905": 4.55077, + "1906": 4.81577, + "1907": 4.9106, + "1908": 5.03029, + "1909": 4.89294, + "1910": 4.7884, + "1911": 4.81269, + "1912": 4.653, + "1913": 4.95098, + "1914": 4.88806, + "1915": 4.86687, + "1916": 4.9302, + "1917": 4.85504, + "1918": 4.87427, + "1919": 4.99557, + "1920": 4.77001, + "1921": 4.88729, + "1922": 4.8196, + "1923": 4.75752, + "1924": 4.8297, + "1925": 5.05687, + "1926": 4.94229, + "1927": 4.93308, + "1928": 4.92739, + "1929": 4.93147, + "1930": 4.917, + "1931": 4.77692, + "1932": 4.86743, + "1933": 4.83532, + "1934": 4.84373, + "1935": 5.11279, + "1936": 4.88728, + "1937": 4.8824, + "1938": 4.80623, + "1939": 4.70831, + "1940": 4.83067, + "1941": 4.74224, + "1942": 4.87785, + "1943": 4.74082, + "1944": 4.7536, + "1945": 4.69017, + "1946": 4.91953, + "1947": 4.87613, + "1948": 4.60452, + "1949": 4.89888, + "1950": 4.79826, + "1951": 4.9677, + "1952": 4.73855, + "1953": 4.79852, + "1954": 4.7398, + "1955": 4.85209, + "1956": 4.88278, + "1957": 4.73599, + "1958": 4.70215, + "1959": 4.76471, + "1960": 4.76967, + "1961": 4.71471, + "1962": 4.83443, + "1963": 4.82459, + "1964": 4.85019, + "1965": 4.87867, + "1966": 4.79219, + "1967": 4.60013, + "1968": 4.83399, + "1969": 4.59632, + "1970": 4.58346, + "1971": 4.90585, + "1972": 4.89941, + "1973": 4.55559, + "1974": 4.8295, + "1975": 4.83261, + "1976": 4.71818, + "1977": 4.58171, + "1978": 5.00781, + "1979": 4.6663, + "1980": 4.74961, + "1981": 4.87741, + "1982": 4.72647, + "1983": 4.89363, + "1984": 4.64954, + "1985": 4.78941, + "1986": 4.70195, + "1987": 4.8185, + "1988": 4.89272, + "1989": 4.63799, + "1990": 4.79789, + "1991": 4.70399, + "1992": 4.80349, + "1993": 4.74121, + "1994": 4.85611, + "1995": 4.5595, + "1996": 4.65792, + "1997": 4.8133, + "1998": 4.68041, + "1999": 4.73244, + "2000": 4.6301 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 26.0, + "2": 32.0, + "3": 38.0, + "4": 33.0, + "5": 32.0, + "6": 30.0, + "7": 33.0, + "8": 34.0, + "9": 40.0, + "10": 31.0, + "11": 26.0, + "12": 33.0, + "13": 28.0, + "14": 29.0, + "15": 28.0, + "16": 27.0, + "17": 32.0, + "18": 28.0, + "19": 31.0, + "20": 39.0, + "21": 22.0, + "22": 29.0, + "23": 39.0, + "24": 35.0, + "25": 31.0, + "26": 40.0, + "27": 39.0, + "28": 42.0, + "29": 53.0, + "30": 51.0, + "31": 48.0, + "32": 51.0, + "33": 38.0, + "34": 48.0, + "35": 47.0, + "36": 49.0, + "37": 42.0, + "38": 43.0, + "39": 52.0, + "40": 55.0, + "41": 39.0, + "42": 54.0, + "43": 57.0, + "44": 53.0, + "45": 46.0, + "46": 61.0, + "47": 52.0, + "48": 54.0, + "49": 64.0, + "50": 64.0, + "51": 42.0, + "52": 55.0, + "53": 48.0, + "54": 71.0, + "55": 56.0, + "56": 74.0, + "57": 70.0, + "58": 57.0, + "59": 53.0, + "60": 67.0, + "61": 63.0, + "62": 59.0, + "63": 66.0, + "64": 70.0, + "65": 59.0, + "66": 74.0, + "67": 81.0, + "68": 74.0, + "69": 60.0, + "70": 60.0, + "71": 66.0, + "72": 75.0, + "73": 67.0, + "74": 63.0, + "75": 60.0, + "76": 60.0, + "77": 78.0, + "78": 78.0, + "79": 58.0, + "80": 63.0, + "81": 63.0, + "82": 50.0, + "83": 63.0, + "84": 72.0, + "85": 69.0, + "86": 80.0, + "87": 70.0, + "88": 68.0, + "89": 69.0, + "90": 63.0, + "91": 58.0, + "92": 87.0, + "93": 65.0, + "94": 50.0, + "95": 67.0, + "96": 71.0, + "97": 70.0, + "98": 81.0, + "99": 66.0, + "100": 76.0, + "101": 67.0, + "102": 44.0, + "103": 60.0, + "104": 68.0, + "105": 84.0, + "106": 61.0, + "107": 76.0, + "108": 68.0, + "109": 76.0, + "110": 74.0, + "111": 75.0, + "112": 78.0, + "113": 58.0, + "114": 66.0, + "115": 71.0, + "116": 63.0, + "117": 74.0, + "118": 52.0, + "119": 74.0, + "120": 52.0, + "121": 76.0, + "122": 66.0, + "123": 81.0, + "124": 76.0, + "125": 87.0, + "126": 49.0, + "127": 56.0, + "128": 78.0, + "129": 53.0, + "130": 76.0, + "131": 86.0, + "132": 61.0, + "133": 72.0, + "134": 62.0, + "135": 59.0, + "136": 60.0, + "137": 57.0, + "138": 81.0, + "139": 74.0, + "140": 59.0, + "141": 50.0, + "142": 64.0, + "143": 54.0, + "144": 49.0, + "145": 57.0, + "146": 51.0, + "147": 49.0, + "148": 69.0, + "149": 49.0, + "150": 66.0, + "151": 57.0, + "152": 51.0, + "153": 61.0, + "154": 58.0, + "155": 68.0, + "156": 68.0, + "157": 51.0, + "158": 68.0, + "159": 60.0, + "160": 64.0, + "161": 66.0, + "162": 75.0, + "163": 40.0, + "164": 84.0, + "165": 50.0, + "166": 68.0, + "167": 54.0, + "168": 58.0, + "169": 65.0, + "170": 71.0, + "171": 54.0, + "172": 64.0, + "173": 81.0, + "174": 55.0, + "175": 63.0, + "176": 69.0, + "177": 80.0, + "178": 68.0, + "179": 69.0, + "180": 64.0, + "181": 41.0, + "182": 63.0, + "183": 66.0, + "184": 67.0, + "185": 77.0, + "186": 77.0, + "187": 61.0, + "188": 62.0, + "189": 50.0, + "190": 57.0, + "191": 60.0, + "192": 67.0, + "193": 70.0, + "194": 72.0, + "195": 60.0, + "196": 81.0, + "197": 56.0, + "198": 47.0, + "199": 50.0, + "200": 86.0, + "201": 52.0, + "202": 64.0, + "203": 58.0, + "204": 63.0, + "205": 40.0, + "206": 72.0, + "207": 50.0, + "208": 42.0, + "209": 69.0, + "210": 68.0, + "211": 56.0, + "212": 64.0, + "213": 60.0, + "214": 62.0, + "215": 66.0, + "216": 58.0, + "217": 59.0, + "218": 70.0, + "219": 80.0, + "220": 81.0, + "221": 51.0, + "222": 57.0, + "223": 67.0, + "224": 53.0, + "225": 61.0, + "226": 68.0, + "227": 76.0, + "228": 59.0, + "229": 44.0, + "230": 50.0, + "231": 58.0, + "232": 65.0, + "233": 90.0, + "234": 60.0, + "235": 98.0, + "236": 49.0, + "237": 92.0, + "238": 71.0, + "239": 68.0, + "240": 79.0, + "241": 67.0, + "242": 75.0, + "243": 66.0, + "244": 59.0, + "245": 81.0, + "246": 80.0, + "247": 88.0, + "248": 81.0, + "249": 79.0, + "250": 80.0, + "251": 74.0, + "252": 72.0, + "253": 57.0, + "254": 67.0, + "255": 79.0, + "256": 86.0, + "257": 66.0, + "258": 94.0, + "259": 69.0, + "260": 70.0, + "261": 64.0, + "262": 77.0, + "263": 74.0, + "264": 70.0, + "265": 68.0, + "266": 67.0, + "267": 66.0, + "268": 59.0, + "269": 73.0, + "270": 85.0, + "271": 67.0, + "272": 81.0, + "273": 71.0, + "274": 69.0, + "275": 72.0, + "276": 72.0, + "277": 82.0, + "278": 61.0, + "279": 94.0, + "280": 56.0, + "281": 55.0, + "282": 73.0, + "283": 90.0, + "284": 85.0, + "285": 49.0, + "286": 50.0, + "287": 90.0, + "288": 71.0, + "289": 85.0, + "290": 75.0, + "291": 88.0, + "292": 88.0, + "293": 91.0, + "294": 84.0, + "295": 85.0, + "296": 102.0, + "297": 70.0, + "298": 65.0, + "299": 80.0, + "300": 80.0, + "301": 91.0, + "302": 94.0, + "303": 71.0, + "304": 74.0, + "305": 59.0, + "306": 72.0, + "307": 73.0, + "308": 91.0, + "309": 88.0, + "310": 82.0, + "311": 84.0, + "312": 73.0, + "313": 97.0, + "314": 74.0, + "315": 69.0, + "316": 96.0, + "317": 61.0, + "318": 99.0, + "319": 67.0, + "320": 77.0, + "321": 86.0, + "322": 70.0, + "323": 86.0, + "324": 96.0, + "325": 74.0, + "326": 97.0, + "327": 73.0, + "328": 99.0, + "329": 93.0, + "330": 96.0, + "331": 81.0, + "332": 79.0, + "333": 97.0, + "334": 81.0, + "335": 84.0, + "336": 81.0, + "337": 99.0, + "338": 89.0, + "339": 93.0, + "340": 101.0, + "341": 93.0, + "342": 57.0, + "343": 81.0, + "344": 105.0, + "345": 88.0, + "346": 85.0, + "347": 91.0, + "348": 82.0, + "349": 78.0, + "350": 101.0, + "351": 105.0, + "352": 76.0, + "353": 112.0, + "354": 72.0, + "355": 79.0, + "356": 104.0, + "357": 86.0, + "358": 77.0, + "359": 99.0, + "360": 102.0, + "361": 64.0, + "362": 123.0, + "363": 96.0, + "364": 95.0, + "365": 85.0, + "366": 82.0, + "367": 84.0, + "368": 83.0, + "369": 77.0, + "370": 118.0, + "371": 76.0, + "372": 77.0, + "373": 96.0, + "374": 68.0, + "375": 92.0, + "376": 84.0, + "377": 98.0, + "378": 99.0, + "379": 108.0, + "380": 96.0, + "381": 92.0, + "382": 75.0, + "383": 89.0, + "384": 100.0, + "385": 73.0, + "386": 85.0, + "387": 73.0, + "388": 93.0, + "389": 88.0, + "390": 90.0, + "391": 115.0, + "392": 88.0, + "393": 99.0, + "394": 104.0, + "395": 125.0, + "396": 80.0, + "397": 78.0, + "398": 67.0, + "399": 104.0, + "400": 96.0, + "401": 105.0, + "402": 88.0, + "403": 97.0, + "404": 101.0, + "405": 85.0, + "406": 114.0, + "407": 76.0, + "408": 98.0, + "409": 84.0, + "410": 102.0, + "411": 81.0, + "412": 56.0, + "413": 68.0, + "414": 90.0, + "415": 95.0, + "416": 93.0, + "417": 90.0, + "418": 60.0, + "419": 86.0, + "420": 76.0, + "421": 110.0, + "422": 89.0, + "423": 78.0, + "424": 82.0, + "425": 94.0, + "426": 80.0, + "427": 96.0, + "428": 86.0, + "429": 92.0, + "430": 84.0, + "431": 87.0, + "432": 80.0, + "433": 81.0, + "434": 93.0, + "435": 83.0, + "436": 82.0, + "437": 91.0, + "438": 62.0, + "439": 72.0, + "440": 79.0, + "441": 87.0, + "442": 106.0, + "443": 106.0, + "444": 58.0, + "445": 93.0, + "446": 89.0, + "447": 97.0, + "448": 79.0, + "449": 90.0, + "450": 83.0, + "451": 63.0, + "452": 70.0, + "453": 63.0, + "454": 80.0, + "455": 114.0, + "456": 98.0, + "457": 101.0, + "458": 70.0, + "459": 69.0, + "460": 65.0, + "461": 115.0, + "462": 63.0, + "463": 73.0, + "464": 69.0, + "465": 95.0, + "466": 76.0, + "467": 77.0, + "468": 90.0, + "469": 65.0, + "470": 91.0, + "471": 76.0, + "472": 60.0, + "473": 94.0, + "474": 69.0, + "475": 90.0, + "476": 66.0, + "477": 75.0, + "478": 78.0, + "479": 63.0, + "480": 73.0, + "481": 80.0, + "482": 77.0, + "483": 78.0, + "484": 84.0, + "485": 70.0, + "486": 84.0, + "487": 69.0, + "488": 88.0, + "489": 77.0, + "490": 59.0, + "491": 83.0, + "492": 57.0, + "493": 83.0, + "494": 69.0, + "495": 50.0, + "496": 56.0, + "497": 97.0, + "498": 77.0, + "499": 75.0, + "500": 60.0, + "501": 64.0, + "502": 64.0, + "503": 71.0, + "504": 77.0, + "505": 68.0, + "506": 65.0, + "507": 80.0, + "508": 42.0, + "509": 63.0, + "510": 77.0, + "511": 81.0, + "512": 57.0, + "513": 61.0, + "514": 60.0, + "515": 71.0, + "516": 61.0, + "517": 85.0, + "518": 43.0, + "519": 72.0, + "520": 82.0, + "521": 50.0, + "522": 58.0, + "523": 74.0, + "524": 70.0, + "525": 82.0, + "526": 60.0, + "527": 71.0, + "528": 63.0, + "529": 66.0, + "530": 67.0, + "531": 69.0, + "532": 72.0, + "533": 81.0, + "534": 62.0, + "535": 66.0, + "536": 61.0, + "537": 60.0, + "538": 55.0, + "539": 62.0, + "540": 63.0, + "541": 61.0, + "542": 61.0, + "543": 55.0, + "544": 64.0, + "545": 73.0, + "546": 77.0, + "547": 69.0, + "548": 75.0, + "549": 61.0, + "550": 61.0, + "551": 63.0, + "552": 71.0, + "553": 78.0, + "554": 67.0, + "555": 65.0, + "556": 74.0, + "557": 61.0, + "558": 62.0, + "559": 62.0, + "560": 71.0, + "561": 56.0, + "562": 65.0, + "563": 77.0, + "564": 67.0, + "565": 55.0, + "566": 58.0, + "567": 42.0, + "568": 70.0, + "569": 56.0, + "570": 60.0, + "571": 58.0, + "572": 41.0, + "573": 71.0, + "574": 69.0, + "575": 85.0, + "576": 44.0, + "577": 50.0, + "578": 69.0, + "579": 62.0, + "580": 67.0, + "581": 59.0, + "582": 58.0, + "583": 55.0, + "584": 47.0, + "585": 60.0, + "586": 41.0, + "587": 47.0, + "588": 53.0, + "589": 55.0, + "590": 46.0, + "591": 69.0, + "592": 50.0, + "593": 52.0, + "594": 56.0, + "595": 47.0, + "596": 44.0, + "597": 33.0, + "598": 61.0, + "599": 50.0, + "600": 88.0, + "601": 55.0, + "602": 64.0, + "603": 60.0, + "604": 57.0, + "605": 57.0, + "606": 45.0, + "607": 54.0, + "608": 45.0, + "609": 40.0, + "610": 45.0, + "611": 53.0, + "612": 52.0, + "613": 73.0, + "614": 53.0, + "615": 52.0, + "616": 64.0, + "617": 44.0, + "618": 59.0, + "619": 50.0, + "620": 72.0, + "621": 50.0, + "622": 58.0, + "623": 57.0, + "624": 56.0, + "625": 56.0, + "626": 71.0, + "627": 50.0, + "628": 49.0, + "629": 50.0, + "630": 50.0, + "631": 40.0, + "632": 45.0, + "633": 42.0, + "634": 38.0, + "635": 51.0, + "636": 36.0, + "637": 55.0, + "638": 45.0, + "639": 63.0, + "640": 52.0, + "641": 51.0, + "642": 52.0, + "643": 49.0, + "644": 51.0, + "645": 57.0, + "646": 57.0, + "647": 69.0, + "648": 60.0, + "649": 49.0, + "650": 49.0, + "651": 66.0, + "652": 49.0, + "653": 59.0, + "654": 42.0, + "655": 42.0, + "656": 46.0, + "657": 49.0, + "658": 50.0, + "659": 44.0, + "660": 53.0, + "661": 46.0, + "662": 60.0, + "663": 43.0, + "664": 61.0, + "665": 37.0, + "666": 30.0, + "667": 42.0, + "668": 41.0, + "669": 44.0, + "670": 44.0, + "671": 59.0, + "672": 53.0, + "673": 47.0, + "674": 42.0, + "675": 54.0, + "676": 43.0, + "677": 68.0, + "678": 41.0, + "679": 38.0, + "680": 46.0, + "681": 50.0, + "682": 33.0, + "683": 38.0, + "684": 52.0, + "685": 40.0, + "686": 43.0, + "687": 61.0, + "688": 57.0, + "689": 51.0, + "690": 35.0, + "691": 45.0, + "692": 55.0, + "693": 36.0, + "694": 50.0, + "695": 50.0, + "696": 51.0, + "697": 41.0, + "698": 37.0, + "699": 47.0, + "700": 42.0, + "701": 37.0, + "702": 33.0, + "703": 39.0, + "704": 43.0, + "705": 45.0, + "706": 32.0, + "707": 38.0, + "708": 38.0, + "709": 46.0, + "710": 35.0, + "711": 48.0, + "712": 35.0, + "713": 48.0, + "714": 37.0, + "715": 48.0, + "716": 36.0, + "717": 34.0, + "718": 26.0, + "719": 36.0, + "720": 34.0, + "721": 36.0, + "722": 35.0, + "723": 29.0, + "724": 47.0, + "725": 32.0, + "726": 39.0, + "727": 40.0, + "728": 39.0, + "729": 47.0, + "730": 36.0, + "731": 48.0, + "732": 43.0, + "733": 39.0, + "734": 51.0, + "735": 40.0, + "736": 49.0, + "737": 44.0, + "738": 27.0, + "739": 46.0, + "740": 38.0, + "741": 38.0, + "742": 45.0, + "743": 44.0, + "744": 52.0, + "745": 48.0, + "746": 50.0, + "747": 53.0, + "748": 52.0, + "749": 48.0, + "750": 46.0, + "751": 40.0, + "752": 50.0, + "753": 44.0, + "754": 43.0, + "755": 48.0, + "756": 38.0, + "757": 45.0, + "758": 40.0, + "759": 56.0, + "760": 46.0, + "761": 44.0, + "762": 48.0, + "763": 54.0, + "764": 49.0, + "765": 42.0, + "766": 57.0, + "767": 45.0, + "768": 51.0, + "769": 60.0, + "770": 51.0, + "771": 31.0, + "772": 41.0, + "773": 60.0, + "774": 37.0, + "775": 43.0, + "776": 37.0, + "777": 34.0, + "778": 42.0, + "779": 37.0, + "780": 34.0, + "781": 41.0, + "782": 25.0, + "783": 30.0, + "784": 39.0, + "785": 34.0, + "786": 38.0, + "787": 47.0, + "788": 41.0, + "789": 50.0, + "790": 44.0, + "791": 34.0, + "792": 38.0, + "793": 53.0, + "794": 45.0, + "795": 52.0, + "796": 39.0, + "797": 41.0, + "798": 39.0, + "799": 44.0, + "800": 46.0, + "801": 44.0, + "802": 40.0, + "803": 47.0, + "804": 34.0, + "805": 45.0, + "806": 43.0, + "807": 46.0, + "808": 36.0, + "809": 35.0, + "810": 35.0, + "811": 44.0, + "812": 47.0, + "813": 41.0, + "814": 36.0, + "815": 41.0, + "816": 52.0, + "817": 43.0, + "818": 35.0, + "819": 52.0, + "820": 40.0, + "821": 29.0, + "822": 34.0, + "823": 44.0, + "824": 47.0, + "825": 36.0, + "826": 40.0, + "827": 29.0, + "828": 35.0, + "829": 32.0, + "830": 30.0, + "831": 36.0, + "832": 34.0, + "833": 39.0, + "834": 50.0, + "835": 38.0, + "836": 37.0, + "837": 50.0, + "838": 45.0, + "839": 52.0, + "840": 37.0, + "841": 35.0, + "842": 30.0, + "843": 50.0, + "844": 23.0, + "845": 45.0, + "846": 25.0, + "847": 32.0, + "848": 25.0, + "849": 34.0, + "850": 39.0, + "851": 46.0, + "852": 41.0, + "853": 43.0, + "854": 45.0, + "855": 27.0, + "856": 47.0, + "857": 47.0, + "858": 46.0, + "859": 35.0, + "860": 45.0, + "861": 30.0, + "862": 39.0, + "863": 21.0, + "864": 26.0, + "865": 46.0, + "866": 44.0, + "867": 48.0, + "868": 27.0, + "869": 42.0, + "870": 45.0, + "871": 33.0, + "872": 49.0, + "873": 32.0, + "874": 56.0, + "875": 38.0, + "876": 41.0, + "877": 40.0, + "878": 37.0, + "879": 22.0, + "880": 39.0, + "881": 40.0, + "882": 49.0, + "883": 39.0, + "884": 35.0, + "885": 32.0, + "886": 45.0, + "887": 41.0, + "888": 34.0, + "889": 35.0, + "890": 37.0, + "891": 41.0, + "892": 42.0, + "893": 42.0, + "894": 34.0, + "895": 38.0, + "896": 37.0, + "897": 41.0, + "898": 33.0, + "899": 35.0, + "900": 39.0, + "901": 37.0, + "902": 39.0, + "903": 42.0, + "904": 38.0, + "905": 32.0, + "906": 34.0, + "907": 38.0, + "908": 39.0, + "909": 52.0, + "910": 34.0, + "911": 26.0, + "912": 46.0, + "913": 40.0, + "914": 48.0, + "915": 25.0, + "916": 49.0, + "917": 36.0, + "918": 31.0, + "919": 26.0, + "920": 40.0, + "921": 34.0, + "922": 38.0, + "923": 41.0, + "924": 24.0, + "925": 27.0, + "926": 43.0, + "927": 31.0, + "928": 40.0, + "929": 32.0, + "930": 42.0, + "931": 33.0, + "932": 34.0, + "933": 38.0, + "934": 41.0, + "935": 26.0, + "936": 44.0, + "937": 36.0, + "938": 37.0, + "939": 28.0, + "940": 33.0, + "941": 34.0, + "942": 31.0, + "943": 26.0, + "944": 37.0, + "945": 29.0, + "946": 31.0, + "947": 34.0, + "948": 41.0, + "949": 31.0, + "950": 35.0, + "951": 31.0, + "952": 38.0, + "953": 47.0, + "954": 43.0, + "955": 46.0, + "956": 35.0, + "957": 40.0, + "958": 37.0, + "959": 52.0, + "960": 35.0, + "961": 38.0, + "962": 41.0, + "963": 45.0, + "964": 43.0, + "965": 51.0, + "966": 38.0, + "967": 31.0, + "968": 32.0, + "969": 35.0, + "970": 48.0, + "971": 38.0, + "972": 43.0, + "973": 38.0, + "974": 40.0, + "975": 43.0, + "976": 29.0, + "977": 44.0, + "978": 31.0, + "979": 43.0, + "980": 39.0, + "981": 33.0, + "982": 30.0, + "983": 54.0, + "984": 43.0, + "985": 48.0, + "986": 40.0, + "987": 30.0, + "988": 38.0, + "989": 38.0, + "990": 42.0, + "991": 36.0, + "992": 48.0, + "993": 47.0, + "994": 50.0, + "995": 35.0, + "996": 29.0, + "997": 51.0, + "998": 42.0, + "999": 35.0, + "1000": 28.0, + "1001": 23.0, + "1002": 35.0, + "1003": 39.0, + "1004": 46.0, + "1005": 42.0, + "1006": 27.0, + "1007": 44.0, + "1008": 32.0, + "1009": 34.0, + "1010": 29.0, + "1011": 31.0, + "1012": 28.0, + "1013": 37.0, + "1014": 29.0, + "1015": 39.0, + "1016": 31.0, + "1017": 37.0, + "1018": 46.0, + "1019": 26.0, + "1020": 34.0, + "1021": 30.0, + "1022": 46.0, + "1023": 38.0, + "1024": 49.0, + "1025": 41.0, + "1026": 55.0, + "1027": 37.0, + "1028": 29.0, + "1029": 38.0, + "1030": 35.0, + "1031": 41.0, + "1032": 42.0, + "1033": 27.0, + "1034": 29.0, + "1035": 32.0, + "1036": 25.0, + "1037": 34.0, + "1038": 32.0, + "1039": 31.0, + "1040": 30.0, + "1041": 24.0, + "1042": 20.0, + "1043": 26.0, + "1044": 44.0, + "1045": 37.0, + "1046": 34.0, + "1047": 27.0, + "1048": 36.0, + "1049": 42.0, + "1050": 37.0, + "1051": 40.0, + "1052": 40.0, + "1053": 32.0, + "1054": 37.0, + "1055": 31.0, + "1056": 36.0, + "1057": 37.0, + "1058": 37.0, + "1059": 35.0, + "1060": 32.0, + "1061": 37.0, + "1062": 45.0, + "1063": 38.0, + "1064": 42.0, + "1065": 35.0, + "1066": 36.0, + "1067": 29.0, + "1068": 30.0, + "1069": 30.0, + "1070": 39.0, + "1071": 33.0, + "1072": 36.0, + "1073": 41.0, + "1074": 47.0, + "1075": 36.0, + "1076": 39.0, + "1077": 45.0, + "1078": 32.0, + "1079": 46.0, + "1080": 43.0, + "1081": 40.0, + "1082": 42.0, + "1083": 42.0, + "1084": 42.0, + "1085": 38.0, + "1086": 42.0, + "1087": 36.0, + "1088": 31.0, + "1089": 42.0, + "1090": 28.0, + "1091": 36.0, + "1092": 35.0, + "1093": 36.0, + "1094": 41.0, + "1095": 37.0, + "1096": 48.0, + "1097": 33.0, + "1098": 24.0, + "1099": 43.0, + "1100": 41.0, + "1101": 38.0, + "1102": 39.0, + "1103": 29.0, + "1104": 33.0, + "1105": 38.0, + "1106": 37.0, + "1107": 30.0, + "1108": 41.0, + "1109": 41.0, + "1110": 42.0, + "1111": 43.0, + "1112": 25.0, + "1113": 40.0, + "1114": 32.0, + "1115": 34.0, + "1116": 45.0, + "1117": 40.0, + "1118": 39.0, + "1119": 31.0, + "1120": 28.0, + "1121": 28.0, + "1122": 28.0, + "1123": 43.0, + "1124": 34.0, + "1125": 26.0, + "1126": 33.0, + "1127": 31.0, + "1128": 33.0, + "1129": 43.0, + "1130": 43.0, + "1131": 40.0, + "1132": 42.0, + "1133": 34.0, + "1134": 32.0, + "1135": 29.0, + "1136": 36.0, + "1137": 42.0, + "1138": 34.0, + "1139": 31.0, + "1140": 38.0, + "1141": 37.0, + "1142": 38.0, + "1143": 44.0, + "1144": 40.0, + "1145": 39.0, + "1146": 42.0, + "1147": 35.0, + "1148": 29.0, + "1149": 40.0, + "1150": 34.0, + "1151": 27.0, + "1152": 22.0, + "1153": 36.0, + "1154": 31.0, + "1155": 41.0, + "1156": 26.0, + "1157": 33.0, + "1158": 35.0, + "1159": 36.0, + "1160": 41.0, + "1161": 40.0, + "1162": 48.0, + "1163": 37.0, + "1164": 43.0, + "1165": 34.0, + "1166": 30.0, + "1167": 34.0, + "1168": 31.0, + "1169": 41.0, + "1170": 27.0, + "1171": 40.0, + "1172": 34.0, + "1173": 23.0, + "1174": 40.0, + "1175": 30.0, + "1176": 50.0, + "1177": 39.0, + "1178": 33.0, + "1179": 42.0, + "1180": 31.0, + "1181": 30.0, + "1182": 38.0, + "1183": 37.0, + "1184": 35.0, + "1185": 31.0, + "1186": 29.0, + "1187": 39.0, + "1188": 34.0, + "1189": 48.0, + "1190": 32.0, + "1191": 41.0, + "1192": 45.0, + "1193": 28.0, + "1194": 46.0, + "1195": 34.0, + "1196": 38.0, + "1197": 51.0, + "1198": 36.0, + "1199": 40.0, + "1200": 29.0, + "1201": 37.0, + "1202": 32.0, + "1203": 35.0, + "1204": 37.0, + "1205": 56.0, + "1206": 40.0, + "1207": 36.0, + "1208": 41.0, + "1209": 31.0, + "1210": 39.0, + "1211": 46.0, + "1212": 45.0, + "1213": 57.0, + "1214": 31.0, + "1215": 33.0, + "1216": 31.0, + "1217": 34.0, + "1218": 42.0, + "1219": 45.0, + "1220": 37.0, + "1221": 44.0, + "1222": 32.0, + "1223": 35.0, + "1224": 34.0, + "1225": 45.0, + "1226": 28.0, + "1227": 34.0, + "1228": 27.0, + "1229": 23.0, + "1230": 25.0, + "1231": 14.0, + "1232": 36.0, + "1233": 39.0, + "1234": 37.0, + "1235": 32.0, + "1236": 41.0, + "1237": 30.0, + "1238": 36.0, + "1239": 37.0, + "1240": 48.0, + "1241": 31.0, + "1242": 34.0, + "1243": 35.0, + "1244": 29.0, + "1245": 28.0, + "1246": 36.0, + "1247": 31.0, + "1248": 38.0, + "1249": 27.0, + "1250": 40.0, + "1251": 26.0, + "1252": 42.0, + "1253": 32.0, + "1254": 39.0, + "1255": 46.0, + "1256": 41.0, + "1257": 30.0, + "1258": 44.0, + "1259": 32.0, + "1260": 25.0, + "1261": 42.0, + "1262": 36.0, + "1263": 34.0, + "1264": 32.0, + "1265": 35.0, + "1266": 34.0, + "1267": 38.0, + "1268": 43.0, + "1269": 30.0, + "1270": 28.0, + "1271": 42.0, + "1272": 32.0, + "1273": 40.0, + "1274": 44.0, + "1275": 38.0, + "1276": 31.0, + "1277": 54.0, + "1278": 46.0, + "1279": 44.0, + "1280": 34.0, + "1281": 26.0, + "1282": 37.0, + "1283": 32.0, + "1284": 43.0, + "1285": 43.0, + "1286": 36.0, + "1287": 46.0, + "1288": 33.0, + "1289": 43.0, + "1290": 37.0, + "1291": 42.0, + "1292": 38.0, + "1293": 43.0, + "1294": 30.0, + "1295": 34.0, + "1296": 31.0, + "1297": 26.0, + "1298": 38.0, + "1299": 40.0, + "1300": 32.0, + "1301": 43.0, + "1302": 35.0, + "1303": 35.0, + "1304": 41.0, + "1305": 30.0, + "1306": 28.0, + "1307": 34.0, + "1308": 32.0, + "1309": 36.0, + "1310": 29.0, + "1311": 43.0, + "1312": 32.0, + "1313": 37.0, + "1314": 35.0, + "1315": 33.0, + "1316": 37.0, + "1317": 33.0, + "1318": 41.0, + "1319": 28.0, + "1320": 42.0, + "1321": 30.0, + "1322": 21.0, + "1323": 28.0, + "1324": 40.0, + "1325": 36.0, + "1326": 43.0, + "1327": 32.0, + "1328": 35.0, + "1329": 33.0, + "1330": 27.0, + "1331": 30.0, + "1332": 36.0, + "1333": 45.0, + "1334": 32.0, + "1335": 41.0, + "1336": 38.0, + "1337": 37.0, + "1338": 38.0, + "1339": 27.0, + "1340": 33.0, + "1341": 47.0, + "1342": 24.0, + "1343": 27.0, + "1344": 34.0, + "1345": 34.0, + "1346": 21.0, + "1347": 33.0, + "1348": 33.0, + "1349": 42.0, + "1350": 30.0, + "1351": 39.0, + "1352": 26.0, + "1353": 36.0, + "1354": 40.0, + "1355": 31.0, + "1356": 46.0, + "1357": 46.0, + "1358": 29.0, + "1359": 29.0, + "1360": 30.0, + "1361": 35.0, + "1362": 40.0, + "1363": 33.0, + "1364": 36.0, + "1365": 34.0, + "1366": 47.0, + "1367": 31.0, + "1368": 37.0, + "1369": 28.0, + "1370": 41.0, + "1371": 30.0, + "1372": 42.0, + "1373": 44.0, + "1374": 34.0, + "1375": 22.0, + "1376": 47.0, + "1377": 29.0, + "1378": 39.0, + "1379": 49.0, + "1380": 44.0, + "1381": 30.0, + "1382": 45.0, + "1383": 44.0, + "1384": 31.0, + "1385": 35.0, + "1386": 31.0, + "1387": 31.0, + "1388": 22.0, + "1389": 32.0, + "1390": 38.0, + "1391": 42.0, + "1392": 34.0, + "1393": 43.0, + "1394": 33.0, + "1395": 39.0, + "1396": 37.0, + "1397": 27.0, + "1398": 33.0, + "1399": 29.0, + "1400": 36.0, + "1401": 28.0, + "1402": 27.0, + "1403": 23.0, + "1404": 28.0, + "1405": 36.0, + "1406": 29.0, + "1407": 36.0, + "1408": 43.0, + "1409": 37.0, + "1410": 37.0, + "1411": 38.0, + "1412": 28.0, + "1413": 48.0, + "1414": 34.0, + "1415": 42.0, + "1416": 35.0, + "1417": 34.0, + "1418": 43.0, + "1419": 38.0, + "1420": 33.0, + "1421": 33.0, + "1422": 53.0, + "1423": 22.0, + "1424": 35.0, + "1425": 43.0, + "1426": 36.0, + "1427": 43.0, + "1428": 31.0, + "1429": 30.0, + "1430": 36.0, + "1431": 29.0, + "1432": 37.0, + "1433": 32.0, + "1434": 47.0, + "1435": 38.0, + "1436": 40.0, + "1437": 47.0, + "1438": 28.0, + "1439": 33.0, + "1440": 25.0, + "1441": 35.0, + "1442": 38.0, + "1443": 42.0, + "1444": 28.0, + "1445": 34.0, + "1446": 28.0, + "1447": 39.0, + "1448": 45.0, + "1449": 41.0, + "1450": 25.0, + "1451": 38.0, + "1452": 27.0, + "1453": 28.0, + "1454": 28.0, + "1455": 32.0, + "1456": 40.0, + "1457": 33.0, + "1458": 37.0, + "1459": 41.0, + "1460": 31.0, + "1461": 34.0, + "1462": 23.0, + "1463": 33.0, + "1464": 42.0, + "1465": 42.0, + "1466": 29.0, + "1467": 27.0, + "1468": 41.0, + "1469": 30.0, + "1470": 35.0, + "1471": 32.0, + "1472": 44.0, + "1473": 53.0, + "1474": 28.0, + "1475": 25.0, + "1476": 47.0, + "1477": 40.0, + "1478": 26.0, + "1479": 33.0, + "1480": 33.0, + "1481": 33.0, + "1482": 33.0, + "1483": 31.0, + "1484": 31.0, + "1485": 45.0, + "1486": 37.0, + "1487": 32.0, + "1488": 26.0, + "1489": 45.0, + "1490": 40.0, + "1491": 44.0, + "1492": 44.0, + "1493": 44.0, + "1494": 33.0, + "1495": 42.0, + "1496": 32.0, + "1497": 39.0, + "1498": 32.0, + "1499": 42.0, + "1500": 42.0, + "1501": 46.0, + "1502": 46.0, + "1503": 39.0, + "1504": 31.0, + "1505": 47.0, + "1506": 41.0, + "1507": 35.0, + "1508": 39.0, + "1509": 32.0, + "1510": 37.0, + "1511": 52.0, + "1512": 29.0, + "1513": 46.0, + "1514": 40.0, + "1515": 41.0, + "1516": 31.0, + "1517": 39.0, + "1518": 40.0, + "1519": 32.0, + "1520": 34.0, + "1521": 44.0, + "1522": 53.0, + "1523": 40.0, + "1524": 39.0, + "1525": 30.0, + "1526": 34.0, + "1527": 19.0, + "1528": 40.0, + "1529": 30.0, + "1530": 38.0, + "1531": 28.0, + "1532": 30.0, + "1533": 43.0, + "1534": 34.0, + "1535": 35.0, + "1536": 34.0, + "1537": 33.0, + "1538": 36.0, + "1539": 32.0, + "1540": 38.0, + "1541": 35.0, + "1542": 50.0, + "1543": 50.0, + "1544": 38.0, + "1545": 38.0, + "1546": 35.0, + "1547": 31.0, + "1548": 39.0, + "1549": 36.0, + "1550": 30.0, + "1551": 42.0, + "1552": 49.0, + "1553": 46.0, + "1554": 41.0, + "1555": 25.0, + "1556": 33.0, + "1557": 46.0, + "1558": 43.0, + "1559": 36.0, + "1560": 30.0, + "1561": 48.0, + "1562": 30.0, + "1563": 38.0, + "1564": 40.0, + "1565": 30.0, + "1566": 34.0, + "1567": 36.0, + "1568": 43.0, + "1569": 35.0, + "1570": 43.0, + "1571": 32.0, + "1572": 34.0, + "1573": 35.0, + "1574": 31.0, + "1575": 39.0, + "1576": 30.0, + "1577": 41.0, + "1578": 46.0, + "1579": 35.0, + "1580": 39.0, + "1581": 43.0, + "1582": 30.0, + "1583": 43.0, + "1584": 36.0, + "1585": 37.0, + "1586": 44.0, + "1587": 37.0, + "1588": 43.0, + "1589": 41.0, + "1590": 46.0, + "1591": 32.0, + "1592": 37.0, + "1593": 32.0, + "1594": 36.0, + "1595": 27.0, + "1596": 40.0, + "1597": 36.0, + "1598": 36.0, + "1599": 32.0, + "1600": 41.0, + "1601": 34.0, + "1602": 38.0, + "1603": 48.0, + "1604": 29.0, + "1605": 42.0, + "1606": 33.0, + "1607": 41.0, + "1608": 40.0, + "1609": 42.0, + "1610": 37.0, + "1611": 35.0, + "1612": 37.0, + "1613": 39.0, + "1614": 51.0, + "1615": 38.0, + "1616": 33.0, + "1617": 45.0, + "1618": 43.0, + "1619": 32.0, + "1620": 43.0, + "1621": 47.0, + "1622": 36.0, + "1623": 50.0, + "1624": 40.0, + "1625": 33.0, + "1626": 39.0, + "1627": 34.0, + "1628": 40.0, + "1629": 30.0, + "1630": 34.0, + "1631": 45.0, + "1632": 39.0, + "1633": 40.0, + "1634": 30.0, + "1635": 53.0, + "1636": 31.0, + "1637": 35.0, + "1638": 39.0, + "1639": 42.0, + "1640": 37.0, + "1641": 43.0, + "1642": 30.0, + "1643": 43.0, + "1644": 36.0, + "1645": 37.0, + "1646": 61.0, + "1647": 34.0, + "1648": 41.0, + "1649": 39.0, + "1650": 42.0, + "1651": 33.0, + "1652": 45.0, + "1653": 25.0, + "1654": 36.0, + "1655": 29.0, + "1656": 45.0, + "1657": 37.0, + "1658": 46.0, + "1659": 38.0, + "1660": 46.0, + "1661": 41.0, + "1662": 35.0, + "1663": 35.0, + "1664": 37.0, + "1665": 30.0, + "1666": 44.0, + "1667": 45.0, + "1668": 40.0, + "1669": 35.0, + "1670": 35.0, + "1671": 37.0, + "1672": 32.0, + "1673": 48.0, + "1674": 41.0, + "1675": 40.0, + "1676": 49.0, + "1677": 35.0, + "1678": 30.0, + "1679": 45.0, + "1680": 40.0, + "1681": 32.0, + "1682": 32.0, + "1683": 42.0, + "1684": 44.0, + "1685": 47.0, + "1686": 30.0, + "1687": 31.0, + "1688": 31.0, + "1689": 40.0, + "1690": 43.0, + "1691": 36.0, + "1692": 31.0, + "1693": 31.0, + "1694": 35.0, + "1695": 41.0, + "1696": 32.0, + "1697": 27.0, + "1698": 39.0, + "1699": 41.0, + "1700": 31.0, + "1701": 35.0, + "1702": 31.0, + "1703": 40.0, + "1704": 36.0, + "1705": 36.0, + "1706": 46.0, + "1707": 26.0, + "1708": 37.0, + "1709": 37.0, + "1710": 39.0, + "1711": 32.0, + "1712": 46.0, + "1713": 44.0, + "1714": 45.0, + "1715": 43.0, + "1716": 30.0, + "1717": 41.0, + "1718": 43.0, + "1719": 28.0, + "1720": 36.0, + "1721": 26.0, + "1722": 42.0, + "1723": 42.0, + "1724": 39.0, + "1725": 28.0, + "1726": 46.0, + "1727": 43.0, + "1728": 40.0, + "1729": 44.0, + "1730": 38.0, + "1731": 26.0, + "1732": 39.0, + "1733": 44.0, + "1734": 39.0, + "1735": 34.0, + "1736": 46.0, + "1737": 46.0, + "1738": 34.0, + "1739": 47.0, + "1740": 44.0, + "1741": 31.0, + "1742": 46.0, + "1743": 43.0, + "1744": 46.0, + "1745": 53.0, + "1746": 42.0, + "1747": 37.0, + "1748": 37.0, + "1749": 47.0, + "1750": 46.0, + "1751": 43.0, + "1752": 35.0, + "1753": 41.0, + "1754": 40.0, + "1755": 32.0, + "1756": 36.0, + "1757": 48.0, + "1758": 34.0, + "1759": 49.0, + "1760": 46.0, + "1761": 36.0, + "1762": 34.0, + "1763": 36.0, + "1764": 39.0, + "1765": 24.0, + "1766": 46.0, + "1767": 46.0, + "1768": 36.0, + "1769": 56.0, + "1770": 28.0, + "1771": 42.0, + "1772": 52.0, + "1773": 45.0, + "1774": 37.0, + "1775": 33.0, + "1776": 43.0, + "1777": 54.0, + "1778": 39.0, + "1779": 33.0, + "1780": 39.0, + "1781": 45.0, + "1782": 35.0, + "1783": 43.0, + "1784": 53.0, + "1785": 36.0, + "1786": 38.0, + "1787": 43.0, + "1788": 45.0, + "1789": 33.0, + "1790": 42.0, + "1791": 44.0, + "1792": 34.0, + "1793": 30.0, + "1794": 40.0, + "1795": 55.0, + "1796": 33.0, + "1797": 30.0, + "1798": 41.0, + "1799": 37.0, + "1800": 41.0, + "1801": 40.0, + "1802": 30.0, + "1803": 36.0, + "1804": 41.0, + "1805": 34.0, + "1806": 39.0, + "1807": 36.0, + "1808": 43.0, + "1809": 45.0, + "1810": 41.0, + "1811": 28.0, + "1812": 33.0, + "1813": 30.0, + "1814": 36.0, + "1815": 35.0, + "1816": 35.0, + "1817": 35.0, + "1818": 42.0, + "1819": 25.0, + "1820": 38.0, + "1821": 48.0, + "1822": 38.0, + "1823": 38.0, + "1824": 49.0, + "1825": 46.0, + "1826": 32.0, + "1827": 47.0, + "1828": 30.0, + "1829": 50.0, + "1830": 43.0, + "1831": 36.0, + "1832": 47.0, + "1833": 42.0, + "1834": 41.0, + "1835": 39.0, + "1836": 39.0, + "1837": 34.0, + "1838": 50.0, + "1839": 35.0, + "1840": 41.0, + "1841": 30.0, + "1842": 34.0, + "1843": 44.0, + "1844": 38.0, + "1845": 41.0, + "1846": 32.0, + "1847": 32.0, + "1848": 36.0, + "1849": 45.0, + "1850": 40.0, + "1851": 36.0, + "1852": 41.0, + "1853": 29.0, + "1854": 35.0, + "1855": 45.0, + "1856": 39.0, + "1857": 33.0, + "1858": 40.0, + "1859": 40.0, + "1860": 48.0, + "1861": 37.0, + "1862": 46.0, + "1863": 47.0, + "1864": 48.0, + "1865": 38.0, + "1866": 51.0, + "1867": 34.0, + "1868": 40.0, + "1869": 42.0, + "1870": 38.0, + "1871": 36.0, + "1872": 42.0, + "1873": 42.0, + "1874": 38.0, + "1875": 51.0, + "1876": 39.0, + "1877": 41.0, + "1878": 26.0, + "1879": 33.0, + "1880": 41.0, + "1881": 50.0, + "1882": 37.0, + "1883": 45.0, + "1884": 39.0, + "1885": 37.0, + "1886": 32.0, + "1887": 36.0, + "1888": 28.0, + "1889": 38.0, + "1890": 37.0, + "1891": 51.0, + "1892": 44.0, + "1893": 50.0, + "1894": 44.0, + "1895": 35.0, + "1896": 34.0, + "1897": 35.0, + "1898": 31.0, + "1899": 39.0, + "1900": 40.0, + "1901": 52.0, + "1902": 31.0, + "1903": 44.0, + "1904": 45.0, + "1905": 32.0, + "1906": 49.0, + "1907": 34.0, + "1908": 33.0, + "1909": 34.0, + "1910": 45.0, + "1911": 41.0, + "1912": 46.0, + "1913": 46.0, + "1914": 51.0, + "1915": 35.0, + "1916": 42.0, + "1917": 40.0, + "1918": 32.0, + "1919": 54.0, + "1920": 41.0, + "1921": 40.0, + "1922": 36.0, + "1923": 34.0, + "1924": 43.0, + "1925": 47.0, + "1926": 42.0, + "1927": 37.0, + "1928": 40.0, + "1929": 40.0, + "1930": 39.0, + "1931": 37.0, + "1932": 40.0, + "1933": 46.0, + "1934": 30.0, + "1935": 50.0, + "1936": 51.0, + "1937": 34.0, + "1938": 38.0, + "1939": 44.0, + "1940": 35.0, + "1941": 39.0, + "1942": 59.0, + "1943": 42.0, + "1944": 46.0, + "1945": 36.0, + "1946": 43.0, + "1947": 39.0, + "1948": 39.0, + "1949": 31.0, + "1950": 36.0, + "1951": 41.0, + "1952": 37.0, + "1953": 26.0, + "1954": 43.0, + "1955": 33.0, + "1956": 37.0, + "1957": 48.0, + "1958": 35.0, + "1959": 44.0, + "1960": 35.0, + "1961": 28.0, + "1962": 51.0, + "1963": 47.0, + "1964": 33.0, + "1965": 56.0, + "1966": 46.0, + "1967": 33.0, + "1968": 53.0, + "1969": 36.0, + "1970": 47.0, + "1971": 35.0, + "1972": 34.0, + "1973": 38.0, + "1974": 46.0, + "1975": 32.0, + "1976": 43.0, + "1977": 38.0, + "1978": 43.0, + "1979": 49.0, + "1980": 32.0, + "1981": 30.0, + "1982": 55.0, + "1983": 41.0, + "1984": 62.0, + "1985": 41.0, + "1986": 48.0, + "1987": 48.0, + "1988": 41.0, + "1989": 50.0, + "1990": 53.0, + "1991": 45.0, + "1992": 46.0, + "1993": 60.0, + "1994": 30.0, + "1995": 41.0, + "1996": 51.0, + "1997": 41.0, + "1998": 45.0, + "1999": 32.0, + "2000": 43.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 302618112.0, + "2": 302618112.0, + "3": 302618112.0, + "4": 302618112.0, + "5": 302618112.0, + "6": 302618112.0, + "7": 302618112.0, + "8": 302618112.0, + "9": 302618112.0, + "10": 302618112.0, + "11": 302618112.0, + "12": 302618112.0, + "13": 302618112.0, + "14": 302618112.0, + "15": 302618112.0, + "16": 302618112.0, + "17": 302618112.0, + "18": 302618112.0, + "19": 302618112.0, + "20": 302618112.0, + "21": 302618112.0, + "22": 302618112.0, + "23": 302618112.0, + "24": 302618112.0, + "25": 302618112.0, + "26": 302618112.0, + "27": 302618112.0, + "28": 302618112.0, + "29": 302618112.0, + "30": 302618112.0, + "31": 302618112.0, + "32": 302618112.0, + "33": 302618112.0, + "34": 302618112.0, + "35": 302618112.0, + "36": 302618112.0, + "37": 302618112.0, + "38": 302618112.0, + "39": 302618112.0, + "40": 302618112.0, + "41": 302618112.0, + "42": 302618112.0, + "43": 302618112.0, + "44": 302618112.0, + "45": 302618112.0, + "46": 302618112.0, + "47": 302618112.0, + "48": 302618112.0, + "49": 302618112.0, + "50": 302618112.0, + "51": 302618112.0, + "52": 302618112.0, + "53": 302618112.0, + "54": 302618112.0, + "55": 302618112.0, + "56": 302618112.0, + "57": 302618112.0, + "58": 302618112.0, + "59": 302618112.0, + "60": 302618112.0, + "61": 302618112.0, + "62": 302618112.0, + "63": 302618112.0, + "64": 302618112.0, + "65": 302618112.0, + "66": 302618112.0, + "67": 302618112.0, + "68": 302618112.0, + "69": 302618112.0, + "70": 302618112.0, + "71": 302618112.0, + "72": 302618112.0, + "73": 302618112.0, + "74": 302618112.0, + "75": 302618112.0, + "76": 302618112.0, + "77": 302618112.0, + "78": 302618112.0, + "79": 302618112.0, + "80": 302618112.0, + "81": 302618112.0, + "82": 302618112.0, + "83": 302618112.0, + "84": 302618112.0, + "85": 302618112.0, + "86": 302618112.0, + "87": 302618112.0, + "88": 302618112.0, + "89": 302618112.0, + "90": 302618112.0, + "91": 302618112.0, + "92": 302618112.0, + "93": 302618112.0, + "94": 302618112.0, + "95": 302618112.0, + "96": 302618112.0, + "97": 302618112.0, + "98": 302618112.0, + "99": 302618112.0, + "100": 302618112.0, + "101": 302618112.0, + "102": 302618112.0, + "103": 302618112.0, + "104": 302618112.0, + "105": 302618112.0, + "106": 302618112.0, + "107": 302618112.0, + "108": 302618112.0, + "109": 302618112.0, + "110": 302618112.0, + "111": 302618112.0, + "112": 302618112.0, + "113": 302618112.0, + "114": 302618112.0, + "115": 302618112.0, + "116": 302618112.0, + "117": 302618112.0, + "118": 302618112.0, + "119": 302618112.0, + "120": 302618112.0, + "121": 302618112.0, + "122": 302618112.0, + "123": 302618112.0, + "124": 302618112.0, + "125": 302618112.0, + "126": 302618112.0, + "127": 302618112.0, + "128": 302618112.0, + "129": 302618112.0, + "130": 302618112.0, + "131": 302618112.0, + "132": 302618112.0, + "133": 302618112.0, + "134": 302618112.0, + "135": 302618112.0, + "136": 302618112.0, + "137": 302618112.0, + "138": 302618112.0, + "139": 302618112.0, + "140": 302618112.0, + "141": 302618112.0, + "142": 302618112.0, + "143": 302618112.0, + "144": 302618112.0, + "145": 302618112.0, + "146": 302618112.0, + "147": 302618112.0, + "148": 302618112.0, + "149": 302618112.0, + "150": 302618112.0, + "151": 302618112.0, + "152": 302618112.0, + "153": 302618112.0, + "154": 302618112.0, + "155": 302618112.0, + "156": 302618112.0, + "157": 302618112.0, + "158": 302618112.0, + "159": 302618112.0, + "160": 302618112.0, + "161": 302618112.0, + "162": 302618112.0, + "163": 302618112.0, + "164": 302618112.0, + "165": 302618112.0, + "166": 302618112.0, + "167": 302618112.0, + "168": 302618112.0, + "169": 302618112.0, + "170": 302618112.0, + "171": 302618112.0, + "172": 302618112.0, + "173": 302618112.0, + "174": 302618112.0, + "175": 302618112.0, + "176": 302618112.0, + "177": 302618112.0, + "178": 302618112.0, + "179": 302618112.0, + "180": 302618112.0, + "181": 302618112.0, + "182": 302618112.0, + "183": 302618112.0, + "184": 302618112.0, + "185": 302618112.0, + "186": 302618112.0, + "187": 302618112.0, + "188": 302618112.0, + "189": 302618112.0, + "190": 302618112.0, + "191": 302618112.0, + "192": 302618112.0, + "193": 302618112.0, + "194": 302618112.0, + "195": 302618112.0, + "196": 302618112.0, + "197": 302618112.0, + "198": 302618112.0, + "199": 302618112.0, + "200": 302618112.0, + "201": 302618112.0, + "202": 302618112.0, + "203": 302618112.0, + "204": 302618112.0, + "205": 302618112.0, + "206": 302618112.0, + "207": 302618112.0, + "208": 302618112.0, + "209": 302618112.0, + "210": 302618112.0, + "211": 302618112.0, + "212": 302618112.0, + "213": 302618112.0, + "214": 302618112.0, + "215": 302618112.0, + "216": 302618112.0, + "217": 302618112.0, + "218": 302618112.0, + "219": 302618112.0, + "220": 302618112.0, + "221": 302618112.0, + "222": 302618112.0, + "223": 302618112.0, + "224": 302618112.0, + "225": 302618112.0, + "226": 302618112.0, + "227": 302618112.0, + "228": 302618112.0, + "229": 302618112.0, + "230": 302618112.0, + "231": 302618112.0, + "232": 302618112.0, + "233": 302618112.0, + "234": 302618112.0, + "235": 302618112.0, + "236": 302618112.0, + "237": 302618112.0, + "238": 302618112.0, + "239": 302618112.0, + "240": 302618112.0, + "241": 302618112.0, + "242": 302618112.0, + "243": 302618112.0, + "244": 302618112.0, + "245": 302618112.0, + "246": 302618112.0, + "247": 302618112.0, + "248": 302618112.0, + "249": 302618112.0, + "250": 302618112.0, + "251": 302618112.0, + "252": 302618112.0, + "253": 302618112.0, + "254": 302618112.0, + "255": 302618112.0, + "256": 302618112.0, + "257": 302618112.0, + "258": 302618112.0, + "259": 302618112.0, + "260": 302618112.0, + "261": 302618112.0, + "262": 302618112.0, + "263": 302618112.0, + "264": 302618112.0, + "265": 302618112.0, + "266": 302618112.0, + "267": 302618112.0, + "268": 302618112.0, + "269": 302618112.0, + "270": 302618112.0, + "271": 302618112.0, + "272": 302618112.0, + "273": 302618112.0, + "274": 302618112.0, + "275": 302618112.0, + "276": 302618112.0, + "277": 302618112.0, + "278": 302618112.0, + "279": 302618112.0, + "280": 302618112.0, + "281": 302618112.0, + "282": 302618112.0, + "283": 302618112.0, + "284": 302618112.0, + "285": 302618112.0, + "286": 302618112.0, + "287": 302618112.0, + "288": 302618112.0, + "289": 302618112.0, + "290": 302618112.0, + "291": 302618112.0, + "292": 302618112.0, + "293": 302618112.0, + "294": 302618112.0, + "295": 302618112.0, + "296": 302618112.0, + "297": 302618112.0, + "298": 302618112.0, + "299": 302618112.0, + "300": 302618112.0, + "301": 302618112.0, + "302": 302618112.0, + "303": 302618112.0, + "304": 302618112.0, + "305": 302618112.0, + "306": 302618112.0, + "307": 302618112.0, + "308": 302618112.0, + "309": 302618112.0, + "310": 302618112.0, + "311": 302618112.0, + "312": 302618112.0, + "313": 302618112.0, + "314": 302618112.0, + "315": 302618112.0, + "316": 302618112.0, + "317": 302618112.0, + "318": 302618112.0, + "319": 302618112.0, + "320": 302618112.0, + "321": 302618112.0, + "322": 302618112.0, + "323": 302618112.0, + "324": 302618112.0, + "325": 302618112.0, + "326": 302618112.0, + "327": 302618112.0, + "328": 302618112.0, + "329": 302618112.0, + "330": 302618112.0, + "331": 302618112.0, + "332": 302618112.0, + "333": 302618112.0, + "334": 302618112.0, + "335": 302618112.0, + "336": 302618112.0, + "337": 302618112.0, + "338": 302618112.0, + "339": 302618112.0, + "340": 302618112.0, + "341": 302618112.0, + "342": 302618112.0, + "343": 302618112.0, + "344": 302618112.0, + "345": 302618112.0, + "346": 302618112.0, + "347": 302618112.0, + "348": 302618112.0, + "349": 302618112.0, + "350": 302618112.0, + "351": 302618112.0, + "352": 302618112.0, + "353": 302618112.0, + "354": 302618112.0, + "355": 302618112.0, + "356": 302618112.0, + "357": 302618112.0, + "358": 302618112.0, + "359": 302618112.0, + "360": 302618112.0, + "361": 302618112.0, + "362": 302618112.0, + "363": 302618112.0, + "364": 302618112.0, + "365": 302618112.0, + "366": 302618112.0, + "367": 302618112.0, + "368": 302618112.0, + "369": 302618112.0, + "370": 302618112.0, + "371": 302618112.0, + "372": 302618112.0, + "373": 302618112.0, + "374": 302618112.0, + "375": 302618112.0, + "376": 302618112.0, + "377": 302618112.0, + "378": 302618112.0, + "379": 302618112.0, + "380": 302618112.0, + "381": 302618112.0, + "382": 302618112.0, + "383": 302618112.0, + "384": 302618112.0, + "385": 302618112.0, + "386": 302618112.0, + "387": 302618112.0, + "388": 302618112.0, + "389": 302618112.0, + "390": 302618112.0, + "391": 302618112.0, + "392": 302618112.0, + "393": 302618112.0, + "394": 302618112.0, + "395": 302618112.0, + "396": 302618112.0, + "397": 302618112.0, + "398": 302618112.0, + "399": 302618112.0, + "400": 302618112.0, + "401": 302618112.0, + "402": 302618112.0, + "403": 302618112.0, + "404": 302618112.0, + "405": 302618112.0, + "406": 302618112.0, + "407": 302618112.0, + "408": 302618112.0, + "409": 302618112.0, + "410": 302618112.0, + "411": 302618112.0, + "412": 302618112.0, + "413": 302618112.0, + "414": 302618112.0, + "415": 302618112.0, + "416": 302618112.0, + "417": 302618112.0, + "418": 302618112.0, + "419": 302618112.0, + "420": 302618112.0, + "421": 302618112.0, + "422": 302618112.0, + "423": 302618112.0, + "424": 302618112.0, + "425": 302618112.0, + "426": 302618112.0, + "427": 302618112.0, + "428": 302618112.0, + "429": 302618112.0, + "430": 302618112.0, + "431": 302618112.0, + "432": 302618112.0, + "433": 302618112.0, + "434": 302618112.0, + "435": 302618112.0, + "436": 302618112.0, + "437": 302618112.0, + "438": 302618112.0, + "439": 302618112.0, + "440": 302618112.0, + "441": 302618112.0, + "442": 302618112.0, + "443": 302618112.0, + "444": 302618112.0, + "445": 302618112.0, + "446": 302618112.0, + "447": 302618112.0, + "448": 302618112.0, + "449": 302618112.0, + "450": 302618112.0, + "451": 302618112.0, + "452": 302618112.0, + "453": 302618112.0, + "454": 302618112.0, + "455": 302618112.0, + "456": 302618112.0, + "457": 302618112.0, + "458": 302618112.0, + "459": 302618112.0, + "460": 302618112.0, + "461": 302618112.0, + "462": 302618112.0, + "463": 302618112.0, + "464": 302618112.0, + "465": 302618112.0, + "466": 302618112.0, + "467": 302618112.0, + "468": 302618112.0, + "469": 302618112.0, + "470": 302618112.0, + "471": 302618112.0, + "472": 302618112.0, + "473": 302618112.0, + "474": 302618112.0, + "475": 302618112.0, + "476": 302618112.0, + "477": 302618112.0, + "478": 302618112.0, + "479": 302618112.0, + "480": 302618112.0, + "481": 302618112.0, + "482": 302618112.0, + "483": 302618112.0, + "484": 302618112.0, + "485": 302618112.0, + "486": 302618112.0, + "487": 302618112.0, + "488": 302618112.0, + "489": 302618112.0, + "490": 302618112.0, + "491": 302618112.0, + "492": 302618112.0, + "493": 302618112.0, + "494": 302618112.0, + "495": 302618112.0, + "496": 302618112.0, + "497": 302618112.0, + "498": 302618112.0, + "499": 302618112.0, + "500": 302618112.0, + "501": 302618112.0, + "502": 302618112.0, + "503": 302618112.0, + "504": 302618112.0, + "505": 302618112.0, + "506": 302618112.0, + "507": 302618112.0, + "508": 302618112.0, + "509": 302618112.0, + "510": 302618112.0, + "511": 302618112.0, + "512": 302618112.0, + "513": 302618112.0, + "514": 302618112.0, + "515": 302618112.0, + "516": 302618112.0, + "517": 302618112.0, + "518": 302618112.0, + "519": 302618112.0, + "520": 302618112.0, + "521": 302618112.0, + "522": 302618112.0, + "523": 302618112.0, + "524": 302618112.0, + "525": 302618112.0, + "526": 302618112.0, + "527": 302618112.0, + "528": 302618112.0, + "529": 302618112.0, + "530": 302618112.0, + "531": 302618112.0, + "532": 302618112.0, + "533": 302618112.0, + "534": 302618112.0, + "535": 302618112.0, + "536": 302618112.0, + "537": 302618112.0, + "538": 302618112.0, + "539": 302618112.0, + "540": 302618112.0, + "541": 302618112.0, + "542": 302618112.0, + "543": 302618112.0, + "544": 302618112.0, + "545": 302618112.0, + "546": 302618112.0, + "547": 302618112.0, + "548": 302618112.0, + "549": 302618112.0, + "550": 302618112.0, + "551": 302618112.0, + "552": 302618112.0, + "553": 302618112.0, + "554": 302618112.0, + "555": 302618112.0, + "556": 302618112.0, + "557": 302618112.0, + "558": 302618112.0, + "559": 302618112.0, + "560": 302618112.0, + "561": 302618112.0, + "562": 302618112.0, + "563": 302618112.0, + "564": 302618112.0, + "565": 302618112.0, + "566": 302618112.0, + "567": 302618112.0, + "568": 302618112.0, + "569": 302618112.0, + "570": 302618112.0, + "571": 302618112.0, + "572": 302618112.0, + "573": 302618112.0, + "574": 302618112.0, + "575": 302618112.0, + "576": 302618112.0, + "577": 302618112.0, + "578": 302618112.0, + "579": 302618112.0, + "580": 302618112.0, + "581": 302618112.0, + "582": 302618112.0, + "583": 302618112.0, + "584": 302618112.0, + "585": 302618112.0, + "586": 302618112.0, + "587": 302618112.0, + "588": 302618112.0, + "589": 302618112.0, + "590": 302618112.0, + "591": 302618112.0, + "592": 302618112.0, + "593": 302618112.0, + "594": 302618112.0, + "595": 302618112.0, + "596": 302618112.0, + "597": 302618112.0, + "598": 302618112.0, + "599": 302618112.0, + "600": 302618112.0, + "601": 302618112.0, + "602": 302618112.0, + "603": 302618112.0, + "604": 302618112.0, + "605": 302618112.0, + "606": 302618112.0, + "607": 302618112.0, + "608": 302618112.0, + "609": 302618112.0, + "610": 302618112.0, + "611": 302618112.0, + "612": 302618112.0, + "613": 302618112.0, + "614": 302618112.0, + "615": 302618112.0, + "616": 302618112.0, + "617": 302618112.0, + "618": 302618112.0, + "619": 302618112.0, + "620": 302618112.0, + "621": 302618112.0, + "622": 302618112.0, + "623": 302618112.0, + "624": 302618112.0, + "625": 302618112.0, + "626": 302618112.0, + "627": 302618112.0, + "628": 302618112.0, + "629": 302618112.0, + "630": 302618112.0, + "631": 302618112.0, + "632": 302618112.0, + "633": 302618112.0, + "634": 302618112.0, + "635": 302618112.0, + "636": 302618112.0, + "637": 302618112.0, + "638": 302618112.0, + "639": 302618112.0, + "640": 302618112.0, + "641": 302618112.0, + "642": 302618112.0, + "643": 302618112.0, + "644": 302618112.0, + "645": 302618112.0, + "646": 302618112.0, + "647": 302618112.0, + "648": 302618112.0, + "649": 302618112.0, + "650": 302618112.0, + "651": 302618112.0, + "652": 302618112.0, + "653": 302618112.0, + "654": 302618112.0, + "655": 302618112.0, + "656": 302618112.0, + "657": 302618112.0, + "658": 302618112.0, + "659": 302618112.0, + "660": 302618112.0, + "661": 302618112.0, + "662": 302618112.0, + "663": 302618112.0, + "664": 302618112.0, + "665": 302618112.0, + "666": 302618112.0, + "667": 302618112.0, + "668": 302618112.0, + "669": 302618112.0, + "670": 302618112.0, + "671": 302618112.0, + "672": 302618112.0, + "673": 302618112.0, + "674": 302618112.0, + "675": 302618112.0, + "676": 302618112.0, + "677": 302618112.0, + "678": 302618112.0, + "679": 302618112.0, + "680": 302618112.0, + "681": 302618112.0, + "682": 302618112.0, + "683": 302618112.0, + "684": 302618112.0, + "685": 302618112.0, + "686": 302618112.0, + "687": 302618112.0, + "688": 302618112.0, + "689": 302618112.0, + "690": 302618112.0, + "691": 302618112.0, + "692": 302618112.0, + "693": 302618112.0, + "694": 302618112.0, + "695": 302618112.0, + "696": 302618112.0, + "697": 302618112.0, + "698": 302618112.0, + "699": 302618112.0, + "700": 302618112.0, + "701": 302618112.0, + "702": 302618112.0, + "703": 302618112.0, + "704": 302618112.0, + "705": 302618112.0, + "706": 302618112.0, + "707": 302618112.0, + "708": 302618112.0, + "709": 302618112.0, + "710": 302618112.0, + "711": 302618112.0, + "712": 302618112.0, + "713": 302618112.0, + "714": 302618112.0, + "715": 302618112.0, + "716": 302618112.0, + "717": 302618112.0, + "718": 302618112.0, + "719": 302618112.0, + "720": 302618112.0, + "721": 302618112.0, + "722": 302618112.0, + "723": 302618112.0, + "724": 302618112.0, + "725": 302618112.0, + "726": 302618112.0, + "727": 302618112.0, + "728": 302618112.0, + "729": 302618112.0, + "730": 302618112.0, + "731": 302618112.0, + "732": 302618112.0, + "733": 302618112.0, + "734": 302618112.0, + "735": 302618112.0, + "736": 302618112.0, + "737": 302618112.0, + "738": 302618112.0, + "739": 302618112.0, + "740": 302618112.0, + "741": 302618112.0, + "742": 302618112.0, + "743": 302618112.0, + "744": 302618112.0, + "745": 302618112.0, + "746": 302618112.0, + "747": 302618112.0, + "748": 302618112.0, + "749": 302618112.0, + "750": 302618112.0, + "751": 302618112.0, + "752": 302618112.0, + "753": 302618112.0, + "754": 302618112.0, + "755": 302618112.0, + "756": 302618112.0, + "757": 302618112.0, + "758": 302618112.0, + "759": 302618112.0, + "760": 302618112.0, + "761": 302618112.0, + "762": 302618112.0, + "763": 302618112.0, + "764": 302618112.0, + "765": 302618112.0, + "766": 302618112.0, + "767": 302618112.0, + "768": 302618112.0, + "769": 302618112.0, + "770": 302618112.0, + "771": 302618112.0, + "772": 302618112.0, + "773": 302618112.0, + "774": 302618112.0, + "775": 302618112.0, + "776": 302618112.0, + "777": 302618112.0, + "778": 302618112.0, + "779": 302618112.0, + "780": 302618112.0, + "781": 302618112.0, + "782": 302618112.0, + "783": 302618112.0, + "784": 302618112.0, + "785": 302618112.0, + "786": 302618112.0, + "787": 302618112.0, + "788": 302618112.0, + "789": 302618112.0, + "790": 302618112.0, + "791": 302618112.0, + "792": 302618112.0, + "793": 302618112.0, + "794": 302618112.0, + "795": 302618112.0, + "796": 302618112.0, + "797": 302618112.0, + "798": 302618112.0, + "799": 302618112.0, + "800": 302618112.0, + "801": 302618112.0, + "802": 302618112.0, + "803": 302618112.0, + "804": 302618112.0, + "805": 302618112.0, + "806": 302618112.0, + "807": 302618112.0, + "808": 302618112.0, + "809": 302618112.0, + "810": 302618112.0, + "811": 302618112.0, + "812": 302618112.0, + "813": 302618112.0, + "814": 302618112.0, + "815": 302618112.0, + "816": 302618112.0, + "817": 302618112.0, + "818": 302618112.0, + "819": 302618112.0, + "820": 302618112.0, + "821": 302618112.0, + "822": 302618112.0, + "823": 302618112.0, + "824": 302618112.0, + "825": 302618112.0, + "826": 302618112.0, + "827": 302618112.0, + "828": 302618112.0, + "829": 302618112.0, + "830": 302618112.0, + "831": 302618112.0, + "832": 302618112.0, + "833": 302618112.0, + "834": 302618112.0, + "835": 302618112.0, + "836": 302618112.0, + "837": 302618112.0, + "838": 302618112.0, + "839": 302618112.0, + "840": 302618112.0, + "841": 302618112.0, + "842": 302618112.0, + "843": 302618112.0, + "844": 302618112.0, + "845": 302618112.0, + "846": 302618112.0, + "847": 302618112.0, + "848": 302618112.0, + "849": 302618112.0, + "850": 302618112.0, + "851": 302618112.0, + "852": 302618112.0, + "853": 302618112.0, + "854": 302618112.0, + "855": 302618112.0, + "856": 302618112.0, + "857": 302618112.0, + "858": 302618112.0, + "859": 302618112.0, + "860": 302618112.0, + "861": 302618112.0, + "862": 302618112.0, + "863": 302618112.0, + "864": 302618112.0, + "865": 302618112.0, + "866": 302618112.0, + "867": 302618112.0, + "868": 302618112.0, + "869": 302618112.0, + "870": 302618112.0, + "871": 302618112.0, + "872": 302618112.0, + "873": 302618112.0, + "874": 302618112.0, + "875": 302618112.0, + "876": 302618112.0, + "877": 302618112.0, + "878": 302618112.0, + "879": 302618112.0, + "880": 302618112.0, + "881": 302618112.0, + "882": 302618112.0, + "883": 302618112.0, + "884": 302618112.0, + "885": 302618112.0, + "886": 302618112.0, + "887": 302618112.0, + "888": 302618112.0, + "889": 302618112.0, + "890": 302618112.0, + "891": 302618112.0, + "892": 302618112.0, + "893": 302618112.0, + "894": 302618112.0, + "895": 302618112.0, + "896": 302618112.0, + "897": 302618112.0, + "898": 302618112.0, + "899": 302618112.0, + "900": 302618112.0, + "901": 302618112.0, + "902": 302618112.0, + "903": 302618112.0, + "904": 302618112.0, + "905": 302618112.0, + "906": 302618112.0, + "907": 302618112.0, + "908": 302618112.0, + "909": 302618112.0, + "910": 302618112.0, + "911": 302618112.0, + "912": 302618112.0, + "913": 302618112.0, + "914": 302618112.0, + "915": 302618112.0, + "916": 302618112.0, + "917": 302618112.0, + "918": 302618112.0, + "919": 302618112.0, + "920": 302618112.0, + "921": 302618112.0, + "922": 302618112.0, + "923": 302618112.0, + "924": 302618112.0, + "925": 302618112.0, + "926": 302618112.0, + "927": 302618112.0, + "928": 302618112.0, + "929": 302618112.0, + "930": 302618112.0, + "931": 302618112.0, + "932": 302618112.0, + "933": 302618112.0, + "934": 302618112.0, + "935": 302618112.0, + "936": 302618112.0, + "937": 302618112.0, + "938": 302618112.0, + "939": 302618112.0, + "940": 302618112.0, + "941": 302618112.0, + "942": 302618112.0, + "943": 302618112.0, + "944": 302618112.0, + "945": 302618112.0, + "946": 302618112.0, + "947": 302618112.0, + "948": 302618112.0, + "949": 302618112.0, + "950": 302618112.0, + "951": 302618112.0, + "952": 302618112.0, + "953": 302618112.0, + "954": 302618112.0, + "955": 302618112.0, + "956": 302618112.0, + "957": 302618112.0, + "958": 302618112.0, + "959": 302618112.0, + "960": 302618112.0, + "961": 302618112.0, + "962": 302618112.0, + "963": 302618112.0, + "964": 302618112.0, + "965": 302618112.0, + "966": 302618112.0, + "967": 302618112.0, + "968": 302618112.0, + "969": 302618112.0, + "970": 302618112.0, + "971": 302618112.0, + "972": 302618112.0, + "973": 302618112.0, + "974": 302618112.0, + "975": 302618112.0, + "976": 302618112.0, + "977": 302618112.0, + "978": 302618112.0, + "979": 302618112.0, + "980": 302618112.0, + "981": 302618112.0, + "982": 302618112.0, + "983": 302618112.0, + "984": 302618112.0, + "985": 302618112.0, + "986": 302618112.0, + "987": 302618112.0, + "988": 302618112.0, + "989": 302618112.0, + "990": 302618112.0, + "991": 302618112.0, + "992": 302618112.0, + "993": 302618112.0, + "994": 302618112.0, + "995": 302618112.0, + "996": 302618112.0, + "997": 302618112.0, + "998": 302618112.0, + "999": 302618112.0, + "1000": 302618112.0, + "1001": 302618112.0, + "1002": 302618112.0, + "1003": 302618112.0, + "1004": 302618112.0, + "1005": 302618112.0, + "1006": 302618112.0, + "1007": 302618112.0, + "1008": 302618112.0, + "1009": 302618112.0, + "1010": 302618112.0, + "1011": 302618112.0, + "1012": 302618112.0, + "1013": 302618112.0, + "1014": 302618112.0, + "1015": 302618112.0, + "1016": 302618112.0, + "1017": 302618112.0, + "1018": 302618112.0, + "1019": 302618112.0, + "1020": 302618112.0, + "1021": 302618112.0, + "1022": 302618112.0, + "1023": 302618112.0, + "1024": 302618112.0, + "1025": 302618112.0, + "1026": 302618112.0, + "1027": 302618112.0, + "1028": 302618112.0, + "1029": 302618112.0, + "1030": 302618112.0, + "1031": 302618112.0, + "1032": 302618112.0, + "1033": 302618112.0, + "1034": 302618112.0, + "1035": 302618112.0, + "1036": 302618112.0, + "1037": 302618112.0, + "1038": 302618112.0, + "1039": 302618112.0, + "1040": 302618112.0, + "1041": 302618112.0, + "1042": 302618112.0, + "1043": 302618112.0, + "1044": 302618112.0, + "1045": 302618112.0, + "1046": 302618112.0, + "1047": 302618112.0, + "1048": 302618112.0, + "1049": 302618112.0, + "1050": 302618112.0, + "1051": 302618112.0, + "1052": 302618112.0, + "1053": 302618112.0, + "1054": 302618112.0, + "1055": 302618112.0, + "1056": 302618112.0, + "1057": 302618112.0, + "1058": 302618112.0, + "1059": 302618112.0, + "1060": 302618112.0, + "1061": 302618112.0, + "1062": 302618112.0, + "1063": 302618112.0, + "1064": 302618112.0, + "1065": 302618112.0, + "1066": 302618112.0, + "1067": 302618112.0, + "1068": 302618112.0, + "1069": 302618112.0, + "1070": 302618112.0, + "1071": 302618112.0, + "1072": 302618112.0, + "1073": 302618112.0, + "1074": 302618112.0, + "1075": 302618112.0, + "1076": 302618112.0, + "1077": 302618112.0, + "1078": 302618112.0, + "1079": 302618112.0, + "1080": 302618112.0, + "1081": 302618112.0, + "1082": 302618112.0, + "1083": 302618112.0, + "1084": 302618112.0, + "1085": 302618112.0, + "1086": 302618112.0, + "1087": 302618112.0, + "1088": 302618112.0, + "1089": 302618112.0, + "1090": 302618112.0, + "1091": 302618112.0, + "1092": 302618112.0, + "1093": 302618112.0, + "1094": 302618112.0, + "1095": 302618112.0, + "1096": 302618112.0, + "1097": 302618112.0, + "1098": 302618112.0, + "1099": 302618112.0, + "1100": 302618112.0, + "1101": 302618112.0, + "1102": 302618112.0, + "1103": 302618112.0, + "1104": 302618112.0, + "1105": 302618112.0, + "1106": 302618112.0, + "1107": 302618112.0, + "1108": 302618112.0, + "1109": 302618112.0, + "1110": 302618112.0, + "1111": 302618112.0, + "1112": 302618112.0, + "1113": 302618112.0, + "1114": 302618112.0, + "1115": 302618112.0, + "1116": 302618112.0, + "1117": 302618112.0, + "1118": 302618112.0, + "1119": 302618112.0, + "1120": 302618112.0, + "1121": 302618112.0, + "1122": 302618112.0, + "1123": 302618112.0, + "1124": 302618112.0, + "1125": 302618112.0, + "1126": 302618112.0, + "1127": 302618112.0, + "1128": 302618112.0, + "1129": 302618112.0, + "1130": 302618112.0, + "1131": 302618112.0, + "1132": 302618112.0, + "1133": 302618112.0, + "1134": 302618112.0, + "1135": 302618112.0, + "1136": 302618112.0, + "1137": 302618112.0, + "1138": 302618112.0, + "1139": 302618112.0, + "1140": 302618112.0, + "1141": 302618112.0, + "1142": 302618112.0, + "1143": 302618112.0, + "1144": 302618112.0, + "1145": 302618112.0, + "1146": 302618112.0, + "1147": 302618112.0, + "1148": 302618112.0, + "1149": 302618112.0, + "1150": 302618112.0, + "1151": 302618112.0, + "1152": 302618112.0, + "1153": 302618112.0, + "1154": 302618112.0, + "1155": 302618112.0, + "1156": 302618112.0, + "1157": 302618112.0, + "1158": 302618112.0, + "1159": 302618112.0, + "1160": 302618112.0, + "1161": 302618112.0, + "1162": 302618112.0, + "1163": 302618112.0, + "1164": 302618112.0, + "1165": 302618112.0, + "1166": 302618112.0, + "1167": 302618112.0, + "1168": 302618112.0, + "1169": 302618112.0, + "1170": 302618112.0, + "1171": 302618112.0, + "1172": 302618112.0, + "1173": 302618112.0, + "1174": 302618112.0, + "1175": 302618112.0, + "1176": 302618112.0, + "1177": 302618112.0, + "1178": 302618112.0, + "1179": 302618112.0, + "1180": 302618112.0, + "1181": 302618112.0, + "1182": 302618112.0, + "1183": 302618112.0, + "1184": 302618112.0, + "1185": 302618112.0, + "1186": 302618112.0, + "1187": 302618112.0, + "1188": 302618112.0, + "1189": 302618112.0, + "1190": 302618112.0, + "1191": 302618112.0, + "1192": 302618112.0, + "1193": 302618112.0, + "1194": 302618112.0, + "1195": 302618112.0, + "1196": 302618112.0, + "1197": 302618112.0, + "1198": 302618112.0, + "1199": 302618112.0, + "1200": 302618112.0, + "1201": 302618112.0, + "1202": 302618112.0, + "1203": 302618112.0, + "1204": 302618112.0, + "1205": 302618112.0, + "1206": 302618112.0, + "1207": 302618112.0, + "1208": 302618112.0, + "1209": 302618112.0, + "1210": 302618112.0, + "1211": 302618112.0, + "1212": 302618112.0, + "1213": 302618112.0, + "1214": 302618112.0, + "1215": 302618112.0, + "1216": 302618112.0, + "1217": 302618112.0, + "1218": 302618112.0, + "1219": 302618112.0, + "1220": 302618112.0, + "1221": 302618112.0, + "1222": 302618112.0, + "1223": 302618112.0, + "1224": 302618112.0, + "1225": 302618112.0, + "1226": 302618112.0, + "1227": 302618112.0, + "1228": 302618112.0, + "1229": 302618112.0, + "1230": 302618112.0, + "1231": 302618112.0, + "1232": 302618112.0, + "1233": 302618112.0, + "1234": 302618112.0, + "1235": 302618112.0, + "1236": 302618112.0, + "1237": 302618112.0, + "1238": 302618112.0, + "1239": 302618112.0, + "1240": 302618112.0, + "1241": 302618112.0, + "1242": 302618112.0, + "1243": 302618112.0, + "1244": 302618112.0, + "1245": 302618112.0, + "1246": 302618112.0, + "1247": 302618112.0, + "1248": 302618112.0, + "1249": 302618112.0, + "1250": 302618112.0, + "1251": 302618112.0, + "1252": 302618112.0, + "1253": 302618112.0, + "1254": 302618112.0, + "1255": 302618112.0, + "1256": 302618112.0, + "1257": 302618112.0, + "1258": 302618112.0, + "1259": 302618112.0, + "1260": 302618112.0, + "1261": 302618112.0, + "1262": 302618112.0, + "1263": 302618112.0, + "1264": 302618112.0, + "1265": 302618112.0, + "1266": 302618112.0, + "1267": 302618112.0, + "1268": 302618112.0, + "1269": 302618112.0, + "1270": 302618112.0, + "1271": 302618112.0, + "1272": 302618112.0, + "1273": 302618112.0, + "1274": 302618112.0, + "1275": 302618112.0, + "1276": 302618112.0, + "1277": 302618112.0, + "1278": 302618112.0, + "1279": 302618112.0, + "1280": 302618112.0, + "1281": 302618112.0, + "1282": 302618112.0, + "1283": 302618112.0, + "1284": 302618112.0, + "1285": 302618112.0, + "1286": 302618112.0, + "1287": 302618112.0, + "1288": 302618112.0, + "1289": 302618112.0, + "1290": 302618112.0, + "1291": 302618112.0, + "1292": 302618112.0, + "1293": 302618112.0, + "1294": 302618112.0, + "1295": 302618112.0, + "1296": 302618112.0, + "1297": 302618112.0, + "1298": 302618112.0, + "1299": 302618112.0, + "1300": 302618112.0, + "1301": 302618112.0, + "1302": 302618112.0, + "1303": 302618112.0, + "1304": 302618112.0, + "1305": 302618112.0, + "1306": 302618112.0, + "1307": 302618112.0, + "1308": 302618112.0, + "1309": 302618112.0, + "1310": 302618112.0, + "1311": 302618112.0, + "1312": 302618112.0, + "1313": 302618112.0, + "1314": 302618112.0, + "1315": 302618112.0, + "1316": 302618112.0, + "1317": 302618112.0, + "1318": 302618112.0, + "1319": 302618112.0, + "1320": 302618112.0, + "1321": 302618112.0, + "1322": 302618112.0, + "1323": 302618112.0, + "1324": 302618112.0, + "1325": 302618112.0, + "1326": 302618112.0, + "1327": 302618112.0, + "1328": 302618112.0, + "1329": 302618112.0, + "1330": 302618112.0, + "1331": 302618112.0, + "1332": 302618112.0, + "1333": 302618112.0, + "1334": 302618112.0, + "1335": 302618112.0, + "1336": 302618112.0, + "1337": 302618112.0, + "1338": 302618112.0, + "1339": 302618112.0, + "1340": 302618112.0, + "1341": 302618112.0, + "1342": 302618112.0, + "1343": 302618112.0, + "1344": 302618112.0, + "1345": 302618112.0, + "1346": 302618112.0, + "1347": 302618112.0, + "1348": 302618112.0, + "1349": 302618112.0, + "1350": 302618112.0, + "1351": 302618112.0, + "1352": 302618112.0, + "1353": 302618112.0, + "1354": 302618112.0, + "1355": 302618112.0, + "1356": 302618112.0, + "1357": 302618112.0, + "1358": 302618112.0, + "1359": 302618112.0, + "1360": 302618112.0, + "1361": 302618112.0, + "1362": 302618112.0, + "1363": 302618112.0, + "1364": 302618112.0, + "1365": 302618112.0, + "1366": 302618112.0, + "1367": 302618112.0, + "1368": 302618112.0, + "1369": 302618112.0, + "1370": 302618112.0, + "1371": 302618112.0, + "1372": 302618112.0, + "1373": 302618112.0, + "1374": 302618112.0, + "1375": 302618112.0, + "1376": 302618112.0, + "1377": 302618112.0, + "1378": 302618112.0, + "1379": 302618112.0, + "1380": 302618112.0, + "1381": 302618112.0, + "1382": 302618112.0, + "1383": 302618112.0, + "1384": 302618112.0, + "1385": 302618112.0, + "1386": 302618112.0, + "1387": 302618112.0, + "1388": 302618112.0, + "1389": 302618112.0, + "1390": 302618112.0, + "1391": 302618112.0, + "1392": 302618112.0, + "1393": 302618112.0, + "1394": 302618112.0, + "1395": 302618112.0, + "1396": 302618112.0, + "1397": 302618112.0, + "1398": 302618112.0, + "1399": 302618112.0, + "1400": 302618112.0, + "1401": 302618112.0, + "1402": 302618112.0, + "1403": 302618112.0, + "1404": 302618112.0, + "1405": 302618112.0, + "1406": 302618112.0, + "1407": 302618112.0, + "1408": 302618112.0, + "1409": 302618112.0, + "1410": 302618112.0, + "1411": 302618112.0, + "1412": 302618112.0, + "1413": 302618112.0, + "1414": 302618112.0, + "1415": 302618112.0, + "1416": 302618112.0, + "1417": 302618112.0, + "1418": 302618112.0, + "1419": 302618112.0, + "1420": 302618112.0, + "1421": 302618112.0, + "1422": 302618112.0, + "1423": 302618112.0, + "1424": 302618112.0, + "1425": 302618112.0, + "1426": 302618112.0, + "1427": 302618112.0, + "1428": 302618112.0, + "1429": 302618112.0, + "1430": 302618112.0, + "1431": 302618112.0, + "1432": 302618112.0, + "1433": 302618112.0, + "1434": 302618112.0, + "1435": 302618112.0, + "1436": 302618112.0, + "1437": 302618112.0, + "1438": 302618112.0, + "1439": 302618112.0, + "1440": 302618112.0, + "1441": 302618112.0, + "1442": 302618112.0, + "1443": 302618112.0, + "1444": 302618112.0, + "1445": 302618112.0, + "1446": 302618112.0, + "1447": 302618112.0, + "1448": 302618112.0, + "1449": 302618112.0, + "1450": 302618112.0, + "1451": 302618112.0, + "1452": 302618112.0, + "1453": 302618112.0, + "1454": 302618112.0, + "1455": 302618112.0, + "1456": 302618112.0, + "1457": 302618112.0, + "1458": 302618112.0, + "1459": 302618112.0, + "1460": 302618112.0, + "1461": 302618112.0, + "1462": 302618112.0, + "1463": 302618112.0, + "1464": 302618112.0, + "1465": 302618112.0, + "1466": 302618112.0, + "1467": 302618112.0, + "1468": 302618112.0, + "1469": 302618112.0, + "1470": 302618112.0, + "1471": 302618112.0, + "1472": 302618112.0, + "1473": 302618112.0, + "1474": 302618112.0, + "1475": 302618112.0, + "1476": 302618112.0, + "1477": 302618112.0, + "1478": 302618112.0, + "1479": 302618112.0, + "1480": 302618112.0, + "1481": 302618112.0, + "1482": 302618112.0, + "1483": 302618112.0, + "1484": 302618112.0, + "1485": 302618112.0, + "1486": 302618112.0, + "1487": 302618112.0, + "1488": 302618112.0, + "1489": 302618112.0, + "1490": 302618112.0, + "1491": 302618112.0, + "1492": 302618112.0, + "1493": 302618112.0, + "1494": 302618112.0, + "1495": 302618112.0, + "1496": 302618112.0, + "1497": 302618112.0, + "1498": 302618112.0, + "1499": 302618112.0, + "1500": 302618112.0, + "1501": 302618112.0, + "1502": 302618112.0, + "1503": 302618112.0, + "1504": 302618112.0, + "1505": 302618112.0, + "1506": 302618112.0, + "1507": 302618112.0, + "1508": 302618112.0, + "1509": 302618112.0, + "1510": 302618112.0, + "1511": 302618112.0, + "1512": 302618112.0, + "1513": 302618112.0, + "1514": 302618112.0, + "1515": 302618112.0, + "1516": 302618112.0, + "1517": 302618112.0, + "1518": 302618112.0, + "1519": 302618112.0, + "1520": 302618112.0, + "1521": 302618112.0, + "1522": 302618112.0, + "1523": 302618112.0, + "1524": 302618112.0, + "1525": 302618112.0, + "1526": 302618112.0, + "1527": 302618112.0, + "1528": 302618112.0, + "1529": 302618112.0, + "1530": 302618112.0, + "1531": 302618112.0, + "1532": 302618112.0, + "1533": 302618112.0, + "1534": 302618112.0, + "1535": 302618112.0, + "1536": 302618112.0, + "1537": 302618112.0, + "1538": 302618112.0, + "1539": 302618112.0, + "1540": 302618112.0, + "1541": 302618112.0, + "1542": 302618112.0, + "1543": 302618112.0, + "1544": 302618112.0, + "1545": 302618112.0, + "1546": 302618112.0, + "1547": 302618112.0, + "1548": 302618112.0, + "1549": 302618112.0, + "1550": 302618112.0, + "1551": 302618112.0, + "1552": 302618112.0, + "1553": 302618112.0, + "1554": 302618112.0, + "1555": 302618112.0, + "1556": 302618112.0, + "1557": 302618112.0, + "1558": 302618112.0, + "1559": 302618112.0, + "1560": 302618112.0, + "1561": 302618112.0, + "1562": 302618112.0, + "1563": 302618112.0, + "1564": 302618112.0, + "1565": 302618112.0, + "1566": 302618112.0, + "1567": 302618112.0, + "1568": 302618112.0, + "1569": 302618112.0, + "1570": 302618112.0, + "1571": 302618112.0, + "1572": 302618112.0, + "1573": 302618112.0, + "1574": 302618112.0, + "1575": 302618112.0, + "1576": 302618112.0, + "1577": 302618112.0, + "1578": 302618112.0, + "1579": 302618112.0, + "1580": 302618112.0, + "1581": 302618112.0, + "1582": 302618112.0, + "1583": 302618112.0, + "1584": 302618112.0, + "1585": 302618112.0, + "1586": 302618112.0, + "1587": 302618112.0, + "1588": 302618112.0, + "1589": 302618112.0, + "1590": 302618112.0, + "1591": 302618112.0, + "1592": 302618112.0, + "1593": 302618112.0, + "1594": 302618112.0, + "1595": 302618112.0, + "1596": 302618112.0, + "1597": 302618112.0, + "1598": 302618112.0, + "1599": 302618112.0, + "1600": 302618112.0, + "1601": 302618112.0, + "1602": 302618112.0, + "1603": 302618112.0, + "1604": 302618112.0, + "1605": 302618112.0, + "1606": 302618112.0, + "1607": 302618112.0, + "1608": 302618112.0, + "1609": 302618112.0, + "1610": 302618112.0, + "1611": 302618112.0, + "1612": 302618112.0, + "1613": 302618112.0, + "1614": 302618112.0, + "1615": 302618112.0, + "1616": 302618112.0, + "1617": 302618112.0, + "1618": 302618112.0, + "1619": 302618112.0, + "1620": 302618112.0, + "1621": 302618112.0, + "1622": 302618112.0, + "1623": 302618112.0, + "1624": 302618112.0, + "1625": 302618112.0, + "1626": 302618112.0, + "1627": 302618112.0, + "1628": 302618112.0, + "1629": 302618112.0, + "1630": 302618112.0, + "1631": 302618112.0, + "1632": 302618112.0, + "1633": 302618112.0, + "1634": 302618112.0, + "1635": 302618112.0, + "1636": 302618112.0, + "1637": 302618112.0, + "1638": 302618112.0, + "1639": 302618112.0, + "1640": 302618112.0, + "1641": 302618112.0, + "1642": 302618112.0, + "1643": 302618112.0, + "1644": 302618112.0, + "1645": 302618112.0, + "1646": 302618112.0, + "1647": 302618112.0, + "1648": 302618112.0, + "1649": 302618112.0, + "1650": 302618112.0, + "1651": 302618112.0, + "1652": 302618112.0, + "1653": 302618112.0, + "1654": 302618112.0, + "1655": 302618112.0, + "1656": 302618112.0, + "1657": 302618112.0, + "1658": 302618112.0, + "1659": 302618112.0, + "1660": 302618112.0, + "1661": 302618112.0, + "1662": 302618112.0, + "1663": 302618112.0, + "1664": 302618112.0, + "1665": 302618112.0, + "1666": 302618112.0, + "1667": 302618112.0, + "1668": 302618112.0, + "1669": 302618112.0, + "1670": 302618112.0, + "1671": 302618112.0, + "1672": 302618112.0, + "1673": 302618112.0, + "1674": 302618112.0, + "1675": 302618112.0, + "1676": 302618112.0, + "1677": 302618112.0, + "1678": 302618112.0, + "1679": 302618112.0, + "1680": 302618112.0, + "1681": 302618112.0, + "1682": 302618112.0, + "1683": 302618112.0, + "1684": 302618112.0, + "1685": 302618112.0, + "1686": 302618112.0, + "1687": 302618112.0, + "1688": 302618112.0, + "1689": 302618112.0, + "1690": 302618112.0, + "1691": 302618112.0, + "1692": 302618112.0, + "1693": 302618112.0, + "1694": 302618112.0, + "1695": 302618112.0, + "1696": 302618112.0, + "1697": 302618112.0, + "1698": 302618112.0, + "1699": 302618112.0, + "1700": 302618112.0, + "1701": 302618112.0, + "1702": 302618112.0, + "1703": 302618112.0, + "1704": 302618112.0, + "1705": 302618112.0, + "1706": 302618112.0, + "1707": 302618112.0, + "1708": 302618112.0, + "1709": 302618112.0, + "1710": 302618112.0, + "1711": 302618112.0, + "1712": 302618112.0, + "1713": 302618112.0, + "1714": 302618112.0, + "1715": 302618112.0, + "1716": 302618112.0, + "1717": 302618112.0, + "1718": 302618112.0, + "1719": 302618112.0, + "1720": 302618112.0, + "1721": 302618112.0, + "1722": 302618112.0, + "1723": 302618112.0, + "1724": 302618112.0, + "1725": 302618112.0, + "1726": 302618112.0, + "1727": 302618112.0, + "1728": 302618112.0, + "1729": 302618112.0, + "1730": 302618112.0, + "1731": 302618112.0, + "1732": 302618112.0, + "1733": 302618112.0, + "1734": 302618112.0, + "1735": 302618112.0, + "1736": 302618112.0, + "1737": 302618112.0, + "1738": 302618112.0, + "1739": 302618112.0, + "1740": 302618112.0, + "1741": 302618112.0, + "1742": 302618112.0, + "1743": 302618112.0, + "1744": 302618112.0, + "1745": 302618112.0, + "1746": 302618112.0, + "1747": 302618112.0, + "1748": 302618112.0, + "1749": 302618112.0, + "1750": 302618112.0, + "1751": 302618112.0, + "1752": 302618112.0, + "1753": 302618112.0, + "1754": 302618112.0, + "1755": 302618112.0, + "1756": 302618112.0, + "1757": 302618112.0, + "1758": 302618112.0, + "1759": 302618112.0, + "1760": 302618112.0, + "1761": 302618112.0, + "1762": 302618112.0, + "1763": 302618112.0, + "1764": 302618112.0, + "1765": 302618112.0, + "1766": 302618112.0, + "1767": 302618112.0, + "1768": 302618112.0, + "1769": 302618112.0, + "1770": 302618112.0, + "1771": 302618112.0, + "1772": 302618112.0, + "1773": 302618112.0, + "1774": 302618112.0, + "1775": 302618112.0, + "1776": 302618112.0, + "1777": 302618112.0, + "1778": 302618112.0, + "1779": 302618112.0, + "1780": 302618112.0, + "1781": 302618112.0, + "1782": 302618112.0, + "1783": 302618112.0, + "1784": 302618112.0, + "1785": 302618112.0, + "1786": 302618112.0, + "1787": 302618112.0, + "1788": 302618112.0, + "1789": 302618112.0, + "1790": 302618112.0, + "1791": 302618112.0, + "1792": 302618112.0, + "1793": 302618112.0, + "1794": 302618112.0, + "1795": 302618112.0, + "1796": 302618112.0, + "1797": 302618112.0, + "1798": 302618112.0, + "1799": 302618112.0, + "1800": 302618112.0, + "1801": 302618112.0, + "1802": 302618112.0, + "1803": 302618112.0, + "1804": 302618112.0, + "1805": 302618112.0, + "1806": 302618112.0, + "1807": 302618112.0, + "1808": 302618112.0, + "1809": 302618112.0, + "1810": 302618112.0, + "1811": 302618112.0, + "1812": 302618112.0, + "1813": 302618112.0, + "1814": 302618112.0, + "1815": 302618112.0, + "1816": 302618112.0, + "1817": 302618112.0, + "1818": 302618112.0, + "1819": 302618112.0, + "1820": 302618112.0, + "1821": 302618112.0, + "1822": 302618112.0, + "1823": 302618112.0, + "1824": 302618112.0, + "1825": 302618112.0, + "1826": 302618112.0, + "1827": 302618112.0, + "1828": 302618112.0, + "1829": 302618112.0, + "1830": 302618112.0, + "1831": 302618112.0, + "1832": 302618112.0, + "1833": 302618112.0, + "1834": 302618112.0, + "1835": 302618112.0, + "1836": 302618112.0, + "1837": 302618112.0, + "1838": 302618112.0, + "1839": 302618112.0, + "1840": 302618112.0, + "1841": 302618112.0, + "1842": 302618112.0, + "1843": 302618112.0, + "1844": 302618112.0, + "1845": 302618112.0, + "1846": 302618112.0, + "1847": 302618112.0, + "1848": 302618112.0, + "1849": 302618112.0, + "1850": 302618112.0, + "1851": 302618112.0, + "1852": 302618112.0, + "1853": 302618112.0, + "1854": 302618112.0, + "1855": 302618112.0, + "1856": 302618112.0, + "1857": 302618112.0, + "1858": 302618112.0, + "1859": 302618112.0, + "1860": 302618112.0, + "1861": 302618112.0, + "1862": 302618112.0, + "1863": 302618112.0, + "1864": 302618112.0, + "1865": 302618112.0, + "1866": 302618112.0, + "1867": 302618112.0, + "1868": 302618112.0, + "1869": 302618112.0, + "1870": 302618112.0, + "1871": 302618112.0, + "1872": 302618112.0, + "1873": 302618112.0, + "1874": 302618112.0, + "1875": 302618112.0, + "1876": 302618112.0, + "1877": 302618112.0, + "1878": 302618112.0, + "1879": 302618112.0, + "1880": 302618112.0, + "1881": 302618112.0, + "1882": 302618112.0, + "1883": 302618112.0, + "1884": 302618112.0, + "1885": 302618112.0, + "1886": 302618112.0, + "1887": 302618112.0, + "1888": 302618112.0, + "1889": 302618112.0, + "1890": 302618112.0, + "1891": 302618112.0, + "1892": 302618112.0, + "1893": 302618112.0, + "1894": 302618112.0, + "1895": 302618112.0, + "1896": 302618112.0, + "1897": 302618112.0, + "1898": 302618112.0, + "1899": 302618112.0, + "1900": 302618112.0, + "1901": 302618112.0, + "1902": 302618112.0, + "1903": 302618112.0, + "1904": 302618112.0, + "1905": 302618112.0, + "1906": 302618112.0, + "1907": 302618112.0, + "1908": 302618112.0, + "1909": 302618112.0, + "1910": 302618112.0, + "1911": 302618112.0, + "1912": 302618112.0, + "1913": 302618112.0, + "1914": 302618112.0, + "1915": 302618112.0, + "1916": 302618112.0, + "1917": 302618112.0, + "1918": 302618112.0, + "1919": 302618112.0, + "1920": 302618112.0, + "1921": 302618112.0, + "1922": 302618112.0, + "1923": 302618112.0, + "1924": 302618112.0, + "1925": 302618112.0, + "1926": 302618112.0, + "1927": 302618112.0, + "1928": 302618112.0, + "1929": 302618112.0, + "1930": 302618112.0, + "1931": 302618112.0, + "1932": 302618112.0, + "1933": 302618112.0, + "1934": 302618112.0, + "1935": 302618112.0, + "1936": 302618112.0, + "1937": 302618112.0, + "1938": 302618112.0, + "1939": 302618112.0, + "1940": 302618112.0, + "1941": 302618112.0, + "1942": 302618112.0, + "1943": 302618112.0, + "1944": 302618112.0, + "1945": 302618112.0, + "1946": 302618112.0, + "1947": 302618112.0, + "1948": 302618112.0, + "1949": 302618112.0, + "1950": 302618112.0, + "1951": 302618112.0, + "1952": 302618112.0, + "1953": 302618112.0, + "1954": 302618112.0, + "1955": 302618112.0, + "1956": 302618112.0, + "1957": 302618112.0, + "1958": 302618112.0, + "1959": 302618112.0, + "1960": 302618112.0, + "1961": 302618112.0, + "1962": 302618112.0, + "1963": 302618112.0, + "1964": 302618112.0, + "1965": 302618112.0, + "1966": 302618112.0, + "1967": 302618112.0, + "1968": 302618112.0, + "1969": 302618112.0, + "1970": 302618112.0, + "1971": 302618112.0, + "1972": 302618112.0, + "1973": 302618112.0, + "1974": 302618112.0, + "1975": 302618112.0, + "1976": 302618112.0, + "1977": 302618112.0, + "1978": 302618112.0, + "1979": 302618112.0, + "1980": 302618112.0, + "1981": 302618112.0, + "1982": 302618112.0, + "1983": 302618112.0, + "1984": 302618112.0, + "1985": 302618112.0, + "1986": 302618112.0, + "1987": 302618112.0, + "1988": 302618112.0, + "1989": 302618112.0, + "1990": 302618112.0, + "1991": 302618112.0, + "1992": 302618112.0, + "1993": 302618112.0, + "1994": 302618112.0, + "1995": 302618112.0, + "1996": 302618112.0, + "1997": 302618112.0, + "1998": 302618112.0, + "1999": 302618112.0, + "2000": 302618112.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 362060288.0, + "2": 428612096.0, + "3": 428612096.0, + "4": 428612096.0, + "5": 428612096.0, + "6": 428612096.0, + "7": 428612096.0, + "8": 428612096.0, + "9": 428612096.0, + "10": 428612096.0, + "11": 428612096.0, + "12": 428612096.0, + "13": 428612096.0, + "14": 428612096.0, + "15": 428612096.0, + "16": 428612096.0, + "17": 428612096.0, + "18": 428612096.0, + "19": 428612096.0, + "20": 428612096.0, + "21": 428612096.0, + "22": 428612096.0, + "23": 428612096.0, + "24": 428612096.0, + "25": 428612096.0, + "26": 428612096.0, + "27": 428612096.0, + "28": 428612096.0, + "29": 428612096.0, + "30": 428612096.0, + "31": 428612096.0, + "32": 428612096.0, + "33": 428612096.0, + "34": 428612096.0, + "35": 428612096.0, + "36": 428612096.0, + "37": 428612096.0, + "38": 428612096.0, + "39": 428612096.0, + "40": 428612096.0, + "41": 428612096.0, + "42": 428612096.0, + "43": 428612096.0, + "44": 428612096.0, + "45": 428612096.0, + "46": 428612096.0, + "47": 428612096.0, + "48": 428612096.0, + "49": 428612096.0, + "50": 428612096.0, + "51": 428612096.0, + "52": 428612096.0, + "53": 428612096.0, + "54": 428612096.0, + "55": 428612096.0, + "56": 428612096.0, + "57": 428612096.0, + "58": 428612096.0, + "59": 428612096.0, + "60": 428612096.0, + "61": 428612096.0, + "62": 428612096.0, + "63": 428612096.0, + "64": 428612096.0, + "65": 428612096.0, + "66": 428612096.0, + "67": 428612096.0, + "68": 428612096.0, + "69": 428612096.0, + "70": 428612096.0, + "71": 428612096.0, + "72": 428612096.0, + "73": 428612096.0, + "74": 428612096.0, + "75": 428612096.0, + "76": 428612096.0, + "77": 428612096.0, + "78": 428612096.0, + "79": 428612096.0, + "80": 428612096.0, + "81": 428612096.0, + "82": 428612096.0, + "83": 428612096.0, + "84": 428612096.0, + "85": 428612096.0, + "86": 428612096.0, + "87": 428612096.0, + "88": 428612096.0, + "89": 428612096.0, + "90": 428612096.0, + "91": 428612096.0, + "92": 428612096.0, + "93": 428612096.0, + "94": 428612096.0, + "95": 428612096.0, + "96": 428612096.0, + "97": 428612096.0, + "98": 428612096.0, + "99": 428612096.0, + "100": 428612096.0, + "101": 428612096.0, + "102": 428612096.0, + "103": 428612096.0, + "104": 428612096.0, + "105": 428612096.0, + "106": 428612096.0, + "107": 428612096.0, + "108": 428612096.0, + "109": 428612096.0, + "110": 428612096.0, + "111": 428612096.0, + "112": 428612096.0, + "113": 428612096.0, + "114": 428612096.0, + "115": 428612096.0, + "116": 428612096.0, + "117": 428612096.0, + "118": 428612096.0, + "119": 428612096.0, + "120": 428612096.0, + "121": 428612096.0, + "122": 428612096.0, + "123": 428612096.0, + "124": 428612096.0, + "125": 428612096.0, + "126": 428612096.0, + "127": 428612096.0, + "128": 428612096.0, + "129": 428612096.0, + "130": 428612096.0, + "131": 428612096.0, + "132": 428612096.0, + "133": 428612096.0, + "134": 428612096.0, + "135": 428612096.0, + "136": 428612096.0, + "137": 428612096.0, + "138": 428612096.0, + "139": 428612096.0, + "140": 428612096.0, + "141": 428612096.0, + "142": 428612096.0, + "143": 428612096.0, + "144": 428612096.0, + "145": 428612096.0, + "146": 428612096.0, + "147": 428612096.0, + "148": 428612096.0, + "149": 428612096.0, + "150": 428612096.0, + "151": 428612096.0, + "152": 428612096.0, + "153": 428612096.0, + "154": 428612096.0, + "155": 428612096.0, + "156": 428612096.0, + "157": 428612096.0, + "158": 428612096.0, + "159": 428612096.0, + "160": 428612096.0, + "161": 428612096.0, + "162": 428612096.0, + "163": 428612096.0, + "164": 428612096.0, + "165": 428612096.0, + "166": 428612096.0, + "167": 428612096.0, + "168": 428612096.0, + "169": 428612096.0, + "170": 428612096.0, + "171": 428612096.0, + "172": 428612096.0, + "173": 428612096.0, + "174": 428612096.0, + "175": 428612096.0, + "176": 428612096.0, + "177": 428612096.0, + "178": 428612096.0, + "179": 428612096.0, + "180": 428612096.0, + "181": 428612096.0, + "182": 428612096.0, + "183": 428612096.0, + "184": 428612096.0, + "185": 428612096.0, + "186": 428612096.0, + "187": 428612096.0, + "188": 428612096.0, + "189": 428612096.0, + "190": 428612096.0, + "191": 428612096.0, + "192": 428612096.0, + "193": 428612096.0, + "194": 428612096.0, + "195": 428612096.0, + "196": 428612096.0, + "197": 428612096.0, + "198": 428612096.0, + "199": 428612096.0, + "200": 428612096.0, + "201": 428612096.0, + "202": 428612096.0, + "203": 428612096.0, + "204": 428612096.0, + "205": 428612096.0, + "206": 428612096.0, + "207": 428612096.0, + "208": 428612096.0, + "209": 428612096.0, + "210": 428612096.0, + "211": 428612096.0, + "212": 428612096.0, + "213": 428612096.0, + "214": 428612096.0, + "215": 428612096.0, + "216": 428612096.0, + "217": 428612096.0, + "218": 428612096.0, + "219": 428612096.0, + "220": 428612096.0, + "221": 428612096.0, + "222": 428612096.0, + "223": 428612096.0, + "224": 428612096.0, + "225": 428612096.0, + "226": 428612096.0, + "227": 428612096.0, + "228": 428612096.0, + "229": 428612096.0, + "230": 428612096.0, + "231": 428612096.0, + "232": 428612096.0, + "233": 428612096.0, + "234": 428612096.0, + "235": 428612096.0, + "236": 428612096.0, + "237": 428612096.0, + "238": 428612096.0, + "239": 428612096.0, + "240": 428612096.0, + "241": 428612096.0, + "242": 428612096.0, + "243": 428612096.0, + "244": 428612096.0, + "245": 428612096.0, + "246": 428612096.0, + "247": 428612096.0, + "248": 428612096.0, + "249": 428612096.0, + "250": 428612096.0, + "251": 428612096.0, + "252": 428612096.0, + "253": 428612096.0, + "254": 428612096.0, + "255": 428612096.0, + "256": 428612096.0, + "257": 428612096.0, + "258": 428612096.0, + "259": 428612096.0, + "260": 428612096.0, + "261": 428612096.0, + "262": 428612096.0, + "263": 428612096.0, + "264": 428612096.0, + "265": 428612096.0, + "266": 428612096.0, + "267": 428612096.0, + "268": 428612096.0, + "269": 428612096.0, + "270": 428612096.0, + "271": 428612096.0, + "272": 428612096.0, + "273": 428612096.0, + "274": 428612096.0, + "275": 428612096.0, + "276": 428612096.0, + "277": 428612096.0, + "278": 428612096.0, + "279": 428612096.0, + "280": 428612096.0, + "281": 428612096.0, + "282": 428612096.0, + "283": 428612096.0, + "284": 428612096.0, + "285": 428612096.0, + "286": 428612096.0, + "287": 428612096.0, + "288": 428612096.0, + "289": 428612096.0, + "290": 428612096.0, + "291": 428612096.0, + "292": 428612096.0, + "293": 428612096.0, + "294": 428612096.0, + "295": 428612096.0, + "296": 428612096.0, + "297": 428612096.0, + "298": 428612096.0, + "299": 428612096.0, + "300": 428612096.0, + "301": 428612096.0, + "302": 428612096.0, + "303": 428612096.0, + "304": 428612096.0, + "305": 428612096.0, + "306": 428612096.0, + "307": 428612096.0, + "308": 428612096.0, + "309": 428612096.0, + "310": 428612096.0, + "311": 428612096.0, + "312": 428612096.0, + "313": 428612096.0, + "314": 428612096.0, + "315": 428612096.0, + "316": 428612096.0, + "317": 428612096.0, + "318": 428612096.0, + "319": 428612096.0, + "320": 428612096.0, + "321": 428612096.0, + "322": 428612096.0, + "323": 428612096.0, + "324": 428612096.0, + "325": 428612096.0, + "326": 428612096.0, + "327": 428612096.0, + "328": 428612096.0, + "329": 428612096.0, + "330": 428612096.0, + "331": 428612096.0, + "332": 428612096.0, + "333": 428612096.0, + "334": 428612096.0, + "335": 428612096.0, + "336": 428612096.0, + "337": 428612096.0, + "338": 428612096.0, + "339": 428612096.0, + "340": 428612096.0, + "341": 428612096.0, + "342": 428612096.0, + "343": 428612096.0, + "344": 428612096.0, + "345": 428612096.0, + "346": 428612096.0, + "347": 428612096.0, + "348": 428612096.0, + "349": 428612096.0, + "350": 428612096.0, + "351": 428612096.0, + "352": 428612096.0, + "353": 428612096.0, + "354": 428612096.0, + "355": 428612096.0, + "356": 428612096.0, + "357": 428612096.0, + "358": 428612096.0, + "359": 428612096.0, + "360": 428612096.0, + "361": 428612096.0, + "362": 428612096.0, + "363": 428612096.0, + "364": 428612096.0, + "365": 428612096.0, + "366": 428612096.0, + "367": 428612096.0, + "368": 428612096.0, + "369": 428612096.0, + "370": 428612096.0, + "371": 428612096.0, + "372": 428612096.0, + "373": 428612096.0, + "374": 428612096.0, + "375": 428612096.0, + "376": 428612096.0, + "377": 428612096.0, + "378": 428612096.0, + "379": 428612096.0, + "380": 428612096.0, + "381": 428612096.0, + "382": 428612096.0, + "383": 428612096.0, + "384": 428612096.0, + "385": 428612096.0, + "386": 428612096.0, + "387": 428612096.0, + "388": 428612096.0, + "389": 428612096.0, + "390": 428612096.0, + "391": 428612096.0, + "392": 428612096.0, + "393": 428612096.0, + "394": 428612096.0, + "395": 428612096.0, + "396": 428612096.0, + "397": 428612096.0, + "398": 428612096.0, + "399": 428612096.0, + "400": 428612096.0, + "401": 428612096.0, + "402": 428612096.0, + "403": 428612096.0, + "404": 428612096.0, + "405": 428612096.0, + "406": 428612096.0, + "407": 428612096.0, + "408": 428612096.0, + "409": 428612096.0, + "410": 428612096.0, + "411": 428612096.0, + "412": 428612096.0, + "413": 428612096.0, + "414": 428612096.0, + "415": 428612096.0, + "416": 428612096.0, + "417": 428612096.0, + "418": 428612096.0, + "419": 428612096.0, + "420": 428612096.0, + "421": 428612096.0, + "422": 428612096.0, + "423": 428612096.0, + "424": 428612096.0, + "425": 428612096.0, + "426": 428612096.0, + "427": 428612096.0, + "428": 428612096.0, + "429": 428612096.0, + "430": 428612096.0, + "431": 428612096.0, + "432": 428612096.0, + "433": 428612096.0, + "434": 428612096.0, + "435": 428612096.0, + "436": 428612096.0, + "437": 428612096.0, + "438": 428612096.0, + "439": 428612096.0, + "440": 428612096.0, + "441": 428612096.0, + "442": 428612096.0, + "443": 428612096.0, + "444": 428612096.0, + "445": 428612096.0, + "446": 428612096.0, + "447": 428612096.0, + "448": 428612096.0, + "449": 428612096.0, + "450": 428612096.0, + "451": 428612096.0, + "452": 428612096.0, + "453": 428612096.0, + "454": 428612096.0, + "455": 428612096.0, + "456": 428612096.0, + "457": 428612096.0, + "458": 428612096.0, + "459": 428612096.0, + "460": 428612096.0, + "461": 428612096.0, + "462": 428612096.0, + "463": 428612096.0, + "464": 428612096.0, + "465": 428612096.0, + "466": 428612096.0, + "467": 428612096.0, + "468": 428612096.0, + "469": 428612096.0, + "470": 428612096.0, + "471": 428612096.0, + "472": 428612096.0, + "473": 428612096.0, + "474": 428612096.0, + "475": 428612096.0, + "476": 428612096.0, + "477": 428612096.0, + "478": 428612096.0, + "479": 428612096.0, + "480": 428612096.0, + "481": 428612096.0, + "482": 428612096.0, + "483": 428612096.0, + "484": 428612096.0, + "485": 428612096.0, + "486": 428612096.0, + "487": 428612096.0, + "488": 428612096.0, + "489": 428612096.0, + "490": 428612096.0, + "491": 428612096.0, + "492": 428612096.0, + "493": 428612096.0, + "494": 428612096.0, + "495": 428612096.0, + "496": 428612096.0, + "497": 428612096.0, + "498": 428612096.0, + "499": 428612096.0, + "500": 428612096.0, + "501": 428612096.0, + "502": 428612096.0, + "503": 428612096.0, + "504": 428612096.0, + "505": 428612096.0, + "506": 428612096.0, + "507": 428612096.0, + "508": 428612096.0, + "509": 428612096.0, + "510": 428612096.0, + "511": 428612096.0, + "512": 428612096.0, + "513": 428612096.0, + "514": 428612096.0, + "515": 428612096.0, + "516": 428612096.0, + "517": 428612096.0, + "518": 428612096.0, + "519": 428612096.0, + "520": 428612096.0, + "521": 428612096.0, + "522": 428612096.0, + "523": 428612096.0, + "524": 428612096.0, + "525": 428612096.0, + "526": 428612096.0, + "527": 428612096.0, + "528": 428612096.0, + "529": 428612096.0, + "530": 428612096.0, + "531": 428612096.0, + "532": 428612096.0, + "533": 428612096.0, + "534": 428612096.0, + "535": 428612096.0, + "536": 428612096.0, + "537": 428612096.0, + "538": 428612096.0, + "539": 428612096.0, + "540": 428612096.0, + "541": 428612096.0, + "542": 428612096.0, + "543": 428612096.0, + "544": 428612096.0, + "545": 428612096.0, + "546": 428612096.0, + "547": 428612096.0, + "548": 428612096.0, + "549": 428612096.0, + "550": 428612096.0, + "551": 428612096.0, + "552": 428612096.0, + "553": 428612096.0, + "554": 428612096.0, + "555": 428612096.0, + "556": 428612096.0, + "557": 428612096.0, + "558": 428612096.0, + "559": 428612096.0, + "560": 428612096.0, + "561": 428612096.0, + "562": 428612096.0, + "563": 428612096.0, + "564": 428612096.0, + "565": 428612096.0, + "566": 428612096.0, + "567": 428612096.0, + "568": 428612096.0, + "569": 428612096.0, + "570": 428612096.0, + "571": 428612096.0, + "572": 428612096.0, + "573": 428612096.0, + "574": 428612096.0, + "575": 428612096.0, + "576": 428612096.0, + "577": 428612096.0, + "578": 428612096.0, + "579": 428612096.0, + "580": 428612096.0, + "581": 428612096.0, + "582": 428612096.0, + "583": 428612096.0, + "584": 428612096.0, + "585": 428612096.0, + "586": 428612096.0, + "587": 428612096.0, + "588": 428612096.0, + "589": 428612096.0, + "590": 428612096.0, + "591": 428612096.0, + "592": 428612096.0, + "593": 428612096.0, + "594": 428612096.0, + "595": 428612096.0, + "596": 428612096.0, + "597": 428612096.0, + "598": 428612096.0, + "599": 428612096.0, + "600": 428612096.0, + "601": 428612096.0, + "602": 428612096.0, + "603": 428612096.0, + "604": 428612096.0, + "605": 428612096.0, + "606": 428612096.0, + "607": 428612096.0, + "608": 428612096.0, + "609": 428612096.0, + "610": 428612096.0, + "611": 428612096.0, + "612": 428612096.0, + "613": 428612096.0, + "614": 428612096.0, + "615": 428612096.0, + "616": 428612096.0, + "617": 428612096.0, + "618": 428612096.0, + "619": 428612096.0, + "620": 428612096.0, + "621": 428612096.0, + "622": 428612096.0, + "623": 428612096.0, + "624": 428612096.0, + "625": 428612096.0, + "626": 428612096.0, + "627": 428612096.0, + "628": 428612096.0, + "629": 428612096.0, + "630": 428612096.0, + "631": 428612096.0, + "632": 428612096.0, + "633": 428612096.0, + "634": 428612096.0, + "635": 428612096.0, + "636": 428612096.0, + "637": 428612096.0, + "638": 428612096.0, + "639": 428612096.0, + "640": 428612096.0, + "641": 428612096.0, + "642": 428612096.0, + "643": 428612096.0, + "644": 428612096.0, + "645": 428612096.0, + "646": 428612096.0, + "647": 428612096.0, + "648": 428612096.0, + "649": 428612096.0, + "650": 428612096.0, + "651": 428612096.0, + "652": 428612096.0, + "653": 428612096.0, + "654": 428612096.0, + "655": 428612096.0, + "656": 428612096.0, + "657": 428612096.0, + "658": 428612096.0, + "659": 428612096.0, + "660": 428612096.0, + "661": 428612096.0, + "662": 428612096.0, + "663": 428612096.0, + "664": 428612096.0, + "665": 428612096.0, + "666": 428612096.0, + "667": 428612096.0, + "668": 428612096.0, + "669": 428612096.0, + "670": 428612096.0, + "671": 428612096.0, + "672": 428612096.0, + "673": 428612096.0, + "674": 428612096.0, + "675": 428612096.0, + "676": 428612096.0, + "677": 428612096.0, + "678": 428612096.0, + "679": 428612096.0, + "680": 428612096.0, + "681": 428612096.0, + "682": 428612096.0, + "683": 428612096.0, + "684": 428612096.0, + "685": 428612096.0, + "686": 428612096.0, + "687": 428612096.0, + "688": 428612096.0, + "689": 428612096.0, + "690": 428612096.0, + "691": 428612096.0, + "692": 428612096.0, + "693": 428612096.0, + "694": 428612096.0, + "695": 428612096.0, + "696": 428612096.0, + "697": 428612096.0, + "698": 428612096.0, + "699": 428612096.0, + "700": 428612096.0, + "701": 428612096.0, + "702": 428612096.0, + "703": 428612096.0, + "704": 428612096.0, + "705": 428612096.0, + "706": 428612096.0, + "707": 428612096.0, + "708": 428612096.0, + "709": 428612096.0, + "710": 428612096.0, + "711": 428612096.0, + "712": 428612096.0, + "713": 428612096.0, + "714": 428612096.0, + "715": 428612096.0, + "716": 428612096.0, + "717": 428612096.0, + "718": 428612096.0, + "719": 428612096.0, + "720": 428612096.0, + "721": 428612096.0, + "722": 428612096.0, + "723": 428612096.0, + "724": 428612096.0, + "725": 428612096.0, + "726": 428612096.0, + "727": 428612096.0, + "728": 428612096.0, + "729": 428612096.0, + "730": 428612096.0, + "731": 428612096.0, + "732": 428612096.0, + "733": 428612096.0, + "734": 428612096.0, + "735": 428612096.0, + "736": 428612096.0, + "737": 428612096.0, + "738": 428612096.0, + "739": 428612096.0, + "740": 428612096.0, + "741": 428612096.0, + "742": 428612096.0, + "743": 428612096.0, + "744": 428612096.0, + "745": 428612096.0, + "746": 428612096.0, + "747": 428612096.0, + "748": 428612096.0, + "749": 428612096.0, + "750": 428612096.0, + "751": 428612096.0, + "752": 428612096.0, + "753": 428612096.0, + "754": 428612096.0, + "755": 428612096.0, + "756": 428612096.0, + "757": 428612096.0, + "758": 428612096.0, + "759": 428612096.0, + "760": 428612096.0, + "761": 428612096.0, + "762": 428612096.0, + "763": 428612096.0, + "764": 428612096.0, + "765": 428612096.0, + "766": 428612096.0, + "767": 428612096.0, + "768": 428612096.0, + "769": 428612096.0, + "770": 428612096.0, + "771": 428612096.0, + "772": 428612096.0, + "773": 428612096.0, + "774": 428612096.0, + "775": 428612096.0, + "776": 428612096.0, + "777": 428612096.0, + "778": 428612096.0, + "779": 428612096.0, + "780": 428612096.0, + "781": 428612096.0, + "782": 428612096.0, + "783": 428612096.0, + "784": 428612096.0, + "785": 428612096.0, + "786": 428612096.0, + "787": 428612096.0, + "788": 428612096.0, + "789": 428612096.0, + "790": 428612096.0, + "791": 428612096.0, + "792": 428612096.0, + "793": 428612096.0, + "794": 428612096.0, + "795": 428612096.0, + "796": 428612096.0, + "797": 428612096.0, + "798": 428612096.0, + "799": 428612096.0, + "800": 428612096.0, + "801": 428612096.0, + "802": 428612096.0, + "803": 428612096.0, + "804": 428612096.0, + "805": 428612096.0, + "806": 428612096.0, + "807": 428612096.0, + "808": 428612096.0, + "809": 428612096.0, + "810": 428612096.0, + "811": 428612096.0, + "812": 428612096.0, + "813": 428612096.0, + "814": 428612096.0, + "815": 428612096.0, + "816": 428612096.0, + "817": 428612096.0, + "818": 428612096.0, + "819": 428612096.0, + "820": 428612096.0, + "821": 428612096.0, + "822": 428612096.0, + "823": 428612096.0, + "824": 428612096.0, + "825": 428612096.0, + "826": 428612096.0, + "827": 428612096.0, + "828": 428612096.0, + "829": 428612096.0, + "830": 428612096.0, + "831": 428612096.0, + "832": 428612096.0, + "833": 428612096.0, + "834": 428612096.0, + "835": 428612096.0, + "836": 428612096.0, + "837": 428612096.0, + "838": 428612096.0, + "839": 428612096.0, + "840": 428612096.0, + "841": 428612096.0, + "842": 428612096.0, + "843": 428612096.0, + "844": 428612096.0, + "845": 428612096.0, + "846": 428612096.0, + "847": 428612096.0, + "848": 428612096.0, + "849": 428612096.0, + "850": 428612096.0, + "851": 428612096.0, + "852": 428612096.0, + "853": 428612096.0, + "854": 428612096.0, + "855": 428612096.0, + "856": 428612096.0, + "857": 428612096.0, + "858": 428612096.0, + "859": 428612096.0, + "860": 428612096.0, + "861": 428612096.0, + "862": 428612096.0, + "863": 428612096.0, + "864": 428612096.0, + "865": 428612096.0, + "866": 428612096.0, + "867": 428612096.0, + "868": 428612096.0, + "869": 428612096.0, + "870": 428612096.0, + "871": 428612096.0, + "872": 428612096.0, + "873": 428612096.0, + "874": 428612096.0, + "875": 428612096.0, + "876": 428612096.0, + "877": 428612096.0, + "878": 428612096.0, + "879": 428612096.0, + "880": 428612096.0, + "881": 428612096.0, + "882": 428612096.0, + "883": 428612096.0, + "884": 428612096.0, + "885": 428612096.0, + "886": 428612096.0, + "887": 428612096.0, + "888": 428612096.0, + "889": 428612096.0, + "890": 428612096.0, + "891": 428612096.0, + "892": 428612096.0, + "893": 428612096.0, + "894": 428612096.0, + "895": 428612096.0, + "896": 428612096.0, + "897": 428612096.0, + "898": 428612096.0, + "899": 428612096.0, + "900": 428612096.0, + "901": 428612096.0, + "902": 428612096.0, + "903": 428612096.0, + "904": 428612096.0, + "905": 428612096.0, + "906": 428612096.0, + "907": 428612096.0, + "908": 428612096.0, + "909": 428612096.0, + "910": 428612096.0, + "911": 428612096.0, + "912": 428612096.0, + "913": 428612096.0, + "914": 428612096.0, + "915": 428612096.0, + "916": 428612096.0, + "917": 428612096.0, + "918": 428612096.0, + "919": 428612096.0, + "920": 428612096.0, + "921": 428612096.0, + "922": 428612096.0, + "923": 428612096.0, + "924": 428612096.0, + "925": 428612096.0, + "926": 428612096.0, + "927": 428612096.0, + "928": 428612096.0, + "929": 428612096.0, + "930": 428612096.0, + "931": 428612096.0, + "932": 428612096.0, + "933": 428612096.0, + "934": 428612096.0, + "935": 428612096.0, + "936": 428612096.0, + "937": 428612096.0, + "938": 428612096.0, + "939": 428612096.0, + "940": 428612096.0, + "941": 428612096.0, + "942": 428612096.0, + "943": 428612096.0, + "944": 428612096.0, + "945": 428612096.0, + "946": 428612096.0, + "947": 428612096.0, + "948": 428612096.0, + "949": 428612096.0, + "950": 428612096.0, + "951": 428612096.0, + "952": 428612096.0, + "953": 428612096.0, + "954": 428612096.0, + "955": 428612096.0, + "956": 428612096.0, + "957": 428612096.0, + "958": 428612096.0, + "959": 428612096.0, + "960": 428612096.0, + "961": 428612096.0, + "962": 428612096.0, + "963": 428612096.0, + "964": 428612096.0, + "965": 428612096.0, + "966": 428612096.0, + "967": 428612096.0, + "968": 428612096.0, + "969": 428612096.0, + "970": 428612096.0, + "971": 428612096.0, + "972": 428612096.0, + "973": 428612096.0, + "974": 428612096.0, + "975": 428612096.0, + "976": 428612096.0, + "977": 428612096.0, + "978": 428612096.0, + "979": 428612096.0, + "980": 428612096.0, + "981": 428612096.0, + "982": 428612096.0, + "983": 428612096.0, + "984": 428612096.0, + "985": 428612096.0, + "986": 428612096.0, + "987": 428612096.0, + "988": 428612096.0, + "989": 428612096.0, + "990": 428612096.0, + "991": 428612096.0, + "992": 428612096.0, + "993": 428612096.0, + "994": 428612096.0, + "995": 428612096.0, + "996": 428612096.0, + "997": 428612096.0, + "998": 428612096.0, + "999": 428612096.0, + "1000": 428612096.0, + "1001": 428612096.0, + "1002": 428612096.0, + "1003": 428612096.0, + "1004": 428612096.0, + "1005": 428612096.0, + "1006": 428612096.0, + "1007": 428612096.0, + "1008": 428612096.0, + "1009": 428612096.0, + "1010": 428612096.0, + "1011": 428612096.0, + "1012": 428612096.0, + "1013": 428612096.0, + "1014": 428612096.0, + "1015": 428612096.0, + "1016": 428612096.0, + "1017": 428612096.0, + "1018": 428612096.0, + "1019": 428612096.0, + "1020": 428612096.0, + "1021": 428612096.0, + "1022": 428612096.0, + "1023": 428612096.0, + "1024": 428612096.0, + "1025": 428612096.0, + "1026": 428612096.0, + "1027": 428612096.0, + "1028": 428612096.0, + "1029": 428612096.0, + "1030": 428612096.0, + "1031": 428612096.0, + "1032": 428612096.0, + "1033": 428612096.0, + "1034": 428612096.0, + "1035": 428612096.0, + "1036": 428612096.0, + "1037": 428612096.0, + "1038": 428612096.0, + "1039": 428612096.0, + "1040": 428612096.0, + "1041": 428612096.0, + "1042": 428612096.0, + "1043": 428612096.0, + "1044": 428612096.0, + "1045": 428612096.0, + "1046": 428612096.0, + "1047": 428612096.0, + "1048": 428612096.0, + "1049": 428612096.0, + "1050": 428612096.0, + "1051": 428612096.0, + "1052": 428612096.0, + "1053": 428612096.0, + "1054": 428612096.0, + "1055": 428612096.0, + "1056": 428612096.0, + "1057": 428612096.0, + "1058": 428612096.0, + "1059": 428612096.0, + "1060": 428612096.0, + "1061": 428612096.0, + "1062": 428612096.0, + "1063": 428612096.0, + "1064": 428612096.0, + "1065": 428612096.0, + "1066": 428612096.0, + "1067": 428612096.0, + "1068": 428612096.0, + "1069": 428612096.0, + "1070": 428612096.0, + "1071": 428612096.0, + "1072": 428612096.0, + "1073": 428612096.0, + "1074": 428612096.0, + "1075": 428612096.0, + "1076": 428612096.0, + "1077": 428612096.0, + "1078": 428612096.0, + "1079": 428612096.0, + "1080": 428612096.0, + "1081": 428612096.0, + "1082": 428612096.0, + "1083": 428612096.0, + "1084": 428612096.0, + "1085": 428612096.0, + "1086": 428612096.0, + "1087": 428612096.0, + "1088": 428612096.0, + "1089": 428612096.0, + "1090": 428612096.0, + "1091": 428612096.0, + "1092": 428612096.0, + "1093": 428612096.0, + "1094": 428612096.0, + "1095": 428612096.0, + "1096": 428612096.0, + "1097": 428612096.0, + "1098": 428612096.0, + "1099": 428612096.0, + "1100": 428612096.0, + "1101": 428612096.0, + "1102": 428612096.0, + "1103": 428612096.0, + "1104": 428612096.0, + "1105": 428612096.0, + "1106": 428612096.0, + "1107": 428612096.0, + "1108": 428612096.0, + "1109": 428612096.0, + "1110": 428612096.0, + "1111": 428612096.0, + "1112": 428612096.0, + "1113": 428612096.0, + "1114": 428612096.0, + "1115": 428612096.0, + "1116": 428612096.0, + "1117": 428612096.0, + "1118": 428612096.0, + "1119": 428612096.0, + "1120": 428612096.0, + "1121": 428612096.0, + "1122": 428612096.0, + "1123": 428612096.0, + "1124": 428612096.0, + "1125": 428612096.0, + "1126": 428612096.0, + "1127": 428612096.0, + "1128": 428612096.0, + "1129": 428612096.0, + "1130": 428612096.0, + "1131": 428612096.0, + "1132": 428612096.0, + "1133": 428612096.0, + "1134": 428612096.0, + "1135": 428612096.0, + "1136": 428612096.0, + "1137": 428612096.0, + "1138": 428612096.0, + "1139": 428612096.0, + "1140": 428612096.0, + "1141": 428612096.0, + "1142": 428612096.0, + "1143": 428612096.0, + "1144": 428612096.0, + "1145": 428612096.0, + "1146": 428612096.0, + "1147": 428612096.0, + "1148": 428612096.0, + "1149": 428612096.0, + "1150": 428612096.0, + "1151": 428612096.0, + "1152": 428612096.0, + "1153": 428612096.0, + "1154": 428612096.0, + "1155": 428612096.0, + "1156": 428612096.0, + "1157": 428612096.0, + "1158": 428612096.0, + "1159": 428612096.0, + "1160": 428612096.0, + "1161": 428612096.0, + "1162": 428612096.0, + "1163": 428612096.0, + "1164": 428612096.0, + "1165": 428612096.0, + "1166": 428612096.0, + "1167": 428612096.0, + "1168": 428612096.0, + "1169": 428612096.0, + "1170": 428612096.0, + "1171": 428612096.0, + "1172": 428612096.0, + "1173": 428612096.0, + "1174": 428612096.0, + "1175": 428612096.0, + "1176": 428612096.0, + "1177": 428612096.0, + "1178": 428612096.0, + "1179": 428612096.0, + "1180": 428612096.0, + "1181": 428612096.0, + "1182": 428612096.0, + "1183": 428612096.0, + "1184": 428612096.0, + "1185": 428612096.0, + "1186": 428612096.0, + "1187": 428612096.0, + "1188": 428612096.0, + "1189": 428612096.0, + "1190": 428612096.0, + "1191": 428612096.0, + "1192": 428612096.0, + "1193": 428612096.0, + "1194": 428612096.0, + "1195": 428612096.0, + "1196": 428612096.0, + "1197": 428612096.0, + "1198": 428612096.0, + "1199": 428612096.0, + "1200": 428612096.0, + "1201": 428612096.0, + "1202": 428612096.0, + "1203": 428612096.0, + "1204": 428612096.0, + "1205": 428612096.0, + "1206": 428612096.0, + "1207": 428612096.0, + "1208": 428612096.0, + "1209": 428612096.0, + "1210": 428612096.0, + "1211": 428612096.0, + "1212": 428612096.0, + "1213": 428612096.0, + "1214": 428612096.0, + "1215": 428612096.0, + "1216": 428612096.0, + "1217": 428612096.0, + "1218": 428612096.0, + "1219": 428612096.0, + "1220": 428612096.0, + "1221": 428612096.0, + "1222": 428612096.0, + "1223": 428612096.0, + "1224": 428612096.0, + "1225": 428612096.0, + "1226": 428612096.0, + "1227": 428612096.0, + "1228": 428612096.0, + "1229": 428612096.0, + "1230": 428612096.0, + "1231": 428612096.0, + "1232": 428612096.0, + "1233": 428612096.0, + "1234": 428612096.0, + "1235": 428612096.0, + "1236": 428612096.0, + "1237": 428612096.0, + "1238": 428612096.0, + "1239": 428612096.0, + "1240": 428612096.0, + "1241": 428612096.0, + "1242": 428612096.0, + "1243": 428612096.0, + "1244": 428612096.0, + "1245": 428612096.0, + "1246": 428612096.0, + "1247": 428612096.0, + "1248": 428612096.0, + "1249": 428612096.0, + "1250": 428612096.0, + "1251": 428612096.0, + "1252": 428612096.0, + "1253": 428612096.0, + "1254": 428612096.0, + "1255": 428612096.0, + "1256": 428612096.0, + "1257": 428612096.0, + "1258": 428612096.0, + "1259": 428612096.0, + "1260": 428612096.0, + "1261": 428612096.0, + "1262": 428612096.0, + "1263": 428612096.0, + "1264": 428612096.0, + "1265": 428612096.0, + "1266": 428612096.0, + "1267": 428612096.0, + "1268": 428612096.0, + "1269": 428612096.0, + "1270": 428612096.0, + "1271": 428612096.0, + "1272": 428612096.0, + "1273": 428612096.0, + "1274": 428612096.0, + "1275": 428612096.0, + "1276": 428612096.0, + "1277": 428612096.0, + "1278": 428612096.0, + "1279": 428612096.0, + "1280": 428612096.0, + "1281": 428612096.0, + "1282": 428612096.0, + "1283": 428612096.0, + "1284": 428612096.0, + "1285": 428612096.0, + "1286": 428612096.0, + "1287": 428612096.0, + "1288": 428612096.0, + "1289": 428612096.0, + "1290": 428612096.0, + "1291": 428612096.0, + "1292": 428612096.0, + "1293": 428612096.0, + "1294": 428612096.0, + "1295": 428612096.0, + "1296": 428612096.0, + "1297": 428612096.0, + "1298": 428612096.0, + "1299": 428612096.0, + "1300": 428612096.0, + "1301": 428612096.0, + "1302": 428612096.0, + "1303": 428612096.0, + "1304": 428612096.0, + "1305": 428612096.0, + "1306": 428612096.0, + "1307": 428612096.0, + "1308": 428612096.0, + "1309": 428612096.0, + "1310": 428612096.0, + "1311": 428612096.0, + "1312": 428612096.0, + "1313": 428612096.0, + "1314": 428612096.0, + "1315": 428612096.0, + "1316": 428612096.0, + "1317": 428612096.0, + "1318": 428612096.0, + "1319": 428612096.0, + "1320": 428612096.0, + "1321": 428612096.0, + "1322": 428612096.0, + "1323": 428612096.0, + "1324": 428612096.0, + "1325": 428612096.0, + "1326": 428612096.0, + "1327": 428612096.0, + "1328": 428612096.0, + "1329": 428612096.0, + "1330": 428612096.0, + "1331": 428612096.0, + "1332": 428612096.0, + "1333": 428612096.0, + "1334": 428612096.0, + "1335": 428612096.0, + "1336": 428612096.0, + "1337": 428612096.0, + "1338": 428612096.0, + "1339": 428612096.0, + "1340": 428612096.0, + "1341": 428612096.0, + "1342": 428612096.0, + "1343": 428612096.0, + "1344": 428612096.0, + "1345": 428612096.0, + "1346": 428612096.0, + "1347": 428612096.0, + "1348": 428612096.0, + "1349": 428612096.0, + "1350": 428612096.0, + "1351": 428612096.0, + "1352": 428612096.0, + "1353": 428612096.0, + "1354": 428612096.0, + "1355": 428612096.0, + "1356": 428612096.0, + "1357": 428612096.0, + "1358": 428612096.0, + "1359": 428612096.0, + "1360": 428612096.0, + "1361": 428612096.0, + "1362": 428612096.0, + "1363": 428612096.0, + "1364": 428612096.0, + "1365": 428612096.0, + "1366": 428612096.0, + "1367": 428612096.0, + "1368": 428612096.0, + "1369": 428612096.0, + "1370": 428612096.0, + "1371": 428612096.0, + "1372": 428612096.0, + "1373": 428612096.0, + "1374": 428612096.0, + "1375": 428612096.0, + "1376": 428612096.0, + "1377": 428612096.0, + "1378": 428612096.0, + "1379": 428612096.0, + "1380": 428612096.0, + "1381": 428612096.0, + "1382": 428612096.0, + "1383": 428612096.0, + "1384": 428612096.0, + "1385": 428612096.0, + "1386": 428612096.0, + "1387": 428612096.0, + "1388": 428612096.0, + "1389": 428612096.0, + "1390": 428612096.0, + "1391": 428612096.0, + "1392": 428612096.0, + "1393": 428612096.0, + "1394": 428612096.0, + "1395": 428612096.0, + "1396": 428612096.0, + "1397": 428612096.0, + "1398": 428612096.0, + "1399": 428612096.0, + "1400": 428612096.0, + "1401": 428612096.0, + "1402": 428612096.0, + "1403": 428612096.0, + "1404": 428612096.0, + "1405": 428612096.0, + "1406": 428612096.0, + "1407": 428612096.0, + "1408": 428612096.0, + "1409": 428612096.0, + "1410": 428612096.0, + "1411": 428612096.0, + "1412": 428612096.0, + "1413": 428612096.0, + "1414": 428612096.0, + "1415": 428612096.0, + "1416": 428612096.0, + "1417": 428612096.0, + "1418": 428612096.0, + "1419": 428612096.0, + "1420": 428612096.0, + "1421": 428612096.0, + "1422": 428612096.0, + "1423": 428612096.0, + "1424": 428612096.0, + "1425": 428612096.0, + "1426": 428612096.0, + "1427": 428612096.0, + "1428": 428612096.0, + "1429": 428612096.0, + "1430": 428612096.0, + "1431": 428612096.0, + "1432": 428612096.0, + "1433": 428612096.0, + "1434": 428612096.0, + "1435": 428612096.0, + "1436": 428612096.0, + "1437": 428612096.0, + "1438": 428612096.0, + "1439": 428612096.0, + "1440": 428612096.0, + "1441": 428612096.0, + "1442": 428612096.0, + "1443": 428612096.0, + "1444": 428612096.0, + "1445": 428612096.0, + "1446": 428612096.0, + "1447": 428612096.0, + "1448": 428612096.0, + "1449": 428612096.0, + "1450": 428612096.0, + "1451": 428612096.0, + "1452": 428612096.0, + "1453": 428612096.0, + "1454": 428612096.0, + "1455": 428612096.0, + "1456": 428612096.0, + "1457": 428612096.0, + "1458": 428612096.0, + "1459": 428612096.0, + "1460": 428612096.0, + "1461": 428612096.0, + "1462": 428612096.0, + "1463": 428612096.0, + "1464": 428612096.0, + "1465": 428612096.0, + "1466": 428612096.0, + "1467": 428612096.0, + "1468": 428612096.0, + "1469": 428612096.0, + "1470": 428612096.0, + "1471": 428612096.0, + "1472": 428612096.0, + "1473": 428612096.0, + "1474": 428612096.0, + "1475": 428612096.0, + "1476": 428612096.0, + "1477": 428612096.0, + "1478": 428612096.0, + "1479": 428612096.0, + "1480": 428612096.0, + "1481": 428612096.0, + "1482": 428612096.0, + "1483": 428612096.0, + "1484": 428612096.0, + "1485": 428612096.0, + "1486": 428612096.0, + "1487": 428612096.0, + "1488": 428612096.0, + "1489": 428612096.0, + "1490": 428612096.0, + "1491": 428612096.0, + "1492": 428612096.0, + "1493": 428612096.0, + "1494": 428612096.0, + "1495": 428612096.0, + "1496": 428612096.0, + "1497": 428612096.0, + "1498": 428612096.0, + "1499": 428612096.0, + "1500": 428612096.0, + "1501": 428612096.0, + "1502": 428612096.0, + "1503": 428612096.0, + "1504": 428612096.0, + "1505": 428612096.0, + "1506": 428612096.0, + "1507": 428612096.0, + "1508": 428612096.0, + "1509": 428612096.0, + "1510": 428612096.0, + "1511": 428612096.0, + "1512": 428612096.0, + "1513": 428612096.0, + "1514": 428612096.0, + "1515": 428612096.0, + "1516": 428612096.0, + "1517": 428612096.0, + "1518": 428612096.0, + "1519": 428612096.0, + "1520": 428612096.0, + "1521": 428612096.0, + "1522": 428612096.0, + "1523": 428612096.0, + "1524": 428612096.0, + "1525": 428612096.0, + "1526": 428612096.0, + "1527": 428612096.0, + "1528": 428612096.0, + "1529": 428612096.0, + "1530": 428612096.0, + "1531": 428612096.0, + "1532": 428612096.0, + "1533": 428612096.0, + "1534": 428612096.0, + "1535": 428612096.0, + "1536": 428612096.0, + "1537": 428612096.0, + "1538": 428612096.0, + "1539": 428612096.0, + "1540": 428612096.0, + "1541": 428612096.0, + "1542": 428612096.0, + "1543": 428612096.0, + "1544": 428612096.0, + "1545": 428612096.0, + "1546": 428612096.0, + "1547": 428612096.0, + "1548": 428612096.0, + "1549": 428612096.0, + "1550": 428612096.0, + "1551": 428612096.0, + "1552": 428612096.0, + "1553": 428612096.0, + "1554": 428612096.0, + "1555": 428612096.0, + "1556": 428612096.0, + "1557": 428612096.0, + "1558": 428612096.0, + "1559": 428612096.0, + "1560": 428612096.0, + "1561": 428612096.0, + "1562": 428612096.0, + "1563": 428612096.0, + "1564": 428612096.0, + "1565": 428612096.0, + "1566": 428612096.0, + "1567": 428612096.0, + "1568": 428612096.0, + "1569": 428612096.0, + "1570": 428612096.0, + "1571": 428612096.0, + "1572": 428612096.0, + "1573": 428612096.0, + "1574": 428612096.0, + "1575": 428612096.0, + "1576": 428612096.0, + "1577": 428612096.0, + "1578": 428612096.0, + "1579": 428612096.0, + "1580": 428612096.0, + "1581": 428612096.0, + "1582": 428612096.0, + "1583": 428612096.0, + "1584": 428612096.0, + "1585": 428612096.0, + "1586": 428612096.0, + "1587": 428612096.0, + "1588": 428612096.0, + "1589": 428612096.0, + "1590": 428612096.0, + "1591": 428612096.0, + "1592": 428612096.0, + "1593": 428612096.0, + "1594": 428612096.0, + "1595": 428612096.0, + "1596": 428612096.0, + "1597": 428612096.0, + "1598": 428612096.0, + "1599": 428612096.0, + "1600": 428612096.0, + "1601": 428612096.0, + "1602": 428612096.0, + "1603": 428612096.0, + "1604": 428612096.0, + "1605": 428612096.0, + "1606": 428612096.0, + "1607": 428612096.0, + "1608": 428612096.0, + "1609": 428612096.0, + "1610": 428612096.0, + "1611": 428612096.0, + "1612": 428612096.0, + "1613": 428612096.0, + "1614": 428612096.0, + "1615": 428612096.0, + "1616": 428612096.0, + "1617": 428612096.0, + "1618": 428612096.0, + "1619": 428612096.0, + "1620": 428612096.0, + "1621": 428612096.0, + "1622": 428612096.0, + "1623": 428612096.0, + "1624": 428612096.0, + "1625": 428612096.0, + "1626": 428612096.0, + "1627": 428612096.0, + "1628": 428612096.0, + "1629": 428612096.0, + "1630": 428612096.0, + "1631": 428612096.0, + "1632": 428612096.0, + "1633": 428612096.0, + "1634": 428612096.0, + "1635": 428612096.0, + "1636": 428612096.0, + "1637": 428612096.0, + "1638": 428612096.0, + "1639": 428612096.0, + "1640": 428612096.0, + "1641": 428612096.0, + "1642": 428612096.0, + "1643": 428612096.0, + "1644": 428612096.0, + "1645": 428612096.0, + "1646": 428612096.0, + "1647": 428612096.0, + "1648": 428612096.0, + "1649": 428612096.0, + "1650": 428612096.0, + "1651": 428612096.0, + "1652": 428612096.0, + "1653": 428612096.0, + "1654": 428612096.0, + "1655": 428612096.0, + "1656": 428612096.0, + "1657": 428612096.0, + "1658": 428612096.0, + "1659": 428612096.0, + "1660": 428612096.0, + "1661": 428612096.0, + "1662": 428612096.0, + "1663": 428612096.0, + "1664": 428612096.0, + "1665": 428612096.0, + "1666": 428612096.0, + "1667": 428612096.0, + "1668": 428612096.0, + "1669": 428612096.0, + "1670": 428612096.0, + "1671": 428612096.0, + "1672": 428612096.0, + "1673": 428612096.0, + "1674": 428612096.0, + "1675": 428612096.0, + "1676": 428612096.0, + "1677": 428612096.0, + "1678": 428612096.0, + "1679": 428612096.0, + "1680": 428612096.0, + "1681": 428612096.0, + "1682": 428612096.0, + "1683": 428612096.0, + "1684": 428612096.0, + "1685": 428612096.0, + "1686": 428612096.0, + "1687": 428612096.0, + "1688": 428612096.0, + "1689": 428612096.0, + "1690": 428612096.0, + "1691": 428612096.0, + "1692": 428612096.0, + "1693": 428612096.0, + "1694": 428612096.0, + "1695": 428612096.0, + "1696": 428612096.0, + "1697": 428612096.0, + "1698": 428612096.0, + "1699": 428612096.0, + "1700": 428612096.0, + "1701": 428612096.0, + "1702": 428612096.0, + "1703": 428612096.0, + "1704": 428612096.0, + "1705": 428612096.0, + "1706": 428612096.0, + "1707": 428612096.0, + "1708": 428612096.0, + "1709": 428612096.0, + "1710": 428612096.0, + "1711": 428612096.0, + "1712": 428612096.0, + "1713": 428612096.0, + "1714": 428612096.0, + "1715": 428612096.0, + "1716": 428612096.0, + "1717": 428612096.0, + "1718": 428612096.0, + "1719": 428612096.0, + "1720": 428612096.0, + "1721": 428612096.0, + "1722": 428612096.0, + "1723": 428612096.0, + "1724": 428612096.0, + "1725": 428612096.0, + "1726": 428612096.0, + "1727": 428612096.0, + "1728": 428612096.0, + "1729": 428612096.0, + "1730": 428612096.0, + "1731": 428612096.0, + "1732": 428612096.0, + "1733": 428612096.0, + "1734": 428612096.0, + "1735": 428612096.0, + "1736": 428612096.0, + "1737": 428612096.0, + "1738": 428612096.0, + "1739": 428612096.0, + "1740": 428612096.0, + "1741": 428612096.0, + "1742": 428612096.0, + "1743": 428612096.0, + "1744": 428612096.0, + "1745": 428612096.0, + "1746": 428612096.0, + "1747": 428612096.0, + "1748": 428612096.0, + "1749": 428612096.0, + "1750": 428612096.0, + "1751": 428612096.0, + "1752": 428612096.0, + "1753": 428612096.0, + "1754": 428612096.0, + "1755": 428612096.0, + "1756": 428612096.0, + "1757": 428612096.0, + "1758": 428612096.0, + "1759": 428612096.0, + "1760": 428612096.0, + "1761": 428612096.0, + "1762": 428612096.0, + "1763": 428612096.0, + "1764": 428612096.0, + "1765": 428612096.0, + "1766": 428612096.0, + "1767": 428612096.0, + "1768": 428612096.0, + "1769": 428612096.0, + "1770": 428612096.0, + "1771": 428612096.0, + "1772": 428612096.0, + "1773": 428612096.0, + "1774": 428612096.0, + "1775": 428612096.0, + "1776": 428612096.0, + "1777": 428612096.0, + "1778": 428612096.0, + "1779": 428612096.0, + "1780": 428612096.0, + "1781": 428612096.0, + "1782": 428612096.0, + "1783": 428612096.0, + "1784": 428612096.0, + "1785": 428612096.0, + "1786": 428612096.0, + "1787": 428612096.0, + "1788": 428612096.0, + "1789": 428612096.0, + "1790": 428612096.0, + "1791": 428612096.0, + "1792": 428612096.0, + "1793": 428612096.0, + "1794": 428612096.0, + "1795": 428612096.0, + "1796": 428612096.0, + "1797": 428612096.0, + "1798": 428612096.0, + "1799": 428612096.0, + "1800": 428612096.0, + "1801": 428612096.0, + "1802": 428612096.0, + "1803": 428612096.0, + "1804": 428612096.0, + "1805": 428612096.0, + "1806": 428612096.0, + "1807": 428612096.0, + "1808": 428612096.0, + "1809": 428612096.0, + "1810": 428612096.0, + "1811": 428612096.0, + "1812": 428612096.0, + "1813": 428612096.0, + "1814": 428612096.0, + "1815": 428612096.0, + "1816": 428612096.0, + "1817": 428612096.0, + "1818": 428612096.0, + "1819": 428612096.0, + "1820": 428612096.0, + "1821": 428612096.0, + "1822": 428612096.0, + "1823": 428612096.0, + "1824": 428612096.0, + "1825": 428612096.0, + "1826": 428612096.0, + "1827": 428612096.0, + "1828": 428612096.0, + "1829": 428612096.0, + "1830": 428612096.0, + "1831": 428612096.0, + "1832": 428612096.0, + "1833": 428612096.0, + "1834": 428612096.0, + "1835": 428612096.0, + "1836": 428612096.0, + "1837": 428612096.0, + "1838": 428612096.0, + "1839": 428612096.0, + "1840": 428612096.0, + "1841": 428612096.0, + "1842": 428612096.0, + "1843": 428612096.0, + "1844": 428612096.0, + "1845": 428612096.0, + "1846": 428612096.0, + "1847": 428612096.0, + "1848": 428612096.0, + "1849": 428612096.0, + "1850": 428612096.0, + "1851": 428612096.0, + "1852": 428612096.0, + "1853": 428612096.0, + "1854": 428612096.0, + "1855": 428612096.0, + "1856": 428612096.0, + "1857": 428612096.0, + "1858": 428612096.0, + "1859": 428612096.0, + "1860": 428612096.0, + "1861": 428612096.0, + "1862": 428612096.0, + "1863": 428612096.0, + "1864": 428612096.0, + "1865": 428612096.0, + "1866": 428612096.0, + "1867": 428612096.0, + "1868": 428612096.0, + "1869": 428612096.0, + "1870": 428612096.0, + "1871": 428612096.0, + "1872": 428612096.0, + "1873": 428612096.0, + "1874": 428612096.0, + "1875": 428612096.0, + "1876": 428612096.0, + "1877": 428612096.0, + "1878": 428612096.0, + "1879": 428612096.0, + "1880": 428612096.0, + "1881": 428612096.0, + "1882": 428612096.0, + "1883": 428612096.0, + "1884": 428612096.0, + "1885": 428612096.0, + "1886": 428612096.0, + "1887": 428612096.0, + "1888": 428612096.0, + "1889": 428612096.0, + "1890": 428612096.0, + "1891": 428612096.0, + "1892": 428612096.0, + "1893": 428612096.0, + "1894": 428612096.0, + "1895": 428612096.0, + "1896": 428612096.0, + "1897": 428612096.0, + "1898": 428612096.0, + "1899": 428612096.0, + "1900": 428612096.0, + "1901": 428612096.0, + "1902": 428612096.0, + "1903": 428612096.0, + "1904": 428612096.0, + "1905": 428612096.0, + "1906": 428612096.0, + "1907": 428612096.0, + "1908": 428612096.0, + "1909": 428612096.0, + "1910": 428612096.0, + "1911": 428612096.0, + "1912": 428612096.0, + "1913": 428612096.0, + "1914": 428612096.0, + "1915": 428612096.0, + "1916": 428612096.0, + "1917": 428612096.0, + "1918": 428612096.0, + "1919": 428612096.0, + "1920": 428612096.0, + "1921": 428612096.0, + "1922": 428612096.0, + "1923": 428612096.0, + "1924": 428612096.0, + "1925": 428612096.0, + "1926": 428612096.0, + "1927": 428612096.0, + "1928": 428612096.0, + "1929": 428612096.0, + "1930": 428612096.0, + "1931": 428612096.0, + "1932": 428612096.0, + "1933": 428612096.0, + "1934": 428612096.0, + "1935": 428612096.0, + "1936": 428612096.0, + "1937": 428612096.0, + "1938": 428612096.0, + "1939": 428612096.0, + "1940": 428612096.0, + "1941": 428612096.0, + "1942": 428612096.0, + "1943": 428612096.0, + "1944": 428612096.0, + "1945": 428612096.0, + "1946": 428612096.0, + "1947": 428612096.0, + "1948": 428612096.0, + "1949": 428612096.0, + "1950": 428612096.0, + "1951": 428612096.0, + "1952": 428612096.0, + "1953": 428612096.0, + "1954": 428612096.0, + "1955": 428612096.0, + "1956": 428612096.0, + "1957": 428612096.0, + "1958": 428612096.0, + "1959": 428612096.0, + "1960": 428612096.0, + "1961": 428612096.0, + "1962": 428612096.0, + "1963": 428612096.0, + "1964": 428612096.0, + "1965": 428612096.0, + "1966": 428612096.0, + "1967": 428612096.0, + "1968": 428612096.0, + "1969": 428612096.0, + "1970": 428612096.0, + "1971": 428612096.0, + "1972": 428612096.0, + "1973": 428612096.0, + "1974": 428612096.0, + "1975": 428612096.0, + "1976": 428612096.0, + "1977": 428612096.0, + "1978": 428612096.0, + "1979": 428612096.0, + "1980": 428612096.0, + "1981": 428612096.0, + "1982": 428612096.0, + "1983": 428612096.0, + "1984": 428612096.0, + "1985": 428612096.0, + "1986": 428612096.0, + "1987": 428612096.0, + "1988": 428612096.0, + "1989": 428612096.0, + "1990": 428612096.0, + "1991": 428612096.0, + "1992": 428612096.0, + "1993": 428612096.0, + "1994": 428612096.0, + "1995": 428612096.0, + "1996": 428612096.0, + "1997": 428612096.0, + "1998": 428612096.0, + "1999": 428612096.0, + "2000": 428612096.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 2000, + "step_interval": 1, + "values": { + "1": 22.43653, + "2": 5.05, + "3": 4.99632, + "4": 5.00941, + "5": 5.30047, + "6": 5.00529, + "7": 4.98693, + "8": 5.03236, + "9": 5.04733, + "10": 5.0355, + "11": 5.05504, + "12": 5.02789, + "13": 5.05026, + "14": 5.03817, + "15": 5.03065, + "16": 5.04414, + "17": 5.00251, + "18": 4.9928, + "19": 4.99792, + "20": 4.99648, + "21": 5.01668, + "22": 4.97973, + "23": 5.06379, + "24": 5.01631, + "25": 4.96187, + "26": 4.95004, + "27": 4.95649, + "28": 4.93702, + "29": 4.93675, + "30": 4.92101, + "31": 4.93325, + "32": 4.92626, + "33": 4.93256, + "34": 4.93518, + "35": 4.95011, + "36": 4.959, + "37": 5.41549, + "38": 5.7108, + "39": 4.96475, + "40": 4.95756, + "41": 5.03533, + "42": 4.94591, + "43": 5.30856, + "44": 4.93166, + "45": 5.29533, + "46": 6.02838, + "47": 4.99271, + "48": 4.93548, + "49": 4.93262, + "50": 4.93589, + "51": 4.93457, + "52": 4.9402, + "53": 4.93593, + "54": 4.93266, + "55": 4.93457, + "56": 4.926, + "57": 4.94015, + "58": 4.93606, + "59": 4.92819, + "60": 4.92679, + "61": 4.92853, + "62": 4.93744, + "63": 4.93014, + "64": 4.92895, + "65": 4.92774, + "66": 4.9263, + "67": 4.92483, + "68": 4.91654, + "69": 4.95386, + "70": 4.95969, + "71": 4.97371, + "72": 4.96736, + "73": 4.98575, + "74": 4.968, + "75": 5.68071, + "76": 4.98487, + "77": 4.98651, + "78": 4.97441, + "79": 4.97854, + "80": 4.97886, + "81": 4.98163, + "82": 4.97647, + "83": 5.33849, + "84": 4.98394, + "85": 4.98, + "86": 4.96888, + "87": 4.9685, + "88": 5.33167, + "89": 5.40565, + "90": 4.97724, + "91": 6.05451, + "92": 4.9699, + "93": 4.96947, + "94": 4.97853, + "95": 5.03234, + "96": 4.9703, + "97": 4.9766, + "98": 4.96386, + "99": 4.97968, + "100": 4.96583, + "101": 4.956, + "102": 4.94425, + "103": 4.96789, + "104": 4.96252, + "105": 4.97853, + "106": 4.98313, + "107": 4.98, + "108": 4.97528, + "109": 4.98226, + "110": 4.98532, + "111": 4.95791, + "112": 4.95409, + "113": 5.66529, + "114": 4.96347, + "115": 4.99625, + "116": 4.99199, + "117": 4.98823, + "118": 4.98114, + "119": 4.97652, + "120": 4.98449, + "121": 4.98578, + "122": 4.98423, + "123": 4.9824, + "124": 4.98111, + "125": 4.98291, + "126": 4.98215, + "127": 4.98484, + "128": 5.35151, + "129": 4.9912, + "130": 4.99188, + "131": 4.98662, + "132": 5.34041, + "133": 4.98063, + "134": 5.33235, + "135": 5.69907, + "136": 5.33587, + "137": 4.98509, + "138": 5.61624, + "139": 5.23864, + "140": 4.92839, + "141": 4.95868, + "142": 4.93611, + "143": 4.9473, + "144": 4.9282, + "145": 4.93563, + "146": 4.92822, + "147": 4.94205, + "148": 4.94037, + "149": 4.93429, + "150": 5.62642, + "151": 4.93794, + "152": 4.9323, + "153": 4.93391, + "154": 4.93581, + "155": 4.93177, + "156": 4.93719, + "157": 4.93775, + "158": 4.93223, + "159": 4.9449, + "160": 4.93898, + "161": 4.94198, + "162": 4.9436, + "163": 4.9355, + "164": 4.93432, + "165": 4.9382, + "166": 4.94332, + "167": 4.93425, + "168": 4.93189, + "169": 4.92717, + "170": 4.94393, + "171": 4.94517, + "172": 4.92976, + "173": 5.303, + "174": 4.92818, + "175": 4.92924, + "176": 4.9385, + "177": 5.27801, + "178": 4.93182, + "179": 5.28092, + "180": 5.99722, + "181": 4.92656, + "182": 4.92594, + "183": 4.92947, + "184": 4.93087, + "185": 4.92967, + "186": 4.93088, + "187": 5.62908, + "188": 4.93498, + "189": 4.9476, + "190": 4.93843, + "191": 4.94101, + "192": 4.93265, + "193": 4.93046, + "194": 4.93133, + "195": 4.94044, + "196": 4.93997, + "197": 4.93336, + "198": 6.32096, + "199": 4.95042, + "200": 4.91888, + "201": 4.91803, + "202": 4.92212, + "203": 4.91738, + "204": 4.93431, + "205": 4.93078, + "206": 4.9288, + "207": 4.9431, + "208": 4.93288, + "209": 4.93152, + "210": 4.92297, + "211": 4.92152, + "212": 4.92078, + "213": 4.93382, + "214": 4.92203, + "215": 4.92628, + "216": 4.92759, + "217": 4.91972, + "218": 4.93018, + "219": 5.30587, + "220": 4.92639, + "221": 4.92815, + "222": 5.28345, + "223": 4.93513, + "224": 5.62954, + "225": 6.35198, + "226": 4.94108, + "227": 4.94033, + "228": 4.94077, + "229": 4.9445, + "230": 4.95277, + "231": 4.93684, + "232": 4.94258, + "233": 4.9386, + "234": 4.94149, + "235": 4.94872, + "236": 4.95361, + "237": 4.94924, + "238": 4.93722, + "239": 4.94342, + "240": 4.95029, + "241": 4.94512, + "242": 4.9423, + "243": 4.93861, + "244": 4.93578, + "245": 4.93502, + "246": 4.94519, + "247": 4.93658, + "248": 4.93761, + "249": 4.94583, + "250": 4.94414, + "251": 4.94331, + "252": 4.94044, + "253": 4.94317, + "254": 4.94161, + "255": 4.95295, + "256": 4.95044, + "257": 4.94816, + "258": 4.94006, + "259": 4.94409, + "260": 4.9408, + "261": 4.94791, + "262": 5.63079, + "263": 4.95361, + "264": 5.3219, + "265": 4.96046, + "266": 4.95564, + "267": 5.30372, + "268": 5.30618, + "269": 4.94954, + "270": 6.01622, + "271": 4.9509, + "272": 4.9579, + "273": 4.9529, + "274": 4.95339, + "275": 4.94721, + "276": 4.95053, + "277": 4.9434, + "278": 4.9389, + "279": 4.94021, + "280": 4.93862, + "281": 4.93834, + "282": 4.93985, + "283": 4.94183, + "284": 4.93716, + "285": 4.9443, + "286": 4.94305, + "287": 4.93467, + "288": 4.93816, + "289": 4.93749, + "290": 4.9349, + "291": 4.939, + "292": 4.93482, + "293": 4.94665, + "294": 4.93648, + "295": 4.93823, + "296": 4.93522, + "297": 4.93472, + "298": 4.93288, + "299": 5.61551, + "300": 4.95418, + "301": 4.95347, + "302": 4.95005, + "303": 4.95224, + "304": 5.01672, + "305": 4.94451, + "306": 4.9469, + "307": 4.94674, + "308": 4.95506, + "309": 5.3147, + "310": 4.97913, + "311": 5.29357, + "312": 4.94239, + "313": 5.28356, + "314": 5.66502, + "315": 5.29945, + "316": 4.94213, + "317": 4.93439, + "318": 4.94085, + "319": 4.93452, + "320": 4.94083, + "321": 4.93407, + "322": 4.93596, + "323": 4.9411, + "324": 4.94091, + "325": 4.93723, + "326": 4.93682, + "327": 4.93712, + "328": 4.99643, + "329": 4.94011, + "330": 4.93777, + "331": 4.93553, + "332": 4.938, + "333": 4.94101, + "334": 4.93199, + "335": 4.93179, + "336": 5.28612, + "337": 5.30266, + "338": 4.96477, + "339": 4.97585, + "340": 4.95959, + "341": 4.95912, + "342": 4.96594, + "343": 4.96105, + "344": 4.96501, + "345": 4.96175, + "346": 4.96452, + "347": 4.9603, + "348": 4.95434, + "349": 4.95658, + "350": 4.95773, + "351": 4.96723, + "352": 5.02353, + "353": 4.95487, + "354": 5.32227, + "355": 4.95601, + "356": 5.29598, + "357": 4.95819, + "358": 5.29935, + "359": 6.01593, + "360": 4.96832, + "361": 4.95302, + "362": 4.95944, + "363": 4.95167, + "364": 4.9483, + "365": 4.94951, + "366": 4.9525, + "367": 4.95364, + "368": 4.94948, + "369": 4.95258, + "370": 4.94974, + "371": 4.96357, + "372": 4.94701, + "373": 4.94584, + "374": 5.27688, + "375": 5.29329, + "376": 4.93553, + "377": 4.93296, + "378": 4.93431, + "379": 4.94158, + "380": 4.98441, + "381": 4.99657, + "382": 4.97634, + "383": 4.98015, + "384": 4.98178, + "385": 4.97595, + "386": 4.97431, + "387": 4.97965, + "388": 4.91884, + "389": 4.92436, + "390": 4.9179, + "391": 4.91999, + "392": 4.92113, + "393": 4.92231, + "394": 4.91815, + "395": 4.92381, + "396": 4.91848, + "397": 4.92412, + "398": 4.91541, + "399": 4.91455, + "400": 5.29982, + "401": 5.26416, + "402": 5.2612, + "403": 4.91795, + "404": 5.63316, + "405": 5.27153, + "406": 4.90744, + "407": 4.9142, + "408": 4.90831, + "409": 4.90838, + "410": 4.92063, + "411": 5.25377, + "412": 5.26322, + "413": 4.91895, + "414": 4.92378, + "415": 4.91866, + "416": 4.91955, + "417": 4.92152, + "418": 4.91929, + "419": 4.9201, + "420": 4.91526, + "421": 4.91974, + "422": 4.92503, + "423": 4.92579, + "424": 4.91791, + "425": 4.92253, + "426": 4.92114, + "427": 4.91774, + "428": 4.91171, + "429": 4.9125, + "430": 4.91411, + "431": 4.90802, + "432": 4.9164, + "433": 4.90723, + "434": 4.92382, + "435": 4.9069, + "436": 4.91154, + "437": 4.90512, + "438": 4.9175, + "439": 4.91782, + "440": 4.91028, + "441": 4.91048, + "442": 4.90894, + "443": 4.88817, + "444": 4.88126, + "445": 5.24853, + "446": 4.87836, + "447": 5.24263, + "448": 5.25398, + "449": 6.28763, + "450": 4.88338, + "451": 4.89491, + "452": 4.88709, + "453": 4.89008, + "454": 4.90322, + "455": 4.90113, + "456": 4.90439, + "457": 4.90223, + "458": 4.90641, + "459": 4.90851, + "460": 4.9009, + "461": 4.89968, + "462": 4.89662, + "463": 4.9081, + "464": 4.88866, + "465": 4.90253, + "466": 4.90724, + "467": 4.89875, + "468": 4.90067, + "469": 4.90495, + "470": 4.89887, + "471": 4.89965, + "472": 4.90145, + "473": 4.88549, + "474": 4.87833, + "475": 4.88274, + "476": 4.87937, + "477": 4.88019, + "478": 4.87808, + "479": 4.88269, + "480": 4.87591, + "481": 4.88072, + "482": 4.87452, + "483": 4.8839, + "484": 4.87834, + "485": 5.21963, + "486": 4.8887, + "487": 5.22473, + "488": 4.88748, + "489": 4.89663, + "490": 5.6108, + "491": 5.24875, + "492": 4.88583, + "493": 5.24488, + "494": 5.59516, + "495": 4.89058, + "496": 4.91601, + "497": 4.88752, + "498": 4.88645, + "499": 4.89008, + "500": 4.89271, + "501": 4.8913, + "502": 4.89039, + "503": 4.8906, + "504": 4.88603, + "505": 4.92691, + "506": 4.91793, + "507": 4.92158, + "508": 4.91981, + "509": 4.92795, + "510": 4.91413, + "511": 4.91073, + "512": 4.90909, + "513": 4.91434, + "514": 4.91509, + "515": 4.91002, + "516": 4.9115, + "517": 4.91722, + "518": 4.91514, + "519": 4.91283, + "520": 4.91403, + "521": 4.91077, + "522": 4.91167, + "523": 5.26088, + "524": 5.27803, + "525": 4.92516, + "526": 4.93143, + "527": 4.9217, + "528": 4.92344, + "529": 4.91786, + "530": 4.9193, + "531": 4.881, + "532": 4.87697, + "533": 4.88329, + "534": 5.23628, + "535": 5.26149, + "536": 4.88132, + "537": 5.23366, + "538": 5.92272, + "539": 4.8822, + "540": 4.87645, + "541": 4.87941, + "542": 4.8726, + "543": 4.87977, + "544": 4.88572, + "545": 4.97915, + "546": 4.94014, + "547": 4.9447, + "548": 4.94585, + "549": 4.93712, + "550": 4.95428, + "551": 4.9405, + "552": 4.94013, + "553": 4.94514, + "554": 4.94542, + "555": 4.94729, + "556": 4.93818, + "557": 4.94632, + "558": 4.95928, + "559": 4.94439, + "560": 5.29538, + "561": 5.29912, + "562": 4.95591, + "563": 4.94545, + "564": 4.9589, + "565": 4.9486, + "566": 4.94487, + "567": 4.94563, + "568": 4.96795, + "569": 4.96332, + "570": 4.95731, + "571": 4.95751, + "572": 4.94401, + "573": 4.94623, + "574": 4.9438, + "575": 4.9342, + "576": 4.93847, + "577": 4.94215, + "578": 4.94036, + "579": 4.95135, + "580": 5.28996, + "581": 5.66625, + "582": 4.93892, + "583": 5.64719, + "584": 5.28091, + "585": 4.95827, + "586": 4.95725, + "587": 4.96107, + "588": 4.95092, + "589": 4.95514, + "590": 4.94845, + "591": 4.94342, + "592": 4.9488, + "593": 4.93576, + "594": 4.93657, + "595": 4.93545, + "596": 4.93595, + "597": 5.29319, + "598": 5.28921, + "599": 4.95347, + "600": 4.94896, + "601": 4.94543, + "602": 4.95405, + "603": 4.94996, + "604": 4.94726, + "605": 4.94394, + "606": 4.9443, + "607": 4.99448, + "608": 4.93032, + "609": 4.96191, + "610": 4.95086, + "611": 4.94486, + "612": 4.94403, + "613": 4.94194, + "614": 4.94624, + "615": 4.94461, + "616": 4.96458, + "617": 4.94658, + "618": 4.94254, + "619": 4.93901, + "620": 4.94138, + "621": 4.94747, + "622": 4.95796, + "623": 4.94579, + "624": 5.30372, + "625": 4.94082, + "626": 5.66834, + "627": 4.93994, + "628": 5.97473, + "629": 4.94152, + "630": 4.94328, + "631": 4.9385, + "632": 4.9688, + "633": 4.93837, + "634": 5.25732, + "635": 4.9147, + "636": 5.25839, + "637": 4.92259, + "638": 4.91081, + "639": 4.92229, + "640": 4.92687, + "641": 4.91335, + "642": 4.91557, + "643": 4.91922, + "644": 4.91847, + "645": 4.92121, + "646": 4.92251, + "647": 4.91255, + "648": 4.91291, + "649": 4.91003, + "650": 4.90867, + "651": 4.91235, + "652": 4.90719, + "653": 4.90865, + "654": 4.90719, + "655": 4.91306, + "656": 4.90861, + "657": 4.90901, + "658": 4.91095, + "659": 4.90726, + "660": 4.90915, + "661": 4.91011, + "662": 4.90721, + "663": 4.90907, + "664": 4.91699, + "665": 4.91095, + "666": 4.90826, + "667": 4.90687, + "668": 4.90738, + "669": 5.25716, + "670": 5.25453, + "671": 5.28603, + "672": 5.25386, + "673": 6.29304, + "674": 4.91719, + "675": 4.9174, + "676": 4.92014, + "677": 4.92048, + "678": 4.90878, + "679": 4.90967, + "680": 4.90981, + "681": 4.91054, + "682": 4.90885, + "683": 4.90932, + "684": 4.915, + "685": 4.90701, + "686": 4.91124, + "687": 4.91733, + "688": 4.91577, + "689": 4.91189, + "690": 4.90854, + "691": 4.90631, + "692": 4.90689, + "693": 4.9142, + "694": 4.90933, + "695": 4.90064, + "696": 4.88962, + "697": 4.89317, + "698": 4.89665, + "699": 4.90473, + "700": 4.90675, + "701": 4.90072, + "702": 4.90347, + "703": 4.90535, + "704": 4.90243, + "705": 4.90653, + "706": 4.90494, + "707": 4.90715, + "708": 4.89971, + "709": 5.25068, + "710": 5.24447, + "711": 4.91173, + "712": 4.91607, + "713": 5.26011, + "714": 4.90966, + "715": 4.90512, + "716": 5.63181, + "717": 5.62011, + "718": 5.23301, + "719": 4.91317, + "720": 4.90779, + "721": 4.90675, + "722": 4.90612, + "723": 4.90554, + "724": 4.90952, + "725": 4.90669, + "726": 4.90589, + "727": 4.9062, + "728": 4.91028, + "729": 4.905, + "730": 4.90848, + "731": 4.90621, + "732": 4.91216, + "733": 4.90248, + "734": 4.90051, + "735": 4.90319, + "736": 4.90401, + "737": 4.90646, + "738": 4.90558, + "739": 4.90438, + "740": 4.90694, + "741": 4.9036, + "742": 4.90521, + "743": 4.90326, + "744": 4.90534, + "745": 4.90658, + "746": 5.24876, + "747": 4.91293, + "748": 5.24944, + "749": 4.90712, + "750": 4.90572, + "751": 4.90977, + "752": 4.90683, + "753": 4.90815, + "754": 4.90611, + "755": 4.91427, + "756": 4.9129, + "757": 4.91264, + "758": 5.25755, + "759": 4.91199, + "760": 5.2647, + "761": 4.91559, + "762": 5.64712, + "763": 5.59149, + "764": 4.91566, + "765": 4.91348, + "766": 4.92052, + "767": 4.9149, + "768": 4.91624, + "769": 4.90919, + "770": 4.9208, + "771": 4.9111, + "772": 4.91242, + "773": 4.91183, + "774": 4.91856, + "775": 4.91524, + "776": 4.91642, + "777": 4.91271, + "778": 4.91587, + "779": 4.91173, + "780": 4.9163, + "781": 4.9101, + "782": 4.90927, + "783": 4.91594, + "784": 5.27562, + "785": 5.29399, + "786": 4.92064, + "787": 4.92508, + "788": 4.91936, + "789": 4.92025, + "790": 4.92839, + "791": 4.91829, + "792": 4.9234, + "793": 4.92615, + "794": 4.91968, + "795": 4.91417, + "796": 4.89214, + "797": 4.87642, + "798": 4.87726, + "799": 4.88691, + "800": 4.87753, + "801": 4.90361, + "802": 4.91538, + "803": 5.25822, + "804": 5.25769, + "805": 4.90985, + "806": 4.91228, + "807": 5.6423, + "808": 5.23836, + "809": 4.9314, + "810": 4.91226, + "811": 4.91382, + "812": 4.91588, + "813": 4.91005, + "814": 4.9202, + "815": 4.90766, + "816": 4.90744, + "817": 4.91497, + "818": 4.91, + "819": 4.90572, + "820": 4.91342, + "821": 5.26215, + "822": 5.25971, + "823": 4.92486, + "824": 4.92645, + "825": 4.91518, + "826": 4.91893, + "827": 4.90862, + "828": 4.9143, + "829": 4.91422, + "830": 4.91829, + "831": 4.90569, + "832": 4.91122, + "833": 4.90584, + "834": 4.90518, + "835": 4.90755, + "836": 4.90656, + "837": 4.90626, + "838": 4.90987, + "839": 4.91189, + "840": 4.90735, + "841": 4.90697, + "842": 4.91064, + "843": 4.90409, + "844": 4.90711, + "845": 4.90385, + "846": 4.90599, + "847": 5.24636, + "848": 4.89752, + "849": 5.24655, + "850": 4.90148, + "851": 4.89501, + "852": 5.98483, + "853": 4.89468, + "854": 4.89653, + "855": 4.8954, + "856": 4.89811, + "857": 4.90026, + "858": 5.24069, + "859": 4.91345, + "860": 5.2538, + "861": 4.91107, + "862": 4.90905, + "863": 4.90289, + "864": 4.90179, + "865": 4.90697, + "866": 4.89969, + "867": 4.89622, + "868": 4.89817, + "869": 4.89734, + "870": 4.89421, + "871": 4.902, + "872": 4.89737, + "873": 4.90082, + "874": 4.8986, + "875": 4.9034, + "876": 4.90213, + "877": 4.89969, + "878": 4.90652, + "879": 4.90216, + "880": 4.90541, + "881": 4.90491, + "882": 4.89798, + "883": 4.89325, + "884": 4.89662, + "885": 4.91, + "886": 4.89481, + "887": 4.90025, + "888": 4.89887, + "889": 4.89458, + "890": 4.89351, + "891": 4.89343, + "892": 5.24625, + "893": 4.90075, + "894": 5.24719, + "895": 4.89439, + "896": 5.95508, + "897": 5.92842, + "898": 4.90126, + "899": 4.91443, + "900": 4.90222, + "901": 4.89928, + "902": 4.89952, + "903": 4.89905, + "904": 4.90536, + "905": 4.90627, + "906": 4.90188, + "907": 4.90671, + "908": 4.90531, + "909": 4.90614, + "910": 4.90319, + "911": 4.90668, + "912": 4.90614, + "913": 4.90641, + "914": 4.90219, + "915": 4.89858, + "916": 4.89788, + "917": 4.90114, + "918": 4.89062, + "919": 4.89675, + "920": 4.89412, + "921": 4.89851, + "922": 4.90258, + "923": 4.89837, + "924": 4.89168, + "925": 4.90558, + "926": 4.88926, + "927": 4.89631, + "928": 4.89481, + "929": 4.89896, + "930": 4.90349, + "931": 4.90254, + "932": 4.89424, + "933": 5.2393, + "934": 4.90447, + "935": 5.24957, + "936": 4.89799, + "937": 5.24757, + "938": 4.90497, + "939": 5.26023, + "940": 4.905, + "941": 4.90603, + "942": 5.89013, + "943": 5.2754, + "944": 4.89903, + "945": 4.90825, + "946": 4.90072, + "947": 4.91095, + "948": 4.89642, + "949": 4.90314, + "950": 4.9027, + "951": 4.90276, + "952": 4.90005, + "953": 4.90591, + "954": 4.89179, + "955": 4.89648, + "956": 4.89739, + "957": 4.90258, + "958": 4.90027, + "959": 4.90627, + "960": 4.89592, + "961": 4.89153, + "962": 4.89826, + "963": 4.89281, + "964": 4.88656, + "965": 4.9056, + "966": 4.88948, + "967": 4.89075, + "968": 4.89128, + "969": 4.88907, + "970": 5.23384, + "971": 4.91197, + "972": 5.24458, + "973": 4.90766, + "974": 4.90557, + "975": 4.9059, + "976": 4.90502, + "977": 4.90392, + "978": 4.90541, + "979": 4.89927, + "980": 4.9047, + "981": 4.90276, + "982": 5.2516, + "983": 5.25121, + "984": 4.90232, + "985": 4.90209, + "986": 5.26939, + "987": 5.52932, + "988": 5.28293, + "989": 4.91742, + "990": 4.90637, + "991": 4.90953, + "992": 4.90864, + "993": 4.9075, + "994": 4.90696, + "995": 4.90473, + "996": 4.90192, + "997": 4.90199, + "998": 4.89181, + "999": 4.89111, + "1000": 4.89025, + "1001": 4.9168, + "1002": 4.90983, + "1003": 4.91875, + "1004": 4.90892, + "1005": 4.92588, + "1006": 4.91678, + "1007": 5.262, + "1008": 4.92447, + "1009": 5.26729, + "1010": 4.92803, + "1011": 4.92461, + "1012": 4.92338, + "1013": 4.9218, + "1014": 4.92051, + "1015": 4.92442, + "1016": 4.91248, + "1017": 4.92113, + "1018": 4.92046, + "1019": 4.91949, + "1020": 4.92623, + "1021": 4.92267, + "1022": 4.92249, + "1023": 4.91899, + "1024": 4.92062, + "1025": 5.26804, + "1026": 4.92131, + "1027": 5.26954, + "1028": 4.91856, + "1029": 4.91681, + "1030": 5.90813, + "1031": 4.92456, + "1032": 4.92325, + "1033": 5.3083, + "1034": 4.91916, + "1035": 4.91422, + "1036": 4.91293, + "1037": 4.91223, + "1038": 4.9211, + "1039": 4.92393, + "1040": 4.92009, + "1041": 4.92106, + "1042": 4.9242, + "1043": 4.92005, + "1044": 5.26878, + "1045": 4.92668, + "1046": 4.93095, + "1047": 5.27312, + "1048": 4.92622, + "1049": 4.92229, + "1050": 4.92078, + "1051": 4.9252, + "1052": 4.92398, + "1053": 4.92467, + "1054": 4.92254, + "1055": 4.92721, + "1056": 4.92594, + "1057": 4.93074, + "1058": 4.9202, + "1059": 4.92339, + "1060": 4.92936, + "1061": 4.92316, + "1062": 4.91832, + "1063": 4.9324, + "1064": 4.96238, + "1065": 4.94321, + "1066": 4.96241, + "1067": 4.93128, + "1068": 4.92665, + "1069": 4.93217, + "1070": 5.29473, + "1071": 5.27044, + "1072": 4.91774, + "1073": 4.92979, + "1074": 5.30092, + "1075": 5.57166, + "1076": 4.9336, + "1077": 4.91975, + "1078": 5.29838, + "1079": 4.92345, + "1080": 4.92265, + "1081": 4.93832, + "1082": 5.28966, + "1083": 4.94183, + "1084": 5.28091, + "1085": 4.94506, + "1086": 4.94668, + "1087": 4.94028, + "1088": 4.93858, + "1089": 4.93937, + "1090": 4.9454, + "1091": 4.95599, + "1092": 4.95023, + "1093": 4.94499, + "1094": 4.96028, + "1095": 4.95213, + "1096": 4.96406, + "1097": 4.93905, + "1098": 4.92198, + "1099": 4.93824, + "1100": 4.92789, + "1101": 4.92981, + "1102": 4.93937, + "1103": 4.91985, + "1104": 4.91889, + "1105": 4.93785, + "1106": 4.94007, + "1107": 4.93618, + "1108": 4.94002, + "1109": 4.96964, + "1110": 4.93965, + "1111": 4.89692, + "1112": 4.89611, + "1113": 4.89245, + "1114": 5.24194, + "1115": 4.89604, + "1116": 5.23738, + "1117": 4.89591, + "1118": 4.89712, + "1119": 6.2207, + "1120": 4.89707, + "1121": 5.24025, + "1122": 4.89987, + "1123": 5.27914, + "1124": 4.9043, + "1125": 4.89477, + "1126": 4.89625, + "1127": 4.90132, + "1128": 4.90216, + "1129": 4.90398, + "1130": 4.89594, + "1131": 4.90153, + "1132": 4.89796, + "1133": 4.89536, + "1134": 4.89807, + "1135": 4.89858, + "1136": 4.89867, + "1137": 4.89681, + "1138": 4.92931, + "1139": 4.92599, + "1140": 4.89538, + "1141": 4.89732, + "1142": 4.89242, + "1143": 4.89262, + "1144": 4.89274, + "1145": 4.93085, + "1146": 4.9294, + "1147": 4.92891, + "1148": 4.91881, + "1149": 4.89129, + "1150": 4.89171, + "1151": 4.8862, + "1152": 4.89315, + "1153": 4.89463, + "1154": 4.89481, + "1155": 4.89194, + "1156": 5.23303, + "1157": 4.89025, + "1158": 4.89312, + "1159": 5.24533, + "1160": 5.25573, + "1161": 5.23949, + "1162": 4.8914, + "1163": 4.89247, + "1164": 4.8896, + "1165": 5.88618, + "1166": 4.91824, + "1167": 4.89232, + "1168": 5.27914, + "1169": 4.88638, + "1170": 4.89624, + "1171": 4.90097, + "1172": 4.89335, + "1173": 4.90022, + "1174": 4.88823, + "1175": 4.91533, + "1176": 4.91702, + "1177": 4.91026, + "1178": 4.89204, + "1179": 4.89341, + "1180": 4.88754, + "1181": 4.89101, + "1182": 4.89528, + "1183": 4.89482, + "1184": 4.88208, + "1185": 4.87829, + "1186": 4.88501, + "1187": 4.88593, + "1188": 4.87526, + "1189": 4.88604, + "1190": 4.90872, + "1191": 4.88218, + "1192": 4.8826, + "1193": 4.88606, + "1194": 5.22378, + "1195": 4.88192, + "1196": 4.8877, + "1197": 5.23842, + "1198": 4.89888, + "1199": 4.89039, + "1200": 4.89543, + "1201": 4.8917, + "1202": 4.88928, + "1203": 4.88428, + "1204": 4.91394, + "1205": 5.27535, + "1206": 5.27273, + "1207": 4.92919, + "1208": 4.92498, + "1209": 5.60645, + "1210": 5.23108, + "1211": 4.91823, + "1212": 4.91107, + "1213": 4.90706, + "1214": 5.33395, + "1215": 4.91341, + "1216": 4.92296, + "1217": 4.92797, + "1218": 4.91436, + "1219": 4.93183, + "1220": 4.92763, + "1221": 4.91189, + "1222": 4.91524, + "1223": 4.92927, + "1224": 4.90762, + "1225": 4.91646, + "1226": 4.95199, + "1227": 4.93657, + "1228": 4.91049, + "1229": 4.90576, + "1230": 4.92418, + "1231": 5.24788, + "1232": 4.90922, + "1233": 4.90828, + "1234": 5.28741, + "1235": 4.93359, + "1236": 4.92651, + "1237": 4.92759, + "1238": 4.91812, + "1239": 4.96161, + "1240": 4.92462, + "1241": 4.9408, + "1242": 4.95151, + "1243": 4.92866, + "1244": 4.94942, + "1245": 4.93202, + "1246": 4.93118, + "1247": 4.92787, + "1248": 4.93195, + "1249": 5.31148, + "1250": 4.96525, + "1251": 5.27677, + "1252": 4.95992, + "1253": 4.89092, + "1254": 5.87598, + "1255": 4.89013, + "1256": 4.89328, + "1257": 4.88679, + "1258": 4.89107, + "1259": 5.26785, + "1260": 4.89071, + "1261": 4.89005, + "1262": 4.89216, + "1263": 4.89212, + "1264": 4.88574, + "1265": 4.88902, + "1266": 4.88642, + "1267": 4.89574, + "1268": 4.88631, + "1269": 5.22724, + "1270": 4.88943, + "1271": 5.23761, + "1272": 4.90353, + "1273": 4.89726, + "1274": 4.92161, + "1275": 4.92347, + "1276": 4.91698, + "1277": 4.92233, + "1278": 4.91979, + "1279": 4.9211, + "1280": 4.9179, + "1281": 4.92209, + "1282": 4.94485, + "1283": 4.92932, + "1284": 4.92976, + "1285": 4.91788, + "1286": 4.93408, + "1287": 4.92359, + "1288": 4.92166, + "1289": 4.9185, + "1290": 4.91424, + "1291": 4.91891, + "1292": 4.92028, + "1293": 4.9117, + "1294": 5.27044, + "1295": 5.29676, + "1296": 4.91703, + "1297": 4.92056, + "1298": 4.92207, + "1299": 5.91394, + "1300": 4.9147, + "1301": 4.9131, + "1302": 4.9176, + "1303": 4.93425, + "1304": 5.304, + "1305": 4.91978, + "1306": 5.27498, + "1307": 4.92043, + "1308": 4.91675, + "1309": 5.27831, + "1310": 4.93667, + "1311": 4.93075, + "1312": 4.92766, + "1313": 4.92554, + "1314": 4.93753, + "1315": 4.93323, + "1316": 4.92326, + "1317": 4.92226, + "1318": 4.9254, + "1319": 4.91683, + "1320": 4.91352, + "1321": 4.93361, + "1322": 4.9202, + "1323": 4.92888, + "1324": 4.94749, + "1325": 4.92427, + "1326": 4.91993, + "1327": 4.94147, + "1328": 4.91569, + "1329": 4.9082, + "1330": 4.90808, + "1331": 4.92463, + "1332": 4.94304, + "1333": 4.91833, + "1334": 4.91915, + "1335": 4.9569, + "1336": 4.91253, + "1337": 4.91228, + "1338": 4.91599, + "1339": 5.26886, + "1340": 4.94108, + "1341": 5.28895, + "1342": 4.92166, + "1343": 4.93148, + "1344": 6.20454, + "1345": 4.93732, + "1346": 4.94109, + "1347": 5.28178, + "1348": 4.92597, + "1349": 5.31528, + "1350": 4.93124, + "1351": 4.9199, + "1352": 4.92145, + "1353": 4.91761, + "1354": 4.91599, + "1355": 4.91867, + "1356": 4.92286, + "1357": 4.91965, + "1358": 4.92454, + "1359": 4.92188, + "1360": 4.91921, + "1361": 4.92021, + "1362": 4.92372, + "1363": 4.91207, + "1364": 4.96107, + "1365": 4.91388, + "1366": 4.91683, + "1367": 4.91413, + "1368": 4.91691, + "1369": 4.91871, + "1370": 4.92278, + "1371": 4.92605, + "1372": 4.92653, + "1373": 4.9264, + "1374": 4.92864, + "1375": 4.92839, + "1376": 4.93185, + "1377": 4.92304, + "1378": 4.92916, + "1379": 4.92701, + "1380": 4.92797, + "1381": 5.27325, + "1382": 4.89544, + "1383": 4.89064, + "1384": 5.60494, + "1385": 5.00482, + "1386": 5.33879, + "1387": 4.92912, + "1388": 4.92575, + "1389": 5.83703, + "1390": 4.91691, + "1391": 4.91717, + "1392": 4.92005, + "1393": 4.92211, + "1394": 4.91895, + "1395": 5.29903, + "1396": 4.92143, + "1397": 4.91551, + "1398": 4.91427, + "1399": 4.91348, + "1400": 4.92556, + "1401": 4.92553, + "1402": 4.91884, + "1403": 4.91856, + "1404": 4.95579, + "1405": 4.88917, + "1406": 4.88886, + "1407": 4.90262, + "1408": 4.88379, + "1409": 4.88976, + "1410": 4.88681, + "1411": 4.8751, + "1412": 4.89308, + "1413": 4.89122, + "1414": 4.88458, + "1415": 4.89489, + "1416": 4.88438, + "1417": 4.88183, + "1418": 5.229, + "1419": 4.96736, + "1420": 4.95735, + "1421": 5.29839, + "1422": 4.92896, + "1423": 4.9679, + "1424": 4.96109, + "1425": 4.96048, + "1426": 4.95854, + "1427": 4.95558, + "1428": 4.90503, + "1429": 5.24486, + "1430": 5.24901, + "1431": 4.8987, + "1432": 4.89075, + "1433": 5.22736, + "1434": 5.47175, + "1435": 4.89209, + "1436": 4.8986, + "1437": 4.8891, + "1438": 4.88697, + "1439": 4.88974, + "1440": 5.27298, + "1441": 4.89403, + "1442": 4.90495, + "1443": 4.89585, + "1444": 4.89766, + "1445": 4.89344, + "1446": 4.89618, + "1447": 4.88721, + "1448": 4.88735, + "1449": 4.89401, + "1450": 4.89435, + "1451": 4.89143, + "1452": 4.88553, + "1453": 4.89139, + "1454": 4.89347, + "1455": 5.23147, + "1456": 4.8987, + "1457": 4.90447, + "1458": 4.89553, + "1459": 5.23187, + "1460": 4.90546, + "1461": 4.89293, + "1462": 4.89652, + "1463": 4.88806, + "1464": 4.94852, + "1465": 4.89339, + "1466": 4.88888, + "1467": 4.89409, + "1468": 4.89028, + "1469": 4.89198, + "1470": 4.89499, + "1471": 4.89853, + "1472": 4.89989, + "1473": 5.245, + "1474": 4.89244, + "1475": 5.24744, + "1476": 4.88786, + "1477": 4.88954, + "1478": 5.81074, + "1479": 4.90603, + "1480": 4.8817, + "1481": 4.88853, + "1482": 4.88913, + "1483": 4.88525, + "1484": 4.88091, + "1485": 5.26103, + "1486": 4.88332, + "1487": 4.88482, + "1488": 4.88349, + "1489": 4.93535, + "1490": 4.93713, + "1491": 4.94008, + "1492": 4.93273, + "1493": 5.26558, + "1494": 4.92625, + "1495": 4.93119, + "1496": 4.93326, + "1497": 5.29661, + "1498": 4.94651, + "1499": 4.94563, + "1500": 4.94732, + "1501": 4.94956, + "1502": 4.93949, + "1503": 4.94314, + "1504": 4.949, + "1505": 4.93848, + "1506": 4.93655, + "1507": 4.93352, + "1508": 4.93376, + "1509": 4.93575, + "1510": 4.93237, + "1511": 4.93325, + "1512": 4.93443, + "1513": 4.93608, + "1514": 4.92875, + "1515": 4.93822, + "1516": 4.92271, + "1517": 4.93602, + "1518": 4.93135, + "1519": 5.28269, + "1520": 5.28601, + "1521": 4.93214, + "1522": 4.93238, + "1523": 4.9331, + "1524": 5.84985, + "1525": 4.93183, + "1526": 4.9312, + "1527": 4.94067, + "1528": 4.94179, + "1529": 4.93283, + "1530": 5.64255, + "1531": 4.93012, + "1532": 4.93237, + "1533": 4.93188, + "1534": 5.28642, + "1535": 4.93295, + "1536": 4.93351, + "1537": 4.93687, + "1538": 4.93395, + "1539": 4.93892, + "1540": 4.93329, + "1541": 4.93178, + "1542": 4.94011, + "1543": 4.93223, + "1544": 4.9238, + "1545": 4.93295, + "1546": 4.92789, + "1547": 4.92723, + "1548": 4.93344, + "1549": 4.93081, + "1550": 4.93484, + "1551": 4.93247, + "1552": 4.94286, + "1553": 4.93871, + "1554": 4.9346, + "1555": 4.93508, + "1556": 4.93254, + "1557": 4.93621, + "1558": 4.93402, + "1559": 4.92552, + "1560": 4.92871, + "1561": 4.9342, + "1562": 4.93981, + "1563": 4.94231, + "1564": 5.28559, + "1565": 5.2926, + "1566": 4.93393, + "1567": 5.27554, + "1568": 5.55669, + "1569": 5.22897, + "1570": 4.93426, + "1571": 5.28382, + "1572": 4.94938, + "1573": 4.95055, + "1574": 4.94811, + "1575": 4.9489, + "1576": 5.33208, + "1577": 4.94524, + "1578": 4.94592, + "1579": 4.94832, + "1580": 4.94832, + "1581": 4.94408, + "1582": 4.93963, + "1583": 5.06791, + "1584": 4.93161, + "1585": 4.93335, + "1586": 4.93849, + "1587": 4.93237, + "1588": 4.93556, + "1589": 4.93066, + "1590": 4.94768, + "1591": 4.93099, + "1592": 4.93258, + "1593": 4.93981, + "1594": 4.92949, + "1595": 4.93453, + "1596": 4.92827, + "1597": 4.92584, + "1598": 4.93755, + "1599": 4.92974, + "1600": 4.94804, + "1601": 4.93191, + "1602": 4.93369, + "1603": 4.93286, + "1604": 4.93069, + "1605": 5.27051, + "1606": 4.92329, + "1607": 4.92495, + "1608": 5.27779, + "1609": 5.28346, + "1610": 5.29602, + "1611": 4.94123, + "1612": 4.93638, + "1613": 5.856, + "1614": 4.94437, + "1615": 4.93653, + "1616": 4.93875, + "1617": 4.93536, + "1618": 4.93896, + "1619": 4.93356, + "1620": 4.93572, + "1621": 5.31736, + "1622": 4.94531, + "1623": 4.94225, + "1624": 4.94386, + "1625": 4.93406, + "1626": 4.93798, + "1627": 4.93633, + "1628": 4.93917, + "1629": 4.93696, + "1630": 4.93053, + "1631": 4.92648, + "1632": 4.92658, + "1633": 4.93841, + "1634": 4.93342, + "1635": 4.9359, + "1636": 4.93181, + "1637": 4.93503, + "1638": 4.93642, + "1639": 4.93683, + "1640": 4.93436, + "1641": 4.9443, + "1642": 5.27794, + "1643": 4.94268, + "1644": 4.91864, + "1645": 4.92135, + "1646": 5.26653, + "1647": 4.93155, + "1648": 4.94793, + "1649": 4.92681, + "1650": 4.92909, + "1651": 4.92222, + "1652": 4.93308, + "1653": 5.27802, + "1654": 5.27831, + "1655": 4.92527, + "1656": 4.92184, + "1657": 4.92535, + "1658": 5.84478, + "1659": 4.93415, + "1660": 4.98533, + "1661": 4.95752, + "1662": 4.94766, + "1663": 4.94933, + "1664": 4.95355, + "1665": 4.94643, + "1666": 5.33217, + "1667": 4.93611, + "1668": 4.93532, + "1669": 4.9092, + "1670": 4.90894, + "1671": 4.9204, + "1672": 4.92236, + "1673": 4.9082, + "1674": 4.91286, + "1675": 4.90919, + "1676": 4.90864, + "1677": 4.91312, + "1678": 4.90871, + "1679": 4.92308, + "1680": 5.26267, + "1681": 4.92022, + "1682": 4.91096, + "1683": 4.91568, + "1684": 5.26065, + "1685": 4.90909, + "1686": 4.90718, + "1687": 4.91023, + "1688": 4.91504, + "1689": 4.9123, + "1690": 4.91353, + "1691": 4.90838, + "1692": 4.90311, + "1693": 4.90235, + "1694": 4.90376, + "1695": 4.90901, + "1696": 4.90724, + "1697": 4.91094, + "1698": 5.25776, + "1699": 4.91455, + "1700": 5.2613, + "1701": 4.90973, + "1702": 4.90149, + "1703": 5.82797, + "1704": 4.9102, + "1705": 4.91831, + "1706": 4.90187, + "1707": 4.89945, + "1708": 4.89865, + "1709": 4.89632, + "1710": 4.90065, + "1711": 5.28146, + "1712": 4.90271, + "1713": 4.90852, + "1714": 4.90365, + "1715": 4.90463, + "1716": 4.91059, + "1717": 5.24655, + "1718": 4.91868, + "1719": 4.90569, + "1720": 4.91426, + "1721": 4.91116, + "1722": 5.25454, + "1723": 4.91058, + "1724": 4.90906, + "1725": 4.92075, + "1726": 4.91839, + "1727": 4.91564, + "1728": 4.91131, + "1729": 4.91291, + "1730": 4.90884, + "1731": 4.91062, + "1732": 4.90638, + "1733": 4.9061, + "1734": 4.90658, + "1735": 4.91543, + "1736": 4.90614, + "1737": 4.91107, + "1738": 4.91084, + "1739": 4.90842, + "1740": 4.91418, + "1741": 4.90881, + "1742": 4.90792, + "1743": 5.26397, + "1744": 4.91738, + "1745": 5.25587, + "1746": 4.90599, + "1747": 4.90321, + "1748": 5.78796, + "1749": 4.90348, + "1750": 4.90858, + "1751": 4.89993, + "1752": 4.90938, + "1753": 4.90593, + "1754": 5.25406, + "1755": 4.9167, + "1756": 4.92732, + "1757": 5.32154, + "1758": 4.93234, + "1759": 5.25874, + "1760": 4.90683, + "1761": 4.90629, + "1762": 4.91525, + "1763": 4.91544, + "1764": 4.91062, + "1765": 4.90636, + "1766": 4.90873, + "1767": 4.91142, + "1768": 4.96573, + "1769": 4.90448, + "1770": 4.8891, + "1771": 4.8932, + "1772": 4.88066, + "1773": 4.87927, + "1774": 4.87496, + "1775": 4.90017, + "1776": 4.88861, + "1777": 4.88943, + "1778": 4.88632, + "1779": 4.89539, + "1780": 4.88673, + "1781": 4.89482, + "1782": 4.89261, + "1783": 4.88921, + "1784": 4.89935, + "1785": 4.88986, + "1786": 4.89061, + "1787": 4.88853, + "1788": 5.24035, + "1789": 5.24993, + "1790": 4.91207, + "1791": 4.91991, + "1792": 5.55415, + "1793": 5.49039, + "1794": 4.899, + "1795": 4.88922, + "1796": 5.25127, + "1797": 4.89889, + "1798": 4.90442, + "1799": 4.89627, + "1800": 4.89346, + "1801": 4.89082, + "1802": 5.2731, + "1803": 4.89886, + "1804": 4.87379, + "1805": 4.87577, + "1806": 4.88484, + "1807": 4.87576, + "1808": 4.86783, + "1809": 4.8917, + "1810": 4.87329, + "1811": 4.87182, + "1812": 4.8594, + "1813": 4.86213, + "1814": 4.86701, + "1815": 4.86025, + "1816": 4.86454, + "1817": 4.86162, + "1818": 4.85688, + "1819": 4.85907, + "1820": 4.85765, + "1821": 4.85878, + "1822": 4.86537, + "1823": 4.86101, + "1824": 4.86218, + "1825": 4.86082, + "1826": 4.85916, + "1827": 4.86304, + "1828": 4.86335, + "1829": 4.85846, + "1830": 5.21054, + "1831": 4.87227, + "1832": 5.20618, + "1833": 4.86815, + "1834": 5.55416, + "1835": 4.87798, + "1836": 4.89752, + "1837": 5.79486, + "1838": 4.90553, + "1839": 4.90533, + "1840": 4.89368, + "1841": 4.89475, + "1842": 4.89469, + "1843": 4.88557, + "1844": 4.89, + "1845": 4.88668, + "1846": 4.89537, + "1847": 5.26263, + "1848": 4.89245, + "1849": 4.89348, + "1850": 4.88835, + "1851": 4.90708, + "1852": 4.90228, + "1853": 4.86785, + "1854": 4.87736, + "1855": 4.87369, + "1856": 4.87811, + "1857": 4.90299, + "1858": 4.88442, + "1859": 4.87297, + "1860": 4.89531, + "1861": 4.90241, + "1862": 4.89309, + "1863": 4.89512, + "1864": 4.90549, + "1865": 4.90854, + "1866": 4.9047, + "1867": 5.2401, + "1868": 4.89946, + "1869": 4.90883, + "1870": 4.90522, + "1871": 4.93888, + "1872": 5.21372, + "1873": 4.87709, + "1874": 4.86464, + "1875": 4.87233, + "1876": 4.88054, + "1877": 4.84923, + "1878": 5.17207, + "1879": 5.1976, + "1880": 4.8445, + "1881": 4.84388, + "1882": 4.84797, + "1883": 5.73664, + "1884": 4.84672, + "1885": 4.84557, + "1886": 4.85201, + "1887": 4.85018, + "1888": 4.84932, + "1889": 4.85617, + "1890": 4.84416, + "1891": 4.85089, + "1892": 4.84881, + "1893": 5.22668, + "1894": 4.8491, + "1895": 4.84681, + "1896": 4.84529, + "1897": 4.84998, + "1898": 4.8507, + "1899": 4.84271, + "1900": 4.84844, + "1901": 4.84365, + "1902": 4.83991, + "1903": 4.84228, + "1904": 5.17846, + "1905": 4.84978, + "1906": 4.84285, + "1907": 4.85138, + "1908": 4.84338, + "1909": 5.19721, + "1910": 4.85138, + "1911": 4.84739, + "1912": 4.84478, + "1913": 4.85226, + "1914": 4.85002, + "1915": 4.85039, + "1916": 4.85444, + "1917": 4.84588, + "1918": 4.8495, + "1919": 4.85217, + "1920": 4.84949, + "1921": 4.84631, + "1922": 4.84476, + "1923": 5.17493, + "1924": 5.19107, + "1925": 4.85154, + "1926": 4.84261, + "1927": 5.44494, + "1928": 5.14044, + "1929": 4.84927, + "1930": 4.84493, + "1931": 4.84048, + "1932": 4.84204, + "1933": 4.84664, + "1934": 4.84105, + "1935": 4.83981, + "1936": 4.841, + "1937": 4.84038, + "1938": 5.22894, + "1939": 4.84209, + "1940": 4.84356, + "1941": 5.20657, + "1942": 4.9004, + "1943": 4.90813, + "1944": 4.90655, + "1945": 4.88214, + "1946": 5.21239, + "1947": 4.86529, + "1948": 4.85849, + "1949": 4.85084, + "1950": 4.86533, + "1951": 4.86, + "1952": 4.85847, + "1953": 4.86113, + "1954": 4.85194, + "1955": 4.85611, + "1956": 4.87124, + "1957": 4.8777, + "1958": 4.84686, + "1959": 4.84732, + "1960": 4.86364, + "1961": 4.8509, + "1962": 4.8663, + "1963": 4.87064, + "1964": 4.86099, + "1965": 4.86103, + "1966": 4.84569, + "1967": 5.17792, + "1968": 4.84796, + "1969": 5.20648, + "1970": 4.84901, + "1971": 4.84838, + "1972": 5.74018, + "1973": 4.85813, + "1974": 4.85367, + "1975": 4.86684, + "1976": 4.87041, + "1977": 4.90603, + "1978": 4.90475, + "1979": 5.25145, + "1980": 4.94444, + "1981": 4.92124, + "1982": 4.90832, + "1983": 4.94722, + "1984": 5.67636, + "1985": 4.939, + "1986": 4.93543, + "1987": 4.96136, + "1988": 4.92447, + "1989": 4.87603, + "1990": 4.86128, + "1991": 4.86822, + "1992": 4.86666, + "1993": 4.85995, + "1994": 4.86025, + "1995": 4.85738, + "1996": 4.86953, + "1997": 4.86535, + "1998": 4.86591, + "1999": 4.86231, + "2000": 4.86466 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml index c16fedc7860..080f669e6a4 100644 --- a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml +++ b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release/model_config.yaml @@ -12,11 +12,12 @@ ENV_VARS: NCCL_DEBUG: VERSION NON_DETERMINSTIC_RESULTS: 1 NVSHMEM_IB_ENABLE_IBGDA: 0 + CUDA_DEVICE_MAX_CONNECTIONS: 1 TEST_TYPE: "release" MODEL_ARGS: # Distributed args --distributed-timeout-minutes: 60 - --tensor-model-parallel-size: 1 + --tensor-model-parallel-size: 2 --pipeline-model-parallel-size: 4 --pipeline-model-parallel-layout: Et*2\\|\\(tt\\|\\)*5t\\|tmL # Et*2|(tt|)*5t|tmL --expert-model-parallel-size: 16 @@ -47,8 +48,8 @@ MODEL_ARGS: # Data args --seq-length: 4096 --data-cache-path: ${DATA_CACHE_PATH} - --tokenizer-type: HuggingFaceTokenizer - --tokenizer-model: ${TOKENIZER_PATH} + --tokenizer-type: GPTSentencePieceTokenizer + --tokenizer-model: ${DATA_PATH}/utils/nemotron_2_256k.model --data-path: $DATA_BLEND --split: 99,1,0 --no-mmap-bin-files: true @@ -81,12 +82,11 @@ MODEL_ARGS: --qk-layernorm: true # Add learning rate args - --lr-decay-samples: 584765624 + --lr-decay-samples: 24413696 --lr-warmup-samples: 1536000 - # Learning rate scaled down from 7.3e-6 (DeepSeek-V3 technical report, GBS=15360) to 3.9e-6 (GBS=8192) - --lr-warmup-init: 3.9e-7 - --lr: 3.9e-6 - --min-lr: 3.9e-7 + --lr-warmup-init: 1e-7 + --lr: 1e-5 + --min-lr: 1e-6 --lr-decay-style: cosine --adam-beta1: 0.9 --adam-beta2: 0.95 @@ -127,8 +127,6 @@ MODEL_ARGS: --eval-interval: 200 # Add checkpointing args - --no-load-optim: true - --no-load-rng: true --auto-detect-ckpt-format: true # Add checkpointing args diff --git a/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml new file mode 100644 index 00000000000..8bab921aa04 --- /dev/null +++ b/tests/functional_tests/test_cases/mixtral/deepseekv3_proxy_flex_tp1pp4emp16etp1cp1_release_sm/model_config.yaml @@ -0,0 +1,166 @@ +# The proxy model is used for local code quality check. +# The proxy model should contain all the necessary components and settings but fewer parameters. +ENV_VARS: + TORCH_NCCL_AVOID_RECORD_STREAMS: 0 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 1 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: 0 + NVTE_FUSED_ATTN: 1 + NVTE_NORM_FWD_USE_CUDNN: 1 + NVTE_NORM_BWD_USE_CUDNN: 1 + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION + NON_DETERMINSTIC_RESULTS: 1 + NVSHMEM_IB_ENABLE_IBGDA: 0 + CUDA_DEVICE_MAX_CONNECTIONS: 1 +TEST_TYPE: "release" +MODEL_ARGS: + # Distributed args + --distributed-timeout-minutes: 60 + --tensor-model-parallel-size: 2 + --pipeline-model-parallel-size: 4 + --pipeline-model-parallel-layout: Et*2\\|\\(tt\\|\\)*5t\\|tmL # Et*2|(tt|)*5t|tmL + --expert-model-parallel-size: 16 + --context-parallel-size: 1 + --expert-tensor-parallel-size: 1 + --use-distributed-optimizer: true + --overlap-grad-reduce: true + --overlap-param-gather: true + + # Training args + --use-mcore-models: true + --sequence-parallel: true + --use-flash-attn: true + --disable-bias-linear: true + --micro-batch-size: 1 + --global-batch-size: 512 + --train-samples: 24414062 + --exit-duration-in-mins: 220 + --no-check-for-nan-in-loss-and-grad: true + --cross-entropy-loss-fusion: true + --cross-entropy-fusion-impl: te + --manual-gc: true + --manual-gc-interval: 10 + + # Transformer Engine args + --transformer-impl: transformer_engine + + # Data args + --seq-length: 4096 + --data-cache-path: ${DATA_CACHE_PATH} + --tokenizer-type: GPTSentencePieceTokenizer + --tokenizer-model: ${DATA_PATH}/utils/nemotron_2_256k.model + --data-path: $DATA_BLEND + --split: 99,1,0 + --no-mmap-bin-files: true + --no-create-attention-mask-in-dataloader: true + --num-workers: 6 + + # Add network size args + --num-layers: 14 # original 61 layers + --hidden-size: 7168 + --ffn-hidden-size: 18432 + --num-attention-heads: 128 + --kv-channels: 128 + --max-position-embeddings: 4096 + --position-embedding-type: rope + --rotary-base: 10000 + --make-vocab-size-divisible-by: 3232 + --normalization: RMSNorm + --norm-epsilon: 1e-6 + --swiglu: true + --untie-embeddings-and-output-weights: true + --multi-latent-attention: true + --mtp-num-layers: 1 + --mtp-loss-scaling-factor: 0.1 + + # Add regularization args + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --clip-grad: 1.0 + --weight-decay: 0.1 + --qk-layernorm: true + + # Add learning rate args + --lr-decay-samples: 24413696 + --lr-warmup-samples: 1536000 + --lr-warmup-init: 1e-7 + --lr: 1e-5 + --min-lr: 1e-6 + --lr-decay-style: cosine + --adam-beta1: 0.9 + --adam-beta2: 0.95 + + # Add MoE args + --num-experts: 64 # local 4 + 1 shared, EP16 + --moe-layer-freq: ([0]*3+[1]*11) + --moe-ffn-hidden-size: 2048 + --moe-shared-expert-intermediate-size: 2048 + --moe-router-load-balancing-type: seq_aux_loss + --moe-router-topk: 8 + --moe-token-dispatcher-type: flex + --moe-enable-deepep: true + --moe-router-pre-softmax: true + --moe-grouped-gemm: true + --moe-aux-loss-coeff: 1e-4 + --moe-router-group-topk: 4 + --moe-router-num-groups: 8 + --moe-router-topk-scaling-factor: 2.5 + --moe-router-score-function: sigmoid + --moe-router-enable-expert-bias: true + --moe-router-bias-update-rate: 1e-3 + --moe-router-dtype: fp32 + --moe-permute-fusion: true + + # Add MLA args + --q-lora-rank: 1536 + --kv-lora-rank: 512 + --qk-head-dim: 128 + --qk-pos-emb-head-dim: 64 + --v-head-dim: 128 + --rotary-scaling-factor: 40 + --mscale: 1.0 + --mscale-all-dim: 1.0 + + # Add validation args + --eval-iters: 32 + --eval-interval: 200 + + # Add checkpointing args + --auto-detect-ckpt-format: + true + # Add checkpointing args + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 500 + --save-retain-interval: 10000 + --dist-ckpt-strictness: log_all + + # Add initialization args + --init-method-std: 0.02 + + # Add logging args + --log-timers-to-tensorboard: true + --log-memory-to-tensorboard: true + --log-num-zeros-in-grad: true + --log-params-norm: true + --log-validation-ppl-to-tensorboard: true + --log-throughput: true + --log-interval: 1 + --logging-level: 40 + --tensorboard-dir: ${TENSORBOARD_PATH} + --wandb-project: megatron-core-release-runs + --wandb-exp-name: ${WANDB_EXPERIMENT} + --wandb-save-dir: ${WANDB_SAVE_PATH} + + # Add mixed precision args + --bf16: true + + # enable experimental + --enable-experimental: true + --exit-interval: 9536 +METRICS: + - "iteration-time" + - "lm loss" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" diff --git a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json index ef8ee741272..b7df693e1f7 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8/golden_values_dev_dgx_h100.json @@ -2,109 +2,343 @@ "lm loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.0475, - "5": 9.43078, - "10": 8.89238, - "15": 7.93732, - "20": 7.77942, - "25": 7.61408, - "30": 7.57234, - "35": 7.15189, - "40": 7.48085, - "45": 7.12056, - "50": 6.96054 + "1": 11.04577, + "2": 11.03578, + "3": 9.5968, + "4": 9.26068, + "5": 9.09365, + "6": 8.97825, + "7": 9.18096, + "8": 8.70673, + "9": 8.55632, + "10": 8.85377, + "11": 8.31245, + "12": 8.35862, + "13": 8.28114, + "14": 7.73951, + "15": 7.91242, + "16": 7.94944, + "17": 7.89918, + "18": 7.64375, + "19": 8.02647, + "20": 7.73813, + "21": 7.44557, + "22": 7.43367, + "23": 7.31291, + "24": 7.30268, + "25": 7.57549, + "26": 6.98093, + "27": 7.50005, + "28": 7.241, + "29": 7.40369, + "30": 7.51839, + "31": 7.29514, + "32": 7.47818, + "33": 7.52568, + "34": 7.57647, + "35": 7.12091, + "36": 6.97439, + "37": 7.30929, + "38": 7.09349, + "39": 7.43659, + "40": 7.45122, + "41": 7.37904, + "42": 7.14627, + "43": 7.13408, + "44": 7.30886, + "45": 7.08523, + "46": 6.8067, + "47": 7.21159, + "48": 7.0245, + "49": 7.50096, + "50": 6.92687 } }, "num-zeros": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 38802620.0, - "5": 243556240.0, - "10": 716187584.0, - "15": 614358336.0, - "20": 677963584.0, - "25": 736321856.0, - "30": 505223648.0, - "35": 548946176.0, - "40": 412329664.0, - "45": 376634624.0, - "50": 205546672.0 + "1": 38802664.0, + "2": 38543552.0, + "3": 38740472.0, + "4": 273766176.0, + "5": 196515488.0, + "6": 432153600.0, + "7": 715038528.0, + "8": 797328960.0, + "9": 696279488.0, + "10": 668928192.0, + "11": 583742720.0, + "12": 595799040.0, + "13": 695916288.0, + "14": 617245056.0, + "15": 629936832.0, + "16": 639940800.0, + "17": 642766016.0, + "18": 664898112.0, + "19": 671247104.0, + "20": 602545216.0, + "21": 542607872.0, + "22": 551419008.0, + "23": 533094816.0, + "24": 527647904.0, + "25": 570717824.0, + "26": 510874176.0, + "27": 498748096.0, + "28": 510353632.0, + "29": 506802112.0, + "30": 486336928.0, + "31": 410143360.0, + "32": 372280800.0, + "33": 369351776.0, + "34": 353666688.0, + "35": 344549376.0, + "36": 278456576.0, + "37": 289517152.0, + "38": 274950816.0, + "39": 242921776.0, + "40": 223597264.0, + "41": 186386944.0, + "42": 180387488.0, + "43": 224573440.0, + "44": 217714800.0, + "45": 143723568.0, + "46": 161525888.0, + "47": 120124336.0, + "48": 183368272.0, + "49": 154411968.0, + "50": 167778288.0 } }, "mem-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 7321331200.0, - "5": 7321333248.0, - "10": 7321333248.0, - "15": 7321333248.0, - "20": 7321333248.0, - "25": 7321333248.0, - "30": 7321333248.0, - "35": 7321333248.0, - "40": 7321333248.0, - "45": 7321333248.0, - "50": 7321333248.0 + "1": 7321336320.0, + "2": 7321338368.0, + "3": 7321338368.0, + "4": 7321338368.0, + "5": 7321338368.0, + "6": 7321338368.0, + "7": 7321338368.0, + "8": 7321338368.0, + "9": 7321338368.0, + "10": 7321338368.0, + "11": 7321338368.0, + "12": 7321338368.0, + "13": 7321338368.0, + "14": 7321338368.0, + "15": 7321338368.0, + "16": 7321338368.0, + "17": 7321338368.0, + "18": 7321338368.0, + "19": 7321338368.0, + "20": 7321338368.0, + "21": 7321338368.0, + "22": 7321338368.0, + "23": 7321338368.0, + "24": 7321338368.0, + "25": 7321338368.0, + "26": 7321338368.0, + "27": 7321338368.0, + "28": 7321338368.0, + "29": 7321338368.0, + "30": 7321338368.0, + "31": 7321338368.0, + "32": 7321338368.0, + "33": 7321338368.0, + "34": 7321338368.0, + "35": 7321338368.0, + "36": 7321338368.0, + "37": 7321338368.0, + "38": 7321338368.0, + "39": 7321338368.0, + "40": 7321338368.0, + "41": 7321338368.0, + "42": 7321338368.0, + "43": 7321338368.0, + "44": 7321338368.0, + "45": 7321338368.0, + "46": 7321338368.0, + "47": 7321338368.0, + "48": 7321338368.0, + "49": 7321338368.0, + "50": 7321338368.0 } }, "mem-max-allocated-bytes": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 53176152064.0, - "5": 55926337536.0, - "10": 55926337536.0, - "15": 55926337536.0, - "20": 55926337536.0, - "25": 56534257664.0, - "30": 57393635328.0, - "35": 57393635328.0, - "40": 57578217472.0, - "45": 57578217472.0, - "50": 57578217472.0 + "1": 54402162688.0, + "2": 57150373888.0, + "3": 57150373888.0, + "4": 57150373888.0, + "5": 57150373888.0, + "6": 57150373888.0, + "7": 57150373888.0, + "8": 57150373888.0, + "9": 57150373888.0, + "10": 57150373888.0, + "11": 57150373888.0, + "12": 57150373888.0, + "13": 57150373888.0, + "14": 57150373888.0, + "15": 57150373888.0, + "16": 57150373888.0, + "17": 57150373888.0, + "18": 57150373888.0, + "19": 57150373888.0, + "20": 57150373888.0, + "21": 57150373888.0, + "22": 57150373888.0, + "23": 57150373888.0, + "24": 57150373888.0, + "25": 57150373888.0, + "26": 57150373888.0, + "27": 57150373888.0, + "28": 57150373888.0, + "29": 57150373888.0, + "30": 57150373888.0, + "31": 57150373888.0, + "32": 57150373888.0, + "33": 57150373888.0, + "34": 57150373888.0, + "35": 57152438272.0, + "36": 57344114688.0, + "37": 57344114688.0, + "38": 57449279488.0, + "39": 57449279488.0, + "40": 57449279488.0, + "41": 57449279488.0, + "42": 57449279488.0, + "43": 57449279488.0, + "44": 57449279488.0, + "45": 57470353408.0, + "46": 57470353408.0, + "47": 57470353408.0, + "48": 57470353408.0, + "49": 57470353408.0, + "50": 57470353408.0 } }, "mtp_1 loss": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 11.0776, - "5": 9.87653, - "10": 9.02332, - "15": 7.91471, - "20": 7.75886, - "25": 7.56825, - "30": 7.53841, - "35": 7.12192, - "40": 7.44579, - "45": 7.09307, - "50": 6.94739 + "1": 11.07769, + "2": 11.07625, + "3": 10.52909, + "4": 10.08687, + "5": 9.82013, + "6": 9.48246, + "7": 9.54169, + "8": 8.83661, + "9": 8.64933, + "10": 8.95821, + "11": 8.32934, + "12": 8.36033, + "13": 8.26936, + "14": 7.73441, + "15": 7.87122, + "16": 7.9153, + "17": 7.86923, + "18": 7.61191, + "19": 7.99919, + "20": 7.72174, + "21": 7.4147, + "22": 7.40336, + "23": 7.27676, + "24": 7.28557, + "25": 7.53782, + "26": 6.94933, + "27": 7.48504, + "28": 7.20219, + "29": 7.38696, + "30": 7.51152, + "31": 7.26613, + "32": 7.45631, + "33": 7.51482, + "34": 7.57527, + "35": 7.10374, + "36": 6.97224, + "37": 7.31053, + "38": 7.08607, + "39": 7.44371, + "40": 7.43612, + "41": 7.37848, + "42": 7.13561, + "43": 7.11558, + "44": 7.30254, + "45": 7.08147, + "46": 6.78911, + "47": 7.21791, + "48": 7.03066, + "49": 7.46668, + "50": 6.93251 } }, "iteration-time": { "start_step": 1, "end_step": 50, - "step_interval": 5, + "step_interval": 1, "values": { - "1": 51.33936, - "5": 1.24167, - "10": 1.14623, - "15": 1.16973, - "20": 1.23165, - "25": 1.13719, - "30": 1.15864, - "35": 1.13509, - "40": 1.14729, - "45": 1.14136, - "50": 1.13625 + "1": 95.02242, + "2": 1.29728, + "3": 1.24413, + "4": 1.67309, + "5": 1.12527, + "6": 1.39226, + "7": 1.33351, + "8": 1.19614, + "9": 1.10737, + "10": 1.09796, + "11": 1.10736, + "12": 1.10105, + "13": 1.10552, + "14": 1.11007, + "15": 1.09853, + "16": 1.10142, + "17": 1.09718, + "18": 1.10103, + "19": 1.10339, + "20": 1.1069, + "21": 1.10541, + "22": 1.10374, + "23": 1.1028, + "24": 1.1, + "25": 1.09935, + "26": 1.09318, + "27": 1.09779, + "28": 1.09457, + "29": 1.09, + "30": 1.09267, + "31": 1.08899, + "32": 1.09268, + "33": 1.08757, + "34": 1.08991, + "35": 1.09705, + "36": 1.09429, + "37": 1.09459, + "38": 1.08857, + "39": 1.09547, + "40": 1.09224, + "41": 1.089, + "42": 1.08879, + "43": 1.0834, + "44": 1.08212, + "45": 1.08363, + "46": 1.08596, + "47": 1.07798, + "48": 1.07329, + "49": 1.07678, + "50": 1.07483 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..13bfff6c765 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81131, + "2": 10.83052, + "3": 10.82065, + "4": 10.81318, + "5": 10.84363, + "6": 10.84747, + "7": 10.85338, + "8": 10.83667, + "9": 10.8468, + "10": 10.7825, + "11": 10.85216, + "12": 10.86296, + "13": 10.85469, + "14": 10.88433, + "15": 10.87748, + "16": 10.84698, + "17": 10.83109, + "18": 10.86619, + "19": 10.84965, + "20": 10.84503, + "21": 10.84788, + "22": 10.79628, + "23": 10.88209, + "24": 10.83272, + "25": 10.82407, + "26": 10.84275, + "27": 10.85284, + "28": 10.87701, + "29": 10.8644, + "30": 10.81288, + "31": 10.78708, + "32": 10.85504, + "33": 10.85616, + "34": 10.84955, + "35": 10.83713, + "36": 10.80378, + "37": 10.83848, + "38": 10.80562, + "39": 10.8422, + "40": 10.80302, + "41": 10.84057, + "42": 10.84402, + "43": 10.81002, + "44": 10.80246, + "45": 10.78649, + "46": 10.80799, + "47": 10.817, + "48": 10.80324, + "49": 10.78157, + "50": 10.80218, + "51": 10.82262, + "52": 10.80415, + "53": 10.83258, + "54": 10.81542, + "55": 10.82524, + "56": 10.77667, + "57": 10.75278, + "58": 10.8075, + "59": 10.79063, + "60": 10.73975, + "61": 10.79974, + "62": 10.81288, + "63": 10.72014, + "64": 10.78563, + "65": 10.68987, + "66": 10.76119, + "67": 10.73431, + "68": 10.80192, + "69": 10.78336, + "70": 10.77619, + "71": 10.76644, + "72": 10.73613, + "73": 10.72971, + "74": 10.62238, + "75": 10.69054, + "76": 10.65471, + "77": 10.82153, + "78": 10.76381, + "79": 10.705, + "80": 10.69388, + "81": 10.72432, + "82": 10.74257, + "83": 10.66783, + "84": 10.69845, + "85": 10.71465, + "86": 10.63873, + "87": 10.71762, + "88": 10.73506, + "89": 10.71394, + "90": 10.74649, + "91": 10.64881, + "92": 10.64684, + "93": 10.60201, + "94": 10.53283, + "95": 10.66127, + "96": 10.67245, + "97": 10.61405, + "98": 10.68482, + "99": 10.52006, + "100": 10.61575 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1216.0, + "2": 1361.0, + "3": 1290.0, + "4": 1255.0, + "5": 1433.0, + "6": 1548.0, + "7": 1277.0, + "8": 1340.0, + "9": 1318.0, + "10": 1284.0, + "11": 1307.0, + "12": 1174.0, + "13": 1268.0, + "14": 1421.0, + "15": 1220.0, + "16": 1229.0, + "17": 1346.0, + "18": 1311.0, + "19": 1252.0, + "20": 1273.0, + "21": 1283.0, + "22": 1145.0, + "23": 1454.0, + "24": 1348.0, + "25": 1258.0, + "26": 1212.0, + "27": 1343.0, + "28": 1389.0, + "29": 1282.0, + "30": 1203.0, + "31": 1152.0, + "32": 1244.0, + "33": 1290.0, + "34": 1082.0, + "35": 1176.0, + "36": 1168.0, + "37": 1242.0, + "38": 1316.0, + "39": 1589.0, + "40": 1218.0, + "41": 1391.0, + "42": 1137.0, + "43": 1234.0, + "44": 1265.0, + "45": 1194.0, + "46": 1124.0, + "47": 1300.0, + "48": 1102.0, + "49": 1124.0, + "50": 1211.0, + "51": 1266.0, + "52": 1269.0, + "53": 1355.0, + "54": 1212.0, + "55": 1137.0, + "56": 1313.0, + "57": 1288.0, + "58": 1341.0, + "59": 1261.0, + "60": 1287.0, + "61": 1139.0, + "62": 1205.0, + "63": 1265.0, + "64": 1350.0, + "65": 1195.0, + "66": 1207.0, + "67": 1121.0, + "68": 1212.0, + "69": 1335.0, + "70": 1356.0, + "71": 1316.0, + "72": 1232.0, + "73": 1121.0, + "74": 1130.0, + "75": 1295.0, + "76": 1335.0, + "77": 1371.0, + "78": 1336.0, + "79": 1042.0, + "80": 1149.0, + "81": 1117.0, + "82": 1202.0, + "83": 1289.0, + "84": 1140.0, + "85": 1323.0, + "86": 1219.0, + "87": 1219.0, + "88": 1221.0, + "89": 1294.0, + "90": 1402.0, + "91": 1197.0, + "92": 1269.0, + "93": 1106.0, + "94": 960.0, + "95": 1192.0, + "96": 1253.0, + "97": 1148.0, + "98": 1218.0, + "99": 1273.0, + "100": 1249.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 994082816.0, + "2": 994053120.0, + "3": 994100224.0, + "4": 994081280.0, + "5": 994103808.0, + "6": 994043392.0, + "7": 994066944.0, + "8": 994074112.0, + "9": 994091008.0, + "10": 994104320.0, + "11": 994077696.0, + "12": 994044416.0, + "13": 994100736.0, + "14": 994012160.0, + "15": 994057216.0, + "16": 993989120.0, + "17": 994107904.0, + "18": 994082304.0, + "19": 994089472.0, + "20": 994008064.0, + "21": 994033152.0, + "22": 994105344.0, + "23": 994081280.0, + "24": 994021888.0, + "25": 994152960.0, + "26": 994058752.0, + "27": 994118144.0, + "28": 994044416.0, + "29": 994075648.0, + "30": 994039296.0, + "31": 994107392.0, + "32": 994037760.0, + "33": 994046976.0, + "34": 994015232.0, + "35": 994064384.0, + "36": 994078208.0, + "37": 994037248.0, + "38": 994120192.0, + "39": 994128896.0, + "40": 994016768.0, + "41": 994044928.0, + "42": 994063872.0, + "43": 994075648.0, + "44": 994180096.0, + "45": 994053632.0, + "46": 994070016.0, + "47": 994091520.0, + "48": 994076672.0, + "49": 994042368.0, + "50": 994061312.0, + "51": 994132992.0, + "52": 994076160.0, + "53": 994139136.0, + "54": 994086400.0, + "55": 994076160.0, + "56": 994066944.0, + "57": 994113536.0, + "58": 994111488.0, + "59": 994096128.0, + "60": 994060288.0, + "61": 994060800.0, + "62": 994054656.0, + "63": 994068992.0, + "64": 994058752.0, + "65": 994064896.0, + "66": 994074624.0, + "67": 994061824.0, + "68": 994071552.0, + "69": 994058240.0, + "70": 994103808.0, + "71": 994077184.0, + "72": 994002944.0, + "73": 994104320.0, + "74": 994116608.0, + "75": 994081792.0, + "76": 994104320.0, + "77": 994054656.0, + "78": 994114048.0, + "79": 994085376.0, + "80": 994039296.0, + "81": 994073600.0, + "82": 994020864.0, + "83": 994123776.0, + "84": 994103296.0, + "85": 994070528.0, + "86": 994070016.0, + "87": 994093056.0, + "88": 994079232.0, + "89": 994066432.0, + "90": 994060800.0, + "91": 994116096.0, + "92": 994098176.0, + "93": 994076672.0, + "94": 994083840.0, + "95": 994082816.0, + "96": 994086400.0, + "97": 994094080.0, + "98": 994070016.0, + "99": 994088448.0, + "100": 994124800.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3209166336.0, + "2": 3482067456.0, + "3": 3514878464.0, + "4": 3514878464.0, + "5": 3515977728.0, + "6": 3515977728.0, + "7": 3515977728.0, + "8": 3515977728.0, + "9": 3515977728.0, + "10": 3519236608.0, + "11": 3519236608.0, + "12": 3519236608.0, + "13": 3519236608.0, + "14": 3519236608.0, + "15": 3519236608.0, + "16": 3519236608.0, + "17": 3519236608.0, + "18": 3519236608.0, + "19": 3519236608.0, + "20": 3519236608.0, + "21": 3519236608.0, + "22": 3519236608.0, + "23": 3519236608.0, + "24": 3519236608.0, + "25": 3549031424.0, + "26": 3549031424.0, + "27": 3549031424.0, + "28": 3549031424.0, + "29": 3549031424.0, + "30": 3549031424.0, + "31": 3549031424.0, + "32": 3549031424.0, + "33": 3549031424.0, + "34": 3549031424.0, + "35": 3549031424.0, + "36": 3549031424.0, + "37": 3549031424.0, + "38": 3549031424.0, + "39": 3549031424.0, + "40": 3549031424.0, + "41": 3549031424.0, + "42": 3549031424.0, + "43": 3549031424.0, + "44": 3560927744.0, + "45": 3560927744.0, + "46": 3560927744.0, + "47": 3560927744.0, + "48": 3560927744.0, + "49": 3560927744.0, + "50": 3560927744.0, + "51": 3560927744.0, + "52": 3560927744.0, + "53": 3560927744.0, + "54": 3560927744.0, + "55": 3560927744.0, + "56": 3560927744.0, + "57": 3560927744.0, + "58": 3560927744.0, + "59": 3560927744.0, + "60": 3560927744.0, + "61": 3560927744.0, + "62": 3560927744.0, + "63": 3560927744.0, + "64": 3560927744.0, + "65": 3560927744.0, + "66": 3560927744.0, + "67": 3560927744.0, + "68": 3560927744.0, + "69": 3560927744.0, + "70": 3560927744.0, + "71": 3560927744.0, + "72": 3560927744.0, + "73": 3560927744.0, + "74": 3560927744.0, + "75": 3560927744.0, + "76": 3560927744.0, + "77": 3560927744.0, + "78": 3560927744.0, + "79": 3560927744.0, + "80": 3560927744.0, + "81": 3560927744.0, + "82": 3560927744.0, + "83": 3560927744.0, + "84": 3560927744.0, + "85": 3560927744.0, + "86": 3560927744.0, + "87": 3560927744.0, + "88": 3560927744.0, + "89": 3560927744.0, + "90": 3560927744.0, + "91": 3560927744.0, + "92": 3560927744.0, + "93": 3560927744.0, + "94": 3560927744.0, + "95": 3560927744.0, + "96": 3560927744.0, + "97": 3560927744.0, + "98": 3560927744.0, + "99": 3560927744.0, + "100": 3560927744.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 12.93942, + "2": 0.24599, + "3": 0.18905, + "4": 0.15958, + "5": 0.17376, + "6": 0.15827, + "7": 0.1625, + "8": 0.15602, + "9": 0.14535, + "10": 0.15058, + "11": 0.15764, + "12": 0.14977, + "13": 0.14045, + "14": 0.14809, + "15": 0.14641, + "16": 0.14226, + "17": 0.14811, + "18": 0.14049, + "19": 0.14226, + "20": 0.14343, + "21": 0.13924, + "22": 0.13727, + "23": 0.14079, + "24": 0.13602, + "25": 0.1322, + "26": 0.14315, + "27": 0.1347, + "28": 0.13221, + "29": 0.14595, + "30": 0.13083, + "31": 0.13326, + "32": 0.14065, + "33": 0.1383, + "34": 0.12953, + "35": 0.12541, + "36": 0.13129, + "37": 0.13317, + "38": 0.13535, + "39": 0.14664, + "40": 0.13368, + "41": 0.13115, + "42": 0.13308, + "43": 0.14022, + "44": 0.12946, + "45": 0.134, + "46": 0.12714, + "47": 0.13354, + "48": 0.13449, + "49": 0.13041, + "50": 0.13278, + "51": 0.14094, + "52": 0.12708, + "53": 0.13344, + "54": 0.13202, + "55": 0.13136, + "56": 0.13508, + "57": 0.13876, + "58": 0.13736, + "59": 0.12763, + "60": 0.13185, + "61": 0.12865, + "62": 0.13343, + "63": 0.13403, + "64": 0.12891, + "65": 0.13097, + "66": 0.12741, + "67": 0.13812, + "68": 0.13131, + "69": 0.13389, + "70": 0.13833, + "71": 0.12822, + "72": 0.12851, + "73": 0.13747, + "74": 0.13403, + "75": 0.12846, + "76": 0.13178, + "77": 0.12922, + "78": 0.12906, + "79": 0.12676, + "80": 0.13361, + "81": 0.12867, + "82": 0.1295, + "83": 0.12961, + "84": 0.12795, + "85": 0.13547, + "86": 0.13067, + "87": 0.13455, + "88": 0.13573, + "89": 0.12632, + "90": 0.13428, + "91": 0.13373, + "92": 0.12985, + "93": 0.1291, + "94": 0.12972, + "95": 0.13089, + "96": 0.13658, + "97": 0.12767, + "98": 0.14125, + "99": 0.13279, + "100": 0.12715 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml new file mode 100644 index 00000000000..1d0ef19232e --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml @@ -0,0 +1,66 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + NCCL_ALGO: Ring + CUBLAS_WORKSPACE_CONFIG: :4096:8 +MODEL_ARGS: + --num-layers: 12 + --hidden-size: 512 + --num-attention-heads: 8 + --log-params-norm: true + --log-num-zeros-in-grad: true + --log-validation-ppl-to-tensorboard: true + --log-timers-to-tensorboard: true + --tensorboard-dir: ${TENSORBOARD_PATH} + --micro-batch-size: 4 + --global-batch-size: 32 + --seq-length: 1024 + --max-position-embeddings: 1024 + --disable-bias-linear: true + --train-iters: 100 + --timing-log-level: 0 + --lr-decay-iters: 320000 + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --split: 949,50,1 + --distributed-backend: nccl + --lr: 0.00015 + --lr-decay-style: cosine + --min-lr: 1.0e-5 + --weight-decay: 1e-2 + --clip-grad: 1.0 + --lr-warmup-fraction: .01 + --log-interval: 1 + --save-interval: 50 + --eval-interval: 1000 + --eval-iters: 10 + --transformer-impl: transformer_engine + --tensor-model-parallel-size: 1 + --pipeline-model-parallel-size: 1 + --expert-model-parallel-size: 8 + --num-experts: 8 + --moe-token-dispatcher-type: allgather + --moe-router-load-balancing-type: aux_loss + --moe-router-topk: 2 + --moe-router-dtype: fp32 + --moe-ffn-hidden-size: 1024 + --moe-grouped-gemm: true + --ckpt-fully-parallel-load: true + --deterministic-mode: true + --no-gradient-accumulation-fusion: true + --attention-softmax-in-fp32: true + --use-checkpoint-opt_param-scheduler: true + --use-mcore-models: true + --ckpt-format: torch_dist + --data-cache-path: ${DATA_CACHE_PATH} + --bf16: true + --no-bias-gelu-fusion: true + --log-memory-to-tensorboard: true + --optimizer: dist_muon + --muon-momentum: 0.9 + --muon-extra-scale-factor: 0.2 + --muon-scale-mode: spectral +TEST_TYPE: ckpt-resume diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..d68eac84aa6 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/golden_values_dev_dgx_h100.json @@ -0,0 +1,537 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.81131, + "2": 10.83052, + "3": 10.82057, + "4": 10.81299, + "5": 10.84355, + "6": 10.84741, + "7": 10.85351, + "8": 10.83654, + "9": 10.84645, + "10": 10.78233, + "11": 10.85218, + "12": 10.86312, + "13": 10.85469, + "14": 10.88472, + "15": 10.87748, + "16": 10.84625, + "17": 10.83069, + "18": 10.86639, + "19": 10.84906, + "20": 10.84503, + "21": 10.84773, + "22": 10.79696, + "23": 10.88222, + "24": 10.8328, + "25": 10.82458, + "26": 10.84297, + "27": 10.85322, + "28": 10.877, + "29": 10.86421, + "30": 10.81318, + "31": 10.78757, + "32": 10.85489, + "33": 10.85622, + "34": 10.84912, + "35": 10.83743, + "36": 10.80413, + "37": 10.83824, + "38": 10.80517, + "39": 10.84145, + "40": 10.8035, + "41": 10.84038, + "42": 10.84445, + "43": 10.80969, + "44": 10.80245, + "45": 10.78719, + "46": 10.80835, + "47": 10.81707, + "48": 10.80285, + "49": 10.78195, + "50": 10.80261, + "51": 10.82279, + "52": 10.80311, + "53": 10.83245, + "54": 10.81518, + "55": 10.8259, + "56": 10.77718, + "57": 10.75221, + "58": 10.80764, + "59": 10.79119, + "60": 10.73952, + "61": 10.8001, + "62": 10.81329, + "63": 10.72056, + "64": 10.7858, + "65": 10.68952, + "66": 10.76117, + "67": 10.73373, + "68": 10.80228, + "69": 10.78363, + "70": 10.77642, + "71": 10.76616, + "72": 10.73585, + "73": 10.72985, + "74": 10.62227, + "75": 10.69038, + "76": 10.65462, + "77": 10.82185, + "78": 10.76375, + "79": 10.70489, + "80": 10.69397, + "81": 10.72436, + "82": 10.74224, + "83": 10.66825, + "84": 10.69825, + "85": 10.71453, + "86": 10.6386, + "87": 10.7184, + "88": 10.73501, + "89": 10.7146, + "90": 10.74633, + "91": 10.64923, + "92": 10.64682, + "93": 10.60192, + "94": 10.53301, + "95": 10.66154, + "96": 10.6726, + "97": 10.61394, + "98": 10.68486, + "99": 10.52044, + "100": 10.61569 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1216.0, + "2": 1361.0, + "3": 1297.0, + "4": 1234.0, + "5": 1371.0, + "6": 1490.0, + "7": 1250.0, + "8": 1336.0, + "9": 1341.0, + "10": 1267.0, + "11": 1273.0, + "12": 1210.0, + "13": 1226.0, + "14": 1433.0, + "15": 1235.0, + "16": 1174.0, + "17": 1327.0, + "18": 1336.0, + "19": 1257.0, + "20": 1235.0, + "21": 1319.0, + "22": 1171.0, + "23": 1345.0, + "24": 1332.0, + "25": 1257.0, + "26": 1210.0, + "27": 1365.0, + "28": 1404.0, + "29": 1274.0, + "30": 1188.0, + "31": 1145.0, + "32": 1314.0, + "33": 1305.0, + "34": 1152.0, + "35": 1224.0, + "36": 1206.0, + "37": 1248.0, + "38": 1368.0, + "39": 1545.0, + "40": 1271.0, + "41": 1388.0, + "42": 1192.0, + "43": 1144.0, + "44": 1229.0, + "45": 1172.0, + "46": 1169.0, + "47": 1310.0, + "48": 1085.0, + "49": 1151.0, + "50": 1199.0, + "51": 1280.0, + "52": 1203.0, + "53": 1365.0, + "54": 1230.0, + "55": 1202.0, + "56": 1298.0, + "57": 1361.0, + "58": 1299.0, + "59": 1247.0, + "60": 1239.0, + "61": 1151.0, + "62": 1215.0, + "63": 1183.0, + "64": 1323.0, + "65": 1172.0, + "66": 1159.0, + "67": 1213.0, + "68": 1241.0, + "69": 1290.0, + "70": 1313.0, + "71": 1275.0, + "72": 1200.0, + "73": 1086.0, + "74": 1107.0, + "75": 1322.0, + "76": 1340.0, + "77": 1340.0, + "78": 1281.0, + "79": 1061.0, + "80": 1193.0, + "81": 1174.0, + "82": 1245.0, + "83": 1294.0, + "84": 1222.0, + "85": 1346.0, + "86": 1190.0, + "87": 1177.0, + "88": 1221.0, + "89": 1238.0, + "90": 1398.0, + "91": 1162.0, + "92": 1279.0, + "93": 1137.0, + "94": 956.0, + "95": 1273.0, + "96": 1243.0, + "97": 1154.0, + "98": 1219.0, + "99": 1212.0, + "100": 1306.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 1096900608.0, + "2": 1096870912.0, + "3": 1096918016.0, + "4": 1096898048.0, + "5": 1096921088.0, + "6": 1096861696.0, + "7": 1096884736.0, + "8": 1096892928.0, + "9": 1096907776.0, + "10": 1096920576.0, + "11": 1096896512.0, + "12": 1096866816.0, + "13": 1096919040.0, + "14": 1096828416.0, + "15": 1096873984.0, + "16": 1096805888.0, + "17": 1096926208.0, + "18": 1096899584.0, + "19": 1096906240.0, + "20": 1096827904.0, + "21": 1096848384.0, + "22": 1096922624.0, + "23": 1096899072.0, + "24": 1096839680.0, + "25": 1096972800.0, + "26": 1096878080.0, + "27": 1096931840.0, + "28": 1096861696.0, + "29": 1096894464.0, + "30": 1096858624.0, + "31": 1096924160.0, + "32": 1096856576.0, + "33": 1096865792.0, + "34": 1096830976.0, + "35": 1096882688.0, + "36": 1096896000.0, + "37": 1096853504.0, + "38": 1096934912.0, + "39": 1096948736.0, + "40": 1096834048.0, + "41": 1096862208.0, + "42": 1096881152.0, + "43": 1096893440.0, + "44": 1096998400.0, + "45": 1096870400.0, + "46": 1096885248.0, + "47": 1096910336.0, + "48": 1096893952.0, + "49": 1096860672.0, + "50": 1096880640.0, + "51": 1096951808.0, + "52": 1096895488.0, + "53": 1096955392.0, + "54": 1096906752.0, + "55": 1096894976.0, + "56": 1096885760.0, + "57": 1096930816.0, + "58": 1096929280.0, + "59": 1096914432.0, + "60": 1096879104.0, + "61": 1096877056.0, + "62": 1096873472.0, + "63": 1096887808.0, + "64": 1096876544.0, + "65": 1096882176.0, + "66": 1096889344.0, + "67": 1096878592.0, + "68": 1096889856.0, + "69": 1096877568.0, + "70": 1096923648.0, + "71": 1096896512.0, + "72": 1096820736.0, + "73": 1096922624.0, + "74": 1096936960.0, + "75": 1096899584.0, + "76": 1096921600.0, + "77": 1096873984.0, + "78": 1096931328.0, + "79": 1096902144.0, + "80": 1096859136.0, + "81": 1096889856.0, + "82": 1096840704.0, + "83": 1096940032.0, + "84": 1096921088.0, + "85": 1096888320.0, + "86": 1096886784.0, + "87": 1096909312.0, + "88": 1096896000.0, + "89": 1096885760.0, + "90": 1096876032.0, + "91": 1096931328.0, + "92": 1096915968.0, + "93": 1096893440.0, + "94": 1096902656.0, + "95": 1096904704.0, + "96": 1096903168.0, + "97": 1096911360.0, + "98": 1096889856.0, + "99": 1096905216.0, + "100": 1096941568.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 3260276224.0, + "2": 3584739840.0, + "3": 3617909248.0, + "4": 3617909248.0, + "5": 3617909248.0, + "6": 3617909248.0, + "7": 3617909248.0, + "8": 3617909248.0, + "9": 3617909248.0, + "10": 3620101120.0, + "11": 3620101120.0, + "12": 3620101120.0, + "13": 3620101120.0, + "14": 3620101120.0, + "15": 3620101120.0, + "16": 3620101120.0, + "17": 3620950528.0, + "18": 3620950528.0, + "19": 3620950528.0, + "20": 3620950528.0, + "21": 3620950528.0, + "22": 3620950528.0, + "23": 3620950528.0, + "24": 3620950528.0, + "25": 3649420288.0, + "26": 3649420288.0, + "27": 3649420288.0, + "28": 3649420288.0, + "29": 3649420288.0, + "30": 3649420288.0, + "31": 3649420288.0, + "32": 3649420288.0, + "33": 3649420288.0, + "34": 3649420288.0, + "35": 3649420288.0, + "36": 3649420288.0, + "37": 3649420288.0, + "38": 3649420288.0, + "39": 3649420288.0, + "40": 3649420288.0, + "41": 3649420288.0, + "42": 3649420288.0, + "43": 3649420288.0, + "44": 3666221056.0, + "45": 3666221056.0, + "46": 3666221056.0, + "47": 3666221056.0, + "48": 3666221056.0, + "49": 3666221056.0, + "50": 3666221056.0, + "51": 3666221056.0, + "52": 3666221056.0, + "53": 3666221056.0, + "54": 3666221056.0, + "55": 3666221056.0, + "56": 3666221056.0, + "57": 3666221056.0, + "58": 3666221056.0, + "59": 3666221056.0, + "60": 3666221056.0, + "61": 3666221056.0, + "62": 3666221056.0, + "63": 3666221056.0, + "64": 3666221056.0, + "65": 3666221056.0, + "66": 3666221056.0, + "67": 3666221056.0, + "68": 3666221056.0, + "69": 3666221056.0, + "70": 3666221056.0, + "71": 3666221056.0, + "72": 3666221056.0, + "73": 3666221056.0, + "74": 3666221056.0, + "75": 3666221056.0, + "76": 3666221056.0, + "77": 3666221056.0, + "78": 3666221056.0, + "79": 3666221056.0, + "80": 3666221056.0, + "81": 3666221056.0, + "82": 3666221056.0, + "83": 3666221056.0, + "84": 3666221056.0, + "85": 3666221056.0, + "86": 3666221056.0, + "87": 3666221056.0, + "88": 3666221056.0, + "89": 3666221056.0, + "90": 3666221056.0, + "91": 3666221056.0, + "92": 3666221056.0, + "93": 3666221056.0, + "94": 3666221056.0, + "95": 3666221056.0, + "96": 3666221056.0, + "97": 3666221056.0, + "98": 3666221056.0, + "99": 3666221056.0, + "100": 3666221056.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 11.88847, + "2": 0.26215, + "3": 0.27604, + "4": 0.224, + "5": 0.22208, + "6": 0.20054, + "7": 0.21235, + "8": 0.20575, + "9": 0.1909, + "10": 0.19325, + "11": 0.19687, + "12": 0.19057, + "13": 0.20285, + "14": 0.19232, + "15": 0.20734, + "16": 0.18953, + "17": 0.18912, + "18": 0.1879, + "19": 0.19392, + "20": 0.18511, + "21": 0.18156, + "22": 0.17441, + "23": 0.19045, + "24": 0.18569, + "25": 0.17676, + "26": 0.18894, + "27": 0.17894, + "28": 0.18713, + "29": 0.19301, + "30": 0.17746, + "31": 0.18105, + "32": 0.18364, + "33": 0.18706, + "34": 0.18723, + "35": 0.18693, + "36": 0.18302, + "37": 0.18341, + "38": 0.18438, + "39": 0.19158, + "40": 0.1883, + "41": 0.18551, + "42": 0.18393, + "43": 0.18993, + "44": 0.17986, + "45": 0.18314, + "46": 0.1799, + "47": 0.18447, + "48": 0.18728, + "49": 0.17581, + "50": 0.18331, + "51": 0.1808, + "52": 0.17423, + "53": 0.18081, + "54": 0.17774, + "55": 0.17619, + "56": 0.18141, + "57": 0.17723, + "58": 0.17154, + "59": 0.17626, + "60": 0.17398, + "61": 0.17639, + "62": 0.17865, + "63": 0.17601, + "64": 0.17189, + "65": 0.17678, + "66": 0.17525, + "67": 0.17732, + "68": 0.17371, + "69": 0.17928, + "70": 0.18002, + "71": 0.17642, + "72": 0.17759, + "73": 0.17701, + "74": 0.1779, + "75": 0.17328, + "76": 0.17709, + "77": 0.18374, + "78": 0.17122, + "79": 0.17448, + "80": 0.17649, + "81": 0.17913, + "82": 0.17842, + "83": 0.17672, + "84": 0.17358, + "85": 0.18273, + "86": 0.17787, + "87": 0.1921, + "88": 0.18275, + "89": 0.1784, + "90": 0.18873, + "91": 0.17603, + "92": 0.17536, + "93": 0.17978, + "94": 0.17667, + "95": 0.17869, + "96": 0.17776, + "97": 0.17418, + "98": 0.18666, + "99": 0.17937, + "100": 0.17448 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml new file mode 100644 index 00000000000..81b023bd86e --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_muon/model_config.yaml @@ -0,0 +1,67 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + NCCL_ALGO: Ring + CUBLAS_WORKSPACE_CONFIG: :4096:8 +MODEL_ARGS: + --num-layers: 12 + --hidden-size: 512 + --num-attention-heads: 8 + --log-params-norm: true + --log-num-zeros-in-grad: true + --log-validation-ppl-to-tensorboard: true + --log-timers-to-tensorboard: true + --tensorboard-dir: ${TENSORBOARD_PATH} + --micro-batch-size: 4 + --global-batch-size: 32 + --seq-length: 1024 + --max-position-embeddings: 1024 + --disable-bias-linear: true + --train-iters: 100 + --timing-log-level: 0 + --lr-decay-iters: 320000 + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --split: 949,50,1 + --distributed-backend: nccl + --lr: 0.00015 + --lr-decay-style: cosine + --min-lr: 1.0e-5 + --weight-decay: 1e-2 + --clip-grad: 1.0 + --lr-warmup-fraction: .01 + --log-interval: 1 + --save-interval: 50 + --eval-interval: 1000 + --eval-iters: 10 + --transformer-impl: transformer_engine + --tensor-model-parallel-size: 1 + --pipeline-model-parallel-size: 1 + --expert-model-parallel-size: 8 + --num-experts: 8 + --moe-token-dispatcher-type: allgather + --moe-router-load-balancing-type: aux_loss + --moe-router-topk: 2 + --moe-router-dtype: fp32 + --moe-ffn-hidden-size: 1024 + --moe-grouped-gemm: true + --ckpt-fully-parallel-load: true + --deterministic-mode: true + --no-gradient-accumulation-fusion: true + --attention-softmax-in-fp32: true + --use-checkpoint-opt_param-scheduler: true + --use-mcore-models: true + --ckpt-format: torch_dist + --ckpt-assume-constant-structure: true + --data-cache-path: ${DATA_CACHE_PATH} + --bf16: true + --no-bias-gelu-fusion: true + --log-memory-to-tensorboard: true + --optimizer: muon + --muon-momentum: 0.9 + --muon-extra-scale-factor: 0.2 + --muon-scale-mode: spectral +TEST_TYPE: ckpt-resume diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..e7f62bbe4af --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -0,0 +1,344 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.06715, + "2": 11.06051, + "3": 10.21154, + "4": 9.95175, + "5": 10.12622, + "6": 8.82146, + "7": 9.52879, + "8": 8.442, + "9": 7.84738, + "10": 7.07075, + "11": 9.31042, + "12": 9.16013, + "13": 7.87292, + "14": 8.2102, + "15": 8.22483, + "16": 8.17879, + "17": 8.21121, + "18": 7.50325, + "19": 8.08274, + "20": 7.62562, + "21": 7.95058, + "22": 7.29789, + "23": 7.93775, + "24": 7.44169, + "25": 8.23817, + "26": 7.74959, + "27": 7.69344, + "28": 7.65487, + "29": 7.75173, + "30": 7.56007, + "31": 7.81567, + "32": 6.46589, + "33": 7.20401, + "34": 7.77921, + "35": 7.72944, + "36": 6.71776, + "37": 8.08311, + "38": 7.6137, + "39": 7.96476, + "40": 7.50072, + "41": 7.50304, + "42": 6.11349, + "43": 7.59404, + "44": 7.91361, + "45": 6.83615, + "46": 7.41293, + "47": 7.79226, + "48": 7.87549, + "49": 7.58763, + "50": 6.84525 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 47165192.0, + "2": 46897912.0, + "3": 52684456.0, + "4": 297127552.0, + "5": 562950784.0, + "6": 668142144.0, + "7": 1027449536.0, + "8": 752259328.0, + "9": 830947776.0, + "10": 718307136.0, + "11": 823731840.0, + "12": 804867840.0, + "13": 639461056.0, + "14": 625408576.0, + "15": 716256960.0, + "16": 870866752.0, + "17": 673817856.0, + "18": 811900096.0, + "19": 892689024.0, + "20": 878114112.0, + "21": 666859968.0, + "22": 792718848.0, + "23": 783683200.0, + "24": 770686976.0, + "25": 651376640.0, + "26": 780070272.0, + "27": 801722496.0, + "28": 670273664.0, + "29": 647960768.0, + "30": 789867776.0, + "31": 801385856.0, + "32": 787688640.0, + "33": 783506816.0, + "34": 792837760.0, + "35": 776103936.0, + "36": 761920512.0, + "37": 775085824.0, + "38": 752868608.0, + "39": 754997184.0, + "40": 745075072.0, + "41": 713941440.0, + "42": 689968512.0, + "43": 663461824.0, + "44": 680285632.0, + "45": 644628992.0, + "46": 641672704.0, + "47": 642439616.0, + "48": 597700608.0, + "49": 603523520.0, + "50": 601014528.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 5290944000.0, + "2": 5291148800.0, + "3": 5291351552.0, + "4": 5290946048.0, + "5": 5291148800.0, + "6": 5291351552.0, + "7": 5291554304.0, + "8": 5291757056.0, + "9": 5291959808.0, + "10": 5292162560.0, + "11": 5292365312.0, + "12": 5292568064.0, + "13": 5292770816.0, + "14": 5292973568.0, + "15": 5293176320.0, + "16": 5293379072.0, + "17": 5293581824.0, + "18": 5293784576.0, + "19": 5293987328.0, + "20": 5294190080.0, + "21": 5294392832.0, + "22": 5294595584.0, + "23": 5294798336.0, + "24": 5295001088.0, + "25": 5295203840.0, + "26": 5295406592.0, + "27": 5295609344.0, + "28": 5295812096.0, + "29": 5296014848.0, + "30": 5296217600.0, + "31": 5296420352.0, + "32": 5296623104.0, + "33": 5296825856.0, + "34": 5297028608.0, + "35": 5297231360.0, + "36": 5297434112.0, + "37": 5297636864.0, + "38": 5297839616.0, + "39": 5298042368.0, + "40": 5298245120.0, + "41": 5298447872.0, + "42": 5298650624.0, + "43": 5298853376.0, + "44": 5299056128.0, + "45": 5299258880.0, + "46": 5299461632.0, + "47": 5299664384.0, + "48": 5299867136.0, + "49": 5300069888.0, + "50": 5300272640.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 6180783616.0, + "2": 8225679872.0, + "3": 8225679872.0, + "4": 8225679872.0, + "5": 8225679872.0, + "6": 8225679872.0, + "7": 8225679872.0, + "8": 8225679872.0, + "9": 8225679872.0, + "10": 8225679872.0, + "11": 8239991296.0, + "12": 8239991296.0, + "13": 8239991296.0, + "14": 8239991296.0, + "15": 8239991296.0, + "16": 8239991296.0, + "17": 8244914688.0, + "18": 8244914688.0, + "19": 8244914688.0, + "20": 8265598464.0, + "21": 8265598464.0, + "22": 8265598464.0, + "23": 8265598464.0, + "24": 8265598464.0, + "25": 8265598464.0, + "26": 8265598464.0, + "27": 8265598464.0, + "28": 8265598464.0, + "29": 8271664640.0, + "30": 8316803584.0, + "31": 8316803584.0, + "32": 8316803584.0, + "33": 8316803584.0, + "34": 8316803584.0, + "35": 8316803584.0, + "36": 8316803584.0, + "37": 8316803584.0, + "38": 8316803584.0, + "39": 8318923264.0, + "40": 8318923264.0, + "41": 8318923264.0, + "42": 8318923264.0, + "43": 8318923264.0, + "44": 8318923264.0, + "45": 8318923264.0, + "46": 8318923264.0, + "47": 8318923264.0, + "48": 8318923264.0, + "49": 8318923264.0, + "50": 8318923264.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.07395, + "2": 11.0927, + "3": 10.82648, + "4": 10.27524, + "5": 10.45343, + "6": 8.32789, + "7": 9.82687, + "8": 8.01561, + "9": 7.47686, + "10": 6.75778, + "11": 8.92977, + "12": 8.98867, + "13": 7.80263, + "14": 8.02637, + "15": 8.11184, + "16": 8.13967, + "17": 8.13444, + "18": 7.44744, + "19": 8.03657, + "20": 7.53993, + "21": 7.90129, + "22": 7.27518, + "23": 7.88304, + "24": 7.37567, + "25": 8.16836, + "26": 7.69935, + "27": 7.6262, + "28": 7.61271, + "29": 7.69819, + "30": 7.4848, + "31": 7.73967, + "32": 6.36884, + "33": 7.14295, + "34": 7.71844, + "35": 7.63485, + "36": 6.61195, + "37": 8.02821, + "38": 7.57841, + "39": 7.89473, + "40": 7.41461, + "41": 7.42116, + "42": 6.01344, + "43": 7.4906, + "44": 7.86418, + "45": 6.74814, + "46": 7.30484, + "47": 7.72617, + "48": 7.79074, + "49": 7.49049, + "50": 6.75504 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 90.97535, + "2": 4.15413, + "3": 4.25282, + "4": 5.50314, + "5": 4.36528, + "6": 4.16016, + "7": 4.60989, + "8": 3.68392, + "9": 3.70951, + "10": 3.66417, + "11": 3.64904, + "12": 3.66094, + "13": 3.68824, + "14": 3.64996, + "15": 3.64159, + "16": 3.68269, + "17": 3.66905, + "18": 4.10783, + "19": 3.63362, + "20": 3.65129, + "21": 3.6431, + "22": 3.64946, + "23": 3.6411, + "24": 3.59707, + "25": 3.55364, + "26": 3.61478, + "27": 3.59779, + "28": 3.58741, + "29": 3.62545, + "30": 3.63538, + "31": 3.58264, + "32": 3.65914, + "33": 3.62764, + "34": 3.61962, + "35": 3.57076, + "36": 3.59244, + "37": 3.68499, + "38": 3.6803, + "39": 3.5849, + "40": 3.59019, + "41": 3.62068, + "42": 3.69144, + "43": 3.71863, + "44": 3.67193, + "45": 3.65673, + "46": 3.66919, + "47": 3.58334, + "48": 3.57229, + "49": 3.66195, + "50": 3.64157 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml new file mode 100644 index 00000000000..c657b9087e7 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading/model_config.yaml @@ -0,0 +1,139 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 32 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: 0 + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION +MODEL_ARGS: + # Distributed args + --distributed-timeout-minutes: 60 + --tensor-model-parallel-size: 2 + --pipeline-model-parallel-size: 2 + --expert-model-parallel-size: 4 + --context-parallel-size: 1 + --expert-tensor-parallel-size: 1 + --use-distributed-optimizer: true + # NOTE: uncomment if TE >= 2.9.0 + # --overlap-grad-reduce: true + # --overlap-param-gather: true + # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN + --attention-backend: unfused # TODO: switch back to fused attention after fix + # Training args + --use-mcore-models: true + --sequence-parallel: true + --disable-bias-linear: true + --micro-batch-size: 1 + --global-batch-size: 8 + --train-iters: 50 + --exit-duration-in-mins: 230 + --no-check-for-nan-in-loss-and-grad: true + --no-rope-fusion: true + --cross-entropy-loss-fusion: true + --cross-entropy-fusion-impl: native + --manual-gc: true + --manual-gc-interval: 100 + --recompute-granularity: selective + --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" + --fine-grained-activation-offloading: true + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]" + # Transformer Engine args + --transformer-impl: transformer_engine + # Data args + --seq-length: 4096 + --data-cache-path: ${DATA_CACHE_PATH} + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --split: 949,50,1 + # Add network size args + --num-layers: 15 + --moe-layer-freq: ([0]*3+[1]*12) + --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6mL # Et*3|(tt|)*6mL + --hidden-size: 1024 + --ffn-hidden-size: 4096 + --num-attention-heads: 32 + --kv-channels: 128 + --max-position-embeddings: 4096 + --position-embedding-type: rope + --rotary-base: 10000 + --make-vocab-size-divisible-by: 3232 + --normalization: RMSNorm + --norm-epsilon: 1e-6 + --swiglu: true + --untie-embeddings-and-output-weights: true + --multi-latent-attention: true + # Comment out the following MTP args to disable MTP + --mtp-num-layers: 1 + --mtp-loss-scaling-factor: 0.1 + # Add regularization args + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --clip-grad: 1.0 + --weight-decay: 0.1 + --qk-layernorm: true + # Add learning rate args + --lr-warmup-fraction: .01 + --lr: 0.00015 + --min-lr: 1.0e-5 + --lr-decay-style: cosine + --adam-beta1: 0.9 + --adam-beta2: 0.95 + # Add MoE args + --num-experts: 32 + --moe-ffn-hidden-size: 1024 + --moe-shared-expert-intermediate-size: 1024 + --moe-router-load-balancing-type: seq_aux_loss + --moe-router-topk: 4 + --moe-token-dispatcher-type: alltoall + --moe-router-pre-softmax: true + --moe-grouped-gemm: true + --moe-aux-loss-coeff: 1e-4 + --moe-router-group-topk: 2 + --moe-router-num-groups: 4 + --moe-router-topk-scaling-factor: 2.0 + --moe-router-score-function: sigmoid + --moe-router-enable-expert-bias: true + --moe-router-bias-update-rate: 1e-3 + --moe-router-dtype: fp32 + --moe-permute-fusion: true + # Add MLA args + --q-lora-rank: 1536 + --kv-lora-rank: 512 + --qk-head-dim: 128 + --qk-pos-emb-head-dim: 64 + --v-head-dim: 128 + --rotary-scaling-factor: 40 + --mscale: 1.0 + --mscale-all-dim: 1.0 + # Add validation args + --eval-iters: 32 + --eval-interval: 200 + # Add checkpointing args + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 25 + # Add initialization args + --init-method-std: 0.02 + # Add logging args + --log-timers-to-tensorboard: true + --log-memory-to-tensorboard: true + --log-num-zeros-in-grad: true + --log-params-norm: true + --log-validation-ppl-to-tensorboard: true + --log-throughput: true + --log-interval: 1 + --logging-level: 40 + --tensorboard-dir: ${TENSORBOARD_PATH} + # Add mixed precision args + --bf16: true + --exit-interval: 50 + --overlap-moe-expert-parallel-comm: true +TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular +METRICS: + # - "iteration-time" + - "lm loss" + - "num-zeros" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" + - "mtp_1 loss" diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..f31e8584055 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/golden_values_dev_dgx_h100.json @@ -0,0 +1,287 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 11.01686, + "2": 11.06264, + "3": 10.17793, + "4": 10.86283, + "5": 9.81719, + "6": 9.10416, + "7": 9.61067, + "8": 8.39543, + "9": 7.79835, + "10": 7.15295, + "11": 9.06686, + "12": 12.40969, + "13": 8.05055, + "14": 8.2476, + "15": 8.25138, + "16": 8.32761, + "17": 8.33769, + "18": 7.57521, + "19": 8.18843, + "20": 7.70464, + "21": 8.00008, + "22": 7.35567, + "23": 7.9428, + "24": 7.49828, + "25": 8.31989, + "26": 7.79139, + "27": 7.72813, + "28": 7.70354, + "29": 7.77157, + "30": 7.56925, + "31": 7.85097, + "32": 6.53309, + "33": 7.24762, + "34": 7.79993, + "35": 7.74601, + "36": 6.74083, + "37": 8.15463, + "38": 7.62637, + "39": 7.97973, + "40": 7.52426, + "41": 7.52118, + "42": 6.11695, + "43": 7.60509, + "44": 7.96979, + "45": 6.84567, + "46": 7.4309, + "47": 7.82486, + "48": 7.87887, + "49": 7.59924, + "50": 6.85064 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 47167816.0, + "2": 46900776.0, + "3": 77860808.0, + "4": 237329376.0, + "5": 471709792.0, + "6": 558041536.0, + "7": 948826176.0, + "8": 723939584.0, + "9": 786891776.0, + "10": 734021888.0, + "11": 688478400.0, + "12": 553228736.0, + "13": 608009792.0, + "14": 741806976.0, + "15": 766532736.0, + "16": 685280512.0, + "17": 654899648.0, + "18": 730146112.0, + "19": 751163904.0, + "20": 884406592.0, + "21": 723541120.0, + "22": 805299648.0, + "23": 789975808.0, + "24": 610294016.0, + "25": 830610048.0, + "26": 824111232.0, + "27": 757678144.0, + "28": 774057088.0, + "29": 805232640.0, + "30": 770995712.0, + "31": 801384640.0, + "32": 790830656.0, + "33": 758341184.0, + "34": 726777280.0, + "35": 750934144.0, + "36": 717880064.0, + "37": 740480704.0, + "38": 724556544.0, + "39": 710957376.0, + "40": 716765760.0, + "41": 531516928.0, + "42": 658507328.0, + "43": 676045888.0, + "44": 680286208.0, + "45": 606880576.0, + "46": 641672384.0, + "47": 633002368.0, + "48": 607136576.0, + "49": 430551968.0, + "50": 563263808.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4315544064.0, + "2": 4315545600.0, + "3": 4315545600.0, + "4": 4315545600.0, + "5": 4315545600.0, + "6": 4315545600.0, + "7": 4315545600.0, + "8": 4315545600.0, + "9": 4315545600.0, + "10": 4315545600.0, + "11": 4315545600.0, + "12": 4315545600.0, + "13": 4315545600.0, + "14": 4315545600.0, + "15": 4315545600.0, + "16": 4315545600.0, + "17": 4315545600.0, + "18": 4315545600.0, + "19": 4315545600.0, + "20": 4315545600.0, + "21": 4315545600.0, + "22": 4315545600.0, + "23": 4315545600.0, + "24": 4315545600.0, + "25": 4315545600.0, + "26": 4315545600.0, + "27": 4315545600.0, + "28": 4315545600.0, + "29": 4315545600.0, + "30": 4315545600.0, + "31": 4315545600.0, + "32": 4315545600.0, + "33": 4315545600.0, + "34": 4315545600.0, + "35": 4315545600.0, + "36": 4315545600.0, + "37": 4315545600.0, + "38": 4315545600.0, + "39": 4315545600.0, + "40": 4315545600.0, + "41": 4315545600.0, + "42": 4315545600.0, + "43": 4315545600.0, + "44": 4315545600.0, + "45": 4315545600.0, + "46": 4315545600.0, + "47": 4315545600.0, + "48": 4315545600.0, + "49": 4315545600.0, + "50": 4315545600.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 4919527424.0, + "2": 5861408768.0, + "3": 5861408768.0, + "4": 5865549824.0, + "5": 5865549824.0, + "6": 5865549824.0, + "7": 5865549824.0, + "8": 5865549824.0, + "9": 5865549824.0, + "10": 5865549824.0, + "11": 5865549824.0, + "12": 5865549824.0, + "13": 5865549824.0, + "14": 5865549824.0, + "15": 5865549824.0, + "16": 5865549824.0, + "17": 5865549824.0, + "18": 5865549824.0, + "19": 5866154496.0, + "20": 5866154496.0, + "21": 5866154496.0, + "22": 5866154496.0, + "23": 5866154496.0, + "24": 5866154496.0, + "25": 5866154496.0, + "26": 5866154496.0, + "27": 5866154496.0, + "28": 5866154496.0, + "29": 5866154496.0, + "30": 5866154496.0, + "31": 5866154496.0, + "32": 5866154496.0, + "33": 5866154496.0, + "34": 5866154496.0, + "35": 5866154496.0, + "36": 5866154496.0, + "37": 5866154496.0, + "38": 5866154496.0, + "39": 5866154496.0, + "40": 5866154496.0, + "41": 5866154496.0, + "42": 5866154496.0, + "43": 5866154496.0, + "44": 5866154496.0, + "45": 5866154496.0, + "46": 5866154496.0, + "47": 5866154496.0, + "48": 5866154496.0, + "49": 5866154496.0, + "50": 5866154496.0 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 50, + "step_interval": 1, + "values": { + "1": 86.37903, + "2": 4.30499, + "3": 5.51749, + "4": 4.16842, + "5": 5.35652, + "6": 3.7018, + "7": 3.68633, + "8": 3.75304, + "9": 3.67596, + "10": 3.70408, + "11": 3.70621, + "12": 3.71713, + "13": 3.73785, + "14": 3.64923, + "15": 3.63825, + "16": 3.64129, + "17": 3.71791, + "18": 3.69956, + "19": 4.27786, + "20": 4.04035, + "21": 3.67423, + "22": 3.66455, + "23": 3.67758, + "24": 4.16675, + "25": 3.71546, + "26": 3.71205, + "27": 3.71193, + "28": 3.60188, + "29": 3.69233, + "30": 3.68235, + "31": 3.69734, + "32": 3.69173, + "33": 3.64974, + "34": 3.73647, + "35": 3.68627, + "36": 3.70357, + "37": 3.71094, + "38": 3.72508, + "39": 3.70553, + "40": 3.6995, + "41": 3.61312, + "42": 3.63624, + "43": 3.68714, + "44": 3.70371, + "45": 3.67257, + "46": 3.73701, + "47": 3.69639, + "48": 3.65815, + "49": 3.63754, + "50": 3.71569 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml new file mode 100644 index 00000000000..5b177ed116d --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading/model_config.yaml @@ -0,0 +1,134 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + NCCL_NVLS_ENABLE: 0 + PYTHONWARNINGS: ignore + NCCL_DEBUG: VERSION +MODEL_ARGS: + # Distributed args + --distributed-timeout-minutes: 60 + --tensor-model-parallel-size: 2 + --pipeline-model-parallel-size: 2 + --expert-model-parallel-size: 4 + --context-parallel-size: 1 + --expert-tensor-parallel-size: 1 + --use-distributed-optimizer: true + # NOTE: uncomment if TE >= 2.9.0 + # --overlap-grad-reduce: true + # --overlap-param-gather: true + # Use unfused attention since MLA with fused attention and deterministic mode leads to NaN + --attention-backend: unfused # TODO: switch back to fused attention after fix + # Training args + --use-mcore-models: true + --sequence-parallel: true + --disable-bias-linear: true + --micro-batch-size: 1 + --global-batch-size: 8 + --train-iters: 50 + --exit-duration-in-mins: 230 + --no-check-for-nan-in-loss-and-grad: true + --no-rope-fusion: true + --cross-entropy-loss-fusion: true + --cross-entropy-fusion-impl: native + --manual-gc: true + --manual-gc-interval: 100 + --recompute-granularity: selective + --recompute-modules: "[layernorm mla_up_proj mlp moe_act]" + --fine-grained-activation-offloading: true + --offload-modules: "[expert_fc1 moe_act attn_norm mlp_norm qkv_linear core_attn attn_proj]" + # Transformer Engine args + --transformer-impl: transformer_engine + # Data args + --seq-length: 4096 + --data-cache-path: ${DATA_CACHE_PATH} + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --split: 949,50,1 + # Add network size args + --num-layers: 15 + --moe-layer-freq: ([0]*3+[1]*12) + --pipeline-model-parallel-layout: Et*3\\|\\(tt\\|\\)*6L # Et*3|(tt|)*6L + --hidden-size: 1024 + --ffn-hidden-size: 4096 + --num-attention-heads: 32 + --kv-channels: 128 + --max-position-embeddings: 4096 + --position-embedding-type: rope + --rotary-base: 10000 + --make-vocab-size-divisible-by: 3232 + --normalization: RMSNorm + --norm-epsilon: 1e-6 + --swiglu: true + --untie-embeddings-and-output-weights: true + --multi-latent-attention: true + # Add regularization args + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --clip-grad: 1.0 + --weight-decay: 0.1 + --qk-layernorm: true + # Add learning rate args + --lr-warmup-fraction: .01 + --lr: 0.00015 + --min-lr: 1.0e-5 + --lr-decay-style: cosine + --adam-beta1: 0.9 + --adam-beta2: 0.95 + # Add MoE args + --num-experts: 32 + --moe-ffn-hidden-size: 1024 + --moe-shared-expert-intermediate-size: 1024 + --moe-router-load-balancing-type: seq_aux_loss + --moe-router-topk: 4 + --moe-token-dispatcher-type: alltoall + --moe-router-pre-softmax: true + --moe-grouped-gemm: true + --moe-aux-loss-coeff: 1e-4 + --moe-router-group-topk: 2 + --moe-router-num-groups: 4 + --moe-router-topk-scaling-factor: 2.0 + --moe-router-score-function: sigmoid + --moe-router-enable-expert-bias: true + --moe-router-bias-update-rate: 1e-3 + --moe-router-dtype: fp32 + --moe-permute-fusion: true + # Add MLA args + --q-lora-rank: 1536 + --kv-lora-rank: 512 + --qk-head-dim: 128 + --qk-pos-emb-head-dim: 64 + --v-head-dim: 128 + --rotary-scaling-factor: 40 + --mscale: 1.0 + --mscale-all-dim: 1.0 + # Add validation args + --eval-iters: 32 + --eval-interval: 200 + # Add checkpointing args + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --save-interval: 25 + # Add initialization args + --init-method-std: 0.02 + # Add logging args + --log-timers-to-tensorboard: true + --log-memory-to-tensorboard: true + --log-num-zeros-in-grad: true + --log-params-norm: true + --log-validation-ppl-to-tensorboard: true + --log-throughput: true + --log-interval: 1 + --logging-level: 40 + --tensorboard-dir: ${TENSORBOARD_PATH} + # Add mixed precision args + --bf16: true + --exit-interval: 50 +TEST_TYPE: regular # Usually ckpt-resume, but as a WAR to #513 set to regular +METRICS: + # - "iteration-time" + - "lm loss" + - "num-zeros" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json new file mode 100644 index 00000000000..309b2533461 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgx_h100.json @@ -0,0 +1,644 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93663, + "2": 10.9327, + "3": 10.94263, + "4": 10.94969, + "5": 10.95052, + "6": 10.94157, + "7": 10.94484, + "8": 10.93674, + "9": 10.94996, + "10": 10.93686, + "11": 10.94102, + "12": 10.93763, + "13": 10.9235, + "14": 10.93428, + "15": 10.88791, + "16": 10.87434, + "17": 10.86896, + "18": 10.86065, + "19": 10.86311, + "20": 10.78063, + "21": 10.73125, + "22": 10.60283, + "23": 10.73278, + "24": 10.61888, + "25": 10.55212, + "26": 10.62704, + "27": 10.6391, + "28": 10.5908, + "29": 10.59809, + "30": 10.37777, + "31": 10.1201, + "32": 10.46078, + "33": 10.45538, + "34": 10.20107, + "35": 10.25779, + "36": 10.20889, + "37": 10.33688, + "38": 10.16827, + "39": 10.40875, + "40": 10.05239, + "41": 10.09432, + "42": 10.17894, + "43": 9.74205, + "44": 9.8904, + "45": 9.74009, + "46": 9.72707, + "47": 10.09139, + "48": 9.75298, + "49": 9.40106, + "50": 9.83667, + "51": 9.77071, + "52": 9.65705, + "53": 10.03051, + "54": 9.87899, + "55": 9.79604, + "56": 9.52924, + "57": 9.36583, + "58": 9.75331, + "59": 9.48065, + "60": 9.40785, + "61": 9.60145, + "62": 9.90753, + "63": 9.2583, + "64": 9.68397, + "65": 8.80003, + "66": 9.60779, + "67": 9.25408, + "68": 9.71438, + "69": 9.71682, + "70": 9.6617, + "71": 9.52466, + "72": 9.47116, + "73": 9.38822, + "74": 8.80223, + "75": 9.33966, + "76": 8.93574, + "77": 9.99333, + "78": 9.64731, + "79": 9.28114, + "80": 9.29588, + "81": 9.39589, + "82": 9.60893, + "83": 9.21629, + "84": 9.33891, + "85": 9.52979, + "86": 8.95817, + "87": 9.51641, + "88": 9.68228, + "89": 9.50664, + "90": 9.75348, + "91": 9.23465, + "92": 9.25972, + "93": 8.94517, + "94": 8.69188, + "95": 9.44591, + "96": 9.4101, + "97": 9.20087, + "98": 9.58175, + "99": 8.75818, + "100": 9.29466 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22750260.0, + "2": 22953110.0, + "3": 22604450.0, + "4": 23266322.0, + "5": 22735560.0, + "6": 23061920.0, + "7": 22793342.0, + "8": 22960820.0, + "9": 22865664.0, + "10": 22950364.0, + "11": 22499674.0, + "12": 22456088.0, + "13": 22948060.0, + "14": 22384512.0, + "15": 22846272.0, + "16": 22856858.0, + "17": 22836412.0, + "18": 22590058.0, + "19": 22627048.0, + "20": 22712308.0, + "21": 22762624.0, + "22": 22816888.0, + "23": 22545124.0, + "24": 22794440.0, + "25": 22841936.0, + "26": 22549680.0, + "27": 22464820.0, + "28": 22453684.0, + "29": 22534640.0, + "30": 22636152.0, + "31": 22989488.0, + "32": 22594070.0, + "33": 22566010.0, + "34": 22855504.0, + "35": 22813688.0, + "36": 22595396.0, + "37": 22499360.0, + "38": 22926126.0, + "39": 22825392.0, + "40": 22675666.0, + "41": 22671586.0, + "42": 22682140.0, + "43": 23013940.0, + "44": 22764458.0, + "45": 22678992.0, + "46": 22915276.0, + "47": 22642868.0, + "48": 22954190.0, + "49": 23786668.0, + "50": 22934008.0, + "51": 23866222.0, + "52": 23807290.0, + "53": 24007532.0, + "54": 22871610.0, + "55": 23571284.0, + "56": 23954310.0, + "57": 24211632.0, + "58": 23914404.0, + "59": 23771838.0, + "60": 23813560.0, + "61": 23797288.0, + "62": 23739984.0, + "63": 23916692.0, + "64": 23895952.0, + "65": 24150562.0, + "66": 23796504.0, + "67": 25032232.0, + "68": 23673188.0, + "69": 23648580.0, + "70": 23903504.0, + "71": 24864636.0, + "72": 24767108.0, + "73": 24850612.0, + "74": 24132990.0, + "75": 24146528.0, + "76": 25025540.0, + "77": 24358472.0, + "78": 24910064.0, + "79": 23810516.0, + "80": 24821440.0, + "81": 25020512.0, + "82": 23851244.0, + "83": 24961024.0, + "84": 25144020.0, + "85": 24823608.0, + "86": 23153096.0, + "87": 24850204.0, + "88": 24749150.0, + "89": 22505554.0, + "90": 24059620.0, + "91": 23839038.0, + "92": 23874568.0, + "93": 24769548.0, + "94": 23992452.0, + "95": 25189838.0, + "96": 23909262.0, + "97": 24713068.0, + "98": 23832506.0, + "99": 23983474.0, + "100": 24101108.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 763142656.0, + "2": 778734592.0, + "3": 772525056.0, + "4": 803593216.0, + "5": 803593216.0, + "6": 803593216.0, + "7": 801299456.0, + "8": 803593216.0, + "9": 801840128.0, + "10": 803593216.0, + "11": 802987008.0, + "12": 803593216.0, + "13": 802987008.0, + "14": 801299456.0, + "15": 803593216.0, + "16": 801840128.0, + "17": 803593216.0, + "18": 802987008.0, + "19": 801299456.0, + "20": 803593216.0, + "21": 801299456.0, + "22": 803593216.0, + "23": 801299456.0, + "24": 803593216.0, + "25": 801299456.0, + "26": 803593216.0, + "27": 801299456.0, + "28": 803593216.0, + "29": 801299456.0, + "30": 803593216.0, + "31": 801299456.0, + "32": 803593216.0, + "33": 801840128.0, + "34": 803593216.0, + "35": 801840128.0, + "36": 803593216.0, + "37": 802987008.0, + "38": 801299456.0, + "39": 803593216.0, + "40": 801299456.0, + "41": 803593216.0, + "42": 801840128.0, + "43": 803593216.0, + "44": 801840128.0, + "45": 803593216.0, + "46": 801840128.0, + "47": 803593216.0, + "48": 801840128.0, + "49": 803593216.0, + "50": 801840128.0, + "51": 801299456.0, + "52": 803593216.0, + "53": 801299456.0, + "54": 803593216.0, + "55": 801840128.0, + "56": 803593216.0, + "57": 801840128.0, + "58": 803593216.0, + "59": 801840128.0, + "60": 803593216.0, + "61": 801299456.0, + "62": 803593216.0, + "63": 801299456.0, + "64": 802987008.0, + "65": 803593216.0, + "66": 801299456.0, + "67": 803593216.0, + "68": 801299456.0, + "69": 803593216.0, + "70": 801840128.0, + "71": 803593216.0, + "72": 801299456.0, + "73": 803593216.0, + "74": 803593216.0, + "75": 802987008.0, + "76": 803593216.0, + "77": 801840128.0, + "78": 803593216.0, + "79": 801299456.0, + "80": 802987008.0, + "81": 803593216.0, + "82": 801840128.0, + "83": 803593216.0, + "84": 801299456.0, + "85": 802987008.0, + "86": 803593216.0, + "87": 801840128.0, + "88": 803593216.0, + "89": 801299456.0, + "90": 802987008.0, + "91": 803593216.0, + "92": 801299456.0, + "93": 803593216.0, + "94": 801299456.0, + "95": 803593216.0, + "96": 801299456.0, + "97": 803593216.0, + "98": 801299456.0, + "99": 802987008.0, + "100": 803593216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 993582592.0, + "2": 1210942464.0, + "3": 1210942464.0, + "4": 1210942464.0, + "5": 1210942464.0, + "6": 1210942464.0, + "7": 1210942464.0, + "8": 1210942464.0, + "9": 1210942464.0, + "10": 1210942464.0, + "11": 1210942464.0, + "12": 1210942464.0, + "13": 1210942464.0, + "14": 1210942464.0, + "15": 1210942464.0, + "16": 1210942464.0, + "17": 1210942464.0, + "18": 1210942464.0, + "19": 1210942464.0, + "20": 1210942464.0, + "21": 1210942464.0, + "22": 1210942464.0, + "23": 1210942464.0, + "24": 1210942464.0, + "25": 1210942464.0, + "26": 1210942464.0, + "27": 1210942464.0, + "28": 1210942464.0, + "29": 1210942464.0, + "30": 1210942464.0, + "31": 1210942464.0, + "32": 1210942464.0, + "33": 1210942464.0, + "34": 1210942464.0, + "35": 1210942464.0, + "36": 1210942464.0, + "37": 1210942464.0, + "38": 1210942464.0, + "39": 1210942464.0, + "40": 1210942464.0, + "41": 1210942464.0, + "42": 1210942464.0, + "43": 1210942464.0, + "44": 1210942464.0, + "45": 1210942464.0, + "46": 1210942464.0, + "47": 1210942464.0, + "48": 1210942464.0, + "49": 1210942464.0, + "50": 1210942464.0, + "51": 1210942464.0, + "52": 1210942464.0, + "53": 1210942464.0, + "54": 1210942464.0, + "55": 1210942464.0, + "56": 1210942464.0, + "57": 1210942464.0, + "58": 1210942464.0, + "59": 1210942464.0, + "60": 1210942464.0, + "61": 1210942464.0, + "62": 1210942464.0, + "63": 1210942464.0, + "64": 1210942464.0, + "65": 1210942464.0, + "66": 1210942464.0, + "67": 1210942464.0, + "68": 1210942464.0, + "69": 1210942464.0, + "70": 1210942464.0, + "71": 1210942464.0, + "72": 1210942464.0, + "73": 1210942464.0, + "74": 1210942464.0, + "75": 1210942464.0, + "76": 1210942464.0, + "77": 1210942464.0, + "78": 1210942464.0, + "79": 1210942464.0, + "80": 1210942464.0, + "81": 1210942464.0, + "82": 1210942464.0, + "83": 1210942464.0, + "84": 1210942464.0, + "85": 1210942464.0, + "86": 1210942464.0, + "87": 1210942464.0, + "88": 1210942464.0, + "89": 1210942464.0, + "90": 1210942464.0, + "91": 1210942464.0, + "92": 1210942464.0, + "93": 1210942464.0, + "94": 1210942464.0, + "95": 1210942464.0, + "96": 1210942464.0, + "97": 1210942464.0, + "98": 1210942464.0, + "99": 1210942464.0, + "100": 1210942464.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88689, + "2": 10.90485, + "3": 10.90869, + "4": 10.86903, + "5": 10.91601, + "6": 10.906, + "7": 10.90268, + "8": 10.88984, + "9": 10.90425, + "10": 10.89144, + "11": 10.93384, + "12": 10.91647, + "13": 10.91108, + "14": 10.91974, + "15": 10.88488, + "16": 10.9077, + "17": 10.87571, + "18": 10.91379, + "19": 10.9092, + "20": 10.87837, + "21": 10.87896, + "22": 10.85583, + "23": 10.88007, + "24": 10.87245, + "25": 10.85859, + "26": 10.8696, + "27": 10.87702, + "28": 10.88641, + "29": 10.88866, + "30": 10.85422, + "31": 10.79713, + "32": 10.86631, + "33": 10.8781, + "34": 10.83982, + "35": 10.84165, + "36": 10.85012, + "37": 10.85556, + "38": 10.83674, + "39": 10.86355, + "40": 10.82887, + "41": 10.8341, + "42": 10.84469, + "43": 10.78828, + "44": 10.82123, + "45": 10.78831, + "46": 10.7823, + "47": 10.82898, + "48": 10.78985, + "49": 10.71269, + "50": 10.77382, + "51": 10.76639, + "52": 10.7397, + "53": 10.80285, + "54": 10.77365, + "55": 10.76066, + "56": 10.71068, + "57": 10.66686, + "58": 10.74378, + "59": 10.69209, + "60": 10.66474, + "61": 10.7073, + "62": 10.77206, + "63": 10.61812, + "64": 10.7178, + "65": 10.49439, + "66": 10.67106, + "67": 10.57534, + "68": 10.6873, + "69": 10.6816, + "70": 10.66836, + "71": 10.64586, + "72": 10.60925, + "73": 10.56508, + "74": 10.37144, + "75": 10.51183, + "76": 10.39914, + "77": 10.75182, + "78": 10.6268, + "79": 10.46827, + "80": 10.47524, + "81": 10.51083, + "82": 10.58769, + "83": 10.4381, + "84": 10.45057, + "85": 10.55084, + "86": 10.28076, + "87": 10.51088, + "88": 10.60323, + "89": 10.50794, + "90": 10.60274, + "91": 10.38238, + "92": 10.38703, + "93": 10.23076, + "94": 10.08438, + "95": 10.42616, + "96": 10.44905, + "97": 10.32215, + "98": 10.4966, + "99": 10.04765, + "100": 10.33491 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 51.30209, + "2": 1.41746, + "3": 1.28029, + "4": 10.57024, + "5": 0.66643, + "6": 0.67893, + "7": 0.65727, + "8": 0.66196, + "9": 0.66227, + "10": 0.65877, + "11": 0.65828, + "12": 0.65862, + "13": 0.65727, + "14": 0.65896, + "15": 0.65851, + "16": 0.66826, + "17": 0.65878, + "18": 0.65573, + "19": 0.65631, + "20": 0.65579, + "21": 0.65091, + "22": 0.65603, + "23": 0.65158, + "24": 0.65266, + "25": 0.65816, + "26": 0.65194, + "27": 0.6541, + "28": 0.65515, + "29": 0.65439, + "30": 0.65241, + "31": 0.65597, + "32": 0.65551, + "33": 0.65318, + "34": 0.6553, + "35": 0.65725, + "36": 0.65926, + "37": 0.65606, + "38": 0.65571, + "39": 0.65846, + "40": 0.65642, + "41": 0.65509, + "42": 0.66105, + "43": 0.65448, + "44": 0.65534, + "45": 0.65304, + "46": 0.65227, + "47": 0.64871, + "48": 0.65257, + "49": 0.65485, + "50": 0.65054, + "51": 0.67883, + "52": 0.6571, + "53": 0.65671, + "54": 0.65877, + "55": 0.65584, + "56": 0.65072, + "57": 0.64951, + "58": 0.65703, + "59": 0.65106, + "60": 0.64536, + "61": 0.64416, + "62": 0.64816, + "63": 0.64084, + "64": 0.6396, + "65": 0.64182, + "66": 0.64004, + "67": 0.64101, + "68": 0.63928, + "69": 0.65723, + "70": 0.6828, + "71": 0.64052, + "72": 0.64287, + "73": 0.64136, + "74": 0.64252, + "75": 0.64617, + "76": 0.64857, + "77": 0.64304, + "78": 0.64068, + "79": 0.64048, + "80": 0.64091, + "81": 0.64179, + "82": 0.64793, + "83": 0.641, + "84": 0.64077, + "85": 0.64011, + "86": 0.64018, + "87": 0.64132, + "88": 0.63901, + "89": 0.6407, + "90": 0.64277, + "91": 0.64132, + "92": 0.64123, + "93": 0.65051, + "94": 0.65036, + "95": 0.64542, + "96": 0.64561, + "97": 0.6504, + "98": 0.64563, + "99": 0.64524, + "100": 0.65049 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json new file mode 100644 index 00000000000..309b2533461 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_coreweave.json @@ -0,0 +1,644 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93663, + "2": 10.9327, + "3": 10.94263, + "4": 10.94969, + "5": 10.95052, + "6": 10.94157, + "7": 10.94484, + "8": 10.93674, + "9": 10.94996, + "10": 10.93686, + "11": 10.94102, + "12": 10.93763, + "13": 10.9235, + "14": 10.93428, + "15": 10.88791, + "16": 10.87434, + "17": 10.86896, + "18": 10.86065, + "19": 10.86311, + "20": 10.78063, + "21": 10.73125, + "22": 10.60283, + "23": 10.73278, + "24": 10.61888, + "25": 10.55212, + "26": 10.62704, + "27": 10.6391, + "28": 10.5908, + "29": 10.59809, + "30": 10.37777, + "31": 10.1201, + "32": 10.46078, + "33": 10.45538, + "34": 10.20107, + "35": 10.25779, + "36": 10.20889, + "37": 10.33688, + "38": 10.16827, + "39": 10.40875, + "40": 10.05239, + "41": 10.09432, + "42": 10.17894, + "43": 9.74205, + "44": 9.8904, + "45": 9.74009, + "46": 9.72707, + "47": 10.09139, + "48": 9.75298, + "49": 9.40106, + "50": 9.83667, + "51": 9.77071, + "52": 9.65705, + "53": 10.03051, + "54": 9.87899, + "55": 9.79604, + "56": 9.52924, + "57": 9.36583, + "58": 9.75331, + "59": 9.48065, + "60": 9.40785, + "61": 9.60145, + "62": 9.90753, + "63": 9.2583, + "64": 9.68397, + "65": 8.80003, + "66": 9.60779, + "67": 9.25408, + "68": 9.71438, + "69": 9.71682, + "70": 9.6617, + "71": 9.52466, + "72": 9.47116, + "73": 9.38822, + "74": 8.80223, + "75": 9.33966, + "76": 8.93574, + "77": 9.99333, + "78": 9.64731, + "79": 9.28114, + "80": 9.29588, + "81": 9.39589, + "82": 9.60893, + "83": 9.21629, + "84": 9.33891, + "85": 9.52979, + "86": 8.95817, + "87": 9.51641, + "88": 9.68228, + "89": 9.50664, + "90": 9.75348, + "91": 9.23465, + "92": 9.25972, + "93": 8.94517, + "94": 8.69188, + "95": 9.44591, + "96": 9.4101, + "97": 9.20087, + "98": 9.58175, + "99": 8.75818, + "100": 9.29466 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22750260.0, + "2": 22953110.0, + "3": 22604450.0, + "4": 23266322.0, + "5": 22735560.0, + "6": 23061920.0, + "7": 22793342.0, + "8": 22960820.0, + "9": 22865664.0, + "10": 22950364.0, + "11": 22499674.0, + "12": 22456088.0, + "13": 22948060.0, + "14": 22384512.0, + "15": 22846272.0, + "16": 22856858.0, + "17": 22836412.0, + "18": 22590058.0, + "19": 22627048.0, + "20": 22712308.0, + "21": 22762624.0, + "22": 22816888.0, + "23": 22545124.0, + "24": 22794440.0, + "25": 22841936.0, + "26": 22549680.0, + "27": 22464820.0, + "28": 22453684.0, + "29": 22534640.0, + "30": 22636152.0, + "31": 22989488.0, + "32": 22594070.0, + "33": 22566010.0, + "34": 22855504.0, + "35": 22813688.0, + "36": 22595396.0, + "37": 22499360.0, + "38": 22926126.0, + "39": 22825392.0, + "40": 22675666.0, + "41": 22671586.0, + "42": 22682140.0, + "43": 23013940.0, + "44": 22764458.0, + "45": 22678992.0, + "46": 22915276.0, + "47": 22642868.0, + "48": 22954190.0, + "49": 23786668.0, + "50": 22934008.0, + "51": 23866222.0, + "52": 23807290.0, + "53": 24007532.0, + "54": 22871610.0, + "55": 23571284.0, + "56": 23954310.0, + "57": 24211632.0, + "58": 23914404.0, + "59": 23771838.0, + "60": 23813560.0, + "61": 23797288.0, + "62": 23739984.0, + "63": 23916692.0, + "64": 23895952.0, + "65": 24150562.0, + "66": 23796504.0, + "67": 25032232.0, + "68": 23673188.0, + "69": 23648580.0, + "70": 23903504.0, + "71": 24864636.0, + "72": 24767108.0, + "73": 24850612.0, + "74": 24132990.0, + "75": 24146528.0, + "76": 25025540.0, + "77": 24358472.0, + "78": 24910064.0, + "79": 23810516.0, + "80": 24821440.0, + "81": 25020512.0, + "82": 23851244.0, + "83": 24961024.0, + "84": 25144020.0, + "85": 24823608.0, + "86": 23153096.0, + "87": 24850204.0, + "88": 24749150.0, + "89": 22505554.0, + "90": 24059620.0, + "91": 23839038.0, + "92": 23874568.0, + "93": 24769548.0, + "94": 23992452.0, + "95": 25189838.0, + "96": 23909262.0, + "97": 24713068.0, + "98": 23832506.0, + "99": 23983474.0, + "100": 24101108.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 763142656.0, + "2": 778734592.0, + "3": 772525056.0, + "4": 803593216.0, + "5": 803593216.0, + "6": 803593216.0, + "7": 801299456.0, + "8": 803593216.0, + "9": 801840128.0, + "10": 803593216.0, + "11": 802987008.0, + "12": 803593216.0, + "13": 802987008.0, + "14": 801299456.0, + "15": 803593216.0, + "16": 801840128.0, + "17": 803593216.0, + "18": 802987008.0, + "19": 801299456.0, + "20": 803593216.0, + "21": 801299456.0, + "22": 803593216.0, + "23": 801299456.0, + "24": 803593216.0, + "25": 801299456.0, + "26": 803593216.0, + "27": 801299456.0, + "28": 803593216.0, + "29": 801299456.0, + "30": 803593216.0, + "31": 801299456.0, + "32": 803593216.0, + "33": 801840128.0, + "34": 803593216.0, + "35": 801840128.0, + "36": 803593216.0, + "37": 802987008.0, + "38": 801299456.0, + "39": 803593216.0, + "40": 801299456.0, + "41": 803593216.0, + "42": 801840128.0, + "43": 803593216.0, + "44": 801840128.0, + "45": 803593216.0, + "46": 801840128.0, + "47": 803593216.0, + "48": 801840128.0, + "49": 803593216.0, + "50": 801840128.0, + "51": 801299456.0, + "52": 803593216.0, + "53": 801299456.0, + "54": 803593216.0, + "55": 801840128.0, + "56": 803593216.0, + "57": 801840128.0, + "58": 803593216.0, + "59": 801840128.0, + "60": 803593216.0, + "61": 801299456.0, + "62": 803593216.0, + "63": 801299456.0, + "64": 802987008.0, + "65": 803593216.0, + "66": 801299456.0, + "67": 803593216.0, + "68": 801299456.0, + "69": 803593216.0, + "70": 801840128.0, + "71": 803593216.0, + "72": 801299456.0, + "73": 803593216.0, + "74": 803593216.0, + "75": 802987008.0, + "76": 803593216.0, + "77": 801840128.0, + "78": 803593216.0, + "79": 801299456.0, + "80": 802987008.0, + "81": 803593216.0, + "82": 801840128.0, + "83": 803593216.0, + "84": 801299456.0, + "85": 802987008.0, + "86": 803593216.0, + "87": 801840128.0, + "88": 803593216.0, + "89": 801299456.0, + "90": 802987008.0, + "91": 803593216.0, + "92": 801299456.0, + "93": 803593216.0, + "94": 801299456.0, + "95": 803593216.0, + "96": 801299456.0, + "97": 803593216.0, + "98": 801299456.0, + "99": 802987008.0, + "100": 803593216.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 993582592.0, + "2": 1210942464.0, + "3": 1210942464.0, + "4": 1210942464.0, + "5": 1210942464.0, + "6": 1210942464.0, + "7": 1210942464.0, + "8": 1210942464.0, + "9": 1210942464.0, + "10": 1210942464.0, + "11": 1210942464.0, + "12": 1210942464.0, + "13": 1210942464.0, + "14": 1210942464.0, + "15": 1210942464.0, + "16": 1210942464.0, + "17": 1210942464.0, + "18": 1210942464.0, + "19": 1210942464.0, + "20": 1210942464.0, + "21": 1210942464.0, + "22": 1210942464.0, + "23": 1210942464.0, + "24": 1210942464.0, + "25": 1210942464.0, + "26": 1210942464.0, + "27": 1210942464.0, + "28": 1210942464.0, + "29": 1210942464.0, + "30": 1210942464.0, + "31": 1210942464.0, + "32": 1210942464.0, + "33": 1210942464.0, + "34": 1210942464.0, + "35": 1210942464.0, + "36": 1210942464.0, + "37": 1210942464.0, + "38": 1210942464.0, + "39": 1210942464.0, + "40": 1210942464.0, + "41": 1210942464.0, + "42": 1210942464.0, + "43": 1210942464.0, + "44": 1210942464.0, + "45": 1210942464.0, + "46": 1210942464.0, + "47": 1210942464.0, + "48": 1210942464.0, + "49": 1210942464.0, + "50": 1210942464.0, + "51": 1210942464.0, + "52": 1210942464.0, + "53": 1210942464.0, + "54": 1210942464.0, + "55": 1210942464.0, + "56": 1210942464.0, + "57": 1210942464.0, + "58": 1210942464.0, + "59": 1210942464.0, + "60": 1210942464.0, + "61": 1210942464.0, + "62": 1210942464.0, + "63": 1210942464.0, + "64": 1210942464.0, + "65": 1210942464.0, + "66": 1210942464.0, + "67": 1210942464.0, + "68": 1210942464.0, + "69": 1210942464.0, + "70": 1210942464.0, + "71": 1210942464.0, + "72": 1210942464.0, + "73": 1210942464.0, + "74": 1210942464.0, + "75": 1210942464.0, + "76": 1210942464.0, + "77": 1210942464.0, + "78": 1210942464.0, + "79": 1210942464.0, + "80": 1210942464.0, + "81": 1210942464.0, + "82": 1210942464.0, + "83": 1210942464.0, + "84": 1210942464.0, + "85": 1210942464.0, + "86": 1210942464.0, + "87": 1210942464.0, + "88": 1210942464.0, + "89": 1210942464.0, + "90": 1210942464.0, + "91": 1210942464.0, + "92": 1210942464.0, + "93": 1210942464.0, + "94": 1210942464.0, + "95": 1210942464.0, + "96": 1210942464.0, + "97": 1210942464.0, + "98": 1210942464.0, + "99": 1210942464.0, + "100": 1210942464.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88689, + "2": 10.90485, + "3": 10.90869, + "4": 10.86903, + "5": 10.91601, + "6": 10.906, + "7": 10.90268, + "8": 10.88984, + "9": 10.90425, + "10": 10.89144, + "11": 10.93384, + "12": 10.91647, + "13": 10.91108, + "14": 10.91974, + "15": 10.88488, + "16": 10.9077, + "17": 10.87571, + "18": 10.91379, + "19": 10.9092, + "20": 10.87837, + "21": 10.87896, + "22": 10.85583, + "23": 10.88007, + "24": 10.87245, + "25": 10.85859, + "26": 10.8696, + "27": 10.87702, + "28": 10.88641, + "29": 10.88866, + "30": 10.85422, + "31": 10.79713, + "32": 10.86631, + "33": 10.8781, + "34": 10.83982, + "35": 10.84165, + "36": 10.85012, + "37": 10.85556, + "38": 10.83674, + "39": 10.86355, + "40": 10.82887, + "41": 10.8341, + "42": 10.84469, + "43": 10.78828, + "44": 10.82123, + "45": 10.78831, + "46": 10.7823, + "47": 10.82898, + "48": 10.78985, + "49": 10.71269, + "50": 10.77382, + "51": 10.76639, + "52": 10.7397, + "53": 10.80285, + "54": 10.77365, + "55": 10.76066, + "56": 10.71068, + "57": 10.66686, + "58": 10.74378, + "59": 10.69209, + "60": 10.66474, + "61": 10.7073, + "62": 10.77206, + "63": 10.61812, + "64": 10.7178, + "65": 10.49439, + "66": 10.67106, + "67": 10.57534, + "68": 10.6873, + "69": 10.6816, + "70": 10.66836, + "71": 10.64586, + "72": 10.60925, + "73": 10.56508, + "74": 10.37144, + "75": 10.51183, + "76": 10.39914, + "77": 10.75182, + "78": 10.6268, + "79": 10.46827, + "80": 10.47524, + "81": 10.51083, + "82": 10.58769, + "83": 10.4381, + "84": 10.45057, + "85": 10.55084, + "86": 10.28076, + "87": 10.51088, + "88": 10.60323, + "89": 10.50794, + "90": 10.60274, + "91": 10.38238, + "92": 10.38703, + "93": 10.23076, + "94": 10.08438, + "95": 10.42616, + "96": 10.44905, + "97": 10.32215, + "98": 10.4966, + "99": 10.04765, + "100": 10.33491 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 51.30209, + "2": 1.41746, + "3": 1.28029, + "4": 10.57024, + "5": 0.66643, + "6": 0.67893, + "7": 0.65727, + "8": 0.66196, + "9": 0.66227, + "10": 0.65877, + "11": 0.65828, + "12": 0.65862, + "13": 0.65727, + "14": 0.65896, + "15": 0.65851, + "16": 0.66826, + "17": 0.65878, + "18": 0.65573, + "19": 0.65631, + "20": 0.65579, + "21": 0.65091, + "22": 0.65603, + "23": 0.65158, + "24": 0.65266, + "25": 0.65816, + "26": 0.65194, + "27": 0.6541, + "28": 0.65515, + "29": 0.65439, + "30": 0.65241, + "31": 0.65597, + "32": 0.65551, + "33": 0.65318, + "34": 0.6553, + "35": 0.65725, + "36": 0.65926, + "37": 0.65606, + "38": 0.65571, + "39": 0.65846, + "40": 0.65642, + "41": 0.65509, + "42": 0.66105, + "43": 0.65448, + "44": 0.65534, + "45": 0.65304, + "46": 0.65227, + "47": 0.64871, + "48": 0.65257, + "49": 0.65485, + "50": 0.65054, + "51": 0.67883, + "52": 0.6571, + "53": 0.65671, + "54": 0.65877, + "55": 0.65584, + "56": 0.65072, + "57": 0.64951, + "58": 0.65703, + "59": 0.65106, + "60": 0.64536, + "61": 0.64416, + "62": 0.64816, + "63": 0.64084, + "64": 0.6396, + "65": 0.64182, + "66": 0.64004, + "67": 0.64101, + "68": 0.63928, + "69": 0.65723, + "70": 0.6828, + "71": 0.64052, + "72": 0.64287, + "73": 0.64136, + "74": 0.64252, + "75": 0.64617, + "76": 0.64857, + "77": 0.64304, + "78": 0.64068, + "79": 0.64048, + "80": 0.64091, + "81": 0.64179, + "82": 0.64793, + "83": 0.641, + "84": 0.64077, + "85": 0.64011, + "86": 0.64018, + "87": 0.64132, + "88": 0.63901, + "89": 0.6407, + "90": 0.64277, + "91": 0.64132, + "92": 0.64123, + "93": 0.65051, + "94": 0.65036, + "95": 0.64542, + "96": 0.64561, + "97": 0.6504, + "98": 0.64563, + "99": 0.64524, + "100": 0.65049 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json new file mode 100644 index 00000000000..e8c2bae571f --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/golden_values_dev_dgxh100_eos.json @@ -0,0 +1,644 @@ +{ + "lm loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.93663, + "2": 10.9327, + "3": 10.94263, + "4": 10.94969, + "5": 10.95052, + "6": 10.94157, + "7": 10.94484, + "8": 10.93674, + "9": 10.94996, + "10": 10.93686, + "11": 10.94102, + "12": 10.93763, + "13": 10.9235, + "14": 10.93428, + "15": 10.88791, + "16": 10.87434, + "17": 10.86896, + "18": 10.86065, + "19": 10.86311, + "20": 10.78063, + "21": 10.73125, + "22": 10.60283, + "23": 10.73278, + "24": 10.61888, + "25": 10.55212, + "26": 10.62704, + "27": 10.6391, + "28": 10.5908, + "29": 10.59809, + "30": 10.37777, + "31": 10.1201, + "32": 10.46078, + "33": 10.45538, + "34": 10.20107, + "35": 10.25779, + "36": 10.20889, + "37": 10.33688, + "38": 10.16827, + "39": 10.40875, + "40": 10.05239, + "41": 10.09432, + "42": 10.17894, + "43": 9.74205, + "44": 9.8904, + "45": 9.74009, + "46": 9.72707, + "47": 10.09139, + "48": 9.75298, + "49": 9.40106, + "50": 9.83667, + "51": 9.77071, + "52": 9.65705, + "53": 10.03051, + "54": 9.87899, + "55": 9.79604, + "56": 9.52924, + "57": 9.36583, + "58": 9.75331, + "59": 9.48065, + "60": 9.40785, + "61": 9.60145, + "62": 9.90753, + "63": 9.2583, + "64": 9.68397, + "65": 8.80003, + "66": 9.60779, + "67": 9.25408, + "68": 9.71438, + "69": 9.71682, + "70": 9.6617, + "71": 9.52466, + "72": 9.47116, + "73": 9.38822, + "74": 8.80223, + "75": 9.33966, + "76": 8.93574, + "77": 9.99333, + "78": 9.64731, + "79": 9.28114, + "80": 9.29588, + "81": 9.39589, + "82": 9.60893, + "83": 9.21629, + "84": 9.33891, + "85": 9.52979, + "86": 8.95817, + "87": 9.51641, + "88": 9.68228, + "89": 9.50664, + "90": 9.75348, + "91": 9.23465, + "92": 9.25972, + "93": 8.94517, + "94": 8.69188, + "95": 9.44591, + "96": 9.4101, + "97": 9.20087, + "98": 9.58175, + "99": 8.75818, + "100": 9.29466 + } + }, + "num-zeros": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 22750260.0, + "2": 22953110.0, + "3": 22604450.0, + "4": 23266322.0, + "5": 22735560.0, + "6": 23061920.0, + "7": 22793342.0, + "8": 22960820.0, + "9": 22865664.0, + "10": 22950364.0, + "11": 22499674.0, + "12": 22456088.0, + "13": 22948060.0, + "14": 22384512.0, + "15": 22846272.0, + "16": 22856858.0, + "17": 22836412.0, + "18": 22590058.0, + "19": 22627048.0, + "20": 22712308.0, + "21": 22762624.0, + "22": 22816888.0, + "23": 22545124.0, + "24": 22794440.0, + "25": 22841936.0, + "26": 22549680.0, + "27": 22464820.0, + "28": 22453684.0, + "29": 22534640.0, + "30": 22636152.0, + "31": 22989488.0, + "32": 22594070.0, + "33": 22566010.0, + "34": 22855504.0, + "35": 22813688.0, + "36": 22595396.0, + "37": 22499360.0, + "38": 22926126.0, + "39": 22825392.0, + "40": 22675666.0, + "41": 22671586.0, + "42": 22682140.0, + "43": 23013940.0, + "44": 22764458.0, + "45": 22678992.0, + "46": 22915276.0, + "47": 22642868.0, + "48": 22954190.0, + "49": 23786668.0, + "50": 22934008.0, + "51": 23866222.0, + "52": 23807290.0, + "53": 24007532.0, + "54": 22871610.0, + "55": 23571284.0, + "56": 23954310.0, + "57": 24211632.0, + "58": 23914404.0, + "59": 23771838.0, + "60": 23813560.0, + "61": 23797288.0, + "62": 23739984.0, + "63": 23916692.0, + "64": 23895952.0, + "65": 24150562.0, + "66": 23796504.0, + "67": 25032232.0, + "68": 23673188.0, + "69": 23648580.0, + "70": 23903504.0, + "71": 24864636.0, + "72": 24767108.0, + "73": 24850612.0, + "74": 24132990.0, + "75": 24146528.0, + "76": 25025540.0, + "77": 24358472.0, + "78": 24910064.0, + "79": 23810516.0, + "80": 24821440.0, + "81": 25020512.0, + "82": 23851244.0, + "83": 24961024.0, + "84": 25144020.0, + "85": 24823608.0, + "86": 23153096.0, + "87": 24850204.0, + "88": 24749150.0, + "89": 22505554.0, + "90": 24059620.0, + "91": 23839038.0, + "92": 23874568.0, + "93": 24769548.0, + "94": 23992452.0, + "95": 25189838.0, + "96": 23909262.0, + "97": 24713068.0, + "98": 23832506.0, + "99": 23983474.0, + "100": 24101108.0 + } + }, + "mem-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 769688064.0, + "2": 775359488.0, + "3": 769690624.0, + "4": 801299456.0, + "5": 803593216.0, + "6": 801299456.0, + "7": 803593216.0, + "8": 803593216.0, + "9": 801299456.0, + "10": 803593216.0, + "11": 801299456.0, + "12": 803593216.0, + "13": 801299456.0, + "14": 803593216.0, + "15": 803593216.0, + "16": 801299456.0, + "17": 803593216.0, + "18": 801299456.0, + "19": 803593216.0, + "20": 801299456.0, + "21": 803593216.0, + "22": 803593216.0, + "23": 801840128.0, + "24": 803593216.0, + "25": 802987008.0, + "26": 801299456.0, + "27": 802987008.0, + "28": 801299456.0, + "29": 801299456.0, + "30": 803593216.0, + "31": 801299456.0, + "32": 803593216.0, + "33": 801299456.0, + "34": 803593216.0, + "35": 801299456.0, + "36": 801299456.0, + "37": 803593216.0, + "38": 801299456.0, + "39": 803593216.0, + "40": 801299456.0, + "41": 803593216.0, + "42": 801299456.0, + "43": 801299456.0, + "44": 803593216.0, + "45": 802987008.0, + "46": 801299456.0, + "47": 803593216.0, + "48": 801299456.0, + "49": 803593216.0, + "50": 801299456.0, + "51": 801299456.0, + "52": 803593216.0, + "53": 802446336.0, + "54": 801299456.0, + "55": 803593216.0, + "56": 802987008.0, + "57": 801299456.0, + "58": 801840128.0, + "59": 801299456.0, + "60": 803593216.0, + "61": 801840128.0, + "62": 801299456.0, + "63": 803593216.0, + "64": 802446336.0, + "65": 803593216.0, + "66": 801840128.0, + "67": 801299456.0, + "68": 803593216.0, + "69": 801840128.0, + "70": 801299456.0, + "71": 803593216.0, + "72": 803593216.0, + "73": 802987008.0, + "74": 801299456.0, + "75": 803593216.0, + "76": 803593216.0, + "77": 801299456.0, + "78": 801299456.0, + "79": 803593216.0, + "80": 801840128.0, + "81": 801299456.0, + "82": 803593216.0, + "83": 801299456.0, + "84": 801299456.0, + "85": 803593216.0, + "86": 801299456.0, + "87": 801299456.0, + "88": 803593216.0, + "89": 801840128.0, + "90": 803593216.0, + "91": 802987008.0, + "92": 801299456.0, + "93": 803593216.0, + "94": 801299456.0, + "95": 801299456.0, + "96": 803593216.0, + "97": 801840128.0, + "98": 803593216.0, + "99": 802987008.0, + "100": 801299456.0 + } + }, + "mem-max-allocated-bytes": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 988765184.0, + "2": 1206831616.0, + "3": 1210116096.0, + "4": 1210116096.0, + "5": 1210116096.0, + "6": 1210116096.0, + "7": 1210116096.0, + "8": 1210116096.0, + "9": 1210116096.0, + "10": 1210116096.0, + "11": 1210116096.0, + "12": 1210116096.0, + "13": 1210116096.0, + "14": 1210116096.0, + "15": 1210116096.0, + "16": 1210116096.0, + "17": 1210116096.0, + "18": 1210116096.0, + "19": 1210116096.0, + "20": 1210116096.0, + "21": 1210116096.0, + "22": 1210116096.0, + "23": 1210116096.0, + "24": 1210116096.0, + "25": 1210116096.0, + "26": 1210116096.0, + "27": 1210116096.0, + "28": 1210116096.0, + "29": 1210116096.0, + "30": 1210116096.0, + "31": 1210116096.0, + "32": 1210116096.0, + "33": 1210116096.0, + "34": 1210116096.0, + "35": 1210116096.0, + "36": 1210116096.0, + "37": 1210116096.0, + "38": 1210116096.0, + "39": 1210116096.0, + "40": 1210116096.0, + "41": 1210116096.0, + "42": 1210116096.0, + "43": 1210116096.0, + "44": 1210116096.0, + "45": 1210116096.0, + "46": 1210116096.0, + "47": 1210116096.0, + "48": 1210116096.0, + "49": 1210116096.0, + "50": 1210116096.0, + "51": 1210116096.0, + "52": 1210116096.0, + "53": 1210116096.0, + "54": 1210116096.0, + "55": 1210116096.0, + "56": 1210116096.0, + "57": 1210116096.0, + "58": 1210116096.0, + "59": 1210116096.0, + "60": 1210116096.0, + "61": 1210116096.0, + "62": 1210116096.0, + "63": 1210116096.0, + "64": 1210116096.0, + "65": 1210116096.0, + "66": 1210116096.0, + "67": 1210116096.0, + "68": 1210116096.0, + "69": 1210116096.0, + "70": 1210116096.0, + "71": 1210116096.0, + "72": 1210116096.0, + "73": 1210116096.0, + "74": 1210116096.0, + "75": 1210116096.0, + "76": 1210116096.0, + "77": 1210116096.0, + "78": 1210116096.0, + "79": 1210116096.0, + "80": 1210116096.0, + "81": 1210116096.0, + "82": 1210116096.0, + "83": 1210116096.0, + "84": 1210116096.0, + "85": 1210116096.0, + "86": 1210116096.0, + "87": 1210116096.0, + "88": 1210116096.0, + "89": 1210116096.0, + "90": 1210116096.0, + "91": 1210116096.0, + "92": 1210116096.0, + "93": 1210116096.0, + "94": 1210116096.0, + "95": 1210116096.0, + "96": 1210116096.0, + "97": 1210116096.0, + "98": 1210116096.0, + "99": 1210116096.0, + "100": 1210116096.0 + } + }, + "mtp_1 loss": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 10.88689, + "2": 10.90485, + "3": 10.90869, + "4": 10.86903, + "5": 10.91601, + "6": 10.906, + "7": 10.90268, + "8": 10.88984, + "9": 10.90425, + "10": 10.89144, + "11": 10.93384, + "12": 10.91647, + "13": 10.91108, + "14": 10.91974, + "15": 10.88488, + "16": 10.9077, + "17": 10.87571, + "18": 10.91379, + "19": 10.9092, + "20": 10.87837, + "21": 10.87896, + "22": 10.85583, + "23": 10.88007, + "24": 10.87245, + "25": 10.85859, + "26": 10.8696, + "27": 10.87702, + "28": 10.88641, + "29": 10.88866, + "30": 10.85422, + "31": 10.79713, + "32": 10.86631, + "33": 10.8781, + "34": 10.83982, + "35": 10.84165, + "36": 10.85012, + "37": 10.85556, + "38": 10.83674, + "39": 10.86355, + "40": 10.82887, + "41": 10.8341, + "42": 10.84469, + "43": 10.78828, + "44": 10.82123, + "45": 10.78831, + "46": 10.7823, + "47": 10.82898, + "48": 10.78985, + "49": 10.71269, + "50": 10.77382, + "51": 10.76639, + "52": 10.7397, + "53": 10.80285, + "54": 10.77365, + "55": 10.76066, + "56": 10.71068, + "57": 10.66686, + "58": 10.74378, + "59": 10.69209, + "60": 10.66474, + "61": 10.7073, + "62": 10.77206, + "63": 10.61812, + "64": 10.7178, + "65": 10.49439, + "66": 10.67106, + "67": 10.57534, + "68": 10.6873, + "69": 10.6816, + "70": 10.66836, + "71": 10.64586, + "72": 10.60925, + "73": 10.56508, + "74": 10.37144, + "75": 10.51183, + "76": 10.39914, + "77": 10.75182, + "78": 10.6268, + "79": 10.46827, + "80": 10.47524, + "81": 10.51083, + "82": 10.58769, + "83": 10.4381, + "84": 10.45057, + "85": 10.55084, + "86": 10.28076, + "87": 10.51088, + "88": 10.60323, + "89": 10.50794, + "90": 10.60274, + "91": 10.38238, + "92": 10.38703, + "93": 10.23076, + "94": 10.08438, + "95": 10.42616, + "96": 10.44905, + "97": 10.32215, + "98": 10.4966, + "99": 10.04765, + "100": 10.33491 + } + }, + "iteration-time": { + "start_step": 1, + "end_step": 100, + "step_interval": 1, + "values": { + "1": 58.67467, + "2": 1.49483, + "3": 1.38721, + "4": 11.78499, + "5": 0.75759, + "6": 0.75678, + "7": 0.76144, + "8": 0.80382, + "9": 0.74706, + "10": 0.74893, + "11": 0.75091, + "12": 0.75087, + "13": 0.74803, + "14": 0.75316, + "15": 0.80396, + "16": 0.75267, + "17": 0.75378, + "18": 0.75457, + "19": 0.75484, + "20": 0.75428, + "21": 0.75639, + "22": 0.81363, + "23": 0.75607, + "24": 0.75553, + "25": 0.75564, + "26": 0.75334, + "27": 0.75722, + "28": 0.76027, + "29": 0.8113, + "30": 0.75278, + "31": 0.75471, + "32": 0.75104, + "33": 0.75271, + "34": 0.74877, + "35": 0.74765, + "36": 0.80549, + "37": 0.75089, + "38": 0.75395, + "39": 0.75254, + "40": 0.76025, + "41": 0.75356, + "42": 0.75573, + "43": 0.79632, + "44": 0.77927, + "45": 0.75515, + "46": 0.75759, + "47": 0.75978, + "48": 0.75749, + "49": 0.75504, + "50": 0.75616, + "51": 0.77974, + "52": 0.76581, + "53": 0.76997, + "54": 0.76705, + "55": 0.76737, + "56": 0.77352, + "57": 0.77833, + "58": 0.81195, + "59": 0.77251, + "60": 0.7711, + "61": 0.77181, + "62": 0.77006, + "63": 0.76957, + "64": 0.77251, + "65": 0.82259, + "66": 0.77112, + "67": 0.7683, + "68": 0.77335, + "69": 0.77022, + "70": 0.77335, + "71": 0.77822, + "72": 0.77769, + "73": 0.79476, + "74": 0.7728, + "75": 0.7711, + "76": 0.76863, + "77": 0.77228, + "78": 0.77031, + "79": 0.76995, + "80": 0.77286, + "81": 0.76616, + "82": 0.76752, + "83": 0.76583, + "84": 0.77264, + "85": 0.76732, + "86": 0.76873, + "87": 0.77239, + "88": 0.77971, + "89": 0.76112, + "90": 0.76225, + "91": 0.75814, + "92": 0.76144, + "93": 0.75796, + "94": 0.76412, + "95": 0.777, + "96": 0.77207, + "97": 0.7628, + "98": 0.76325, + "99": 0.76204, + "100": 0.7668 + } + } +} \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml new file mode 100644 index 00000000000..f0d1cc0afd3 --- /dev/null +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph/model_config.yaml @@ -0,0 +1,96 @@ +ENV_VARS: + CUDA_DEVICE_MAX_CONNECTIONS: 1 + NVTE_ALLOW_NONDETERMINISTIC_ALGO: 0 + NCCL_ALGO: Ring + CUBLAS_WORKSPACE_CONFIG: :4096:8 +MODEL_ARGS: + --num-layers: 13 + --hidden-size: 512 + --num-attention-heads: 8 + --mtp-num-layers: 1 + --micro-batch-size: 2 + --global-batch-size: 32 + --seq-length: 1024 + --max-position-embeddings: 1024 + --position-embedding-type: rope + --rotary-base: 10000 + --untie-embeddings-and-output-weights: true + --disable-bias-linear: true + --attention-dropout: 0.0 + --hidden-dropout: 0.0 + --train-iters: 100 + --lr-decay-iters: 320000 + --split: 949,50,1 + --distributed-backend: nccl + --lr: 0.00015 + --lr-decay-style: cosine + --min-lr: 1.0e-5 + --weight-decay: 1e-2 + --clip-grad: 1.0 + --lr-warmup-fraction: .01 + --transformer-impl: transformer_engine + --tensor-model-parallel-size: 4 + --pipeline-model-parallel-size: 2 + --expert-model-parallel-size: 2 + --expert-tensor-parallel-size: 2 + --pipeline-model-parallel-layout: Et\\|\\(tt\\|\\)*6mL # Et|(tt|)*6mL + --sequence-parallel: true + --num-experts: 8 + --use-distributed-optimizer: true + --overlap-grad-reduce: true + --overlap-param-gather: true + --moe-token-dispatcher-type: alltoall + --moe-router-load-balancing-type: global_aux_loss + --moe-router-topk: 2 + --moe-router-dtype: fp32 + --moe-router-fusion: true + --moe-router-enable-expert-bias: true + --moe-router-score-function: sigmoid + --moe-router-pre-softmax: true + --moe-ffn-hidden-size: 1024 + --moe-shared-expert-intermediate-size: 512 + --moe-grouped-gemm: true + --moe-layer-freq: ([0]*4+[1]*9) + --moe-permute-fusion: true + --deterministic-mode: true + --no-gradient-accumulation-fusion: true + --attention-softmax-in-fp32: true + --use-checkpoint-opt_param-scheduler: true + --use-mcore-models: true + --bf16: true + --fp8-format: hybrid + --fp8-recipe: blockwise + --first-last-layers-bf16: true + --no-bias-gelu-fusion: true + --recompute-granularity: selective + --recompute-modules: "[moe_act]" + --cuda-graph-impl: transformer_engine + --cuda-graph-scope: "[attn mlp moe_router moe_preprocess]" + --log-memory-to-tensorboard: true + --log-params-norm: true + --log-num-zeros-in-grad: true + --log-validation-ppl-to-tensorboard: true + --log-timers-to-tensorboard: true + --tensorboard-dir: ${TENSORBOARD_PATH} + --log-interval: 1 + --timing-log-level: 0 + --save-interval: 50 + --eval-interval: 1000 + --eval-iters: 10 + --data-path: ${DATA_PATH}/text/the_pile/shard00/my-gpt3_00_text_document + --data-cache-path: ${DATA_CACHE_PATH} + --vocab-file: ${DATA_PATH}/text/the_pile/shard00/bpe/vocab.json + --merge-file: ${DATA_PATH}/text/the_pile/shard00/bpe/merges.txt + --save: ${CHECKPOINT_SAVE_PATH} + --load: ${CHECKPOINT_LOAD_PATH} + --ckpt-fully-parallel-load: true + --ckpt-format: torch_dist + --ckpt-assume-constant-structure: true +TEST_TYPE: ckpt-resume +METRICS: + # - "iteration-time" + - "lm loss" + - "num-zeros" + - "mem-allocated-bytes" + - "mem-max-allocated-bytes" + - "mtp_1 loss" diff --git a/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml b/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml index 852fbf9819d..b684a2ebb54 100644 --- a/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml +++ b/tests/functional_tests/test_cases/t5/t5_release/model_config.yaml @@ -37,7 +37,7 @@ MODEL_ARGS: --pipeline-model-parallel-size: 1 # Data args --data-path: ${DATA_BLEND} - --vocab-file: ${DATA_PATH}/text/the_pile/t5_shard00/bert-large-cased-vocab.txt + --vocab-file: ${DATA_PATH}/bert-large-cased-vocab.txt --tokenizer-type: BertWordPieceCase --split: 99982,9,9 --data-cache-path: ${DATA_CACHE_PATH} diff --git a/tests/test_utils/recipes/bert.yaml b/tests/test_utils/recipes/bert.yaml deleted file mode 100644 index 7d010df93ba..00000000000 --- a/tests/test_utils/recipes/bert.yaml +++ /dev/null @@ -1,93 +0,0 @@ -type: basic -format_version: 1 -maintainers: [mcore] -loggers: [stdout] -spec: - name: "{test_case}_{environment}_{platforms}" - model: bert - nodes: 1 - build: mcore-pyt-{environment} - gpus: 8 - platforms: dgx_a100 - time_limit: - n_repeat: - script_setup: | - unset https_proxy - echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc - - # Checkout latest - cd /opt - rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm - git init - git remote add origin $MCORE_REPO - git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' - git fetch origin $MCORE_MR_COMMIT - git checkout $MCORE_MR_COMMIT - git rev-parse HEAD - - # Checkout backwards-ref - cd /opt - rm -rf /opt/megatron-lm-legacy; mkdir megatron-lm-legacy; cd megatron-lm-legacy - git init - git remote add origin $MCORE_REPO - git fetch origin $MCORE_BACKWARDS_COMMIT - git checkout $MCORE_BACKWARDS_COMMIT - git rev-parse HEAD - rm -rf megatron; cp -a /opt/megatron-lm/megatron ./ - script: |- - ls - cd /opt/megatron-lm - NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g') - ARGUMENTS=( - "DATA_PATH=/mnt/artifacts" - "DATA_CACHE_PATH=/workspace/data/cache" - "OUTPUT_PATH={assets_dir}" - "TENSORBOARD_PATH={assets_dir}/tensorboard" - "CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints" - "CHECKPOINT_LOAD_PATH=/mnt/artifacts" - "TRAINING_SCRIPT_PATH=pretrain_bert.py" - "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" - "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json" - "N_REPEAT={n_repeat}" - "ENABLE_LIGHTWEIGHT_MODE=${{ENABLE_LIGHTWEIGHT_MODE}}" - "RECORD_CHECKPOINTS=${{RECORD_CHECKPOINTS}}" - ) - - bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}} - -products: - - test_case: [bert_mcore_tp2_pp2] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp2_pp2_local_spec] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp2_pp2_resume_torch_dist] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp2_pp2_resume_torch_dist_local_spec] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp1_pp2] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp1_pp4_vp2] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_h100] - - test_case: [bert_mcore_tp4_pp1] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/gpt.yaml b/tests/test_utils/recipes/gpt.yaml index 1f407112463..2e011372e22 100644 --- a/tests/test_utils/recipes/gpt.yaml +++ b/tests/test_utils/recipes/gpt.yaml @@ -468,7 +468,7 @@ products: - environment: [lts] scope: [mr, mr-github] - environment: [dev] - scope: [mr, mr-slim] + scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] - test_case: [gpt3_mcore_te_tp4_pp2_resume_torch_dist_reshard_8x1xNone] products: diff --git a/tests/test_utils/recipes/mamba-static-inference.yaml b/tests/test_utils/recipes/mamba-static-inference.yaml index c33bdff3be1..06107618916 100644 --- a/tests/test_utils/recipes/mamba-static-inference.yaml +++ b/tests/test_utils/recipes/mamba-static-inference.yaml @@ -63,4 +63,4 @@ products: products: - environment: [dev] scope: [mr, mr-github] - platforms: [dg x_h100] + platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/moe.yaml b/tests/test_utils/recipes/moe.yaml index 5dc65df3619..2d4e8c4c94c 100644 --- a/tests/test_utils/recipes/moe.yaml +++ b/tests/test_utils/recipes/moe.yaml @@ -60,51 +60,16 @@ products: ####################################################################### # Nightly tests: Run both DEV and LTS unless something is flaky # ####################################################################### - - test_case: [gpt3_mcore_tp2_pp2_ep2_te_4experts2parallel] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - environment: [lts] - scope: [nightly] - - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel] products: - environment: [dev] scope: [nightly] platforms: [dgx_a100, dgx_h100] - - environment: [lts] - scope: [nightly] - - test_case: [gpt3_mcore_tp2_pp2_ep2_etp2_te_4experts2parallel_dp_last] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - environment: [lts] - scope: [nightly] - test_case: [gpt3_mcore_tp2_cp2_pp2_ep2_te_4experts2parallel_dp_last] products: - environment: [dev] scope: [nightly] platforms: [dgx_a100, dgx_h100] - - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - environment: [lts] - scope: [nightly] - - test_case: [gpt3_mcore_cp2_pp2_ep2_te_4experts2parallel_nondeterministic_dp_last] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - environment: [lts] - scope: [nightly] # - test_case: [gpt3_mcore_tp2_pp2_resume_torch_dist_te_2experts] # products: # non-determinism: #478 # - environment: [dev, lts] @@ -121,11 +86,6 @@ products: - environment: [dev] scope: [mr, mr-github] platforms: [dgx_h100] - # - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] - # products: - # - environment: [dev] - # scope: [mr, mr-github] - # platforms: [dgx_h100] # hang: #513 - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_resume_torch_dist_attn_cudagraph] products: - environment: [dev] @@ -136,49 +96,55 @@ products: # - environment: [dev] # scope: [mr, mr-github] # platforms: [dgx_h100] # hang: #513 - - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_dist_optimizer] + - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] products: - environment: [dev] scope: [mr, mr-github] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_pp1_resume_torch_dist_te_8experts2parallel_multi_dist_optimizer_instances] + - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] products: - environment: [dev] scope: [mr, mr-github] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_overlap_grad_reduce_param_gather_groupedGEMM] + - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts_etp1_ep4] - products: - # - environment: [dev] - # scope: [mr, mr-github] - # platforms: [dgx_h100] # hang: #513 - - environment: [lts] - scope: [nightly] - - test_case: [gpt3_mcore_te_tp2_pp1_te_a2a_ovlp_8experts_etp1_ep4] + - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] products: - environment: [dev] scope: [mr, mr-github] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_zp_z3_resume_torch_dist_te_8experts2parallel_top2router] + - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] products: - environment: [dev] scope: [mr, mr-github] platforms: [dgx_h100] - - test_case: [gpt3_mcore_te_tp2_pp1_te_8experts2parallel_ddp_average_in_collective] + # - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon] + # products: + # - environment: [dev] + # scope: [mr, mr-github, mr-slim] + # platforms: [dgx_h100] + # - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_muon] + # products: + # - environment: [dev] + # scope: [mr, mr-github, mr-slim] + # platforms: [dgx_h100] + - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_no_mtp_no_a2a_ovlp_fine_grained_offloading] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr] platforms: [dgx_h100] - - environment: [lts] - scope: [nightly] - - test_case: [gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_optimizer] + - test_case: [gpt3_moe_mcore_te_tp2_pp2_ep4_etp1_fine_grained_offloading] products: - environment: [dev] - scope: [mr, mr-github] + scope: [mr] + platforms: [dgx_h100] + - test_case: [gpt3_moe_mcore_te_tp4_ep2_etp2_pp2_scoped_cudagraph] + products: + - environment: [dev] + scope: [mr] platforms: [dgx_h100] ####################################################################### # Super important mr, mr-github tests that run for both DEV and LTS per mr, mr-github # @@ -206,3 +172,11 @@ products: - environment: [dev] scope: [mr, mr-github, mr-slim] platforms: [dgx_h100] + - test_case: [gpt3_mcore_te_tp2_pp2_ep4_etp1_mtp_resume_torch_dist_fp8] + products: + - environment: [dev] + scope: [mr] + platforms: [dgx_h100] # hang: #513 + - environment: [dev] + scope: [mr-slim] + platforms: [dgx_h100] diff --git a/tests/test_utils/recipes/t5.yaml b/tests/test_utils/recipes/t5.yaml deleted file mode 100644 index c6b065b6a2b..00000000000 --- a/tests/test_utils/recipes/t5.yaml +++ /dev/null @@ -1,113 +0,0 @@ -type: basic -format_version: 1 -maintainers: [mcore] -loggers: [stdout] -spec: - name: "{test_case}_{environment}_{platforms}" - model: t5 - build: mcore-pyt-{environment} - nodes: 1 - gpus: 8 - platforms: dgx_a100 - script_setup: | - unset https_proxy - echo "machine gitlab-master.nvidia.com login okoenig password $RO_API_TOKEN" | tee -a /root/.netrc - - # Checkout latest - cd /opt - rm -rf /opt/megatron-lm; mkdir megatron-lm; cd megatron-lm - git init - git remote add origin $MCORE_REPO - git fetch origin '+refs/merge-requests/*:refs/remotes/merge-requests/*' - git fetch origin $MCORE_MR_COMMIT - git checkout $MCORE_MR_COMMIT - git rev-parse HEAD - - # Checkout backwards-ref - cd /opt - rm -rf /opt/megatron-lm-legacy; mkdir megatron-lm-legacy; cd megatron-lm-legacy - git init - git remote add origin $MCORE_REPO - git fetch origin $MCORE_BACKWARDS_COMMIT - git checkout $MCORE_BACKWARDS_COMMIT - git rev-parse HEAD - rm -rf megatron; cp -a /opt/megatron-lm/megatron ./ - script: |- - ls - cd /opt/megatron-lm - - NAME=$(echo {test_case}_{environment} | sed 's/dgx_h100/dgx_a100/g') - - ARGUMENTS=( - "DATA_PATH=/mnt/artifacts" - "DATA_CACHE_PATH=/workspace/data/cache" - "OUTPUT_PATH={assets_dir}" - "TENSORBOARD_PATH={assets_dir}/tensorboard" - "CHECKPOINT_SAVE_PATH={artifacts_dir}/checkpoints" - "CHECKPOINT_LOAD_PATH=/mnt/artifacts" - "TRAINING_SCRIPT_PATH=pretrain_t5.py" - "TRAINING_PARAMS_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/model_config.yaml" - "GOLDEN_VALUES_PATH=./tests/functional_tests/test_cases/{model}/{test_case}/golden_values_{environment}_{platforms}.json" - "N_REPEAT={n_repeat}" - "ENABLE_LIGHTWEIGHT_MODE=${{ENABLE_LIGHTWEIGHT_MODE}}" - "RECORD_CHECKPOINTS=${{RECORD_CHECKPOINTS}}" - ) - - bash ./tests/functional_tests/shell_test_utils/run_ci_test.sh ${{ARGUMENTS[@]}} - -products: - - test_case: [t5_11b_mcore_tp4_pp1] - products: - - environment: [dev] - scope: [mr] - platforms: [dgx_h100] - - test_case: [t5_mcore_te_tp4_pp1] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [t5_mcore_te_tp4_pp1_resume_torch_dist] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [t5_mcore_tp4_pp1] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [t5_mcore_tp4_pp1_resume_torch_dist] - products: - - environment: [dev] - scope: [mr, mr-github] - platforms: [dgx_h100] - - test_case: [t5_mcore_te_tp1_pp1_vp1_resume_torch] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - test_case: [t5_mcore_te_tp2_pp1_vp1] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - test_case: [t5_mcore_te_tp2_pp1_vp1_sequence_parallel] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - test_case: [t5_mcore_tp1_pp1_vp1] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - test_case: [t5_mcore_tp1_pp1_vp1_resume_torch] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] - - test_case: [t5_mcore_tp2_pp1_vp1] - products: - - environment: [dev] - scope: [nightly] - platforms: [dgx_a100, dgx_h100] diff --git a/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py b/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py index 2dd0f20fe2c..81e61a3404a 100644 --- a/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py +++ b/tests/unit_tests/a2a_overlap/test_schedule_chunk_1f1b.py @@ -96,7 +96,7 @@ def test_1f1b_schedule_model_chunk(self, mtp_layers, dispatcher_type, fp8_flag, # create TransformerConfig extra_kwargs = {"moe_token_dispatcher_type": dispatcher_type} if dispatcher_type == "flex": - extra_kwargs["moe_enable_deepep"] = True + extra_kwargs["moe_flex_dispatcher_backend"] = "deepep" extra_kwargs["moe_router_dtype"] = "fp32" if fp8_flag is not None: extra_kwargs["fp8"] = fp8_flag[0] diff --git a/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py b/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py index 0c80a3fb4d9..3ebffb810e5 100644 --- a/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py +++ b/tests/unit_tests/a2a_overlap/test_schedule_layer_1f1b.py @@ -358,7 +358,7 @@ def test_transformer_layer_overlap(self, dispatcher_type, fp8_flag): extra_kwargs = {"moe_token_dispatcher_type": dispatcher_type} if dispatcher_type == "flex": - extra_kwargs["moe_enable_deepep"] = True + extra_kwargs["moe_flex_dispatcher_backend"] = "deepep" extra_kwargs["moe_router_dtype"] = "fp32" if fp8_flag is not None: extra_kwargs["fp8"] = fp8_flag[0] @@ -408,7 +408,7 @@ def test_mtp_layer_overlap(self, dispatcher_type, fp8_flag): "mtp_loss_scaling_factor": 1.1, } if dispatcher_type == "flex": - extra_kwargs["moe_enable_deepep"] = True + extra_kwargs["moe_flex_dispatcher_backend"] = "deepep" extra_kwargs["moe_router_dtype"] = "fp32" if fp8_flag is not None: extra_kwargs["fp8_recipe"] = fp8_flag[1] diff --git a/tests/unit_tests/dist_checkpointing/models/common.py b/tests/unit_tests/dist_checkpointing/models/common.py index 31b5d9db3c9..8cb1dc4df65 100644 --- a/tests/unit_tests/dist_checkpointing/models/common.py +++ b/tests/unit_tests/dist_checkpointing/models/common.py @@ -91,7 +91,8 @@ def common_test_parallel_reconfiguration_e2e( save(gpt_model_A.sharded_state_dict(metadata=metadata), ckpt_dir_A, save_strategy) regular_state_dict_A = gpt_model_A.state_dict() Utils.destroy_model_parallel() - + if metadata is not None: + metadata.pop("dp_cp_group") # Load checkpoint A with different TP/PP and save as checkpoint B # No FPS this time, only FPL Utils.initialize_model_parallel(*dest_tp_pp, **(dst_tp_pp_kwargs or {}), order=store_order) diff --git a/tests/unit_tests/dist_checkpointing/models/test_mamba.py b/tests/unit_tests/dist_checkpointing/models/test_mamba.py index ff2c6309977..85fbe5dd045 100644 --- a/tests/unit_tests/dist_checkpointing/models/test_mamba.py +++ b/tests/unit_tests/dist_checkpointing/models/test_mamba.py @@ -130,6 +130,8 @@ def test_parallel_reconfiguration_e2e( ) save(sharded_state_dict, ckpt_dir_A, save_strategy) Utils.destroy_model_parallel() + if metadata is not None: + metadata.pop("dp_cp_group") # Load checkpoint A with different TP/PP/expert/CP and save as checkpoint B # No FPS this time, only FPL diff --git a/tests/unit_tests/dist_checkpointing/models/test_mlp_glu.py b/tests/unit_tests/dist_checkpointing/models/test_mlp_glu.py index 18cfbf67cee..0970e2adc8a 100644 --- a/tests/unit_tests/dist_checkpointing/models/test_mlp_glu.py +++ b/tests/unit_tests/dist_checkpointing/models/test_mlp_glu.py @@ -71,6 +71,9 @@ def test_parallel_reconfiguration_e2e( save(mlp_A.sharded_state_dict(prefix=layer_prefix, metadata=metadata), ckpt_dir_A) Utils.destroy_model_parallel() + if "dp_cp_group" in metadata.keys(): + del metadata["dp_cp_group"] + # Load checkpoint A with different TP/PP and save as checkpoint B Utils.initialize_model_parallel(*dest_tp_pp) mlp_B = initialize_mlp() diff --git a/tests/unit_tests/dist_checkpointing/models/test_moe_experts.py b/tests/unit_tests/dist_checkpointing/models/test_moe_experts.py index b116d2cb603..ca546d746af 100644 --- a/tests/unit_tests/dist_checkpointing/models/test_moe_experts.py +++ b/tests/unit_tests/dist_checkpointing/models/test_moe_experts.py @@ -190,6 +190,9 @@ def test_parallel_reconfiguration_e2e( save(sharded_state_dict, ckpt_dir_A, save_strategy) Utils.destroy_model_parallel() + if "dp_cp_group" in metadata.keys(): + del metadata["dp_cp_group"] + # Load checkpoint A with different TP/PP/EP and save as checkpoint B # No FPS this time, only FPL Utils.initialize_model_parallel( @@ -276,6 +279,9 @@ def test_sequential_grouped_mlp_interchangeable( save(sharded_state_dict, ckpt_dir_A, save_strategy) Utils.destroy_model_parallel() + if "dp_cp_group" in metadata.keys(): + del metadata["dp_cp_group"] + Utils.initialize_model_parallel(dest_tp, dest_pp, expert_model_parallel_size=dest_exp) model_B = initialize_expert_layer(1, use_glu, expert_type=dest_module) load_strategy = None @@ -351,6 +357,9 @@ def test_sequential_grouped_mlp_extra_state( save(sharded_state_dict, ckpt_dir_A, save_strategy) Utils.destroy_model_parallel() + if "dp_cp_group" in metadata.keys(): + del metadata["dp_cp_group"] + Utils.initialize_model_parallel(dest_tp, dest_pp, expert_model_parallel_size=dest_exp) load_strategy = None diff --git a/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py b/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py new file mode 100644 index 00000000000..69fc37bb773 --- /dev/null +++ b/tests/unit_tests/dist_checkpointing/test_layer_wise_optimizer.py @@ -0,0 +1,603 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +from copy import deepcopy +from functools import partial +from unittest import mock + +import pytest +import torch + +from megatron.core import parallel_state +from megatron.core.dist_checkpointing import load, save +from megatron.core.dist_checkpointing.dict_utils import nested_values +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_decoder_block_spec +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_with_transformer_engine_spec as gpt_te_spec, +) +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.optimizer import ChainedOptimizer +from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed +from megatron.core.transformer import MLATransformerConfig, TransformerConfig +from megatron.core.utils import get_pg_size +from megatron.training.arguments import parse_args +from megatron.training.checkpointing import load_checkpoint, save_checkpoint +from tests.unit_tests.dist_checkpointing import ( + TempNamedDir, + init_basic_mock_args, + init_checkpointing_mock_args, + initialize_gpt_model, + setup_model_and_optimizer, + setup_moe_model_and_optimizer, +) +from tests.unit_tests.test_utilities import Utils + + +def check_equal(input_1, input_2): + """Check if two inputs are equal, used for checking checkpointing.""" + if isinstance(input_1, dict) and isinstance(input_2, dict): + assert input_1.keys() == input_2.keys() + for key in input_1.keys(): + check_equal(input_1[key], input_2[key]) + elif isinstance(input_1, list) and isinstance(input_2, list): + assert len(input_1) == len(input_2) + for i in range(len(input_1)): + check_equal(input_1[i], input_2[i]) + elif isinstance(input_1, torch.Tensor) and isinstance(input_2, torch.Tensor): + assert torch.all(input_1 == input_2), f"Input 1: {input_1} != Input 2: {input_2}" + elif type(input_1) != type(input_2): + assert False, f"Input 1 type: {type(input_1)} != Input 2 type: {type(input_2)}" + else: + assert input_1 == input_2, f"Input 1: {input_1} != Input 2: {input_2}" + + +def initialize_real_model( + seed, + pre_process, + post_process, + vp_stage=None, + is_moe=False, + is_mla=False, + virtual_pipeline_model_parallel_size=None, + **config_kwargs, +): + torch.manual_seed(seed) + model_parallel_cuda_manual_seed(seed) + + default_config_kwargs = dict( + num_layers=6, + hidden_size=16, + num_attention_heads=8, + use_cpu_initialization=True, + pipeline_dtype=torch.bfloat16, + bf16=True, + virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, + ) + if is_moe: + default_config_kwargs["moe_ffn_hidden_size"] = 128 + default_config_kwargs["num_moe_experts"] = 4 + default_config_kwargs["add_bias_linear"] = False + # Pop unused fields + config_kwargs.pop("use_sp") + config_kwargs.pop("use_te") + config_kwargs.pop("use_grouped_mlp") + config_kwargs.pop("use_glu") + if is_mla: + default_config_kwargs["multi_latent_attention"] = True + default_config_kwargs["q_lora_rank"] = 96 + default_config_kwargs["kv_lora_rank"] = 512 + default_config_kwargs["qk_head_dim"] = 64 + default_config_kwargs["qk_pos_emb_head_dim"] = 32 + default_config_kwargs["v_head_dim"] = 64 + default_config_kwargs.update(**config_kwargs) + config_cls = MLATransformerConfig if is_mla else TransformerConfig + transformer_config = config_cls(**default_config_kwargs) + + if is_moe: + layer_spec = get_gpt_decoder_block_spec( + transformer_config, use_transformer_engine=True, vp_stage=vp_stage + ) + else: + layer_spec = gpt_te_spec(multi_latent_attention=is_mla) + this_model = GPTModel( + config=transformer_config, + transformer_layer_spec=layer_spec, + vocab_size=128, + max_sequence_length=4, + pre_process=pre_process, + post_process=post_process, + vp_stage=vp_stage, + ) + + return this_model + + +def load_checkpoint_no_arg_checks(*args, **kwargs): + with mock.patch('megatron.training.checkpointing.check_checkpoint_args'): + with mock.patch('megatron.training.checkpointing.update_num_microbatches'): + return load_checkpoint(*args, **kwargs) + + +class TestLayerWiseOptimizer: + """Tests for LayerWiseDistributedOptimizer functionality.""" + + def setup_method(self, method): + pass + + def teardown_method(self, method): + Utils.destroy_model_parallel() + + def test_parameter_sharding(self): + """Test that parameters are correctly sharded across DP ranks.""" + Utils.initialize_model_parallel(1, 1) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=1, + pp=1, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Check if optimizer is ChainedOptimizer (expected for standard setup) + if isinstance(optimizer, ChainedOptimizer): + total_params = sum( + len(group['params']) + for opt in optimizer.chained_optimizers + for group in opt.param_groups + ) + assert total_params > 0, "No parameters found in optimizer" + + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2, 4]) + def test_broadcast_params(self, tp, pp): + """Test that parameter broadcasting works correctly across DP ranks.""" + if tp * pp > 8: + pytest.skip(f"TP*PP > 8 is larger than world size") + + Utils.initialize_model_parallel(tp, pp) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # If this is a LayerWiseDistributedOptimizer, test broadcast + if isinstance(optimizer, LayerWiseDistributedOptimizer): + # Store original param values + original_params = {} + for name, param in model[0].named_parameters(): + original_params[name] = param.data.clone() + + # Call broadcast (should be idempotent if no updates) + optimizer.broadcast_params() + + # Check params are unchanged after broadcast without step + for name, param in model[0].named_parameters(): + assert torch.allclose(param.data, original_params[name]) + + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2, 4]) + @pytest.mark.parametrize('bf16', [True, False]) + def test_layer_wise_optimizer_save_load(self, tmp_path_dist_ckpt, tp, pp, bf16): + """Test save/load of LayerWiseDistributedOptimizer checkpoints.""" + if tp * pp > 8: + pytest.skip(f"TP*PP > 8 is larger than world size") + + Utils.initialize_model_parallel(tp, pp) + + with TempNamedDir( + tmp_path_dist_ckpt / 'test_layer_wise_optimizer_A', sync=True + ) as ckpt_dir_A: + with TempNamedDir( + tmp_path_dist_ckpt / 'test_layer_wise_optimizer_B', sync=True + ) as ckpt_dir_B: + # Create model and optimizer A + model_A, optimizer_A = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + bf16=bf16, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Save checkpoint A + model_sharded_sd_A = model_A[0].sharded_state_dict() + optim_sd_A = optimizer_A.sharded_state_dict(model_sharded_sd_A) + save(optim_sd_A, ckpt_dir_A) + + # Create model and optimizer B with different seed + model_B, optimizer_B = setup_model_and_optimizer( + seed=3, + tp=tp, + pp=pp, + bf16=bf16, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Load checkpoint A into optimizer B + model_sharded_sd_B = model_B[0].sharded_state_dict() + load_sharded_sd = optimizer_B.sharded_state_dict( + model_sharded_sd_B, is_loading=True + ) + state_dict = load(load_sharded_sd, ckpt_dir_A) + optimizer_B.load_state_dict(state_dict) + + # Save as checkpoint B + optim_sd_B = optimizer_B.sharded_state_dict(model_sharded_sd_B) + save(optim_sd_B, ckpt_dir_B) + + Utils.destroy_model_parallel() + + # Compare checkpoints + Utils.initialize_model_parallel(1, 1) + from megatron.core.dist_checkpointing import load_plain_tensors + + plain_sd_A = load_plain_tensors(ckpt_dir_A) + plain_sd_B = load_plain_tensors(ckpt_dir_B) + + check_equal(plain_sd_A, plain_sd_B) + + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2, 4]) + def test_layer_wise_optimizer_grad_norm(self, tp, pp): + """Test that gradient norm calculation works correctly.""" + if tp * pp > 8: + pytest.skip(f"TP*PP > 8 is larger than world size") + + Utils.initialize_model_parallel(tp, pp) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Create dummy gradients + for param in model[0].parameters(): + if param.requires_grad: + param.grad = torch.randn_like(param.data) + + # Test grad norm calculation + if isinstance(optimizer, LayerWiseDistributedOptimizer): + grad_norm = optimizer.get_grad_norm() + assert grad_norm is not None + assert grad_norm >= 0 + + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2, 4]) + def test_layer_wise_optimizer_count_zeros(self, tp, pp): + """Test that zero counting in gradients works correctly.""" + if tp * pp > 8: + pytest.skip(f"TP*PP > 8 is larger than world size") + + Utils.initialize_model_parallel(tp, pp) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Create dummy gradients with some zeros + for param in model[0].parameters(): + if param.requires_grad: + grad = torch.randn_like(param.data) + # Set some values to zero + grad[grad < 0] = 0 + param.grad = grad + + # Test zero counting + if isinstance(optimizer, LayerWiseDistributedOptimizer): + num_zeros = optimizer.count_zeros() + assert num_zeros >= 0 + + @pytest.mark.parametrize('src_tp', [1, 2, 4]) + @pytest.mark.parametrize('src_pp', [1, 2, 4]) + @pytest.mark.parametrize('dest_tp', [1, 2, 4]) + @pytest.mark.parametrize('dest_pp', [1, 2, 4]) + def test_layer_wise_optimizer_resharding( + self, tmp_path_dist_ckpt, src_tp, src_pp, dest_tp, dest_pp + ): + """Test resharding of LayerWiseDistributedOptimizer across different TP/PP.""" + if src_tp * src_pp > 8: + pytest.skip(f"SRC_TP*SRC_PP > 8 is larger than world size") + + if dest_tp * dest_pp > 8: + pytest.skip(f"DEST_TP*DEST_PP > 8 is larger than world size") + + Utils.initialize_model_parallel(src_tp, src_pp) + + with TempNamedDir( + tmp_path_dist_ckpt / 'test_layer_wise_resharding_A', sync=True + ) as ckpt_dir: + # Create and save with source configuration + model_A, optimizer_A = setup_model_and_optimizer( + seed=2, + tp=src_tp, + pp=src_pp, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + model_sharded_sd = model_A[0].sharded_state_dict() + optim_sd = optimizer_A.sharded_state_dict(model_sharded_sd) + save(optim_sd, ckpt_dir) + + Utils.destroy_model_parallel() + + # Load with destination configuration + Utils.initialize_model_parallel(dest_tp, dest_pp) + model_B, optimizer_B = setup_model_and_optimizer( + seed=3, + tp=dest_tp, + pp=dest_pp, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + model_sharded_sd = model_B[0].sharded_state_dict() + load_sharded_sd = optimizer_B.sharded_state_dict(model_sharded_sd, is_loading=True) + + state_dict = load(load_sharded_sd, ckpt_dir) + optimizer_B.load_state_dict(state_dict) + + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2, 4]) + @pytest.mark.parametrize('ep', [1, 2, 4]) + def test_layer_wise_optimizer_with_moe(self, tmp_path_dist_ckpt, tp, pp, ep): + """Test LayerWiseDistributedOptimizer with MoE models.""" + if tp * pp * ep > 8: + pytest.skip(f"TP*PP > 8 is larger than world size") + + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp, + pipeline_model_parallel_size=pp, + expert_model_parallel_size=ep, + ) + + with TempNamedDir(tmp_path_dist_ckpt / 'test_layer_wise_moe', sync=True) as ckpt_dir: + # Create MoE model with optimizer + model, optimizer = setup_moe_model_and_optimizer( + seed=2, tp=tp, pp=pp, ep=ep, bf16=True, dist_opt=False, optimizer='dist_muon' + ) + + # Test that optimizer handles expert parallel parameters + if isinstance(optimizer, LayerWiseDistributedOptimizer): + # Check that expt_dp_params_list exists if EP > 1 + if ep > 1: + assert hasattr(optimizer, 'expt_dp_params_list') + + # Test save/load + model_sharded_sd = model[0].sharded_state_dict() + optim_sd = optimizer.sharded_state_dict(model_sharded_sd) + save(optim_sd, ckpt_dir) + + # Create new optimizer and load + model_new, optimizer_new = setup_moe_model_and_optimizer( + seed=3, tp=tp, pp=pp, ep=ep, bf16=True, dist_opt=False, optimizer='dist_muon' + ) + + model_sharded_sd = model_new[0].sharded_state_dict() + load_sharded_sd = optimizer_new.sharded_state_dict(model_sharded_sd, is_loading=True) + state_dict = load(load_sharded_sd, ckpt_dir) + optimizer_new.load_state_dict(state_dict) + + def test_layer_wise_optimizer_replica_id(self): + """Test that LayerWiseDistributedOptimizer sets replica_id correctly.""" + Utils.initialize_model_parallel(2, 2) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=2, + pp=2, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + if isinstance(optimizer, LayerWiseDistributedOptimizer): + model_sharded_sd = model[0].sharded_state_dict() + optim_sd = optimizer.sharded_state_dict(model_sharded_sd) + + # Extract ShardedTensors and check replica_id + from megatron.core.dist_checkpointing import ShardedTensor + + for sh_base in nested_values(optim_sd): + if isinstance(sh_base, ShardedTensor): + # Check that replica_id has been modified + assert len(sh_base.replica_id) == 3 + # DP component should be 0 for layer-wise optimizer + assert sh_base.replica_id[2] == 0 + + @pytest.mark.parametrize('dp_size', [1, 2, 4]) + def test_layer_wise_optimizer_dp_sizes(self, dp_size): + """Test LayerWiseDistributedOptimizer with different DP sizes.""" + # Use TP to vary DP size while keeping world size constant + world_size = 8 + if world_size % dp_size != 0: + pytest.skip(f"World size {world_size} not divisible by DP size {dp_size}") + + pp = 1 + tp = world_size // dp_size + + if tp == 0: + pytest.skip(f"Invalid TP configuration") + + Utils.initialize_model_parallel(tp, pp) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=1, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + if isinstance(optimizer, LayerWiseDistributedOptimizer): + # Check parameter sharding based on DP size + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True) + + actual_dp_size = get_pg_size(pg_collection.dp_cp) + + if actual_dp_size > 1: + assert optimizer.dp_cp_params_list is not None + assert len(optimizer.dp_cp_params_list) == actual_dp_size + else: + assert optimizer.dp_cp_params_list is None + + def test_layer_wise_optimizer_step(self): + """Test that step function works and returns expected values.""" + Utils.initialize_model_parallel(2, 2) + + model, optimizer = setup_model_and_optimizer( + seed=2, + tp=2, + pp=2, + bf16=True, + dist_opt=False, + initialize_fn=initialize_gpt_model, + optimizer='dist_muon', + ) + + # Create dummy gradients + for param in model[0].parameters(): + if param.requires_grad: + param.grad = torch.randn_like(param.data) + + if isinstance(optimizer, LayerWiseDistributedOptimizer): + # Perform step + update_successful, grad_norm, num_zeros = optimizer.step() + + # Check return values + assert isinstance(update_successful, bool) + assert grad_norm is None or grad_norm >= 0 + assert num_zeros is None or num_zeros >= 0 + + # TODO(@boxiangw): Add test for loading with different TP/PP sizes + @pytest.mark.parametrize("fully_parallel", [True, False]) + @pytest.mark.parametrize('optimizer_type', ['dist_muon', 'muon']) + @pytest.mark.parametrize('tp', [1, 2, 4]) + @pytest.mark.parametrize('pp', [1, 2]) + @pytest.mark.parametrize('ep', [1, 2, 4]) + @pytest.mark.parametrize('is_moe', [True, False]) + @pytest.mark.parametrize('is_mla', [True, False]) + def test_optimizer_common_state_dict( + self, tmp_path_dist_ckpt, fully_parallel, tp, pp, ep, is_moe, is_mla, optimizer_type + ): + if tp * pp * ep > 8: + pytest.skip(f"TP*PP*EP > 8 is larger than world size") + + if ep > 1 and not is_moe: + pytest.skip(f"EP > 1 needs to be used with MoE") + + initialize_fn = partial(initialize_real_model, is_moe=is_moe, is_mla=is_mla) + + # Initialize parallel + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp, + pipeline_model_parallel_size=pp, + expert_model_parallel_size=ep, + ) + rank = torch.distributed.get_rank() + + with TempNamedDir(tmp_path_dist_ckpt / 'test_dp_sharding', sync=True) as ckpt_dir: + mock_args = parse_args(ignore_unknown_args=True) + mock_args.use_distributed_optimizer = False + with mock.patch('megatron.training.checkpointing.get_args', new=lambda: mock_args): + # Initialize model and optimizer A + if is_moe: + model, optimizer_A = setup_moe_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + ep=ep, + initialize_fn=initialize_fn, + dist_opt=False, + optimizer=optimizer_type, + ) + else: + model, optimizer_A = setup_model_and_optimizer( + seed=2, + tp=tp, + pp=pp, + initialize_fn=initialize_fn, + dist_opt=False, + optimizer=optimizer_type, + ) + + # Save checkpoint + init_checkpointing_mock_args(mock_args, ckpt_dir, fully_parallel=fully_parallel) + from megatron.training.training import preprocess_common_state_dict + + save_checkpoint( + 10, + model, + optimizer_A, + None, + 0, + preprocess_common_state_dict_fn=preprocess_common_state_dict, + ) + + # Get optimizer A param state + optim_param_state_A = optimizer_A.state_dict() + + # Initialize model and optimizer B + if is_moe: + model, optimizer_B = setup_moe_model_and_optimizer( + seed=3, + tp=tp, + pp=pp, + ep=ep, + initialize_fn=initialize_fn, + dist_opt=False, + optimizer=optimizer_type, + ) + else: + model, optimizer_B = setup_model_and_optimizer( + seed=3, + tp=tp, + pp=pp, + initialize_fn=initialize_fn, + dist_opt=False, + optimizer=optimizer_type, + ) + + # Load optimizer B from checkpoint + load_checkpoint_no_arg_checks(model, optimizer_B, None) + + # Get optimizer B param state + optim_param_state_B = optimizer_B.state_dict() + + # Test both param state dicts are equal + check_equal(optim_param_state_A, optim_param_state_B) + + Utils.destroy_model_parallel() diff --git a/tests/unit_tests/dist_checkpointing/test_local.py b/tests/unit_tests/dist_checkpointing/test_local.py index 1b8597e1f1c..5ce3422c726 100644 --- a/tests/unit_tests/dist_checkpointing/test_local.py +++ b/tests/unit_tests/dist_checkpointing/test_local.py @@ -26,6 +26,7 @@ LocalCheckpointManager, ) +from megatron.core import parallel_state from megatron.core.dist_checkpointing import ShardedTensor from megatron.core.dist_checkpointing.dict_utils import diff from megatron.core.dist_checkpointing.mapping import ShardedBase, ShardedTensorFactory @@ -78,7 +79,11 @@ def test_sharded_tensors(self, tp, pp, use_torch_fsdp2): opt_param_scheduler = None rng_state = None iteration = None - optim_sd_kwargs = dict(sharding_type='fully_sharded_model_space') + metadata = dict( + dp_cp_group=parallel_state.get_data_parallel_group(with_context_parallel=True) + ) + model_sd_kwargs = dict(metadata=metadata) + optim_sd_kwargs = dict(sharding_type='fully_sharded_model_space', metadata=metadata) mock_args = parse_args(ignore_unknown_args=True) mock_args.no_save_optim = False mock_args.no_save_rng = True @@ -91,6 +96,7 @@ def test_sharded_tensors(self, tp, pp, use_torch_fsdp2): opt_param_scheduler, rng_state, iteration=iteration, + model_sd_kwargs=model_sd_kwargs, optim_sd_kwargs=optim_sd_kwargs, ) sharded_tensor_factories = find_matching_values( diff --git a/tests/unit_tests/dist_checkpointing/utils.py b/tests/unit_tests/dist_checkpointing/utils.py index e722ebe79ca..60157760a5a 100644 --- a/tests/unit_tests/dist_checkpointing/utils.py +++ b/tests/unit_tests/dist_checkpointing/utils.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + from functools import partial from typing import Any, Callable, Tuple, Union from unittest import mock @@ -10,6 +12,7 @@ get_gpt_layer_with_transformer_engine_spec, ) from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer +from megatron.core.optimizer.muon import get_megatron_muon_optimizer from megatron.core.tensor_parallel import model_parallel_cuda_manual_seed from megatron.core.transformer import TransformerConfig from megatron.training.arguments import parse_args @@ -158,14 +161,13 @@ def init_checkpointing_mock_args(args, ckpt_dir, fully_parallel=False): def setup_model_and_optimizer( - seed, - tp, - pp, - initialize_fn=initialize_gpt_model, - bf16=True, - dist_opt=True, - data_parallel_sharding_strategy="optim_grads_params", + seed, tp, pp, initialize_fn=initialize_gpt_model, bf16=True, dist_opt=True, optimizer='adam' ): + if 'muon' in optimizer and dist_opt: + raise ValueError( + "Layer-wise distributed optimizer with Muon is not supported with distributed optimizer." + ) + mock_args = parse_args(ignore_unknown_args=True) with mock.patch('megatron.training.training.get_args', new=lambda: mock_args): init_basic_mock_args(mock_args, tp, pp, bf16=bf16) @@ -184,17 +186,39 @@ def setup_model_and_optimizer( bf16=bf16, params_dtype=torch.bfloat16 if bf16 else torch.float, use_distributed_optimizer=dist_opt, + optimizer=optimizer, ) - optimizer = get_megatron_optimizer(config, model) + + if 'muon' in optimizer: + # Use layer-wise distributed optimizer with Muon + optimizer_type = optimizer + optimizer = get_megatron_muon_optimizer(config, model) + else: + optimizer_type = optimizer + optimizer = get_megatron_optimizer(config, model) torch.manual_seed(seed + 1) model_parallel_cuda_manual_seed(seed + 1) - for group in optimizer.optimizer.param_groups: - for p in group['params']: - if len(optimizer.optimizer.state[p]) == 0: - optimizer.optimizer.state[p]['exp_avg'] = torch.rand_like(p.data) - optimizer.optimizer.state[p]['exp_avg_sq'] = torch.rand_like(p.data) + if not 'muon' in optimizer_type: + for group in optimizer.optimizer.param_groups: + for p in group['params']: + if len(optimizer.optimizer.state[p]) == 0: + optimizer.optimizer.state[p]['exp_avg'] = torch.rand_like(p.data) + optimizer.optimizer.state[p]['exp_avg_sq'] = torch.rand_like(p.data) + else: + for group in optimizer.chained_optimizers[0].param_groups: + for p in group['params']: + if len(optimizer.chained_optimizers[0].state[p]) == 0: + optimizer.chained_optimizers[0].state[p]['momentum_buffer'] = torch.rand_like( + p.data + ) + + for group in optimizer.chained_optimizers[1].param_groups: + for p in group['params']: + if len(optimizer.chained_optimizers[1].state[p]) == 0: + optimizer.chained_optimizers[1].state[p]['exp_avg'] = torch.rand_like(p.data) + optimizer.chained_optimizers[1].state[p]['exp_avg_sq'] = torch.rand_like(p.data) optimizer.reload_model_params() @@ -237,7 +261,12 @@ def setup_moe_model_and_optimizer( use_te=False, use_grouped_mlp=False, use_glu=False, + optimizer='adam', ): + if 'muon' in optimizer and dist_opt: + raise ValueError( + "Layer-wise distributed optimizer with Muon is not supported with distributed optimizer." + ) mock_args = parse_args(ignore_unknown_args=True) with mock.patch('megatron.training.training.get_args', new=lambda: mock_args): init_basic_mock_args(mock_args, tp, pp, bf16=bf16) @@ -261,18 +290,39 @@ def setup_moe_model_and_optimizer( bf16=bf16, params_dtype=torch.bfloat16 if bf16 else torch.float, use_distributed_optimizer=dist_opt, + optimizer=optimizer, ) - optimizer = get_megatron_optimizer(config, model) + + if 'muon' in optimizer: + optimizer_type = optimizer + optimizer = get_megatron_muon_optimizer(config, model) + else: + optimizer_type = optimizer + optimizer = get_megatron_optimizer(config, model) torch.manual_seed(seed + 1) model_parallel_cuda_manual_seed(seed + 1) - for opt in optimizer.chained_optimizers: - for group in opt.param_groups: + if not 'muon' in optimizer_type: + for opt in optimizer.chained_optimizers: + for group in opt.param_groups: + for p in group['params']: + if len(opt.state[p]) == 0: + opt.state[p]['exp_avg'] = torch.rand_like(p.data) + opt.state[p]['exp_avg_sq'] = torch.rand_like(p.data) + else: + for group in optimizer.chained_optimizers[0].param_groups: + for p in group['params']: + if len(optimizer.chained_optimizers[0].state[p]) == 0: + optimizer.chained_optimizers[0].state[p]['momentum_buffer'] = torch.rand_like( + p.data + ) + + for group in optimizer.chained_optimizers[1].param_groups: for p in group['params']: - if len(opt.state[p]) == 0: - opt.state[p]['exp_avg'] = torch.rand_like(p.data) - opt.state[p]['exp_avg_sq'] = torch.rand_like(p.data) + if len(optimizer.chained_optimizers[1].state[p]) == 0: + optimizer.chained_optimizers[1].state[p]['exp_avg'] = torch.rand_like(p.data) + optimizer.chained_optimizers[1].state[p]['exp_avg_sq'] = torch.rand_like(p.data) optimizer.reload_model_params() diff --git a/tests/unit_tests/fusions/test_torch_softmax.py b/tests/unit_tests/fusions/test_torch_softmax.py index edc650a0994..af7e016b253 100644 --- a/tests/unit_tests/fusions/test_torch_softmax.py +++ b/tests/unit_tests/fusions/test_torch_softmax.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import pytest import torch @@ -21,9 +23,6 @@ def setup_method(self, method): scale=None, ) - def teardown_method(self): - get_default_causal_mask.cache_clear() - def test_output_shape(self): x = torch.randn(8, 2, 4, 4, device="cuda") y = self.softmax(x, None, None) @@ -126,9 +125,6 @@ def test_causal_mask_equal_scores(self): class TestFusedScaleMaskSoftmaxComprehensive: """Comprehensive tests for FusedScaleMaskSoftmax including window attention and scaling.""" - def teardown_method(self): - get_default_causal_mask.cache_clear() - def test_scaling_factor(self): """Test softmax with different scaling factors.""" x = torch.randn(2, 4, 8, 8, device="cuda") diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine.py b/tests/unit_tests/inference/engines/test_dynamic_engine.py index f3c05257552..0ac4b296746 100644 --- a/tests/unit_tests/inference/engines/test_dynamic_engine.py +++ b/tests/unit_tests/inference/engines/test_dynamic_engine.py @@ -103,7 +103,7 @@ class DynamicEngineTestConfig: return_log_probs: bool = False materialize_only_last_token_logits: bool = True skip_prompt_log_probs: bool = False - cuda_graph_scope: str = "full_iteration" + cuda_graph_scope: List[str] = None force_build_cuda_graphs: bool = False # If False, do not build cuda graphs in the tests, even if # num_cuda_graphs is set. @@ -136,6 +136,9 @@ def __post_init__(self): if self.context_max_tokens_override is None: self.context_max_tokens_override = self.num_requests * self.max_sequence_length + if self.cuda_graph_scope is None: + self.cuda_graph_scope = ["full_iteration"] + @dataclass class DynamicEngineTestEnv: @@ -511,7 +514,7 @@ def teardown_method(self, method): ) @pytest.mark.parametrize("model_provider", ["gpt", "mamba"]) @pytest.mark.parametrize("num_cuda_graphs", [None, 1, 4]) - @pytest.mark.parametrize("cuda_graph_scope", ["full", "full_iteration"]) + @pytest.mark.parametrize("cuda_graph_scope", [[], ["full_iteration"]]) def test_simple(self, model_provider, num_cuda_graphs, cuda_graph_scope) -> None: """Simple test that runs without errors, and validates output.""" skip_if_mamba_sequence_packing_not_available(model_provider) diff --git a/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py new file mode 100644 index 00000000000..7c1b7f1fe4b --- /dev/null +++ b/tests/unit_tests/pipeline_parallel/test_fine_grained_activation_offloading.py @@ -0,0 +1,187 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import gc + +import pytest +import torch + +EPSILON = 0.1 + +# Skip all tests if CUDA is not available +cuda_available = torch.cuda.is_available() + + +def _reset_cuda_memory(): + gc.collect() + if cuda_available: + torch.cuda.empty_cache() + + +class ToyModel(torch.nn.Module): + def __init__(self, hidden_size: int = 2048, num_layers: int = 4, dtype=torch.bfloat16): + super().__init__() + layers = [] + for _ in range(num_layers): + layers.append( + torch.nn.Linear(hidden_size, hidden_size, bias=True, dtype=dtype, device="cuda") + ) + self.net = torch.nn.Sequential(*layers).to(device="cuda", dtype=dtype) + self.hidden_size = hidden_size + self.num_layers = num_layers + self.dtype = dtype + + # Prevent weights/bias from being considered activation tensors for offload; + # ensure we only count activation tensors (inputs x) in memory accounting. + for p in self.parameters(): + try: + setattr(p, "offloading_activation", False) + except Exception: + pass + + def forward(self, x, use_offload: bool = False): + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + if use_offload: + # Initialize a new chunk (microbatch) and enable offload context. + with off.get_fine_grained_offloading_context(True): + off.fine_grained_offloading_init_chunk_handler( + vp_size=1, vp_stage=None, min_offloaded_tensor_size=1 + ) + for i, layer in enumerate(self.net): + # Group by module; with this linear-only model, each group corresponds to a layer. + off.fine_grained_offloading_set_last_layer(i == len(self.net) - 1) + x = off.fine_grained_offloading_group_start(x, name=f"layer_{i}") + x = layer(x) + # Commit the group; returns a tuple of tensors + (x,) = off.fine_grained_offloading_group_commit( + x, name=f"layer_{i}", forced_released_tensors=[] + ) + return x + # Baseline path (no offload hooks) + with ( + torch.autocast(device_type="cuda", dtype=self.dtype) + if self.dtype in (torch.float16, torch.bfloat16) + else torch.cuda.amp.autocast(enabled=False) + ): + for layer in self.net: + x = layer(x) + return x + + +@pytest.fixture(autouse=True) +def _monkeypatch_offload_deps(monkeypatch): + # Avoid requiring torch.distributed initialization and NVML in tests + import megatron.core.pipeline_parallel.fine_grained_activation_offload as off + + monkeypatch.setattr(off, "debug_rank", lambda *args, **kwargs: None, raising=False) + monkeypatch.setattr(off, "set_ideal_affinity_for_current_gpu", lambda: None, raising=False) + # Ensure a clean state each test + off.fine_grained_offloading_reset() + yield + off.fine_grained_offloading_reset() + + +def test_fine_grained_activation_offload_memory_reduction(): + torch.manual_seed(1234) + # Use a linear-only stack so theoretical saved memory equals sum of per-layer input x bytes. + model = ToyModel(hidden_size=2048, num_layers=8, dtype=torch.bfloat16).eval() + + # Create input + inp = torch.randn( + (2048, model.hidden_size), device="cuda", dtype=torch.bfloat16, requires_grad=True + ) + + # Warmup to stabilize allocator behavior + _reset_cuda_memory() + out = model(inp, use_offload=False) + (out.sum()).backward() + torch.cuda.synchronize() + _reset_cuda_memory() + + # Baseline memory measurement (no offload) + _reset_cuda_memory() + inp_baseline = inp.detach().clone().requires_grad_(True) + baseline_mem_before = torch.cuda.memory_allocated() / (1024**2) + out_base = model(inp_baseline, use_offload=False) + baseline_mem_after = (torch.cuda.memory_allocated() - out_base.nbytes) / (1024**2) + (out_base.sum()).backward() + torch.cuda.synchronize() + baseline_delta = baseline_mem_after - baseline_mem_before + + # Offload memory measurement + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + off.fine_grained_offloading_reset() + _reset_cuda_memory() + inp_off = inp.detach().clone().requires_grad_(True) + offload_mem_before = torch.cuda.memory_allocated() / (1024**2) + out_off = model(inp_off, use_offload=True) + offload_mem_after = (torch.cuda.memory_allocated() - out_off.nbytes) / (1024**2) + (out_off.sum()).backward() + torch.cuda.synchronize() + offload_delta = offload_mem_after - offload_mem_before + + # Offload should reduce peak cached memory usage after forward + assert ( + offload_delta < baseline_delta + ), f"offload did not reduce memory: off={offload_delta:.2f}MiB base={baseline_delta:.2f}MiB" + + # Theoretical savings: storing per-layer input x (same shape each layer). + bytes_per_elem = inp.element_size() # 2 for bfloat16 + input_bytes = inp.numel() * bytes_per_elem + # -2 because the first and last activations are not offloaded + expected_saved_mib = (model.num_layers - 2) * (input_bytes / (1024**2)) + + # Actual savings ≈ baseline_delta - offload_delta (both exclude output tensor memory). + actual_saved_mib = baseline_delta - offload_delta + + # Allow slack for allocator jitter and extra intermediates; magnitudes should match. + rel_err = abs(actual_saved_mib - expected_saved_mib) / max(expected_saved_mib, 1e-6) + assert ( + rel_err <= EPSILON + ), f"saved mismatch: actual={actual_saved_mib:.2f}MiB expected~={expected_saved_mib:.2f}MiB (rel_err={rel_err:.2f})" + + +def test_fine_grained_activation_offload_output_and_grad_consistency(): + torch.manual_seed(2025) + hidden = 1024 + layers = 3 + + # Create identical models by resetting seed + torch.manual_seed(2025) + model_base = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() + torch.manual_seed(2025) + model_off = ToyModel(hidden_size=hidden, num_layers=layers, dtype=torch.bfloat16).train() + + # Same input and target + inp = torch.randn((32, hidden), device="cuda", dtype=torch.bfloat16, requires_grad=True) + target = torch.randn_like(inp) + + # Baseline forward/backward + out_base = model_base(inp, use_offload=False) + loss_base = torch.nn.functional.mse_loss(out_base, target) + loss_base.backward() + grads_base = [ + p.grad.detach().clone() if p.grad is not None else None for p in model_base.parameters() + ] + + # Offload forward/backward + from megatron.core.pipeline_parallel import fine_grained_activation_offload as off + + off.fine_grained_offloading_reset() + out_off = model_off(inp.detach().clone().requires_grad_(True), use_offload=True) + loss_off = torch.nn.functional.mse_loss(out_off, target) + loss_off.backward() + grads_off = [ + p.grad.detach().clone() if p.grad is not None else None for p in model_off.parameters() + ] + + # Compare outputs + assert torch.allclose(out_off.float(), out_base.float(), rtol=1e-3, atol=1e-3) + + # Compare gradients parameter-wise + for gb, go in zip(grads_base, grads_off): + if gb is None and go is None: + continue + assert gb is not None and go is not None + assert torch.allclose(go.float(), gb.float(), rtol=1e-3, atol=1e-3) diff --git a/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py b/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py new file mode 100644 index 00000000000..73739859f42 --- /dev/null +++ b/tests/unit_tests/pipeline_parallel/test_multimodule_communicator.py @@ -0,0 +1,780 @@ +import logging +import os +import sys + +import pytest +import torch +import torch.distributed as dist +from packaging import version + +from megatron.core import parallel_state +from megatron.core.hyper_comm_grid import HyperCommGrid +from megatron.core.model_parallel_config import ModelParallelConfig +from megatron.core.pipeline_parallel.multimodule_communicator import MultiModulePipelineCommunicator +from tests.unit_tests.pipeline_parallel.test_bridge_communicator import ( + _avg_params, + _create_transformer_block, + _get_pg_collection_from_grid, + create_hypercomm_grid, + get_transformer_block_and_grid, +) +from tests.unit_tests.test_utilities import Utils + + +class TestMultiModulePipelineCommunicator: + + @classmethod + def setup_class(cls): + """Set up distributed environment for the entire test class.""" + if not dist.is_initialized(): + dist.init_process_group(backend="nccl") + if torch.cuda.is_available(): + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + + world_size = dist.get_world_size() + if world_size != 8: + pytest.skip( + f"These tests require 8 GPUs, but only {world_size} are available.", + allow_module_level=True, + ) + + def teardown_class(cls): + Utils.destroy_model_parallel() + + def test_multimodule_communicator_init(self): + """Test MultiModulePipelineCommunicator initialization.""" + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1) + generator_grid = create_hypercomm_grid(offset=6, tp=2, cp=1, pp=1, dp=1) + + # Define module-grid mapping + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + # Define module computation topology + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [], + } + config = ModelParallelConfig(bf16=True) + # Initialize communicator + mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config) + # Test attributes match expectations + assert mllm_comm.module_to_grid_map == module_to_grid_map + assert mllm_comm.topology == topology + assert mllm_comm.config == config + assert mllm_comm.current_rank == dist.get_rank() + + def test_compute_total_pipeline_stages(self): + """Test compute_total_pipeline_stages for overall chain and until specific ranks.""" + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1) + generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2) + + # Define module-grid mapping and topology + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [], + } + + # Overall total pipeline stages: max(1,1) + 2 + 1 = 4 + total = MultiModulePipelineCommunicator.compute_total_pipeline_stages( + topology, module_to_grid_map + ) + assert total == 4 + + llm_pp_rank = MultiModulePipelineCommunicator.compute_total_pipeline_stages( + topology, module_to_grid_map, rank=2, module_name='llm' + ) + assert llm_pp_rank == 2 + + def test_send_forward_recv_forward(self): + """Test send_forward and recv_forward operations.""" + if not dist.is_initialized(): + pytest.skip("Distributed not initialized") + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1) + generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2) + + # Set up module-grid mapping and topology + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [], + } + config = ModelParallelConfig(pipeline_dtype=torch.float) + mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config) + + # Simulate forward communication for each module + if mllm_comm.is_current_rank_in_grid(image_encoder_grid): + # Image encoder sends output forward + output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()} + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(audio_encoder_grid): + # Audio encoder sends output forward + output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()} + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(llm_grid): + output_dict = {'llm': torch.randn(2, 32, 128).cuda()} + if dist.get_rank() == 2 or dist.get_rank() == 3: + # LLM stage receives both image and audio outputs + input_dict = mllm_comm.recv_forward() + assert input_dict['image_encoder'].shape == (2, 8, 128) + assert input_dict['audio_encoder'].shape == (2, 16, 128) + mllm_comm.send_forward(output_dict) + else: + # LLM stage receives concatenated LLM outputs + input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128)) + assert input_dict['llm'].shape == (2, 32, 128) + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(generator_grid): + # Generator module receives final LLM output + input_dict = mllm_comm.recv_forward() + assert input_dict['llm'].shape == (1, 32, 128) + + def test_send_forward_recv_forward_with_different_pp_size(self): + """Test for the case when pp(image_encoder) != pp(audio_encoder).""" + if not dist.is_initialized(): + pytest.skip("Distributed not initialized") + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=2, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=4, tp=1, cp=1, pp=4, dp=1) + + # Set up module-grid mapping and topology + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + } + topology = {'image_encoder': ['llm'], 'audio_encoder': ['llm'], 'llm': []} + config = ModelParallelConfig(pipeline_dtype=torch.float) + mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config) + + # Simulate forward communication for each module + if mllm_comm.is_current_rank_in_grid(image_encoder_grid): + output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()} + if dist.get_rank() == 0: + # Image encoder sends output forward + mllm_comm.send_forward(output_dict) + else: + # Image stage receives image outputs + input_dict = mllm_comm.recv_forward(tensor_shape=(2, 8, 128)) + assert input_dict['image_encoder'].shape == (2, 8, 128) + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(audio_encoder_grid): + # Audio encoder sends output forward + output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()} + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(llm_grid): + output_dict = {'llm': torch.randn(2, 32, 128).cuda()} + if dist.get_rank() == 4: + # LLM stage receives both image and audio outputs + input_dict = mllm_comm.recv_forward() + assert input_dict['image_encoder'].shape == (2, 8, 128) + assert input_dict['audio_encoder'].shape == (2, 16, 128) + mllm_comm.send_forward(output_dict) + elif dist.get_rank() == 5 or dist.get_rank() == 6: + # LLM stage receives concatenated LLM outputs + input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128)) + assert input_dict['llm'].shape == (2, 32, 128) + mllm_comm.send_forward(output_dict) + elif dist.get_rank() == 7: + # LLM stage receives concatenated LLM outputs + input_dict = mllm_comm.recv_forward(tensor_shape=(2, 32, 128)) + assert input_dict['llm'].shape == (2, 32, 128) + + def test_send_backward_recv_backward(self): + """Test send_backward and recv_backward operations.""" + if not dist.is_initialized(): + pytest.skip("Distributed not initialized") + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1) + generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2) + + # Set up module-grid mapping and topology + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [], + } + config = ModelParallelConfig(pipeline_dtype=torch.float) + mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config) + + # Simulate backward communication for each module + if mllm_comm.is_current_rank_in_grid(generator_grid): + # Generator sends gradient backward + grad_dict = {'llm': torch.randn(1, 32, 128).cuda()} + mllm_comm.send_backward(grad_dict) + if mllm_comm.is_current_rank_in_grid(llm_grid): + if dist.get_rank() == 4 or dist.get_rank() == 5: + # LLM receives expanded gradient and sends backward + received_grad = mllm_comm.recv_backward() + assert received_grad['llm'].shape == (2, 32, 128) + grad_dict = {'llm': torch.randn(2, 32, 128).cuda()} + mllm_comm.send_backward(grad_dict) + else: + # LLM receives gradient and sends backward to both image/audio encoders + received_grad = mllm_comm.recv_backward(tensor_shape=(2, 32, 128)) + assert received_grad['llm'].shape == (2, 32, 128) + grad_dict = { + 'image_encoder': torch.randn(2, 8, 128).cuda(), + 'audio_encoder': torch.randn(2, 16, 128).cuda(), + } + mllm_comm.send_backward(grad_dict) + if mllm_comm.is_current_rank_in_grid(image_encoder_grid): + # Image encoder receives its gradient + received_grad = mllm_comm.recv_backward() + assert received_grad['image_encoder'].shape == (2, 8, 128) + if mllm_comm.is_current_rank_in_grid(audio_encoder_grid): + # Audio encoder receives its gradient + received_grad = mllm_comm.recv_backward() + assert received_grad['audio_encoder'].shape == (2, 16, 128) + + @pytest.mark.skipif( + version.parse(torch.__version__) < version.parse('2.3.0'), + reason="Feature requires PyTorch 2.3 or later", + ) + def test_send_forward_recv_backward_send_backward_recv_forward(self): + """Test send_forward_recv_backward and send_backward_recv_forward operations.""" + if not dist.is_initialized(): + pytest.skip("Distributed not initialized") + + # Create process group grids for each module + image_encoder_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=1) + audio_encoder_grid = create_hypercomm_grid(offset=1, tp=1, cp=1, pp=1, dp=1) + llm_grid = create_hypercomm_grid(offset=2, tp=2, cp=1, pp=2, dp=1) + generator_grid = create_hypercomm_grid(offset=6, tp=1, cp=1, pp=1, dp=2) + + # Set up module-grid mapping and topology + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + topology = { + 'image_encoder': ['llm'], + 'audio_encoder': ['llm'], + 'llm': ['generator'], + 'generator': [], + } + config = ModelParallelConfig(pipeline_dtype=torch.float) + mllm_comm = MultiModulePipelineCommunicator(module_to_grid_map, topology, config) + + # Simulate bidirectional send/recv for forward and backward in pipeline + + # Encoder stages send forward to the first stage of LLM, and receive backward from the first stage of LLM + if mllm_comm.is_current_rank_in_grid(image_encoder_grid): + output_dict = {'image_encoder': torch.randn(2, 8, 128).cuda()} + received_grad = mllm_comm.send_forward_recv_backward(output_dict) + assert received_grad['image_encoder'].shape == (2, 8, 128) + if mllm_comm.is_current_rank_in_grid(audio_encoder_grid): + output_dict = {'audio_encoder': torch.randn(2, 16, 128).cuda()} + received_grad = mllm_comm.send_forward_recv_backward(output_dict) + assert received_grad['audio_encoder'].shape == (2, 16, 128) + if mllm_comm.is_current_rank_in_grid(llm_grid): + if dist.get_rank() == 2 or dist.get_rank() == 3: + grad_dict = { + 'image_encoder': torch.randn(2, 8, 128).cuda(), + 'audio_encoder': torch.randn(2, 16, 128).cuda(), + } + input_dict = mllm_comm.send_backward_recv_forward(grad_dict) + assert input_dict['image_encoder'].shape == (2, 8, 128) + assert input_dict['audio_encoder'].shape == (2, 16, 128) + + # First stage of LLM sends forward to the second stage of LLM, and receive backward from the second stage of LLM + if mllm_comm.is_current_rank_in_grid(llm_grid): + if dist.get_rank() == 2 or dist.get_rank() == 3: + output_dict = {'llm': torch.randn(2, 32, 128).cuda()} + received_grad = mllm_comm.send_forward_recv_backward( + output_dict, tensor_shape=(2, 32, 128) + ) + assert received_grad['llm'].shape == (2, 32, 128) + if dist.get_rank() == 4 or dist.get_rank() == 5: + grad_dict = {'llm': torch.randn(2, 32, 128).cuda()} + input_dict = mllm_comm.send_backward_recv_forward( + grad_dict, tensor_shape=(2, 32, 128) + ) + assert input_dict['llm'].shape == (2, 32, 128) + + # Second stage of LLM sends forward to generator, and receive backward from generator + if mllm_comm.is_current_rank_in_grid(llm_grid): + if dist.get_rank() == 4 or dist.get_rank() == 5: + output_dict = {'llm': torch.randn(2, 32, 128).cuda()} + received_grad = mllm_comm.send_forward_recv_backward(output_dict) + assert received_grad['llm'].shape == (2, 32, 128) + if mllm_comm.is_current_rank_in_grid(generator_grid): + grad_dict = {'llm': torch.randn(1, 32, 128).cuda()} + input_dict = mllm_comm.send_backward_recv_forward(grad_dict) + assert input_dict['llm'].shape == (1, 32, 128) + + @pytest.mark.skipif( + version.parse(torch.__version__) < version.parse('2.3.0'), + reason="Feature requires PyTorch 2.3 or later", + ) + def test_send_forward_recv_forward_with_transformer_blocks(self): + """Test send_forward and recv_forward operations.""" + + # Set model/test dimensions for easier debugging and output comparison + hidden_size = 16 + sequence_length = 2 + micro_batch_size = 2 + + # For reproducibility, set a fixed seed + torch.manual_seed(12345) + dtype = torch.float32 + + # Create random input hidden states tensor + hidden_states = torch.randn( + (sequence_length, micro_batch_size, hidden_size), device="cuda" + ).to(dtype) + current_rank = dist.get_rank() + + # ========== Initialize tensor model-parallel environment ========== + parallel_state_tp = 2 + Utils.initialize_model_parallel(tensor_model_parallel_size=2) + + # ========== Build reference 1D grid and transformer block for weight sharing ========== + ref_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=8) + ref_pg_collection = _get_pg_collection_from_grid(ref_grid) + ref_block = _create_transformer_block( + dtype=dtype, hidden_size=hidden_size, pg_collection=ref_pg_collection + ) + _avg_params( + ref_block, ref_grid.get_pg("dp") + ) # Ensure parameters are averaged across data parallel (DP) + + # ========== Create different transformer blocks for each model stage ========== + # Image encoder + image_encoder_block, image_encoder_grid = get_transformer_block_and_grid( + ref_block, + tp_size=1, + cp_size=1, + pp_size=1, + dp_size=1, + grid_offset=0, + hidden_size=hidden_size, + dtype=dtype, + ) + # Audio encoder + audio_encoder_block, audio_encoder_grid = get_transformer_block_and_grid( + ref_block, + tp_size=1, + cp_size=1, + pp_size=1, + dp_size=1, + grid_offset=1, + hidden_size=hidden_size, + dtype=dtype, + ) + # LLM (Large Language Model) block with tensor & pipeline parallelism + llm_block, llm_grid = get_transformer_block_and_grid( + ref_block, + tp_size=2, + cp_size=1, + pp_size=2, + dp_size=1, + grid_offset=2, + hidden_size=hidden_size, + dtype=dtype, + ) + # Generator block (final stage) with DP=2 + generator_block, generator_grid = get_transformer_block_and_grid( + ref_block, + tp_size=1, + cp_size=1, + pp_size=1, + dp_size=2, + grid_offset=6, + hidden_size=hidden_size, + dtype=dtype, + ) + + # ========== Define module-to-grid correspondence and pipeline topology ========== + module_to_grid_map = { + 'image_encoder': image_encoder_grid, + 'audio_encoder': audio_encoder_grid, + 'llm': llm_grid, + 'generator': generator_grid, + } + topology = { + 'image_encoder': ['llm'], # image_encoder sends output to llm + 'audio_encoder': ['llm'], # audio_encoder sends output to llm + 'llm': ['generator'], # llm sends output to generator + 'generator': [], # generator is the final module + } + config = ModelParallelConfig(pipeline_dtype=torch.float) + # Define dimension mapping for sequence, batch, hidden + dim_mapping = {'s': 0, 'h': 2, 'b': 1} + seq_dim = dim_mapping['s'] + + # Communication handler for multi-module pipeline (send/recv abstraction) + mllm_comm = MultiModulePipelineCommunicator( + module_to_grid_map, topology, config, dim_mapping=dim_mapping + ) + + # ========== Run actual distributed pipeline blocks (per process, depending on role) ========== + if mllm_comm.is_current_rank_in_grid(image_encoder_grid): + # Image encoder rank: run forward and send output + image_encoder_output = image_encoder_block( + hidden_states=hidden_states, attention_mask=None + ) + output_dict = {'image_encoder': image_encoder_output} + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(audio_encoder_grid): + # Audio encoder rank: run forward and send output + audio_encoder_output = audio_encoder_block( + hidden_states=hidden_states, attention_mask=None + ) + output_dict = {'audio_encoder': audio_encoder_output} + mllm_comm.send_forward(output_dict) + if mllm_comm.is_current_rank_in_grid(llm_grid): + if dist.get_rank() == 2 or dist.get_rank() == 3: + # LLM stage 0 (receives both image and audio, concatenates along seq_dim) + input_dict = mllm_comm.recv_forward() + llm_output = llm_block( + hidden_states=torch.cat( + [input_dict['image_encoder'], input_dict['audio_encoder']], dim=seq_dim + ), + attention_mask=None, + ) + output_dict = {'llm': llm_output} + mllm_comm.send_forward(output_dict) + else: + # LLM stage 1 (receives output of previous LLM stage) + input_dict = mllm_comm.recv_forward( + tensor_shape=(sequence_length * 2, micro_batch_size, hidden_size) + ) + llm_output = llm_block(hidden_states=input_dict['llm'], attention_mask=None) + output_dict = {'llm': llm_output} + mllm_comm.send_forward(output_dict) + + if mllm_comm.is_current_rank_in_grid(generator_grid): + # Generator block: only receives from llm and runs forward + input_dict = mllm_comm.recv_forward() + generator_output = generator_block(hidden_states=input_dict['llm'], attention_mask=None) + + # ========== Build a reference (serial/global) pipeline for correctness checking ========== + global_image_encoder_block, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + global_audio_encoder_block, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + global_llm_block_pp_rank_0, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + global_llm_block_pp_rank_1, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + global_generator_block, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + + # Run each stage sequentially as a global pipeline (for truth) + global_image_encoder_output = global_image_encoder_block( + hidden_states=hidden_states, attention_mask=None + ) + global_audio_encoder_output = global_audio_encoder_block( + hidden_states=hidden_states, attention_mask=None + ) + # Compare output between global and distributed blocks for image/audio stage + if current_rank == 0: + torch.testing.assert_close( + global_image_encoder_output, image_encoder_output, rtol=1e-3, atol=1e-3 + ) + if current_rank == 1: + torch.testing.assert_close( + global_audio_encoder_output, audio_encoder_output, rtol=1e-3, atol=1e-3 + ) + + # Feed outputs to LLM stages (emulate pipeline cut with concatenation) + global_llm_input = torch.cat( + [global_image_encoder_output, global_audio_encoder_output], dim=seq_dim + ) + global_llm_pp_rank_0_output = global_llm_block_pp_rank_0( + hidden_states=global_llm_input, attention_mask=None + ) + if current_rank == 2 or current_rank == 3: + torch.testing.assert_close( + global_llm_pp_rank_0_output, llm_output, rtol=1e-3, atol=1e-3 + ) + global_llm_pp_rank_1_output = global_llm_block_pp_rank_1( + hidden_states=global_llm_pp_rank_0_output, attention_mask=None + ) + if current_rank == 4 or current_rank == 5: + torch.testing.assert_close( + global_llm_pp_rank_1_output, llm_output, rtol=1e-3, atol=1e-3 + ) + + # Generator output and comparison to distributed output (for each DP chunk) + global_generator_block_output = global_generator_block( + hidden_states=global_llm_pp_rank_1_output, attention_mask=None + ) + global_generator_block_chunks = torch.split( + global_generator_block_output, global_generator_block_output.shape[1] // 2, dim=1 + ) + if current_rank == 6: + torch.testing.assert_close( + global_generator_block_chunks[0], generator_output, rtol=1e-3, atol=1e-3 + ) + if current_rank == 7: + torch.testing.assert_close( + global_generator_block_chunks[1], generator_output, rtol=1e-3, atol=1e-3 + ) + + @pytest.mark.skipif( + version.parse(torch.__version__) < version.parse('2.3.0'), + reason="Feature requires PyTorch 2.3 or later", + ) + @pytest.mark.parametrize( + "grid1_tp, grid1_pp, grid1_dp, grid2_tp, grid2_pp, grid2_dp, parallel_state_tp", + [ + (2, 1, 1, 2, 1, 1, 2), # TP2PP1DP1 to TP2PP1DP1 + (2, 1, 1, 2, 2, 1, 2), # TP2PP1DP1 to TP2PP2DP1 + (2, 2, 1, 2, 2, 1, 2), # TP2PP2DP1 to TP2PP2DP1 + (4, 1, 1, 4, 1, 1, 4), # TP4DP1 to TP4DP1 + (2, 1, 2, 4, 1, 1, 2), # TP2DP2 to TP4DP1 + (4, 1, 1, 2, 1, 2, 2), # TP4DP1 to TP2DP2 + (2, 1, 2, 1, 1, 4, 2), # TP2DP2 to TP1DP4 + ], + ) + def test_send_forward_recv_forward_with_transformer_blocks_and_different_parallelisms( + self, grid1_tp, grid1_pp, grid1_dp, grid2_tp, grid2_pp, grid2_dp, parallel_state_tp + ): + """Test bridge communicator with two transformer blocks having different process group configurations.""" + # Model and input configuration + hidden_size = 16 + sequence_length = 2 + micro_batch_size = 8 + torch.manual_seed(12345) + dtype = torch.float32 + + # Create random input tensor on CUDA + hidden_states = torch.randn( + (sequence_length, micro_batch_size, hidden_size), device="cuda" + ).to(dtype) + hidden_states_ref = hidden_states.clone() + current_rank = dist.get_rank() + + # Initialize model parallel with desired TP + Utils.initialize_model_parallel(tensor_model_parallel_size=parallel_state_tp) + + # Build a reference grid and block for parameter sharing & DP averaging + ref_grid = create_hypercomm_grid(offset=0, tp=1, cp=1, pp=1, dp=8) + ref_pg_collection = _get_pg_collection_from_grid(ref_grid) + ref_block = _create_transformer_block( + dtype=dtype, hidden_size=hidden_size, pg_collection=ref_pg_collection + ) + _avg_params( + ref_block, ref_grid.get_pg("dp") + ) # Synchronize parameters across DP for reproducibility + + # ====== Create two transformer block+grid pairs with different TP/DP settings ====== + block_grid_1, grid_1 = get_transformer_block_and_grid( + ref_block, + tp_size=grid1_tp, + pp_size=grid1_pp, + dp_size=grid1_dp, + grid_offset=0, + hidden_size=hidden_size, + dtype=dtype, + ) + + block_grid_2, grid_2 = get_transformer_block_and_grid( + ref_block, + tp_size=grid2_tp, + pp_size=grid2_pp, + dp_size=grid2_dp, + grid_offset=grid_1.size, + hidden_size=hidden_size, + dtype=dtype, + ) + + dist.barrier() # Synchronize ranks before communication + + # Module-grid map and pipeline communication topology + module_to_grid_map = {'image_encoder': grid_1, 'llm': grid_2} + topology = { + 'image_encoder': ['llm'], # image_encoder sends forward results to llm + 'llm': [], # llm is the last stage here + } + config = ModelParallelConfig(pipeline_dtype=torch.float) + mllm_comm = MultiModulePipelineCommunicator( + module_to_grid_map, topology, config, dim_mapping={'s': 0, 'h': 2, 'b': 1} + ) + + output_grid_2 = None + # If current rank is in the first grid, run first block and send output + if grid_1 is not None and mllm_comm.is_current_rank_in_grid(grid_1): + rank_module_info = mllm_comm.rank_module_map['image_encoder'] + if rank_module_info.pp_rank == 0: + hidden_states = block_grid_1(hidden_states=hidden_states, attention_mask=None) + mllm_comm.send_forward({'image_encoder': hidden_states}) + else: + input_dict = mllm_comm.recv_forward( + tensor_shape=(sequence_length, micro_batch_size, hidden_size) + ) + hidden_states = input_dict['image_encoder'] + hidden_states = block_grid_1(hidden_states=hidden_states, attention_mask=None) + mllm_comm.send_forward({'image_encoder': hidden_states}) + + # If current rank is in second grid, receive and run the second block + if grid_2 is not None and mllm_comm.is_current_rank_in_grid(grid_2): + rank_module_info = mllm_comm.rank_module_map['llm'] + if rank_module_info.pp_rank == 0: + input_dict = mllm_comm.recv_forward() + hidden_states = input_dict['image_encoder'] + hidden_states = block_grid_2(hidden_states=hidden_states, attention_mask=None) + if rank_module_info.pp_rank == rank_module_info.pp_size - 1: + output_grid_2 = hidden_states + else: + mllm_comm.send_forward({'llm': hidden_states}) + elif rank_module_info.pp_rank < rank_module_info.pp_size - 1: + input_dict = mllm_comm.recv_forward( + tensor_shape=( + sequence_length, + (grid1_dp * micro_batch_size) // grid2_dp, + hidden_size, + ) + ) + hidden_states = input_dict['llm'] + hidden_states = block_grid_2(hidden_states=hidden_states, attention_mask=None) + mllm_comm.send_forward({'llm': hidden_states}) + else: + input_dict = mllm_comm.recv_forward( + tensor_shape=( + sequence_length, + (grid1_dp * micro_batch_size) // grid2_dp, + hidden_size, + ) + ) + hidden_states = input_dict['llm'] + output_grid_2 = block_grid_2(hidden_states=hidden_states, attention_mask=None) + + # Compute expected output shape based on change in DP size (chunk/expand batch dimension appropriately) + factor = max(grid1_dp, grid2_dp) // min(grid1_dp, grid2_dp) + expected_output_shape = ( + sequence_length, + ( + micro_batch_size * factor + if grid1_dp > grid2_dp + else micro_batch_size // factor + ), + hidden_size, + ) + assert ( + output_grid_2.shape == expected_output_shape + ), f"Output2 shape mismatch: {output_grid_2.shape}" + + # ====== Reference: global (replicated) pipeline forward for correctness checking ====== + global_block_1, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + global_block_2, _ = get_transformer_block_and_grid( + ref_block, + tp_size=parallel_state_tp, + use_global_parallel_state=True, + hidden_size=hidden_size, + dtype=dtype, + ) + + for i in range(grid1_pp): + hidden_states_ref = global_block_1(hidden_states=hidden_states_ref, attention_mask=None) + + for i in range(grid2_pp): + hidden_states_ref = global_block_2(hidden_states=hidden_states_ref, attention_mask=None) + + # Output comparison under different DP compositions between grids + if ( + grid_2 is not None + and mllm_comm.is_current_rank_in_grid(grid_2) + and rank_module_info.pp_rank == rank_module_info.pp_size - 1 + ): + if grid1_dp == grid2_dp: + # DP size matches: all outputs directly compared + torch.testing.assert_close(hidden_states_ref, output_grid_2, rtol=1e-3, atol=1e-3) + elif grid1_dp < grid2_dp: + # If grid2 expands DP: each output_grid_2 chunk corresponds to a split of the reference output + grid2_dp_ranks = grid_2._gen_rank_enum([x for x in grid_2.dim_names if x != "dp"]) + global_block_2_chunks = torch.split( + hidden_states_ref, hidden_states_ref.shape[1] // (grid2_dp // grid1_dp), dim=1 + ) + relevant_chunk = None + for i, dp_ranks in enumerate(grid2_dp_ranks): + if current_rank in dp_ranks: + relevant_chunk = global_block_2_chunks[i % len(global_block_2_chunks)] + torch.testing.assert_close(relevant_chunk, output_grid_2, rtol=1e-3, atol=1e-3) + else: + # If DP shrinks (grid1_dp > grid2_dp): just compare the relevant first chunk + output_grid_2_first_chunk = torch.chunk(output_grid_2, grid1_dp // grid2_dp, dim=1)[ + 0 + ] + torch.testing.assert_close( + hidden_states_ref, output_grid_2_first_chunk, rtol=1e-3, atol=1e-3 + ) diff --git a/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py b/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py index 04880fb432c..9b1261312b2 100644 --- a/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py +++ b/tests/unit_tests/pipeline_parallel/test_pipeline_layout.py @@ -1,4 +1,4 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import os from pathlib import Path @@ -21,6 +21,7 @@ ) from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.multi_token_prediction import mtp_on_this_rank from megatron.core.transformer.transformer_config import TransformerConfig from megatron.training.checkpointing import load_checkpoint, save_checkpoint from megatron.training.global_vars import set_args @@ -53,6 +54,8 @@ def initialize_gpt_model( virtual_pipeline_model_parallel_size=virtual_pipeline_model_parallel_size, hidden_dropout=0.0, attention_dropout=0.0, + mtp_num_layers=1 if with_mtp else None, + mtp_loss_scaling_factor=1.0 if with_mtp else None, ) default_config_kwargs.update(**config_kwargs) transformer_config = TransformerConfig(**default_config_kwargs) @@ -61,9 +64,6 @@ def initialize_gpt_model( transformer_config.moe_ffn_hidden_size = 128 transformer_config.num_moe_experts = 4 transformer_config.add_bias_linear = False - if with_mtp: - transformer_config.mtp_num_layers = 1 - transformer_config.mtp_loss_scaling_factor = 1.0 model = [] for i in range(virtual_pipeline_model_parallel_size or 1): if is_moe: @@ -71,8 +71,11 @@ def initialize_gpt_model( else: layer_spec = layer_spec_fn() - if is_moe and with_mtp and mpu.is_pipeline_last_stage(ignore_virtual=False, vp_stage=i): - transformer_layer_spec_for_mtp = gpt_te_spec(transformer_config) + if with_mtp and mtp_on_this_rank(transformer_config, ignore_virtual=False, vp_stage=i): + if is_moe: + transformer_layer_spec_for_mtp = gpt_te_spec(transformer_config) + else: + transformer_layer_spec_for_mtp = layer_spec mtp_block_spec = get_gpt_mtp_block_spec( transformer_config, transformer_layer_spec_for_mtp, @@ -81,6 +84,10 @@ def initialize_gpt_model( ) else: mtp_block_spec = None + + # print("========================") + # print("[DEBUG] mtp_block_spec is ", mtp_block_spec) + # exit() pre_process = mpu.is_pipeline_first_stage(ignore_virtual=False, vp_stage=i) post_process = mpu.is_pipeline_last_stage(ignore_virtual=False, vp_stage=i) this_model = ( @@ -163,7 +170,7 @@ def create_args(): [], ["decoder"], ["decoder"], - ["decoder"] * 2 + ["loss"], + ["decoder"] * 2 + ["mtp"] + ["loss"], ], False, True, @@ -185,7 +192,19 @@ def create_args(): False, ), ((1, 2, None), [["embedding"] + ["decoder"] * 4, ["decoder"] * 4 + ["loss"]], True, False), - ((1, 4, 2), "E|t*3|(t|)*5L", True, True), + ((1, 4, 2), "E|t*3|(t|)*5mL", True, True), # mtp in the last stage + ( + (1, 4, 2), + "E|t*3|(t|)*4tm|L", + True, + True, + ), # mtp in the second last stage with a decoder layer + ( + (1, 4, 2), + "E|t*3|(t|)*3tt|m|L", + True, + True, + ), # mtp in the second last stage with no other layers ], ) def test_forward_vpp(create_args, tmp_path_dist_ckpt, tp_pp_vpp, pp_layout, is_moe, with_mtp): diff --git a/tests/unit_tests/post_training/test_modelopt_module_spec.py b/tests/unit_tests/post_training/test_modelopt_module_spec.py index f27a22390f7..ec80fcb1a72 100644 --- a/tests/unit_tests/post_training/test_modelopt_module_spec.py +++ b/tests/unit_tests/post_training/test_modelopt_module_spec.py @@ -6,7 +6,7 @@ import torch from packaging.version import Version -from megatron.core import dist_checkpointing +from megatron.core import dist_checkpointing, parallel_state from megatron.core.inference.contexts import StaticInferenceContext from megatron.core.models.gpt.gpt_layer_specs import ( get_gpt_decoder_block_spec, @@ -92,8 +92,11 @@ def setup_method(self, method): def test_sharded_state_dict_restore(self, tmp_path_dist_ckpt): """Save with the default TE spec and restore using the ModelOpt spec.""" _dist_checkpoint_name = "default_model" - te_fused_sharded_state_dict = self.default_model.sharded_state_dict() - modelopt_sharded_state_dict = self.modelopt_model.sharded_state_dict() + metadata = { + "dp_cp_group": parallel_state.get_data_parallel_group(with_context_parallel=True) + } + te_fused_sharded_state_dict = self.default_model.sharded_state_dict(metadata=metadata) + modelopt_sharded_state_dict = self.modelopt_model.sharded_state_dict(metadata=metadata) with TempNamedDir(tmp_path_dist_ckpt / _dist_checkpoint_name, sync=True) as tmpdirname: dist_checkpointing.save(te_fused_sharded_state_dict, tmpdirname) diff --git a/tests/unit_tests/ssm/test_gated_delta_net.py b/tests/unit_tests/ssm/test_gated_delta_net.py new file mode 100644 index 00000000000..dbf8d203634 --- /dev/null +++ b/tests/unit_tests/ssm/test_gated_delta_net.py @@ -0,0 +1,319 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + +from functools import partial +from unittest import mock + +import pytest +import torch +import torch.nn.functional as F + +from megatron.core import parallel_state +from megatron.core.models.common.embeddings.rope_utils import ( + get_pos_emb_on_this_cp_rank as get_tensor_on_this_cp_rank, +) +from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.models.gpt.gpt_model import GPTModel +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.ssm.gated_delta_net import GatedDeltaNet +from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed +from megatron.core.transformer import TransformerConfig +from megatron.training.arguments import parse_args +from megatron.training.checkpointing import load_checkpoint, save_checkpoint +from megatron.training.global_vars import set_args +from megatron.training.training import get_model +from megatron.training.utils import unwrap_model +from tests.unit_tests.dist_checkpointing import ( + TempNamedDir, + init_basic_mock_args, + init_checkpointing_mock_args, +) +from tests.unit_tests.test_utilities import Utils + +try: + import fla + + HAVE_FLA = True +except ImportError: + HAVE_FLA = False + + +@pytest.mark.parametrize( + ("tp_size", "sp", "cp_size"), + [ + (1, False, 1), + (2, False, 1), + (2, True, 1), + # GDN does not support CP for now. Leave it for future work. + ], +) +@pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.") +@pytest.mark.internal +class TestGatedDeltaNet: + + @pytest.fixture(scope='function', autouse=True) + def setup_method(self, tp_size, sp, cp_size): + # Initialize parallel and random seed + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp_size, + pipeline_model_parallel_size=1, + context_parallel_size=cp_size, + ) + model_parallel_cuda_manual_seed(123) + self.tp_size = tp_size + self.cp_size = cp_size + self.sp_size = tp_size if sp else 1 + + # Get TP and CP process groups from device mesh + tp_group = parallel_state.get_tensor_model_parallel_group() + cp_group = parallel_state.get_context_parallel_group() + pg_collection = ProcessGroupCollection(tp=tp_group, cp=cp_group) + + # Initialize model + self.transformer_config = TransformerConfig( + hidden_size=256, + linear_conv_kernel_dim=2, + linear_key_head_dim=64, + linear_value_head_dim=64, + linear_num_key_heads=4, + linear_num_value_heads=8, + num_layers=1, + normalization="RMSNorm", + use_cpu_initialization=True, + layernorm_zero_centered_gamma=True, + num_attention_heads=8, + activation_func=F.silu, + bf16=True, + tensor_model_parallel_size=tp_size, + sequence_parallel=sp, + context_parallel_size=cp_size, + ) + gdn_submodules = get_gpt_layer_with_transformer_engine_spec( + linear_attention_type="gated_delta_net", normalization="RMSNorm" + ).submodules.self_attention.submodules + + self.gdn = GatedDeltaNet( + self.transformer_config, + submodules=gdn_submodules, + layer_number=1, + bias=False, + conv_bias=False, + conv_init=1.0, + use_qk_l2norm=True, + A_init_range=(1, 16), + pg_collection=pg_collection, + ) + self.gdn = self.gdn.cuda().bfloat16() + + def teardown_method(self): + Utils.destroy_model_parallel() + + def test_gpu_forward(self): + gdn = self.gdn + + micro_batch_size = 2 + seq_length = 64 + hidden_states = torch.ones( + (seq_length // self.sp_size // self.cp_size, micro_batch_size, gdn.config.hidden_size), + device=torch.cuda.current_device(), + dtype=torch.bfloat16, + ) + attention_mask = None + + output, bias = gdn(hidden_states, attention_mask) + + assert output.dim() == 3, f"Output too many dimensions ({output.shape=})" + assert output.shape[0] == seq_length // self.sp_size // self.cp_size, ( + f"Output shape {output.shape[0]=} mismatch with " + f" {seq_length=} // {self.sp_size=} // {self.cp_size=}." + ) + assert ( + output.shape[1] == micro_batch_size + ), f"Output shape {output.shape[1]=} mismatch with {micro_batch_size=}" + assert ( + output.shape[2] == gdn.config.hidden_size + ), f"Output shape {output.shape[2]=} mismatch with {gdn.config.hidden_size=}" + assert ( + output.dtype == hidden_states.dtype + ), f"Output dtype {output.dtype=} mismatch with {hidden_states.dtype=}" + + +@pytest.mark.parametrize( + ("tp", "sp", "cp"), + [ + (4, False, 1), # TP w/o SP + (4, True, 1), # TP w/ SP + # CP does not support GDN for now. Add it once it is supported. + ], +) +@pytest.mark.skipif(not HAVE_FLA, reason="FLA is not installed.") +def test_parallel_gated_delta_net_correctness(tmp_path_dist_ckpt, tp, sp, cp): + # Constants + seed = 123 + sequence_length = 256 + micro_batch_size = 4 + hidden_size = 128 + normalization = "RMSNorm" + + # Model initialization function + def initialize_gpt_model(config, pre_process=True, post_process=True, vp_stage=None): + layer_spec = get_gpt_layer_with_transformer_engine_spec( + linear_attention_type="gated_delta_net", normalization=normalization + ) + gpt_model = GPTModel( + config=config, + transformer_layer_spec=layer_spec, + vocab_size=128, + max_sequence_length=sequence_length, + pre_process=pre_process, + post_process=post_process, + vp_stage=vp_stage, + ) + return gpt_model + + # Initialize baseline parallel state + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, pipeline_model_parallel_size=1, context_parallel_size=1 + ) + + # Initialize input hidden states + torch.manual_seed(seed) + model_parallel_cuda_manual_seed(seed) + input_hidden_states = ( + torch.rand((sequence_length, micro_batch_size, hidden_size)) + .cuda() + .bfloat16() + .requires_grad_(True) + ) + + # Initialize transformer config + transformer_config = TransformerConfig( + hidden_size=128, + linear_conv_kernel_dim=2, + linear_key_head_dim=32, + linear_value_head_dim=32, + linear_num_key_heads=4, + linear_num_value_heads=8, + num_layers=1, + normalization=normalization, + use_cpu_initialization=True, + layernorm_zero_centered_gamma=True, + num_attention_heads=8, + activation_func=F.silu, + bf16=True, + ) + + with TempNamedDir(tmp_path_dist_ckpt / 'test_parallel_gdn', sync=True) as ckpt_dir: + # Set argument + mock_args = parse_args(ignore_unknown_args=True) + set_args(mock_args) + + # Initialize baseline model + init_basic_mock_args(mock_args, 1, 1, bf16=True) + mock_args.context_parallel_size = 1 + mock_args.sequence_parallel = 1 + gpt_model = unwrap_model( + get_model(partial(initialize_gpt_model, config=transformer_config)) + ) + + # Initialize args and save checkpoint + init_checkpointing_mock_args(mock_args, ckpt_dir, False) + mock_args.no_save_optim = True + mock_args.no_save_rng = True + mock_args.no_load_optim = True + mock_args.no_load_rng = True + save_checkpoint(10, gpt_model, None, None, 0) + + # Calculate baseline output + attention = gpt_model[0].decoder.layers[0].self_attention + output_hidden_states_baseline, bias_hidden_states_baseline = attention( + input_hidden_states, attention_mask=None + ) + output_hidden_states_baseline.sum().backward() + + # Save baseline output + input_grad_baseline = input_hidden_states.grad.detach() + output_hidden_states_baseline = output_hidden_states_baseline.detach() + + # Initialize parallel model + Utils.destroy_model_parallel() + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp, pipeline_model_parallel_size=1, context_parallel_size=cp + ) + torch.manual_seed(seed) + model_parallel_cuda_manual_seed(seed) + transformer_config.context_parallel_size = cp + transformer_config.tensor_model_parallel_size = tp + transformer_config.sequence_parallel = sp + init_basic_mock_args(mock_args, tp, 1, bf16=True) + mock_args.context_parallel_size = cp + mock_args.sequence_parallel = sp + gpt_model = unwrap_model( + get_model(partial(initialize_gpt_model, config=transformer_config)) + ) + with mock.patch('megatron.training.checkpointing.check_checkpoint_args'): + with mock.patch('megatron.training.checkpointing.update_num_microbatches'): + load_checkpoint(gpt_model, None, None) + + # Function to get tensor on this tp and cp rank + cp_group = parallel_state.get_context_parallel_group() + tp_rank = parallel_state.get_tensor_model_parallel_rank() + + def get_tensor_on_this_rank(tensor): + if cp > 1: + tensor = get_tensor_on_this_cp_rank(tensor, 0, cp_group) + if tp > 1 and sp: + sp_seg = sequence_length // tp // cp + tensor = tensor[tp_rank * sp_seg : (tp_rank + 1) * sp_seg] + return tensor + + # Calculate parallel model output + input_hidden_states = get_tensor_on_this_rank(input_hidden_states) + input_hidden_states = input_hidden_states.detach().requires_grad_(True) + parallel_attention = gpt_model[0].decoder.layers[0].self_attention + output_hidden_states_parallel, bias_hidden_states_parallel = parallel_attention( + input_hidden_states, attention_mask=None + ) + output_hidden_states_parallel.sum().backward() + input_grad_parallel = input_hidden_states.grad.detach() + + # Check if the output is the same + if cp: + atol, rtol = 5e-3, 5e-3 + else: + atol, rtol = 5e-4, 5e-4 + output_hidden_states_baseline = get_tensor_on_this_rank(output_hidden_states_baseline) + input_grad_baseline = get_tensor_on_this_rank(input_grad_baseline) + + assert torch.all( + ~torch.isnan(output_hidden_states_baseline) + ), "output_hidden_states_baseline contains nan" + assert torch.all( + ~torch.isinf(output_hidden_states_baseline) + ), "output_hidden_states_baseline contains inf" + assert torch.all(~torch.isnan(input_grad_baseline)), "input_grad_baseline contains nan" + assert torch.all(~torch.isinf(input_grad_baseline)), "input_grad_baseline contains inf" + assert torch.all( + ~torch.isnan(output_hidden_states_parallel) + ), "output_hidden_states_parallel contains nan" + assert torch.all( + ~torch.isinf(output_hidden_states_parallel) + ), "output_hidden_states_parallel contains inf" + assert torch.all(~torch.isnan(input_grad_parallel)), "input_grad_parallel contains nan" + assert torch.all(~torch.isinf(input_grad_parallel)), "input_grad_parallel contains inf" + + torch.testing.assert_close( + output_hidden_states_baseline, + output_hidden_states_parallel, + atol=atol, + rtol=rtol, + msg=lambda msg: f"Mismatch in output_hidden_states: {msg}", + ) + torch.testing.assert_close( + input_grad_baseline, + input_grad_parallel, + atol=atol, + rtol=rtol, + msg=lambda msg: f"Mismatch in input_grad: {msg}", + ) + + Utils.destroy_model_parallel() diff --git a/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py b/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py new file mode 100644 index 00000000000..f7a518e8e88 --- /dev/null +++ b/tests/unit_tests/tensor_parallel/test_tp_attrs_without_init.py @@ -0,0 +1,87 @@ +import pytest +import torch + +from megatron.core.tensor_parallel.layers import ( + ColumnParallelLinear, + RowParallelLinear, + VocabParallelEmbedding, +) +from megatron.core.transformer.transformer_config import TransformerConfig +from tests.unit_tests.test_utilities import Utils + + +class TestTPAttributesWithoutInitialization: + + def teardown_method(self, method): + Utils.destroy_model_parallel() + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize("use_cpu_init", [True, False]) + def test_vocab_parallel_embedding_tp_attrs_no_init(self, use_cpu_init): + Utils.initialize_model_parallel(tensor_model_parallel_size=2) + cfg = TransformerConfig( + num_layers=1, + hidden_size=8, + num_attention_heads=4, + use_cpu_initialization=use_cpu_init, + perform_initialization=False, + ) + + emb = VocabParallelEmbedding( + num_embeddings=16, embedding_dim=8, init_method=cfg.init_method, config=cfg + ) + w = emb.weight + assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True + assert hasattr(w, "partition_dim") and w.partition_dim == 0 + assert hasattr(w, "partition_stride") and w.partition_stride == 1 + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize("use_cpu_init", [True, False]) + def test_column_parallel_linear_tp_attrs_no_init(self, use_cpu_init): + Utils.initialize_model_parallel(tensor_model_parallel_size=2) + cfg = TransformerConfig( + num_layers=1, + hidden_size=8, + num_attention_heads=4, + use_cpu_initialization=use_cpu_init, + perform_initialization=False, + ) + + layer = ColumnParallelLinear( + input_size=8, + output_size=8, + init_method=cfg.init_method, + bias=True, + config=cfg, + skip_bias_add=False, + ) + w = layer.weight + assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True + assert hasattr(w, "partition_dim") and w.partition_dim == 0 + assert hasattr(w, "partition_stride") and w.partition_stride == 1 + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") + @pytest.mark.parametrize("use_cpu_init", [True, False]) + def test_row_parallel_linear_tp_attrs_no_init(self, use_cpu_init): + Utils.initialize_model_parallel(tensor_model_parallel_size=2) + cfg = TransformerConfig( + num_layers=1, + hidden_size=8, + num_attention_heads=4, + use_cpu_initialization=use_cpu_init, + perform_initialization=False, + ) + + layer = RowParallelLinear( + input_size=8, + output_size=8, + init_method=cfg.init_method, + bias=True, + input_is_parallel=True, + config=cfg, + skip_bias_add=False, + ) + w = layer.weight + assert hasattr(w, "tensor_model_parallel") and w.tensor_model_parallel is True + assert hasattr(w, "partition_dim") and w.partition_dim == 1 + assert hasattr(w, "partition_stride") and w.partition_stride == 1 diff --git a/tests/unit_tests/test_fp8_param.py b/tests/unit_tests/test_fp8_param.py index 0cbba273507..0b8d41769ec 100644 --- a/tests/unit_tests/test_fp8_param.py +++ b/tests/unit_tests/test_fp8_param.py @@ -1,6 +1,7 @@ # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. import contextlib +import gc import os import sys @@ -8,6 +9,7 @@ import torch from transformer_engine.pytorch.fp8 import check_fp8_support +from megatron.core.distributed import DistributedDataParallel as DDP from megatron.core.enums import ModelType from megatron.core.fp8_utils import is_float8tensor from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec @@ -29,6 +31,34 @@ _SEED = 1234 fp8_available, reason_for_no_fp8 = check_fp8_support() +cuda_graph_supported = False +reason_for_no_cuda_graph = "" +try: + from transformer_engine.pytorch.tensor.utils import post_all_gather_processing + + cuda_graph_supported = True +except ImportError: + reason_for_no_cuda_graph = "Need newer TransformerEngine" + + +def enable_forward_pre_hook(model_chunks): + for model_chunk in model_chunks: + assert isinstance(model_chunk, DDP) + model_chunk.enable_forward_pre_hook() + + +def disable_forward_pre_hook(model_chunks, param_sync=True): + for model_chunk in model_chunks: + assert isinstance(model_chunk, DDP) + model_chunk.disable_forward_pre_hook(param_sync=param_sync) + + +def should_disable_forward_pre_hook(args): + """Block forward pre-hook for certain configurations.""" + return ( + not args.use_megatron_fsdp and args.use_distributed_optimizer and args.overlap_param_gather + ) + class TestFP8Param: @@ -41,6 +71,7 @@ def teardown_method(self, method): Utils.destroy_model_parallel() destroy_global_vars() destroy_num_microbatches_calculator() + gc.collect() def model_provider( self, @@ -68,7 +99,15 @@ def model_provider( ) def create_test_args( - self, tp, recipe, sequence_length, micro_batch_size, inference, fp8_param_gather, **kwargs + self, + tp, + recipe, + sequence_length, + micro_batch_size, + inference, + fp8_param_gather, + use_cuda_graph, + **kwargs, ): destroy_global_vars() destroy_num_microbatches_calculator() @@ -102,6 +141,10 @@ def create_test_args( if recipe == "mxfp8" and fp8_param_gather: args.reuse_grad_buf_for_mxfp8_param_ag = True + if use_cuda_graph: + args.cuda_graph_impl = "transformer_engine" + args.cuda_graph_warmup_steps = 0 + for key, value in kwargs.items(): assert hasattr(args, key) setattr(args, key, value) @@ -122,7 +165,13 @@ def get_batch(self, seq_length, micro_batch_size): return input_ids, labels, position_ids, attention_mask, loss_mask def _run_test_helper( - self, tp_size, recipe, inference: bool = False, fp8_param_gather: bool = True, **kwargs + self, + tp_size, + recipe, + inference: bool = False, + fp8_param_gather: bool = True, + use_cuda_graph: bool = False, + **kwargs, ): """Test fp8_param with gpt_model.""" args = self.create_test_args( @@ -132,6 +181,7 @@ def _run_test_helper( self.micro_batch_size, inference, fp8_param_gather, + use_cuda_graph, **kwargs, ) @@ -143,6 +193,7 @@ def _run_test_helper( set_args(args) torch.manual_seed(_SEED) Utils.initialize_model_parallel(tensor_model_parallel_size=tp_size) + input_ids, labels, position_ids, attention_mask, loss_mask = self.get_batch( self.seq_length, self.micro_batch_size ) @@ -158,6 +209,20 @@ def _run_test_helper( ) assert len(gpt_model) == 1 # Assume only one model in the model provider. + cuda_graph_helper = None + # Hard coded to use cuda_graph_impl="transformer_engine" + cuda_graph_impl = "transformer_engine" + if use_cuda_graph and cuda_graph_impl == "transformer_engine": + from megatron.core.transformer.cuda_graphs import TECudaGraphHelper + + cuda_graph_helper = TECudaGraphHelper( + model=gpt_model, + config=gpt_model[0].config, + seq_length=self.seq_length, + micro_batch_size=self.micro_batch_size, + optimizers=[optimizer], + ) + num_fp8_params = 0 for _, param in gpt_model[0].named_parameters(): if not inference: @@ -182,6 +247,17 @@ def _run_test_helper( gpt_model[0].zero_grad_buffer() optimizer.zero_grad() + # Capture CUDA graphs after warmup if helper is provided. + # Hard coded cuda_graph_warmup_steps = 0. + cuda_graph_warmup_steps = 0 + if cuda_graph_helper is not None and i == cuda_graph_warmup_steps: + if should_disable_forward_pre_hook(args): + disable_forward_pre_hook(gpt_model, param_sync=False) + cuda_graph_helper.create_cudagraphs() + if should_disable_forward_pre_hook(args): + enable_forward_pre_hook(gpt_model) + cuda_graph_helper.cuda_graph_set_manual_hooks() + # For the mxfp8_param with reuse_grad_buf_for_mxfp8_param_ag and dp_ag_overlap, # we need to call the _copy_main_params_to_param_buffer() after the grad buffer # is zeroed by zero_grad_buffer() because param and grad buffer are shared. @@ -239,6 +315,15 @@ def run_test(self, tp_size, recipe, inference: bool = False, **kwargs): ) torch.testing.assert_close(loss_list, loss_list_ref, atol=1e-4, rtol=1e-4) + def run_test_with_cuda_graph(self, tp_size, recipe, **kwargs): + loss = self._run_test_helper( + tp_size, recipe, fp8_param_gather=True, use_cuda_graph=True, **kwargs + ) + loss_ref = self._run_test_helper( + tp_size, recipe, fp8_param_gather=True, use_cuda_graph=False, **kwargs + ) + torch.testing.assert_close(loss, loss_ref, atol=0, rtol=0) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) @pytest.mark.parametrize("tp_size", [2]) @pytest.mark.parametrize("dp_overlap", [(True, True)]) @@ -246,6 +331,14 @@ def test_delayed_scaling(self, tp_size, dp_overlap): kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} self.run_test(tp_size=tp_size, recipe="delayed", **kwargs) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) + @pytest.mark.parametrize("tp_size", [2]) + @pytest.mark.parametrize("dp_overlap", [(True, True)]) + @pytest.mark.skipif(not cuda_graph_supported, reason=reason_for_no_cuda_graph) + def test_delayed_scaling_with_cuda_graph(self, tp_size, dp_overlap): + kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} + self.run_test_with_cuda_graph(tp_size, "delayed", **kwargs) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) @pytest.mark.skipif(not is_te_min_version("2.2.0"), reason="TE 2.2.0 is required") @pytest.mark.parametrize("tp_size", [2]) @@ -260,6 +353,15 @@ def test_tensorwise_scaling(self, tp_size, dp_overlap): def test_tensorwise_scaling_inference(self, tp_size): self.run_test(tp_size=tp_size, recipe="tensorwise", inference=True) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) + @pytest.mark.skipif(not is_te_min_version("2.2.0"), reason="TE 2.2.0 is required") + @pytest.mark.parametrize("tp_size", [2]) + @pytest.mark.parametrize("dp_overlap", [(True, True)]) + @pytest.mark.skipif(not cuda_graph_supported, reason=reason_for_no_cuda_graph) + def test_tensorwise_scaling_with_cuda_graph(self, tp_size, dp_overlap): + kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} + self.run_test_with_cuda_graph(tp_size, "tensorwise", **kwargs) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) @pytest.mark.skipif(not is_te_min_version("2.2.0"), reason="TE 2.2.0 is required") @pytest.mark.parametrize("tp_size", [2]) @@ -282,6 +384,18 @@ def test_blockwise_scaling(self, tp_size, dp_overlap): kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} self.run_test(tp_size=tp_size, recipe="blockwise") + @pytest.mark.skipif( + get_device_arch_version() != 9, reason="blockwise is only supported on Hopper architecture" + ) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) + @pytest.mark.skipif(not is_te_min_version("2.4.0.dev0"), reason="TE 2.4.0.dev0 is required") + @pytest.mark.parametrize("tp_size", [2]) + @pytest.mark.parametrize("dp_overlap", [(True, True)]) + @pytest.mark.skipif(not cuda_graph_supported, reason=reason_for_no_cuda_graph) + def test_blockwise_scaling_with_cuda_graph(self, tp_size, dp_overlap): + kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} + self.run_test_with_cuda_graph(tp_size, "blockwise", **kwargs) + @pytest.mark.skipif( get_device_arch_version() < 10, reason="MXFP8 is supported since Blackwell architecture" ) @@ -296,6 +410,21 @@ def test_mxfp8(self, tp_size, dp_overlap): kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} self.run_test(tp_size=tp_size, recipe="mxfp8", **kwargs) + @pytest.mark.skipif( + get_device_arch_version() < 10, reason="MXFP8 is supported since Blackwell architecture" + ) + @pytest.mark.skipif(not fp8_available, reason=reason_for_no_fp8) + @pytest.mark.skipif(not is_te_min_version("2.3.0.dev0"), reason="TE 2.3.0.dev0 is required") + @pytest.mark.parametrize("tp_size", [2]) + @pytest.mark.parametrize("dp_overlap", [(False, False), (False, True), (True, True)]) + @pytest.mark.skipif(not cuda_graph_supported, reason=reason_for_no_cuda_graph) + def test_mxfp8_with_cuda_graph(self, tp_size, dp_overlap): + """ + dp_overlap: (overlap_param_gather, overlap_grad_reduce) + """ + kwargs = {"overlap_param_gather": dp_overlap[0], "overlap_grad_reduce": dp_overlap[1]} + self.run_test_with_cuda_graph(tp_size=tp_size, recipe="mxfp8", **kwargs) + @pytest.mark.skipif( get_device_arch_version() != 9, reason="blockwise is only supported on Hopper architecture" ) diff --git a/tests/unit_tests/test_inference.py b/tests/unit_tests/test_inference.py index 518aa7f4126..5b2bc07e6e1 100644 --- a/tests/unit_tests/test_inference.py +++ b/tests/unit_tests/test_inference.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import argparse import unittest.mock diff --git a/tests/unit_tests/test_layer_wise_optimizer.py b/tests/unit_tests/test_layer_wise_optimizer.py new file mode 100644 index 00000000000..05ce26bcfa0 --- /dev/null +++ b/tests/unit_tests/test_layer_wise_optimizer.py @@ -0,0 +1,440 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import os + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from packaging.version import Version + +from megatron.core import parallel_state +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig, get_megatron_optimizer +from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer +from megatron.core.optimizer.optimizer import Float16OptimizerWithFloat16Params, FP32Optimizer +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer import TransformerConfig +from megatron.core.utils import get_pg_size +from tests.unit_tests.test_utilities import Utils + +# Skip all tests in this file for LTS versions +pytestmark = pytest.mark.skipif( + Version(os.getenv('NVIDIA_PYTORCH_VERSION', "24.01")) <= Version("25.05"), + reason="Skip layer-wise optimizer for LTS test", +) + + +class SimpleModel(nn.Module): + """Simple model for testing LayerWiseDistributedOptimizer. + + Model with 5 layers to ensure more than 8 parameters (10 total: 5 weights + 5 biases). + """ + + def __init__(self, input_size=80, hidden_size=48, output_size=10): + super().__init__() + self.fc1 = nn.Linear(input_size, hidden_size) + self.fc2 = nn.Linear(hidden_size, 32) + self.fc3 = nn.Linear(32, 24) + self.fc4 = nn.Linear(24, 16) + self.fc5 = nn.Linear(16, output_size) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + x = F.relu(self.fc4(x)) + x = self.fc5(x) + return x + + +class TinyModel(nn.Module): + """Tiny model with only 1 layer (2 parameters: weight and bias).""" + + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(10, 5) + + def forward(self, x): + return self.fc1(x) + + +@pytest.mark.skipif( + int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1" +) +class TestLayerWiseOptimizer: + """Test class for LayerWiseDistributedOptimizer with common setup code.""" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + """Setup and teardown for each test.""" + world = int(os.getenv('WORLD_SIZE', '1')) + rank = int(os.getenv('RANK', '0')) + Utils.initialize_model_parallel() + yield + Utils.destroy_model_parallel() + + def create_model_and_optimizer( + self, + model_class=SimpleModel, + clip_grad=1.0, + model_kwargs=None, + use_layer_wise=True, + copy_from=None, + ): + """Create model, DDP wrapper, and optimizer. + + Args: + model_class: Model class to instantiate + clip_grad: Optional gradient clipping value + model_kwargs: Optional kwargs for model initialization + use_layer_wise: If True, wrap optimizer in LayerWiseDistributedOptimizer; + if False, use get_megatron_optimizer instead (for reference) + + Returns: + tuple: (model, optimizer, pg_collection) + """ + if model_kwargs is None: + model_kwargs = {} + + model = model_class(**model_kwargs).bfloat16().cuda() + model.requires_grad_(True) + + ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False) + model = DistributedDataParallel( + TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model + ) + if copy_from: + model.module.load_state_dict(copy_from.module.state_dict()) + else: + model.broadcast_params() + + optimizer_config = OptimizerConfig( + optimizer='adam', + lr=0.01, + weight_decay=0.01, + bf16=not use_layer_wise, + use_distributed_optimizer=False, + clip_grad=clip_grad, + ) + + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True) + pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group() + + optimizer = get_megatron_optimizer(optimizer_config, [model]) + if use_layer_wise: + optimizer_config.bf16 = True + optimizer = LayerWiseDistributedOptimizer( + optimizer.chained_optimizers, optimizer_config, pg_collection + ) + return model, optimizer, pg_collection + + def create_reference_model(self, model): + """Create a reference model by cloning the current model.""" + reference_model = type(model.module)().bfloat16().cuda() + reference_model.load_state_dict(model.module.state_dict()) + return reference_model + + def test_basic(self): + """Test basic LayerWiseDistributedOptimizer initialization and step with bf16.""" + model, optimizer, pg_collection = self.create_model_and_optimizer() + + # Verify basic properties + assert optimizer is not None, "Optimizer should not be None" + assert hasattr(optimizer, 'chained_optimizers'), "Should be a ChainedOptimizer" + + reference_model = self.create_reference_model(model) + + input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + update_successful, grad_norm, num_zeros = optimizer.step() + + assert update_successful, "Optimizer step should be successful" + + # Verify parameters were updated + params_updated = 0 + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + if not torch.equal(param.data, ref_param.data): + params_updated += 1 + + assert params_updated > 0, "At least some parameters should be updated" + + # Verify all ranks have the same updated parameters (test allgather) + dp_size = get_pg_size(pg_collection.dp_cp) + + if dp_size > 1: + for name, param in model.named_parameters(): + # Gather parameters from all ranks + param_list = [torch.zeros_like(param.data) for _ in range(dp_size)] + torch.distributed.all_gather(param_list, param.data, group=pg_collection.dp_cp) + + # Verify all ranks have the same parameter values + for i in range(1, dp_size): + try: + torch.testing.assert_close(param_list[0], param_list[i]) + except AssertionError as e: + # Append additional context without overwriting the default message + raise AssertionError( + f"Parameter {name} differs between rank 0 and rank {i}. {str(e)}" + ) from None + + def test_get_grad_norm(self): + """Test LayerWiseDistributedOptimizer gradient norm computation.""" + model, optimizer, pg_collection = self.create_model_and_optimizer() + reference_model, reference_optimizer, _ = self.create_model_and_optimizer( + use_layer_wise=False + ) + + # Set same gradients on both models + # note that model is different at this point but we're only testing grad norm here + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + grad_value = torch.randn_like(param) + torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp) + param.main_grad = grad_value.float().detach() + ref_param.main_grad = grad_value.float().detach() + + # Test get_grad_norm on both optimizers + optimizer.prepare_grads() + grad_norm = optimizer.get_grad_norm() + + reference_optimizer.prepare_grads() + reference_grad_norm = reference_optimizer.get_grad_norm() + + assert grad_norm is not None, "Grad norm should not be None" + assert grad_norm >= 0, "Grad norm should be non-negative" + + # Compare with reference optimizer grad norm + torch.testing.assert_close(grad_norm, reference_grad_norm, rtol=1e-5, atol=1e-5) + + def test_state_dict(self): + """Test LayerWiseDistributedOptimizer state dict save and load.""" + model, optimizer, pg_collection = self.create_model_and_optimizer() + + for param in model.parameters(): + param.grad = torch.randn_like(param) + optimizer.step() + + # Test state_dict + state_dict = optimizer.state_dict() + + # Test load_state_dict + # TODO(deyuf): fix this. not going through get() will cause missing keys like wd_mult + # optimizer.load_state_dict(state_dict) + + def test_sharded_state_dict(self): + """Test LayerWiseDistributedOptimizer sharded_state_dict method.""" + model, optimizer, pg_collection = self.create_model_and_optimizer() + + for param in model.parameters(): + param.grad = torch.randn_like(param) + optimizer.step() + + # Get model sharded state dict + model_sharded_state_dict = model.sharded_state_dict() + + # Test sharded_state_dict + sharded_state_dict = optimizer.sharded_state_dict(model_sharded_state_dict) + + # Verify the sharded_state_dict is not None and has expected structure + assert sharded_state_dict is not None, "Sharded state dict should not be None" + assert ( + 'optimizer' in sharded_state_dict + ), "Sharded state dict should contain 'optimizer' key" + + # Verify that replica_id is set correctly (should be 0 for DP dimension) + from megatron.core.dist_checkpointing import ShardedTensor + from megatron.core.dist_checkpointing.dict_utils import nested_values + + for sh_base in nested_values(sharded_state_dict): + if isinstance(sh_base, ShardedTensor): + assert ( + len(sh_base.replica_id) == 3 + ), f'Expected replica_id format (PP, TP, DP), got: {sh_base.replica_id}' + assert ( + sh_base.replica_id[2] == 0 + ), f'Expected DP replica_id to be 0 for layer-wise optimizer, got: {sh_base.replica_id[2]}' + + def test_multiple_optimizers(self): + """Test LayerWiseDistributedOptimizer with multiple chained optimizers. + + This test properly tests allgather functionality with multiple ranks. + """ + model = SimpleModel().bfloat16().cuda() + model.requires_grad_(True) + + ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False) + model = DistributedDataParallel( + TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model + ) + + optimizer_config = OptimizerConfig( + optimizer='adam', lr=0.01, bf16=True, use_distributed_optimizer=False + ) + + # Split parameters into two groups for testing multiple optimizers + params = list(model.parameters()) + mid_point = len(params) // 2 + param_groups_1 = [{'params': params[:mid_point]}] + param_groups_2 = [{'params': params[mid_point:]}] + + # Create two separate base optimizers + base_optimizer_1 = torch.optim.Adam(param_groups_1, lr=optimizer_config.lr) + base_optimizer_2 = torch.optim.Adam(param_groups_2, lr=optimizer_config.lr) + + wrapped_optimizer_1 = FP32Optimizer(base_optimizer_1, optimizer_config, None) + wrapped_optimizer_2 = FP32Optimizer(base_optimizer_2, optimizer_config, None) + + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True) + pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group() + + optimizer = LayerWiseDistributedOptimizer( + [wrapped_optimizer_1, wrapped_optimizer_2], optimizer_config, pg_collection + ) + + assert len(optimizer.chained_optimizers) == 2, "Should have two chained optimizers" + + # Set gradients and test optimizer step - this will trigger allgather + for param in model.parameters(): + param.grad = torch.randn_like(param) + + update_successful, grad_norm, num_zeros = optimizer.step() + + assert update_successful, "Optimizer step should be successful" + + def test_bf16_wrapping(self): + """Test LayerWiseDistributedOptimizer automatically wraps optimizer with bf16.""" + model, optimizer, pg_collection = self.create_model_and_optimizer() + + # Verify bf16 wrapping happened + assert isinstance( + optimizer.chained_optimizers[0], Float16OptimizerWithFloat16Params + ), "Optimizer should be wrapped in Float16OptimizerWithFloat16Params" + + for param in model.parameters(): + param.grad = torch.randn_like(param) + + update_successful, grad_norm, num_zeros = optimizer.step() + + assert update_successful, "Optimizer step should be successful" + + def test_bf16_error(self): + """Test LayerWiseDistributedOptimizer raises error when receiving pre-wrapped Float16 optimizer.""" + model = SimpleModel().bfloat16().cuda() + model.requires_grad_(True) + + ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False) + model = DistributedDataParallel( + TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model + ) + + optimizer_config = OptimizerConfig( + optimizer='adam', lr=0.01, bf16=True, use_distributed_optimizer=False + ) + + # Create base optimizer and manually wrap in Float16 optimizer + param_groups = [{'params': list(model.parameters())}] + base_optimizer = torch.optim.Adam(param_groups, lr=optimizer_config.lr) + wrapped_optimizer = Float16OptimizerWithFloat16Params( + base_optimizer, optimizer_config, None, None + ) + + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + pg_collection.dp_cp = parallel_state.get_data_parallel_group(with_context_parallel=True) + pg_collection.expt_dp = parallel_state.get_expert_data_parallel_group() + + # Should raise TypeError when receiving already-wrapped Float16 optimizer + with pytest.raises( + TypeError, match='LayerWiseDistributedOptimizer received Float16 optimizer already' + ): + LayerWiseDistributedOptimizer([wrapped_optimizer], optimizer_config, pg_collection) + + def _run_parameter_update_test(self, model_class=SimpleModel): + """Helper method to test parameter updates with a given model class. + + Args: + model_class: Model class to use for testing + """ + model, optimizer, pg_collection = self.create_model_and_optimizer(model_class=model_class) + + # Create reference model and optimizer using the same function + reference_model, reference_optimizer, _ = self.create_model_and_optimizer( + model_class=model_class, use_layer_wise=False, copy_from=model + ) + + # Set same gradients on both models + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + assert torch.equal(param.data, ref_param.data) + torch.testing.assert_close(param.data, ref_param.data, rtol=1e-5, atol=1e-5) + grad_value = torch.randn_like(param) + torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp) + param.main_grad = grad_value.clone().detach() + ref_param.main_grad = grad_value.clone().detach() + + optimizer.step() + + # Verify at least some parameters were updated + params_updated = 0 + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + if not torch.equal(param.data, ref_param.data): + params_updated += 1 + + assert params_updated > 0, "At least some parameters should be updated" + + reference_optimizer.step() + + # Verify updated values match reference optimizer + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + torch.testing.assert_close(param.data, ref_param.data, rtol=1e-5, atol=1e-5) + + def test_parameter_updates(self): + """Test LayerWiseDistributedOptimizer actually updates model parameters.""" + self._run_parameter_update_test() + + def test_parameter_updates_insufficient_parameters(self): + """Test LayerWiseDistributedOptimizer when there are insufficient parameters for all ranks. + + Uses a tiny model with only 1 layer (2 parameters: weight and bias). + This will be insufficient when world size > 2. + """ + self._run_parameter_update_test(model_class=TinyModel) + + def test_broadcast_vs_allgather(self): + """Test LayerWiseDistributedOptimizer allgather code agains broadcast code.""" + model, optimizer, pg_collection = self.create_model_and_optimizer(model_class=SimpleModel) + + # Create reference model and optimizer using the same function + reference_model, reference_optimizer, _ = self.create_model_and_optimizer( + model_class=SimpleModel, copy_from=model + ) + + # Set same gradients on both models + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + assert torch.equal(param.data, ref_param.data) + torch.testing.assert_close(param.data, ref_param.data, rtol=0, atol=0) + grad_value = torch.randn_like(param) + torch.distributed.broadcast(grad_value, src=0, group=pg_collection.dp_cp) + param.main_grad = grad_value.clone().detach() + ref_param.main_grad = grad_value.clone().detach() + + optimizer.step() + + # Verify at least some parameters were updated + params_updated = 0 + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + if not torch.equal(param.data, ref_param.data): + params_updated += 1 + + assert params_updated > 0, "At least some parameters should be updated" + + # step() internal call allgather_params. replace reference object with bcast + reference_optimizer.allgather_params = reference_optimizer.broadcast_params + reference_optimizer.step() + + # Verify updated values match reference optimizer + for param, ref_param in zip(model.parameters(), reference_model.parameters()): + torch.testing.assert_close(param.data, ref_param.data, rtol=0, atol=0) diff --git a/tests/unit_tests/test_muon_optimizer.py b/tests/unit_tests/test_muon_optimizer.py new file mode 100644 index 00000000000..cc99f7a16e6 --- /dev/null +++ b/tests/unit_tests/test_muon_optimizer.py @@ -0,0 +1,670 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +import os + +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +from packaging.version import Version + +from megatron.core import parallel_state +from megatron.core.distributed import DistributedDataParallel, DistributedDataParallelConfig +from megatron.core.optimizer import OptimizerConfig +from megatron.core.optimizer.muon import TensorParallelMuon, get_megatron_muon_optimizer +from megatron.core.process_groups_config import ProcessGroupCollection +from megatron.core.transformer import TransformerConfig +from tests.unit_tests.test_utilities import Utils + +# Skip all tests in this file for LTS versions +pytestmark = pytest.mark.skipif( + Version(os.getenv('NVIDIA_PYTORCH_VERSION', "24.01")) <= Version("25.05"), + reason="Skip muon optimizer for LTS test", +) + + +class Net(nn.Module): + def __init__(self): + super().__init__() + self.fc1 = nn.Linear(80, 48) + self.fc2 = nn.Linear(48, 32) + self.fc3 = nn.Linear(32, 24) + self.fc4 = nn.Linear(24, 16) + self.fc5 = nn.Linear(16, 10) + + def forward(self, x): + x = F.relu(self.fc1(x)) + x = F.relu(self.fc2(x)) + x = F.relu(self.fc3(x)) + x = F.relu(self.fc4(x)) + x = self.fc5(x) + return x + + +def test_muon_optimizer_smoke(): + """Smoke test for TensorParallelMuon optimizer.""" + # Create a simple linear model for testing + model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + # Create TensorParallelMuon optimizer + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + momentum_beta=0.95, + use_nesterov=True, + weight_decay=0.01, + use_decoupled_weight_decay=True, + split_qkv=False, + fp32_matmul_prec="medium", + num_ns_steps=5, + scale_mode="spectral", + extra_scale_factor=1.0, + pg_collection=None, + mode="duplicated", + ) + + # Test basic properties + assert optimizer is not None, "Optimizer should not be None" + assert hasattr(optimizer, 'param_groups'), "Optimizer should have param_groups" + assert len(optimizer.param_groups) > 0, "Optimizer should have at least one parameter group" + + # Test forward and backward pass + input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + # Store original weight + original_weight = model.weight.data.clone() + + # Test optimizer step + optimizer.step() + + # Verify weight was updated + assert not torch.equal( + model.weight.data, original_weight + ), "Weight should be updated after optimizer step" + + # Test zero_grad + optimizer.zero_grad() + assert model.weight.grad is None or torch.all( + model.weight.grad == 0 + ), "Gradients should be zeroed" + + # Test state_dict and load_state_dict + state_dict = optimizer.state_dict() + assert 'state' in state_dict, "State dict should contain state" + assert 'param_groups' in state_dict, "State dict should contain param_groups" + + # Load state dict should not raise error + optimizer.load_state_dict(state_dict) + + +@pytest.mark.skipif( + int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1" +) +class TestMuonOptimizerMultiRank: + """Test class for Muon optimizer with multi-rank setup.""" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + """Setup and teardown for each test.""" + Utils.initialize_model_parallel() + yield + Utils.destroy_model_parallel() + + def create_ddp_model(self, model): + """Wrap model in DDP. + + Args: + model: Model to wrap + + Returns: + DDP-wrapped model + """ + ddp_config = DistributedDataParallelConfig(use_distributed_optimizer=False) + return DistributedDataParallel( + TransformerConfig(num_attention_heads=1, num_layers=1), ddp_config, model + ) + + def test_get_megatron_muon_optimizer_smoke(self): + """Smoke test for get_megatron_muon_optimizer function.""" + model = Net().bfloat16().cuda() + model.requires_grad_(True) + model = self.create_ddp_model(model) + + # Ensure all parameters require gradients + for param in model.parameters(): + assert param.requires_grad, "All parameters should require gradients" + + # Create optimizer config for Muon + optimizer_config = OptimizerConfig( + optimizer='muon', # This will be changed internally to 'adam' for non-linear params + lr=0.01, + weight_decay=0.01, + bf16=True, + use_distributed_optimizer=False, # Muon doesn't support distributed optimizer + muon_momentum=0.95, + muon_use_nesterov=True, + muon_fp32_matmul_prec="medium", + muon_num_ns_steps=5, + muon_scale_mode="spectral", + muon_tp_mode="duplicated", + ) + + # Test creating the optimizer + optimizer = get_megatron_muon_optimizer( + config=optimizer_config, + model_chunks=[model], + use_gloo_process_groups=True, + layer_wise_distributed_optimizer=False, + ) + + # Test basic properties + assert optimizer is not None, "Optimizer should not be None" + assert hasattr(optimizer, 'param_groups'), "Optimizer should have param_groups" + assert hasattr(optimizer, 'chained_optimizers'), "Should be a ChainedOptimizer" + assert len(optimizer.chained_optimizers) >= 1, "Should have at least one chained optimizer" + + # Test forward and backward pass + input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + # Store original parameters + original_params = {} + for name, param in model.named_parameters(): + original_params[name] = param.data.clone() + + # Test optimizer step + optimizer.step() + + # Verify at least some parameters were updated + params_updated = 0 + for name, param in model.named_parameters(): + if not torch.equal(param.data, original_params[name]): + params_updated += 1 + + assert params_updated > 0, "At least some parameters should be updated after optimizer step" + + # Test zero_grad + optimizer.zero_grad() + for param in model.parameters(): + assert param.grad is None or torch.all( + param.grad == 0 + ), f"Gradients should be zeroed for all parameters" + + # Test state_dict and load_state_dict + state_dict = optimizer.state_dict() + assert isinstance(state_dict, list), "State dict should be a list" + + # Load state dict should not raise error + optimizer.load_state_dict(state_dict) + + def test_get_megatron_muon_optimizer_validation(self): + """Test validation logic for get_megatron_muon_optimizer.""" + model = torch.nn.Linear(100, 50, bias=False, dtype=torch.bfloat16, device='cuda') + model.requires_grad_(True) + model = self.create_ddp_model(model) + + # Test 1: Distributed optimizer should raise exception + optimizer_config_dist = OptimizerConfig( + optimizer='muon', + lr=0.01, + bf16=True, + use_distributed_optimizer=True, # This should cause an exception + ) + + with pytest.raises(Exception, match='muon with dist optimizer is not supported'): + get_megatron_muon_optimizer(config=optimizer_config_dist, model_chunks=[model]) + + # Test 2: FP16 should raise exception + optimizer_config_fp16 = OptimizerConfig( + optimizer='muon', + lr=0.01, + fp16=True, # This should cause an exception + use_distributed_optimizer=False, + ) + + with pytest.raises(Exception, match='muon with fp16 is not supported'): + get_megatron_muon_optimizer(config=optimizer_config_fp16, model_chunks=[model]) + + # Test 3: Invalid num_ns_steps should raise exception + optimizer_config_invalid_ns = OptimizerConfig( + optimizer='muon', + lr=0.01, + bf16=True, + use_distributed_optimizer=False, + muon_num_ns_steps=0, # This should cause an exception + ) + + with pytest.raises(ValueError, match='num_ns_steps must be at least 1'): + get_megatron_muon_optimizer(config=optimizer_config_invalid_ns, model_chunks=[model]) + + def test_get_megatron_muon_optimizer_layer_wise(self): + """Test get_megatron_muon_optimizer with layer-wise distributed optimizer.""" + model = Net().bfloat16().cuda() + model.requires_grad_(True) + model = self.create_ddp_model(model) + + optimizer_config = OptimizerConfig( + optimizer='muon', + lr=0.01, + weight_decay=0.01, + bf16=True, + use_distributed_optimizer=False, + muon_momentum=0.95, + muon_use_nesterov=True, + muon_fp32_matmul_prec="medium", + muon_num_ns_steps=5, + muon_scale_mode="spectral", + muon_tp_mode="duplicated", + ) + + # Test with layer_wise_distributed_optimizer=True + optimizer = get_megatron_muon_optimizer( + config=optimizer_config, + model_chunks=[model], + use_gloo_process_groups=True, + layer_wise_distributed_optimizer=True, + ) + + # Verify it's a LayerWiseDistributedOptimizer + from megatron.core.optimizer.layer_wise_optimizer import LayerWiseDistributedOptimizer + + assert isinstance( + optimizer, LayerWiseDistributedOptimizer + ), "Should return LayerWiseDistributedOptimizer" + + # Test forward and backward pass + input_tensor = torch.randn(16, 80, dtype=torch.bfloat16, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + # Test optimizer step + update_successful, grad_norm, num_zeros = optimizer.step() + + assert update_successful, "Optimizer step should be successful" + assert grad_norm is not None or grad_norm is None, "Grad norm should be returned" + + +@pytest.mark.parametrize("mode", ["duplicated", "blockwise", "distributed"]) +def test_muon_optimizer_different_modes_single_rank(mode): + """Test TensorParallelMuon optimizer with different modes on single rank. + + When TP size is 1, all modes should produce the same result. + """ + # Set random seed for reproducibility + torch.manual_seed(42) + torch.cuda.manual_seed(42) + + model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.normal_(0, 0.02) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + momentum_beta=0.95, + weight_decay=0.0, # Disable weight decay for deterministic comparison + num_ns_steps=5, + pg_collection=None, + mode=mode, + ) + + # Use fixed input for deterministic results + torch.manual_seed(42) + input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda') + + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + # Verify weight was updated + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with mode={mode}" + + +@pytest.mark.skipif( + int(os.getenv('WORLD_SIZE', '1')) == 1, reason="Multi-rank test requires WORLD_SIZE > 1" +) +class TestMuonOptimizerMultiRankTP: + """Test class for Muon optimizer with multi-rank and tensor parallel setup.""" + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + """Setup and teardown for each test with tensor parallel.""" + world = int(os.getenv('WORLD_SIZE', '1')) + Utils.initialize_model_parallel(tensor_model_parallel_size=min(world, 2)) + yield + Utils.destroy_model_parallel() + + def create_tp_model_and_optimizer(self, mode): + """Create model with TP and optimizer. + + Args: + mode: Muon optimizer mode + + Returns: + tuple: (model, optimizer, pg_collection) + """ + rank = int(os.getenv('RANK', '0')) + pg_collection = ProcessGroupCollection.use_mpu_process_groups() + + # Create model with partition_dim for TP + torch.manual_seed(42 + rank) + model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.normal_(0, 0.02) + model.weight.partition_dim = 0 # Set partition dimension for TP + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + momentum_beta=0.95, + weight_decay=0.0, + num_ns_steps=5, + pg_collection=pg_collection, + mode=mode, + ) + + return model, optimizer + + @pytest.mark.parametrize("mode", ["duplicated", "distributed"]) + def test_muon_optimizer_modes_multirank_same_result(self, mode): + """Test that duplicated and distributed modes produce same results with TP > 1.""" + model, optimizer = self.create_tp_model_and_optimizer(mode) + + # Use fixed input for deterministic results + torch.manual_seed(42) + input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda') + + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + # Verify weight was updated + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with mode={mode}" + + def test_muon_optimizer_blockwise_mode_different_result(self): + """Test that blockwise mode produces different results than duplicated/distributed with TP > 1.""" + model, optimizer = self.create_tp_model_and_optimizer("blockwise") + + # Use fixed input for deterministic results + torch.manual_seed(42) + input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda') + + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + # Verify weight was updated + assert not torch.equal( + model.weight.data, original_weight + ), "Weight should be updated with mode=blockwise" + + +@pytest.mark.parametrize( + "coefficient_type_and_steps", [("simple", 3), ("quintic", 5), ("polar_express", 8)] +) +def test_muon_optimizer_coefficient_types(coefficient_type_and_steps): + """Test TensorParallelMuon optimizer with different coefficient types.""" + model = torch.nn.Linear(80, 40, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + coefficient_type=coefficient_type_and_steps[0], + num_ns_steps=coefficient_type_and_steps[1], + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, 80, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with coefficient_type={coefficient_type_and_steps[0]} and num_ns_steps={coefficient_type_and_steps[1]}" + + +@pytest.mark.parametrize("scale_mode", ["spectral", "unit_rms_norm", "shape_scaling"]) +def test_muon_optimizer_scale_modes(scale_mode): + """Test TensorParallelMuon optimizer with different scale modes.""" + model = torch.nn.Linear(60, 30, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + scale_mode=scale_mode, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, 60, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with scale_mode={scale_mode}" + + +@pytest.mark.parametrize("use_nesterov", [True, False]) +def test_muon_optimizer_nesterov(use_nesterov): + """Test TensorParallelMuon optimizer with and without Nesterov momentum.""" + model = torch.nn.Linear(50, 25, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + momentum_beta=0.9, + use_nesterov=use_nesterov, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, 50, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with use_nesterov={use_nesterov}" + + +def test_muon_optimizer_multiple_steps(): + """Test TensorParallelMuon optimizer across multiple optimization steps.""" + model = torch.nn.Linear(100, 50, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + momentum_beta=0.95, + weight_decay=0.01, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + weights_history = [model.weight.data.clone()] + + for i in range(3): + input_tensor = torch.randn(32, 100, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + optimizer.step() + optimizer.zero_grad() + weights_history.append(model.weight.data.clone()) + + # Verify weights changed at each step + for i in range(len(weights_history) - 1): + assert not torch.equal( + weights_history[i], weights_history[i + 1] + ), f"Weight should change at step {i}" + + +def test_muon_optimizer_qkv_split(): + """Test TensorParallelMuon optimizer with QKV splitting.""" + # Create a model with QKV-like parameter + qkv_size = 3 * 64 * 16 # Combined Q, K, V dimensions, 16 heads x 64 per head + hidden_size = 1024 + model = torch.nn.Linear(hidden_size, qkv_size, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + # Mark parameter as QKV + model.weight.is_qkv = True + + # QKV split shapes: [Q_size, K_size, V_size] + qkv_split_shapes = (64, 64, 64) + + # Test with split_qkv=True + optimizer_split = TensorParallelMuon( + params=[model.weight], + lr=0.01, + split_qkv=True, + is_qkv_fn=lambda p: getattr(p, 'is_qkv', False), + qkv_split_shapes=qkv_split_shapes, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, hidden_size, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer_split.step() + weight_with_split = model.weight.data.clone() + + assert not torch.equal( + weight_with_split, original_weight + ), "QKV weight should be updated with split_qkv=True" + + # Reset model and test with split_qkv=False + model.weight.data.fill_(1.0) + optimizer_no_split = TensorParallelMuon( + params=[model.weight], + lr=0.01, + split_qkv=False, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + output = model(input_tensor) + loss = output.sum() + loss.backward() + + optimizer_no_split.step() + weight_without_split = model.weight.data.clone() + + assert not torch.equal( + weight_without_split, original_weight + ), "QKV weight should be updated with split_qkv=False" + + # Ensure the two results are different + assert not torch.equal( + weight_with_split, weight_without_split + ), "Weights should be different between split_qkv=True and split_qkv=False" + + +def test_muon_optimizer_extra_scale_factor(): + """Test TensorParallelMuon optimizer with different extra_scale_factor values.""" + model = torch.nn.Linear(80, 40, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + extra_scale_factor=2.0, + num_ns_steps=5, + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, 80, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + assert not torch.equal( + model.weight.data, original_weight + ), "Weight should be updated with extra_scale_factor" + + +@pytest.mark.parametrize("num_ns_steps", [5, 15, 25]) +def test_muon_optimizer_num_ns_steps(num_ns_steps): + """Test TensorParallelMuon optimizer with different numbers of Newton-Schulz steps.""" + model = torch.nn.Linear(60, 30, bias=False, dtype=torch.float32, device='cuda') + model.requires_grad_(True) + model.weight.data.fill_(1.0) + + optimizer = TensorParallelMuon( + params=[model.weight], + lr=0.01, + coefficient_type="quintic", + num_ns_steps=num_ns_steps, + pg_collection=None, + mode="duplicated", + ) + + input_tensor = torch.randn(16, 60, dtype=torch.float32, device='cuda') + output = model(input_tensor) + loss = output.sum() + loss.backward() + + original_weight = model.weight.data.clone() + optimizer.step() + + assert not torch.equal( + model.weight.data, original_weight + ), f"Weight should be updated with num_ns_steps={num_ns_steps}" diff --git a/tests/unit_tests/test_optimizer.py b/tests/unit_tests/test_optimizer.py index d8f6e3a2eeb..3d6b4b3c15e 100644 --- a/tests/unit_tests/test_optimizer.py +++ b/tests/unit_tests/test_optimizer.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + import os import pytest @@ -244,24 +246,13 @@ def run_model(model, input, optim, fp8_recipe, fp8_recipe_settings): test_model, input, test_optim, fp8_recipe, fp8_recipe_settings ) - rtol = 1e-3 # relative tolerance - atol = 1e-5 # absolute tolerance + rtol, atol = 1.6e-2, 1e-5 # Compare grad norms - allow small difference due to precision - rel_diff = abs(test_grad_norm - baseline_grad_norm) / ( - abs(baseline_grad_norm) + 1e-7 # avoid div by 0 - ) - abs_diff = abs(test_grad_norm - baseline_grad_norm) - assert ( - rel_diff <= rtol or abs_diff <= atol - ), f"Grad norm mismatch: baseline={baseline_grad_norm}, test={test_grad_norm}, rel_diff={rel_diff}, abs_diff={abs_diff}" + torch.testing.assert_close(test_grad_norm, baseline_grad_norm, atol=atol, rtol=rtol) # Compare losses - allow small difference due to precision - loss_rel_diff = abs(test_loss - baseline_loss) / (abs(baseline_loss) + 1e-7) - loss_abs_diff = abs(test_loss - baseline_loss) - assert ( - loss_rel_diff <= rtol or loss_abs_diff <= atol - ), f"Loss mismatch: baseline={baseline_loss}, test={test_loss}, rel_diff={loss_rel_diff}, abs_diff={loss_abs_diff}" + torch.testing.assert_close(test_loss, baseline_loss, atol=atol, rtol=rtol) # Save and reload state dict for the test model state_dict = test_optim.state_dict() diff --git a/tests/unit_tests/transformer/moe/test_shared_experts.py b/tests/unit_tests/transformer/moe/test_shared_experts.py index f721c482937..6df4d2fd369 100644 --- a/tests/unit_tests/transformer/moe/test_shared_experts.py +++ b/tests/unit_tests/transformer/moe/test_shared_experts.py @@ -20,7 +20,8 @@ def teardown_method(self, method): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @pytest.mark.internal - def test_gpu_forward(self): + @pytest.mark.parametrize("shared_expert_gate", [False, True]) + def test_gpu_forward(self, shared_expert_gate): Utils.initialize_model_parallel(1, 1) model_parallel_cuda_manual_seed(123) print("done intializing") @@ -38,6 +39,7 @@ def test_gpu_forward(self): moe_router_load_balancing_type="sinkhorn", moe_router_topk=1, add_bias_linear=False, + moe_shared_expert_gate=shared_expert_gate, ) transformer_layer_spec = get_gpt_layer_local_spec( num_experts=num_moe_experts, moe_grouped_gemm=False @@ -49,7 +51,10 @@ def test_gpu_forward(self): assert isinstance(self.moe_layer, MoELayer) num_weights = sum([p.numel() for p in self.moe_layer.parameters()]) - assert num_weights == 3480 + 1152 + if shared_expert_gate: + assert num_weights == 3480 + 1152 + 12 # 12 is the weight of the gate + else: + assert num_weights == 3480 + 1152 assert self.moe_layer.shared_experts is not None assert self.moe_layer.shared_experts.stream is None assert self.moe_layer.token_dispatcher.shared_experts is None diff --git a/tests/unit_tests/transformer/moe/test_token_dispatcher.py b/tests/unit_tests/transformer/moe/test_token_dispatcher.py index 328b8837790..80b0d2bca69 100644 --- a/tests/unit_tests/transformer/moe/test_token_dispatcher.py +++ b/tests/unit_tests/transformer/moe/test_token_dispatcher.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy import dataclasses @@ -91,7 +91,7 @@ def __init__( sequence_parallel=tp_size > 1, add_bias_linear=kwargs.get("add_bias_linear", False), moe_permute_fusion=kwargs.get("moe_permute_fusion", False), - moe_enable_deepep=kwargs.get("moe_enable_deepep", False), + moe_flex_dispatcher_backend=kwargs.get("moe_flex_dispatcher_backend", None), ) # init moe layer @@ -115,6 +115,7 @@ def __del__(self): torch.cuda.synchronize() Utils.destroy_model_parallel() + @pytest.mark.flaky_in_dev @pytest.mark.internal def dispatcher_dropless_test(self): moe_layer = self.moe_layer @@ -157,6 +158,7 @@ def dispatcher_dropless_test(self): hidden_states.grad, ans ), "Restored hidden states do not match original hidden states" + @pytest.mark.flaky_in_dev @pytest.mark.internal def dispatcher_capacity_test(self): moe_layer = self.moe_layer @@ -210,6 +212,7 @@ def dispatcher_capacity_test(self): hidden_states.grad, restored_hidden_states_answer ), "Gradient of hidden states should be same as hidden states" + @pytest.mark.flaky_in_dev @pytest.mark.internal def dispatcher_drop_and_pad_test(self): """Test if the tokens are dropped and padded correctly. @@ -275,20 +278,21 @@ def dispatcher_drop_and_pad_test(self): hidden_states.grad, backward_answer ), "Gradient of hidden states should be same as hidden states" + @pytest.mark.flaky_in_dev @pytest.mark.internal def dispatcher_router_padding_for_fp8_test(self): """Test if the routing map is padded correctly for FP8 training. The test runs the forward flow twice: - 1. First with moe_router_padding_for_fp8=False - 2. Then with moe_router_padding_for_fp8=True + 1. First with moe_router_padding_for_quantization=False + 2. Then with moe_router_padding_for_quantization=True We verify that: 1. The results are the same in both cases 2. The number of tokens received by each expert is padded to a multiple of 16 """ - # First run with moe_router_padding_for_fp8 = False - moe_layer = self.new_moe_layer(moe_router_padding_for_fp8=False) + # First run with moe_router_padding_for_quantization = False + moe_layer = self.new_moe_layer(moe_router_padding_for_quantization=False) num_tokens = 32 hidden_states = torch.randn( @@ -309,8 +313,8 @@ def dispatcher_router_padding_for_fp8_test(self): grad_1 = hidden_states.grad.clone() hidden_states.grad = None - # Run with moe_router_padding_for_fp8 = True - moe_layer_2 = self.new_moe_layer(moe_router_padding_for_fp8=True, fp8="hybrid") + # Run with moe_router_padding_for_quantization = True + moe_layer_2 = self.new_moe_layer(moe_router_padding_for_quantization=True, fp8="hybrid") moe_layer_2.load_state_dict(moe_layer.state_dict()) probs_2, indices_2 = moe_layer_2.router(hidden_states) @@ -367,6 +371,7 @@ def teardown_method(self, method): @pytest.mark.internal @pytest.mark.parametrize("tp_size,ep_size", [(8, 1), (1, 8), (2, 4), (1, 1)]) @pytest.mark.parametrize("permute_fusion", permute_fusion_params) + @pytest.mark.flaky_in_dev def test_forward_backward(self, tp_size, ep_size, permute_fusion): container = MoEModelTestContainer( tp_size=tp_size, @@ -411,7 +416,13 @@ def is_deep_ep_available(): return HAVE_DEEP_EP -@pytest.mark.skipif(not is_deep_ep_available(), reason="Deep EP is not available") +def is_hybrid_ep_available(): + from megatron.core.transformer.moe.fused_a2a import HAVE_HYBRIDEP + + return HAVE_HYBRIDEP + + +@pytest.mark.skipif(True, reason="Deep EP and Hybrid EP are not available") class TestFlexDispatcher: def setup_method(self, method): pass @@ -421,9 +432,14 @@ def teardown_method(self, method): @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @pytest.mark.internal - @pytest.mark.parametrize("tp_size,ep_size", [(8, 1), (1, 8), (2, 4)]) + @pytest.mark.parametrize("tp_size,ep_size", [(1, 8), (8, 1), (4, 2)]) @pytest.mark.parametrize("permute_fusion", permute_fusion_params) - def test_forward_backward(self, tp_size, ep_size, permute_fusion): + @pytest.mark.parametrize("moe_flex_dispatcher_backend", ["deepep", "hybridep"]) + def test_forward_backward(self, tp_size, ep_size, permute_fusion, moe_flex_dispatcher_backend): + if moe_flex_dispatcher_backend == "deepep" and not is_deep_ep_available(): + pytest.skip("Deep EP is not available") + if moe_flex_dispatcher_backend == "hybridep" and not is_hybrid_ep_available(): + pytest.skip("Hybrid EP is not available") if permute_fusion: config.ENABLE_EXPERIMENTAL = True container = MoEModelTestContainer( @@ -435,8 +451,8 @@ def test_forward_backward(self, tp_size, ep_size, permute_fusion): moe_router_load_balancing_type="aux_loss", moe_token_dispatcher_type="flex", moe_permute_fusion=permute_fusion, - hidden_size=32, - moe_enable_deepep=True, + hidden_size=1024, + moe_flex_dispatcher_backend=moe_flex_dispatcher_backend, test_dtype=torch.bfloat16, ) container.dispatcher_dropless_test() @@ -448,7 +464,14 @@ def test_forward_backward(self, tp_size, ep_size, permute_fusion): @pytest.mark.timeout(120) @pytest.mark.parametrize("tp_size,ep_size", [(1, 8), (8, 1), (4, 2)]) @pytest.mark.parametrize("permute_fusion", permute_fusion_params) - def test_capacity_forward_backward(self, tp_size, ep_size, permute_fusion): + @pytest.mark.parametrize("moe_flex_dispatcher_backend", ["deepep", "hybridep"]) + def test_capacity_forward_backward( + self, tp_size, ep_size, permute_fusion, moe_flex_dispatcher_backend + ): + if moe_flex_dispatcher_backend == "deepep" and not is_deep_ep_available(): + pytest.skip("Deep EP is not available") + if moe_flex_dispatcher_backend == "hybridep" and not is_hybrid_ep_available(): + pytest.skip("Hybrid EP is not available") if permute_fusion: config.ENABLE_EXPERIMENTAL = True container = MoEModelTestContainer( @@ -463,8 +486,8 @@ def test_capacity_forward_backward(self, tp_size, ep_size, permute_fusion): moe_expert_capacity_factor=0.5, moe_pad_expert_input_to_capacity=False, moe_permute_fusion=permute_fusion, - hidden_size=32, - moe_enable_deepep=True, + hidden_size=1024, + moe_flex_dispatcher_backend=moe_flex_dispatcher_backend, test_dtype=torch.bfloat16, ) container.dispatcher_capacity_test() @@ -478,7 +501,14 @@ def test_capacity_forward_backward(self, tp_size, ep_size, permute_fusion): @pytest.mark.timeout(120) @pytest.mark.parametrize("tp_size,ep_size", [(1, 8), (8, 1), (4, 2)]) @pytest.mark.parametrize("permute_fusion", [True]) - def test_router_padding_for_fp8_forward_backward(self, tp_size, ep_size, permute_fusion): + @pytest.mark.parametrize("moe_flex_dispatcher_backend", ["deepep", "hybridep"]) + def test_router_padding_for_fp8_forward_backward( + self, tp_size, ep_size, permute_fusion, moe_flex_dispatcher_backend + ): + if moe_flex_dispatcher_backend == "deepep" and not is_deep_ep_available(): + pytest.skip("Deep EP is not available") + if moe_flex_dispatcher_backend == "hybridep" and not is_hybrid_ep_available(): + pytest.skip("Hybrid EP is not available") if permute_fusion: config.ENABLE_EXPERIMENTAL = True container = MoEModelTestContainer( @@ -491,8 +521,8 @@ def test_router_padding_for_fp8_forward_backward(self, tp_size, ep_size, permute moe_token_dispatcher_type="flex", moe_pad_expert_input_to_capacity=False, moe_permute_fusion=permute_fusion, - hidden_size=32, - moe_enable_deepep=True, + hidden_size=1024, + moe_flex_dispatcher_backend=moe_flex_dispatcher_backend, test_dtype=torch.bfloat16, ) container.dispatcher_router_padding_for_fp8_test() diff --git a/tests/unit_tests/transformer/test_attention.py b/tests/unit_tests/transformer/test_attention.py index 7e0e8c55807..de32ede6e8d 100644 --- a/tests/unit_tests/transformer/test_attention.py +++ b/tests/unit_tests/transformer/test_attention.py @@ -1,20 +1,45 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. import copy +from functools import partial +from unittest import mock +import einops import pytest import torch from packaging import version +from torch.nn import functional as F import megatron.core.parallel_state as parallel_state from megatron.core.hyper_comm_grid import HyperCommGrid -from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core.models.common.embeddings.rope_utils import ( + get_pos_emb_on_this_cp_rank as get_tensor_on_this_cp_rank, +) +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, +) +from megatron.core.models.gpt.gpt_model import GPTModel from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.tensor_parallel.random import model_parallel_cuda_manual_seed from megatron.core.transformer import TransformerConfig from megatron.core.transformer.attention import SelfAttention +from megatron.core.transformer.dot_product_attention_context_parallel import ( + AttentionFuncionWithContextParallel, + to_zz_mask_attn_bias, +) from megatron.core.transformer.enums import AttnMaskType from megatron.core.utils import is_te_min_version +from megatron.training.arguments import parse_args +from megatron.training.checkpointing import load_checkpoint, save_checkpoint +from megatron.training.global_vars import set_args +from megatron.training.training import get_model +from megatron.training.utils import unwrap_model +from tests.unit_tests.dist_checkpointing import ( + TempNamedDir, + init_basic_mock_args, + init_checkpointing_mock_args, +) from tests.unit_tests.test_utilities import Utils try: @@ -25,9 +50,20 @@ HAVE_FUSED_QKV_ROPE = False +@pytest.mark.parametrize("output_gate", [False, True]) +@pytest.mark.parametrize( + ("transformer_impl", "fallback_to_eager_attn"), + [("transformer_engine", False), ("transformer_engine", True), ("native", False)], +) class TestParallelAttention: - def setup_method(self, method): + @pytest.fixture(scope='function', autouse=True) + def setup_method(self, output_gate, transformer_impl, fallback_to_eager_attn): + if output_gate: + if transformer_impl == "native": + pytest.skip("Native implementation does not support output gate.") + if fallback_to_eager_attn: + pytest.skip("No need to test output gate for fallback_to_eager_attn = True.") Utils.initialize_model_parallel(1, 1) model_parallel_cuda_manual_seed(123) self.transformer_config = TransformerConfig( @@ -37,14 +73,22 @@ def setup_method(self, method): use_cpu_initialization=True, bf16=True, params_dtype=torch.bfloat16, + attention_output_gate=output_gate, + transformer_impl=transformer_impl, + fallback_to_eager_attn=fallback_to_eager_attn, ) + if transformer_impl == "transformer_engine": + layer_spec = get_gpt_layer_with_transformer_engine_spec( + fallback_to_eager_attn=fallback_to_eager_attn + ) + else: + layer_spec = get_gpt_layer_local_spec() + attn_layer_spec = layer_spec.submodules.self_attention.submodules self.parallel_attention = SelfAttention( - self.transformer_config, - get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, - layer_number=1, + self.transformer_config, attn_layer_spec, layer_number=1 ) - def teardown_method(self, method): + def teardown_method(self): Utils.destroy_model_parallel() def test_constructor(self): @@ -52,7 +96,19 @@ def test_constructor(self): assert self.parallel_attention.layer_number == 1 num_weights = sum([p.numel() for p in self.parallel_attention.parameters()]) - assert num_weights == 66304 + + hidden_size = self.transformer_config.hidden_size + standard_num_weights = ( + hidden_size * hidden_size * 4 + hidden_size * 4 # QKVO weight # QKVO bias + ) + if self.transformer_config.attention_output_gate: + standard_num_weights += hidden_size * hidden_size + hidden_size # Gate weight and bias + if self.transformer_config.transformer_impl == "transformer_engine": + standard_num_weights += hidden_size * 2 # fused pre layernorm weight and bias + + assert ( + num_weights == standard_num_weights + ), f"{num_weights=} does not match {standard_num_weights=}." def test_cpu_forward(self): # we can't currently do this because the global memory buffer is on GPU @@ -87,9 +143,13 @@ def test_gpu_forward(self): @pytest.mark.parametrize("rotary_interleaved", [True, False]) @pytest.mark.parametrize("fused_qkv_rope", [True, False]) def test_fused_rope_gpu_forward(self, rotary_interleaved, fused_qkv_rope): + if self.transformer_config.fallback_to_eager_attn: + pytest.skip("No need to test fused RoPE for fallback_to_eager_attn = True.") self.parallel_attention.config.apply_rope_fusion = True if rotary_interleaved and not is_te_min_version("2.3.0"): pytest.skip("Only TE >= 2.3.0 supports interleaved fused RoPE.") + if fused_qkv_rope and self.parallel_attention.config.attention_output_gate: + pytest.skip("Fused QKV RoPE does not support gated attention for now.") if fused_qkv_rope and not HAVE_FUSED_QKV_ROPE: pytest.skip("Fused QKV RoPE not available.") self.parallel_attention.config.rotary_interleaved = rotary_interleaved @@ -157,26 +217,226 @@ def test_checkpointed_gpu_forward(self): assert bias.shape[0] == config.hidden_size -class TestSelfAttention: +@pytest.mark.skipif(not is_te_min_version("2.9.0"), reason="QK clipping requires TE >= 2.9.0") +class TestClipQK: def setup_method(self, method): - Utils.destroy_model_parallel() + Utils.initialize_model_parallel(1, 1) + model_parallel_cuda_manual_seed(123) def teardown_method(self, method): Utils.destroy_model_parallel() + def test_clip_qk_disabled_raises_error(self): + """Test that clip_qk raises ValueError when qk_clip is not enabled.""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=4, + use_cpu_initialization=True, + qk_clip=False, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + + with pytest.raises(ValueError, match="qk_clip option needs to be enabled"): + attention.clip_qk() + + def test_clip_qk_none_logits_raises_error(self): + """Test that clip_qk raises ValueError when current_max_attn_logits is None.""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=4, + use_cpu_initialization=True, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + + with pytest.raises(ValueError, match="current_max_attn_logits is None"): + attention.clip_qk() + + def test_clip_qk_below_threshold_no_update(self): + """Test that weights are not updated when max logits are below threshold.""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=4, + use_cpu_initialization=True, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + attention.cuda() + + # Save original weights + original_weight = attention.linear_qkv.weight.data.clone() + + # Set current_max_attn_logits below threshold + attention.core_attention.current_max_attn_logits = torch.tensor( + [50.0, 60.0, 70.0, 80.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should not be updated + assert torch.equal(attention.linear_qkv.weight.data, original_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_above_threshold_updates_weights(self): + """Test that weights are updated when max logits exceed threshold.""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=4, + use_cpu_initialization=True, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + attention.cuda() + + # Save original weights + original_weight = attention.linear_qkv.weight.data.clone() + + # Set current_max_attn_logits above threshold + attention.core_attention.current_max_attn_logits = torch.tensor( + [150.0, 160.0, 170.0, 180.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should be updated + assert not torch.equal(attention.linear_qkv.weight.data, original_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_gqa_configuration(self): + """Test clip_qk with GQA (Grouped Query Attention) configuration.""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=8, + num_query_groups=4, # GQA with 2 heads per group + use_cpu_initialization=True, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + attention.cuda() + + # Save original weights + original_weight = attention.linear_qkv.weight.data.clone() + + # Set current_max_attn_logits for all heads (8 heads) + attention.core_attention.current_max_attn_logits = torch.tensor( + [150.0, 160.0, 170.0, 180.0, 190.0, 200.0, 210.0, 220.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should be updated + assert not torch.equal(attention.linear_qkv.weight.data, original_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_mixed_logits(self): + """Test clip_qk with mixed logits (some above, some below threshold).""" + transformer_config = TransformerConfig( + num_layers=2, + hidden_size=128, + num_attention_heads=4, + use_cpu_initialization=True, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = SelfAttention( + transformer_config, + get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + layer_number=1, + ) + attention.cuda() + + # Save original weights + original_weight = attention.linear_qkv.weight.data.clone() + + # Set mixed current_max_attn_logits (some above, some below threshold) + attention.core_attention.current_max_attn_logits = torch.tensor( + [80.0, 150.0, 90.0, 200.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should be updated since at least one head exceeds threshold + assert not torch.equal(attention.linear_qkv.weight.data, original_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + +@pytest.mark.parametrize("output_gate", [False, True]) +@pytest.mark.parametrize("transformer_impl", ["transformer_engine", "native"]) +class TestSelfAttention: + + @pytest.fixture(scope='function', autouse=True) + def setup_method(self, output_gate, transformer_impl): + if transformer_impl == "native": + if output_gate: + pytest.skip("Native implementation does not support output gate.") + self.transformer_impl = transformer_impl + self.output_gate = output_gate + Utils.destroy_model_parallel() + + def teardown_method(self): + Utils.destroy_model_parallel() + def run_self_attention(self, pg_collection): tensor_model_parallel_size = torch.distributed.get_world_size(pg_collection.tp) self.transformer_config = TransformerConfig( num_layers=2, hidden_size=128, num_attention_heads=4, + attention_output_gate=self.output_gate, tensor_model_parallel_size=tensor_model_parallel_size, use_cpu_initialization=False, + transformer_impl=self.transformer_impl, ) + if self.transformer_impl == "transformer_engine": + get_gpt_layer_spec_fn = get_gpt_layer_with_transformer_engine_spec + else: + get_gpt_layer_spec_fn = get_gpt_layer_local_spec self.self_attention = SelfAttention( self.transformer_config, - get_gpt_layer_with_transformer_engine_spec().submodules.self_attention.submodules, + get_gpt_layer_spec_fn().submodules.self_attention.submodules, layer_number=1, attn_mask_type=AttnMaskType.causal, pg_collection=pg_collection, @@ -249,3 +509,374 @@ def test_self_attention_independent_pg_smoke(self): pg_collection = ProcessGroupCollection(tp=tp_group, cp=cp_group) self.run_self_attention(pg_collection) + + +def _test_parallel_attention_correctness( + transformer_config, + transformer_layer_spec, + tmp_path_dist_ckpt, + atol, + rtol, + tp=1, + sp=False, + cp=1, + seed=123, + sequence_length=256, + micro_batch_size=4, +): + # Model initialization function + def initialize_gpt_model(config, pre_process=True, post_process=True, vp_stage=None): + gpt_model = GPTModel( + config=config, + transformer_layer_spec=transformer_layer_spec, + vocab_size=128, + max_sequence_length=sequence_length, + pre_process=pre_process, + post_process=post_process, + vp_stage=vp_stage, + ) + return gpt_model + + # Initialize baseline parallel state + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, pipeline_model_parallel_size=1, context_parallel_size=1 + ) + + # Initialize input hidden states + torch.manual_seed(seed) + model_parallel_cuda_manual_seed(seed) + input_hidden_states = ( + torch.rand((sequence_length, micro_batch_size, transformer_config.hidden_size)) + .cuda() + .bfloat16() + .requires_grad_(True) + ) + + with TempNamedDir(tmp_path_dist_ckpt / 'test_parallel_attn', sync=True) as ckpt_dir: + # Set argument + mock_args = parse_args(ignore_unknown_args=True) + set_args(mock_args) + + # Initialize baseline model + init_basic_mock_args(mock_args, 1, 1, bf16=True) + mock_args.context_parallel_size = 1 + mock_args.sequence_parallel = 1 + gpt_model = unwrap_model( + get_model(partial(initialize_gpt_model, config=transformer_config)) + ) + + # Initialize args and save checkpoint + init_checkpointing_mock_args(mock_args, ckpt_dir, False) + mock_args.no_save_optim = True + mock_args.no_save_rng = True + mock_args.no_load_optim = True + mock_args.no_load_rng = True + save_checkpoint(10, gpt_model, None, None, 0) + + # Calculate baseline output + attention = gpt_model[0].decoder.layers[0].self_attention + output_hidden_states_baseline, bias_hidden_states_baseline = attention( + input_hidden_states, attention_mask=None + ) + output_hidden_states_baseline.sum().backward() + + # Save baseline output + input_grad_baseline = input_hidden_states.grad.detach() + output_hidden_states_baseline = output_hidden_states_baseline.detach() + bias_hidden_states_baseline = bias_hidden_states_baseline + if bias_hidden_states_baseline is not None: + bias_hidden_states_baseline = bias_hidden_states_baseline.detach() + has_bias = True + else: + has_bias = False + + # Initialize parallel model + Utils.destroy_model_parallel() + Utils.initialize_model_parallel( + tensor_model_parallel_size=tp, pipeline_model_parallel_size=1, context_parallel_size=cp + ) + torch.manual_seed(seed) + model_parallel_cuda_manual_seed(seed) + transformer_config.context_parallel_size = cp + transformer_config.tensor_model_parallel_size = tp + transformer_config.sequence_parallel = sp + init_basic_mock_args(mock_args, tp, 1, bf16=True) + mock_args.context_parallel_size = cp + mock_args.sequence_parallel = sp + gpt_model = unwrap_model( + get_model(partial(initialize_gpt_model, config=transformer_config)) + ) + with mock.patch('megatron.training.checkpointing.check_checkpoint_args'): + with mock.patch('megatron.training.checkpointing.update_num_microbatches'): + load_checkpoint(gpt_model, None, None) + + # Function to get tensor on this tp and cp rank + cp_group = parallel_state.get_context_parallel_group() + tp_rank = parallel_state.get_tensor_model_parallel_rank() + + def get_tensor_on_this_rank(tensor): + if cp > 1: + tensor = get_tensor_on_this_cp_rank(tensor, 0, cp_group) + if tp > 1 and sp: + sp_seg = sequence_length // tp // cp + tensor = tensor[tp_rank * sp_seg : (tp_rank + 1) * sp_seg] + return tensor + + # Calculate parallel model output + input_hidden_states = get_tensor_on_this_rank(input_hidden_states) + input_hidden_states = input_hidden_states.detach().requires_grad_(True) + parallel_attention = gpt_model[0].decoder.layers[0].self_attention + output_hidden_states_parallel, bias_hidden_states_parallel = parallel_attention( + input_hidden_states, attention_mask=None + ) + output_hidden_states_parallel.sum().backward() + input_grad_parallel = input_hidden_states.grad.detach() + + # Check if the output is close + output_hidden_states_baseline = get_tensor_on_this_rank(output_hidden_states_baseline) + input_grad_baseline = get_tensor_on_this_rank(input_grad_baseline) + + assert torch.all( + ~torch.isnan(output_hidden_states_baseline) + ), "output_hidden_states_baseline contains nan" + assert torch.all( + ~torch.isinf(output_hidden_states_baseline) + ), "output_hidden_states_baseline contains inf" + assert torch.all(~torch.isnan(input_grad_baseline)), "input_grad_baseline contains nan" + assert torch.all(~torch.isinf(input_grad_baseline)), "input_grad_baseline contains inf" + assert torch.all( + ~torch.isnan(output_hidden_states_parallel) + ), "output_hidden_states_parallel contains nan" + assert torch.all( + ~torch.isinf(output_hidden_states_parallel) + ), "output_hidden_states_parallel contains inf" + assert torch.all(~torch.isnan(input_grad_parallel)), "input_grad_parallel contains nan" + assert torch.all(~torch.isinf(input_grad_parallel)), "input_grad_parallel contains inf" + if has_bias: + assert torch.all( + ~torch.isnan(bias_hidden_states_baseline) + ), "bias_hidden_states_baseline contains nan" + assert torch.all( + ~torch.isinf(bias_hidden_states_baseline) + ), "bias_hidden_states_baseline contains inf" + assert torch.all( + ~torch.isnan(bias_hidden_states_parallel) + ), "bias_hidden_states_parallel contains nan" + assert torch.all( + ~torch.isinf(bias_hidden_states_parallel) + ), "bias_hidden_states_parallel contains inf" + + torch.testing.assert_close( + output_hidden_states_baseline, + output_hidden_states_parallel, + atol=atol, + rtol=rtol, + msg=lambda msg: f"Mismatch in output_hidden_states: {msg}", + ) + torch.testing.assert_close( + input_grad_baseline, + input_grad_parallel, + atol=atol, + rtol=rtol, + msg=lambda msg: f"Mismatch in input_grad: {msg}", + ) + if has_bias: + torch.testing.assert_close( + bias_hidden_states_baseline, + bias_hidden_states_parallel, + atol=atol, + rtol=rtol, + msg=lambda msg: f"Mismatch in bias_hidden_states: {msg}", + ) + + Utils.destroy_model_parallel() + + +@pytest.mark.parametrize("apply_rope_fusion", [False, True]) +@pytest.mark.parametrize( + ("tp", "sp", "cp"), + [ + (4, False, 1), # TP w/o SP + (4, True, 1), # TP w/ SP + (1, False, 4), # CP + (2, False, 2), # CP + TP w/o SP + (2, True, 2), # CP + TP w/ SP + ], +) +@pytest.mark.parametrize("qk_layernorm", [False, True]) +@pytest.mark.parametrize("fallback_to_eager_attn", [False, True]) +@pytest.mark.parametrize("output_gate", [False, True]) +def test_parallel_attention_correctness( + tmp_path_dist_ckpt, + apply_rope_fusion, + tp, + sp, + cp, + qk_layernorm, + fallback_to_eager_attn, + output_gate, +): + transformer_config = TransformerConfig( + num_layers=1, + hidden_size=128, + num_attention_heads=4, + context_parallel_size=1, + tensor_model_parallel_size=1, + sequence_parallel=False, + bf16=True, + qk_layernorm=qk_layernorm, + apply_rope_fusion=apply_rope_fusion, + attention_output_gate=output_gate, + hidden_dropout=0.0, + attention_dropout=0.0, + ) + + transformer_layer_spec = get_gpt_layer_with_transformer_engine_spec( + fallback_to_eager_attn=fallback_to_eager_attn, + normalization="RMSNorm", + qk_layernorm=qk_layernorm, + ) + if cp > 1: + if qk_layernorm: + atol, rtol = 2e-2, 2e-2 + else: + atol, rtol = 5e-3, 5e-3 + else: + if qk_layernorm: + atol, rtol = 1e-2, 1e-2 + else: + atol, rtol = 2e-3, 2e-3 + + _test_parallel_attention_correctness( + transformer_config, transformer_layer_spec, tmp_path_dist_ckpt, tp, sp, cp + ) + + +def _torch_native_attention(query, key, value, attention_mask, sinks, scaling: float): + """Torch native attention implementation + This was not in the original implementation and slightly affect results; + it prevents overflow in BF16/FP16 when training with batch size > 1 we clamp max values. + """ + # Rearrange query, key, value to (b, h, s, d) + query = einops.rearrange(query, 's b h d -> b h s d') + key = einops.rearrange(key, 's b h d -> b h s d') + value = einops.rearrange(value, 's b h d -> b h s d') + + # Compute attention weights + attn_weights = torch.matmul(query, key.transpose(2, 3)) * scaling + if attention_mask is not None: + nheads = query.shape[1] + nheads_k = key.shape[1] + heads_k_stride = 1 + mask_bias = to_zz_mask_attn_bias( + attention_mask, 1, nheads, nheads_k, heads_k_stride, query.device, query.dtype + ) + attn_weights = attn_weights + mask_bias + + # Add sinks to attention weights + if sinks is None: + combined_logits = attn_weights + else: + sinks = sinks.reshape(1, -1, 1, 1).expand(query.shape[0], -1, query.shape[-2], -1) + combined_logits = torch.cat([attn_weights, sinks], dim=-1) + + # Compute attention scores + probs = F.softmax(combined_logits, dim=-1, dtype=combined_logits.dtype) + if sinks is None: + scores = probs + else: + scores = probs[..., :-1] + + # Compute attention output + attn_output = torch.matmul(scores, value) + attn_output = einops.rearrange(attn_output, 'b h s d -> s b h d') + attn_output = attn_output.contiguous() + return attn_output + + +def test_eager_attention_function_correctness(): + """Test the correctness of the context parallel eager attention function""" + + # Configuration + batch_size = 4 + num_heads = 2 + head_dim = 256 + seq_len_q = 512 + seq_len_k = 2048 + scale = 1 / (head_dim**2) + + # Initialize inputs + q = torch.rand( + (seq_len_q, batch_size, num_heads, head_dim), + device='cuda', + dtype=torch.bfloat16, + requires_grad=True, + ) + k = torch.rand( + (seq_len_k, batch_size, num_heads, head_dim), + device='cuda', + dtype=torch.bfloat16, + requires_grad=True, + ) + v = torch.rand( + (seq_len_k, batch_size, num_heads, head_dim), + device='cuda', + dtype=torch.bfloat16, + requires_grad=True, + ) + + def randbool(shape, **kwargs): + return torch.randn(shape, **kwargs) > 0 + + attn_bias = randbool((batch_size, 1, seq_len_q, seq_len_k), device='cuda') + sinks = None + + # Torch native attention forward and backward pass + out_torch = _torch_native_attention( + query=q, key=k, value=v, attention_mask=attn_bias, sinks=sinks, scaling=scale + ) + loss_torch = out_torch.sum() + loss_torch.backward() + torch_q_grad = q.grad.clone() + torch_k_grad = k.grad.clone() + torch_v_grad = v.grad.clone() + q.grad.zero_() + k.grad.zero_() + v.grad.zero_() + if sinks is not None: + torch_sinks_grad = sinks.grad.clone() + sinks.grad.zero_() + else: + torch_sinks_grad = None + + # Custom attention forward and backward pass + out_custom = AttentionFuncionWithContextParallel.apply( + q, k, v, attn_bias, 0.0, scale, None # dropout + ) + loss_custom = out_custom.sum() + loss_custom.backward() + custom_q_grad = q.grad.clone() + custom_k_grad = k.grad.clone() + custom_v_grad = v.grad.clone() + q.grad.zero_() + k.grad.zero_() + v.grad.zero_() + if sinks is not None: + custom_sinks_grad = sinks.grad.clone() + sinks.grad.zero_() + else: + custom_sinks_grad = None + + # Check attention output and gradients + assert torch.equal(out_custom, out_torch), "Mismatch in attention output" + tol = {"atol": 1e-4, "rtol": 1e-4} + for tensor_name, tensor_torch, tensor_custom in [ + ("q_grad", torch_q_grad, custom_q_grad), + ("k_grad", torch_k_grad, custom_k_grad), + ("v_grad", torch_v_grad, custom_v_grad), + ("sinks_grad", torch_sinks_grad, custom_sinks_grad), + ]: + if (tensor_torch is not None) and (tensor_custom is not None): + torch.testing.assert_close( + out_custom, out_torch, **tol, msg=lambda msg: f"Mismatch in {tensor_name}: {msg}" + ) diff --git a/tests/unit_tests/transformer/test_cuda_graphs.py b/tests/unit_tests/transformer/test_cuda_graphs.py index b92ff383d82..fb3567074f1 100644 --- a/tests/unit_tests/transformer/test_cuda_graphs.py +++ b/tests/unit_tests/transformer/test_cuda_graphs.py @@ -1,11 +1,37 @@ # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +import gc +import os +import random +import sys +import time +import types + import pytest import torch -from megatron.core.models.gpt.gpt_layer_specs import get_gpt_layer_with_transformer_engine_spec +from megatron.core import parallel_state +from megatron.core.enums import ModelType +from megatron.core.inference.contexts import DynamicInferenceContext +from megatron.core.inference.engines import DynamicInferenceEngine +from megatron.core.inference.model_inference_wrappers.gpt.gpt_inference_wrapper import ( + GPTInferenceWrapper, +) +from megatron.core.inference.model_inference_wrappers.inference_wrapper_config import ( + InferenceWrapperConfig, +) +from megatron.core.inference.sampling_params import SamplingParams +from megatron.core.inference.text_generation_controllers.text_generation_controller import ( + TextGenerationController, +) +from megatron.core.models.gpt.gpt_layer_specs import ( + get_gpt_layer_local_spec, + get_gpt_layer_with_transformer_engine_spec, + get_gpt_mtp_block_spec, +) from megatron.core.models.gpt.gpt_model import GPTModel from megatron.core.models.mamba.mamba_layer_specs import mamba_stack_spec +from megatron.core.num_microbatches_calculator import destroy_num_microbatches_calculator from megatron.core.pipeline_parallel.schedules import set_current_microbatch from megatron.core.process_groups_config import ProcessGroupCollection from megatron.core.ssm.mamba_block import MambaStack @@ -18,6 +44,14 @@ from megatron.core.transformer.transformer_block import TransformerBlock from megatron.core.transformer.transformer_config import TransformerConfig from megatron.core.utils import is_fa_min_version, is_te_min_version +from megatron.training.arguments import core_transformer_config_from_args, parse_args, validate_args +from megatron.training.global_vars import ( + destroy_global_vars, + get_args, + set_args, + set_global_variables, +) +from megatron.training.training import setup_model_and_optimizer from tests.unit_tests.test_utilities import Utils @@ -497,6 +531,461 @@ def test_gpu_cudagraph(self): del parallel_mamba_block.layers[_].cudagraph_manager.cudagraph_runners[0].fwd_graph +class TestCaptureFreezeGC: + + def capture_cuda_graphs(self, cuda_graph_capture_freeze_gc: bool) -> None: + """Capture multiple cuda graphs by initializing the `DynamicInferenceEngine`. + + The `DynamicInferenceEngine` is used here because it is currently (as of + August 2025) one of the heaviest users of multiple cuda graphs, and so + its setup tests a realistic use-case of multi-batch size cuda graphs. + + Args: + cuda_graph_capture_freeze_gc (bool): Flag that determines whether to + freeze garbage collection. + """ + + # Set freeze-gc environment variable. + os.environ["CUDA_GRAPH_CAPTURE_FREEZE_GC"] = str(int(cuda_graph_capture_freeze_gc)) + + # Configuration. + random_seed = 123 + vocab_size = 100 + num_tokens_to_prompt = 128 + num_tokens_to_generate = 32 + max_sequence_length = num_tokens_to_prompt + num_tokens_to_generate + num_cuda_graphs = 4 + + # Rounder values. + rounder = 4 + DynamicInferenceContext.ROUNDER = rounder # For backwards compatibility + DynamicInferenceContext.TOKEN_ROUNDER = rounder + DynamicInferenceContext.REQUEST_ROUNDER = rounder + + # Random state. + random.seed(random_seed) + torch.manual_seed(random_seed) + model_parallel_cuda_manual_seed( + seed=random_seed, + inference_rng_tracker=True, + use_cudagraphable_rng=False, + force_reset_rng=True, + ) + + # Transformer config. + transformer_config = TransformerConfig( + params_dtype=torch.bfloat16, + num_layers=4, + hidden_size=32, + num_attention_heads=4, + use_cpu_initialization=True, + cuda_graph_impl="local", + inference_rng_tracker=True, + tensor_model_parallel_size=1, # needed? + ) + + # Sampling params. + sampling_params = SamplingParams(num_tokens_to_generate=num_tokens_to_generate) + + # GPT model. + model = GPTModel( + config=transformer_config, + transformer_layer_spec=get_gpt_layer_local_spec(), + vocab_size=vocab_size, + max_sequence_length=max_sequence_length, + parallel_output=True, + ).cuda() + + for param in model.parameters(): + param.data = param.data.to(transformer_config.params_dtype) + + model.eval() + + # Inference config. + inference_config = InferenceWrapperConfig( + hidden_size=transformer_config.hidden_size, + inference_batch_times_seqlen_threshold=400, + fp32_residual_connection=False, + params_dtype=transformer_config.params_dtype, + padded_vocab_size=vocab_size, + ) + + # Inference context. + context = DynamicInferenceContext( + params_dtype=transformer_config.params_dtype, + num_layers=transformer_config.num_layers, + kv_channels=transformer_config.kv_channels, + num_attention_heads=transformer_config.num_query_groups, + max_sequence_length=max_sequence_length, + num_cuda_graphs=num_cuda_graphs, + buffer_size_gb=20, + buffer_guaranteed_fraction=0.05, + block_size_tokens=256, + buffer_overflow_factor=1.1, + max_requests_override=512, + max_tokens_override=8196, + tensor_model_parallel_size=transformer_config.tensor_model_parallel_size, + ) + + # Inference model wrapper. + inference_wrapped_model = GPTInferenceWrapper(model, inference_config, context) + + # Note: the following is taken from AbstractModelInferenceWrapper.prep_model_for_inference(). + inference_wrapped_model.model_is_pipeline_parallel = not ( + parallel_state.is_pipeline_first_stage() and parallel_state.is_pipeline_last_stage() + ) + + # Text generation controller. + text_generation_controller = TextGenerationController( + inference_wrapped_model=inference_wrapped_model, + tokenizer=types.SimpleNamespace(vocab_size=vocab_size), + ) + + # Inference engine. + engine = DynamicInferenceEngine( + text_generation_controller, + context, + termination_id=vocab_size - 1, + random_seed=random_seed, + ) + + return engine.capture_stats + + @pytest.mark.flaky_in_dev # Issue #2855 + @pytest.mark.experimental + @pytest.mark.skipif( + not is_fa_min_version("2.7.3"), reason="need latest flash attn for dynamic batching" + ) + def test_capture_freeze_gc(self): + """Test cuda graph capture while freezing the GC.""" + + Utils.initialize_model_parallel( + tensor_model_parallel_size=1, pipeline_model_parallel_size=1 + ) + + # Run tests with GC freeze off/on. + result_map = {} + for freeze_gc in (False, True): + + # Reset global cuda graph state. + _CudagraphGlobalRecord.cudagraph_created = False + _CudagraphGlobalRecord.cudagraph_record = [] + CudaGraphManager.global_mempool = None + + # Capture multiple cuda graphs by initializing DynamicInferenceEngine. + mem_stats_start = torch.cuda.memory_stats() + time_start = time.time() + internal_stats = self.capture_cuda_graphs(freeze_gc) + time_end = time.time() + mem_stats_end = torch.cuda.memory_stats() + + # Track local (external) stats, in addition to internal stats. + external_stats = { + "time": time_end - time_start, + "allocated_bytes": ( + mem_stats_end["allocated_bytes.all.current"] + - mem_stats_start["allocated_bytes.all.current"] + ), + "reserved_bytes": ( + mem_stats_end["reserved_bytes.all.current"] + - mem_stats_start["reserved_bytes.all.current"] + ), + } + + # Record results. + result_map[freeze_gc] = {"internal": internal_stats, "external": external_stats} + + # Extract results. + freeze_off_results = result_map[False] + freeze_on_results = result_map[True] + print( + "test capture | freeze off: internal %.3f, external %.3f." + % (freeze_off_results["internal"]["time"], freeze_off_results["external"]["time"]) + ) + print( + "test capture | freeze on: internal %.3f, external %.3f." + % (freeze_on_results["internal"]["time"], freeze_on_results["external"]["time"]) + ) + + # Validate time and memory usage. + assert freeze_on_results["internal"]["time"] < 0.3 * freeze_off_results["internal"]["time"] + assert freeze_on_results["external"]["time"] < 0.3 * freeze_off_results["external"]["time"] + assert ( + freeze_on_results["internal"]["allocated_bytes"] + <= freeze_off_results["internal"]["allocated_bytes"] + ) + assert ( + freeze_on_results["external"]["allocated_bytes"] + <= freeze_off_results["external"]["allocated_bytes"] + ) + assert ( + freeze_on_results["internal"]["reserved_bytes"] + <= freeze_off_results["internal"]["reserved_bytes"] + ) + assert ( + freeze_on_results["external"]["reserved_bytes"] + <= freeze_off_results["external"]["reserved_bytes"] + ) + + +def is_deep_ep_available(): + from megatron.core.transformer.moe.fused_a2a import HAVE_DEEP_EP + + return HAVE_DEEP_EP + + +def is_hybrid_ep_available(): + from megatron.core.transformer.moe.fused_a2a import HAVE_HYBRIDEP + + return HAVE_HYBRIDEP + + +class TestPartialCudaGraph: + """Test that CUDA graph outputs match non-CUDA graph outputs for various scopes.""" + + def setup_method(self, method): + self.seq_length = 512 + self.micro_batch_size = 2 + # Store original environment variable values + self.original_env = { + 'CUDA_DEVICE_MAX_CONNECTIONS': os.environ.get('CUDA_DEVICE_MAX_CONNECTIONS'), + 'NVTE_ALLOW_NONDETERMINISTIC_ALGO': os.environ.get('NVTE_ALLOW_NONDETERMINISTIC_ALGO'), + } + os.environ['CUDA_DEVICE_MAX_CONNECTIONS'] = '1' + os.environ['NVTE_ALLOW_NONDETERMINISTIC_ALGO'] = '0' + + def teardown_method(self, method): + # Restore original environment variable values + for key, value in self.original_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + Utils.destroy_model_parallel() + destroy_global_vars() + destroy_num_microbatches_calculator() + gc.collect() + + def model_provider( + self, + pre_process=True, + post_process=True, + layer_spec_fn=get_gpt_layer_with_transformer_engine_spec, + **config_kwargs, + ): + model_parallel_cuda_manual_seed(123) + args = get_args() + config = core_transformer_config_from_args(args) + transformer_layer_spec = layer_spec_fn() + if args.mtp_num_layers: + mtp_block_spec = get_gpt_mtp_block_spec( + config, transformer_layer_spec, use_transformer_engine=True + ) + else: + mtp_block_spec = None + return GPTModel( + config=config, + transformer_layer_spec=transformer_layer_spec, + vocab_size=args.vocab_size, + max_sequence_length=args.max_position_embeddings, + pre_process=pre_process, + post_process=post_process, + fp16_lm_cross_entropy=args.fp16_lm_cross_entropy, + parallel_output=True, + share_embeddings_and_output_weights=not args.untie_embeddings_and_output_weights, + position_embedding_type=args.position_embedding_type, + rotary_percent=args.rotary_percent, + mtp_block_spec=mtp_block_spec, + ) + + def create_test_args( + self, cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, ep_size, **kwargs + ): + destroy_global_vars() + destroy_num_microbatches_calculator() + + sys.argv = ['test_cuda_graphs.py'] + args = parse_args() + args.num_layers = 4 + args.mtp_num_layers = 1 + args.vocab_size = 1024 + args.hidden_size = 128 + args.num_attention_heads = 8 + args.max_position_embeddings = 512 + args.global_batch_size = self.micro_batch_size * 8 + args.micro_batch_size = self.micro_batch_size + args.create_attention_mask_in_dataloader = True + args.seq_length = self.seq_length + args.tensor_model_parallel_size = 2 + args.sequence_parallel = True + args.pipeline_model_parallel_size = 1 + args.context_parallel_size = 1 + args.expert_model_parallel_size = ep_size + args.train_iters = 10 + args.lr = 3e-5 + args.bf16 = True + args.add_bias_linear = False + args.swiglu = True + args.use_distributed_optimizer = True + args.position_embedding_type = "rope" + args.rotary_percent = 1.0 + args.hidden_dropout = 0.0 + args.attention_dropout = 0.0 + + # MoE settings + args.num_experts = 4 + args.expert_model_parallel_size = ep_size + args.moe_shared_expert_intermediate_size = 1024 + args.moe_layer_freq = "[0,0,1,1]" + args.moe_permute_fusion = True + args.moe_router_fusion = True + args.moe_router_topk = 2 + + # CUDA graph settings + args.cuda_graph_impl = cuda_graph_impl + args.cuda_graph_scope = cuda_graph_scope + args.cuda_graph_warmup_steps = cuda_graph_warmup_steps + args.use_te_rng_tracker = cuda_graph_impl != "none" + + for key, value in kwargs.items(): + assert hasattr(args, key) + setattr(args, key, value) + + validate_args(args) + set_global_variables(args, False) + return args + + def get_batch(self, seq_length, micro_batch_size): + data = list(range(seq_length)) + input_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda() + labels = 1 + torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda() + position_ids = torch.tensor(data, dtype=torch.int64).repeat((micro_batch_size, 1)).cuda() + attention_mask = torch.ones( + (micro_batch_size, 1, seq_length, seq_length), dtype=bool + ).cuda() + loss_mask = torch.ones(seq_length).repeat((micro_batch_size, 1)).cuda() + return input_ids, labels, position_ids, attention_mask, loss_mask + + def _run_test_helper( + self, ep_size, cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, **kwargs + ): + """Test fp8_param with gpt_model.""" + args = self.create_test_args( + cuda_graph_impl, cuda_graph_scope, cuda_graph_warmup_steps, ep_size, **kwargs + ) + + set_args(args) + torch.manual_seed(123) + Utils.initialize_model_parallel( + tensor_model_parallel_size=2, expert_model_parallel_size=ep_size + ) + + input_ids, labels, position_ids, attention_mask, loss_mask = self.get_batch( + self.seq_length, self.micro_batch_size + ) + + gpt_model, optimizer, _ = setup_model_and_optimizer( + self.model_provider, ModelType.encoder_or_decoder + ) + assert len(gpt_model) == 1 # Assume only one model in the model provider. + + loss_list = [] + + cuda_graph_helper = None + if cuda_graph_impl == "transformer_engine": + from megatron.core.transformer.cuda_graphs import TECudaGraphHelper + + cuda_graph_helper = TECudaGraphHelper( + model=gpt_model, + config=gpt_model[0].config, + seq_length=self.seq_length, + micro_batch_size=self.micro_batch_size, + optimizers=[optimizer], + ) + + for i in range(100): + gpt_model[0].zero_grad_buffer() + optimizer.zero_grad() + + # Capture CUDA graphs after warmup if helper is provided + if cuda_graph_helper is not None and i == cuda_graph_warmup_steps: + cuda_graph_helper.create_cudagraphs() + + output = gpt_model[0].forward( + input_ids=input_ids, + position_ids=position_ids, + attention_mask=attention_mask, + labels=labels, + loss_mask=loss_mask, + ) + + # Check output shapes + assert output.shape[0] == self.micro_batch_size + assert output.shape[1] == self.seq_length + + # Verify gradients + loss = output.mean() + loss.backward() + + for param in gpt_model[0].parameters(): + assert param.main_grad is not None + + update_successful, _, _ = optimizer.step() + assert update_successful + + loss_list.append(loss.item()) + + return torch.tensor(loss_list) + + @pytest.mark.skipif( + not (HAVE_TE and is_te_min_version("1.14.0")), + reason="Partial CUDA graph support requires TransformerEngine version >= 1.14.0", + ) + @pytest.mark.parametrize("ep_size", [1, 4]) + @pytest.mark.parametrize("moe_dropless_dispatcher", [False, True]) + @pytest.mark.parametrize("moe_dispatcher_type", ["alltoall", "deepep", "hybridep"]) + def test_moe_partial_cudagraph(self, ep_size, moe_dropless_dispatcher, moe_dispatcher_type): + extra_kwargs = {} + if moe_dispatcher_type == "deepep": + if not is_deep_ep_available(): + pytest.skip("Deep EP is not available") + extra_kwargs["moe_token_dispatcher_type"] = "flex" + extra_kwargs["moe_flex_dispatcher_backend"] = "deepep" + elif moe_dispatcher_type == "hybridep": + if not is_hybrid_ep_available(): + pytest.skip("Hybrid EP is not available") + extra_kwargs["moe_token_dispatcher_type"] = "flex" + extra_kwargs["moe_flex_dispatcher_backend"] = "hybridep" + else: + extra_kwargs["moe_token_dispatcher_type"] = moe_dispatcher_type + if not moe_dropless_dispatcher: + if moe_dispatcher_type == "deepep": + pytest.skip("Deep EP doesn't support drop&pad MoE") + extra_kwargs["moe_expert_capacity_factor"] = 1.0 + extra_kwargs["moe_pad_expert_input_to_capacity"] = True + + loss_list_ref = self._run_test_helper(ep_size, "none", None, 0, **extra_kwargs) + for cuda_graph_scope in [ + None, + ["attn"], + ["moe"], + ["mlp", "moe_router"], + ["attn", "mlp", "moe_router", "moe_preprocess"], + ]: + if moe_dropless_dispatcher and (cuda_graph_scope is None or "moe" in cuda_graph_scope): + # Dropless MoE doesn't work with "moe" scope cudagraph. Skip. + continue + cuda_graph_warmup_steps = 3 + loss_list = self._run_test_helper( + ep_size, + "transformer_engine", + cuda_graph_scope, + cuda_graph_warmup_steps, + **extra_kwargs, + ) + assert torch.equal(loss_list, loss_list_ref) + + if __name__ == "__main__": test = TestParallelTransformerBlockCudagraphs() @@ -508,3 +997,11 @@ def test_gpu_cudagraph(self): llava_test.setup_method(method=None) llava_test.test_llava_cudagraph_is_last_layer_logic() llava_test.teardown_method(method=None) + + test = TestCaptureFreezeGC() + test.test_capture_freeze_gc() + + test = TestPartialCudaGraph() + test.setup_method(method=None) + test.test_moe_partial_cudagraph(4, True, "alltoall") + test.teardown_method(method=None) diff --git a/tests/unit_tests/transformer/test_multi_latent_attention.py b/tests/unit_tests/transformer/test_multi_latent_attention.py index 8ade4b6bcb8..ad156dfda13 100644 --- a/tests/unit_tests/transformer/test_multi_latent_attention.py +++ b/tests/unit_tests/transformer/test_multi_latent_attention.py @@ -1034,6 +1034,234 @@ def test_gpu_forward_thd_precision(self): os.environ.update(_environ) +@pytest.mark.skipif(not is_te_min_version("2.9.0"), reason="QK clipping requires TE >= 2.9.0") +@pytest.mark.parametrize("rope_type", ('yarn', 'rope')) +class TestMLAClipQK: + + @pytest.fixture(scope='function', autouse=True) + def setup_and_teardown(self, rope_type): + Utils.initialize_model_parallel(1, 1) + model_parallel_cuda_manual_seed(123) + self.transformer_config = MLATransformerConfig( + num_layers=2, + hidden_size=12, + num_attention_heads=4, + use_cpu_initialization=True, + q_lora_rank=32, + kv_lora_rank=32, + qk_head_dim=128, + v_head_dim=128, + qk_pos_emb_head_dim=64, + rope_type=rope_type, + rotary_base=10000, + original_max_position_embeddings=32, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + + def teardown_method(self, method): + Utils.destroy_model_parallel() + + def test_clip_qk_disabled_raises_error(self): + """Test that clip_qk raises ValueError when qk_clip is not enabled.""" + if is_te_min_version("1.10.0"): + # Create config without qk_clip + config = MLATransformerConfig( + num_layers=2, + hidden_size=12, + num_attention_heads=4, + use_cpu_initialization=True, + q_lora_rank=32, + kv_lora_rank=32, + qk_head_dim=128, + v_head_dim=128, + qk_pos_emb_head_dim=64, + rotary_base=10000, + original_max_position_embeddings=32, + qk_clip=False, + ) + attention = MLASelfAttention( + config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + + with pytest.raises(ValueError, match="qk_clip option needs to be enabled"): + attention.clip_qk() + + def test_clip_qk_none_logits_raises_error(self): + """Test that clip_qk raises ValueError when current_max_attn_logits is None.""" + if is_te_min_version("1.10.0"): + attention = MLASelfAttention( + self.transformer_config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + + with pytest.raises(ValueError, match="current_max_attn_logits is None"): + attention.clip_qk() + + def test_clip_qk_below_threshold_no_update(self): + """Test that weights are not updated when max logits are below threshold.""" + if not is_te_min_version("1.10.0"): + pytest.skip("MLA requires TransformerEngine >= 1.10.0") + + attention = MLASelfAttention( + self.transformer_config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + attention.cuda() + + # Save original weights + if self.transformer_config.q_lora_rank is None: + original_q_weight = attention.linear_q_proj.weight.data.clone() + else: + original_q_weight = attention.linear_q_up_proj.weight.data.clone() + original_kv_weight = attention.linear_kv_up_proj.weight.data.clone() + + # Set current_max_attn_logits below threshold + attention.core_attention.current_max_attn_logits = torch.tensor( + [50.0, 60.0, 70.0, 80.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should not be updated + if self.transformer_config.q_lora_rank is None: + assert torch.equal(attention.linear_q_proj.weight.data, original_q_weight) + else: + assert torch.equal(attention.linear_q_up_proj.weight.data, original_q_weight) + assert torch.equal(attention.linear_kv_up_proj.weight.data, original_kv_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_above_threshold_updates_weights(self): + """Test that weights are updated when max logits exceed threshold.""" + if not is_te_min_version("1.10.0"): + pytest.skip("MLA requires TransformerEngine >= 1.10.0") + + attention = MLASelfAttention( + self.transformer_config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + attention.cuda() + + # Save original weights + if self.transformer_config.q_lora_rank is None: + original_q_weight = attention.linear_q_proj.weight.data.clone() + else: + original_q_weight = attention.linear_q_up_proj.weight.data.clone() + original_kv_weight = attention.linear_kv_up_proj.weight.data.clone() + + # Set current_max_attn_logits above threshold + attention.core_attention.current_max_attn_logits = torch.tensor( + [150.0, 160.0, 170.0, 180.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should be updated + if self.transformer_config.q_lora_rank is None: + assert not torch.equal(attention.linear_q_proj.weight.data, original_q_weight) + else: + assert not torch.equal(attention.linear_q_up_proj.weight.data, original_q_weight) + assert not torch.equal(attention.linear_kv_up_proj.weight.data, original_kv_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_mixed_logits(self): + """Test clip_qk with mixed logits (some above, some below threshold).""" + if not is_te_min_version("1.10.0"): + pytest.skip("MLA requires TransformerEngine >= 1.10.0") + + attention = MLASelfAttention( + self.transformer_config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + attention.cuda() + + # Save original weights + if self.transformer_config.q_lora_rank is None: + original_q_weight = attention.linear_q_proj.weight.data.clone() + else: + original_q_weight = attention.linear_q_up_proj.weight.data.clone() + original_kv_weight = attention.linear_kv_up_proj.weight.data.clone() + + # Set mixed current_max_attn_logits (some above, some below threshold) + attention.core_attention.current_max_attn_logits = torch.tensor( + [80.0, 150.0, 90.0, 200.0], device='cuda' + ) + + # Call clip_qk + attention.clip_qk() + + # Weights should be updated since at least one head exceeds threshold + if self.transformer_config.q_lora_rank is None: + assert not torch.equal(attention.linear_q_proj.weight.data, original_q_weight) + else: + assert not torch.equal(attention.linear_q_up_proj.weight.data, original_q_weight) + assert not torch.equal(attention.linear_kv_up_proj.weight.data, original_kv_weight) + # current_max_attn_logits should be reset + assert attention.core_attention.current_max_attn_logits is None + + def test_clip_qk_with_absorption_raises_error(self): + """Test that clip_qk raises ValueError when in absorption mode.""" + if not is_te_min_version("1.10.0"): + pytest.skip("MLA requires TransformerEngine >= 1.10.0") + + # Create config with cache_mla_latents enabled + config = MLATransformerConfig( + num_layers=2, + hidden_size=12, + num_attention_heads=4, + use_cpu_initialization=True, + q_lora_rank=32, + kv_lora_rank=32, + qk_head_dim=128, + v_head_dim=128, + qk_pos_emb_head_dim=64, + rotary_base=10000, + original_max_position_embeddings=32, + qk_clip=True, + qk_clip_threshold=100.0, + qk_clip_alpha=0.5, + ) + attention = MLASelfAttention( + config, + get_mla_self_attn_submodules(), + layer_number=1, + attn_mask_type=AttnMaskType.causal, + ) + attention.cuda() + + # Simulate absorption mode by setting cache_mla_latents and deleting linear_kv_up_proj + attention.cache_mla_latents = True + if hasattr(attention, 'linear_kv_up_proj'): + delattr(attention, 'linear_kv_up_proj') + + # Set current_max_attn_logits + attention.core_attention.current_max_attn_logits = torch.tensor( + [150.0, 160.0, 170.0, 180.0], device='cuda' + ) + + with pytest.raises( + ValueError, + match="qk_clip is not supported when cache_mla_latents is enabled and absorption is active", + ): + attention.clip_qk() + + @pytest.mark.experimental @pytest.mark.parametrize( ("rope_type", "apply_rope_fusion"), diff --git a/tests/unit_tests/transformer/test_multi_token_prediction.py b/tests/unit_tests/transformer/test_multi_token_prediction.py index 65e58eaede4..9b9d2c67881 100644 --- a/tests/unit_tests/transformer/test_multi_token_prediction.py +++ b/tests/unit_tests/transformer/test_multi_token_prediction.py @@ -101,7 +101,7 @@ def test_constructor_local(self, tp): assert num_weights == 15216 * config.mtp_num_layers @pytest.mark.skipif(not HAVE_TE, reason="transformer_engine not available") - @pytest.mark.parametrize(('tp', 'cp'), [(1, 1), (1, 2), (2, 1), (2, 2)]) + @pytest.mark.parametrize(('tp', 'cp'), [(1, 1), (2, 1), (2, 2)]) def test_constructor_ues_te(self, tp, cp): """Test basic construction of MTP module.""" torch.manual_seed(_SEED) @@ -249,7 +249,7 @@ def get_batch(self, seq_length, micro_batch_size): not HAVE_TE or not is_te_min_version("2.1.0"), reason="grouped_gemm requires TransformerEngine >= 2.1.0", ) - @pytest.mark.parametrize(("tp", "cp"), [(1, 1), (1, 2), (2, 1), (2, 2)]) + @pytest.mark.parametrize(("tp", "cp"), [(2, 1), (2, 2)]) def test_sharded_state_dict(self, tp, cp): """Test MTP with different tensor parallel sizes.""" args = self.create_test_args(tp, cp, self.seq_length, self.micro_batch_size) @@ -268,9 +268,8 @@ def test_sharded_state_dict(self, tp, cp): not HAVE_TE or not is_te_min_version("2.1.0"), reason="grouped_gemm requires TransformerEngine >= 2.1.0", ) - @pytest.mark.parametrize("full_recompute", [False, True]) @pytest.mark.parametrize( - ("tp", "cp"), [(1, 1), (1, 2), (1, 4), (2, 1), (2, 2), (2, 4), (4, 1), (4, 2)] + ("tp", "cp", "full_recompute"), [(1, 1, False), (1, 4, False), (2, 4, False), (4, 1, True)] ) def test_forward_backward(self, tmp_path_dist_ckpt, tp, cp, full_recompute): """Test MTP forward and backward with gptmodel.""" diff --git a/tests/unit_tests/transformer/test_submodule_callables.py b/tests/unit_tests/transformer/test_submodule_callables.py index 922d2b4b832..1ccb6fd5be8 100644 --- a/tests/unit_tests/transformer/test_submodule_callables.py +++ b/tests/unit_tests/transformer/test_submodule_callables.py @@ -143,7 +143,7 @@ def test_1f1b_overlap(self, dispatcher_type, grouped_gemm, permute_fusion): "moe_permute_fusion": permute_fusion, } if dispatcher_type == "flex": - extra_kwargs["moe_enable_deepep"] = True + extra_kwargs["moe_flex_dispatcher_backend"] = "deepep" extra_kwargs["moe_router_dtype"] = "fp32" config = get_test_config(extra_kwargs=extra_kwargs, moe_grouped_gemm=grouped_gemm) microbatches = 4 diff --git a/tools/checkpoint/checkpoint_inspector.py b/tools/checkpoint/checkpoint_inspector.py index c62f0ca7417..3d03f4db959 100644 --- a/tools/checkpoint/checkpoint_inspector.py +++ b/tools/checkpoint/checkpoint_inspector.py @@ -1,3 +1,5 @@ +# Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # python checkpoint_inspector.py inspect /path/to/checkpoint # torchrun --nproc_per_node=8 --nnodes=1 checkpoint_inspector.py convert-torch-dist-to-fsdp-dtensor /path/to/input_checkpoint /path/to/output_checkpoint --swiglu import gc diff --git a/uv.lock b/uv.lock index 3de5e29b62b..8b2e0f3d13e 100644 --- a/uv.lock +++ b/uv.lock @@ -1,14 +1,16 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", @@ -16,11 +18,13 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", @@ -29,12 +33,14 @@ resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", @@ -300,6 +306,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" }, ] +[[package]] +name = "apache-tvm-ffi" +version = "0.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d8/e8/7db1ca6db40877d190a8538cc378f740aae247c6fe063815898607c2d2ca/apache_tvm_ffi-0.1.1.tar.gz", hash = "sha256:728ce3f4ae02b89a7147b718f7f670afac3c6d1f96df38d488757274643709fc", size = 1259223, upload-time = "2025-11-04T02:43:38.154Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/ad/550aff4c9652ee8297f90a04c3ab4143ece1d373101010d85b5c9a9a2e7d/apache_tvm_ffi-0.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:af0de7bb9581ac9e090276cba37c4e7ffaeed601a2b2b546bf0e2daed3810cec", size = 1723658, upload-time = "2025-11-04T02:42:37.628Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/01e65f4a6c2b146f7c40f6d8d663d76b60c3be324159f8fb8223ea505738/apache_tvm_ffi-0.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7d6828652803cb8c0e13d1f06d01fc6bfb8e79e77e3de7e6fd4b5fae5ee9d2", size = 1882437, upload-time = "2025-11-04T02:42:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/6b/bd/b52b71d03637d7a82388c2e90d48dddec2c46121be1333c9851d6a135824/apache_tvm_ffi-0.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1fe072b55a7949720a792a9d455c0659aa097825e709a16a4667d720137b8b5c", size = 1954949, upload-time = "2025-11-04T02:42:41.119Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ef/ff85926928694785f2399a4c5b793bcfecf8c3cf806dedf9202b7db73b8b/apache_tvm_ffi-0.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b25178b265903dabd9a35bd767db26928be3b7869f681fe1d6e1aed93d7c0799", size = 1837395, upload-time = "2025-11-04T02:42:42.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/69/f048bda5e5445a89200737062a202cb39097d3b1902e886654de9cd6b624/apache_tvm_ffi-0.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5552af3c625750361d1b7d646d499a28caf94858967e74c9cce6ed7d4629b28", size = 1947740, upload-time = "2025-11-04T02:42:44.49Z" }, + { url = "https://files.pythonhosted.org/packages/dc/df/295f71613502edeb39a39b30c8bbb9ec8fcc06bd95b3043dd99b55fa98a8/apache_tvm_ffi-0.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:c102ba5899ce106c8068a3f21155c106790b5b0141fba52a52ed6e9aeb286aff", size = 1710966, upload-time = "2025-11-04T02:42:46.037Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a9/544767d7058f825c0ceb5bc25760ad3a821b2efcc6a3dbe2e3988a3aee86/apache_tvm_ffi-0.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7cbf31c472920cdc5b3f75f2d2720b8a6b37ddbdb11d573fa94524815ea5a144", size = 1725662, upload-time = "2025-11-04T02:42:47.528Z" }, + { url = "https://files.pythonhosted.org/packages/54/c3/fe1a9f8968d5ce2d3b674e397c2bf01961e32a72b723817478c67c9780e3/apache_tvm_ffi-0.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7602bc37019387a4705677b6e742059c7e1973a899b6918af235febcb3d3b47", size = 1884278, upload-time = "2025-11-04T02:42:48.998Z" }, + { url = "https://files.pythonhosted.org/packages/24/b9/80cbba18b2d7d9013031d8c13671986912275b9ca6aaea70a1dd9b361c39/apache_tvm_ffi-0.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7941f82a2ae4549f55c07d82d37c5765628d70f29dace98628393fcea525e870", size = 1957018, upload-time = "2025-11-04T02:42:50.538Z" }, + { url = "https://files.pythonhosted.org/packages/b4/0c/d27beb98d6841a3929468648433ed2c53e4da953fadb73c754b9372b2356/apache_tvm_ffi-0.1.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e0d6d8e0888ee3a3defd2cbe1eff7a65c05900b4e8fa0e18c890048fc6a44a6", size = 1839279, upload-time = "2025-11-04T02:42:52.438Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d7cf7779c65047ad2ca652234a174c2908d936cb69bc4f5156e17382fa91/apache_tvm_ffi-0.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:549c2150e1c2d7ca7912cad173f62a192aec90cd981c024bd246161283ea5d78", size = 1950476, upload-time = "2025-11-04T02:42:54.159Z" }, + { url = "https://files.pythonhosted.org/packages/53/71/bb5ee4bca52a37a8f9580ab1f1de1be5366808a194981c324a756dabbe15/apache_tvm_ffi-0.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:3fbcfe526b458bc8edeafdc769388782d3bb4321c46a987e50bcece93ae78af8", size = 1711278, upload-time = "2025-11-04T02:42:55.56Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1e/f8d16dbe2303d1e7348037b4207d6c1093c554573484c97c8f3cde61a060/apache_tvm_ffi-0.1.1-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:f2c0164a5c6286f9c333ddedeb448b855cbc1225688d0a4c9aeab006ddfa1180", size = 1701072, upload-time = "2025-11-04T02:42:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/3d/47/f7a55e9b5b741f901ed9101a3ef46fd250f2c1519a6479e055432ff4f308/apache_tvm_ffi-0.1.1-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:33cc35738e0c44f2a67e550457b6b7dc7de9109ca64422a9e7063b1ba43c336e", size = 1854467, upload-time = "2025-11-04T02:43:00.158Z" }, + { url = "https://files.pythonhosted.org/packages/f2/db/f3adbe1e2d092fbb18908971a25ceb5496669ec65d01a28b7dd57f471ae0/apache_tvm_ffi-0.1.1-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9db6484259120b1bdc600f736084ee3d574775b1f4a3e8fef110323e3a9d2b6", size = 1930968, upload-time = "2025-11-04T02:43:01.96Z" }, + { url = "https://files.pythonhosted.org/packages/3b/da/7f678675ccc8af1c7d313322f3875e2c829f1faaa58c0d982431beeb3b3e/apache_tvm_ffi-0.1.1-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7bd812058ce9046cb69fd7b3e18538d1d0eefa1719822a1441b00bb841f7af4", size = 1811173, upload-time = "2025-11-04T02:43:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/e1/11/c8b3b7d69ceebd219dcb06f5e4a3997edea3bc2e0bbdd8f57ae65bba4f2f/apache_tvm_ffi-0.1.1-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:807def3039fb336a228c120ca8c32eb794bdfd2d7aff218c8611f287ad913736", size = 1922690, upload-time = "2025-11-04T02:43:04.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/0b/f816735d761049e53eb388264238655f58fcb42a31e0d1848a4fb6a6556b/apache_tvm_ffi-0.1.1-cp312-abi3-win_amd64.whl", hash = "sha256:624b4430ca3949f85fffd9ef498ebaf1155ff0ac659fc764eec6c6fd66ec7986", size = 1690969, upload-time = "2025-11-04T02:43:06.581Z" }, + { url = "https://files.pythonhosted.org/packages/12/aa/df81df8f8b39d3c41fbac41b1e6661d192d9987a3ef317fabcefecf727a6/apache_tvm_ffi-0.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c93d9de81c1ba9560fcc696cf84d777f88016eb53f05ee2d6288ddcb95a5e72f", size = 1732582, upload-time = "2025-11-04T02:43:08.042Z" }, + { url = "https://files.pythonhosted.org/packages/a8/55/861090532e4accd855e119f0e67e0e482b42abb866c9505edd8956148ebc/apache_tvm_ffi-0.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f9e0227179a0ce83384132b34757fd05f492270f1c031eae615870a5641b5039", size = 1870196, upload-time = "2025-11-04T02:43:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/2a/c6/470493934559e371ad699e1764649176efc5e022267c6dd0a565217177ad/apache_tvm_ffi-0.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78e75e193d675b9639e6fd0c33c60c3a4259d4c9f848f60baa6a3194df7e1fea", size = 1941999, upload-time = "2025-11-04T02:43:11.467Z" }, + { url = "https://files.pythonhosted.org/packages/85/b8/84eba0d266c9b10beae59a6863ef5c68044e20a6f12d46a42116e80db774/apache_tvm_ffi-0.1.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49792622720421525a18e378d848411731d32fcb05a00b6e54b84d05ff46cc22", size = 1823965, upload-time = "2025-11-04T02:43:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/64/73/ca73a43260a1374b1f34d0e6fcf6f8af16f66867a89dfd562b26184af1bd/apache_tvm_ffi-0.1.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:039293086d44e7f601bf8231e369198afe7ad38986330969ddb1a5fc7622976b", size = 1933779, upload-time = "2025-11-04T02:43:14.543Z" }, + { url = "https://files.pythonhosted.org/packages/5b/91/687c3b9ff3313addeebc1188ac50b299a82944ef1784b91890fc6f250ebd/apache_tvm_ffi-0.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:3f6cbd214bee2e52719d5264f05a2685c955ae7b096980f0361d917a5a9f47a6", size = 1751905, upload-time = "2025-11-04T02:43:16.286Z" }, +] + +[[package]] +name = "asciitree" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", size = 3951, upload-time = "2016-09-05T19:10:42.681Z" } + [[package]] name = "astroid" version = "3.2.4" @@ -483,75 +530,88 @@ wheels = [ [[package]] name = "bitarray" -version = "3.7.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e8/c1/644ea86b6f1a0864f656a3b3ee5bf8c29daa895cb3233942315fe065ea3a/bitarray-3.7.2.tar.gz", hash = "sha256:27a59bb7c64c0d094057a3536e15fdd693f8520771ee75d9344b82d0a5ade2d0", size = 150586, upload-time = "2025-10-08T14:29:03.309Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/76/78/488d06983f34f974c857ff6494dc8ef4f4b9d5e237d91d99ba47298bf446/bitarray-3.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:10b53d85e98c5e6189f2c5e9744171901eec4fc7b7d5a0ea47d0902fb9738abc", size = 147139, upload-time = "2025-10-08T14:26:51.915Z" }, - { url = "https://files.pythonhosted.org/packages/d5/aa/8ebe4ed9c2acba77d49e4a2c1fd25fefb841ceaba39bf6b3949c6c6e1a18/bitarray-3.7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7770f1b8f880acca6743da21665578082378014415c85454ca9eabf41c06b1d1", size = 144090, upload-time = "2025-10-08T14:26:53.472Z" }, - { url = "https://files.pythonhosted.org/packages/2f/20/0d93b1b54cb8944e582109814712fe37d934c8cd51d3faeaa69effacac0a/bitarray-3.7.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e83f074d9c52565cd14ec24f9895d6bbab14f123ac64c81415c2911d09b827ea", size = 321801, upload-time = "2025-10-08T14:26:54.963Z" }, - { url = "https://files.pythonhosted.org/packages/88/e8/bca798cdd87d9bd54f77bb0f304cd61c9a83d7610cc0c7c287207a6c3c39/bitarray-3.7.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5abc46f51e5414ef792172ee52e6af1f43d15cfab32f326b26e52be860476a25", size = 349665, upload-time = "2025-10-08T14:26:56.375Z" }, - { url = "https://files.pythonhosted.org/packages/f3/d3/052aca85c4d92f2ee221598249bd8332d72c716255beedbbb8799660909f/bitarray-3.7.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7f950ac47ead99959fca0017de8af3e966c16654399d04d6da84e395d7f1e3d", size = 359949, upload-time = "2025-10-08T14:26:57.879Z" }, - { url = "https://files.pythonhosted.org/packages/7b/6e/0633ffbbe621fc1820765b509deff640a34a35ccb8400ca2f574a50daf92/bitarray-3.7.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bfaf2f91616fca3025804100effa9153da8bfb3312232cfea9e2917956fd15f", size = 328142, upload-time = "2025-10-08T14:26:59.643Z" }, - { url = "https://files.pythonhosted.org/packages/30/68/9d34bbef00de3caf2d5a555c985b265736d264f884f54bb74d2735795f0d/bitarray-3.7.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f7cadb42a1a71203dfa3717e22a26779b60e7eea224efc063e11615ceeffe02d", size = 319634, upload-time = "2025-10-08T14:27:01.03Z" }, - { url = "https://files.pythonhosted.org/packages/29/88/f27c5f68fb4b59d873e63a036b71735900602caf62029153ed5fa72cad7f/bitarray-3.7.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:041fb7c29156b0d2779d600abdcf632e67e2e2b93664a00cd39d7d492a769ac4", size = 347811, upload-time = "2025-10-08T14:27:02.445Z" }, - { url = "https://files.pythonhosted.org/packages/92/a9/a3a60a4ae40c1ca726ea856e3e66090f9290d36df95fe179f01c8e3e3d00/bitarray-3.7.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:3ffb9a746282687b26d3475ab2288fa932e8321acdeae8b5cef26c0daeb43e4c", size = 344003, upload-time = "2025-10-08T14:27:03.68Z" }, - { url = "https://files.pythonhosted.org/packages/59/34/d9a7dc4187a72c30c7d98c995ba1b77f3280c5afe64e7409a3257c66d938/bitarray-3.7.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:042076dd5663401a6c881e8da40b883409cb1c7dbfb11959d80c0961d62ae388", size = 325038, upload-time = "2025-10-08T14:27:06.054Z" }, - { url = "https://files.pythonhosted.org/packages/a4/c8/926a2833ab80e15b5514d8270fb4b69ae858a84fadfe64c968ca663e35de/bitarray-3.7.2-cp310-cp310-win32.whl", hash = "sha256:90265fa81144e62ec00038796096d53a0b4e6b695c26628dc19a6a1805d018d8", size = 141257, upload-time = "2025-10-08T14:27:07.122Z" }, - { url = "https://files.pythonhosted.org/packages/be/88/f80eab0d7501bcfdf59efc2dd5e4da4784af7d1ccec263c3fda14a9d8bac/bitarray-3.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:72d7c86fd64dbad5b8e35eddff90f63a2db380760b84dab1287b0fe5a112eb76", size = 148071, upload-time = "2025-10-08T14:27:08.511Z" }, - { url = "https://files.pythonhosted.org/packages/f0/43/0ebf3c1f8aa39135f43c45f64c2b86bcc1bf8d0a1cc7cba43fe5e11b09fa/bitarray-3.7.2-cp310-cp310-win_arm64.whl", hash = "sha256:0bf145b35ce851778aa14c922376f0801a66b5d9cc69ed37b94a9582ff3c6af8", size = 145059, upload-time = "2025-10-08T14:27:09.562Z" }, - { url = "https://files.pythonhosted.org/packages/39/92/267e7234faefe3cddd3b1360b4c2946a2ebf7179513b8c7a5744f8b2ffd0/bitarray-3.7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f0b36a06fd7416feb2bf4651cf9ebd692ab457df38f2cf577209ba4b805bb4b6", size = 147140, upload-time = "2025-10-08T14:27:10.716Z" }, - { url = "https://files.pythonhosted.org/packages/4a/96/fb1081a580cc2ef7fdaba7bb8a6e34dafdef3b0c1697dbc6617216795987/bitarray-3.7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ee7c41ac961c5625791f463280c9de2dfb8cb1852843689c9701244a528b9b48", size = 144089, upload-time = "2025-10-08T14:27:11.74Z" }, - { url = "https://files.pythonhosted.org/packages/fb/f6/beeb02dec52bc970db9789b75eebee07467e9b824aad66eba53a763eef15/bitarray-3.7.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fffd2387aee2df25da51ae232e038febc6537a34e25d986aa8d4a1b43547d994", size = 329875, upload-time = "2025-10-08T14:27:12.772Z" }, - { url = "https://files.pythonhosted.org/packages/c9/a4/9d9fed4db42b4ac9e367786f007bde6cf41dcb3f0863b5176d95d8db5d10/bitarray-3.7.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:739153b628389e72e7a10a766c5f417a1f60e14f7d6c18e1d601c77ee3b03360", size = 357727, upload-time = "2025-10-08T14:27:13.965Z" }, - { url = "https://files.pythonhosted.org/packages/0d/35/1371a5ce63d3238abfad179dcb371dbe2491c9a2f5fe7fac41c1216a5955/bitarray-3.7.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f69aedc2f995e8f046b6bf9625866fdff9d6502cc02df59f57ada4196a03647", size = 368179, upload-time = "2025-10-08T14:27:15.097Z" }, - { url = "https://files.pythonhosted.org/packages/42/e3/85e699578e1eca1ad6104f6934596c10efc0cde0d87c75be331d8c7203a8/bitarray-3.7.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad2462fd80e12fc61776fbf4c0ec5b37ddc03cdbca479110329aa11ca5487e8", size = 335984, upload-time = "2025-10-08T14:27:16.234Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/785aa7925645c250cfa36e782cc078a0bc136140dd72293a51acab99a7bb/bitarray-3.7.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a0480c8cab13646a362dd4b925f486af0186c4334cc9ddfd4083b56fdb39fc1d", size = 327573, upload-time = "2025-10-08T14:27:17.368Z" }, - { url = "https://files.pythonhosted.org/packages/ed/2b/a06e84b85884666e2b784bd4d9f605484f5fedc6dd557729585a09ca5b28/bitarray-3.7.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:0617cd06aef4d38c5fa81475efcfd0ca5b73683331736042381b8c58be2bd913", size = 355752, upload-time = "2025-10-08T14:27:18.428Z" }, - { url = "https://files.pythonhosted.org/packages/66/06/57728ce036c1a8e740d22d229d325a7307d5aed07587a58a9aded7752b71/bitarray-3.7.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:13033bd04178695a93cbde503dde4a4f07f3b0d33927713ce55befffea5c20dc", size = 352164, upload-time = "2025-10-08T14:27:20.313Z" }, - { url = "https://files.pythonhosted.org/packages/4b/fe/719e62f924073be1d935266964a6a186a67389b1ad8e876271ad5f2ac5b4/bitarray-3.7.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a52a00c73afbea40522346b11850338bbc3c83c54572f3fd5eb4d28e7bb5fb4", size = 333041, upload-time = "2025-10-08T14:27:21.762Z" }, - { url = "https://files.pythonhosted.org/packages/73/f7/584d7f0c4267f845fb01d20fb16f9ad1d977d270557bfdd6b31824e9d08b/bitarray-3.7.2-cp311-cp311-win32.whl", hash = "sha256:4f9ea4014e7635570b609445ff10043eb23b60948f4fd70fb206fdc2b4cbcc60", size = 141405, upload-time = "2025-10-08T14:27:23.163Z" }, - { url = "https://files.pythonhosted.org/packages/ac/b2/ec72d1233fb2dfbe6d248ba567818c26038644c270926b865da411d958c7/bitarray-3.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:682d23e555dfb729039ddc7f36ac6fce1c84811c441d91714b0cb75d14a6dc98", size = 148254, upload-time = "2025-10-08T14:27:24.718Z" }, - { url = "https://files.pythonhosted.org/packages/2e/f1/6b6c76f92ba21a7eef847b8b5977c2b4f89736295616b996dc1da2ede9ca/bitarray-3.7.2-cp311-cp311-win_arm64.whl", hash = "sha256:824cb8f94268249755dc0fb2d72e09ddc5a48a1d4915f1c709a37099ae5b1e64", size = 145352, upload-time = "2025-10-08T14:27:25.951Z" }, - { url = "https://files.pythonhosted.org/packages/2d/6f/f91eda05b138e69e842c913461765b3cab4e22269f0ad756e530ae4aa932/bitarray-3.7.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0be3705631c15394231b205f19bfac1cfd67d86024c3ee0325305b8557303a8c", size = 147237, upload-time = "2025-10-08T14:27:27.336Z" }, - { url = "https://files.pythonhosted.org/packages/21/90/dd90023aa54d698d1afdbcac2cc76f0b67840dc2c44334543c057b43817b/bitarray-3.7.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:48e2551ba3562464ed3b0a6d10ae3505cbcd63b5a5fb8effcf13c65d5a39931c", size = 144020, upload-time = "2025-10-08T14:27:28.729Z" }, - { url = "https://files.pythonhosted.org/packages/49/15/7d5dc84ef3e8e12ec376ff06f1593c2f2cc5e16c9f3a1cb946b999031e78/bitarray-3.7.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f88ef6412eefee6bd99ad8b6f985f140da37e8e21cbcb84a4090be433267c8c9", size = 331886, upload-time = "2025-10-08T14:27:30.612Z" }, - { url = "https://files.pythonhosted.org/packages/e3/0f/77a1de93cf3a5878f555bb5f689b3f4c97b41cc1f4a8fd4a02e9fee5b9aa/bitarray-3.7.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5e113bc700a1c97fbb9442f129de9bcf10008bfafb5b12dc97f689d37002badd", size = 359759, upload-time = "2025-10-08T14:27:31.728Z" }, - { url = "https://files.pythonhosted.org/packages/d7/b4/fe070c3903e9b7b03b8198110b1b5c2f80bf91bb8abfe926b7b5fae5b1b4/bitarray-3.7.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:de5c6f960f279f716571ffb9146a601d5f64921264c41f2fc4316b86f996a648", size = 370990, upload-time = "2025-10-08T14:27:32.984Z" }, - { url = "https://files.pythonhosted.org/packages/ce/1a/fc71d713832d36b6221eee7f98a3422aa6febea1f55f3ee82fbbd5133d77/bitarray-3.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:06ebdf0d663e38a6c77aeaec16b89c8bb00110696aae12ef369413990ed467da", size = 339304, upload-time = "2025-10-08T14:27:34.04Z" }, - { url = "https://files.pythonhosted.org/packages/be/40/737018176f57265ec73164c98b7919345798eb984bd1ac311eb9eb156101/bitarray-3.7.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:890dd8431b6cc2c4cdaa24539d191c949802a11a20dca4cc0678452b1e527daf", size = 329607, upload-time = "2025-10-08T14:27:35.683Z" }, - { url = "https://files.pythonhosted.org/packages/a8/40/47087cdde8a70c1a77754a4c6f8a7a636289a83fb14e3e0608bc010a4719/bitarray-3.7.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:43330e929846790ac6d76b52de58da5b550fcef0627b4632de01f405c223b612", size = 357175, upload-time = "2025-10-08T14:27:36.761Z" }, - { url = "https://files.pythonhosted.org/packages/7d/0c/f06abce6637156efcfc836e4637e24be475478e5e81c9b050a1d1885e9c3/bitarray-3.7.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:b9f8d8d116925d12ab9d2992b12781bf34eeb2a6a329dcf1ea1c7407e6c07e07", size = 355268, upload-time = "2025-10-08T14:27:37.924Z" }, - { url = "https://files.pythonhosted.org/packages/03/3e/d50498496f97d12e65d48bb96e831db537b17344dd071293353171ed1633/bitarray-3.7.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ddf01e86d866e6d321f71ee6e63d6381957797530125fa558ebca76b54567958", size = 335971, upload-time = "2025-10-08T14:27:39.738Z" }, - { url = "https://files.pythonhosted.org/packages/a8/28/ad7a934b37a8d20cd7673d0dcb3b1c125a077059309abb555518a7901d64/bitarray-3.7.2-cp312-cp312-win32.whl", hash = "sha256:73a29c49a81426a1b0d153064045f3f4fde6cb88ae38ada1d99d200486cf53a3", size = 141539, upload-time = "2025-10-08T14:27:41.021Z" }, - { url = "https://files.pythonhosted.org/packages/9f/20/6bbbd4309801ccad39624f66fc6407a3c9c95827074e8270591c9a6d3599/bitarray-3.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:283e5a5b735a7574a5242ed2ecbb0b09c9521ed78ff4067089efd2ba856e2332", size = 148533, upload-time = "2025-10-08T14:27:42.371Z" }, - { url = "https://files.pythonhosted.org/packages/88/e8/eb9bb20c8ad309c0e404b4d7b9d0e37b0d265b842998fcc4e9a12cd6895e/bitarray-3.7.2-cp312-cp312-win_arm64.whl", hash = "sha256:9c3f1d983c12dd1e54a808b78d685ccd9b96b7c43ef20fbf9b85fa076e491cec", size = 145496, upload-time = "2025-10-08T14:27:43.606Z" }, - { url = "https://files.pythonhosted.org/packages/7f/2e/45239f89c02dde9059360d20ef8b1f3979da4547fafc14571b6a1f4560a1/bitarray-3.7.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0eacd088bbad701d691da4a90e19f39469665d323a3809b82cb9e5abaf30aeea", size = 147218, upload-time = "2025-10-08T14:27:44.622Z" }, - { url = "https://files.pythonhosted.org/packages/c0/56/5f91439e970ed1ca7149e5a54bfa466b9142521378d7d972eab601ea5640/bitarray-3.7.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dde42566197f8148daeed354c0dbb0450b834c4fda6a94645810de64d39328fc", size = 143999, upload-time = "2025-10-08T14:27:45.772Z" }, - { url = "https://files.pythonhosted.org/packages/3e/2d/bbce096e1357615374707238e3e331d903771bdd2768fa7c955f1c21ef59/bitarray-3.7.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d595b7159318249064b94879b8a8d947e5ab11647ae975ade7e86b132bed091", size = 331956, upload-time = "2025-10-08T14:27:46.809Z" }, - { url = "https://files.pythonhosted.org/packages/89/7e/34739b627b804087aa20748df7ac2ec64b01499817f603cda5eb80d81961/bitarray-3.7.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba9a45ff8a96ada0d215e5111971f1b432064e9ab0e1fae668603cb0023086eb", size = 359825, upload-time = "2025-10-08T14:27:48.205Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c5/d548f3ca9b9f413768c91b58d127240b0464d6964b98ed091cf5a3284de3/bitarray-3.7.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:aabfd2ebd43f295a4eb945a4e3ca7f4de63ce196341b7f25dcf464147d8fd5b3", size = 371028, upload-time = "2025-10-08T14:27:49.595Z" }, - { url = "https://files.pythonhosted.org/packages/95/a3/8acb092a2ae90539b4f2dac41f6aed36761c382d9f44ba8d2baab75bff6d/bitarray-3.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c10c893ce03693bf5084470c782429f242dc84e836a6442155f25c3ba77948de", size = 339372, upload-time = "2025-10-08T14:27:50.726Z" }, - { url = "https://files.pythonhosted.org/packages/2d/a9/d265a635cf29ccfe0f7dcfd980b487c6ba82de3b9c13f2da07b25624eee8/bitarray-3.7.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:164ae38aed41f8ee663d2b4f950dc2502799a17cd2e5d004180c63b8f3640c72", size = 329601, upload-time = "2025-10-08T14:27:52.139Z" }, - { url = "https://files.pythonhosted.org/packages/cc/91/f7f97b7094702972350af0e0d9305e677e93bdde0e772497c67038bd137f/bitarray-3.7.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3496f761d08ccda94a07cd782fc97b23c818dfc1aaef5551349004174aa0cb85", size = 357191, upload-time = "2025-10-08T14:27:53.783Z" }, - { url = "https://files.pythonhosted.org/packages/96/7a/4530b77264e7ea887ba61fcb209a001871730720b1c6f47edc94a9190ac6/bitarray-3.7.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f18ca6039ec011e81a641cc622a168e7c4cbcf336bf854b7c075d49dd8dd85e0", size = 355262, upload-time = "2025-10-08T14:27:55.407Z" }, - { url = "https://files.pythonhosted.org/packages/6c/da/d7f8e7078b9dd291cfb97ab5f45dde399b86b411e6c0345c63727fac48d2/bitarray-3.7.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c0e96c88f7bd202bde53ad0d58d0d1b669ab2745152ed4b909c5d7e80558b44b", size = 335986, upload-time = "2025-10-08T14:27:56.576Z" }, - { url = "https://files.pythonhosted.org/packages/0e/8a/26f8dd9d14baa436b1a67b7460e684c16e26b92d2054675a99f982b445db/bitarray-3.7.2-cp313-cp313-win32.whl", hash = "sha256:5056531cbf9732cddacaf96b2732097c546f28a0a1b778e1d389852d43af7853", size = 141522, upload-time = "2025-10-08T14:27:57.705Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b9/c5cc21204d1457c42bcbbf93246e707f66fcd9ec93c2c57cb5f246386187/bitarray-3.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:ddc67e003e0065feaf70e529366425d0d5747a6487bbfffbec6f9e229960cdd6", size = 148540, upload-time = "2025-10-08T14:27:58.802Z" }, - { url = "https://files.pythonhosted.org/packages/f3/5e/4ee20ac951069e30b87964239666ee5e572bacb9f60c515445b079465e4d/bitarray-3.7.2-cp313-cp313-win_arm64.whl", hash = "sha256:ce782a6ee535042ea1bed8c57b5dbb45e59f208297abb079fa56a61aa8b120a6", size = 145505, upload-time = "2025-10-08T14:27:59.845Z" }, - { url = "https://files.pythonhosted.org/packages/2a/d6/235e9cc42d0e254b2e7a9c52dcff4e7a3f6cb0d045c8f533f48c78d3121c/bitarray-3.7.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:019bbd454feff2607c2af171eb5e8268925aa24ce3d1b43bfd87f2f0dddefc0e", size = 147209, upload-time = "2025-10-08T14:28:01.276Z" }, - { url = "https://files.pythonhosted.org/packages/82/1c/66179ed5f7b78583e8e4678bb68f6637cfcad5ea4febf46c3e4bada36e06/bitarray-3.7.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5f323773d6e3c22f25c2b9a2b96caee9a7aa5420861144f190ae0e183621e1b2", size = 144060, upload-time = "2025-10-08T14:28:02.68Z" }, - { url = "https://files.pythonhosted.org/packages/e4/65/e3a977864a9c0150885cf583e066a0303a612b6e829cfe3c1170a1e672c9/bitarray-3.7.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f95e5861b96b83b13d962f20b2e8fba26296e5cefde2c9015385e945798916da", size = 331856, upload-time = "2025-10-08T14:28:03.792Z" }, - { url = "https://files.pythonhosted.org/packages/91/31/965f75c78378fadd22824910f5a19c90e9c4aebc3bc78cd576761cb0f4e4/bitarray-3.7.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea5b4c553176b22438d89b4ec953124119dc0c5f51f80039947d5a49e920a3a7", size = 359879, upload-time = "2025-10-08T14:28:05.864Z" }, - { url = "https://files.pythonhosted.org/packages/18/24/fb4e32b5345067971262310ca19d751b0e87c9e03d622939015e755b9967/bitarray-3.7.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:884792b4e6c19dc6529ca28f2de82133d31c52039eb0c4bc034ae4f8d19afee2", size = 370605, upload-time = "2025-10-08T14:28:07.38Z" }, - { url = "https://files.pythonhosted.org/packages/54/33/1f861aa36b58c6d9351b71f9c26facb5badf0450d35b934cbe68df39bdfe/bitarray-3.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bff701d1852aed8c21ad071a284ff3ff51e1b48c36be39ea273a374cb7c971d", size = 339088, upload-time = "2025-10-08T14:28:08.552Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d7/6c891c2ef20ffbaa3a61272b1375849b7ba449fb236bd954588af80a45b9/bitarray-3.7.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eba43046de6ddaa2e917c189a25ae0a92c57ec9789c1a0ebd5cc9de1fab0d4f0", size = 329798, upload-time = "2025-10-08T14:28:09.83Z" }, - { url = "https://files.pythonhosted.org/packages/d3/be/e956c75c07a8a06ccfbe0610dc2276ea656d0f2dabfd47adae1b0688d901/bitarray-3.7.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:de77dfd695e599ea2dabd0c3d990548cde8ace15eeeb55b17bddbb8d2eab67a0", size = 357447, upload-time = "2025-10-08T14:28:11.066Z" }, - { url = "https://files.pythonhosted.org/packages/a1/16/4feb2544d21ba828d4d7f2e827060d8f278a30fba27c57d5e1561d3cf968/bitarray-3.7.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:a6dea053e7e5bcabae669e6d7730b894283ef7611d035798d85df12522dae6ff", size = 354724, upload-time = "2025-10-08T14:28:12.613Z" }, - { url = "https://files.pythonhosted.org/packages/b6/29/a49e9673d29646d659538b59c012df0e9d9201f84b5c84093d3810cef57b/bitarray-3.7.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:13985244301c1186760fa2e0107e838807c368fb1fc589601c54b72af0cf997c", size = 335984, upload-time = "2025-10-08T14:28:14.212Z" }, - { url = "https://files.pythonhosted.org/packages/71/1e/cab11929caaed8290b5a5c280beccd00c492e1affbd7c4312de1dfc34810/bitarray-3.7.2-cp314-cp314-win32.whl", hash = "sha256:c8462c9dd4be7c68eacc407f5214056f310b989aa62ba26280ef992170e78ff3", size = 140698, upload-time = "2025-10-08T14:28:15.82Z" }, - { url = "https://files.pythonhosted.org/packages/82/96/1d788e9e21c6600a0a13d6952edd2c5c2cb50a147536d72f9ea29ee986ea/bitarray-3.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:5edb42097a39ae253e19b5c8343c0bda0b8a0df486b6fce548992fa9141a2af7", size = 147312, upload-time = "2025-10-08T14:28:17.148Z" }, - { url = "https://files.pythonhosted.org/packages/08/ef/4dd74fd4a982b75bade2ce061dde8cbc52f7cadfffecca102edbc8f5dd8f/bitarray-3.7.2-cp314-cp314-win_arm64.whl", hash = "sha256:6cab44b1963e54017fcda240a9a96d01f64fd9e03e29aea6e12cd49c0e0a1bc7", size = 144704, upload-time = "2025-10-08T14:28:18.63Z" }, +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/06/92fdc84448d324ab8434b78e65caf4fb4c6c90b4f8ad9bdd4c8021bfaf1e/bitarray-3.8.0.tar.gz", hash = "sha256:3eae38daffd77c9621ae80c16932eea3fb3a4af141fb7cc724d4ad93eff9210d", size = 151991, upload-time = "2025-11-02T21:41:15.117Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/b9/8a645fd36fc4c01ee223f97eccd4699c2f2e91681ccb33c0e963881c8e58/bitarray-3.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f08342dc8d19214faa7ef99574dea6c37a2790d6d04a9793ef8fa76c188dc08d", size = 148504, upload-time = "2025-11-02T21:38:54.596Z" }, + { url = "https://files.pythonhosted.org/packages/c0/f4/11b562e13ff732bd0674376f367f0a272034ebc28b8efbafbeb924552d21/bitarray-3.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:792462abfeeca6cc8c6c1e6d27e14319682f0182f6b0ba37befe911af794db70", size = 145481, upload-time = "2025-11-02T21:38:56.253Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7c/5a2487da579491b38abab3b437e01d3b05be6e16e69cc5eb304040dcebd5/bitarray-3.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0df69d26f21a9d2f1b20266f6737fa43f08aa5015c99900fb69f255fbe4dabb4", size = 322760, upload-time = "2025-11-02T21:38:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/8d/59/f0ef82d6a878d4af1b4961d208a716317929aa172fc0dfa5f4115319a873/bitarray-3.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4f10d3f304be7183fac79bf2cd997f82e16aa9a9f37343d76c026c6e435a8a8", size = 350332, upload-time = "2025-11-02T21:38:58.238Z" }, + { url = "https://files.pythonhosted.org/packages/e5/ec/d444b22fce853327d4a8adec1de9987e11b28fcc2d7204dcbc544e196ed9/bitarray-3.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fc98ff43abad61f00515ad9a06213b7716699146e46eabd256cdfe7cb522bd97", size = 360787, upload-time = "2025-11-02T21:38:59.239Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9e/60b205f52ea9ff155e9f12249090475159c909039daa29e47cd95e115dd5/bitarray-3.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81c6b4a6c1af800d52a6fa32389ef8f4281583f4f99dc1a40f2bb47667281541", size = 329050, upload-time = "2025-11-02T21:39:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/2ce373b423bc85a0eb93ee1cba3977971259a92a116932632f417b1b04d2/bitarray-3.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f3fd8df63c41ff6a676d031956aebf68ebbc687b47c507da25501eb22eec341f", size = 320507, upload-time = "2025-11-02T21:39:01.714Z" }, + { url = "https://files.pythonhosted.org/packages/2a/88/437408a2674b8bdb02063dd1535969b9c73cb8fdd197485de431e506c50e/bitarray-3.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f0ce9d9e07c75da8027c62b4c9f45771d1d8aae7dc9ad7fb606c6a5aedbe9741", size = 348449, upload-time = "2025-11-02T21:39:03.124Z" }, + { url = "https://files.pythonhosted.org/packages/97/46/d799e7e731c778b6dcb4627bafd395102065e5ab15a4a31f4222a3e20706/bitarray-3.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8a9c962c64a4c08def58b9799333e33af94ec53038cf151d36edacdb41f81646", size = 344776, upload-time = "2025-11-02T21:39:04.147Z" }, + { url = "https://files.pythonhosted.org/packages/b3/9a/129fff56d22d316b1c848c6e13e64191485756b5cd6ceb08e640edb80020/bitarray-3.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1a54d7e7999735faacdcbe8128e30207abc2caf9f9fd7102d180b32f1b78bfce", size = 325899, upload-time = "2025-11-02T21:39:05.118Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/4b01e99452ecc39f4abccf9bf83fe0f01c390e9794dad2d04b2c8b893c5f/bitarray-3.8.0-cp310-cp310-win32.whl", hash = "sha256:3ea52df96566457735314794422274bd1962066bfb609e7eea9113d70cf04ffe", size = 142756, upload-time = "2025-11-02T21:39:06.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/3f/c83635a67d90f45f88012468566c233eed1e9e9a9184fa882ba4039fadb3/bitarray-3.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:82a07de83dce09b4fa1bccbdc8bde8f188b131666af0dc9048ba0a0e448d8a3b", size = 149527, upload-time = "2025-11-02T21:39:07.377Z" }, + { url = "https://files.pythonhosted.org/packages/33/46/391b3902a523d4555313640746460b19d317c6233d9379e150af97fa1554/bitarray-3.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:c5ba07e58fd98c9782201e79eb8dd4225733d212a5a3700f9a84d329bd0463a6", size = 146453, upload-time = "2025-11-02T21:39:08.624Z" }, + { url = "https://files.pythonhosted.org/packages/bc/7d/63558f1d0eb09217a3d30c1c847890879973e224a728fcff9391fab999b8/bitarray-3.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:25b9cff6c9856bc396232e2f609ea0c5ec1a8a24c500cee4cca96ba8a3cd50b6", size = 148502, upload-time = "2025-11-02T21:39:09.993Z" }, + { url = "https://files.pythonhosted.org/packages/5e/7b/f957ad211cb0172965b5f0881b67b99e2b6d41512af0a1001f44a44ddf4a/bitarray-3.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d9984017314da772f5f7460add7a0301a4ffc06c72c2998bb16c300a6253607", size = 145484, upload-time = "2025-11-02T21:39:10.904Z" }, + { url = "https://files.pythonhosted.org/packages/9f/dc/897973734f14f91467a3a795a4624752238053ecffaec7c8bbda1e363fda/bitarray-3.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbbbfbb7d039b20d289ce56b1beb46138d65769d04af50c199c6ac4cb6054d52", size = 330909, upload-time = "2025-11-02T21:39:12.276Z" }, + { url = "https://files.pythonhosted.org/packages/67/be/24b4b792426d92de289e73e09682915d567c2e69d47e8857586cbdc865d0/bitarray-3.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f1f723e260c35e1c7c57a09d3a6ebe681bd56c83e1208ae3ce1869b7c0d10d4f", size = 358469, upload-time = "2025-11-02T21:39:13.766Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0e/2eda69a7a59a6998df8fb57cc9d1e0e62888c599fb5237b0a8b479a01afb/bitarray-3.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cbd1660fb48827381ce3a621a4fdc237959e1cd4e98b098952a8f624a0726425", size = 369131, upload-time = "2025-11-02T21:39:15.041Z" }, + { url = "https://files.pythonhosted.org/packages/f7/7b/8a372d6635a6b2622477b2f96a569b2cd0318a62bc95a4a2144c7942c987/bitarray-3.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df6d7bf3e15b7e6e202a16ff4948a51759354016026deb04ab9b5acbbe35e096", size = 337089, upload-time = "2025-11-02T21:39:16.124Z" }, + { url = "https://files.pythonhosted.org/packages/93/f0/8eca934dbe5dee47a0e5ef44eeb72e85acacc8097c27cd164337bc4ec5d3/bitarray-3.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d5c931ec1c03111718cabf85f6012bb2815fa0ce578175567fa8d6f2cc15d3b4", size = 328504, upload-time = "2025-11-02T21:39:17.321Z" }, + { url = "https://files.pythonhosted.org/packages/88/dd/928b8e23a9950f8a8bfc42bc1e7de41f4e27f57de01a716308be5f683c2b/bitarray-3.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:41b53711f89008ba2de62e4c2d2260a8b357072fd4f18e1351b28955db2719dc", size = 356461, upload-time = "2025-11-02T21:39:18.396Z" }, + { url = "https://files.pythonhosted.org/packages/a9/93/4fb58417aff47fa2fe1874a39c9346b589a1d78c93a9cb24cccede5dc737/bitarray-3.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4f298daaaea58d45e245a132d6d2bdfb6f856da50dc03d75ebb761439fb626cf", size = 353008, upload-time = "2025-11-02T21:39:19.828Z" }, + { url = "https://files.pythonhosted.org/packages/da/54/aa04e4a7b45aa5913f08ee377d43319b0979925e3c0407882eb29df3be66/bitarray-3.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:30989a2451b693c3f9359d91098a744992b5431a0be4858f1fdf0ec76b457125", size = 334048, upload-time = "2025-11-02T21:39:20.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/52/e851f41076df014c05d6ac1ce34fbf7db5fa31241da3e2f09bb2be9e283d/bitarray-3.8.0-cp311-cp311-win32.whl", hash = "sha256:e5aed4754895942ae15ffa48c52d181e1c1463236fda68d2dba29c03aa61786b", size = 142907, upload-time = "2025-11-02T21:39:22.312Z" }, + { url = "https://files.pythonhosted.org/packages/28/01/db0006148b1dd13b4ac2686df8fa57d12f5887df313a506e939af0cb0997/bitarray-3.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:22c540ed20167d3dbb1e2d868ca935180247d620c40eace90efa774504a40e3b", size = 149670, upload-time = "2025-11-02T21:39:23.341Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ea/b7d55ee269b1426f758a535c9ec2a07c056f20f403fa981685c3c8b4798c/bitarray-3.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:84b52b2cf77bb7f703d16c4007b021078dbbe6cf8ffb57abe81a7bacfc175ef2", size = 146709, upload-time = "2025-11-02T21:39:24.343Z" }, + { url = "https://files.pythonhosted.org/packages/82/a0/0c41d893eda756315491adfdbf9bc928aee3d377a7f97a8834d453aa5de1/bitarray-3.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f2fcbe9b3a5996b417e030aa33a562e7e20dfc86271e53d7e841fc5df16268b8", size = 148575, upload-time = "2025-11-02T21:39:25.718Z" }, + { url = "https://files.pythonhosted.org/packages/0e/30/12ab2f4a4429bd844b419c37877caba93d676d18be71354fbbeb21d9f4cc/bitarray-3.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cd761d158f67e288fd0ebe00c3b158095ce80a4bc7c32b60c7121224003ba70d", size = 145454, upload-time = "2025-11-02T21:39:26.695Z" }, + { url = "https://files.pythonhosted.org/packages/26/58/314b3e3f219533464e120f0c51ac5123e7b1c1b91f725a4073fb70c5a858/bitarray-3.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c394a3f055b49f92626f83c1a0b6d6cd2c628f1ccd72481c3e3c6aa4695f3b20", size = 332949, upload-time = "2025-11-02T21:39:27.801Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ce/ca8c706bd8341c7a22dd92d2a528af71f7e5f4726085d93f81fd768cb03b/bitarray-3.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:969fd67de8c42affdb47b38b80f1eaa79ac0ef17d65407cdd931db1675315af1", size = 360599, upload-time = "2025-11-02T21:39:28.964Z" }, + { url = "https://files.pythonhosted.org/packages/ef/dc/aa181df85f933052d962804906b282acb433cb9318b08ec2aceb4ee34faf/bitarray-3.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:99d25aff3745c54e61ab340b98400c52ebec04290a62078155e0d7eb30380220", size = 371972, upload-time = "2025-11-02T21:39:30.228Z" }, + { url = "https://files.pythonhosted.org/packages/ff/d9/b805bfa158c7bcf4df0ac19b1be581b47e1ddb792c11023aed80a7058e78/bitarray-3.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e645b4c365d6f1f9e0799380ad6395268f3c3b898244a650aaeb8d9d27b74c35", size = 340303, upload-time = "2025-11-02T21:39:31.342Z" }, + { url = "https://files.pythonhosted.org/packages/1f/42/5308cc97ea929e30727292617a3a88293470166851e13c9e3f16f395da55/bitarray-3.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2fa23fdb3beab313950bbb49674e8a161e61449332d3997089fe3944953f1b77", size = 330494, upload-time = "2025-11-02T21:39:32.769Z" }, + { url = "https://files.pythonhosted.org/packages/4c/89/64f1596cb80433323efdbc8dcd0d6e57c40dfbe6ea3341623f34ec397edd/bitarray-3.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:165052a0e61c880f7093808a0c524ce1b3555bfa114c0dfb5c809cd07918a60d", size = 358123, upload-time = "2025-11-02T21:39:34.331Z" }, + { url = "https://files.pythonhosted.org/packages/27/fd/f3d49c5443b57087f888b5e118c8dd78bb7c8e8cfeeed250f8e92128a05f/bitarray-3.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:337c8cd46a4c6568d367ed676cbf2d7de16f890bb31dbb54c44c1d6bb6d4a1de", size = 356046, upload-time = "2025-11-02T21:39:35.449Z" }, + { url = "https://files.pythonhosted.org/packages/aa/db/1fd0b402bd2b47142e958b6930dbb9445235d03fa703c9a24caa6e576ae2/bitarray-3.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:21ca6a47bf20db9e7ad74ca04b3d479e4d76109b68333eb23535553d2705339e", size = 336872, upload-time = "2025-11-02T21:39:36.891Z" }, + { url = "https://files.pythonhosted.org/packages/58/73/680b47718f1313b4538af479c4732eaca0aeda34d93fc5b869f87932d57d/bitarray-3.8.0-cp312-cp312-win32.whl", hash = "sha256:178c5a4c7fdfb5cd79e372ae7f675390e670f3732e5bc68d327e01a5b3ff8d55", size = 143025, upload-time = "2025-11-02T21:39:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/f8/11/7792587c19c79a8283e8838f44709fa4338a8f7d2a3091dfd81c07ae89c7/bitarray-3.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:75a3b6e9c695a6570ea488db75b84bb592ff70a944957efa1c655867c575018b", size = 149969, upload-time = "2025-11-02T21:39:39.715Z" }, + { url = "https://files.pythonhosted.org/packages/9a/00/9df64b5d8a84e8e9ec392f6f9ce93f50626a5b301cb6c6b3fe3406454d66/bitarray-3.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:5591daf81313096909d973fb2612fccd87528fdfdd39f6478bdce54543178954", size = 146907, upload-time = "2025-11-02T21:39:40.815Z" }, + { url = "https://files.pythonhosted.org/packages/3e/35/480364d4baf1e34c79076750914664373f561c58abb5c31c35b3fae613ff/bitarray-3.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:18214bac86341f1cc413772e66447d6cca10981e2880b70ecaf4e826c04f95e9", size = 148582, upload-time = "2025-11-02T21:39:42.268Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a8/718b95524c803937f4edbaaf6480f39c80f6ed189d61357b345e8361ffb6/bitarray-3.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:01c5f0dc080b0ebb432f7a68ee1e88a76bd34f6d89c9568fcec65fb16ed71f0e", size = 145433, upload-time = "2025-11-02T21:39:43.552Z" }, + { url = "https://files.pythonhosted.org/packages/03/66/4a10f30dc9e2e01e3b4ecd44a511219f98e63c86b0e0f704c90fac24059b/bitarray-3.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:86685fa04067f7175f9718489ae755f6acde03593a1a9ca89305554af40e14fd", size = 332986, upload-time = "2025-11-02T21:39:44.656Z" }, + { url = "https://files.pythonhosted.org/packages/53/25/4c08774d847f80a1166e4c704b4e0f1c417c0afe6306eae0bc5e70d35faa/bitarray-3.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56896ceeffe25946c4010320629e2d858ca763cd8ded273c81672a5edbcb1e0a", size = 360634, upload-time = "2025-11-02T21:39:45.798Z" }, + { url = "https://files.pythonhosted.org/packages/a5/8f/bf8ad26169ebd0b2746d5c7564db734453ca467f8aab87e9d43b0a794383/bitarray-3.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9858dcbc23ba7eaadcd319786b982278a1a2b2020720b19db43e309579ff76fb", size = 371992, upload-time = "2025-11-02T21:39:46.968Z" }, + { url = "https://files.pythonhosted.org/packages/a9/16/ce166754e7c9d10650e02914552fa637cf3b2591f7ed16632bbf6b783312/bitarray-3.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa7dec53c25f1949513457ef8b0ea1fb40e76c672cc4d2daa8ad3c8d6b73491a", size = 340315, upload-time = "2025-11-02T21:39:48.182Z" }, + { url = "https://files.pythonhosted.org/packages/de/2a/fbba3a106ddd260e84b9a624f730257c32ba51a8a029565248dfedfdf6f2/bitarray-3.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:15a2eff91f54d2b1f573cca8ca6fb58763ce8fea80e7899ab028f3987ef71cd5", size = 330473, upload-time = "2025-11-02T21:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/68/97/56cf3c70196e7307ad32318a9d6ed969dbdc6a4534bbe429112fa7dfe42e/bitarray-3.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b1572ee0eb1967e71787af636bb7d1eb9c6735d5337762c450650e7f51844594", size = 358129, upload-time = "2025-11-02T21:39:51.189Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/afd391a5c0896d3339613321b2f94af853f29afc8bd3fbc327431244c642/bitarray-3.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5bfac7f236ba1a4d402644bdce47fb9db02a7cf3214a1f637d3a88390f9e5428", size = 356005, upload-time = "2025-11-02T21:39:52.355Z" }, + { url = "https://files.pythonhosted.org/packages/ae/08/a8e1a371babba29bad3378bb3a2cdca2b012170711e7fe1f22031a6b7b95/bitarray-3.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f0a55cf02d2cdd739b40ce10c09bbdd520e141217696add7a48b56e67bdfdfe6", size = 336862, upload-time = "2025-11-02T21:39:54.345Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/6dc1d0fdc06991c8dc3b1fcfe1ae49fbaced42064cd1b5f24278e73fe05f/bitarray-3.8.0-cp313-cp313-win32.whl", hash = "sha256:a2ba92f59e30ce915e9e79af37649432e3a212ddddf416d4d686b1b4825bcdb2", size = 143018, upload-time = "2025-11-02T21:39:56.361Z" }, + { url = "https://files.pythonhosted.org/packages/2e/72/76e13f5cd23b8b9071747909663ce3b02da24a5e7e22c35146338625db35/bitarray-3.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:1c8f2a5d8006db5a555e06f9437e76bf52537d3dfd130cb8ae2b30866aca32c9", size = 149977, upload-time = "2025-11-02T21:39:57.718Z" }, + { url = "https://files.pythonhosted.org/packages/01/37/60f336c32336cc3ec03b0c61076f16ea2f05d5371c8a56e802161d218b77/bitarray-3.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:50ddbe3a7b4b6ab96812f5a4d570f401a2cdb95642fd04c062f98939610bbeee", size = 146930, upload-time = "2025-11-02T21:39:59.308Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b0/411327a6c7f6b2bead64bb06fe60b92e0344957ec1ab0645d5ccc25fdafe/bitarray-3.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8cbd4bfc933b33b85c43ef4c1f4d5e3e9d91975ea6368acf5fbac02bac06ea89", size = 148563, upload-time = "2025-11-02T21:40:01.006Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bc/ff80d97c627d774f879da0ea93223adb1267feab7e07d5c17580ffe6d632/bitarray-3.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9d35d8f8a1c9ed4e2b08187b513f8a3c71958600129db3aa26d85ea3abfd1310", size = 145422, upload-time = "2025-11-02T21:40:02.535Z" }, + { url = "https://files.pythonhosted.org/packages/66/e7/b4cb6c5689aacd0a32f3aa8a507155eaa33528c63de2f182b60843fbf700/bitarray-3.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:99f55e14e7c56f4fafe1343480c32b110ef03836c21ff7c48bae7add6818f77c", size = 332852, upload-time = "2025-11-02T21:40:03.645Z" }, + { url = "https://files.pythonhosted.org/packages/e7/91/fbd1b047e3e2f4b65590f289c8151df1d203d75b005f5aae4e072fe77d76/bitarray-3.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dfbe2aa45b273f49e715c5345d94874cb65a28482bf231af408891c260601b8d", size = 360801, upload-time = "2025-11-02T21:40:04.827Z" }, + { url = "https://files.pythonhosted.org/packages/ef/4a/63064c593627bac8754fdafcb5343999c93ab2aeb27bcd9d270a010abea5/bitarray-3.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:64af877116edf051375b45f0bda648143176a017b13803ec7b3a3111dc05f4c5", size = 371408, upload-time = "2025-11-02T21:40:05.985Z" }, + { url = "https://files.pythonhosted.org/packages/46/97/ddc07723767bdafd170f2ff6e173c940fa874192783ee464aa3c1dedf07d/bitarray-3.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cdfbb27f2c46bb5bbdcee147530cbc5ca8ab858d7693924e88e30ada21b2c5e2", size = 340033, upload-time = "2025-11-02T21:40:07.189Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1e/e1ea9f1146fd4af032817069ff118918d73e5de519854ce3860e2ed560ff/bitarray-3.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4d73d4948dcc5591d880db8933004e01f1dd2296df9de815354d53469beb26fe", size = 330774, upload-time = "2025-11-02T21:40:08.496Z" }, + { url = "https://files.pythonhosted.org/packages/cf/9f/8242296c124a48d1eab471fd0838aeb7ea9c6fd720302d99ab7855d3e6d3/bitarray-3.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:28a85b056c0eb7f5d864c0ceef07034117e8ebfca756f50648c71950a568ba11", size = 358337, upload-time = "2025-11-02T21:40:10.035Z" }, + { url = "https://files.pythonhosted.org/packages/b5/6b/9095d75264c67d479f298c80802422464ce18c3cdd893252eeccf4997611/bitarray-3.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:79ec4498a545733ecace48d780d22407411b07403a2e08b9a4d7596c0b97ebd7", size = 355639, upload-time = "2025-11-02T21:40:11.485Z" }, + { url = "https://files.pythonhosted.org/packages/a0/af/c93c0ae5ef824136e90ac7ddf6cceccb1232f34240b2f55a922f874da9b4/bitarray-3.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:33af25c4ff7723363cb8404dfc2eefeab4110b654f6c98d26aba8a08c745d860", size = 336999, upload-time = "2025-11-02T21:40:12.709Z" }, + { url = "https://files.pythonhosted.org/packages/81/0f/72c951f5997b2876355d5e671f78dd2362493254876675cf22dbd24389ae/bitarray-3.8.0-cp314-cp314-win32.whl", hash = "sha256:2c3bb96b6026643ce24677650889b09073f60b9860a71765f843c99f9ab38b25", size = 142169, upload-time = "2025-11-02T21:40:14.031Z" }, + { url = "https://files.pythonhosted.org/packages/8a/55/ef1b4de8107bf13823da8756c20e1fbc9452228b4e837f46f6d9ddba3eb3/bitarray-3.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:847c7f61964225fc489fe1d49eda7e0e0d253e98862c012cecf845f9ad45cdf4", size = 148737, upload-time = "2025-11-02T21:40:15.436Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/bc0784136775024ac56cc67c0d6f9aa77a7770de7f82c3a7c9be11c217cd/bitarray-3.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:a2cb35a6efaa0e3623d8272471371a12c7e07b51a33e5efce9b58f655d864b4e", size = 146083, upload-time = "2025-11-02T21:40:17.135Z" }, + { url = "https://files.pythonhosted.org/packages/6e/64/57984e64264bf43d93a1809e645972771566a2d0345f4896b041ce20b000/bitarray-3.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:15e8d0597cc6e8496de6f4dea2a6880c57e1251502a7072f5631108a1aa28521", size = 149455, upload-time = "2025-11-02T21:40:18.558Z" }, + { url = "https://files.pythonhosted.org/packages/81/c0/0d5f2eaef1867f462f764bdb07d1e116c33a1bf052ea21889aefe4282f5b/bitarray-3.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8ffe660e963ae711cb9e2b8d8461c9b1ad6167823837fc17d59d5e539fb898fa", size = 146491, upload-time = "2025-11-02T21:40:19.665Z" }, + { url = "https://files.pythonhosted.org/packages/65/c6/bc1261f7a8862c0c59220a484464739e52235fd1e2afcb24d7f7d3fb5702/bitarray-3.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4779f356083c62e29b4198d290b7b17a39a69702d150678b7efff0fdddf494a8", size = 339721, upload-time = "2025-11-02T21:40:21.277Z" }, + { url = "https://files.pythonhosted.org/packages/81/d8/289ca55dd2939ea17b1108dc53bffc0fdc5160ba44f77502dfaae35d08c6/bitarray-3.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:025d133bf4ca8cf75f904eeb8ea946228d7c043231866143f31946a6f4dd0bf3", size = 367823, upload-time = "2025-11-02T21:40:22.463Z" }, + { url = "https://files.pythonhosted.org/packages/91/a2/61e7461ca9ac0fcb70f327a2e84b006996d2a840898e69037a39c87c6d06/bitarray-3.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:451f9958850ea98440d542278368c8d1e1ea821e2494b204570ba34a340759df", size = 377341, upload-time = "2025-11-02T21:40:23.789Z" }, + { url = "https://files.pythonhosted.org/packages/6c/87/4a0c9c8bdb13916d443e04d8f8542eef9190f31425da3c17c3478c40173f/bitarray-3.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6d79f659965290af60d6acc8e2716341865fe74609a7ede2a33c2f86ad893b8f", size = 344985, upload-time = "2025-11-02T21:40:25.261Z" }, + { url = "https://files.pythonhosted.org/packages/17/4c/ff9259b916efe53695b631772e5213699c738efc2471b5ffe273f4000994/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fbf05678c2ae0064fb1b8de7e9e8f0fc30621b73c8477786dd0fb3868044a8c8", size = 336796, upload-time = "2025-11-02T21:40:26.942Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4b/51b2468bbddbade5e2f3b8d5db08282c5b309e8687b0f02f75a8b5ff559c/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c396358023b876cff547ce87f4e8ff8a2280598873a137e8cc69e115262260b8", size = 365085, upload-time = "2025-11-02T21:40:28.224Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/53473bfc2e052c6dbb628cdc1b156be621c77aaeb715918358b01574be55/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed3493a369fe849cce98542d7405c88030b355e4d2e113887cb7ecc86c205773", size = 361012, upload-time = "2025-11-02T21:40:29.635Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/242bf2e44bfc69e73fa2b954b425d761a8e632f78ea31008f1c3cfad0854/bitarray-3.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:c764fb167411d5afaef88138542a4bfa28bd5e5ded5e8e42df87cef965efd6e9", size = 340644, upload-time = "2025-11-02T21:40:31.089Z" }, + { url = "https://files.pythonhosted.org/packages/cf/01/12e5ecf30a5de28a32485f226cad4b8a546845f65f755ce0365057ab1e92/bitarray-3.8.0-cp314-cp314t-win32.whl", hash = "sha256:e12769d3adcc419e65860de946df8d2ed274932177ac1cdb05186e498aaa9149", size = 143630, upload-time = "2025-11-02T21:40:32.351Z" }, + { url = "https://files.pythonhosted.org/packages/b6/92/6b6ade587b08024a8a890b07724775d29da9cf7497be5c3cbe226185e463/bitarray-3.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0ca70ccf789446a6dfde40b482ec21d28067172cd1f8efd50d5548159fccad9e", size = 150250, upload-time = "2025-11-02T21:40:33.596Z" }, + { url = "https://files.pythonhosted.org/packages/ed/40/be3858ffed004e47e48a2cefecdbf9b950d41098b780f9dc3aa609a88351/bitarray-3.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:2a3d1b05ffdd3e95687942ae7b13c63689f85d3f15c39b33329e3cb9ce6c015f", size = 147015, upload-time = "2025-11-02T21:40:35.064Z" }, ] [[package]] @@ -671,7 +731,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "implementation_name != 'PyPy' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -842,7 +902,7 @@ name = "click" version = "8.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } wheels = [ @@ -980,6 +1040,82 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] +[[package]] +name = "crc32c" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/a0/28b4686a8db0bb0f77970f4c6ccede90d1d5740a1d4b4703bd54c3e75655/crc32c-2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2c0f4eb01fe7c0a3e3f973a418e04d52101bb077dd77626fd80c658ec60aaf95", size = 66321, upload-time = "2025-10-17T06:18:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/76/1f/1697f5b8b770f715ed9b264d79e36b4f77ae0527f81f3c749ef08937a32e/crc32c-2.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6baefcfbca82b1a9678455416da24f18629769a76920c640d5a538620a7d12bb", size = 62985, upload-time = "2025-10-17T06:18:54.97Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e5/333cfa5ffa8d5779733aced2b984b5e5139b4a8ceaa2c6bc563e9a1092f3/crc32c-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7f959fcf6c5aad1c4a653ee1a50f05760dab1d1c35d98ec4d7f0f68643f7612", size = 61517, upload-time = "2025-10-17T06:18:55.795Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d8/362a009e8140dd926a153b44d56753e3aa7cb50aca243779a84adadbff11/crc32c-2.8-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bb678507a4e4cf3f0506607b046ecc4ed1c58a19e08a3fb3c2d25441c480bf1", size = 79385, upload-time = "2025-10-17T06:18:56.598Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0d4ea3aa71ffb15f1285669d23024cc40779388ce32157d339dc2584491c/crc32c-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a16f7ffa4c242a909558565567cbba95148603717b53538ea299c98da68e7a9", size = 80965, upload-time = "2025-10-17T06:18:57.384Z" }, + { url = "https://files.pythonhosted.org/packages/20/44/d77657aaca4a2c0283f2356a3da6f8e91b003567bb8f09daaf540cbf192f/crc32c-2.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0184369aad562d801f91f454c81f56b9ecb966f6b96684c4d6cf82fc8741d2ad", size = 79993, upload-time = "2025-10-17T06:18:58.503Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c0/07017a93ebf85d9408028b7e03ef96d5c6bfb14cb77cfe90d35eedcc1501/crc32c-2.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86d2eeb5f0189bd803720abe7387019328ea34c4acde62999e5723f789bc316b", size = 79243, upload-time = "2025-10-17T06:18:59.273Z" }, + { url = "https://files.pythonhosted.org/packages/c7/1a/b3c5ac4cf2fd1f82395173d0bd8e1a15d09f0bc1eccdf10ea7f8caaccd67/crc32c-2.8-cp310-cp310-win32.whl", hash = "sha256:51da61904a9e753780a2e6011885677d601db1fa840be4b68799643a113e6f08", size = 64888, upload-time = "2025-10-17T06:19:00.089Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f2/60c45fc7bb2221d3c93c7a872e921be591f40d45228fe46f879b1d8c0424/crc32c-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:b2d6a1f2500daaf2e4b08f97ad0349aa2eff5faaaa5fd3350314a26eade334cd", size = 66639, upload-time = "2025-10-17T06:19:00.974Z" }, + { url = "https://files.pythonhosted.org/packages/dc/0b/5e03b22d913698e9cc563f39b9f6bbd508606bf6b8e9122cd6bf196b87ea/crc32c-2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e560a97fbb96c9897cb1d9b5076ef12fc12e2e25622530a1afd0de4240f17e1f", size = 66329, upload-time = "2025-10-17T06:19:01.771Z" }, + { url = "https://files.pythonhosted.org/packages/6b/38/2fe0051ffe8c6a650c8b1ac0da31b8802d1dbe5fa40a84e4b6b6f5583db5/crc32c-2.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6762d276d90331a490ef7e71ffee53b9c0eb053bd75a272d786f3b08d3fe3671", size = 62988, upload-time = "2025-10-17T06:19:02.953Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/5837a71c014be83aba1469c58820d287fc836512a0cad6b8fdd43868accd/crc32c-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:60670569f5ede91e39f48fb0cb4060e05b8d8704dd9e17ede930bf441b2f73ef", size = 61522, upload-time = "2025-10-17T06:19:03.796Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/63972fc1452778e2092ae998c50cbfc2fc93e3fa9798a0278650cd6169c5/crc32c-2.8-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:711743da6ccc70b3c6718c328947b0b6f34a1fe6a6c27cc6c1d69cc226bf70e9", size = 80200, upload-time = "2025-10-17T06:19:04.617Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3a/60eb49d7bdada4122b3ffd45b0df54bdc1b8dd092cda4b069a287bdfcff4/crc32c-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eb4094a2054774f13b26f21bf56792bb44fa1fcee6c6ad099387a43ffbfb4fa", size = 81757, upload-time = "2025-10-17T06:19:05.496Z" }, + { url = "https://files.pythonhosted.org/packages/f5/63/6efc1b64429ef7d23bd58b75b7ac24d15df327e3ebbe9c247a0f7b1c2ed1/crc32c-2.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fff15bf2bd3e95780516baae935ed12be88deaa5ebe6143c53eb0d26a7bdc7b7", size = 80830, upload-time = "2025-10-17T06:19:06.621Z" }, + { url = "https://files.pythonhosted.org/packages/e1/eb/0ae9f436f8004f1c88f7429e659a7218a3879bd11a6b18ed1257aad7e98b/crc32c-2.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c0e11e3826668121fa53e0745635baf5e4f0ded437e8ff63ea56f38fc4f970a", size = 80095, upload-time = "2025-10-17T06:19:07.381Z" }, + { url = "https://files.pythonhosted.org/packages/9e/81/4afc9d468977a4cd94a2eb62908553345009a7c0d30e74463a15d4b48ec3/crc32c-2.8-cp311-cp311-win32.whl", hash = "sha256:38f915336715d1f1353ab07d7d786f8a789b119e273aea106ba55355dfc9101d", size = 64886, upload-time = "2025-10-17T06:19:08.497Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e8/94e839c9f7e767bf8479046a207afd440a08f5c59b52586e1af5e64fa4a0/crc32c-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:60e0a765b1caab8d31b2ea80840639253906a9351d4b861551c8c8625ea20f86", size = 66639, upload-time = "2025-10-17T06:19:09.338Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab", size = 66379, upload-time = "2025-10-17T06:19:10.115Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68", size = 63042, upload-time = "2025-10-17T06:19:10.946Z" }, + { url = "https://files.pythonhosted.org/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2", size = 61528, upload-time = "2025-10-17T06:19:11.768Z" }, + { url = "https://files.pythonhosted.org/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2", size = 80028, upload-time = "2025-10-17T06:19:12.554Z" }, + { url = "https://files.pythonhosted.org/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1", size = 81531, upload-time = "2025-10-17T06:19:13.406Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e", size = 80608, upload-time = "2025-10-17T06:19:14.223Z" }, + { url = "https://files.pythonhosted.org/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c", size = 79886, upload-time = "2025-10-17T06:19:15.345Z" }, + { url = "https://files.pythonhosted.org/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76", size = 64896, upload-time = "2025-10-17T06:19:16.457Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2", size = 66645, upload-time = "2025-10-17T06:19:17.235Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, + { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, + { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, + { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, + { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, + { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, + { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, + { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, + { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, + { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, + { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, + { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, + { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/dd926c68eb8aac8b142a1a10b8eb62d95212c1cf81775644373fe7cceac2/crc32c-2.8-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5833f4071da7ea182c514ba17d1eee8aec3c5be927d798222fbfbbd0f5eea02c", size = 62345, upload-time = "2025-10-17T06:20:09.39Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/803404e5abea2ef2c15042edca04bbb7f625044cca879e47f186b43887c2/crc32c-2.8-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1dc4da036126ac07b39dd9d03e93e585ec615a2ad28ff12757aef7de175295a8", size = 61229, upload-time = "2025-10-17T06:20:10.236Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3a/00cc578cd27ed0b22c9be25cef2c24539d92df9fa80ebd67a3fc5419724c/crc32c-2.8-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:15905fa78344654e241371c47e6ed2411f9eeb2b8095311c68c88eccf541e8b4", size = 64108, upload-time = "2025-10-17T06:20:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/6b/bc/0587ef99a1c7629f95dd0c9d4f3d894de383a0df85831eb16c48a6afdae4/crc32c-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c596f918688821f796434e89b431b1698396c38bf0b56de873621528fe3ecb1e", size = 64815, upload-time = "2025-10-17T06:20:11.919Z" }, + { url = "https://files.pythonhosted.org/packages/73/42/94f2b8b92eae9064fcfb8deef2b971514065bd606231f8857ff8ae02bebd/crc32c-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8d23c4fe01b3844cb6e091044bc1cebdef7d16472e058ce12d9fadf10d2614af", size = 66659, upload-time = "2025-10-17T06:20:12.766Z" }, +] + [[package]] name = "cryptography" version = "42.0.8" @@ -1018,63 +1154,118 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/38/2c8dae407d301eaf942e377a5b2b30485cfa0df03c6c2dcc2ac044054ed9/cryptography-42.0.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d", size = 2801764, upload-time = "2024-06-04T19:54:25.455Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/98/0666ee759cd2e5306f911cbc95d2c6c814326906ed6b9c09e817a4b4a7c8/cuda_bindings-13.0.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d56e46a9e984bb754e56b9d060cf027fe99f08a97651ce6d8aa1c2032476d01e", size = 11762523, upload-time = "2025-10-21T15:08:45.913Z" }, + { url = "https://files.pythonhosted.org/packages/e1/36/2b2a43c8a6f8d8ff7a5ec7de1357ba3f1438ea69281c9deb90df29d55d56/cuda_bindings-13.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f797ce534a303525259be0ae7ee9cfcf4f7874b22f1f9b8e85555509dccb83", size = 12136098, upload-time = "2025-10-21T15:08:48.233Z" }, + { url = "https://files.pythonhosted.org/packages/47/67/5de1d48189511114859a1a131193896f88271c067a64b1159787e2d9f89b/cuda_bindings-13.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:74307cea1feee6c32a6e27b42e77beb22cd21cff4b7764fd214fa6ff89f8bd69", size = 11106982, upload-time = "2025-10-21T15:08:50.433Z" }, + { url = "https://files.pythonhosted.org/packages/dc/67/9e171ee6359d4aabf2d8202802c85487cae6c2eb52b9352bb7754583802f/cuda_bindings-13.0.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfd66c25a133365c4f93e3396c38c64b04400ccafd18c3a889ae251a1bfabaa1", size = 11807212, upload-time = "2025-10-21T15:08:52.988Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/d7036d9e402e6b5b57877a7496aba3bf2a0090f4fa3f072743fce3373eba/cuda_bindings-13.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9afede5937474864aa794eb57399dbdf5d2b05427aadcac275209b0857528a61", size = 12198791, upload-time = "2025-10-21T15:08:55.687Z" }, + { url = "https://files.pythonhosted.org/packages/83/25/620ce2afb6ea6d5da89d98375c85641f691924eef574247f7f0dd99f8bee/cuda_bindings-13.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:fce6d6b162457475b35e1a259ab643e683d1d20a84459fea898782e2f1e10a3b", size = 11138783, upload-time = "2025-10-21T15:08:57.741Z" }, + { url = "https://files.pythonhosted.org/packages/61/3c/c33fd3aa5fcc89aa1c135e477a0561f29142ab5fe028ca425fc87f7f0a74/cuda_bindings-13.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b899e5a513c11eaa18648f9bf5265d8de2a93f76ef66a6bfca0a2887303965cd", size = 11709086, upload-time = "2025-10-21T15:09:00.005Z" }, + { url = "https://files.pythonhosted.org/packages/21/ac/6b34452a3836c9fbabcd360689a353409d15f500dd9d9ced7f837549e383/cuda_bindings-13.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf41d9e69019939aa15296fa66ea7d3fdb8d2c6383f729f4b1156c8b37808a06", size = 12128303, upload-time = "2025-10-21T15:09:02.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/ad9cc2f0496886c37aefbc00256197a6043a3f04bbe959481e6908310afe/cuda_bindings-13.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:6b12ccd98f447aea9589d32caf9efda0c193994080752a60f790b646d519fe8c", size = 11237397, upload-time = "2025-10-21T15:09:05.421Z" }, + { url = "https://files.pythonhosted.org/packages/2f/36/41ccc303eb6be8ae82c5edd2ccae938876e8a794660e8bb96a193174a978/cuda_bindings-13.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fb16a7f769c9c67469add7a1d9f6c14dd44637f6921cb6b9eb82cb5015b35c3d", size = 11537064, upload-time = "2025-10-21T15:09:07.84Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ac/699889100536f1b63779646291e74eefa818087a0974eb271314d850f5dc/cuda_bindings-13.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:512d0d803a5e47a8a42d5a34ce0932802bf72fe952fdb11ac798715a35c6e5cb", size = 11910447, upload-time = "2025-10-21T15:09:09.942Z" }, + { url = "https://files.pythonhosted.org/packages/8c/f9/a2f5910aaf21f4cd43f456ea80f47f1424eece5b8f063dac1980304b8ef0/cuda_bindings-13.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:dd83e8d79587e265b82d3e589ba6b061770537443dfb1bb4a74f755c8b13f62b", size = 11211659, upload-time = "2025-10-21T15:09:12.639Z" }, + { url = "https://files.pythonhosted.org/packages/11/67/9656e003f18c5b32e1a2496998b24f4355ec978c5f3639b0eb9f6d0ff83f/cuda_bindings-13.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c859e326c776a47e66c50386a10c84fe34291eb6e711610c9fd7cc27d446334f", size = 11522409, upload-time = "2025-10-21T15:09:14.674Z" }, + { url = "https://files.pythonhosted.org/packages/18/d8/a83379caa7c1bed4195e704c24467a6c07fe8e29c7055ccd4f00c5702363/cuda_bindings-13.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e675dbd009fb5e66d63fd13a8ff35f849120f01bcc4dafadbced3004605c3588", size = 11903148, upload-time = "2025-10-21T15:09:16.918Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e0/ff1eeda06364df8c750843432ac6efb33a06df38261f0a1ceee59bb7dac2/cuda_bindings-13.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:193762306b6032c00a141fc38bcef92c6fb4d332fd2d6a550c7f950e7fd8acd8", size = 11543153, upload-time = "2025-10-21T15:09:19.252Z" }, + { url = "https://files.pythonhosted.org/packages/e8/99/0042dc5e98e3364480b1aaabc0f5c150d037825b264bba35ac7a883e46ee/cuda_bindings-13.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c7e6e89cdfc9b34f16a065cc6ad6c4bab19ce5dcef8da3ace8ad10bda899fa0", size = 11594384, upload-time = "2025-10-21T15:09:21.938Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c4/a931a90ce763bd7d587e18e73e4ce246b8547c78247c4f50ee24efc0e984/cuda_bindings-13.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e93866465e7ff4b7ebdf711cf9cd680499cd875f992058c68be08d4775ac233d", size = 11920899, upload-time = "2025-10-21T15:09:26.306Z" }, + { url = "https://files.pythonhosted.org/packages/14/3e/5725b2e5b9ac22bf19a50ec5f7611301ab6111c98ccf1b6b125fdaa71550/cuda_bindings-13.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bd2364bc49925837ce18dda259c3a36e539977ca0297799a54891cae1d5213f5", size = 11160621, upload-time = "2025-10-21T15:09:28.7Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/ec611e27ba48a9056f3b0610c5e27727e539f3905356cfe07acea18e772c/cuda_bindings-13.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ed06ef3507bd0aefb0da367e3d15676a8c7443bd68a88f298562d60b41078c20", size = 11521928, upload-time = "2025-10-21T15:09:30.714Z" }, + { url = "https://files.pythonhosted.org/packages/d4/2e/02cebf281ef5201b6bb9ea193b1a4d26e6233c46571cfb04c4a7dede12b9/cuda_bindings-13.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ab845487ca2c14accdcb393a559a3070469ea4b591d05e6ef439471f47f3e24", size = 11902749, upload-time = "2025-10-21T15:09:32.688Z" }, + { url = "https://files.pythonhosted.org/packages/36/d2/088c28751f54df7a251259ef3f99d34c428e12653f15db02fd62a96247af/cuda_bindings-13.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:aaa0934e16aa20ec10fbb1ecc53a6961b8d1c06a970fe05cc6ee7d2a805a090f", size = 11697137, upload-time = "2025-10-21T15:09:35.232Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/b2/a4982b5c7315d2dd211092d1ab226cb0c69b902480d0a58fde89b5991d71/cuda_pathfinder-1.3.2-py3-none-any.whl", hash = "sha256:7bd2774bc6be93aea226d579f415a63803b2b2c062207ed06c1d6dfc9cfacc3c", size = 27375, upload-time = "2025-10-29T21:51:45.342Z" }, +] + +[[package]] +name = "cuda-python" +version = "13.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings" }, + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/5f/beaa12a11b051027eec0b041df01c6690db4f02e3b2e8fadd5a0eeb4df52/cuda_python-13.0.3-py3-none-any.whl", hash = "sha256:914cd7e2dd075bd06a2d5121c1d9ccdd3d0c94b03ea5a44dbd98d24d8ed93bab", size = 7605, upload-time = "2025-10-21T15:48:59.222Z" }, +] + [[package]] name = "cython" -version = "3.1.6" +version = "3.2.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/58/6a8321cc0791876dc2509d7a22fc75535a1a7aa770b3496772f58b0a53a4/cython-3.1.6.tar.gz", hash = "sha256:ff4ccffcf98f30ab5723fc45a39c0548a3f6ab14f01d73930c5bfaea455ff01c", size = 3192329, upload-time = "2025-10-23T12:38:20.786Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/bb/23e917f1d2a11834730ff07cdb7e7c87ab72c16090b3d61b86477a38cc68/cython-3.1.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c4027b4d1bf7781fdfb2dbe1c1d81ccac9b910831511747e2c9fc8452fb3ea6b", size = 2989648, upload-time = "2025-10-23T12:38:38.272Z" }, - { url = "https://files.pythonhosted.org/packages/cd/72/9ec7797714c65bf45d11fb33361fd5cb522556d8a2a2e808f17db6a3aaf6/cython-3.1.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:141dea9df09f9c711af3b95510bd417c58b2abd33676eef1cb61f25581f7090a", size = 2914302, upload-time = "2025-10-23T12:38:39.888Z" }, - { url = "https://files.pythonhosted.org/packages/30/cd/63d551eb65273e144e9ee84bf697190586201dd02d2fd719b68e7da724e2/cython-3.1.6-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:486376a988268408b7e8ea7b4cccffb914aa497c498b41589fb4a862ba47e050", size = 3507159, upload-time = "2025-10-23T12:38:41.988Z" }, - { url = "https://files.pythonhosted.org/packages/44/bd/c451e15cd89ee98fa5207689505f9a211f79cdb4d18f2f96a7c9c6e7f3f6/cython-3.1.6-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdc6e63a04ead11812752a5198b85b7fc079688c76712348d072403f18fdeb49", size = 3261427, upload-time = "2025-10-23T12:38:43.838Z" }, - { url = "https://files.pythonhosted.org/packages/5d/dc/a4102de1a15a2ef56fc46e4486da112a8701b63ff98077d0ebaa39792e44/cython-3.1.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47e79f0bfbf403a5d6008bc9e7214e81e647794ca95cae6716399ba21abcc706", size = 3423208, upload-time = "2025-10-23T12:38:45.953Z" }, - { url = "https://files.pythonhosted.org/packages/e0/d6/dff399500588611e2bf189f191cc03bc985c80aaa263242c3abcd93122f7/cython-3.1.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2379f729f1d5a445adb4621f279f7c23aeb6245f036f96cce14b5b2fd1f5ff0a", size = 3276605, upload-time = "2025-10-23T12:38:47.825Z" }, - { url = "https://files.pythonhosted.org/packages/09/b1/af3d75e6b4363abd8efbe18cf90709b7dee38108846f3c7377ee50b8adcb/cython-3.1.6-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1089e18d938b6e742f077e398d52e1701080213c4f203755afde6f1b33d9e051", size = 3521386, upload-time = "2025-10-23T12:38:49.929Z" }, - { url = "https://files.pythonhosted.org/packages/0c/58/6fc30fba52c9cf35bb5d02effc7b16cdc9aa3d3aa56b07e47429c59ee657/cython-3.1.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73576246abbc62397db85cbdde74d2e5d73dabfdb7e593fdbb3671275ffb50ce", size = 3437394, upload-time = "2025-10-23T12:38:52.145Z" }, - { url = "https://files.pythonhosted.org/packages/69/c9/10bde13a679d1dc90b86bba754d94b126637686f4bba7637e14a923b8962/cython-3.1.6-cp310-cp310-win32.whl", hash = "sha256:f48eae3275b3352ba7eb550fc5321b0fb1ba8d916fa9985fb2f02ce42ae69ddd", size = 2480812, upload-time = "2025-10-23T12:38:54.126Z" }, - { url = "https://files.pythonhosted.org/packages/c9/60/c5dd9af41c9ec6ee406b423458065d2d3427422e0eb1bb91794c8ab3b787/cython-3.1.6-cp310-cp310-win_amd64.whl", hash = "sha256:4066908ee24a18572880966de1d0865d178f5ab9828a9249faa97e1ffdfbed9f", size = 2705655, upload-time = "2025-10-23T12:38:56.064Z" }, - { url = "https://files.pythonhosted.org/packages/a7/44/631939fd36577fccf0c47c9cd14fdc3d8125cde166ed2b2f1abdf9a505cc/cython-3.1.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5a1aedd8990f470d108b76ca768d9f1766d6610cf2546b73075dbe1e523daebe", size = 3022464, upload-time = "2025-10-23T12:38:57.677Z" }, - { url = "https://files.pythonhosted.org/packages/ec/68/700aef24fcf73f77940fec7efa27c18da68f6a5446dfce5e3a253ab707e3/cython-3.1.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f75c33e83e224737b1a68b2868bc08bddaabc6f04aef74864ff6069fe2e68341", size = 2952046, upload-time = "2025-10-23T12:38:59.684Z" }, - { url = "https://files.pythonhosted.org/packages/fd/9e/5dba03cc21190bd6756bb4717038a16cc87930ef32399c6d0e6bbbe538b3/cython-3.1.6-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:91b8fb3e961b3344bf257b851f2ce679727f44857fec94d643bcc458601dab54", size = 3408110, upload-time = "2025-10-23T12:39:01.442Z" }, - { url = "https://files.pythonhosted.org/packages/cb/45/81897d8802666d10086639b0f70702d2f9d03bb5358b012bb109b08b4dd1/cython-3.1.6-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1cfeb04d43464f5ff8398b499ba46c6eef22093da0e74b25f972576e768880e7", size = 3187425, upload-time = "2025-10-23T12:39:03.661Z" }, - { url = "https://files.pythonhosted.org/packages/3b/ed/1a1e93703edf37ee822c03013246d2b4c05a8ea689105051205150dadf07/cython-3.1.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f32366c198ac663a540ff4fa6ed55801d113183616c51100f4cc533568d2c4cf", size = 3309991, upload-time = "2025-10-23T12:39:05.801Z" }, - { url = "https://files.pythonhosted.org/packages/6e/11/147aefe4bdc5aa4f273283ea62949001d877808f4ad8a3b4774baf05f0ac/cython-3.1.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9856e8cd7f7a95a3f10a8f15fef4d17e5a4a57fb5185fe3482cec4adb0536635", size = 3202048, upload-time = "2025-10-23T12:39:07.52Z" }, - { url = "https://files.pythonhosted.org/packages/ef/82/6a43a68a1c9e22bef7476eb5a4fd8987812972b6746991b7b16b599aa872/cython-3.1.6-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6966f4d4ee13eceade2d952dc63bdf313f413c0c3f165aef0d6f62e6f27dab02", size = 3424512, upload-time = "2025-10-23T12:39:09.241Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d1/40dfa6c02bde72669525a2666aff5b0c75b0ec6f9d965b4beb1582ad4b6c/cython-3.1.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dffb14bc986626be50003f4edc614a2c0a56cbaaf87259f6c763a6d21da14921", size = 3326637, upload-time = "2025-10-23T12:39:11.376Z" }, - { url = "https://files.pythonhosted.org/packages/58/7c/c8dab163f2c9f8e3c4972aee31a45307f2b96733f799aa036ba05292efa8/cython-3.1.6-cp311-cp311-win32.whl", hash = "sha256:cde4748d37483b6c91df9f4327768e2828b1e374cb61bcee06d618958de59b7b", size = 2478500, upload-time = "2025-10-23T12:39:12.958Z" }, - { url = "https://files.pythonhosted.org/packages/e0/34/895cda4ac7e93460cedb28f609a7c056f09c1db5694ed38058f680c56386/cython-3.1.6-cp311-cp311-win_amd64.whl", hash = "sha256:29d6141b0c9697dfcaf5940eceb06353bec76f51f0579658964c0d29418000df", size = 2709986, upload-time = "2025-10-23T12:39:15.042Z" }, - { url = "https://files.pythonhosted.org/packages/70/cd/6e7bb9ef074d35c1b62af91c9f92126fae992d5a8fb6b47fdd1ade67bf56/cython-3.1.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0d2c32e8f6c65854e8203b381ff7ab540820763756b7c326e2c8dc18c9bbb44e", size = 3059014, upload-time = "2025-10-23T12:39:16.823Z" }, - { url = "https://files.pythonhosted.org/packages/13/04/a1b4fe2a4c72eb8fdcdf6b680908328f920f813caeb72f1b5d2cea40e45c/cython-3.1.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be24fcde7300a81712af279467ebc79baafc8483eb4dfa4daebf8ee90a826d39", size = 2966746, upload-time = "2025-10-23T12:39:18.56Z" }, - { url = "https://files.pythonhosted.org/packages/57/44/347f48b0ccfaa8233860a64b88a9df851138058ea923583e68625528710f/cython-3.1.6-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5012025af433bd7188fe1f7705df1c4a67e7add80c71658f6c6bc35ea876cc68", size = 3383297, upload-time = "2025-10-23T12:39:20.231Z" }, - { url = "https://files.pythonhosted.org/packages/98/80/e065d0725614ce9ff43624ae1d9f81647c5fd2d88ecffc2614dde703482d/cython-3.1.6-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b3520e2d4484f927c3ec00d32ffda75ec72cfd6a2ee07adac721cce339fa26f", size = 3164391, upload-time = "2025-10-23T12:39:22.036Z" }, - { url = "https://files.pythonhosted.org/packages/95/e1/3f86f321ff6bfd31310a5478f5ac56eaac3ea0743f6b76543ff5fbcb2b4e/cython-3.1.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a01d241d775319bcd7adb4144b070e1c4b01cdf841a62032492f07fad9efdc", size = 3316085, upload-time = "2025-10-23T12:39:23.795Z" }, - { url = "https://files.pythonhosted.org/packages/94/b5/677a2f4faa1c036cedbb715edc933b09de3e235891f1fcdaa82f8c3fdc85/cython-3.1.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fd88799fa7bb177182423e0745c9197c50938c6839ebfbe6fd01539582ed488e", size = 3176911, upload-time = "2025-10-23T12:39:25.749Z" }, - { url = "https://files.pythonhosted.org/packages/f8/e4/21117a7768ab19fcd766f2dd81f0a61d2d24e7a3649eff306349c2ab99a8/cython-3.1.6-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f809bae2e00b79c01ff5daf9a260df7c1bc9fda087b9d625592fa28c1a2248a9", size = 3396231, upload-time = "2025-10-23T12:39:28.168Z" }, - { url = "https://files.pythonhosted.org/packages/b5/4e/1152e9bfa0357d2237449fad94673c273f72c011a54c7227bb1291dd4423/cython-3.1.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f657e7a4b2242d159de603f280928d8e458dfba48144714774ad76c08f5a530", size = 3327101, upload-time = "2025-10-23T12:39:30.361Z" }, - { url = "https://files.pythonhosted.org/packages/39/fe/b7f9dc5ba8ce221aa7d40587d1d7175871b2ea61917c7fa4d5e85a7c042f/cython-3.1.6-cp312-cp312-win32.whl", hash = "sha256:6502f3e58db0ab3e2c983bec2c8c9e45d602e2c7ff921a5a8515b0008d918102", size = 2483823, upload-time = "2025-10-23T12:39:31.986Z" }, - { url = "https://files.pythonhosted.org/packages/40/d5/60261f023b0bdb28f0b9e8f00690b8bdbef692995184bc57f33811f8a936/cython-3.1.6-cp312-cp312-win_amd64.whl", hash = "sha256:71d099d8d6094c5de63a32e67b29964565aed889a218e8d16a94083f4239b904", size = 2701846, upload-time = "2025-10-23T12:39:33.769Z" }, - { url = "https://files.pythonhosted.org/packages/cc/96/22b43125180d9b2814da4271d9450a5cc4623a6c6439b6b1d8faa7675c81/cython-3.1.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f0d6b9f803eacf48e9e80ea12a03f54e5f5ac48914341b0a6b81554b3b3154", size = 3047517, upload-time = "2025-10-23T12:39:35.641Z" }, - { url = "https://files.pythonhosted.org/packages/db/09/8abf6ccb13d1e2589e60320423f861952cf4c4ec092cd8536e1beb018e9c/cython-3.1.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ec79615d0e29fa29fd4283bc7a2ed9c3d00532086a0031532d64b724db8c3e8e", size = 2954975, upload-time = "2025-10-23T12:39:37.568Z" }, - { url = "https://files.pythonhosted.org/packages/a6/4d/c3455fb738f52d536e7a113749c0a2242943251ce2d0dfac0e42ebba2fc0/cython-3.1.6-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:037d457738cf4fc12260946c6524b745f488cf413428099f2a064af7612d181f", size = 3355462, upload-time = "2025-10-23T12:39:39.462Z" }, - { url = "https://files.pythonhosted.org/packages/6b/b4/923f4d7ca7d987573aa2df0ca48fa9a103a48ddf1aec9cd8fcef9618b787/cython-3.1.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b036cb4ed7abcbc89cc04311832b22ad386c532fdd1fe690e1364aa992a54c7", size = 3150852, upload-time = "2025-10-23T12:39:41.416Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2c/985dd11b6cc3ac2e460c5e0b59030aebca66a85f9423db90e5186e8e9087/cython-3.1.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0fb2694327834c5bda7c5a07605f76437354d0ff76bb8739e77b479d176cf52", size = 3304059, upload-time = "2025-10-23T12:39:43.154Z" }, - { url = "https://files.pythonhosted.org/packages/69/af/b3af74d1d10a0f6d4d9fcdd836959ae54dabb36f84f316b09ccb84dbd8e0/cython-3.1.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92eb7a39e60426165a5b2a219af181e5695c4dedd598e317a7a4d9086bd66b91", size = 3166353, upload-time = "2025-10-23T12:39:45.146Z" }, - { url = "https://files.pythonhosted.org/packages/f1/2d/48130ecef876f141aaded34a961f32be45d2f36aa285de08d2e81aa5fec3/cython-3.1.6-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c475018b28f4e7111148bd02b600595090e0aac6cc49615c4586bb4e7f164a22", size = 3368659, upload-time = "2025-10-23T12:39:46.908Z" }, - { url = "https://files.pythonhosted.org/packages/2f/b2/0cd9ff5be3f0d224bc139eea8a8e83066d61ad424cf7fd0f43c3c4b791d4/cython-3.1.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b1b4bb661103cb95c6ca70daf5d39992b2d89fd260b02a54d92e365095ed37eb", size = 3316247, upload-time = "2025-10-23T12:39:48.699Z" }, - { url = "https://files.pythonhosted.org/packages/f9/0f/55f95e166c591fb8fd8caeb1f2c86cf86ef6f7f929a56094615ab757dc11/cython-3.1.6-cp313-cp313-win32.whl", hash = "sha256:69b1bea23b51628b8c9f14c3e0bb4c7dd5be63781bfbaa581b1c683b473c728a", size = 2477610, upload-time = "2025-10-23T12:39:51.014Z" }, - { url = "https://files.pythonhosted.org/packages/2e/07/23aa4577513a5e918c0deaf8a2ab8a9a5e6703e3fe554e3bc2c3bda1ef58/cython-3.1.6-cp313-cp313-win_amd64.whl", hash = "sha256:c844004712a9fe2a6f2ed4d6fe02aabb2e0e34f88c150724aad1afec7caff37a", size = 2699460, upload-time = "2025-10-23T12:39:54.146Z" }, - { url = "https://files.pythonhosted.org/packages/5b/16/e399f6fd33912116aba8bcdfeadd6093ff14996d7b5b72212fe4301e9f96/cython-3.1.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8893619efa77fc83934c1255c619d522711a5cf5933cef0d5c2b9755e8e5fabc", size = 3042822, upload-time = "2025-10-23T12:39:56.081Z" }, - { url = "https://files.pythonhosted.org/packages/94/aa/5500ff58f8972431c0e74783546b8cdc39511493aa44b74a7fde1ec4e654/cython-3.1.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bb49c74220af0b098f406701f0b87876b1c7614716d39786306986b9feea774b", size = 2963154, upload-time = "2025-10-23T12:39:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/cb/04/caa7893a4259e4bdb333a40a2105d58b53294445d9d2cf948eac9f0346b5/cython-3.1.6-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:defbf9571fca78e8a6e21b93d35c0a491d6af77a8e6180a0146da1b3c8eb8ce6", size = 3356015, upload-time = "2025-10-23T12:39:59.856Z" }, - { url = "https://files.pythonhosted.org/packages/df/da/6736caaf38a4d9f09db4b8dd76d0c8f7937820c2eef4d899f80259566298/cython-3.1.6-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cd7ea8c6ce0adf52d142bf37c4d54b8d0356818144a4584a24f2a0b9cdae6b8", size = 3188923, upload-time = "2025-10-23T12:40:01.926Z" }, - { url = "https://files.pythonhosted.org/packages/e8/ba/5dbee7f80c11c57a68b1e26d285e106ab259e7cf50536369b28f952b5809/cython-3.1.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c47fcc47553214e0a139fd33199d825c5d13970cd6c1039d2594af855ffb338", size = 3308343, upload-time = "2025-10-23T12:40:03.673Z" }, - { url = "https://files.pythonhosted.org/packages/81/c0/2759f4e2ec2f10ac941b2963de217f0ee6c0f6b2767ddcbaeba799c77dec/cython-3.1.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:92489385bca6d1935913540e35701a979618fdfeed4dbec6cad1be924fb487bf", size = 3205352, upload-time = "2025-10-23T12:40:05.431Z" }, - { url = "https://files.pythonhosted.org/packages/c7/fc/077b0084300d42bc69f4c9468c1946882884db859daa48b2b98b8f194fad/cython-3.1.6-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:926a3efd9b7012cdb3df0d1886e6f0e32e0b72a5d311ac2d3f48c0716fd91c6d", size = 3371256, upload-time = "2025-10-23T12:40:07.174Z" }, - { url = "https://files.pythonhosted.org/packages/60/71/4461521017e51b66a2d8dd443a596d636c87149e2d6ae95d664cbfdb1303/cython-3.1.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e35118eedfa0138154a43fb6b14e83703dae93193ba9940c747c170ed845cca7", size = 3319689, upload-time = "2025-10-23T12:40:09.181Z" }, - { url = "https://files.pythonhosted.org/packages/5b/53/f8dfff20e06dd3a6a39ed7b5ba784a9797eb206ec7df56f35c0e0ca31a49/cython-3.1.6-cp314-cp314-win32.whl", hash = "sha256:27f2b26442737d6e080900284883e078aae0276dfd7715a49b338f1a9481f7b9", size = 2499779, upload-time = "2025-10-23T12:40:11.306Z" }, - { url = "https://files.pythonhosted.org/packages/0a/cd/fef529bcc8eb6b55caf8bda524ee6194593137579fdc4ee616ff2a40dd2a/cython-3.1.6-cp314-cp314-win_amd64.whl", hash = "sha256:7f75ead2a7cad5ee719427b915711c70e40a114f045b2a9b5bd983484a0b83a7", size = 2731204, upload-time = "2025-10-23T12:40:13.878Z" }, - { url = "https://files.pythonhosted.org/packages/18/d5/7a04640bf559bb890455ffb28978daf7d44f667c3f04a4d422c655c1ba92/cython-3.1.6-py3-none-any.whl", hash = "sha256:91dcf7eb9b6a089ce4e9e1140e571d84c3bca834afb77ec269be7aa9d31a8157", size = 1223550, upload-time = "2025-10-23T12:38:16.732Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/52/82/01f0b63287cb922e5ba96c5147c30f1e51f541ce91bd178025bb3518b1ba/cython-3.2.0.tar.gz", hash = "sha256:41fdce8237baee2d961c292ed0386903dfe126f131e450a62de0fd7a5280d4b2", size = 3267264, upload-time = "2025-11-05T13:35:04.231Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/8d/b2e9578d960d38b1b04a278bf66e13008486aa73e73967186f2015d63d1c/cython-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee408125b2d218ec7d7a061e09d24715fcab9bf7ea1a4ac01907c3f8ec8730b3", size = 2953775, upload-time = "2025-11-05T13:35:22.291Z" }, + { url = "https://files.pythonhosted.org/packages/19/dd/cfd684f98bac9e0f505af1cbb7998498c59d713275e920a72b40dab03bfa/cython-3.2.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c93ce307b05fcd86a5bb0e4a7d7fab238e2f0e9936636097a60bc0e21f2def30", size = 3361627, upload-time = "2025-11-05T13:35:24.519Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c1/75acdbe9f6292514f0bb92ab1b78df5eedd7049235f4cbd194d2c6c46bfc/cython-3.2.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:191cfc2fa84642ad41a52d5abaacfb330d9a6653a465e4bf0a5681f66197a967", size = 3529751, upload-time = "2025-11-05T13:35:26.341Z" }, + { url = "https://files.pythonhosted.org/packages/f2/ce/d0468eb6d87b956902b02909f5007ad61e3839d4c07ab235b514911d869b/cython-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:a259053037ef82959b743b7fde238bd191ee43f88eb8e51101d5f3d8849f1e32", size = 2758839, upload-time = "2025-11-05T13:35:28.36Z" }, + { url = "https://files.pythonhosted.org/packages/ff/2b/904493fceda95747ba83971b40a66c8cc29ff009313429903f38ee620140/cython-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e9e4b2248dc3a98b86aeba65e9862d2cc881d072c163c0fb31b511d4d72e93c8", size = 2946248, upload-time = "2025-11-05T13:35:30.406Z" }, + { url = "https://files.pythonhosted.org/packages/89/fe/abe926699fe6c580967e30bc4035da54b5e31355ba9b1f4c0cf574228a84/cython-3.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02fb4990a83d5d6f780dda18ed8baa8d587cb6523f57b4d72bc0b41ad3766c96", size = 3236384, upload-time = "2025-11-05T13:35:32.233Z" }, + { url = "https://files.pythonhosted.org/packages/1b/36/6b6266549802234286438298d494152deb19922a94928d9dcd256659ebd1/cython-3.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a98925517819d62ea25d2cf40057df60a9bcf75fdd1d6ed3882e6ae0730d82f", size = 3372915, upload-time = "2025-11-05T13:35:34.082Z" }, + { url = "https://files.pythonhosted.org/packages/29/fa/5cf15466b428f9248e38a28515cf0fd98078ae869aa395cfb300315964c4/cython-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:4c959a5d4cd6331e8498822ba47200bd2ff4bf74517c0c91475d5bc21da3b4d5", size = 2762735, upload-time = "2025-11-05T13:35:35.806Z" }, + { url = "https://files.pythonhosted.org/packages/57/d3/2e6f5f2552c860bb9c00653d092103521846114f6a2ae0648ecf84c0816c/cython-3.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511d823d9f8a1b850178ec355d6df0a1731b9c20b08ee6d1a780f68215e9013f", size = 2959932, upload-time = "2025-11-05T13:35:37.518Z" }, + { url = "https://files.pythonhosted.org/packages/dd/bf/7bdc7f231fff6780f78586f939c1740475adecaa03bf256fcb62b2353952/cython-3.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbadeedcb2d135655bcce7380fb28c9e2a75b6810426c12b6e5a6fe6106fafb4", size = 3218588, upload-time = "2025-11-05T13:35:39.642Z" }, + { url = "https://files.pythonhosted.org/packages/be/81/7d7a81010897dc5abee59691f5fc85849dcc4c8a7687b22ed01bc8d86a7a/cython-3.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92d2394a3e3fe704210b5324eb8118333b514af72c98b1e02a6503945825b231", size = 3381940, upload-time = "2025-11-05T13:35:41.886Z" }, + { url = "https://files.pythonhosted.org/packages/4f/9d/35e7fb7b591bd9912685a772fcc773d7bb951a8feb6fb9be20addbc38928/cython-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:73435e56654a34ece57d4c3304a4556a8402cc4ae2d0e30f71c237a985dc5246", size = 2750886, upload-time = "2025-11-05T13:35:43.629Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d0/dc4b260e8fde81b23ab4dca56948b3e69617ef470247ec6a3e09370a9849/cython-3.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d900e58e826f9a5a27b0e2b50e33473e9986a5bae375c39b0f2e19f2c545fa23", size = 2950437, upload-time = "2025-11-05T13:35:45.427Z" }, + { url = "https://files.pythonhosted.org/packages/c8/53/c322bf0486a938ad954a645866b67e978777d79183cf0a042bda6bea11de/cython-3.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9d38cd3aab720d21fa6d6ee168228352f69aea0a95bd4fb84e8879c6ed38fbb", size = 3209331, upload-time = "2025-11-05T13:35:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/cd/48/55d02dba0606768d3450afd088e2bbcd6f8a54977dce041c2c3c1894631c/cython-3.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92b31d0b7b0a49b3d2aa94faaf75d44a03174cff2616b341a8853c919e511d51", size = 3370974, upload-time = "2025-11-05T13:35:49.534Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bd/6dab19652b68464572b7a137d07a91ebe86db2a81c35842ff5e49ef23403/cython-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:2847b74e76dbad612f6fc7182c12a5f78cffb0d05808fd2c4b638cf02d1aade6", size = 2746274, upload-time = "2025-11-05T13:35:51.522Z" }, + { url = "https://files.pythonhosted.org/packages/e2/db/de5331ca6489da1761078825709257e1f24e543b4040f86a2502a4b841f9/cython-3.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a0a8274959d538d12f865193dcd67bb5630906e020190c890d2b7c13d31713c6", size = 2961164, upload-time = "2025-11-05T13:35:53.826Z" }, + { url = "https://files.pythonhosted.org/packages/54/3e/64e37e419331f7c4c540ad25c0b3e6d8f44d597f21ab8861afbc66aa7e02/cython-3.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a1c800833c25195833805c7c3626a2c30b3baaaa9ba361a1af3bbc379662a8d", size = 3249627, upload-time = "2025-11-05T13:35:55.524Z" }, + { url = "https://files.pythonhosted.org/packages/9b/fc/9faedfcc2de807f77115d97a4910c260dd4693f4fa9e0e3be0d9ae89e260/cython-3.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df15af08c21c18a2e848df5954d6fd3310735089b60405132fa4111e2cf7482a", size = 3375458, upload-time = "2025-11-05T13:35:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/31/e0/30d449cd97ee0d6395aba18f2646b61b52ab3dc5a3851a346e2d363a7d85/cython-3.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:9d6876af2132757fff1b42a2f4eaa72482f991863160e3f0dc8f2c812b300ebf", size = 2783210, upload-time = "2025-11-05T13:35:59.54Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6b/9e1e171fe19274465d84dffa4610d46f434b1ae945e946802db396695d67/cython-3.2.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:04821ce06598a3aa5c9e0270d98960cfe6556dedbd1418c65e4479162b8ae74a", size = 2869249, upload-time = "2025-11-05T13:36:08.944Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f1/f461726f664668a96072b2a245bdfae566d68e2eb1393ec72780cc59c21e/cython-3.2.0-cp39-abi3-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:54b5b1c72a63da822b3f4739a0e31546c0a19f8e834b174906bf817ed5f9d65f", size = 3204332, upload-time = "2025-11-05T13:36:11.386Z" }, + { url = "https://files.pythonhosted.org/packages/78/d8/73c07ce64cae496e5f5a6dfe3e53574af1a8ef777e2a834d10dae8b67a4e/cython-3.2.0-cp39-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6155a6c360e32af1aaa16fa10b0119b49deeadff42a1958973324150870af1b5", size = 2851317, upload-time = "2025-11-05T13:36:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d9/d9f321637b8034b5028fa5fe7d1085ffa9351fea350af6510d5cb924c014/cython-3.2.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:861258ac3878b76c57b9b5a379787d772a0bc47fec9167b43986777de542c474", size = 2987155, upload-time = "2025-11-05T13:36:15.018Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b5/9f9e7d261f083b4066d734b27a7872b0c584fd4c3578196652dbf72b3f62/cython-3.2.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:85dbf955e3193893d0288105afa0fa5f4e835ff587061681f240a4f0487c44fb", size = 2884219, upload-time = "2025-11-05T13:36:17.334Z" }, + { url = "https://files.pythonhosted.org/packages/88/64/5aeb6e43e0ded9efedc5a516f87a487fdca8e434491cc352e5a805380459/cython-3.2.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:3b3f13822526726bac43275c0e92916bbcc2c30e9f559edc4c1132670b70498d", size = 3218067, upload-time = "2025-11-05T13:36:19.493Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a0/1958f54cd79d8251a330b9c9652b2a5ceba6a3fcec10782dd03e2a23c74f/cython-3.2.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ab18d09673d219008be5b6174bcbb6dbfd50904e66371f104a8a4698b791472d", size = 3108277, upload-time = "2025-11-05T13:36:21.203Z" }, + { url = "https://files.pythonhosted.org/packages/9c/84/9b8112160cab922b97edef00616ed18771567d88b5ba9d30d1736880c345/cython-3.2.0-cp39-abi3-win32.whl", hash = "sha256:c9fd986413fc52929b916187630a9abab9f876299951488c4b905ad5346afee6", size = 2430852, upload-time = "2025-11-05T13:36:23.049Z" }, + { url = "https://files.pythonhosted.org/packages/8f/57/65d3de140b51c45dd6892846bfabdfaaa032e2418f1cb1a2f46058c1fe42/cython-3.2.0-cp39-abi3-win_arm64.whl", hash = "sha256:ee2ea79ddeb721f912e7efea039b9db059c81767ff04fbf9a995f64e1187df99", size = 2435793, upload-time = "2025-11-05T13:36:25.139Z" }, + { url = "https://files.pythonhosted.org/packages/20/58/1f798ddb7fe6bfddf85f4f97d2d4ad63a491a7b643e85c1e274d0f09138e/cython-3.2.0-py3-none-any.whl", hash = "sha256:73f7f4c75acde5b5b4df05b11fdc2705ec637b99241d1bc2f4ebf345f7a2ea90", size = 1252818, upload-time = "2025-11-05T13:35:00.391Z" }, +] + +[[package]] +name = "datasets" +version = "4.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"], marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/bf/0dae295d6d1ba0b1a200a9dd216838464b5bbd05da01407cb1330b377445/datasets-4.4.1.tar.gz", hash = "sha256:80322699aa8c0bbbdb7caa87906da689c3c2e29523cff698775c67f28fdab1fc", size = 585341, upload-time = "2025-11-05T16:00:38.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5e/6f8d874366788ad5d549e9ba258037d974dda6e004843be1bda794571701/datasets-4.4.1-py3-none-any.whl", hash = "sha256:c1163de5211e42546079ab355cc0250c7e6db16eb209ac5ac6252f801f596c44", size = 511591, upload-time = "2025-11-05T16:00:36.365Z" }, ] [[package]] @@ -1100,8 +1291,8 @@ name = "deprecated" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ @@ -1149,6 +1340,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] +[[package]] +name = "donfig" +version = "0.8.1.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" }, +] + [[package]] name = "ebmlite" version = "3.4.1" @@ -1167,6 +1370,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/62/9773de14fe6c45c23649e98b83231fffd7b9892b6cf863251dc2afa73643/einops-0.8.1-py3-none-any.whl", hash = "sha256:919387eb55330f5757c6bea9165c5ff5cfe63a642682ea788a6d472576d81737", size = 64359, upload-time = "2025-02-09T03:17:01.998Z" }, ] +[[package]] +name = "emerging-optimizers" +version = "0.1.0" +source = { git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=cf9909b777ffac18e05b67a6708282cadc000942#cf9909b777ffac18e05b67a6708282cadc000942" } +dependencies = [ + { name = "absl-py" }, + { name = "torch", marker = "sys_platform == 'never'" }, + { name = "typing-extensions" }, +] + [[package]] name = "exceptiongroup" version = "1.3.0" @@ -1196,7 +1409,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.120.2" +version = "0.121.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -1204,9 +1417,18 @@ dependencies = [ { name = "starlette" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a0/fb/79e556bc8f9d360e5cc2fa7364a7ad6bda6f1736938b43a2791fa8baee7b/fastapi-0.120.2.tar.gz", hash = "sha256:4c5ab43e2a90335bbd8326d1b659eac0f3dbcc015e2af573c4f5de406232c4ac", size = 338684, upload-time = "2025-10-29T13:47:35.802Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e3/77a2df0946703973b9905fd0cde6172c15e0781984320123b4f5079e7113/fastapi-0.121.0.tar.gz", hash = "sha256:06663356a0b1ee93e875bbf05a31fb22314f5bed455afaaad2b2dad7f26e98fa", size = 342412, upload-time = "2025-11-03T10:25:54.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/2c/42277afc1ba1a18f8358561eee40785d27becab8f80a1f945c0a3051c6eb/fastapi-0.121.0-py3-none-any.whl", hash = "sha256:8bdf1b15a55f4e4b0d6201033da9109ea15632cb76cf156e7b8b4019f2172106", size = 109183, upload-time = "2025-11-03T10:25:53.27Z" }, +] + +[[package]] +name = "fasteners" +version = "0.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz", hash = "sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", size = 25087, upload-time = "2025-08-11T10:19:37.785Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/cc/1c33d05f62c9349bb80dfe789cc9a7409bdfb337a63fa347fd651d25294a/fastapi-0.120.2-py3-none-any.whl", hash = "sha256:bedcf2c14240e43d56cb9a339b32bcf15104fe6b5897c0222603cb7ec416c8eb", size = 108383, upload-time = "2025-10-29T13:47:32.978Z" }, + { url = "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl", hash = "sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", size = 18702, upload-time = "2025-08-11T10:19:35.716Z" }, ] [[package]] @@ -1242,6 +1464,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, ] +[[package]] +name = "fla-core" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "einops" }, + { name = "torch", marker = "sys_platform == 'never'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/67/c6/10a1149b07e6bab45b2cb2d07f6b827716c2baf5f3404161753f25c6389b/fla_core-0.3.2.tar.gz", hash = "sha256:d38db16bc4e1c6fa8c04df442f246da1e6926a209426bc6ef703d41bfbc37c92", size = 296725, upload-time = "2025-09-10T07:43:40.155Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/74947b33c07682280e65adbdf17c4ee94b30232df2f728bafecf13d1d820/fla_core-0.3.2-py3-none-any.whl", hash = "sha256:e751d5a41e33eee721a6fb6588bd857f6f36e0d14719a23b1ebdbd617d307209", size = 413594, upload-time = "2025-09-10T07:43:37.786Z" }, +] + [[package]] name = "flake8" version = "7.1.0" @@ -1256,6 +1491,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/43/d5147aadaa52558e94e024811f2f9543b4bd7203b3a9659eeb5dff9c61b3/flake8-7.1.0-py2.py3-none-any.whl", hash = "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a", size = 57569, upload-time = "2024-06-15T21:37:05.342Z" }, ] +[[package]] +name = "flash-linear-attention" +version = "0.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "datasets" }, + { name = "fla-core" }, + { name = "pytest" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/f6/e62c1e562a288557eba7f06f168a7615813d1a227327b8beb8ba426da2c5/flash_linear_attention-0.3.2.tar.gz", hash = "sha256:9147747316c2951fed4ebeb4fa87977c05d807dc70c93b46250b68a6eb1183e2", size = 150880, upload-time = "2025-09-10T07:43:41.37Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/d0/35ce9eac5f52c72005095aaa12a393d2656ed7ffedf925b2381a6b76d10c/flash_linear_attention-0.3.2-py3-none-any.whl", hash = "sha256:604e73361437ba786420ab195e2caa3fd19280503761e703fa353c5ce5c65376", size = 274592, upload-time = "2025-09-10T07:43:39.107Z" }, +] + [[package]] name = "flash-mla" version = "1.0.0+9edee0c" @@ -1263,22 +1513,27 @@ source = { git = "https://github.com/deepseek-ai/FlashMLA?rev=9edee0c022cd093814 [[package]] name = "flashinfer-python" -version = "0.3.1.post1" +version = "0.5.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "apache-tvm-ffi" }, { name = "click" }, { name = "einops" }, { name = "ninja" }, { name = "numpy" }, { name = "nvidia-cudnn-frontend" }, + { name = "nvidia-cutlass-dsl" }, + { name = "nvidia-ml-py" }, { name = "packaging" }, - { name = "pynvml" }, { name = "requests" }, { name = "tabulate" }, { name = "torch", marker = "sys_platform == 'never'" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/a7/f5bd3878f94fc47e25ecc0828f910233022366f7e832dfa02f3617fad41f/flashinfer_python-0.3.1.post1.tar.gz", hash = "sha256:d32218c7e33bcbf907719d3e51ddbea84d94a87fd0425378d70bcd28728f342e", size = 3817448, upload-time = "2025-09-26T04:26:25.177Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/bb/897c3b9d683dcf6490f70e468efb585eebcd673970b13a04ed947b491982/flashinfer_python-0.5.1.tar.gz", hash = "sha256:f12b32d88d8cc10a396456df8ab017f1c4661fbf257e14f4d2461961ec0d090e", size = 4627606, upload-time = "2025-11-04T05:55:02.376Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/f1/33dedad087a2bc3d66244126bd5d1c79721ea22d1f2124299f9e5bdaf3b1/flashinfer_python-0.5.1-py3-none-any.whl", hash = "sha256:ec8434d21e53a0ec333734a3c61946a0f7d2f972e344aefa99ba5b87e63aa76a", size = 6932706, upload-time = "2025-11-04T05:55:00.335Z" }, +] [[package]] name = "flask" @@ -1435,11 +1690,16 @@ wheels = [ [[package]] name = "fsspec" -version = "2025.9.0" +version = "2025.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, ] [[package]] @@ -1653,7 +1913,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging" }, { name = "pyyaml" }, { name = "requests" }, @@ -1699,11 +1959,13 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.12.*' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", @@ -1711,7 +1973,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'linux'", ] dependencies = [ - { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, + { name = "zipp", marker = "extra == 'extra-13-megatron-core-dev'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/08/c1395a292bb23fd03bdf572a1357c5a733d3eecbab877641ceacab23db6e/importlib_metadata-8.6.1.tar.gz", hash = "sha256:310b41d755445d74569f993ccfc22838295d9fe005425094fad953d7f15c8580", size = 55767, upload-time = "2025-01-20T22:21:30.429Z" } wheels = [ @@ -1723,23 +1985,39 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] dependencies = [ - { name = "zipp", marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "zipp", marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -1902,70 +2180,70 @@ wheels = [ [[package]] name = "libcst" -version = "1.8.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyyaml", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "pyyaml-ft", marker = "python_full_version >= '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/55/ca4552d7fe79a91b2a7b4fa39991e8a45a17c8bfbcaf264597d95903c777/libcst-1.8.5.tar.gz", hash = "sha256:e72e1816eed63f530668e93a4c22ff1cf8b91ddce0ec53e597d3f6c53e103ec7", size = 884582, upload-time = "2025-09-26T05:29:44.101Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/61/92115569ba7d5ccf0bd74d33641d261d184a09a9ed58699a8463c44b79d5/libcst-1.8.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:373011a1a995c6201cf76c72ab598cedc27de9a5d665428620610f599bfc5f20", size = 2206397, upload-time = "2025-09-26T05:27:47.397Z" }, - { url = "https://files.pythonhosted.org/packages/fc/2f/199d716d211b4938ba9a6cd0406b9c4fe36432a0e843230d6fab18d3ef67/libcst-1.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:774df1b40d338d245bb2d4e368ed99feb72a4642984125a5db62a3f4013a6e87", size = 2090397, upload-time = "2025-09-26T05:27:49.578Z" }, - { url = "https://files.pythonhosted.org/packages/f5/38/0e058e52bd6ac3436f45315a943b8a214b856f3fe0b1fcdb5ca65c3d7e66/libcst-1.8.5-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:08762c19eaf3d72162150ac0f0e1aa70378a10182ee539b8ecdf55c7f83b7f82", size = 2231629, upload-time = "2025-09-26T05:27:51.54Z" }, - { url = "https://files.pythonhosted.org/packages/d5/7e/d458480534bd30f4f9800228e38b96d647fc5f59d10f92ae2ab4d1271e5b/libcst-1.8.5-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:54a50034c29d477fd3ceed2bcc02e17142b354e4039831246c32fde59281d116", size = 2294203, upload-time = "2025-09-26T05:27:53.039Z" }, - { url = "https://files.pythonhosted.org/packages/54/ea/1900b3896b4a1bccdbee2484490cf50a00045f759e721e5e9915e5b4a4cf/libcst-1.8.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:667ec0b245b8fa1e4afaa69ab4640ff124d4f5e7a480196fedde705db69b8c56", size = 2297115, upload-time = "2025-09-26T05:27:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/e4/f8/ab4b1e7d0be5273948bd1e8f4ae1c3072dc4bf69f5fda559e04b30408e1b/libcst-1.8.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3b7e5142768418094fb8f673e107f01cfdfa70b72d6c97749f3619e2e8beacb1", size = 2398228, upload-time = "2025-09-26T05:27:56.654Z" }, - { url = "https://files.pythonhosted.org/packages/da/fb/2f334f94c6f61f1a8116da5234b2977efc2b752e23c7a6df0dd712a4f248/libcst-1.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:4ad060e43bd3ba54b4fefcc5f619fc2480fd5a7dbec6768b598bfe0eb46e3da9", size = 2106562, upload-time = "2025-09-26T05:27:58.439Z" }, - { url = "https://files.pythonhosted.org/packages/c1/16/f281fd995028ab2dec978b3468b43e3d8a465b7d804c1df0b3f7da03284e/libcst-1.8.5-cp310-cp310-win_arm64.whl", hash = "sha256:985303bbc3c748c8fb71f994b56cc2806385b423acd53f5dd1cc191b3c2df6d3", size = 1992204, upload-time = "2025-09-26T05:28:00.539Z" }, - { url = "https://files.pythonhosted.org/packages/b8/a0/4efb5b33c184f72554409516c73c8900909f87de528538d194b2cb5898ac/libcst-1.8.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dd5a292ce2b6410bc100aeac2b18ba3554fd8a8f6aa0ee6a9238bb4031c521ca", size = 2206056, upload-time = "2025-09-26T05:28:02.503Z" }, - { url = "https://files.pythonhosted.org/packages/26/b0/8b1dca00aebfc89f8e538212e5582548cedfc0b8f3aa4e73a815fe87bdfd/libcst-1.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f43915cd523a6967ba1dfe137627ed3804892005330c3bf53674a2ab4ff3dad", size = 2090132, upload-time = "2025-09-26T05:28:04.511Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1f/78ad030ca973f2c58fa58c3f30d94c2239473d3aba6c9dd1bdedd5047ddd/libcst-1.8.5-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:9a756bd314b87b87dec9f0f900672c37719645b1c8bb2b53fe37b5b5fe7ee2c2", size = 2231559, upload-time = "2025-09-26T05:28:06.492Z" }, - { url = "https://files.pythonhosted.org/packages/33/8a/2ee78c01070c919de3d6736a06d1d9ecaedcbe1f367f4eee3c34ae5f801e/libcst-1.8.5-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:26e9d5e756447873eeda78441fa7d1fe640c0b526e5be2b6b7ee0c8f03c4665f", size = 2293973, upload-time = "2025-09-26T05:28:08.456Z" }, - { url = "https://files.pythonhosted.org/packages/50/cf/ef4cb1c1b16f4bd32b0d7a5f01b18168fd833010a916bc062958dd6bcd8a/libcst-1.8.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b5b33ec61f62ff6122dc9c5bf1401bc8a9f9a2f0663ca15661d21d14d9dc4de0", size = 2297099, upload-time = "2025-09-26T05:28:10.4Z" }, - { url = "https://files.pythonhosted.org/packages/75/3e/ccd2e449f09c745ded6925804a6fe66f4c96ef82a0330de646becb8c6140/libcst-1.8.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a80e14836ecbdf5374c2c82cd5cd290abaa7290ecfafe9259d0615a1ebccb30c", size = 2398032, upload-time = "2025-09-26T05:28:12.124Z" }, - { url = "https://files.pythonhosted.org/packages/1f/16/277d0666e77d53d0061cb73327053b114f516ab7b36c9d4c71963fb5e806/libcst-1.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:588acde1588544b3bfe06069c118ee731e6712f323f26a026733f0ec4512596e", size = 2106472, upload-time = "2025-09-26T05:28:13.945Z" }, - { url = "https://files.pythonhosted.org/packages/bd/25/b1594abbec644a10b61ee1c1bab935ccc992a17b3880aa50234b9b4e9b06/libcst-1.8.5-cp311-cp311-win_arm64.whl", hash = "sha256:a8146f945f1eb46406fab676f86de3b7f88aca9e5d421f6366f7a63c8a950254", size = 1991976, upload-time = "2025-09-26T05:28:15.939Z" }, - { url = "https://files.pythonhosted.org/packages/13/bb/c7abe0654fcf00292d6959256948ce4ae07785c4f65a45c3e25cc4637074/libcst-1.8.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27c7733aba7b43239157661207b1e3a9f3711a7fc061a0eca6a33f0716fdfd21", size = 2196690, upload-time = "2025-09-26T05:28:17.839Z" }, - { url = "https://files.pythonhosted.org/packages/49/25/e7c02209e8ce66e7b75a66d132118f6f812a8b03cd31ee7d96de56c733a1/libcst-1.8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b8c3cfbbf6049e3c587713652e4b3c88cfbf7df7878b2eeefaa8dd20a48dc607", size = 2082616, upload-time = "2025-09-26T05:28:19.794Z" }, - { url = "https://files.pythonhosted.org/packages/32/68/a4f49d99e3130256e225d639722440ba2682c12812a30ebd7ba64fd0fd31/libcst-1.8.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:31d86025d8997c853f85c4b5d494f04a157fb962e24f187b4af70c7755c9b27d", size = 2229037, upload-time = "2025-09-26T05:28:21.459Z" }, - { url = "https://files.pythonhosted.org/packages/b2/62/4fa21600a0bf3eb9f4d4f8bbb50ef120fb0b2990195eabba997b0b889566/libcst-1.8.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:ff9c535cfe99f0be79ac3024772b288570751fc69fc472b44fca12d1912d1561", size = 2292806, upload-time = "2025-09-26T05:28:23.033Z" }, - { url = "https://files.pythonhosted.org/packages/14/df/a01e8d54b62060698e37e3e28f77559ecb70c7b93ffee00d17e40221f419/libcst-1.8.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e8204607504563d3606bbaea2b9b04e0cef2b3bdc14c89171a702c1e09b9318a", size = 2294836, upload-time = "2025-09-26T05:28:24.937Z" }, - { url = "https://files.pythonhosted.org/packages/75/4f/c410e7f7ceda0558f688c1ca5dfb3a40ff8dfc527f8e6015fa749e11a650/libcst-1.8.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5e6cd3df72d47701b205fa3349ba8899566df82cef248c2fdf5f575d640419c4", size = 2396004, upload-time = "2025-09-26T05:28:26.582Z" }, - { url = "https://files.pythonhosted.org/packages/f0/07/bb77dcb94badad0ad3e5a1e992a4318dbdf40632eac3b5cf18299858ad7d/libcst-1.8.5-cp312-cp312-win_amd64.whl", hash = "sha256:197c2f86dd0ca5c6464184ddef7f6440d64c8da39b78d16fc053da6701ed1209", size = 2107301, upload-time = "2025-09-26T05:28:28.235Z" }, - { url = "https://files.pythonhosted.org/packages/79/70/e688e6d99d6920c3f97bf8bbaec33ac2c71a947730772a1d32dd899dbbf1/libcst-1.8.5-cp312-cp312-win_arm64.whl", hash = "sha256:c5ca109c9a81dff3d947dceba635a08f9c3dfeb7f61b0b824a175ef0a98ea69b", size = 1990870, upload-time = "2025-09-26T05:28:29.858Z" }, - { url = "https://files.pythonhosted.org/packages/b0/77/ca1d2499881c774121ebb7c78c22f371c179f18317961e1e529dafc1af52/libcst-1.8.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9e9563dcd754b65557ba9cdff9a5af32cfa5f007be0db982429580db45bfe", size = 2196687, upload-time = "2025-09-26T05:28:31.769Z" }, - { url = "https://files.pythonhosted.org/packages/ef/1c/fdb7c226ad82fcf3b1bb19c24d8e895588a0c1fd2bc81e30792d041e15bc/libcst-1.8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:61d56839d237e9bf3310e6479ffaf6659f298940f0e0d2460ce71ee67a5375df", size = 2082639, upload-time = "2025-09-26T05:28:33.358Z" }, - { url = "https://files.pythonhosted.org/packages/af/1a/c6e89455483355971d13f6d71ad717624686b50558f7e2c12393c2c8e2f1/libcst-1.8.5-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b084769dcda2036265fc426eec5894c658af8d4b0e0d0255ab6bb78c8c9d6eb4", size = 2229202, upload-time = "2025-09-26T05:28:35.276Z" }, - { url = "https://files.pythonhosted.org/packages/02/9c/3e4ce737a34c0ada15a35f51d0dbd8bf0ac0cef0c4560ddc0a8364e3f712/libcst-1.8.5-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c20384b8a4a7801b4416ef96173f1fbb7fafad7529edfdf151811ef70423118a", size = 2293220, upload-time = "2025-09-26T05:28:37.201Z" }, - { url = "https://files.pythonhosted.org/packages/1a/74/a68fcb3625b0c218c01aaefef9366f505654a1aa64af99cfe7ff7c97bf41/libcst-1.8.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:271b0b363972ff7d2b8116add13977e7c3b2668c7a424095851d548d222dab18", size = 2295146, upload-time = "2025-09-26T05:28:39.122Z" }, - { url = "https://files.pythonhosted.org/packages/37/c3/f4b6edf204f919c6968eb2d111c338098aebbe3fb5d5d95aceacfcf65d9a/libcst-1.8.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ba728c7aee73b330f49f2df0f0b56b74c95302eeb78860f8d5ff0e0fc52c887", size = 2396597, upload-time = "2025-09-26T05:28:41.162Z" }, - { url = "https://files.pythonhosted.org/packages/d0/94/b5cbe122db8f60e7e05bd56743f91d176f3da9b2101f8234e25bb3c5e493/libcst-1.8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0abf0e87570cd3b06a8cafbb5378a9d1cbf12e4583dc35e0fff2255100da55a1", size = 2107479, upload-time = "2025-09-26T05:28:43.094Z" }, - { url = "https://files.pythonhosted.org/packages/05/4d/5e47752c37b33ea6fd1fac76f62e2caa37a6f78d841338bb8fd3dcf51498/libcst-1.8.5-cp313-cp313-win_arm64.whl", hash = "sha256:757390c3cf0b45d7ae1d1d4070c839b082926e762e65eab144f37a63ad33b939", size = 1990992, upload-time = "2025-09-26T05:28:44.993Z" }, - { url = "https://files.pythonhosted.org/packages/88/df/d0eaaed2c402f945fd049b990c98242cb6eace640258e9f8d484206a9666/libcst-1.8.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:f8934763389cd21ce3ed229b63b994b79dac8be7e84a9da144823f46bc1ffc5c", size = 2187746, upload-time = "2025-09-26T05:28:46.946Z" }, - { url = "https://files.pythonhosted.org/packages/19/05/ca62c80dc5f2cf26c2d5d1428612950c6f04df66f765ab0ca8b7d42b4ba1/libcst-1.8.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b873caf04862b6649a2a961fce847f7515ba882be02376a924732cf82c160861", size = 2072530, upload-time = "2025-09-26T05:28:48.451Z" }, - { url = "https://files.pythonhosted.org/packages/1a/38/34a5825bd87badaf8bc0725e5816d395f43ea2f8d1f3cb6982cccc70a1a2/libcst-1.8.5-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:50e095d18c4f76da0e03f25c50b52a2999acbcbe4598a3cf41842ee3c13b54f1", size = 2219819, upload-time = "2025-09-26T05:28:50.328Z" }, - { url = "https://files.pythonhosted.org/packages/74/ea/10407cc1c06231079f5ee6c5e2c2255a2c3f876a7a7f13af734f9bb6ee0e/libcst-1.8.5-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:3a3c967725cc3e8fa5c7251188d57d48eec8835f44c6b53f7523992bec595fa0", size = 2283011, upload-time = "2025-09-26T05:28:51.808Z" }, - { url = "https://files.pythonhosted.org/packages/5b/fc/c4e4c03b4804ac78b8209e83a3c15e449aa68ddd0e602d5c2cc4b7e1b9ed/libcst-1.8.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eed454ab77f4b18100c41d8973b57069e503943ea4e5e5bbb660404976a0fe7a", size = 2283315, upload-time = "2025-09-26T05:28:53.33Z" }, - { url = "https://files.pythonhosted.org/packages/bb/39/75e07c2933b55815b71b1971e5388a24d1d1475631266251249eaed8af28/libcst-1.8.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:39130e59868b8fa49f6eeedd46f008d3456fc13ded57e1c85b211636eb6425f3", size = 2387279, upload-time = "2025-09-26T05:28:54.872Z" }, - { url = "https://files.pythonhosted.org/packages/04/44/0315fb0f2ee8913d209a5caf57932db8efb3f562dbcdc5fb157de92fb098/libcst-1.8.5-cp313-cp313t-win_amd64.whl", hash = "sha256:a7b1cc3abfdba5ce36907f94f07e079528d4be52c07dfffa26f0e68eb1d25d45", size = 2098827, upload-time = "2025-09-26T05:28:56.877Z" }, - { url = "https://files.pythonhosted.org/packages/45/c2/1335fe9feb7d75526df454a8f9db77615460c69691c27af0a57621ca9e47/libcst-1.8.5-cp313-cp313t-win_arm64.whl", hash = "sha256:20354c4217e87afea936e9ea90c57fe0b2c5651f41b3ee59f5df8a53ab417746", size = 1979853, upload-time = "2025-09-26T05:28:58.408Z" }, - { url = "https://files.pythonhosted.org/packages/9e/4e/4d961f15e7cc3f9924c4865158cf23de3cb1d9727be5bc5ec1f6b2e0e991/libcst-1.8.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:f350ff2867b3075ba97a022de694f2747c469c25099216cef47b58caaee96314", size = 2196843, upload-time = "2025-09-26T05:29:00.64Z" }, - { url = "https://files.pythonhosted.org/packages/47/b5/706b51025218b31346335c8aa1e316e91dbd82b9bd60483a23842a59033b/libcst-1.8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b95db09d04d125619a63f191c9534853656c4c76c303b8b4c5f950c8e610fba", size = 2082306, upload-time = "2025-09-26T05:29:02.498Z" }, - { url = "https://files.pythonhosted.org/packages/eb/78/53816b76257d9d149f074ac0b913be1c94d54fb07b3a77f3e11333659d36/libcst-1.8.5-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:60e62e966b45b7dee6f0ec0fd7687704d29be18ae670c5bc6c9c61a12ccf589f", size = 2230603, upload-time = "2025-09-26T05:29:04.123Z" }, - { url = "https://files.pythonhosted.org/packages/a6/06/4497c456ad0ace0f60a38f0935d6e080600532bcddeaf545443d4d7c4db2/libcst-1.8.5-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:7cbb330a352dde570059c73af7b7bbfaa84ae121f54d2ce46c5530351f57419d", size = 2293110, upload-time = "2025-09-26T05:29:05.685Z" }, - { url = "https://files.pythonhosted.org/packages/14/fc/9ef8cc7c0a9cca722b6f176cc82b5925dbcdfcee6e17cd6d3056d45af38e/libcst-1.8.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:71b2b1ef2305cba051252342a1a4f8e94e6b8e95d7693a7c15a00ce8849ef722", size = 2296366, upload-time = "2025-09-26T05:29:07.451Z" }, - { url = "https://files.pythonhosted.org/packages/2d/7e/799dac0cd086cc5dab3837ead9c72dd4e29a79323795dc52b2ebb3aac9a0/libcst-1.8.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0f504d06dfba909d1ba6a4acf60bfe3f22275444d6e0d07e472a5da4a209b0be", size = 2397188, upload-time = "2025-09-26T05:29:09.084Z" }, - { url = "https://files.pythonhosted.org/packages/1b/5c/e4f32439818db04ea43b1d6de1d375dcdd5ff33b828864900c340f26436c/libcst-1.8.5-cp314-cp314-win_amd64.whl", hash = "sha256:c69d2b39e360dea5490ccb5dcf5957dcbb1067d27dc1f3f0787d4e287f7744e2", size = 2183599, upload-time = "2025-09-26T05:29:11.039Z" }, - { url = "https://files.pythonhosted.org/packages/e2/f9/a457c3da610aef4b5f5c00f1feb67192594b77fb9dddab8f654161c1ea6f/libcst-1.8.5-cp314-cp314-win_arm64.whl", hash = "sha256:63405cb548b2d7b78531535a7819231e633b13d3dee3eb672d58f0f3322892ca", size = 2071025, upload-time = "2025-09-26T05:29:12.546Z" }, - { url = "https://files.pythonhosted.org/packages/4a/b6/37abad6fc44df268cd8c2a903ddb2108bd8ac324ef000c2dfcb03d763a41/libcst-1.8.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8a5921105610f35921cc4db6fa5e68e941c6da20ce7f9f93b41b6c66b5481353", size = 2187762, upload-time = "2025-09-26T05:29:14.322Z" }, - { url = "https://files.pythonhosted.org/packages/b4/19/d1118c0b25612a3f50fb2c4b2010562fbf7e7df30ad821bab0aae9cf7e4f/libcst-1.8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:abded10e8d92462fa982d19b064c6f24ed7ead81cf3c3b71011e9764cb12923d", size = 2072565, upload-time = "2025-09-26T05:29:16.37Z" }, - { url = "https://files.pythonhosted.org/packages/f7/c8/f72515e2774234c4f92909222d762789cc4be2247ed4189bc0639ade1f8c/libcst-1.8.5-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dd7bdb14545c4b77a6c0eb39c86a76441fe833da800f6ca63e917e1273621029", size = 2219884, upload-time = "2025-09-26T05:29:18.118Z" }, - { url = "https://files.pythonhosted.org/packages/f4/b8/b267b28cbb0cae19e8c7887cdeda72288ae1020d1c22b6c9955f065b296e/libcst-1.8.5-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6dc28d33ab8750a84c28b5625f7916846ecbecefd89bf75a5292a35644b6efbd", size = 2282790, upload-time = "2025-09-26T05:29:19.578Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8a/46f2b01bb6782dbc0f4e917ed029b1236278a5dc6d263e55ee986a83a88e/libcst-1.8.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:970b7164a71c65e13c961965f9677bbbbeb21ce2e7e6655294f7f774156391c4", size = 2283591, upload-time = "2025-09-26T05:29:21.024Z" }, - { url = "https://files.pythonhosted.org/packages/e4/ca/3097729b5f6ab1d5e3a753492912d1d8b483a320421d3c0e9e26f1ecef0c/libcst-1.8.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd74c543770e6a61dcb8846c9689dfcce2ad686658896f77f3e21b6ce94bcb2e", size = 2386780, upload-time = "2025-09-26T05:29:22.922Z" }, - { url = "https://files.pythonhosted.org/packages/bb/cc/4fc91968779b70429106797ddb2265a18b0026e17ec6ba805c34427d2fb9/libcst-1.8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:3d8e80cd1ed6577166f0bab77357f819f12564c2ed82307612e2bcc93e684d72", size = 2174807, upload-time = "2025-09-26T05:29:24.799Z" }, - { url = "https://files.pythonhosted.org/packages/79/3c/db47e1cf0c98a13cbea2cb5611e7b6913ac5e63845b0e41ee7020b03f523/libcst-1.8.5-cp314-cp314t-win_arm64.whl", hash = "sha256:a026aaa19cb2acd8a4d9e2a215598b0a7e2c194bf4482eb9dec4d781ec6e10b2", size = 2059048, upload-time = "2025-09-26T05:29:28.425Z" }, +version = "1.8.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version != '3.13.*' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "pyyaml-ft", marker = "python_full_version == '3.13.*' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/cd/337df968b38d94c5aabd3e1b10630f047a2b345f6e1d4456bd9fe7417537/libcst-1.8.6.tar.gz", hash = "sha256:f729c37c9317126da9475bdd06a7208eb52fcbd180a6341648b45a56b4ba708b", size = 891354, upload-time = "2025-11-03T22:33:30.621Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/52/97d5454dee9d014821fe0c88f3dc0e83131b97dd074a4d49537056a75475/libcst-1.8.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a20c5182af04332cc94d8520792befda06d73daf2865e6dddc5161c72ea92cb9", size = 2211698, upload-time = "2025-11-03T22:31:50.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a4/d1205985d378164687af3247a9c8f8bdb96278b0686ac98ab951bc6d336a/libcst-1.8.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:36473e47cb199b7e6531d653ee6ffed057de1d179301e6c67f651f3af0b499d6", size = 2093104, upload-time = "2025-11-03T22:31:52.189Z" }, + { url = "https://files.pythonhosted.org/packages/9e/de/1338da681b7625b51e584922576d54f1b8db8fc7ff4dc79121afc5d4d2cd/libcst-1.8.6-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:06fc56335a45d61b7c1b856bfab4587b84cfe31e9d6368f60bb3c9129d900f58", size = 2237419, upload-time = "2025-11-03T22:31:53.526Z" }, + { url = "https://files.pythonhosted.org/packages/50/06/ee66f2d83b870534756e593d464d8b33b0914c224dff3a407e0f74dc04e0/libcst-1.8.6-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:6b23d14a7fc0addd9795795763af26b185deb7c456b1e7cc4d5228e69dab5ce8", size = 2300820, upload-time = "2025-11-03T22:31:55.995Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ca/959088729de8e0eac8dd516e4fb8623d8d92bad539060fa85c9e94d418a5/libcst-1.8.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:16cfe0cfca5fd840e1fb2c30afb628b023d3085b30c3484a79b61eae9d6fe7ba", size = 2301201, upload-time = "2025-11-03T22:31:57.347Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4c/2a21a8c452436097dfe1da277f738c3517f3f728713f16d84b9a3d67ca8d/libcst-1.8.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:455f49a93aea4070132c30ebb6c07c2dea0ba6c1fde5ffde59fc45dbb9cfbe4b", size = 2408213, upload-time = "2025-11-03T22:31:59.221Z" }, + { url = "https://files.pythonhosted.org/packages/3e/26/8f7b671fad38a515bb20b038718fd2221ab658299119ac9bcec56c2ced27/libcst-1.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:72cca15800ffc00ba25788e4626189fe0bc5fe2a0c1cb4294bce2e4df21cc073", size = 2119189, upload-time = "2025-11-03T22:32:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/5b/bf/ffb23a48e27001165cc5c81c5d9b3d6583b21b7f5449109e03a0020b060c/libcst-1.8.6-cp310-cp310-win_arm64.whl", hash = "sha256:6cad63e3a26556b020b634d25a8703b605c0e0b491426b3e6b9e12ed20f09100", size = 2001736, upload-time = "2025-11-03T22:32:02.986Z" }, + { url = "https://files.pythonhosted.org/packages/dc/15/95c2ecadc0fb4af8a7057ac2012a4c0ad5921b9ef1ace6c20006b56d3b5f/libcst-1.8.6-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3649a813660fbffd7bc24d3f810b1f75ac98bd40d9d6f56d1f0ee38579021073", size = 2211289, upload-time = "2025-11-03T22:32:04.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/c3/7e1107acd5ed15cf60cc07c7bb64498a33042dc4821874aea3ec4942f3cd/libcst-1.8.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0cbe17067055829607c5ba4afa46bfa4d0dd554c0b5a583546e690b7367a29b6", size = 2092927, upload-time = "2025-11-03T22:32:06.209Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ff/0d2be87f67e2841a4a37d35505e74b65991d30693295c46fc0380ace0454/libcst-1.8.6-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:59a7e388c57d21d63722018978a8ddba7b176e3a99bd34b9b84a576ed53f2978", size = 2237002, upload-time = "2025-11-03T22:32:07.559Z" }, + { url = "https://files.pythonhosted.org/packages/69/99/8c4a1b35c7894ccd7d33eae01ac8967122f43da41325223181ca7e4738fe/libcst-1.8.6-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:b6c1248cc62952a3a005792b10cdef2a4e130847be9c74f33a7d617486f7e532", size = 2301048, upload-time = "2025-11-03T22:32:08.869Z" }, + { url = "https://files.pythonhosted.org/packages/9b/8b/d1aa811eacf936cccfb386ae0585aa530ea1221ccf528d67144e041f5915/libcst-1.8.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6421a930b028c5ef4a943b32a5a78b7f1bf15138214525a2088f11acbb7d3d64", size = 2300675, upload-time = "2025-11-03T22:32:10.579Z" }, + { url = "https://files.pythonhosted.org/packages/c6/6b/7b65cd41f25a10c1fef2389ddc5c2b2cc23dc4d648083fa3e1aa7e0eeac2/libcst-1.8.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6d8b67874f2188399a71a71731e1ba2d1a2c3173b7565d1cc7ffb32e8fbaba5b", size = 2407934, upload-time = "2025-11-03T22:32:11.856Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/401cfff374bb3b785adfad78f05225225767ee190997176b2a9da9ed9460/libcst-1.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:b0d8c364c44ae343937f474b2e492c1040df96d94530377c2f9263fb77096e4f", size = 2119247, upload-time = "2025-11-03T22:32:13.279Z" }, + { url = "https://files.pythonhosted.org/packages/f1/17/085f59eaa044b6ff6bc42148a5449df2b7f0ba567307de7782fe85c39ee2/libcst-1.8.6-cp311-cp311-win_arm64.whl", hash = "sha256:5dcaaebc835dfe5755bc85f9b186fb7e2895dda78e805e577fef1011d51d5a5c", size = 2001774, upload-time = "2025-11-03T22:32:14.647Z" }, + { url = "https://files.pythonhosted.org/packages/0c/3c/93365c17da3d42b055a8edb0e1e99f1c60c776471db6c9b7f1ddf6a44b28/libcst-1.8.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0c13d5bd3d8414a129e9dccaf0e5785108a4441e9b266e1e5e9d1f82d1b943c9", size = 2206166, upload-time = "2025-11-03T22:32:16.012Z" }, + { url = "https://files.pythonhosted.org/packages/1d/cb/7530940e6ac50c6dd6022349721074e19309eb6aa296e942ede2213c1a19/libcst-1.8.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1472eeafd67cdb22544e59cf3bfc25d23dc94058a68cf41f6654ff4fcb92e09", size = 2083726, upload-time = "2025-11-03T22:32:17.312Z" }, + { url = "https://files.pythonhosted.org/packages/1b/cf/7e5eaa8c8f2c54913160671575351d129170db757bb5e4b7faffed022271/libcst-1.8.6-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:089c58e75cb142ec33738a1a4ea7760a28b40c078ab2fd26b270dac7d2633a4d", size = 2235755, upload-time = "2025-11-03T22:32:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/55/54/570ec2b0e9a3de0af9922e3bb1b69a5429beefbc753a7ea770a27ad308bd/libcst-1.8.6-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c9d7aeafb1b07d25a964b148c0dda9451efb47bbbf67756e16eeae65004b0eb5", size = 2301473, upload-time = "2025-11-03T22:32:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/11/4c/163457d1717cd12181c421a4cca493454bcabd143fc7e53313bc6a4ad82a/libcst-1.8.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207481197afd328aa91d02670c15b48d0256e676ce1ad4bafb6dc2b593cc58f1", size = 2298899, upload-time = "2025-11-03T22:32:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/35/1d/317ddef3669883619ef3d3395ea583305f353ef4ad87d7a5ac1c39be38e3/libcst-1.8.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:375965f34cc6f09f5f809244d3ff9bd4f6cb6699f571121cebce53622e7e0b86", size = 2408239, upload-time = "2025-11-03T22:32:23.275Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a1/f47d8cccf74e212dd6044b9d6dbc223636508da99acff1d54786653196bc/libcst-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:da95b38693b989eaa8d32e452e8261cfa77fe5babfef1d8d2ac25af8c4aa7e6d", size = 2119660, upload-time = "2025-11-03T22:32:24.822Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/dd313bf6a7942cdf951828f07ecc1a7695263f385065edc75ef3016a3cb5/libcst-1.8.6-cp312-cp312-win_arm64.whl", hash = "sha256:bff00e1c766658adbd09a175267f8b2f7616e5ee70ce45db3d7c4ce6d9f6bec7", size = 1999824, upload-time = "2025-11-03T22:32:26.131Z" }, + { url = "https://files.pythonhosted.org/packages/90/01/723cd467ec267e712480c772aacc5aa73f82370c9665162fd12c41b0065b/libcst-1.8.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7445479ebe7d1aff0ee094ab5a1c7718e1ad78d33e3241e1a1ec65dcdbc22ffb", size = 2206386, upload-time = "2025-11-03T22:32:27.422Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/b944944f910f24c094f9b083f76f61e3985af5a376f5342a21e01e2d1a81/libcst-1.8.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fc3fef8a2c983e7abf5d633e1884c5dd6fa0dcb8f6e32035abd3d3803a3a196", size = 2083945, upload-time = "2025-11-03T22:32:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/36/a1/bd1b2b2b7f153d82301cdaddba787f4a9fc781816df6bdb295ca5f88b7cf/libcst-1.8.6-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1a3a5e4ee870907aa85a4076c914ae69066715a2741b821d9bf16f9579de1105", size = 2235818, upload-time = "2025-11-03T22:32:30.504Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ab/f5433988acc3b4d188c4bb154e57837df9488cc9ab551267cdeabd3bb5e7/libcst-1.8.6-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6609291c41f7ad0bac570bfca5af8fea1f4a27987d30a1fa8b67fe5e67e6c78d", size = 2301289, upload-time = "2025-11-03T22:32:31.812Z" }, + { url = "https://files.pythonhosted.org/packages/5d/57/89f4ba7a6f1ac274eec9903a9e9174890d2198266eee8c00bc27eb45ecf7/libcst-1.8.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:25eaeae6567091443b5374b4c7d33a33636a2d58f5eda02135e96fc6c8807786", size = 2299230, upload-time = "2025-11-03T22:32:33.242Z" }, + { url = "https://files.pythonhosted.org/packages/f2/36/0aa693bc24cce163a942df49d36bf47a7ed614a0cd5598eee2623bc31913/libcst-1.8.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04030ea4d39d69a65873b1d4d877def1c3951a7ada1824242539e399b8763d30", size = 2408519, upload-time = "2025-11-03T22:32:34.678Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/6dd055b5f15afa640fb3304b2ee9df8b7f72e79513814dbd0a78638f4a0e/libcst-1.8.6-cp313-cp313-win_amd64.whl", hash = "sha256:8066f1b70f21a2961e96bedf48649f27dfd5ea68be5cd1bed3742b047f14acde", size = 2119853, upload-time = "2025-11-03T22:32:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ed/5ddb2a22f0b0abdd6dcffa40621ada1feaf252a15e5b2733a0a85dfd0429/libcst-1.8.6-cp313-cp313-win_arm64.whl", hash = "sha256:c188d06b583900e662cd791a3f962a8c96d3dfc9b36ea315be39e0a4c4792ebf", size = 1999808, upload-time = "2025-11-03T22:32:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/25/d3/72b2de2c40b97e1ef4a1a1db4e5e52163fc7e7740ffef3846d30bc0096b5/libcst-1.8.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c41c76e034a1094afed7057023b1d8967f968782433f7299cd170eaa01ec033e", size = 2190553, upload-time = "2025-11-03T22:32:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/0d/20/983b7b210ccc3ad94a82db54230e92599c4a11b9cfc7ce3bc97c1d2df75c/libcst-1.8.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5432e785322aba3170352f6e72b32bea58d28abd141ac37cc9b0bf6b7c778f58", size = 2074717, upload-time = "2025-11-03T22:32:41.373Z" }, + { url = "https://files.pythonhosted.org/packages/13/f2/9e01678fedc772e09672ed99930de7355757035780d65d59266fcee212b8/libcst-1.8.6-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:85b7025795b796dea5284d290ff69de5089fc8e989b25d6f6f15b6800be7167f", size = 2225834, upload-time = "2025-11-03T22:32:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/4a/0d/7bed847b5c8c365e9f1953da274edc87577042bee5a5af21fba63276e756/libcst-1.8.6-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:536567441182a62fb706e7aa954aca034827b19746832205953b2c725d254a93", size = 2287107, upload-time = "2025-11-03T22:32:44.549Z" }, + { url = "https://files.pythonhosted.org/packages/02/f0/7e51fa84ade26c518bfbe7e2e4758b56d86a114c72d60309ac0d350426c4/libcst-1.8.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f04d3672bde1704f383a19e8f8331521abdbc1ed13abb349325a02ac56e5012", size = 2288672, upload-time = "2025-11-03T22:32:45.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cd/15762659a3f5799d36aab1bc2b7e732672722e249d7800e3c5f943b41250/libcst-1.8.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f04febcd70e1e67917be7de513c8d4749d2e09206798558d7fe632134426ea4", size = 2392661, upload-time = "2025-11-03T22:32:47.232Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6b/b7f9246c323910fcbe021241500f82e357521495dcfe419004dbb272c7cb/libcst-1.8.6-cp313-cp313t-win_amd64.whl", hash = "sha256:1dc3b897c8b0f7323412da3f4ad12b16b909150efc42238e19cbf19b561cc330", size = 2105068, upload-time = "2025-11-03T22:32:49.145Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0b/4fd40607bc4807ec2b93b054594373d7fa3d31bb983789901afcb9bcebe9/libcst-1.8.6-cp313-cp313t-win_arm64.whl", hash = "sha256:44f38139fa95e488db0f8976f9c7ca39a64d6bc09f2eceef260aa1f6da6a2e42", size = 1985181, upload-time = "2025-11-03T22:32:50.597Z" }, + { url = "https://files.pythonhosted.org/packages/3a/60/4105441989e321f7ad0fd28ffccb83eb6aac0b7cfb0366dab855dcccfbe5/libcst-1.8.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b188e626ce61de5ad1f95161b8557beb39253de4ec74fc9b1f25593324a0279c", size = 2204202, upload-time = "2025-11-03T22:32:52.311Z" }, + { url = "https://files.pythonhosted.org/packages/67/2f/51a6f285c3a183e50cfe5269d4a533c21625aac2c8de5cdf2d41f079320d/libcst-1.8.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:87e74f7d7dfcba9efa91127081e22331d7c42515f0a0ac6e81d4cf2c3ed14661", size = 2083581, upload-time = "2025-11-03T22:32:54.269Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/921b1c19b638860af76cdb28bc81d430056592910b9478eea49e31a7f47a/libcst-1.8.6-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:3a926a4b42015ee24ddfc8ae940c97bd99483d286b315b3ce82f3bafd9f53474", size = 2236495, upload-time = "2025-11-03T22:32:55.723Z" }, + { url = "https://files.pythonhosted.org/packages/12/a8/b00592f9bede618cbb3df6ffe802fc65f1d1c03d48a10d353b108057d09c/libcst-1.8.6-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:3f4fbb7f569e69fd9e89d9d9caa57ca42c577c28ed05062f96a8c207594e75b8", size = 2301466, upload-time = "2025-11-03T22:32:57.337Z" }, + { url = "https://files.pythonhosted.org/packages/af/df/790d9002f31580fefd0aec2f373a0f5da99070e04c5e8b1c995d0104f303/libcst-1.8.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:08bd63a8ce674be431260649e70fca1d43f1554f1591eac657f403ff8ef82c7a", size = 2300264, upload-time = "2025-11-03T22:32:58.852Z" }, + { url = "https://files.pythonhosted.org/packages/21/de/dc3f10e65bab461be5de57850d2910a02c24c3ddb0da28f0e6e4133c3487/libcst-1.8.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e00e275d4ba95d4963431ea3e409aa407566a74ee2bf309a402f84fc744abe47", size = 2408572, upload-time = "2025-11-03T22:33:00.552Z" }, + { url = "https://files.pythonhosted.org/packages/20/3b/35645157a7590891038b077db170d6dd04335cd2e82a63bdaa78c3297dfe/libcst-1.8.6-cp314-cp314-win_amd64.whl", hash = "sha256:fea5c7fa26556eedf277d4f72779c5ede45ac3018650721edd77fd37ccd4a2d4", size = 2193917, upload-time = "2025-11-03T22:33:02.354Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a2/1034a9ba7d3e82f2c2afaad84ba5180f601aed676d92b76325797ad60951/libcst-1.8.6-cp314-cp314-win_arm64.whl", hash = "sha256:bb9b4077bdf8857b2483879cbbf70f1073bc255b057ec5aac8a70d901bb838e9", size = 2078748, upload-time = "2025-11-03T22:33:03.707Z" }, + { url = "https://files.pythonhosted.org/packages/95/a1/30bc61e8719f721a5562f77695e6154e9092d1bdf467aa35d0806dcd6cea/libcst-1.8.6-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:55ec021a296960c92e5a33b8d93e8ad4182b0eab657021f45262510a58223de1", size = 2188980, upload-time = "2025-11-03T22:33:05.152Z" }, + { url = "https://files.pythonhosted.org/packages/2c/14/c660204532407c5628e3b615015a902ed2d0b884b77714a6bdbe73350910/libcst-1.8.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ba9ab2b012fbd53b36cafd8f4440a6b60e7e487cd8b87428e57336b7f38409a4", size = 2074828, upload-time = "2025-11-03T22:33:06.864Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/c497c354943dff644749f177ee9737b09ed811b8fc842b05709a40fe0d1b/libcst-1.8.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c0a0cc80aebd8aa15609dd4d330611cbc05e9b4216bcaeabba7189f99ef07c28", size = 2225568, upload-time = "2025-11-03T22:33:08.354Z" }, + { url = "https://files.pythonhosted.org/packages/86/ef/45999676d07bd6d0eefa28109b4f97124db114e92f9e108de42ba46a8028/libcst-1.8.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:42a4f68121e2e9c29f49c97f6154e8527cd31021809cc4a941c7270aa64f41aa", size = 2286523, upload-time = "2025-11-03T22:33:10.206Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6c/517d8bf57d9f811862f4125358caaf8cd3320a01291b3af08f7b50719db4/libcst-1.8.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a434c521fadaf9680788b50d5c21f4048fa85ed19d7d70bd40549fbaeeecab1", size = 2288044, upload-time = "2025-11-03T22:33:11.628Z" }, + { url = "https://files.pythonhosted.org/packages/83/ce/24d7d49478ffb61207f229239879845da40a374965874f5ee60f96b02ddb/libcst-1.8.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6a65f844d813ab4ef351443badffa0ae358f98821561d19e18b3190f59e71996", size = 2392605, upload-time = "2025-11-03T22:33:12.962Z" }, + { url = "https://files.pythonhosted.org/packages/39/c3/829092ead738b71e96a4e96896c96f276976e5a8a58b4473ed813d7c962b/libcst-1.8.6-cp314-cp314t-win_amd64.whl", hash = "sha256:bdb14bc4d4d83a57062fed2c5da93ecb426ff65b0dc02ddf3481040f5f074a82", size = 2181581, upload-time = "2025-11-03T22:33:14.514Z" }, + { url = "https://files.pythonhosted.org/packages/98/6d/5d6a790a02eb0d9d36c4aed4f41b277497e6178900b2fa29c35353aa45ed/libcst-1.8.6-cp314-cp314t-win_arm64.whl", hash = "sha256:819c8081e2948635cab60c603e1bbdceccdfe19104a242530ad38a36222cb88f", size = 2065000, upload-time = "2025-11-03T22:33:16.257Z" }, ] [[package]] @@ -1998,11 +2276,11 @@ sdist = { url = "https://files.pythonhosted.org/packages/b6/0c/9373a469ff7a33bdd [[package]] name = "markdown" -version = "3.9" +version = "3.10" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" }, + { url = "https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" }, ] [[package]] @@ -2146,6 +2424,8 @@ dev = [ { name = "av" }, { name = "causal-conv1d" }, { name = "einops" }, + { name = "emerging-optimizers" }, + { name = "flash-linear-attention" }, { name = "flashinfer-python" }, { name = "mamba-ssm" }, { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-dev'" }, @@ -2155,7 +2435,7 @@ dev = [ { name = "nvidia-resiliency-ext" }, { name = "nvtx" }, { name = "onnxscript", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "onnxscript", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnxscript", version = "0.5.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "opentelemetry-api" }, { name = "setuptools" }, { name = "tensorstore", version = "0.1.74", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, @@ -2165,23 +2445,16 @@ dev = [ { name = "wget" }, ] lts = [ - { name = "av" }, - { name = "causal-conv1d" }, { name = "einops" }, - { name = "flashinfer-python" }, - { name = "mamba-ssm" }, - { name = "megatron-energon", extra = ["av-decode"], marker = "extra == 'extra-13-megatron-core-lts'" }, - { name = "multi-storage-client" }, - { name = "nv-grouped-gemm" }, { name = "nvtx" }, - { name = "onnxscript", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "onnxscript", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "opentelemetry-api" }, { name = "setuptools" }, { name = "tensorstore", version = "0.1.74", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tensorstore", version = "0.1.78", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "tqdm" }, + { name = "transformers" }, { name = "wget" }, + { name = "zarr", version = "2.18.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "zarr", version = "3.1.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] mlm = [ { name = "flask-restful" }, @@ -2239,53 +2512,48 @@ test = [ { name = "pytest-random-order" }, { name = "pyyaml" }, { name = "tensorboard" }, - { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, - { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "wrapt", version = "1.17.3", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "wrapt", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, ] [package.metadata] requires-dist = [ { name = "av", marker = "extra == 'dev'", specifier = "<16.0.0" }, - { name = "av", marker = "extra == 'lts'", specifier = "<16.0.0" }, { name = "causal-conv1d", marker = "extra == 'dev'", specifier = "~=1.5" }, - { name = "causal-conv1d", marker = "extra == 'lts'", specifier = "~=1.5" }, { name = "einops", marker = "extra == 'dev'", specifier = "~=0.8" }, - { name = "einops", marker = "extra == 'lts'", specifier = "~=0.8" }, + { name = "einops", marker = "extra == 'lts'" }, + { name = "emerging-optimizers", marker = "extra == 'dev'", git = "https://github.com/NVIDIA-NeMo/Emerging-Optimizers.git?rev=cf9909b777ffac18e05b67a6708282cadc000942" }, + { name = "flash-linear-attention", marker = "extra == 'dev'", specifier = "~=0.3.2" }, { name = "flashinfer-python", marker = "extra == 'dev'" }, - { name = "flashinfer-python", marker = "extra == 'lts'" }, { name = "flask-restful", marker = "extra == 'mlm'" }, { name = "mamba-ssm", marker = "extra == 'dev'", specifier = "~=2.2" }, - { name = "mamba-ssm", marker = "extra == 'lts'", specifier = "~=2.2" }, { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'dev'", specifier = "~=6.0" }, - { name = "megatron-energon", extras = ["av-decode"], marker = "extra == 'lts'", specifier = "~=6.0" }, { name = "multi-storage-client", marker = "extra == 'dev'", specifier = "~=0.27" }, - { name = "multi-storage-client", marker = "extra == 'lts'", specifier = "~=0.27" }, { name = "numpy", specifier = "<2.0.0" }, { name = "nv-grouped-gemm", marker = "extra == 'dev'", specifier = "~=1.1" }, - { name = "nv-grouped-gemm", marker = "extra == 'lts'", specifier = "~=1.1" }, - { name = "nvidia-modelopt", extras = ["torch"], marker = "sys_platform != 'darwin' and extra == 'dev'" }, - { name = "nvidia-resiliency-ext", marker = "extra == 'dev'" }, + { name = "nvidia-modelopt", extras = ["torch"], marker = "sys_platform != 'darwin' and extra == 'dev'", specifier = ">=0.33.0a0,<0.34.0" }, + { name = "nvidia-resiliency-ext", marker = "extra == 'dev'", specifier = ">=0.4.0a0,<0.5.0" }, { name = "nvtx", marker = "extra == 'dev'", specifier = "~=0.2" }, - { name = "nvtx", marker = "extra == 'lts'", specifier = "~=0.2" }, + { name = "nvtx", marker = "extra == 'lts'" }, { name = "onnxscript", marker = "extra == 'dev'" }, - { name = "onnxscript", marker = "extra == 'lts'" }, { name = "opentelemetry-api", marker = "extra == 'dev'", specifier = "~=1.33.1" }, - { name = "opentelemetry-api", marker = "extra == 'lts'", specifier = "~=1.33.1" }, { name = "packaging", specifier = ">=24.2" }, { name = "sentencepiece", marker = "extra == 'mlm'" }, { name = "setuptools", marker = "extra == 'dev'", specifier = "<80.0.0" }, { name = "setuptools", marker = "extra == 'lts'", specifier = "<80.0.0" }, { name = "tensorstore", marker = "extra == 'dev'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, - { name = "tensorstore", marker = "extra == 'lts'", specifier = "~=0.1,!=0.1.46,!=0.1.72" }, + { name = "tensorstore", marker = "extra == 'lts'", specifier = "!=0.1.46,!=0.1.72" }, { name = "tiktoken", marker = "extra == 'mlm'" }, { name = "torch" }, { name = "tqdm", marker = "extra == 'dev'" }, { name = "tqdm", marker = "extra == 'lts'" }, { name = "transformer-engine", extras = ["pytorch"], marker = "extra == 'dev'", git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.9" }, + { name = "transformers", marker = "extra == 'lts'" }, { name = "transformers", marker = "extra == 'mlm'" }, { name = "wandb", marker = "extra == 'mlm'" }, { name = "wget", marker = "extra == 'dev'" }, { name = "wget", marker = "extra == 'lts'" }, + { name = "zarr", marker = "extra == 'lts'" }, ] provides-extras = ["mlm", "dev", "lts"] @@ -2374,10 +2642,12 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", ] dependencies = [ { name = "numpy", marker = "python_full_version >= '3.13'" }, @@ -2692,6 +2962,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" }, ] +[[package]] +name = "multiprocess" +version = "0.70.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/f8/7f9a8f08bf98cea1dfaa181e05cc8bbcb59cecf044b5a9ac3cce39f9c449/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:25d4012dcaaf66b9e8e955f58482b42910c2ee526d532844d8bcf661bbc604df", size = 135083, upload-time = "2025-04-17T03:11:04.223Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/b7b10dbfc17b2b3ce07d4d30b3ba8367d0ed32d6d46cd166e298f161dd46/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:06b19433de0d02afe5869aec8931dd5c01d99074664f806c73896b0d9e527213", size = 135128, upload-time = "2025-04-17T03:11:06.045Z" }, + { url = "https://files.pythonhosted.org/packages/c1/a3/5f8d3b9690ea5580bee5868ab7d7e2cfca74b7e826b28192b40aa3881cdc/multiprocess-0.70.18-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6fa1366f994373aaf2d4738b0f56e707caeaa05486e97a7f71ee0853823180c2", size = 135132, upload-time = "2025-04-17T03:11:07.533Z" }, + { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" }, + { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" }, + { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917, upload-time = "2025-04-17T03:11:24.044Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" }, + { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -2761,33 +3054,39 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", @@ -2838,6 +3137,83 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/90/81ac364ef94209c100e12579629dc92bf7a709a84af32f8c551b02c07e94/nltk-3.9.2-py3-none-any.whl", hash = "sha256:1e209d2b3009110635ed9709a67a1a3e33a10f799490fa71cf4bec218c11c88a", size = 1513404, upload-time = "2025-10-01T07:19:21.648Z" }, ] +[[package]] +name = "numcodecs" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/56/8895a76abe4ec94ebd01eeb6d74f587bc4cddd46569670e1402852a5da13/numcodecs-0.13.1.tar.gz", hash = "sha256:a3cf37881df0898f3a9c0d4477df88133fe85185bffe57ba31bcc2fa207709bc", size = 5955215, upload-time = "2024-10-09T16:28:00.188Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/c0/6d72cde772bcec196b7188731d41282993b2958440f77fdf0db216f722da/numcodecs-0.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:96add4f783c5ce57cc7e650b6cac79dd101daf887c479a00a29bc1487ced180b", size = 1580012, upload-time = "2024-10-09T16:27:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/94/1d/f81fc1fa9210bbea97258242393a1f9feab4f6d8fb201f81f76003005e4b/numcodecs-0.13.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:237b7171609e868a20fd313748494444458ccd696062f67e198f7f8f52000c15", size = 1176919, upload-time = "2024-10-09T16:27:21.634Z" }, + { url = "https://files.pythonhosted.org/packages/16/e4/b9ec2f4dfc34ecf724bc1beb96a9f6fa9b91801645688ffadacd485089da/numcodecs-0.13.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96e42f73c31b8c24259c5fac6adba0c3ebf95536e37749dc6c62ade2989dca28", size = 8625842, upload-time = "2024-10-09T16:27:24.168Z" }, + { url = "https://files.pythonhosted.org/packages/fe/90/299952e1477954ec4f92813fa03e743945e3ff711bb4f6c9aace431cb3da/numcodecs-0.13.1-cp310-cp310-win_amd64.whl", hash = "sha256:eda7d7823c9282e65234731fd6bd3986b1f9e035755f7fed248d7d366bb291ab", size = 828638, upload-time = "2024-10-09T16:27:27.063Z" }, + { url = "https://files.pythonhosted.org/packages/f0/78/34b8e869ef143e88d62e8231f4dbfcad85e5c41302a11fc5bd2228a13df5/numcodecs-0.13.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2eda97dd2f90add98df6d295f2c6ae846043396e3d51a739ca5db6c03b5eb666", size = 1580199, upload-time = "2024-10-09T16:27:29.336Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cf/f70797d86bb585d258d1e6993dced30396f2044725b96ce8bcf87a02be9c/numcodecs-0.13.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2a86f5367af9168e30f99727ff03b27d849c31ad4522060dde0bce2923b3a8bc", size = 1177203, upload-time = "2024-10-09T16:27:31.011Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b5/d14ad69b63fde041153dfd05d7181a49c0d4864de31a7a1093c8370da957/numcodecs-0.13.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233bc7f26abce24d57e44ea8ebeb5cd17084690b4e7409dd470fdb75528d615f", size = 8868743, upload-time = "2024-10-09T16:27:32.833Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/27a7b5af0b33f6d61e198faf177fbbf3cb83ff10d9d1a6857b7efc525ad5/numcodecs-0.13.1-cp311-cp311-win_amd64.whl", hash = "sha256:796b3e6740107e4fa624cc636248a1580138b3f1c579160f260f76ff13a4261b", size = 829603, upload-time = "2024-10-09T16:27:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/37/3a/bc09808425e7d3df41e5fc73fc7a802c429ba8c6b05e55f133654ade019d/numcodecs-0.13.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5195bea384a6428f8afcece793860b1ab0ae28143c853f0b2b20d55a8947c917", size = 1575806, upload-time = "2024-10-09T16:27:37.804Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cc/dc74d0bfdf9ec192332a089d199f1e543e747c556b5659118db7a437dcca/numcodecs-0.13.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3501a848adaddce98a71a262fee15cd3618312692aa419da77acd18af4a6a3f6", size = 1178233, upload-time = "2024-10-09T16:27:40.169Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/434e8e3970b8e92ae9ab6d9db16cb9bc7aa1cd02e17c11de6848224100a1/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53", size = 8857827, upload-time = "2024-10-09T16:27:42.743Z" }, + { url = "https://files.pythonhosted.org/packages/83/e7/1d8b1b266a92f9013c755b1c146c5ad71a2bff147ecbc67f86546a2e4d6a/numcodecs-0.13.1-cp312-cp312-win_amd64.whl", hash = "sha256:e5db4824ebd5389ea30e54bc8aeccb82d514d28b6b68da6c536b8fa4596f4bca", size = 826539, upload-time = "2024-10-09T16:27:44.808Z" }, + { url = "https://files.pythonhosted.org/packages/83/8b/06771dead2cc4a8ae1ea9907737cf1c8d37a323392fa28f938a586373468/numcodecs-0.13.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a60d75179fd6692e301ddfb3b266d51eb598606dcae7b9fc57f986e8d65cb43", size = 1571660, upload-time = "2024-10-09T16:27:47.125Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ea/d925bf85f92dfe4635356018da9fe4bfecb07b1c72f62b01c1bc47f936b1/numcodecs-0.13.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f593c7506b0ab248961a3b13cb148cc6e8355662ff124ac591822310bc55ecf", size = 1169925, upload-time = "2024-10-09T16:27:49.512Z" }, + { url = "https://files.pythonhosted.org/packages/0f/d6/643a3839d571d8e439a2c77dc4b0b8cab18d96ac808e4a81dbe88e959ab6/numcodecs-0.13.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d3071465f03522e776a31045ddf2cfee7f52df468b977ed3afdd7fe5869701", size = 8814257, upload-time = "2024-10-09T16:27:52.059Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c5/f3e56bc9b4e438a287fff738993d6d11abef368c0328a612ac2842ba9fca/numcodecs-0.13.1-cp313-cp313-win_amd64.whl", hash = "sha256:90d3065ae74c9342048ae0046006f99dcb1388b7288da5a19b3bddf9c30c3176", size = 821887, upload-time = "2024-10-09T16:27:55.039Z" }, +] + +[[package]] +name = "numcodecs" +version = "0.16.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", +] +dependencies = [ + { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/48/6188e359b90a9d8a1850f2bc888c023e66f4a8b2b496820babbea414f008/numcodecs-0.16.3.tar.gz", hash = "sha256:53d705865faaf0a7927c973af3777532001c8fbb653de119c1e844608614d799", size = 6275704, upload-time = "2025-09-18T18:54:57.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/cc/917a85972537498f2bbd7914047efc98babc8667587ceb9dcb228378978a/numcodecs-0.16.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:95c9f2a49bef10cf91ad614a761cba9bfe96656b60c12540e1080de5d909b4ca", size = 1642356, upload-time = "2025-09-18T18:54:36.402Z" }, + { url = "https://files.pythonhosted.org/packages/3b/6a/64c25a089e8537441fe67c09ecb7f3f7fb5d98cd04faf01f605d43aca41c/numcodecs-0.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2afe73d5ebaf9ca0cd5c83aad945da80d29a33d860a80d43a7248491d8813ff", size = 1169186, upload-time = "2025-09-18T18:54:37.838Z" }, + { url = "https://files.pythonhosted.org/packages/d8/a0/0de627baeb43e2045a3d4b3de99bf8b69af329a33df1ed4cda468d70c1fb/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:913f08194d82dcb37594e6705e6d4ae6ccd4b6571500b832fb3e4a155de1dfe8", size = 8341668, upload-time = "2025-09-18T18:54:39.444Z" }, + { url = "https://files.pythonhosted.org/packages/b6/0f/49d1f74a216149240c4b9403218111f11670bd11af0919fda357bb056bf2/numcodecs-0.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85a7f1cae9eb18b85709af46570bf9c60056e7155c4c8f610e8080c68124d0e5", size = 8866611, upload-time = "2025-09-18T18:54:41.168Z" }, + { url = "https://files.pythonhosted.org/packages/aa/51/03aece765108fe247717105b5131856546e5428f22a56a14ffdebd017424/numcodecs-0.16.3-cp311-cp311-win_amd64.whl", hash = "sha256:f7bb7f2c46eb7ec8a1c5f8d8fe1a72c222256dd6d6df5af9eaac7a6b905f3575", size = 806787, upload-time = "2025-09-18T18:54:42.78Z" }, + { url = "https://files.pythonhosted.org/packages/0d/78/e4b34803a3aa1d0769919695de4b133266c18c80c474d32ebc462fa1a9bd/numcodecs-0.16.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c77454d92941a335d148b0b822f5d4783103f392774d5d76283bbf7f21b49529", size = 1681108, upload-time = "2025-09-18T18:54:43.856Z" }, + { url = "https://files.pythonhosted.org/packages/25/cf/ca36f463b03a4097767d2a1c1b72f31810e8c6384e9449dd9b925203783c/numcodecs-0.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:270e7a33ee96bdf5c957acf25a2487002a233811a125a155c400c2f036b69c73", size = 1165589, upload-time = "2025-09-18T18:54:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ae/670260c3c4b5ed34a0674561355f3d4ce7fcbdf09a667e5bc841526d271c/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12f43fa4a347d1dba775c4506a1c9b15b90144c258433b81f79f1c1b1a990db5", size = 8316365, upload-time = "2025-09-18T18:54:46.073Z" }, + { url = "https://files.pythonhosted.org/packages/bb/fa/94e022419c751a60ff0f53642ebae5ef81ed3cc3640f958588e3ad3dc18d/numcodecs-0.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44869ef564a50aa545215c6a0d42ba5bbc34e9715523fb2336ada3d1fb2b331d", size = 8846228, upload-time = "2025-09-18T18:54:47.858Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/f23733589f3e059bf8589508acd23ffeec230bdf179f138a54f5ab16e0a6/numcodecs-0.16.3-cp312-cp312-win_amd64.whl", hash = "sha256:9aae6996172ba10c5f5111b2998709071b5aeba6b58b1ee0b26b61ed6aa7f2f4", size = 806260, upload-time = "2025-09-18T18:54:49.41Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d5/d3536d06ac1e5fb848a3186958204082b68b106364c9a3669652dd786731/numcodecs-0.16.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:947406b01c20f2ce7ce2e631e7f21b782e8a9d4b57b374a41c9e7b1341a8f3a2", size = 1677129, upload-time = "2025-09-18T18:54:50.5Z" }, + { url = "https://files.pythonhosted.org/packages/e1/fd/b0513a3428dc2b38ec85eea771703ae69c49f09b9650d6c44c9105c80073/numcodecs-0.16.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7cf50e351398a34b45817974c411527629e88937b7683695e276afd65da6ed6f", size = 1159058, upload-time = "2025-09-18T18:54:51.675Z" }, + { url = "https://files.pythonhosted.org/packages/98/05/b7c127283cfb154a97abb284363825401b69302d71a28608af66f73257cc/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7938502fcc060ed9543814f38ca67048b33d7bd2667756e36e6b1060455b17e", size = 8260987, upload-time = "2025-09-18T18:54:52.883Z" }, + { url = "https://files.pythonhosted.org/packages/ff/46/320d960aff884bc63abaaf846ffa3de4803e83e8070b6f84c5688464839c/numcodecs-0.16.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:010d628c95be1214536fb22c0df4ced58da954b404b1fcb25ddebf64e4a3f7f3", size = 8805295, upload-time = "2025-09-18T18:54:54.698Z" }, + { url = "https://files.pythonhosted.org/packages/31/ae/acc2e0f1f49ba32afa2174578f170673139248ef86f77e334f2619133867/numcodecs-0.16.3-cp313-cp313-win_amd64.whl", hash = "sha256:e83115e3c32de798c7b7164503e06aae9f9746c1cef564d029616eb44bd6cd90", size = 803204, upload-time = "2025-09-18T18:54:56.192Z" }, +] + +[package.optional-dependencies] +crc32c = [ + { name = "crc32c", marker = "python_full_version >= '3.11'" }, +] + [[package]] name = "numpy" version = "1.26.4" @@ -2881,6 +3257,59 @@ dependencies = [ ] sdist = { url = "https://files.pythonhosted.org/packages/05/79/87c45f32e661b25e0aaa1e325ba166511f57be5dff8f0fcabc12d3e73b64/nv_grouped_gemm-1.1.4.post6.tar.gz", hash = "sha256:dad6115f4b4ff7ceb0bc40ad44e923c13a24fc88cfe1e20b1a6b4c9cf24c445c", size = 26508, upload-time = "2025-10-10T18:52:29.508Z" } +[[package]] +name = "nvidia-cublas-cu12" +version = "12.8.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/99/db44d685f0e257ff0e213ade1964fc459b4a690a73293220e98feb3307cf/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b86f6dd8935884615a0683b663891d43781b819ac4f2ba2b0c9604676af346d0", size = 590537124, upload-time = "2025-03-07T01:43:53.556Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/e24b560ab2e2eaeb3c839129175fb330dfcfc29e5203196e5541a4c44682/nvidia_cublas_cu12-12.8.4.1-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8ac4e771d5a348c551b2a426eda6193c19aa630236b418086020df5ba9667142", size = 594346921, upload-time = "2025-03-07T01:44:31.254Z" }, + { url = "https://files.pythonhosted.org/packages/70/61/7d7b3c70186fb651d0fbd35b01dbfc8e755f69fd58f817f3d0f642df20c3/nvidia_cublas_cu12-12.8.4.1-py3-none-win_amd64.whl", hash = "sha256:47e9b82132fa8d2b4944e708049229601448aaad7e6f296f630f2d1a32de35af", size = 567544208, upload-time = "2025-03-07T01:53:30.535Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/1f/b3bd73445e5cb342727fd24fe1f7b748f690b460acadc27ea22f904502c8/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4412396548808ddfed3f17a467b104ba7751e6b58678a4b840675c56d21cf7ed", size = 9533318, upload-time = "2025-03-07T01:40:10.421Z" }, + { url = "https://files.pythonhosted.org/packages/f8/02/2adcaa145158bf1a8295d83591d22e4103dbfd821bcaf6f3f53151ca4ffa/nvidia_cuda_cupti_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ea0cb07ebda26bb9b29ba82cda34849e73c166c18162d3913575b0c9db9a6182", size = 10248621, upload-time = "2025-03-07T01:40:21.213Z" }, + { url = "https://files.pythonhosted.org/packages/41/bc/83f5426095d93694ae39fe1311431b5d5a9bb82e48bf0dd8e19be2765942/nvidia_cuda_cupti_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:bb479dcdf7e6d4f8b0b01b115260399bf34154a1a2e9fe11c85c517d87efd98e", size = 7015759, upload-time = "2025-03-07T01:51:11.355Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/6b/32f747947df2da6994e999492ab306a903659555dddc0fbdeb9d71f75e52/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a7756528852ef889772a84c6cd89d41dfa74667e24cca16bb31f8f061e3e9994", size = 88040029, upload-time = "2025-03-07T01:42:13.562Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d1/e50d0acaab360482034b84b6e27ee83c6738f7d32182b987f9c7a4e32962/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fc1fec1e1637854b4c0a65fb9a8346b51dd9ee69e61ebaccc82058441f15bce8", size = 43106076, upload-time = "2025-03-07T01:41:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/45/51/52a3d84baa2136cc8df15500ad731d74d3a1114d4c123e043cb608d4a32b/nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:7a4b6b2904850fe78e0bd179c4b655c404d4bb799ef03ddc60804247099ae909", size = 73586838, upload-time = "2025-03-07T01:52:13.483Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/75/f865a3b236e4647605ea34cc450900854ba123834a5f1598e160b9530c3a/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:52bf7bbee900262ffefe5e9d5a2a69a30d97e2bc5bb6cc866688caa976966e3d", size = 965265, upload-time = "2025-03-07T01:39:43.533Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9b/a997b638fcd068ad6e4d53b8551a7d30fe8b404d6f1804abf1df69838932/nvidia_cuda_runtime_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:adade8dcbd0edf427b7204d480d6066d33902cab2a4707dcfc48a2d0fd44ab90", size = 954765, upload-time = "2025-03-07T01:40:01.615Z" }, + { url = "https://files.pythonhosted.org/packages/30/a5/a515b7600ad361ea14bfa13fb4d6687abf500adc270f19e89849c0590492/nvidia_cuda_runtime_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:c0c6027f01505bfed6c3b21ec546f69c687689aad5f1a377554bc6ca4aa993a8", size = 944318, upload-time = "2025-03-07T01:51:01.794Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.10.2.21" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/e79269ce215c857c935fd86bcfe91a451a584dfc27f1e068f568b9ad1ab7/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c9132cc3f8958447b4910a1720036d9eff5928cc3179b0a51fb6d167c6cc87d8", size = 705026878, upload-time = "2025-06-06T21:52:51.348Z" }, + { url = "https://files.pythonhosted.org/packages/ba/51/e123d997aa098c61d029f76663dedbfb9bc8dcf8c60cbd6adbe42f76d049/nvidia_cudnn_cu12-9.10.2.21-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:949452be657fa16687d0930933f032835951ef0892b37d2d53824d1a84dc97a8", size = 706758467, upload-time = "2025-06-06T21:54:08.597Z" }, + { url = "https://files.pythonhosted.org/packages/3d/90/0bd6e586701b3a890fd38aa71c387dab4883d619d6e5ad912ccbd05bfd67/nvidia_cudnn_cu12-9.10.2.21-py3-none-win_amd64.whl", hash = "sha256:c6288de7d63e6cf62988f0923f96dc339cea362decb1bf5b3141883392a7d65e", size = 692992268, upload-time = "2025-06-06T21:55:18.114Z" }, +] + [[package]] name = "nvidia-cudnn-frontend" version = "1.15.0" @@ -2900,6 +3329,96 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/04/519fd6e3ea12fe7fe98c497c4d51f6c5c87763d02e90ea3102cef32a6ef1/nvidia_cudnn_frontend-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:7c8c6f12534b73b0cd55956c5e9419b7840a01e4c260837606112450ce1ca0d9", size = 1297324, upload-time = "2025-10-10T18:46:53.104Z" }, ] +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.3.83" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/bc/7771846d3a0272026c416fbb7e5f4c1f146d6d80704534d0b187dd6f4800/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:848ef7224d6305cdb2a4df928759dca7b1201874787083b6e7550dd6765ce69a", size = 193109211, upload-time = "2025-03-07T01:44:56.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/13/ee4e00f30e676b66ae65b4f08cb5bcbb8392c03f54f2d5413ea99a5d1c80/nvidia_cufft_cu12-11.3.3.83-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4d2dd21ec0b88cf61b62e6b43564355e5222e4a3fb394cac0db101f2dd0d4f74", size = 193118695, upload-time = "2025-03-07T01:45:27.821Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ec/ce1629f1e478bb5ccd208986b5f9e0316a78538dd6ab1d0484f012f8e2a1/nvidia_cufft_cu12-11.3.3.83-py3-none-win_amd64.whl", hash = "sha256:7a64a98ef2a7c47f905aaf8931b69a3a43f27c55530c698bb2ed7c75c0b42cb7", size = 192216559, upload-time = "2025-03-07T01:53:57.106Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.1.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/fe/1bcba1dfbfb8d01be8d93f07bfc502c93fa23afa6fd5ab3fc7c1df71038a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1d069003be650e131b21c932ec3d8969c1715379251f8d23a1860554b1cb24fc", size = 1197834, upload-time = "2025-03-07T01:45:50.723Z" }, + { url = "https://files.pythonhosted.org/packages/1e/f5/5607710447a6fe9fd9b3283956fceeee8a06cda1d2f56ce31371f595db2a/nvidia_cufile_cu12-1.13.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:4beb6d4cce47c1a0f1013d72e02b0994730359e17801d395bdcbf20cfb3bb00a", size = 1120705, upload-time = "2025-03-07T01:45:41.434Z" }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.9.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/5e/92aa15eca622a388b80fbf8375d4760738df6285b1e92c43d37390a33a9a/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:dfab99248034673b779bc6decafdc3404a8a6f502462201f2f31f11354204acd", size = 63625754, upload-time = "2025-03-07T01:46:10.735Z" }, + { url = "https://files.pythonhosted.org/packages/fb/aa/6584b56dc84ebe9cf93226a5cde4d99080c8e90ab40f0c27bda7a0f29aa1/nvidia_curand_cu12-10.3.9.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:b32331d4f4df5d6eefa0554c565b626c7216f87a06a4f56fab27c3b68a830ec9", size = 63619976, upload-time = "2025-03-07T01:46:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/b9/75/70c05b2f3ed5be3bb30b7102b6eb78e100da4bbf6944fd6725c012831cab/nvidia_curand_cu12-10.3.9.90-py3-none-win_amd64.whl", hash = "sha256:f149a8ca457277da854f89cf282d6ef43176861926c7ac85b2a0fbd237c587ec", size = 62765309, upload-time = "2025-03-07T01:54:20.478Z" }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.3.90" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/32/f7cd6ce8a7690544d084ea21c26e910a97e077c9b7f07bf5de623ee19981/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:db9ed69dbef9715071232caa9b69c52ac7de3a95773c2db65bdba85916e4e5c0", size = 267229841, upload-time = "2025-03-07T01:46:54.356Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/9a13d2975803e8cf2777d5ed57b87a0b6ca2cc795f9a4f59796a910bfb80/nvidia_cusolver_cu12-11.7.3.90-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4376c11ad263152bd50ea295c05370360776f8c3427b30991df774f9fb26c450", size = 267506905, upload-time = "2025-03-07T01:47:16.273Z" }, + { url = "https://files.pythonhosted.org/packages/13/c0/76ca8551b8a84146ffa189fec81c26d04adba4bc0dbe09cd6e6fd9b7de04/nvidia_cusolver_cu12-11.7.3.90-py3-none-win_amd64.whl", hash = "sha256:4a550db115fcabc4d495eb7d39ac8b58d4ab5d8e63274d3754df1c0ad6a22d34", size = 256720438, upload-time = "2025-03-07T01:54:39.898Z" }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.8.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/f7/cd777c4109681367721b00a106f491e0d0d15cfa1fd59672ce580ce42a97/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:9b6c161cb130be1a07a27ea6923df8141f3c295852f4b260c65f18f3e0a091dc", size = 288117129, upload-time = "2025-03-07T01:47:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f5/e1854cb2f2bcd4280c44736c93550cc300ff4b8c95ebe370d0aa7d2b473d/nvidia_cusparse_cu12-12.5.8.93-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ec05d76bbbd8b61b06a80e1eaf8cf4959c3d4ce8e711b65ebd0443bb0ebb13b", size = 288216466, upload-time = "2025-03-07T01:48:13.779Z" }, + { url = "https://files.pythonhosted.org/packages/62/07/f3b2ad63f8e3d257a599f422ae34eb565e70c41031aecefa3d18b62cabd1/nvidia_cusparse_cu12-12.5.8.93-py3-none-win_amd64.whl", hash = "sha256:9a33604331cb2cac199f2e7f5104dfbb8a5a898c367a53dfda9ff2acb6b6b4dd", size = 284937404, upload-time = "2025-03-07T01:55:07.742Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/b9/598f6ff36faaece4b3c50d26f50e38661499ff34346f00e057760b35cc9d/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8878dce784d0fac90131b6817b607e803c36e629ba34dc5b433471382196b6a5", size = 283835557, upload-time = "2025-02-26T00:16:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/56/79/12978b96bd44274fe38b5dde5cfb660b1d114f70a65ef962bcbbed99b549/nvidia_cusparselt_cu12-0.7.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f1bb701d6b930d5a7cea44c19ceb973311500847f81b634d802b7b539dc55623", size = 287193691, upload-time = "2025-02-26T00:15:44.104Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d8/a6b0d0d0c2435e9310f3e2bb0d9c9dd4c33daef86aa5f30b3681defd37ea/nvidia_cusparselt_cu12-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f67fbb5831940ec829c9117b7f33807db9f9678dc2a617fbe781cac17b4e1075", size = 271020911, upload-time = "2025-02-26T00:14:47.204Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl" +version = "4.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/0f/1e96ce9fbe07e8c39484fae4d2cf36e328bdf434b311d88ccedccbfed7db/nvidia_cutlass_dsl-4.2.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:1628bacedde042c60c7ebb1aeccce5a82501197f5e5c4fbbf803712fa45fba59", size = 58540319, upload-time = "2025-09-23T14:38:00.634Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e3/bc6071743d0ad43d837bf633139bfe1202260c28d893e30f247cf0aa8019/nvidia_cutlass_dsl-4.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aec74b50f700a8ef455f15863de4cb5f1486f72b7bd4becea88624c58c555a13", size = 62233601, upload-time = "2025-09-23T14:39:50.44Z" }, + { url = "https://files.pythonhosted.org/packages/1d/2a/e65312728338e5bb00b592ce0be12b51e7594a3ef288cd8c99bc1c456968/nvidia_cutlass_dsl-4.2.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:04e605417773957405cad0ac6c2d46139a88aca07a783b4f66e1363f3a91a835", size = 58540069, upload-time = "2025-09-23T14:38:56.002Z" }, + { url = "https://files.pythonhosted.org/packages/be/f3/20eacdf9876abd892668c191003edc5d7100e45fabfa027d9f3f99d21871/nvidia_cutlass_dsl-4.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:874aa3620b3d3dc6598af2226fa3b78f2e7998b8656929b492259e0c9f778786", size = 62233009, upload-time = "2025-09-23T14:39:23.308Z" }, + { url = "https://files.pythonhosted.org/packages/1e/1d/f168a3dbd8570e5dbbe0deca217d7b374c977b4a4970ebadf3b6d0f1174f/nvidia_cutlass_dsl-4.2.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:10ace6e2005cb0bc04d158c7660f8ec104ab29aeffb26f1ed3bb0b5a577ccc34", size = 58535504, upload-time = "2025-09-23T14:38:29.028Z" }, + { url = "https://files.pythonhosted.org/packages/02/ab/5bcc0c8c620af5d4acbc71abce10e3eb3023e50342e6bc29b6461f72530e/nvidia_cutlass_dsl-4.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d7ddc9c1f5bb803718d736c907fac857fc606f1fce630c0b1d741935a72723b9", size = 62230361, upload-time = "2025-09-23T14:40:18.156Z" }, + { url = "https://files.pythonhosted.org/packages/cf/d5/9b79faaec3fa12c52b7de1e727af94c54184b00f280c79b667ab045550db/nvidia_cutlass_dsl-4.2.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c0985124a74ba435e1f756aa78e89f64c6d01e4f54de1d5a5d218ebbc1c92eff", size = 58535424, upload-time = "2025-09-23T14:37:33.064Z" }, + { url = "https://files.pythonhosted.org/packages/43/86/78c8cd3fa1a684f3976535d7ac69e54f4ede165b5abca7979fd0820f74f2/nvidia_cutlass_dsl-4.2.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:9356604afc8f62aac46634b3a12baf8cb3f3a6f2e44e398dcfe6ec98ff1a8d1b", size = 62230122, upload-time = "2025-09-23T14:40:46.621Z" }, +] + [[package]] name = "nvidia-mathdx" version = "25.1.1" @@ -2919,12 +3438,13 @@ wheels = [ [[package]] name = "nvidia-modelopt" -version = "0.37.0" +version = "0.33.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ninja" }, { name = "numpy" }, { name = "nvidia-ml-py" }, + { name = "nvidia-modelopt-core" }, { name = "packaging" }, { name = "pulp" }, { name = "pydantic" }, @@ -2935,10 +3455,63 @@ dependencies = [ { name = "scipy", version = "1.16.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "torch", marker = "sys_platform == 'never'" }, { name = "torchprofile" }, + { name = "torchvision", marker = "sys_platform == 'never'" }, { name = "tqdm" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/27/a8/85034a33753a56ef120b931dda1183d98efef1010e379c54d3c3214a5048/nvidia_modelopt-0.37.0-py3-none-any.whl", hash = "sha256:3490b6d6aea3541aa5d475d81230fee627e2c16ff47bbab1cba4b80a1eb119a2", size = 831271, upload-time = "2025-10-08T18:37:03.951Z" }, + { url = "https://files.pythonhosted.org/packages/ca/cb/4af39357792a96f334c7877ea0380c9337aec210ff4794a7dd95beb7c349/nvidia_modelopt-0.33.1-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:6c51091683a117cd40fdb96a0ec28579f2276f6b627db7ccddc370df544e1dd7", size = 751683, upload-time = "2025-08-12T18:37:48.832Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b1/fc2f468d140ef58e90fac584759d0cc449db9bc4f64668cdff750ef38fef/nvidia_modelopt-0.33.1-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:ef78a98901890f265596ec413dffac177d4a1865201d89a14f29f4fa0cf8e710", size = 751683, upload-time = "2025-08-12T18:36:59.964Z" }, +] + +[[package]] +name = "nvidia-modelopt-core" +version = "0.33.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/21/d12ca11f5554340684d11958aae6c6e7755cf0aaae10a2d2c9db217228cf/nvidia_modelopt_core-0.33.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f25f6a817609c693ee39d1bcf2d3aeef462b9769f971590133de8b1b0310885b", size = 1307716, upload-time = "2025-08-12T18:41:12.086Z" }, + { url = "https://files.pythonhosted.org/packages/eb/df/7bead24d4854274d9f2818f1ae780fc24260aab60b7b6f73e1af4f056ce5/nvidia_modelopt_core-0.33.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:195f32f06d19bc9f9d858811f1864bddcc1db6278974d98ea6309cb3553427f1", size = 1326896, upload-time = "2025-08-12T18:39:48.243Z" }, + { url = "https://files.pythonhosted.org/packages/a1/36/3318980c670292d827ace5ac6110ab6054d0f2d87e507382842ea9e7c78f/nvidia_modelopt_core-0.33.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ffd008a90d8867660ae41c98002156b526e368a4cdf39e225fe20f478adce8b2", size = 1376104, upload-time = "2025-08-12T18:41:47.358Z" }, + { url = "https://files.pythonhosted.org/packages/27/97/99d1ddabe01ab262c18621619c996e1c2c119bc058607d2bc9ce7eb85fe7/nvidia_modelopt_core-0.33.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:be49121b2f74db4cb73955396a7bb83935d92232c5a20bcfd7b8e7cae68e482f", size = 1393729, upload-time = "2025-08-12T18:40:07.86Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba79b1c52b634b24e45dca409f133f947217a5c7ec5c256266e4ec5fa3eb/nvidia_modelopt_core-0.33.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1ddd9279d8312f8e972b302692a26e6180f1c9fd277232f5925a5589f42b1b76", size = 1338081, upload-time = "2025-08-12T18:40:36.156Z" }, + { url = "https://files.pythonhosted.org/packages/13/40/4427583475dfd8eb1b8c7522d75d4d059f0512ff03dcc62d6986a22ab918/nvidia_modelopt_core-0.33.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:69d5ace564f2b056c916117be2023f2b7fc01cd1501073915e6b2ced2b8a5394", size = 1363366, upload-time = "2025-08-12T18:39:28.854Z" }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.27.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/1c/857979db0ef194ca5e21478a0612bcdbbe59458d7694361882279947b349/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31432ad4d1fb1004eb0c56203dc9bc2178a1ba69d1d9e02d64a6938ab5e40e7a", size = 322400625, upload-time = "2025-06-26T04:11:04.496Z" }, + { url = "https://files.pythonhosted.org/packages/6e/89/f7a07dc961b60645dbbf42e80f2bc85ade7feb9a491b11a1e973aa00071f/nvidia_nccl_cu12-2.27.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ad730cf15cb5d25fe849c6e6ca9eb5b76db16a80f13f425ac68d8e2e55624457", size = 322348229, upload-time = "2025-06-26T04:11:28.385Z" }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.8.93" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/74/86a07f1d0f42998ca31312f998bd3b9a7eff7f52378f4f270c8679c77fb9/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:81ff63371a7ebd6e6451970684f916be2eab07321b73c9d244dc2b4da7f73b88", size = 39254836, upload-time = "2025-03-07T01:49:55.661Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a2/8cee5da30d13430e87bf99bb33455d2724d0a4a9cb5d7926d80ccb96d008/nvidia_nvjitlink_cu12-12.8.93-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:adccd7161ace7261e01bb91e44e88da350895c270d23f744f0820c818b7229e7", size = 38386204, upload-time = "2025-03-07T01:49:43.612Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d7/34f02dad2e30c31b10a51f6b04e025e5dd60e5f936af9045a9b858a05383/nvidia_nvjitlink_cu12-12.8.93-py3-none-win_amd64.whl", hash = "sha256:bd93fbeeee850917903583587f4fc3a4eafa022e34572251368238ab5e6bd67f", size = 268553710, upload-time = "2025-03-07T01:56:24.13Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu12" +version = "3.3.20" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/3dd98852568fb845ec1f7902c90a22b240fe1cbabda411ccedf2fd737b7b/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0b0b960da3842212758e4fa4696b94f129090b30e5122fea3c5345916545cff0", size = 124484616, upload-time = "2025-08-04T20:24:59.172Z" }, + { url = "https://files.pythonhosted.org/packages/3b/6c/99acb2f9eb85c29fc6f3a7ac4dccfd992e22666dd08a642b303311326a97/nvidia_nvshmem_cu12-3.3.20-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d00f26d3f9b2e3c3065be895e3059d6479ea5c638a3f38c9fec49b1b9dd7c1e5", size = 124657145, upload-time = "2025-08-04T20:25:19.995Z" }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.8.90" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/c0/1b303feea90d296f6176f32a2a70b5ef230f9bdeb3a72bddb0dc922dc137/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d7ad891da111ebafbf7e015d34879f7112832fc239ff0d7d776b6cb685274615", size = 91161, upload-time = "2025-03-07T01:42:23.922Z" }, + { url = "https://files.pythonhosted.org/packages/a2/eb/86626c1bbc2edb86323022371c39aa48df6fd8b0a1647bc274577f72e90b/nvidia_nvtx_cu12-12.8.90-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5b17e2001cc0d751a5bc2c6ec6d26ad95913324a4adb86788c944f8ce9ba441f", size = 89954, upload-time = "2025-03-07T01:42:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/9f/99/4c9c0c329bf9fc125008c3b54c7c94c0023518d06fc025ae36431375e1fe/nvidia_nvtx_cu12-12.8.90-py3-none-win_amd64.whl", hash = "sha256:619c8304aedc69f02ea82dd244541a83c3d9d40993381b3b590f1adaed3db41e", size = 56492, upload-time = "2025-03-07T01:52:24.69Z" }, ] [[package]] @@ -3014,8 +3587,8 @@ name = "onnx" version = "1.19.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version >= '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (python_full_version < '3.13' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "ml-dtypes", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "numpy" }, { name = "protobuf" }, { name = "typing-extensions" }, @@ -3059,10 +3632,12 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", ] dependencies = [ { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -3105,10 +3680,12 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", ] dependencies = [ { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -3125,7 +3702,7 @@ wheels = [ [[package]] name = "onnxscript" -version = "0.5.4" +version = "0.5.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.12.*' and sys_platform == 'linux'", @@ -3143,9 +3720,9 @@ dependencies = [ { name = "packaging", marker = "python_full_version < '3.13'" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/88/c3fad7bcb0f57c6f7795e28c479545cd47ca9109b174765f7cca0121148e/onnxscript-0.5.4.tar.gz", hash = "sha256:311e08ce513484a1fadfc19f3ad28f83ed41a8ce73eca4d038cd3623f6f52104", size = 584052, upload-time = "2025-10-16T17:55:27.615Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/4b/eed2199327bbf12c3443d7835893e3c4c23b1c1a4aa13efe0f7fbe0a6bf9/onnxscript-0.5.6.tar.gz", hash = "sha256:cc3338b2976daffd2af0bb6ac4866a4dca76aefface1666a0d7bc65ad9850822", size = 587017, upload-time = "2025-10-31T03:50:38.656Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/b2/bd42c04bd731769667df20de57f80b9753dbffb72bb24fddf4bb87a65433/onnxscript-0.5.4-py3-none-any.whl", hash = "sha256:640d2aa55364b9b0d66c280cc4b22cd2c918fde9a1a72c2460d2fbe24b9181dc", size = 679346, upload-time = "2025-10-16T17:55:29.886Z" }, + { url = "https://files.pythonhosted.org/packages/fd/1e/a5462bfe28a2add00dc0abec7dd9b742ac3207b73e5c97bde9747b503971/onnxscript-0.5.6-py3-none-any.whl", hash = "sha256:b0c3355fea3eecab8ca291da8b77afddcaacd3ada5ee59294390a049ea123938", size = 683045, upload-time = "2025-10-31T03:50:41.15Z" }, ] [[package]] @@ -3557,28 +4134,28 @@ wheels = [ [[package]] name = "psutil" -version = "7.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cd/ec/7b8e6b9b1d22708138630ef34c53ab2b61032c04f16adfdbb96791c8c70c/psutil-7.1.2.tar.gz", hash = "sha256:aa225cdde1335ff9684708ee8c72650f6598d5ed2114b9a7c5802030b1785018", size = 487424, upload-time = "2025-10-25T10:46:34.931Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b8/d9/b56cc9f883140ac10021a8c9b0f4e16eed1ba675c22513cdcbce3ba64014/psutil-7.1.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0cc5c6889b9871f231ed5455a9a02149e388fffcb30b607fb7a8896a6d95f22e", size = 238575, upload-time = "2025-10-25T10:46:38.728Z" }, - { url = "https://files.pythonhosted.org/packages/36/eb/28d22de383888deb252c818622196e709da98816e296ef95afda33f1c0a2/psutil-7.1.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8e9e77a977208d84aa363a4a12e0f72189d58bbf4e46b49aae29a2c6e93ef206", size = 239297, upload-time = "2025-10-25T10:46:41.347Z" }, - { url = "https://files.pythonhosted.org/packages/89/5d/220039e2f28cc129626e54d63892ab05c0d56a29818bfe7268dcb5008932/psutil-7.1.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d9623a5e4164d2220ecceb071f4b333b3c78866141e8887c072129185f41278", size = 280420, upload-time = "2025-10-25T10:46:44.122Z" }, - { url = "https://files.pythonhosted.org/packages/ba/7a/286f0e1c167445b2ef4a6cbdfc8c59fdb45a5a493788950cf8467201dc73/psutil-7.1.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:364b1c10fe4ed59c89ec49e5f1a70da353b27986fa8233b4b999df4742a5ee2f", size = 283049, upload-time = "2025-10-25T10:46:47.095Z" }, - { url = "https://files.pythonhosted.org/packages/aa/cc/7eb93260794a42e39b976f3a4dde89725800b9f573b014fac142002a5c98/psutil-7.1.2-cp313-cp313t-win_amd64.whl", hash = "sha256:f101ef84de7e05d41310e3ccbdd65a6dd1d9eed85e8aaf0758405d022308e204", size = 248713, upload-time = "2025-10-25T10:46:49.573Z" }, - { url = "https://files.pythonhosted.org/packages/ab/1a/0681a92b53366e01f0a099f5237d0c8a2f79d322ac589cccde5e30c8a4e2/psutil-7.1.2-cp313-cp313t-win_arm64.whl", hash = "sha256:20c00824048a95de67f00afedc7b08b282aa08638585b0206a9fb51f28f1a165", size = 244644, upload-time = "2025-10-25T10:46:51.924Z" }, - { url = "https://files.pythonhosted.org/packages/56/9e/f1c5c746b4ed5320952acd3002d3962fe36f30524c00ea79fdf954cc6779/psutil-7.1.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:e09cfe92aa8e22b1ec5e2d394820cf86c5dff6367ac3242366485dfa874d43bc", size = 238640, upload-time = "2025-10-25T10:46:54.089Z" }, - { url = "https://files.pythonhosted.org/packages/32/ee/fd26216a735395cc25c3899634e34aeb41fb1f3dbb44acc67d9e594be562/psutil-7.1.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fa6342cf859c48b19df3e4aa170e4cfb64aadc50b11e06bb569c6c777b089c9e", size = 239303, upload-time = "2025-10-25T10:46:56.932Z" }, - { url = "https://files.pythonhosted.org/packages/3c/cd/7d96eaec4ef7742b845a9ce2759a2769ecce4ab7a99133da24abacbc9e41/psutil-7.1.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:625977443498ee7d6c1e63e93bacca893fd759a66c5f635d05e05811d23fb5ee", size = 281717, upload-time = "2025-10-25T10:46:59.116Z" }, - { url = "https://files.pythonhosted.org/packages/bc/1a/7f0b84bdb067d35fe7fade5fff888408688caf989806ce2d6dae08c72dd5/psutil-7.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a24bcd7b7f2918d934af0fb91859f621b873d6aa81267575e3655cd387572a7", size = 284575, upload-time = "2025-10-25T10:47:00.944Z" }, - { url = "https://files.pythonhosted.org/packages/de/05/7820ef8f7b275268917e0c750eada5834581206d9024ca88edce93c4b762/psutil-7.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:329f05610da6380982e6078b9d0881d9ab1e9a7eb7c02d833bfb7340aa634e31", size = 249491, upload-time = "2025-10-25T10:47:03.174Z" }, - { url = "https://files.pythonhosted.org/packages/db/9a/58de399c7cb58489f08498459ff096cd76b3f1ddc4f224ec2c5ef729c7d0/psutil-7.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:7b04c29e3c0c888e83ed4762b70f31e65c42673ea956cefa8ced0e31e185f582", size = 244880, upload-time = "2025-10-25T10:47:05.228Z" }, - { url = "https://files.pythonhosted.org/packages/ae/89/b9f8d47ddbc52d7301fc868e8224e5f44ed3c7f55e6d0f54ecaf5dd9ff5e/psutil-7.1.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c9ba5c19f2d46203ee8c152c7b01df6eec87d883cfd8ee1af2ef2727f6b0f814", size = 237244, upload-time = "2025-10-25T10:47:07.086Z" }, - { url = "https://files.pythonhosted.org/packages/c8/7a/8628c2f6b240680a67d73d8742bb9ff39b1820a693740e43096d5dcb01e5/psutil-7.1.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a486030d2fe81bec023f703d3d155f4823a10a47c36784c84f1cc7f8d39bedb", size = 238101, upload-time = "2025-10-25T10:47:09.523Z" }, - { url = "https://files.pythonhosted.org/packages/30/28/5e27f4d5a0e347f8e3cc16cd7d35533dbce086c95807f1f0e9cd77e26c10/psutil-7.1.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3efd8fc791492e7808a51cb2b94889db7578bfaea22df931424f874468e389e3", size = 258675, upload-time = "2025-10-25T10:47:11.082Z" }, - { url = "https://files.pythonhosted.org/packages/e5/5c/79cf60c9acf36d087f0db0f82066fca4a780e97e5b3a2e4c38209c03d170/psutil-7.1.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2aeb9b64f481b8eabfc633bd39e0016d4d8bbcd590d984af764d80bf0851b8a", size = 260203, upload-time = "2025-10-25T10:47:13.226Z" }, - { url = "https://files.pythonhosted.org/packages/f7/03/0a464404c51685dcb9329fdd660b1721e076ccd7b3d97dee066bcc9ffb15/psutil-7.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:8e17852114c4e7996fe9da4745c2bdef001ebbf2f260dec406290e66628bdb91", size = 246714, upload-time = "2025-10-25T10:47:15.093Z" }, - { url = "https://files.pythonhosted.org/packages/6a/32/97ca2090f2f1b45b01b6aa7ae161cfe50671de097311975ca6eea3e7aabc/psutil-7.1.2-cp37-abi3-win_arm64.whl", hash = "sha256:3e988455e61c240cc879cb62a008c2699231bf3e3d061d7fce4234463fd2abb4", size = 243742, upload-time = "2025-10-25T10:47:17.302Z" }, +version = "7.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/88/bdd0a41e5857d5d703287598cbf08dad90aed56774ea52ae071bae9071b6/psutil-7.1.3.tar.gz", hash = "sha256:6c86281738d77335af7aec228328e944b30930899ea760ecf33a4dba66be5e74", size = 489059, upload-time = "2025-11-02T12:25:54.619Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/93/0c49e776b8734fef56ec9c5c57f923922f2cf0497d62e0f419465f28f3d0/psutil-7.1.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0005da714eee687b4b8decd3d6cc7c6db36215c9e74e5ad2264b90c3df7d92dc", size = 239751, upload-time = "2025-11-02T12:25:58.161Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8d/b31e39c769e70780f007969815195a55c81a63efebdd4dbe9e7a113adb2f/psutil-7.1.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:19644c85dcb987e35eeeaefdc3915d059dac7bd1167cdcdbf27e0ce2df0c08c0", size = 240368, upload-time = "2025-11-02T12:26:00.491Z" }, + { url = "https://files.pythonhosted.org/packages/62/61/23fd4acc3c9eebbf6b6c78bcd89e5d020cfde4acf0a9233e9d4e3fa698b4/psutil-7.1.3-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:95ef04cf2e5ba0ab9eaafc4a11eaae91b44f4ef5541acd2ee91d9108d00d59a7", size = 287134, upload-time = "2025-11-02T12:26:02.613Z" }, + { url = "https://files.pythonhosted.org/packages/30/1c/f921a009ea9ceb51aa355cb0cc118f68d354db36eae18174bab63affb3e6/psutil-7.1.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1068c303be3a72f8e18e412c5b2a8f6d31750fb152f9cb106b54090296c9d251", size = 289904, upload-time = "2025-11-02T12:26:05.207Z" }, + { url = "https://files.pythonhosted.org/packages/a6/82/62d68066e13e46a5116df187d319d1724b3f437ddd0f958756fc052677f4/psutil-7.1.3-cp313-cp313t-win_amd64.whl", hash = "sha256:18349c5c24b06ac5612c0428ec2a0331c26443d259e2a0144a9b24b4395b58fa", size = 249642, upload-time = "2025-11-02T12:26:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/df/ad/c1cd5fe965c14a0392112f68362cfceb5230819dbb5b1888950d18a11d9f/psutil-7.1.3-cp313-cp313t-win_arm64.whl", hash = "sha256:c525ffa774fe4496282fb0b1187725793de3e7c6b29e41562733cae9ada151ee", size = 245518, upload-time = "2025-11-02T12:26:09.719Z" }, + { url = "https://files.pythonhosted.org/packages/2e/bb/6670bded3e3236eb4287c7bcdc167e9fae6e1e9286e437f7111caed2f909/psutil-7.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b403da1df4d6d43973dc004d19cee3b848e998ae3154cc8097d139b77156c353", size = 239843, upload-time = "2025-11-02T12:26:11.968Z" }, + { url = "https://files.pythonhosted.org/packages/b8/66/853d50e75a38c9a7370ddbeefabdd3d3116b9c31ef94dc92c6729bc36bec/psutil-7.1.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ad81425efc5e75da3f39b3e636293360ad8d0b49bed7df824c79764fb4ba9b8b", size = 240369, upload-time = "2025-11-02T12:26:14.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/bd/313aba97cb5bfb26916dc29cf0646cbe4dd6a89ca69e8c6edce654876d39/psutil-7.1.3-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f33a3702e167783a9213db10ad29650ebf383946e91bc77f28a5eb083496bc9", size = 288210, upload-time = "2025-11-02T12:26:16.699Z" }, + { url = "https://files.pythonhosted.org/packages/c2/fa/76e3c06e760927a0cfb5705eb38164254de34e9bd86db656d4dbaa228b04/psutil-7.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fac9cd332c67f4422504297889da5ab7e05fd11e3c4392140f7370f4208ded1f", size = 291182, upload-time = "2025-11-02T12:26:18.848Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1d/5774a91607035ee5078b8fd747686ebec28a962f178712de100d00b78a32/psutil-7.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:3792983e23b69843aea49c8f5b8f115572c5ab64c153bada5270086a2123c7e7", size = 250466, upload-time = "2025-11-02T12:26:21.183Z" }, + { url = "https://files.pythonhosted.org/packages/00/ca/e426584bacb43a5cb1ac91fae1937f478cd8fbe5e4ff96574e698a2c77cd/psutil-7.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:31d77fcedb7529f27bb3a0472bea9334349f9a04160e8e6e5020f22c59893264", size = 245756, upload-time = "2025-11-02T12:26:23.148Z" }, + { url = "https://files.pythonhosted.org/packages/ef/94/46b9154a800253e7ecff5aaacdf8ebf43db99de4a2dfa18575b02548654e/psutil-7.1.3-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2bdbcd0e58ca14996a42adf3621a6244f1bb2e2e528886959c72cf1e326677ab", size = 238359, upload-time = "2025-11-02T12:26:25.284Z" }, + { url = "https://files.pythonhosted.org/packages/68/3a/9f93cff5c025029a36d9a92fef47220ab4692ee7f2be0fba9f92813d0cb8/psutil-7.1.3-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc31fa00f1fbc3c3802141eede66f3a2d51d89716a194bf2cd6fc68310a19880", size = 239171, upload-time = "2025-11-02T12:26:27.23Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b1/5f49af514f76431ba4eea935b8ad3725cdeb397e9245ab919dbc1d1dc20f/psutil-7.1.3-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bb428f9f05c1225a558f53e30ccbad9930b11c3fc206836242de1091d3e7dd3", size = 263261, upload-time = "2025-11-02T12:26:29.48Z" }, + { url = "https://files.pythonhosted.org/packages/e0/95/992c8816a74016eb095e73585d747e0a8ea21a061ed3689474fabb29a395/psutil-7.1.3-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56d974e02ca2c8eb4812c3f76c30e28836fffc311d55d979f1465c1feeb2b68b", size = 264635, upload-time = "2025-11-02T12:26:31.74Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/c3ed1a622b6ae2fd3c945a366e64eb35247a31e4db16cf5095e269e8eb3c/psutil-7.1.3-cp37-abi3-win_amd64.whl", hash = "sha256:f39c2c19fe824b47484b96f9692932248a54c43799a84282cfe58d05a6449efd", size = 247633, upload-time = "2025-11-02T12:26:33.887Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ad/33b2ccec09bf96c2b2ef3f9a6f66baac8253d7565d8839e024a6b905d45d/psutil-7.1.3-cp37-abi3-win_arm64.whl", hash = "sha256:bd0d69cee829226a761e92f28140bec9a5ee9d5b4fb4b0cc589068dbfff559b1", size = 244608, upload-time = "2025-11-02T12:26:36.136Z" }, ] [[package]] @@ -3590,6 +4167,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/6c/64cafaceea3f99927e84b38a362ec6a8f24f33061c90bda77dfe1cd4c3c6/pulp-3.3.0-py3-none-any.whl", hash = "sha256:dd6ad2d63f196d1254eddf9dcff5cd224912c1f046120cb7c143c5b0eda63fae", size = 16387700, upload-time = "2025-09-18T08:14:53.368Z" }, ] +[[package]] +name = "pyarrow" +version = "22.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/53/04a7fdc63e6056116c9ddc8b43bc28c12cdd181b85cbeadb79278475f3ae/pyarrow-22.0.0.tar.gz", hash = "sha256:3d600dc583260d845c7d8a6db540339dd883081925da2bd1c5cb808f720b3cd9", size = 1151151, upload-time = "2025-10-24T12:30:00.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/9b/cb3f7e0a345353def531ca879053e9ef6b9f38ed91aebcf68b09ba54dec0/pyarrow-22.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:77718810bd3066158db1e95a63c160ad7ce08c6b0710bc656055033e39cdad88", size = 34223968, upload-time = "2025-10-24T10:03:31.21Z" }, + { url = "https://files.pythonhosted.org/packages/6c/41/3184b8192a120306270c5307f105b70320fdaa592c99843c5ef78aaefdcf/pyarrow-22.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:44d2d26cda26d18f7af7db71453b7b783788322d756e81730acb98f24eb90ace", size = 35942085, upload-time = "2025-10-24T10:03:38.146Z" }, + { url = "https://files.pythonhosted.org/packages/d9/3d/a1eab2f6f08001f9fb714b8ed5cfb045e2fe3e3e3c0c221f2c9ed1e6d67d/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b9d71701ce97c95480fecb0039ec5bb889e75f110da72005743451339262f4ce", size = 44964613, upload-time = "2025-10-24T10:03:46.516Z" }, + { url = "https://files.pythonhosted.org/packages/46/46/a1d9c24baf21cfd9ce994ac820a24608decf2710521b29223d4334985127/pyarrow-22.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:710624ab925dc2b05a6229d47f6f0dac1c1155e6ed559be7109f684eba048a48", size = 47627059, upload-time = "2025-10-24T10:03:55.353Z" }, + { url = "https://files.pythonhosted.org/packages/3a/4c/f711acb13075c1391fd54bc17e078587672c575f8de2a6e62509af026dcf/pyarrow-22.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f963ba8c3b0199f9d6b794c90ec77545e05eadc83973897a4523c9e8d84e9340", size = 47947043, upload-time = "2025-10-24T10:04:05.408Z" }, + { url = "https://files.pythonhosted.org/packages/4e/70/1f3180dd7c2eab35c2aca2b29ace6c519f827dcd4cfeb8e0dca41612cf7a/pyarrow-22.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd0d42297ace400d8febe55f13fdf46e86754842b860c978dfec16f081e5c653", size = 50206505, upload-time = "2025-10-24T10:04:15.786Z" }, + { url = "https://files.pythonhosted.org/packages/80/07/fea6578112c8c60ffde55883a571e4c4c6bc7049f119d6b09333b5cc6f73/pyarrow-22.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:00626d9dc0f5ef3a75fe63fd68b9c7c8302d2b5bbc7f74ecaedba83447a24f84", size = 28101641, upload-time = "2025-10-24T10:04:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b7/18f611a8cdc43417f9394a3ccd3eace2f32183c08b9eddc3d17681819f37/pyarrow-22.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:3e294c5eadfb93d78b0763e859a0c16d4051fc1c5231ae8956d61cb0b5666f5a", size = 34272022, upload-time = "2025-10-24T10:04:28.973Z" }, + { url = "https://files.pythonhosted.org/packages/26/5c/f259e2526c67eb4b9e511741b19870a02363a47a35edbebc55c3178db22d/pyarrow-22.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:69763ab2445f632d90b504a815a2a033f74332997052b721002298ed6de40f2e", size = 35995834, upload-time = "2025-10-24T10:04:35.467Z" }, + { url = "https://files.pythonhosted.org/packages/50/8d/281f0f9b9376d4b7f146913b26fac0aa2829cd1ee7e997f53a27411bbb92/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:b41f37cabfe2463232684de44bad753d6be08a7a072f6a83447eeaf0e4d2a215", size = 45030348, upload-time = "2025-10-24T10:04:43.366Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e5/53c0a1c428f0976bf22f513d79c73000926cb00b9c138d8e02daf2102e18/pyarrow-22.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:35ad0f0378c9359b3f297299c3309778bb03b8612f987399a0333a560b43862d", size = 47699480, upload-time = "2025-10-24T10:04:51.486Z" }, + { url = "https://files.pythonhosted.org/packages/95/e1/9dbe4c465c3365959d183e6345d0a8d1dc5b02ca3f8db4760b3bc834cf25/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8382ad21458075c2e66a82a29d650f963ce51c7708c7c0ff313a8c206c4fd5e8", size = 48011148, upload-time = "2025-10-24T10:04:59.585Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b4/7caf5d21930061444c3cf4fa7535c82faf5263e22ce43af7c2759ceb5b8b/pyarrow-22.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1a812a5b727bc09c3d7ea072c4eebf657c2f7066155506ba31ebf4792f88f016", size = 50276964, upload-time = "2025-10-24T10:05:08.175Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f3/cec89bd99fa3abf826f14d4e53d3d11340ce6f6af4d14bdcd54cd83b6576/pyarrow-22.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:ec5d40dd494882704fb876c16fa7261a69791e784ae34e6b5992e977bd2e238c", size = 28106517, upload-time = "2025-10-24T10:05:14.314Z" }, + { url = "https://files.pythonhosted.org/packages/af/63/ba23862d69652f85b615ca14ad14f3bcfc5bf1b99ef3f0cd04ff93fdad5a/pyarrow-22.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bea79263d55c24a32b0d79c00a1c58bb2ee5f0757ed95656b01c0fb310c5af3d", size = 34211578, upload-time = "2025-10-24T10:05:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/b1/d0/f9ad86fe809efd2bcc8be32032fa72e8b0d112b01ae56a053006376c5930/pyarrow-22.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:12fe549c9b10ac98c91cf791d2945e878875d95508e1a5d14091a7aaa66d9cf8", size = 35989906, upload-time = "2025-10-24T10:05:29.485Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a8/f910afcb14630e64d673f15904ec27dd31f1e009b77033c365c84e8c1e1d/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:334f900ff08ce0423407af97e6c26ad5d4e3b0763645559ece6fbf3747d6a8f5", size = 45021677, upload-time = "2025-10-24T10:05:38.274Z" }, + { url = "https://files.pythonhosted.org/packages/13/95/aec81f781c75cd10554dc17a25849c720d54feafb6f7847690478dcf5ef8/pyarrow-22.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c6c791b09c57ed76a18b03f2631753a4960eefbbca80f846da8baefc6491fcfe", size = 47726315, upload-time = "2025-10-24T10:05:47.314Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d4/74ac9f7a54cfde12ee42734ea25d5a3c9a45db78f9def949307a92720d37/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c3200cb41cdbc65156e5f8c908d739b0dfed57e890329413da2748d1a2cd1a4e", size = 47990906, upload-time = "2025-10-24T10:05:58.254Z" }, + { url = "https://files.pythonhosted.org/packages/2e/71/fedf2499bf7a95062eafc989ace56572f3343432570e1c54e6599d5b88da/pyarrow-22.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ac93252226cf288753d8b46280f4edf3433bf9508b6977f8dd8526b521a1bbb9", size = 50306783, upload-time = "2025-10-24T10:06:08.08Z" }, + { url = "https://files.pythonhosted.org/packages/68/ed/b202abd5a5b78f519722f3d29063dda03c114711093c1995a33b8e2e0f4b/pyarrow-22.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:44729980b6c50a5f2bfcc2668d36c569ce17f8b17bccaf470c4313dcbbf13c9d", size = 27972883, upload-time = "2025-10-24T10:06:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/a6/d6/d0fac16a2963002fc22c8fa75180a838737203d558f0ed3b564c4a54eef5/pyarrow-22.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e6e95176209257803a8b3d0394f21604e796dadb643d2f7ca21b66c9c0b30c9a", size = 34204629, upload-time = "2025-10-24T10:06:20.274Z" }, + { url = "https://files.pythonhosted.org/packages/c6/9c/1d6357347fbae062ad3f17082f9ebc29cc733321e892c0d2085f42a2212b/pyarrow-22.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:001ea83a58024818826a9e3f89bf9310a114f7e26dfe404a4c32686f97bd7901", size = 35985783, upload-time = "2025-10-24T10:06:27.301Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c0/782344c2ce58afbea010150df07e3a2f5fdad299cd631697ae7bd3bac6e3/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ce20fe000754f477c8a9125543f1936ea5b8867c5406757c224d745ed033e691", size = 45020999, upload-time = "2025-10-24T10:06:35.387Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8b/5362443737a5307a7b67c1017c42cd104213189b4970bf607e05faf9c525/pyarrow-22.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:e0a15757fccb38c410947df156f9749ae4a3c89b2393741a50521f39a8cf202a", size = 47724601, upload-time = "2025-10-24T10:06:43.551Z" }, + { url = "https://files.pythonhosted.org/packages/69/4d/76e567a4fc2e190ee6072967cb4672b7d9249ac59ae65af2d7e3047afa3b/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cedb9dd9358e4ea1d9bce3665ce0797f6adf97ff142c8e25b46ba9cdd508e9b6", size = 48001050, upload-time = "2025-10-24T10:06:52.284Z" }, + { url = "https://files.pythonhosted.org/packages/01/5e/5653f0535d2a1aef8223cee9d92944cb6bccfee5cf1cd3f462d7cb022790/pyarrow-22.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:252be4a05f9d9185bb8c18e83764ebcfea7185076c07a7a662253af3a8c07941", size = 50307877, upload-time = "2025-10-24T10:07:02.405Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f8/1d0bd75bf9328a3b826e24a16e5517cd7f9fbf8d34a3184a4566ef5a7f29/pyarrow-22.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:a4893d31e5ef780b6edcaf63122df0f8d321088bb0dee4c8c06eccb1ca28d145", size = 27977099, upload-time = "2025-10-24T10:08:07.259Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/db56870c997805bf2b0f6eeeb2d68458bf4654652dccdcf1bf7a42d80903/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:f7fe3dbe871294ba70d789be16b6e7e52b418311e166e0e3cba9522f0f437fb1", size = 34336685, upload-time = "2025-10-24T10:07:11.47Z" }, + { url = "https://files.pythonhosted.org/packages/1c/98/0727947f199aba8a120f47dfc229eeb05df15bcd7a6f1b669e9f882afc58/pyarrow-22.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ba95112d15fd4f1105fb2402c4eab9068f0554435e9b7085924bcfaac2cc306f", size = 36032158, upload-time = "2025-10-24T10:07:18.626Z" }, + { url = "https://files.pythonhosted.org/packages/96/b4/9babdef9c01720a0785945c7cf550e4acd0ebcd7bdd2e6f0aa7981fa85e2/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:c064e28361c05d72eed8e744c9605cbd6d2bb7481a511c74071fd9b24bc65d7d", size = 44892060, upload-time = "2025-10-24T10:07:26.002Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ca/2f8804edd6279f78a37062d813de3f16f29183874447ef6d1aadbb4efa0f/pyarrow-22.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:6f9762274496c244d951c819348afbcf212714902742225f649cf02823a6a10f", size = 47504395, upload-time = "2025-10-24T10:07:34.09Z" }, + { url = "https://files.pythonhosted.org/packages/b9/f0/77aa5198fd3943682b2e4faaf179a674f0edea0d55d326d83cb2277d9363/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a9d9ffdc2ab696f6b15b4d1f7cec6658e1d788124418cb30030afbae31c64746", size = 48066216, upload-time = "2025-10-24T10:07:43.528Z" }, + { url = "https://files.pythonhosted.org/packages/79/87/a1937b6e78b2aff18b706d738c9e46ade5bfcf11b294e39c87706a0089ac/pyarrow-22.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ec1a15968a9d80da01e1d30349b2b0d7cc91e96588ee324ce1b5228175043e95", size = 50288552, upload-time = "2025-10-24T10:07:53.519Z" }, + { url = "https://files.pythonhosted.org/packages/60/ae/b5a5811e11f25788ccfdaa8f26b6791c9807119dffcf80514505527c384c/pyarrow-22.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bba208d9c7decf9961998edf5c65e3ea4355d5818dd6cd0f6809bec1afb951cc", size = 28262504, upload-time = "2025-10-24T10:08:00.932Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b0/0fa4d28a8edb42b0a7144edd20befd04173ac79819547216f8a9f36f9e50/pyarrow-22.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:9bddc2cade6561f6820d4cd73f99a0243532ad506bc510a75a5a65a522b2d74d", size = 34224062, upload-time = "2025-10-24T10:08:14.101Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a8/7a719076b3c1be0acef56a07220c586f25cd24de0e3f3102b438d18ae5df/pyarrow-22.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e70ff90c64419709d38c8932ea9fe1cc98415c4f87ea8da81719e43f02534bc9", size = 35990057, upload-time = "2025-10-24T10:08:21.842Z" }, + { url = "https://files.pythonhosted.org/packages/89/3c/359ed54c93b47fb6fe30ed16cdf50e3f0e8b9ccfb11b86218c3619ae50a8/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:92843c305330aa94a36e706c16209cd4df274693e777ca47112617db7d0ef3d7", size = 45068002, upload-time = "2025-10-24T10:08:29.034Z" }, + { url = "https://files.pythonhosted.org/packages/55/fc/4945896cc8638536ee787a3bd6ce7cec8ec9acf452d78ec39ab328efa0a1/pyarrow-22.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6dda1ddac033d27421c20d7a7943eec60be44e0db4e079f33cc5af3b8280ccde", size = 47737765, upload-time = "2025-10-24T10:08:38.559Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5e/7cb7edeb2abfaa1f79b5d5eb89432356155c8426f75d3753cbcb9592c0fd/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:84378110dd9a6c06323b41b56e129c504d157d1a983ce8f5443761eb5256bafc", size = 48048139, upload-time = "2025-10-24T10:08:46.784Z" }, + { url = "https://files.pythonhosted.org/packages/88/c6/546baa7c48185f5e9d6e59277c4b19f30f48c94d9dd938c2a80d4d6b067c/pyarrow-22.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:854794239111d2b88b40b6ef92aa478024d1e5074f364033e73e21e3f76b25e0", size = 50314244, upload-time = "2025-10-24T10:08:55.771Z" }, + { url = "https://files.pythonhosted.org/packages/3c/79/755ff2d145aafec8d347bf18f95e4e81c00127f06d080135dfc86aea417c/pyarrow-22.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:b883fe6fd85adad7932b3271c38ac289c65b7337c2c132e9569f9d3940620730", size = 28757501, upload-time = "2025-10-24T10:09:59.891Z" }, + { url = "https://files.pythonhosted.org/packages/0e/d2/237d75ac28ced3147912954e3c1a174df43a95f4f88e467809118a8165e0/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:7a820d8ae11facf32585507c11f04e3f38343c1e784c9b5a8b1da5c930547fe2", size = 34355506, upload-time = "2025-10-24T10:09:02.953Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/733dfffe6d3069740f98e57ff81007809067d68626c5faef293434d11bd6/pyarrow-22.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:c6ec3675d98915bf1ec8b3c7986422682f7232ea76cad276f4c8abd5b7319b70", size = 36047312, upload-time = "2025-10-24T10:09:10.334Z" }, + { url = "https://files.pythonhosted.org/packages/7c/2b/29d6e3782dc1f299727462c1543af357a0f2c1d3c160ce199950d9ca51eb/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:3e739edd001b04f654b166204fc7a9de896cf6007eaff33409ee9e50ceaff754", size = 45081609, upload-time = "2025-10-24T10:09:18.61Z" }, + { url = "https://files.pythonhosted.org/packages/8d/42/aa9355ecc05997915af1b7b947a7f66c02dcaa927f3203b87871c114ba10/pyarrow-22.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7388ac685cab5b279a41dfe0a6ccd99e4dbf322edfb63e02fc0443bf24134e91", size = 47703663, upload-time = "2025-10-24T10:09:27.369Z" }, + { url = "https://files.pythonhosted.org/packages/ee/62/45abedde480168e83a1de005b7b7043fd553321c1e8c5a9a114425f64842/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f633074f36dbc33d5c05b5dc75371e5660f1dbf9c8b1d95669def05e5425989c", size = 48066543, upload-time = "2025-10-24T10:09:34.908Z" }, + { url = "https://files.pythonhosted.org/packages/84/e9/7878940a5b072e4f3bf998770acafeae13b267f9893af5f6d4ab3904b67e/pyarrow-22.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4c19236ae2402a8663a2c8f21f1870a03cc57f0bef7e4b6eb3238cc82944de80", size = 50288838, upload-time = "2025-10-24T10:09:44.394Z" }, + { url = "https://files.pythonhosted.org/packages/7b/03/f335d6c52b4a4761bcc83499789a1e2e16d9d201a58c327a9b5cc9a41bd9/pyarrow-22.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0c34fe18094686194f204a3b1787a27456897d8a2d62caf84b61e8dfbc0252ae", size = 29185594, upload-time = "2025-10-24T10:09:53.111Z" }, +] + [[package]] name = "pybind11" version = "3.0.1" @@ -3619,7 +4253,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.12.3" +version = "2.12.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -3627,123 +4261,127 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/1e/4f0a3233767010308f2fd6bd0814597e3f63f1dc98304a9112b8759df4ff/pydantic-2.12.3.tar.gz", hash = "sha256:1da1c82b0fc140bb0103bc1441ffe062154c8d38491189751ee00fd8ca65ce74", size = 819383, upload-time = "2025-10-17T15:04:21.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/6b/83661fa77dcefa195ad5f8cd9af3d1a7450fd57cc883ad04d65446ac2029/pydantic-2.12.3-py3-none-any.whl", hash = "sha256:6986454a854bc3bc6e5443e1369e06a3a456af9d339eda45510f517d9ea5c6bf", size = 462431, upload-time = "2025-10-17T15:04:19.346Z" }, + { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.4" +version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/df/18/d0944e8eaaa3efd0a91b0f1fc537d3be55ad35091b6a87638211ba691964/pydantic_core-2.41.4.tar.gz", hash = "sha256:70e47929a9d4a1905a67e4b687d5946026390568a8e952b92824118063cee4d5", size = 457557, upload-time = "2025-10-14T10:23:47.909Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/3d/9b8ca77b0f76fcdbf8bc6b72474e264283f461284ca84ac3fde570c6c49a/pydantic_core-2.41.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2442d9a4d38f3411f22eb9dd0912b7cbf4b7d5b6c92c4173b75d3e1ccd84e36e", size = 2111197, upload-time = "2025-10-14T10:19:43.303Z" }, - { url = "https://files.pythonhosted.org/packages/59/92/b7b0fe6ed4781642232755cb7e56a86e2041e1292f16d9ae410a0ccee5ac/pydantic_core-2.41.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:30a9876226dda131a741afeab2702e2d127209bde3c65a2b8133f428bc5d006b", size = 1917909, upload-time = "2025-10-14T10:19:45.194Z" }, - { url = "https://files.pythonhosted.org/packages/52/8c/3eb872009274ffa4fb6a9585114e161aa1a0915af2896e2d441642929fe4/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d55bbac04711e2980645af68b97d445cdbcce70e5216de444a6c4b6943ebcccd", size = 1969905, upload-time = "2025-10-14T10:19:46.567Z" }, - { url = "https://files.pythonhosted.org/packages/f4/21/35adf4a753bcfaea22d925214a0c5b880792e3244731b3f3e6fec0d124f7/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e1d778fb7849a42d0ee5927ab0f7453bf9f85eef8887a546ec87db5ddb178945", size = 2051938, upload-time = "2025-10-14T10:19:48.237Z" }, - { url = "https://files.pythonhosted.org/packages/7d/d0/cdf7d126825e36d6e3f1eccf257da8954452934ede275a8f390eac775e89/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b65077a4693a98b90ec5ad8f203ad65802a1b9b6d4a7e48066925a7e1606706", size = 2250710, upload-time = "2025-10-14T10:19:49.619Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1c/af1e6fd5ea596327308f9c8d1654e1285cc3d8de0d584a3c9d7705bf8a7c/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62637c769dee16eddb7686bf421be48dfc2fae93832c25e25bc7242e698361ba", size = 2367445, upload-time = "2025-10-14T10:19:51.269Z" }, - { url = "https://files.pythonhosted.org/packages/d3/81/8cece29a6ef1b3a92f956ea6da6250d5b2d2e7e4d513dd3b4f0c7a83dfea/pydantic_core-2.41.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dfe3aa529c8f501babf6e502936b9e8d4698502b2cfab41e17a028d91b1ac7b", size = 2072875, upload-time = "2025-10-14T10:19:52.671Z" }, - { url = "https://files.pythonhosted.org/packages/e3/37/a6a579f5fc2cd4d5521284a0ab6a426cc6463a7b3897aeb95b12f1ba607b/pydantic_core-2.41.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca2322da745bf2eeb581fc9ea3bbb31147702163ccbcbf12a3bb630e4bf05e1d", size = 2191329, upload-time = "2025-10-14T10:19:54.214Z" }, - { url = "https://files.pythonhosted.org/packages/ae/03/505020dc5c54ec75ecba9f41119fd1e48f9e41e4629942494c4a8734ded1/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e8cd3577c796be7231dcf80badcf2e0835a46665eaafd8ace124d886bab4d700", size = 2151658, upload-time = "2025-10-14T10:19:55.843Z" }, - { url = "https://files.pythonhosted.org/packages/cb/5d/2c0d09fb53aa03bbd2a214d89ebfa6304be7df9ed86ee3dc7770257f41ee/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:1cae8851e174c83633f0833e90636832857297900133705ee158cf79d40f03e6", size = 2316777, upload-time = "2025-10-14T10:19:57.607Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4b/c2c9c8f5e1f9c864b57d08539d9d3db160e00491c9f5ee90e1bfd905e644/pydantic_core-2.41.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a26d950449aae348afe1ac8be5525a00ae4235309b729ad4d3399623125b43c9", size = 2320705, upload-time = "2025-10-14T10:19:59.016Z" }, - { url = "https://files.pythonhosted.org/packages/28/c3/a74c1c37f49c0a02c89c7340fafc0ba816b29bd495d1a31ce1bdeacc6085/pydantic_core-2.41.4-cp310-cp310-win32.whl", hash = "sha256:0cf2a1f599efe57fa0051312774280ee0f650e11152325e41dfd3018ef2c1b57", size = 1975464, upload-time = "2025-10-14T10:20:00.581Z" }, - { url = "https://files.pythonhosted.org/packages/d6/23/5dd5c1324ba80303368f7569e2e2e1a721c7d9eb16acb7eb7b7f85cb1be2/pydantic_core-2.41.4-cp310-cp310-win_amd64.whl", hash = "sha256:a8c2e340d7e454dc3340d3d2e8f23558ebe78c98aa8f68851b04dcb7bc37abdc", size = 2024497, upload-time = "2025-10-14T10:20:03.018Z" }, - { url = "https://files.pythonhosted.org/packages/62/4c/f6cbfa1e8efacd00b846764e8484fe173d25b8dab881e277a619177f3384/pydantic_core-2.41.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:28ff11666443a1a8cf2a044d6a545ebffa8382b5f7973f22c36109205e65dc80", size = 2109062, upload-time = "2025-10-14T10:20:04.486Z" }, - { url = "https://files.pythonhosted.org/packages/21/f8/40b72d3868896bfcd410e1bd7e516e762d326201c48e5b4a06446f6cf9e8/pydantic_core-2.41.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61760c3925d4633290292bad462e0f737b840508b4f722247d8729684f6539ae", size = 1916301, upload-time = "2025-10-14T10:20:06.857Z" }, - { url = "https://files.pythonhosted.org/packages/94/4d/d203dce8bee7faeca791671c88519969d98d3b4e8f225da5b96dad226fc8/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eae547b7315d055b0de2ec3965643b0ab82ad0106a7ffd29615ee9f266a02827", size = 1968728, upload-time = "2025-10-14T10:20:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/65/f5/6a66187775df87c24d526985b3a5d78d861580ca466fbd9d4d0e792fcf6c/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef9ee5471edd58d1fcce1c80ffc8783a650e3e3a193fe90d52e43bb4d87bff1f", size = 2050238, upload-time = "2025-10-14T10:20:09.766Z" }, - { url = "https://files.pythonhosted.org/packages/5e/b9/78336345de97298cf53236b2f271912ce11f32c1e59de25a374ce12f9cce/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:15dd504af121caaf2c95cb90c0ebf71603c53de98305621b94da0f967e572def", size = 2249424, upload-time = "2025-10-14T10:20:11.732Z" }, - { url = "https://files.pythonhosted.org/packages/99/bb/a4584888b70ee594c3d374a71af5075a68654d6c780369df269118af7402/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a926768ea49a8af4d36abd6a8968b8790f7f76dd7cbd5a4c180db2b4ac9a3a2", size = 2366047, upload-time = "2025-10-14T10:20:13.647Z" }, - { url = "https://files.pythonhosted.org/packages/5f/8d/17fc5de9d6418e4d2ae8c675f905cdafdc59d3bf3bf9c946b7ab796a992a/pydantic_core-2.41.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6916b9b7d134bff5440098a4deb80e4cb623e68974a87883299de9124126c2a8", size = 2071163, upload-time = "2025-10-14T10:20:15.307Z" }, - { url = "https://files.pythonhosted.org/packages/54/e7/03d2c5c0b8ed37a4617430db68ec5e7dbba66358b629cd69e11b4d564367/pydantic_core-2.41.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5cf90535979089df02e6f17ffd076f07237efa55b7343d98760bde8743c4b265", size = 2190585, upload-time = "2025-10-14T10:20:17.3Z" }, - { url = "https://files.pythonhosted.org/packages/be/fc/15d1c9fe5ad9266a5897d9b932b7f53d7e5cfc800573917a2c5d6eea56ec/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7533c76fa647fade2d7ec75ac5cc079ab3f34879626dae5689b27790a6cf5a5c", size = 2150109, upload-time = "2025-10-14T10:20:19.143Z" }, - { url = "https://files.pythonhosted.org/packages/26/ef/e735dd008808226c83ba56972566138665b71477ad580fa5a21f0851df48/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:37e516bca9264cbf29612539801ca3cd5d1be465f940417b002905e6ed79d38a", size = 2315078, upload-time = "2025-10-14T10:20:20.742Z" }, - { url = "https://files.pythonhosted.org/packages/90/00/806efdcf35ff2ac0f938362350cd9827b8afb116cc814b6b75cf23738c7c/pydantic_core-2.41.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c19cb355224037c83642429b8ce261ae108e1c5fbf5c028bac63c77b0f8646e", size = 2318737, upload-time = "2025-10-14T10:20:22.306Z" }, - { url = "https://files.pythonhosted.org/packages/41/7e/6ac90673fe6cb36621a2283552897838c020db343fa86e513d3f563b196f/pydantic_core-2.41.4-cp311-cp311-win32.whl", hash = "sha256:09c2a60e55b357284b5f31f5ab275ba9f7f70b7525e18a132ec1f9160b4f1f03", size = 1974160, upload-time = "2025-10-14T10:20:23.817Z" }, - { url = "https://files.pythonhosted.org/packages/e0/9d/7c5e24ee585c1f8b6356e1d11d40ab807ffde44d2db3b7dfd6d20b09720e/pydantic_core-2.41.4-cp311-cp311-win_amd64.whl", hash = "sha256:711156b6afb5cb1cb7c14a2cc2c4a8b4c717b69046f13c6b332d8a0a8f41ca3e", size = 2021883, upload-time = "2025-10-14T10:20:25.48Z" }, - { url = "https://files.pythonhosted.org/packages/33/90/5c172357460fc28b2871eb4a0fb3843b136b429c6fa827e4b588877bf115/pydantic_core-2.41.4-cp311-cp311-win_arm64.whl", hash = "sha256:6cb9cf7e761f4f8a8589a45e49ed3c0d92d1d696a45a6feaee8c904b26efc2db", size = 1968026, upload-time = "2025-10-14T10:20:27.039Z" }, - { url = "https://files.pythonhosted.org/packages/e9/81/d3b3e95929c4369d30b2a66a91db63c8ed0a98381ae55a45da2cd1cc1288/pydantic_core-2.41.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ab06d77e053d660a6faaf04894446df7b0a7e7aba70c2797465a0a1af00fc887", size = 2099043, upload-time = "2025-10-14T10:20:28.561Z" }, - { url = "https://files.pythonhosted.org/packages/58/da/46fdac49e6717e3a94fc9201403e08d9d61aa7a770fab6190b8740749047/pydantic_core-2.41.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c53ff33e603a9c1179a9364b0a24694f183717b2e0da2b5ad43c316c956901b2", size = 1910699, upload-time = "2025-10-14T10:20:30.217Z" }, - { url = "https://files.pythonhosted.org/packages/1e/63/4d948f1b9dd8e991a5a98b77dd66c74641f5f2e5225fee37994b2e07d391/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:304c54176af2c143bd181d82e77c15c41cbacea8872a2225dd37e6544dce9999", size = 1952121, upload-time = "2025-10-14T10:20:32.246Z" }, - { url = "https://files.pythonhosted.org/packages/b2/a7/e5fc60a6f781fc634ecaa9ecc3c20171d238794cef69ae0af79ac11b89d7/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:025ba34a4cf4fb32f917d5d188ab5e702223d3ba603be4d8aca2f82bede432a4", size = 2041590, upload-time = "2025-10-14T10:20:34.332Z" }, - { url = "https://files.pythonhosted.org/packages/70/69/dce747b1d21d59e85af433428978a1893c6f8a7068fa2bb4a927fba7a5ff/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9f5f30c402ed58f90c70e12eff65547d3ab74685ffe8283c719e6bead8ef53f", size = 2219869, upload-time = "2025-10-14T10:20:35.965Z" }, - { url = "https://files.pythonhosted.org/packages/83/6a/c070e30e295403bf29c4df1cb781317b6a9bac7cd07b8d3acc94d501a63c/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd96e5d15385d301733113bcaa324c8bcf111275b7675a9c6e88bfb19fc05e3b", size = 2345169, upload-time = "2025-10-14T10:20:37.627Z" }, - { url = "https://files.pythonhosted.org/packages/f0/83/06d001f8043c336baea7fd202a9ac7ad71f87e1c55d8112c50b745c40324/pydantic_core-2.41.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98f348cbb44fae6e9653c1055db7e29de67ea6a9ca03a5fa2c2e11a47cff0e47", size = 2070165, upload-time = "2025-10-14T10:20:39.246Z" }, - { url = "https://files.pythonhosted.org/packages/14/0a/e567c2883588dd12bcbc110232d892cf385356f7c8a9910311ac997ab715/pydantic_core-2.41.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec22626a2d14620a83ca583c6f5a4080fa3155282718b6055c2ea48d3ef35970", size = 2189067, upload-time = "2025-10-14T10:20:41.015Z" }, - { url = "https://files.pythonhosted.org/packages/f4/1d/3d9fca34273ba03c9b1c5289f7618bc4bd09c3ad2289b5420481aa051a99/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3a95d4590b1f1a43bf33ca6d647b990a88f4a3824a8c4572c708f0b45a5290ed", size = 2132997, upload-time = "2025-10-14T10:20:43.106Z" }, - { url = "https://files.pythonhosted.org/packages/52/70/d702ef7a6cd41a8afc61f3554922b3ed8d19dd54c3bd4bdbfe332e610827/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:f9672ab4d398e1b602feadcffcdd3af44d5f5e6ddc15bc7d15d376d47e8e19f8", size = 2307187, upload-time = "2025-10-14T10:20:44.849Z" }, - { url = "https://files.pythonhosted.org/packages/68/4c/c06be6e27545d08b802127914156f38d10ca287a9e8489342793de8aae3c/pydantic_core-2.41.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:84d8854db5f55fead3b579f04bda9a36461dab0730c5d570e1526483e7bb8431", size = 2305204, upload-time = "2025-10-14T10:20:46.781Z" }, - { url = "https://files.pythonhosted.org/packages/b0/e5/35ae4919bcd9f18603419e23c5eaf32750224a89d41a8df1a3704b69f77e/pydantic_core-2.41.4-cp312-cp312-win32.whl", hash = "sha256:9be1c01adb2ecc4e464392c36d17f97e9110fbbc906bcbe1c943b5b87a74aabd", size = 1972536, upload-time = "2025-10-14T10:20:48.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/49c5bb6d2a49eb2ee3647a93e3dae7080c6409a8a7558b075027644e879c/pydantic_core-2.41.4-cp312-cp312-win_amd64.whl", hash = "sha256:d682cf1d22bab22a5be08539dca3d1593488a99998f9f412137bc323179067ff", size = 2031132, upload-time = "2025-10-14T10:20:50.421Z" }, - { url = "https://files.pythonhosted.org/packages/06/23/936343dbcba6eec93f73e95eb346810fc732f71ba27967b287b66f7b7097/pydantic_core-2.41.4-cp312-cp312-win_arm64.whl", hash = "sha256:833eebfd75a26d17470b58768c1834dfc90141b7afc6eb0429c21fc5a21dcfb8", size = 1969483, upload-time = "2025-10-14T10:20:52.35Z" }, - { url = "https://files.pythonhosted.org/packages/13/d0/c20adabd181a029a970738dfe23710b52a31f1258f591874fcdec7359845/pydantic_core-2.41.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85e050ad9e5f6fe1004eec65c914332e52f429bc0ae12d6fa2092407a462c746", size = 2105688, upload-time = "2025-10-14T10:20:54.448Z" }, - { url = "https://files.pythonhosted.org/packages/00/b6/0ce5c03cec5ae94cca220dfecddc453c077d71363b98a4bbdb3c0b22c783/pydantic_core-2.41.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7393f1d64792763a48924ba31d1e44c2cfbc05e3b1c2c9abb4ceeadd912cced", size = 1910807, upload-time = "2025-10-14T10:20:56.115Z" }, - { url = "https://files.pythonhosted.org/packages/68/3e/800d3d02c8beb0b5c069c870cbb83799d085debf43499c897bb4b4aaff0d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94dab0940b0d1fb28bcab847adf887c66a27a40291eedf0b473be58761c9799a", size = 1956669, upload-time = "2025-10-14T10:20:57.874Z" }, - { url = "https://files.pythonhosted.org/packages/60/a4/24271cc71a17f64589be49ab8bd0751f6a0a03046c690df60989f2f95c2c/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:de7c42f897e689ee6f9e93c4bec72b99ae3b32a2ade1c7e4798e690ff5246e02", size = 2051629, upload-time = "2025-10-14T10:21:00.006Z" }, - { url = "https://files.pythonhosted.org/packages/68/de/45af3ca2f175d91b96bfb62e1f2d2f1f9f3b14a734afe0bfeff079f78181/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:664b3199193262277b8b3cd1e754fb07f2c6023289c815a1e1e8fb415cb247b1", size = 2224049, upload-time = "2025-10-14T10:21:01.801Z" }, - { url = "https://files.pythonhosted.org/packages/af/8f/ae4e1ff84672bf869d0a77af24fd78387850e9497753c432875066b5d622/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95b253b88f7d308b1c0b417c4624f44553ba4762816f94e6986819b9c273fb2", size = 2342409, upload-time = "2025-10-14T10:21:03.556Z" }, - { url = "https://files.pythonhosted.org/packages/18/62/273dd70b0026a085c7b74b000394e1ef95719ea579c76ea2f0cc8893736d/pydantic_core-2.41.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1351f5bbdbbabc689727cb91649a00cb9ee7203e0a6e54e9f5ba9e22e384b84", size = 2069635, upload-time = "2025-10-14T10:21:05.385Z" }, - { url = "https://files.pythonhosted.org/packages/30/03/cf485fff699b4cdaea469bc481719d3e49f023241b4abb656f8d422189fc/pydantic_core-2.41.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1affa4798520b148d7182da0615d648e752de4ab1a9566b7471bc803d88a062d", size = 2194284, upload-time = "2025-10-14T10:21:07.122Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7e/c8e713db32405dfd97211f2fc0a15d6bf8adb7640f3d18544c1f39526619/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7b74e18052fea4aa8dea2fb7dbc23d15439695da6cbe6cfc1b694af1115df09d", size = 2137566, upload-time = "2025-10-14T10:21:08.981Z" }, - { url = "https://files.pythonhosted.org/packages/04/f7/db71fd4cdccc8b75990f79ccafbbd66757e19f6d5ee724a6252414483fb4/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:285b643d75c0e30abda9dc1077395624f314a37e3c09ca402d4015ef5979f1a2", size = 2316809, upload-time = "2025-10-14T10:21:10.805Z" }, - { url = "https://files.pythonhosted.org/packages/76/63/a54973ddb945f1bca56742b48b144d85c9fc22f819ddeb9f861c249d5464/pydantic_core-2.41.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:f52679ff4218d713b3b33f88c89ccbf3a5c2c12ba665fb80ccc4192b4608dbab", size = 2311119, upload-time = "2025-10-14T10:21:12.583Z" }, - { url = "https://files.pythonhosted.org/packages/f8/03/5d12891e93c19218af74843a27e32b94922195ded2386f7b55382f904d2f/pydantic_core-2.41.4-cp313-cp313-win32.whl", hash = "sha256:ecde6dedd6fff127c273c76821bb754d793be1024bc33314a120f83a3c69460c", size = 1981398, upload-time = "2025-10-14T10:21:14.584Z" }, - { url = "https://files.pythonhosted.org/packages/be/d8/fd0de71f39db91135b7a26996160de71c073d8635edfce8b3c3681be0d6d/pydantic_core-2.41.4-cp313-cp313-win_amd64.whl", hash = "sha256:d081a1f3800f05409ed868ebb2d74ac39dd0c1ff6c035b5162356d76030736d4", size = 2030735, upload-time = "2025-10-14T10:21:16.432Z" }, - { url = "https://files.pythonhosted.org/packages/72/86/c99921c1cf6650023c08bfab6fe2d7057a5142628ef7ccfa9921f2dda1d5/pydantic_core-2.41.4-cp313-cp313-win_arm64.whl", hash = "sha256:f8e49c9c364a7edcbe2a310f12733aad95b022495ef2a8d653f645e5d20c1564", size = 1973209, upload-time = "2025-10-14T10:21:18.213Z" }, - { url = "https://files.pythonhosted.org/packages/36/0d/b5706cacb70a8414396efdda3d72ae0542e050b591119e458e2490baf035/pydantic_core-2.41.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ed97fd56a561f5eb5706cebe94f1ad7c13b84d98312a05546f2ad036bafe87f4", size = 1877324, upload-time = "2025-10-14T10:21:20.363Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/cba1fa02cfdea72dfb3a9babb067c83b9dff0bbcb198368e000a6b756ea7/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a870c307bf1ee91fc58a9a61338ff780d01bfae45922624816878dce784095d2", size = 1884515, upload-time = "2025-10-14T10:21:22.339Z" }, - { url = "https://files.pythonhosted.org/packages/07/ea/3df927c4384ed9b503c9cc2d076cf983b4f2adb0c754578dfb1245c51e46/pydantic_core-2.41.4-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d25e97bc1f5f8f7985bdc2335ef9e73843bb561eb1fa6831fdfc295c1c2061cf", size = 2042819, upload-time = "2025-10-14T10:21:26.683Z" }, - { url = "https://files.pythonhosted.org/packages/6a/ee/df8e871f07074250270a3b1b82aad4cd0026b588acd5d7d3eb2fcb1471a3/pydantic_core-2.41.4-cp313-cp313t-win_amd64.whl", hash = "sha256:d405d14bea042f166512add3091c1af40437c2e7f86988f3915fabd27b1e9cd2", size = 1995866, upload-time = "2025-10-14T10:21:28.951Z" }, - { url = "https://files.pythonhosted.org/packages/fc/de/b20f4ab954d6d399499c33ec4fafc46d9551e11dc1858fb7f5dca0748ceb/pydantic_core-2.41.4-cp313-cp313t-win_arm64.whl", hash = "sha256:19f3684868309db5263a11bace3c45d93f6f24afa2ffe75a647583df22a2ff89", size = 1970034, upload-time = "2025-10-14T10:21:30.869Z" }, - { url = "https://files.pythonhosted.org/packages/54/28/d3325da57d413b9819365546eb9a6e8b7cbd9373d9380efd5f74326143e6/pydantic_core-2.41.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e9205d97ed08a82ebb9a307e92914bb30e18cdf6f6b12ca4bedadb1588a0bfe1", size = 2102022, upload-time = "2025-10-14T10:21:32.809Z" }, - { url = "https://files.pythonhosted.org/packages/9e/24/b58a1bc0d834bf1acc4361e61233ee217169a42efbdc15a60296e13ce438/pydantic_core-2.41.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:82df1f432b37d832709fbcc0e24394bba04a01b6ecf1ee87578145c19cde12ac", size = 1905495, upload-time = "2025-10-14T10:21:34.812Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a4/71f759cc41b7043e8ecdaab81b985a9b6cad7cec077e0b92cff8b71ecf6b/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3b4cc4539e055cfa39a3763c939f9d409eb40e85813257dcd761985a108554", size = 1956131, upload-time = "2025-10-14T10:21:36.924Z" }, - { url = "https://files.pythonhosted.org/packages/b0/64/1e79ac7aa51f1eec7c4cda8cbe456d5d09f05fdd68b32776d72168d54275/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b1eb1754fce47c63d2ff57fdb88c351a6c0150995890088b33767a10218eaa4e", size = 2052236, upload-time = "2025-10-14T10:21:38.927Z" }, - { url = "https://files.pythonhosted.org/packages/e9/e3/a3ffc363bd4287b80f1d43dc1c28ba64831f8dfc237d6fec8f2661138d48/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e6ab5ab30ef325b443f379ddb575a34969c333004fca5a1daa0133a6ffaad616", size = 2223573, upload-time = "2025-10-14T10:21:41.574Z" }, - { url = "https://files.pythonhosted.org/packages/28/27/78814089b4d2e684a9088ede3790763c64693c3d1408ddc0a248bc789126/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:31a41030b1d9ca497634092b46481b937ff9397a86f9f51bd41c4767b6fc04af", size = 2342467, upload-time = "2025-10-14T10:21:44.018Z" }, - { url = "https://files.pythonhosted.org/packages/92/97/4de0e2a1159cb85ad737e03306717637842c88c7fd6d97973172fb183149/pydantic_core-2.41.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a44ac1738591472c3d020f61c6df1e4015180d6262ebd39bf2aeb52571b60f12", size = 2063754, upload-time = "2025-10-14T10:21:46.466Z" }, - { url = "https://files.pythonhosted.org/packages/0f/50/8cb90ce4b9efcf7ae78130afeb99fd1c86125ccdf9906ef64b9d42f37c25/pydantic_core-2.41.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d72f2b5e6e82ab8f94ea7d0d42f83c487dc159c5240d8f83beae684472864e2d", size = 2196754, upload-time = "2025-10-14T10:21:48.486Z" }, - { url = "https://files.pythonhosted.org/packages/34/3b/ccdc77af9cd5082723574a1cc1bcae7a6acacc829d7c0a06201f7886a109/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:c4d1e854aaf044487d31143f541f7aafe7b482ae72a022c664b2de2e466ed0ad", size = 2137115, upload-time = "2025-10-14T10:21:50.63Z" }, - { url = "https://files.pythonhosted.org/packages/ca/ba/e7c7a02651a8f7c52dc2cff2b64a30c313e3b57c7d93703cecea76c09b71/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b568af94267729d76e6ee5ececda4e283d07bbb28e8148bb17adad93d025d25a", size = 2317400, upload-time = "2025-10-14T10:21:52.959Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ba/6c533a4ee8aec6b812c643c49bb3bd88d3f01e3cebe451bb85512d37f00f/pydantic_core-2.41.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6d55fb8b1e8929b341cc313a81a26e0d48aa3b519c1dbaadec3a6a2b4fcad025", size = 2312070, upload-time = "2025-10-14T10:21:55.419Z" }, - { url = "https://files.pythonhosted.org/packages/22/ae/f10524fcc0ab8d7f96cf9a74c880243576fd3e72bd8ce4f81e43d22bcab7/pydantic_core-2.41.4-cp314-cp314-win32.whl", hash = "sha256:5b66584e549e2e32a1398df11da2e0a7eff45d5c2d9db9d5667c5e6ac764d77e", size = 1982277, upload-time = "2025-10-14T10:21:57.474Z" }, - { url = "https://files.pythonhosted.org/packages/b4/dc/e5aa27aea1ad4638f0c3fb41132f7eb583bd7420ee63204e2d4333a3bbf9/pydantic_core-2.41.4-cp314-cp314-win_amd64.whl", hash = "sha256:557a0aab88664cc552285316809cab897716a372afaf8efdbef756f8b890e894", size = 2024608, upload-time = "2025-10-14T10:21:59.557Z" }, - { url = "https://files.pythonhosted.org/packages/3e/61/51d89cc2612bd147198e120a13f150afbf0bcb4615cddb049ab10b81b79e/pydantic_core-2.41.4-cp314-cp314-win_arm64.whl", hash = "sha256:3f1ea6f48a045745d0d9f325989d8abd3f1eaf47dd00485912d1a3a63c623a8d", size = 1967614, upload-time = "2025-10-14T10:22:01.847Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c2/472f2e31b95eff099961fa050c376ab7156a81da194f9edb9f710f68787b/pydantic_core-2.41.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6c1fe4c5404c448b13188dd8bd2ebc2bdd7e6727fa61ff481bcc2cca894018da", size = 1876904, upload-time = "2025-10-14T10:22:04.062Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/ea8eeb91173807ecdae4f4a5f4b150a520085b35454350fc219ba79e66a3/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:523e7da4d43b113bf8e7b49fa4ec0c35bf4fe66b2230bfc5c13cc498f12c6c3e", size = 1882538, upload-time = "2025-10-14T10:22:06.39Z" }, - { url = "https://files.pythonhosted.org/packages/1e/29/b53a9ca6cd366bfc928823679c6a76c7a4c69f8201c0ba7903ad18ebae2f/pydantic_core-2.41.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5729225de81fb65b70fdb1907fcf08c75d498f4a6f15af005aabb1fdadc19dfa", size = 2041183, upload-time = "2025-10-14T10:22:08.812Z" }, - { url = "https://files.pythonhosted.org/packages/c7/3d/f8c1a371ceebcaf94d6dd2d77c6cf4b1c078e13a5837aee83f760b4f7cfd/pydantic_core-2.41.4-cp314-cp314t-win_amd64.whl", hash = "sha256:de2cfbb09e88f0f795fd90cf955858fc2c691df65b1f21f0aa00b99f3fbc661d", size = 1993542, upload-time = "2025-10-14T10:22:11.332Z" }, - { url = "https://files.pythonhosted.org/packages/8a/ac/9fc61b4f9d079482a290afe8d206b8f490e9fd32d4fc03ed4fc698214e01/pydantic_core-2.41.4-cp314-cp314t-win_arm64.whl", hash = "sha256:d34f950ae05a83e0ede899c595f312ca976023ea1db100cd5aa188f7005e3ab0", size = 1973897, upload-time = "2025-10-14T10:22:13.444Z" }, - { url = "https://files.pythonhosted.org/packages/b0/12/5ba58daa7f453454464f92b3ca7b9d7c657d8641c48e370c3ebc9a82dd78/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:a1b2cfec3879afb742a7b0bcfa53e4f22ba96571c9e54d6a3afe1052d17d843b", size = 2122139, upload-time = "2025-10-14T10:22:47.288Z" }, - { url = "https://files.pythonhosted.org/packages/21/fb/6860126a77725c3108baecd10fd3d75fec25191d6381b6eb2ac660228eac/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:d175600d975b7c244af6eb9c9041f10059f20b8bbffec9e33fdd5ee3f67cdc42", size = 1936674, upload-time = "2025-10-14T10:22:49.555Z" }, - { url = "https://files.pythonhosted.org/packages/de/be/57dcaa3ed595d81f8757e2b44a38240ac5d37628bce25fb20d02c7018776/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f184d657fa4947ae5ec9c47bd7e917730fa1cbb78195037e32dcbab50aca5ee", size = 1956398, upload-time = "2025-10-14T10:22:52.19Z" }, - { url = "https://files.pythonhosted.org/packages/2f/1d/679a344fadb9695f1a6a294d739fbd21d71fa023286daeea8c0ed49e7c2b/pydantic_core-2.41.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed810568aeffed3edc78910af32af911c835cc39ebbfacd1f0ab5dd53028e5c", size = 2138674, upload-time = "2025-10-14T10:22:54.499Z" }, - { url = "https://files.pythonhosted.org/packages/c4/48/ae937e5a831b7c0dc646b2ef788c27cd003894882415300ed21927c21efa/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:4f5d640aeebb438517150fdeec097739614421900e4a08db4a3ef38898798537", size = 2112087, upload-time = "2025-10-14T10:22:56.818Z" }, - { url = "https://files.pythonhosted.org/packages/5e/db/6db8073e3d32dae017da7e0d16a9ecb897d0a4d92e00634916e486097961/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:4a9ab037b71927babc6d9e7fc01aea9e66dc2a4a34dff06ef0724a4049629f94", size = 1920387, upload-time = "2025-10-14T10:22:59.342Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c1/dd3542d072fcc336030d66834872f0328727e3b8de289c662faa04aa270e/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4dab9484ec605c3016df9ad4fd4f9a390bc5d816a3b10c6550f8424bb80b18c", size = 1951495, upload-time = "2025-10-14T10:23:02.089Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c6/db8d13a1f8ab3f1eb08c88bd00fd62d44311e3456d1e85c0e59e0a0376e7/pydantic_core-2.41.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd8a5028425820731d8c6c098ab642d7b8b999758e24acae03ed38a66eca8335", size = 2139008, upload-time = "2025-10-14T10:23:04.539Z" }, - { url = "https://files.pythonhosted.org/packages/5d/d4/912e976a2dd0b49f31c98a060ca90b353f3b73ee3ea2fd0030412f6ac5ec/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e5ab4fc177dd41536b3c32b2ea11380dd3d4619a385860621478ac2d25ceb00", size = 2106739, upload-time = "2025-10-14T10:23:06.934Z" }, - { url = "https://files.pythonhosted.org/packages/71/f0/66ec5a626c81eba326072d6ee2b127f8c139543f1bf609b4842978d37833/pydantic_core-2.41.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:3d88d0054d3fa11ce936184896bed3c1c5441d6fa483b498fac6a5d0dd6f64a9", size = 1932549, upload-time = "2025-10-14T10:23:09.24Z" }, - { url = "https://files.pythonhosted.org/packages/c4/af/625626278ca801ea0a658c2dcf290dc9f21bb383098e99e7c6a029fccfc0/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b2a054a8725f05b4b6503357e0ac1c4e8234ad3b0c2ac130d6ffc66f0e170e2", size = 2135093, upload-time = "2025-10-14T10:23:11.626Z" }, - { url = "https://files.pythonhosted.org/packages/20/f6/2fba049f54e0f4975fef66be654c597a1d005320fa141863699180c7697d/pydantic_core-2.41.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0d9db5a161c99375a0c68c058e227bee1d89303300802601d76a3d01f74e258", size = 2187971, upload-time = "2025-10-14T10:23:14.437Z" }, - { url = "https://files.pythonhosted.org/packages/0e/80/65ab839a2dfcd3b949202f9d920c34f9de5a537c3646662bdf2f7d999680/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6273ea2c8ffdac7b7fda2653c49682db815aebf4a89243a6feccf5e36c18c347", size = 2147939, upload-time = "2025-10-14T10:23:16.831Z" }, - { url = "https://files.pythonhosted.org/packages/44/58/627565d3d182ce6dfda18b8e1c841eede3629d59c9d7cbc1e12a03aeb328/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:4c973add636efc61de22530b2ef83a65f39b6d6f656df97f678720e20de26caa", size = 2311400, upload-time = "2025-10-14T10:23:19.234Z" }, - { url = "https://files.pythonhosted.org/packages/24/06/8a84711162ad5a5f19a88cead37cca81b4b1f294f46260ef7334ae4f24d3/pydantic_core-2.41.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b69d1973354758007f46cf2d44a4f3d0933f10b6dc9bf15cf1356e037f6f731a", size = 2316840, upload-time = "2025-10-14T10:23:21.738Z" }, - { url = "https://files.pythonhosted.org/packages/aa/8b/b7bb512a4682a2f7fbfae152a755d37351743900226d29bd953aaf870eaa/pydantic_core-2.41.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:3619320641fd212aaf5997b6ca505e97540b7e16418f4a241f44cdf108ffb50d", size = 2149135, upload-time = "2025-10-14T10:23:24.379Z" }, - { url = "https://files.pythonhosted.org/packages/7e/7d/138e902ed6399b866f7cfe4435d22445e16fff888a1c00560d9dc79a780f/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:491535d45cd7ad7e4a2af4a5169b0d07bebf1adfd164b0368da8aa41e19907a5", size = 2104721, upload-time = "2025-10-14T10:23:26.906Z" }, - { url = "https://files.pythonhosted.org/packages/47/13/0525623cf94627f7b53b4c2034c81edc8491cbfc7c28d5447fa318791479/pydantic_core-2.41.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:54d86c0cada6aba4ec4c047d0e348cbad7063b87ae0f005d9f8c9ad04d4a92a2", size = 1931608, upload-time = "2025-10-14T10:23:29.306Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f9/744bc98137d6ef0a233f808bfc9b18cf94624bf30836a18d3b05d08bf418/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca1124aced216b2500dc2609eade086d718e8249cb9696660ab447d50a758bd", size = 2132986, upload-time = "2025-10-14T10:23:32.057Z" }, - { url = "https://files.pythonhosted.org/packages/17/c8/629e88920171173f6049386cc71f893dff03209a9ef32b4d2f7e7c264bcf/pydantic_core-2.41.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6c9024169becccf0cb470ada03ee578d7348c119a0d42af3dcf9eda96e3a247c", size = 2187516, upload-time = "2025-10-14T10:23:34.871Z" }, - { url = "https://files.pythonhosted.org/packages/2e/0f/4f2734688d98488782218ca61bcc118329bf5de05bb7fe3adc7dd79b0b86/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:26895a4268ae5a2849269f4991cdc97236e4b9c010e51137becf25182daac405", size = 2146146, upload-time = "2025-10-14T10:23:37.342Z" }, - { url = "https://files.pythonhosted.org/packages/ed/f2/ab385dbd94a052c62224b99cf99002eee99dbec40e10006c78575aead256/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:ca4df25762cf71308c446e33c9b1fdca2923a3f13de616e2a949f38bf21ff5a8", size = 2311296, upload-time = "2025-10-14T10:23:40.145Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8e/e4f12afe1beeb9823bba5375f8f258df0cc61b056b0195fb1cf9f62a1a58/pydantic_core-2.41.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:5a28fcedd762349519276c36634e71853b4541079cab4acaaac60c4421827308", size = 2315386, upload-time = "2025-10-14T10:23:42.624Z" }, - { url = "https://files.pythonhosted.org/packages/48/f7/925f65d930802e3ea2eb4d5afa4cb8730c8dc0d2cb89a59dc4ed2fcb2d74/pydantic_core-2.41.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c173ddcd86afd2535e2b695217e82191580663a1d1928239f877f5a1649ef39f", size = 2147775, upload-time = "2025-10-14T10:23:45.406Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, + { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, + { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, + { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, + { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, + { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, + { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, + { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, + { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, + { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, + { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, + { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, + { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, + { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, + { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, + { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, + { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, + { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, + { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, + { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, + { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, + { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, + { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, + { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, + { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, + { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, + { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, + { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, + { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, + { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, + { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, + { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, + { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, + { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, + { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, + { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, + { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, + { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, + { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, + { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, + { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, + { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, + { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, + { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, + { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, + { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, + { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, + { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] [[package]] @@ -4159,7 +4797,7 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -4168,109 +4806,109 @@ wheels = [ [[package]] name = "regex" -version = "2025.10.23" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/c8/1d2160d36b11fbe0a61acb7c3c81ab032d9ec8ad888ac9e0a61b85ab99dd/regex-2025.10.23.tar.gz", hash = "sha256:8cbaf8ceb88f96ae2356d01b9adf5e6306fa42fa6f7eab6b97794e37c959ac26", size = 401266, upload-time = "2025-10-21T15:58:20.23Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/11/849d5d23633a77047465eaae4cc0cbf24ded7aa496c02e8b9710e28b1687/regex-2025.10.23-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:17bbcde374bef1c5fad9b131f0e28a6a24856dd90368d8c0201e2b5a69533daa", size = 487957, upload-time = "2025-10-21T15:54:26.151Z" }, - { url = "https://files.pythonhosted.org/packages/87/12/5985386e7e3200a0d6a6417026d2c758d783a932428a5efc0a42ca1ddf74/regex-2025.10.23-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b4e10434279cc8567f99ca6e018e9025d14f2fded2a603380b6be2090f476426", size = 290419, upload-time = "2025-10-21T15:54:28.804Z" }, - { url = "https://files.pythonhosted.org/packages/67/cf/a8615923f962f8fdc41a3a6093a48726955e8b1993f4614b26a41d249f9b/regex-2025.10.23-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c9bb421cbe7012c744a5a56cf4d6c80829c72edb1a2991677299c988d6339c8", size = 288285, upload-time = "2025-10-21T15:54:30.47Z" }, - { url = "https://files.pythonhosted.org/packages/4e/3d/6a3a1e12c86354cd0b3cbf8c3dd6acbe853609ee3b39d47ecd3ce95caf84/regex-2025.10.23-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:275cd1c2ed8c4a78ebfa489618d7aee762e8b4732da73573c3e38236ec5f65de", size = 781458, upload-time = "2025-10-21T15:54:31.978Z" }, - { url = "https://files.pythonhosted.org/packages/46/47/76a8da004489f2700361754859e373b87a53d043de8c47f4d1583fd39d78/regex-2025.10.23-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7b426ae7952f3dc1e73a86056d520bd4e5f021397484a6835902fc5648bcacce", size = 850605, upload-time = "2025-10-21T15:54:33.753Z" }, - { url = "https://files.pythonhosted.org/packages/67/05/fa886461f97d45a6f4b209699cb994dc6d6212d6e219d29444dac5005775/regex-2025.10.23-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c5cdaf5b6d37c7da1967dbe729d819461aab6a98a072feef65bbcff0a6e60649", size = 898563, upload-time = "2025-10-21T15:54:35.431Z" }, - { url = "https://files.pythonhosted.org/packages/2d/db/3ddd8d01455f23cabad7499f4199de0df92f5e96d39633203ff9d0b592dc/regex-2025.10.23-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3bfeff0b08f296ab28b4332a7e03ca31c437ee78b541ebc874bbf540e5932f8d", size = 791535, upload-time = "2025-10-21T15:54:37.269Z" }, - { url = "https://files.pythonhosted.org/packages/7c/ae/0fa5cbf41ca92b6ec3370222fcb6c68b240d68ab10e803d086c03a19fd9e/regex-2025.10.23-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f97236a67307b775f30a74ef722b64b38b7ab7ba3bb4a2508518a5de545459c", size = 782461, upload-time = "2025-10-21T15:54:39.187Z" }, - { url = "https://files.pythonhosted.org/packages/d4/23/70af22a016df11af4def27870eb175c2c7235b72d411ecf75a4b4a422cb6/regex-2025.10.23-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:be19e7de499940cd72475fb8e46ab2ecb1cf5906bebdd18a89f9329afb1df82f", size = 774583, upload-time = "2025-10-21T15:54:41.018Z" }, - { url = "https://files.pythonhosted.org/packages/7a/ee/a54a6851f6905f33d3c4ed64e8737b1d85ed01b5724712530ddc0f9abdb1/regex-2025.10.23-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:883df76ee42d9ecb82b37ff8d01caea5895b3f49630a64d21111078bbf8ef64c", size = 845649, upload-time = "2025-10-21T15:54:42.615Z" }, - { url = "https://files.pythonhosted.org/packages/80/7d/c3ec1cae14e01fab00e38c41ed35f47a853359e95e9c023e9a4381bb122c/regex-2025.10.23-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2e9117d1d35fc2addae6281019ecc70dc21c30014b0004f657558b91c6a8f1a7", size = 836037, upload-time = "2025-10-21T15:54:44.63Z" }, - { url = "https://files.pythonhosted.org/packages/15/ae/45771140dd43c4d67c87b54d3728078ed6a96599d9fc7ba6825086236782/regex-2025.10.23-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0ff1307f531a5d8cf5c20ea517254551ff0a8dc722193aab66c656c5a900ea68", size = 779705, upload-time = "2025-10-21T15:54:46.08Z" }, - { url = "https://files.pythonhosted.org/packages/b8/95/074e2581760eafce7c816a352b7d3a322536e5b68c346d1a8bacd895545c/regex-2025.10.23-cp310-cp310-win32.whl", hash = "sha256:7888475787cbfee4a7cd32998eeffe9a28129fa44ae0f691b96cb3939183ef41", size = 265663, upload-time = "2025-10-21T15:54:47.854Z" }, - { url = "https://files.pythonhosted.org/packages/f7/c7/a25f56a718847e34d3f1608c72eadeb67653bff1a0411da023dd8f4c647b/regex-2025.10.23-cp310-cp310-win_amd64.whl", hash = "sha256:ec41a905908496ce4906dab20fb103c814558db1d69afc12c2f384549c17936a", size = 277587, upload-time = "2025-10-21T15:54:49.571Z" }, - { url = "https://files.pythonhosted.org/packages/d3/e5/63eb17c6b5deaefd93c2bbb1feae7c0a8d2157da25883a6ca2569cf7a663/regex-2025.10.23-cp310-cp310-win_arm64.whl", hash = "sha256:b2b7f19a764d5e966d5a62bf2c28a8b4093cc864c6734510bdb4aeb840aec5e6", size = 269979, upload-time = "2025-10-21T15:54:51.375Z" }, - { url = "https://files.pythonhosted.org/packages/82/e5/74b7cd5cd76b4171f9793042045bb1726f7856dd56e582fc3e058a7a8a5e/regex-2025.10.23-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6c531155bf9179345e85032052a1e5fe1a696a6abf9cea54b97e8baefff970fd", size = 487960, upload-time = "2025-10-21T15:54:53.253Z" }, - { url = "https://files.pythonhosted.org/packages/b9/08/854fa4b3b20471d1df1c71e831b6a1aa480281e37791e52a2df9641ec5c6/regex-2025.10.23-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:912e9df4e89d383681268d38ad8f5780d7cccd94ba0e9aa09ca7ab7ab4f8e7eb", size = 290425, upload-time = "2025-10-21T15:54:55.21Z" }, - { url = "https://files.pythonhosted.org/packages/ab/d3/6272b1dd3ca1271661e168762b234ad3e00dbdf4ef0c7b9b72d2d159efa7/regex-2025.10.23-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f375c61bfc3138b13e762fe0ae76e3bdca92497816936534a0177201666f44f", size = 288278, upload-time = "2025-10-21T15:54:56.862Z" }, - { url = "https://files.pythonhosted.org/packages/14/8f/c7b365dd9d9bc0a36e018cb96f2ffb60d2ba8deb589a712b437f67de2920/regex-2025.10.23-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e248cc9446081119128ed002a3801f8031e0c219b5d3c64d3cc627da29ac0a33", size = 793289, upload-time = "2025-10-21T15:54:58.352Z" }, - { url = "https://files.pythonhosted.org/packages/d4/fb/b8fbe9aa16cf0c21f45ec5a6c74b4cecbf1a1c0deb7089d4a6f83a9c1caa/regex-2025.10.23-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b52bf9282fdf401e4f4e721f0f61fc4b159b1307244517789702407dd74e38ca", size = 860321, upload-time = "2025-10-21T15:54:59.813Z" }, - { url = "https://files.pythonhosted.org/packages/b0/81/bf41405c772324926a9bd8a640dedaa42da0e929241834dfce0733070437/regex-2025.10.23-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5c084889ab2c59765a0d5ac602fd1c3c244f9b3fcc9a65fdc7ba6b74c5287490", size = 907011, upload-time = "2025-10-21T15:55:01.968Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fb/5ad6a8b92d3f88f3797b51bb4ef47499acc2d0b53d2fbe4487a892f37a73/regex-2025.10.23-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d80e8eb79009bdb0936658c44ca06e2fbbca67792013e3818eea3f5f228971c2", size = 800312, upload-time = "2025-10-21T15:55:04.15Z" }, - { url = "https://files.pythonhosted.org/packages/42/48/b4efba0168a2b57f944205d823f8e8a3a1ae6211a34508f014ec2c712f4f/regex-2025.10.23-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6f259118ba87b814a8ec475380aee5f5ae97a75852a3507cf31d055b01b5b40", size = 782839, upload-time = "2025-10-21T15:55:05.641Z" }, - { url = "https://files.pythonhosted.org/packages/13/2a/c9efb4c6c535b0559c1fa8e431e0574d229707c9ca718600366fcfef6801/regex-2025.10.23-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9b8c72a242683dcc72d37595c4f1278dfd7642b769e46700a8df11eab19dfd82", size = 854270, upload-time = "2025-10-21T15:55:07.27Z" }, - { url = "https://files.pythonhosted.org/packages/34/2d/68eecc1bdaee020e8ba549502291c9450d90d8590d0552247c9b543ebf7b/regex-2025.10.23-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a8d7b7a0a3df9952f9965342159e0c1f05384c0f056a47ce8b61034f8cecbe83", size = 845771, upload-time = "2025-10-21T15:55:09.477Z" }, - { url = "https://files.pythonhosted.org/packages/a5/cd/a1ae499cf9b87afb47a67316bbf1037a7c681ffe447c510ed98c0aa2c01c/regex-2025.10.23-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:413bfea20a484c524858125e92b9ce6ffdd0a4b97d4ff96b5859aa119b0f1bdd", size = 788778, upload-time = "2025-10-21T15:55:11.396Z" }, - { url = "https://files.pythonhosted.org/packages/38/f9/70765e63f5ea7d43b2b6cd4ee9d3323f16267e530fb2a420d92d991cf0fc/regex-2025.10.23-cp311-cp311-win32.whl", hash = "sha256:f76deef1f1019a17dad98f408b8f7afc4bd007cbe835ae77b737e8c7f19ae575", size = 265666, upload-time = "2025-10-21T15:55:13.306Z" }, - { url = "https://files.pythonhosted.org/packages/9c/1a/18e9476ee1b63aaec3844d8e1cb21842dc19272c7e86d879bfc0dcc60db3/regex-2025.10.23-cp311-cp311-win_amd64.whl", hash = "sha256:59bba9f7125536f23fdab5deeea08da0c287a64c1d3acc1c7e99515809824de8", size = 277600, upload-time = "2025-10-21T15:55:15.087Z" }, - { url = "https://files.pythonhosted.org/packages/1d/1b/c019167b1f7a8ec77251457e3ff0339ed74ca8bce1ea13138dc98309c923/regex-2025.10.23-cp311-cp311-win_arm64.whl", hash = "sha256:b103a752b6f1632ca420225718d6ed83f6a6ced3016dd0a4ab9a6825312de566", size = 269974, upload-time = "2025-10-21T15:55:16.841Z" }, - { url = "https://files.pythonhosted.org/packages/f6/57/eeb274d83ab189d02d778851b1ac478477522a92b52edfa6e2ae9ff84679/regex-2025.10.23-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7a44d9c00f7a0a02d3b777429281376370f3d13d2c75ae74eb94e11ebcf4a7fc", size = 489187, upload-time = "2025-10-21T15:55:18.322Z" }, - { url = "https://files.pythonhosted.org/packages/55/5c/7dad43a9b6ea88bf77e0b8b7729a4c36978e1043165034212fd2702880c6/regex-2025.10.23-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b83601f84fde939ae3478bb32a3aef36f61b58c3208d825c7e8ce1a735f143f2", size = 291122, upload-time = "2025-10-21T15:55:20.2Z" }, - { url = "https://files.pythonhosted.org/packages/66/21/38b71e6f2818f0f4b281c8fba8d9d57cfca7b032a648fa59696e0a54376a/regex-2025.10.23-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ec13647907bb9d15fd192bbfe89ff06612e098a5709e7d6ecabbdd8f7908fc45", size = 288797, upload-time = "2025-10-21T15:55:21.932Z" }, - { url = "https://files.pythonhosted.org/packages/be/95/888f069c89e7729732a6d7cca37f76b44bfb53a1e35dda8a2c7b65c1b992/regex-2025.10.23-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78d76dd2957d62501084e7012ddafc5fcd406dd982b7a9ca1ea76e8eaaf73e7e", size = 798442, upload-time = "2025-10-21T15:55:23.747Z" }, - { url = "https://files.pythonhosted.org/packages/76/70/4f903c608faf786627a8ee17c06e0067b5acade473678b69c8094b248705/regex-2025.10.23-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8668e5f067e31a47699ebb354f43aeb9c0ef136f915bd864243098524482ac43", size = 864039, upload-time = "2025-10-21T15:55:25.656Z" }, - { url = "https://files.pythonhosted.org/packages/62/19/2df67b526bf25756c7f447dde554fc10a220fd839cc642f50857d01e4a7b/regex-2025.10.23-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a32433fe3deb4b2d8eda88790d2808fed0dc097e84f5e683b4cd4f42edef6cca", size = 912057, upload-time = "2025-10-21T15:55:27.309Z" }, - { url = "https://files.pythonhosted.org/packages/99/14/9a39b7c9e007968411bc3c843cc14cf15437510c0a9991f080cab654fd16/regex-2025.10.23-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d97d73818c642c938db14c0668167f8d39520ca9d983604575ade3fda193afcc", size = 803374, upload-time = "2025-10-21T15:55:28.9Z" }, - { url = "https://files.pythonhosted.org/packages/d4/f7/3495151dd3ca79949599b6d069b72a61a2c5e24fc441dccc79dcaf708fe6/regex-2025.10.23-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bca7feecc72ee33579e9f6ddf8babbe473045717a0e7dbc347099530f96e8b9a", size = 787714, upload-time = "2025-10-21T15:55:30.628Z" }, - { url = "https://files.pythonhosted.org/packages/28/65/ee882455e051131869957ee8597faea45188c9a98c0dad724cfb302d4580/regex-2025.10.23-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7e24af51e907d7457cc4a72691ec458320b9ae67dc492f63209f01eecb09de32", size = 858392, upload-time = "2025-10-21T15:55:32.322Z" }, - { url = "https://files.pythonhosted.org/packages/53/25/9287fef5be97529ebd3ac79d256159cb709a07eb58d4be780d1ca3885da8/regex-2025.10.23-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d10bcde58bbdf18146f3a69ec46dd03233b94a4a5632af97aa5378da3a47d288", size = 850484, upload-time = "2025-10-21T15:55:34.037Z" }, - { url = "https://files.pythonhosted.org/packages/f3/b4/b49b88b4fea2f14dc73e5b5842755e782fc2e52f74423d6f4adc130d5880/regex-2025.10.23-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:44383bc0c933388516c2692c9a7503e1f4a67e982f20b9a29d2fb70c6494f147", size = 789634, upload-time = "2025-10-21T15:55:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/b6/3c/2f8d199d0e84e78bcd6bdc2be9b62410624f6b796e2893d1837ae738b160/regex-2025.10.23-cp312-cp312-win32.whl", hash = "sha256:6040a86f95438a0114bba16e51dfe27f1bc004fd29fe725f54a586f6d522b079", size = 266060, upload-time = "2025-10-21T15:55:37.902Z" }, - { url = "https://files.pythonhosted.org/packages/d7/67/c35e80969f6ded306ad70b0698863310bdf36aca57ad792f45ddc0e2271f/regex-2025.10.23-cp312-cp312-win_amd64.whl", hash = "sha256:436b4c4352fe0762e3bfa34a5567079baa2ef22aa9c37cf4d128979ccfcad842", size = 276931, upload-time = "2025-10-21T15:55:39.502Z" }, - { url = "https://files.pythonhosted.org/packages/f5/a1/4ed147de7d2b60174f758412c87fa51ada15cd3296a0ff047f4280aaa7ca/regex-2025.10.23-cp312-cp312-win_arm64.whl", hash = "sha256:f4b1b1991617055b46aff6f6db24888c1f05f4db9801349d23f09ed0714a9335", size = 270103, upload-time = "2025-10-21T15:55:41.24Z" }, - { url = "https://files.pythonhosted.org/packages/28/c6/195a6217a43719d5a6a12cc192a22d12c40290cecfa577f00f4fb822f07d/regex-2025.10.23-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:b7690f95404a1293923a296981fd943cca12c31a41af9c21ba3edd06398fc193", size = 488956, upload-time = "2025-10-21T15:55:42.887Z" }, - { url = "https://files.pythonhosted.org/packages/4c/93/181070cd1aa2fa541ff2d3afcf763ceecd4937b34c615fa92765020a6c90/regex-2025.10.23-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a32d77aeaea58a13230100dd8797ac1a84c457f3af2fdf0d81ea689d5a9105b", size = 290997, upload-time = "2025-10-21T15:55:44.53Z" }, - { url = "https://files.pythonhosted.org/packages/b6/c5/9d37fbe3a40ed8dda78c23e1263002497540c0d1522ed75482ef6c2000f0/regex-2025.10.23-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b24b29402f264f70a3c81f45974323b41764ff7159655360543b7cabb73e7d2f", size = 288686, upload-time = "2025-10-21T15:55:46.186Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e7/db610ff9f10c2921f9b6ac0c8d8be4681b28ddd40fc0549429366967e61f/regex-2025.10.23-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:563824a08c7c03d96856d84b46fdb3bbb7cfbdf79da7ef68725cda2ce169c72a", size = 798466, upload-time = "2025-10-21T15:55:48.24Z" }, - { url = "https://files.pythonhosted.org/packages/90/10/aab883e1fa7fe2feb15ac663026e70ca0ae1411efa0c7a4a0342d9545015/regex-2025.10.23-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0ec8bdd88d2e2659c3518087ee34b37e20bd169419ffead4240a7004e8ed03b", size = 863996, upload-time = "2025-10-21T15:55:50.478Z" }, - { url = "https://files.pythonhosted.org/packages/a2/b0/8f686dd97a51f3b37d0238cd00a6d0f9ccabe701f05b56de1918571d0d61/regex-2025.10.23-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b577601bfe1d33913fcd9276d7607bbac827c4798d9e14d04bf37d417a6c41cb", size = 912145, upload-time = "2025-10-21T15:55:52.215Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ca/639f8cd5b08797bca38fc5e7e07f76641a428cf8c7fca05894caf045aa32/regex-2025.10.23-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c9f2c68ac6cb3de94eea08a437a75eaa2bd33f9e97c84836ca0b610a5804368", size = 803370, upload-time = "2025-10-21T15:55:53.944Z" }, - { url = "https://files.pythonhosted.org/packages/0d/1e/a40725bb76959eddf8abc42a967bed6f4851b39f5ac4f20e9794d7832aa5/regex-2025.10.23-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:89f8b9ea3830c79468e26b0e21c3585f69f105157c2154a36f6b7839f8afb351", size = 787767, upload-time = "2025-10-21T15:55:56.004Z" }, - { url = "https://files.pythonhosted.org/packages/3d/d8/8ee9858062936b0f99656dce390aa667c6e7fb0c357b1b9bf76fb5e2e708/regex-2025.10.23-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:98fd84c4e4ea185b3bb5bf065261ab45867d8875032f358a435647285c722673", size = 858335, upload-time = "2025-10-21T15:55:58.185Z" }, - { url = "https://files.pythonhosted.org/packages/d8/0a/ed5faaa63fa8e3064ab670e08061fbf09e3a10235b19630cf0cbb9e48c0a/regex-2025.10.23-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:1e11d3e5887b8b096f96b4154dfb902f29c723a9556639586cd140e77e28b313", size = 850402, upload-time = "2025-10-21T15:56:00.023Z" }, - { url = "https://files.pythonhosted.org/packages/79/14/d05f617342f4b2b4a23561da500ca2beab062bfcc408d60680e77ecaf04d/regex-2025.10.23-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f13450328a6634348d47a88367e06b64c9d84980ef6a748f717b13f8ce64e87", size = 789739, upload-time = "2025-10-21T15:56:01.967Z" }, - { url = "https://files.pythonhosted.org/packages/f9/7b/e8ce8eef42a15f2c3461f8b3e6e924bbc86e9605cb534a393aadc8d3aff8/regex-2025.10.23-cp313-cp313-win32.whl", hash = "sha256:37be9296598a30c6a20236248cb8b2c07ffd54d095b75d3a2a2ee5babdc51df1", size = 266054, upload-time = "2025-10-21T15:56:05.291Z" }, - { url = "https://files.pythonhosted.org/packages/71/2d/55184ed6be6473187868d2f2e6a0708195fc58270e62a22cbf26028f2570/regex-2025.10.23-cp313-cp313-win_amd64.whl", hash = "sha256:ea7a3c283ce0f06fe789365841e9174ba05f8db16e2fd6ae00a02df9572c04c0", size = 276917, upload-time = "2025-10-21T15:56:07.303Z" }, - { url = "https://files.pythonhosted.org/packages/9c/d4/927eced0e2bd45c45839e556f987f8c8f8683268dd3c00ad327deb3b0172/regex-2025.10.23-cp313-cp313-win_arm64.whl", hash = "sha256:d9a4953575f300a7bab71afa4cd4ac061c7697c89590a2902b536783eeb49a4f", size = 270105, upload-time = "2025-10-21T15:56:09.857Z" }, - { url = "https://files.pythonhosted.org/packages/3e/b3/95b310605285573341fc062d1d30b19a54f857530e86c805f942c4ff7941/regex-2025.10.23-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7d6606524fa77b3912c9ef52a42ef63c6cfbfc1077e9dc6296cd5da0da286044", size = 491850, upload-time = "2025-10-21T15:56:11.685Z" }, - { url = "https://files.pythonhosted.org/packages/a4/8f/207c2cec01e34e56db1eff606eef46644a60cf1739ecd474627db90ad90b/regex-2025.10.23-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c037aadf4d64bdc38af7db3dbd34877a057ce6524eefcb2914d6d41c56f968cc", size = 292537, upload-time = "2025-10-21T15:56:13.963Z" }, - { url = "https://files.pythonhosted.org/packages/98/3b/025240af4ada1dc0b5f10d73f3e5122d04ce7f8908ab8881e5d82b9d61b6/regex-2025.10.23-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:99018c331fb2529084a0c9b4c713dfa49fafb47c7712422e49467c13a636c656", size = 290904, upload-time = "2025-10-21T15:56:16.016Z" }, - { url = "https://files.pythonhosted.org/packages/81/8e/104ac14e2d3450c43db18ec03e1b96b445a94ae510b60138f00ce2cb7ca1/regex-2025.10.23-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd8aba965604d70306eb90a35528f776e59112a7114a5162824d43b76fa27f58", size = 807311, upload-time = "2025-10-21T15:56:17.818Z" }, - { url = "https://files.pythonhosted.org/packages/19/63/78aef90141b7ce0be8a18e1782f764f6997ad09de0e05251f0d2503a914a/regex-2025.10.23-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:238e67264b4013e74136c49f883734f68656adf8257bfa13b515626b31b20f8e", size = 873241, upload-time = "2025-10-21T15:56:19.941Z" }, - { url = "https://files.pythonhosted.org/packages/b3/a8/80eb1201bb49ae4dba68a1b284b4211ed9daa8e74dc600018a10a90399fb/regex-2025.10.23-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b2eb48bd9848d66fd04826382f5e8491ae633de3233a3d64d58ceb4ecfa2113a", size = 914794, upload-time = "2025-10-21T15:56:22.488Z" }, - { url = "https://files.pythonhosted.org/packages/f0/d5/1984b6ee93281f360a119a5ca1af6a8ca7d8417861671388bf750becc29b/regex-2025.10.23-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d36591ce06d047d0c0fe2fc5f14bfbd5b4525d08a7b6a279379085e13f0e3d0e", size = 812581, upload-time = "2025-10-21T15:56:24.319Z" }, - { url = "https://files.pythonhosted.org/packages/c4/39/11ebdc6d9927172a64ae237d16763145db6bd45ebb4055c17b88edab72a7/regex-2025.10.23-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b5d4ece8628d6e364302006366cea3ee887db397faebacc5dacf8ef19e064cf8", size = 795346, upload-time = "2025-10-21T15:56:26.232Z" }, - { url = "https://files.pythonhosted.org/packages/3b/b4/89a591bcc08b5e436af43315284bd233ba77daf0cf20e098d7af12f006c1/regex-2025.10.23-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:39a7e8083959cb1c4ff74e483eecb5a65d3b3e1d821b256e54baf61782c906c6", size = 868214, upload-time = "2025-10-21T15:56:28.597Z" }, - { url = "https://files.pythonhosted.org/packages/3d/ff/58ba98409c1dbc8316cdb20dafbc63ed267380a07780cafecaf5012dabc9/regex-2025.10.23-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:842d449a8fefe546f311656cf8c0d6729b08c09a185f1cad94c756210286d6a8", size = 854540, upload-time = "2025-10-21T15:56:30.875Z" }, - { url = "https://files.pythonhosted.org/packages/9a/f2/4a9e9338d67626e2071b643f828a482712ad15889d7268e11e9a63d6f7e9/regex-2025.10.23-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d614986dc68506be8f00474f4f6960e03e4ca9883f7df47744800e7d7c08a494", size = 799346, upload-time = "2025-10-21T15:56:32.725Z" }, - { url = "https://files.pythonhosted.org/packages/63/be/543d35c46bebf6f7bf2be538cca74d6585f25714700c36f37f01b92df551/regex-2025.10.23-cp313-cp313t-win32.whl", hash = "sha256:a5b7a26b51a9df473ec16a1934d117443a775ceb7b39b78670b2e21893c330c9", size = 268657, upload-time = "2025-10-21T15:56:34.577Z" }, - { url = "https://files.pythonhosted.org/packages/14/9f/4dd6b7b612037158bb2c9bcaa710e6fb3c40ad54af441b9c53b3a137a9f1/regex-2025.10.23-cp313-cp313t-win_amd64.whl", hash = "sha256:ce81c5544a5453f61cb6f548ed358cfb111e3b23f3cd42d250a4077a6be2a7b6", size = 280075, upload-time = "2025-10-21T15:56:36.767Z" }, - { url = "https://files.pythonhosted.org/packages/81/7a/5bd0672aa65d38c8da6747c17c8b441bdb53d816c569e3261013af8e83cf/regex-2025.10.23-cp313-cp313t-win_arm64.whl", hash = "sha256:e9bf7f6699f490e4e43c44757aa179dab24d1960999c84ab5c3d5377714ed473", size = 271219, upload-time = "2025-10-21T15:56:39.033Z" }, - { url = "https://files.pythonhosted.org/packages/73/f6/0caf29fec943f201fbc8822879c99d31e59c1d51a983d9843ee5cf398539/regex-2025.10.23-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:5b5cb5b6344c4c4c24b2dc87b0bfee78202b07ef7633385df70da7fcf6f7cec6", size = 488960, upload-time = "2025-10-21T15:56:40.849Z" }, - { url = "https://files.pythonhosted.org/packages/8e/7d/ebb7085b8fa31c24ce0355107cea2b92229d9050552a01c5d291c42aecea/regex-2025.10.23-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a6ce7973384c37bdf0f371a843f95a6e6f4e1489e10e0cf57330198df72959c5", size = 290932, upload-time = "2025-10-21T15:56:42.875Z" }, - { url = "https://files.pythonhosted.org/packages/27/41/43906867287cbb5ca4cee671c3cc8081e15deef86a8189c3aad9ac9f6b4d/regex-2025.10.23-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:2ee3663f2c334959016b56e3bd0dd187cbc73f948e3a3af14c3caaa0c3035d10", size = 288766, upload-time = "2025-10-21T15:56:44.894Z" }, - { url = "https://files.pythonhosted.org/packages/ab/9e/ea66132776700fc77a39b1056e7a5f1308032fead94507e208dc6716b7cd/regex-2025.10.23-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2003cc82a579107e70d013482acce8ba773293f2db534fb532738395c557ff34", size = 798884, upload-time = "2025-10-21T15:56:47.178Z" }, - { url = "https://files.pythonhosted.org/packages/d5/99/aed1453687ab63819a443930770db972c5c8064421f0d9f5da9ad029f26b/regex-2025.10.23-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:182c452279365a93a9f45874f7f191ec1c51e1f1eb41bf2b16563f1a40c1da3a", size = 864768, upload-time = "2025-10-21T15:56:49.793Z" }, - { url = "https://files.pythonhosted.org/packages/99/5d/732fe747a1304805eb3853ce6337eea16b169f7105a0d0dd9c6a5ffa9948/regex-2025.10.23-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b1249e9ff581c5b658c8f0437f883b01f1edcf424a16388591e7c05e5e9e8b0c", size = 911394, upload-time = "2025-10-21T15:56:52.186Z" }, - { url = "https://files.pythonhosted.org/packages/5e/48/58a1f6623466522352a6efa153b9a3714fc559d9f930e9bc947b4a88a2c3/regex-2025.10.23-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2b841698f93db3ccc36caa1900d2a3be281d9539b822dc012f08fc80b46a3224", size = 803145, upload-time = "2025-10-21T15:56:55.142Z" }, - { url = "https://files.pythonhosted.org/packages/ea/f6/7dea79be2681a5574ab3fc237aa53b2c1dfd6bd2b44d4640b6c76f33f4c1/regex-2025.10.23-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:956d89e0c92d471e8f7eee73f73fdff5ed345886378c45a43175a77538a1ffe4", size = 787831, upload-time = "2025-10-21T15:56:57.203Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ad/07b76950fbbe65f88120ca2d8d845047c401450f607c99ed38862904671d/regex-2025.10.23-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5c259cb363299a0d90d63b5c0d7568ee98419861618a95ee9d91a41cb9954462", size = 859162, upload-time = "2025-10-21T15:56:59.195Z" }, - { url = "https://files.pythonhosted.org/packages/41/87/374f3b2021b22aa6a4fc0b750d63f9721e53d1631a238f7a1c343c1cd288/regex-2025.10.23-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:185d2b18c062820b3a40d8fefa223a83f10b20a674bf6e8c4a432e8dfd844627", size = 849899, upload-time = "2025-10-21T15:57:01.747Z" }, - { url = "https://files.pythonhosted.org/packages/12/4a/7f7bb17c5a5a9747249807210e348450dab9212a46ae6d23ebce86ba6a2b/regex-2025.10.23-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:281d87fa790049c2b7c1b4253121edd80b392b19b5a3d28dc2a77579cb2a58ec", size = 789372, upload-time = "2025-10-21T15:57:04.018Z" }, - { url = "https://files.pythonhosted.org/packages/c9/dd/9c7728ff544fea09bbc8635e4c9e7c423b11c24f1a7a14e6ac4831466709/regex-2025.10.23-cp314-cp314-win32.whl", hash = "sha256:63b81eef3656072e4ca87c58084c7a9c2b81d41a300b157be635a8a675aacfb8", size = 271451, upload-time = "2025-10-21T15:57:06.266Z" }, - { url = "https://files.pythonhosted.org/packages/48/f8/ef7837ff858eb74079c4804c10b0403c0b740762e6eedba41062225f7117/regex-2025.10.23-cp314-cp314-win_amd64.whl", hash = "sha256:0967c5b86f274800a34a4ed862dfab56928144d03cb18821c5153f8777947796", size = 280173, upload-time = "2025-10-21T15:57:08.206Z" }, - { url = "https://files.pythonhosted.org/packages/8e/d0/d576e1dbd9885bfcd83d0e90762beea48d9373a6f7ed39170f44ed22e336/regex-2025.10.23-cp314-cp314-win_arm64.whl", hash = "sha256:c70dfe58b0a00b36aa04cdb0f798bf3e0adc31747641f69e191109fd8572c9a9", size = 273206, upload-time = "2025-10-21T15:57:10.367Z" }, - { url = "https://files.pythonhosted.org/packages/a6/d0/2025268315e8b2b7b660039824cb7765a41623e97d4cd421510925400487/regex-2025.10.23-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1f5799ea1787aa6de6c150377d11afad39a38afd033f0c5247aecb997978c422", size = 491854, upload-time = "2025-10-21T15:57:12.526Z" }, - { url = "https://files.pythonhosted.org/packages/44/35/5681c2fec5e8b33454390af209c4353dfc44606bf06d714b0b8bd0454ffe/regex-2025.10.23-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:a9639ab7540cfea45ef57d16dcbea2e22de351998d614c3ad2f9778fa3bdd788", size = 292542, upload-time = "2025-10-21T15:57:15.158Z" }, - { url = "https://files.pythonhosted.org/packages/5d/17/184eed05543b724132e4a18149e900f5189001fcfe2d64edaae4fbaf36b4/regex-2025.10.23-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:08f52122c352eb44c3421dab78b9b73a8a77a282cc8314ae576fcaa92b780d10", size = 290903, upload-time = "2025-10-21T15:57:17.108Z" }, - { url = "https://files.pythonhosted.org/packages/25/d0/5e3347aa0db0de382dddfa133a7b0ae72f24b4344f3989398980b44a3924/regex-2025.10.23-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebf1baebef1c4088ad5a5623decec6b52950f0e4d7a0ae4d48f0a99f8c9cb7d7", size = 807546, upload-time = "2025-10-21T15:57:19.179Z" }, - { url = "https://files.pythonhosted.org/packages/d2/bb/40c589bbdce1be0c55e9f8159789d58d47a22014f2f820cf2b517a5cd193/regex-2025.10.23-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:16b0f1c2e2d566c562d5c384c2b492646be0a19798532fdc1fdedacc66e3223f", size = 873322, upload-time = "2025-10-21T15:57:21.36Z" }, - { url = "https://files.pythonhosted.org/packages/fe/56/a7e40c01575ac93360e606278d359f91829781a9f7fb6e5aa435039edbda/regex-2025.10.23-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7ada5d9dceafaab92646aa00c10a9efd9b09942dd9b0d7c5a4b73db92cc7e61", size = 914855, upload-time = "2025-10-21T15:57:24.044Z" }, - { url = "https://files.pythonhosted.org/packages/5c/4b/d55587b192763db3163c3f508b3b67b31bb6f5e7a0e08b83013d0a59500a/regex-2025.10.23-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a36b4005770044bf08edecc798f0e41a75795b9e7c9c12fe29da8d792ef870c", size = 812724, upload-time = "2025-10-21T15:57:26.123Z" }, - { url = "https://files.pythonhosted.org/packages/33/20/18bac334955fbe99d17229f4f8e98d05e4a501ac03a442be8facbb37c304/regex-2025.10.23-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:af7b2661dcc032da1fae82069b5ebf2ac1dfcd5359ef8b35e1367bfc92181432", size = 795439, upload-time = "2025-10-21T15:57:28.497Z" }, - { url = "https://files.pythonhosted.org/packages/67/46/c57266be9df8549c7d85deb4cb82280cb0019e46fff677534c5fa1badfa4/regex-2025.10.23-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1cb976810ac1416a67562c2e5ba0accf6f928932320fef302e08100ed681b38e", size = 868336, upload-time = "2025-10-21T15:57:30.867Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f3/bd5879e41ef8187fec5e678e94b526a93f99e7bbe0437b0f2b47f9101694/regex-2025.10.23-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:1a56a54be3897d62f54290190fbcd754bff6932934529fbf5b29933da28fcd43", size = 854567, upload-time = "2025-10-21T15:57:33.062Z" }, - { url = "https://files.pythonhosted.org/packages/e6/57/2b6bbdbd2f24dfed5b028033aa17ad8f7d86bb28f1a892cac8b3bc89d059/regex-2025.10.23-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8f3e6d202fb52c2153f532043bbcf618fd177df47b0b306741eb9b60ba96edc3", size = 799565, upload-time = "2025-10-21T15:57:35.153Z" }, - { url = "https://files.pythonhosted.org/packages/c7/ba/a6168f542ba73b151ed81237adf6b869c7b2f7f8d51618111296674e20ee/regex-2025.10.23-cp314-cp314t-win32.whl", hash = "sha256:1fa1186966b2621b1769fd467c7b22e317e6ba2d2cdcecc42ea3089ef04a8521", size = 274428, upload-time = "2025-10-21T15:57:37.996Z" }, - { url = "https://files.pythonhosted.org/packages/ef/a0/c84475e14a2829e9b0864ebf77c3f7da909df9d8acfe2bb540ff0072047c/regex-2025.10.23-cp314-cp314t-win_amd64.whl", hash = "sha256:08a15d40ce28362eac3e78e83d75475147869c1ff86bc93285f43b4f4431a741", size = 284140, upload-time = "2025-10-21T15:57:40.027Z" }, - { url = "https://files.pythonhosted.org/packages/51/33/6a08ade0eee5b8ba79386869fa6f77afeb835b60510f3525db987e2fffc4/regex-2025.10.23-cp314-cp314t-win_arm64.whl", hash = "sha256:a93e97338e1c8ea2649e130dcfbe8cd69bba5e1e163834752ab64dcb4de6d5ed", size = 274497, upload-time = "2025-10-21T15:57:42.389Z" }, +version = "2025.11.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/a9/546676f25e573a4cf00fe8e119b78a37b6a8fe2dc95cda877b30889c9c45/regex-2025.11.3.tar.gz", hash = "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", size = 414669, upload-time = "2025-11-03T21:34:22.089Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/d6/d788d52da01280a30a3f6268aef2aa71043bff359c618fea4c5b536654d5/regex-2025.11.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2b441a4ae2c8049106e8b39973bfbddfb25a179dda2bdb99b0eeb60c40a6a3af", size = 488087, upload-time = "2025-11-03T21:30:47.317Z" }, + { url = "https://files.pythonhosted.org/packages/69/39/abec3bd688ec9bbea3562de0fd764ff802976185f5ff22807bf0a2697992/regex-2025.11.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2fa2eed3f76677777345d2f81ee89f5de2f5745910e805f7af7386a920fa7313", size = 290544, upload-time = "2025-11-03T21:30:49.912Z" }, + { url = "https://files.pythonhosted.org/packages/39/b3/9a231475d5653e60002508f41205c61684bb2ffbf2401351ae2186897fc4/regex-2025.11.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d8b4a27eebd684319bdf473d39f1d79eed36bf2cd34bd4465cdb4618d82b3d56", size = 288408, upload-time = "2025-11-03T21:30:51.344Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c5/1929a0491bd5ac2d1539a866768b88965fa8c405f3e16a8cef84313098d6/regex-2025.11.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cf77eac15bd264986c4a2c63353212c095b40f3affb2bc6b4ef80c4776c1a28", size = 781584, upload-time = "2025-11-03T21:30:52.596Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fd/16aa16cf5d497ef727ec966f74164fbe75d6516d3d58ac9aa989bc9cdaad/regex-2025.11.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b7f9ee819f94c6abfa56ec7b1dbab586f41ebbdc0a57e6524bd5e7f487a878c7", size = 850733, upload-time = "2025-11-03T21:30:53.825Z" }, + { url = "https://files.pythonhosted.org/packages/e6/49/3294b988855a221cb6565189edf5dc43239957427df2d81d4a6b15244f64/regex-2025.11.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:838441333bc90b829406d4a03cb4b8bf7656231b84358628b0406d803931ef32", size = 898691, upload-time = "2025-11-03T21:30:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/14/62/b56d29e70b03666193369bdbdedfdc23946dbe9f81dd78ce262c74d988ab/regex-2025.11.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cfe6d3f0c9e3b7e8c0c694b24d25e677776f5ca26dce46fd6b0489f9c8339391", size = 791662, upload-time = "2025-11-03T21:30:57.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/fc/e4c31d061eced63fbf1ce9d853975f912c61a7d406ea14eda2dd355f48e7/regex-2025.11.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2ab815eb8a96379a27c3b6157fcb127c8f59c36f043c1678110cea492868f1d5", size = 782587, upload-time = "2025-11-03T21:30:58.788Z" }, + { url = "https://files.pythonhosted.org/packages/b2/bb/5e30c7394bcf63f0537121c23e796be67b55a8847c3956ae6068f4c70702/regex-2025.11.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:728a9d2d173a65b62bdc380b7932dd8e74ed4295279a8fe1021204ce210803e7", size = 774709, upload-time = "2025-11-03T21:31:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c4/fce773710af81b0cb37cb4ff0947e75d5d17dee304b93d940b87a67fc2f4/regex-2025.11.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:509dc827f89c15c66a0c216331260d777dd6c81e9a4e4f830e662b0bb296c313", size = 845773, upload-time = "2025-11-03T21:31:01.583Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/9466a7ec4b8ec282077095c6eb50a12a389d2e036581134d4919e8ca518c/regex-2025.11.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:849202cd789e5f3cf5dcc7822c34b502181b4824a65ff20ce82da5524e45e8e9", size = 836164, upload-time = "2025-11-03T21:31:03.244Z" }, + { url = "https://files.pythonhosted.org/packages/95/18/82980a60e8ed1594eb3c89eb814fb276ef51b9af7caeab1340bfd8564af6/regex-2025.11.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b6f78f98741dcc89607c16b1e9426ee46ce4bf31ac5e6b0d40e81c89f3481ea5", size = 779832, upload-time = "2025-11-03T21:31:04.876Z" }, + { url = "https://files.pythonhosted.org/packages/03/cc/90ab0fdbe6dce064a42015433f9152710139fb04a8b81b4fb57a1cb63ffa/regex-2025.11.3-cp310-cp310-win32.whl", hash = "sha256:149eb0bba95231fb4f6d37c8f760ec9fa6fabf65bab555e128dde5f2475193ec", size = 265802, upload-time = "2025-11-03T21:31:06.581Z" }, + { url = "https://files.pythonhosted.org/packages/34/9d/e9e8493a85f3b1ddc4a5014465f5c2b78c3ea1cbf238dcfde78956378041/regex-2025.11.3-cp310-cp310-win_amd64.whl", hash = "sha256:ee3a83ce492074c35a74cc76cf8235d49e77b757193a5365ff86e3f2f93db9fd", size = 277722, upload-time = "2025-11-03T21:31:08.144Z" }, + { url = "https://files.pythonhosted.org/packages/15/c4/b54b24f553966564506dbf873a3e080aef47b356a3b39b5d5aba992b50db/regex-2025.11.3-cp310-cp310-win_arm64.whl", hash = "sha256:38af559ad934a7b35147716655d4a2f79fcef2d695ddfe06a06ba40ae631fa7e", size = 270289, upload-time = "2025-11-03T21:31:10.267Z" }, + { url = "https://files.pythonhosted.org/packages/f7/90/4fb5056e5f03a7048abd2b11f598d464f0c167de4f2a51aa868c376b8c70/regex-2025.11.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031", size = 488081, upload-time = "2025-11-03T21:31:11.946Z" }, + { url = "https://files.pythonhosted.org/packages/85/23/63e481293fac8b069d84fba0299b6666df720d875110efd0338406b5d360/regex-2025.11.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4", size = 290554, upload-time = "2025-11-03T21:31:13.387Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/b101d0262ea293a0066b4522dfb722eb6a8785a8c3e084396a5f2c431a46/regex-2025.11.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50", size = 288407, upload-time = "2025-11-03T21:31:14.809Z" }, + { url = "https://files.pythonhosted.org/packages/0c/64/79241c8209d5b7e00577ec9dca35cd493cc6be35b7d147eda367d6179f6d/regex-2025.11.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f", size = 793418, upload-time = "2025-11-03T21:31:16.556Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e2/23cd5d3573901ce8f9757c92ca4db4d09600b865919b6d3e7f69f03b1afd/regex-2025.11.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118", size = 860448, upload-time = "2025-11-03T21:31:18.12Z" }, + { url = "https://files.pythonhosted.org/packages/2a/4c/aecf31beeaa416d0ae4ecb852148d38db35391aac19c687b5d56aedf3a8b/regex-2025.11.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2", size = 907139, upload-time = "2025-11-03T21:31:20.753Z" }, + { url = "https://files.pythonhosted.org/packages/61/22/b8cb00df7d2b5e0875f60628594d44dba283e951b1ae17c12f99e332cc0a/regex-2025.11.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e", size = 800439, upload-time = "2025-11-03T21:31:22.069Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/c4b20330a5cdc7a8eb265f9ce593f389a6a88a0c5f280cf4d978f33966bc/regex-2025.11.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0", size = 782965, upload-time = "2025-11-03T21:31:23.598Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4c/ae3e52988ae74af4b04d2af32fee4e8077f26e51b62ec2d12d246876bea2/regex-2025.11.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58", size = 854398, upload-time = "2025-11-03T21:31:25.008Z" }, + { url = "https://files.pythonhosted.org/packages/06/d1/a8b9cf45874eda14b2e275157ce3b304c87e10fb38d9fc26a6e14eb18227/regex-2025.11.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab", size = 845897, upload-time = "2025-11-03T21:31:26.427Z" }, + { url = "https://files.pythonhosted.org/packages/ea/fe/1830eb0236be93d9b145e0bd8ab499f31602fe0999b1f19e99955aa8fe20/regex-2025.11.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e", size = 788906, upload-time = "2025-11-03T21:31:28.078Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/dc2577c1f95f188c1e13e2e69d8825a5ac582ac709942f8a03af42ed6e93/regex-2025.11.3-cp311-cp311-win32.whl", hash = "sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf", size = 265812, upload-time = "2025-11-03T21:31:29.72Z" }, + { url = "https://files.pythonhosted.org/packages/50/1e/15f08b2f82a9bbb510621ec9042547b54d11e83cb620643ebb54e4eb7d71/regex-2025.11.3-cp311-cp311-win_amd64.whl", hash = "sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a", size = 277737, upload-time = "2025-11-03T21:31:31.422Z" }, + { url = "https://files.pythonhosted.org/packages/f4/fc/6500eb39f5f76c5e47a398df82e6b535a5e345f839581012a418b16f9cc3/regex-2025.11.3-cp311-cp311-win_arm64.whl", hash = "sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc", size = 270290, upload-time = "2025-11-03T21:31:33.041Z" }, + { url = "https://files.pythonhosted.org/packages/e8/74/18f04cb53e58e3fb107439699bd8375cf5a835eec81084e0bddbd122e4c2/regex-2025.11.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", size = 489312, upload-time = "2025-11-03T21:31:34.343Z" }, + { url = "https://files.pythonhosted.org/packages/78/3f/37fcdd0d2b1e78909108a876580485ea37c91e1acf66d3bb8e736348f441/regex-2025.11.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", size = 291256, upload-time = "2025-11-03T21:31:35.675Z" }, + { url = "https://files.pythonhosted.org/packages/bf/26/0a575f58eb23b7ebd67a45fccbc02ac030b737b896b7e7a909ffe43ffd6a/regex-2025.11.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", size = 288921, upload-time = "2025-11-03T21:31:37.07Z" }, + { url = "https://files.pythonhosted.org/packages/ea/98/6a8dff667d1af907150432cf5abc05a17ccd32c72a3615410d5365ac167a/regex-2025.11.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", size = 798568, upload-time = "2025-11-03T21:31:38.784Z" }, + { url = "https://files.pythonhosted.org/packages/64/15/92c1db4fa4e12733dd5a526c2dd2b6edcbfe13257e135fc0f6c57f34c173/regex-2025.11.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", size = 864165, upload-time = "2025-11-03T21:31:40.559Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/3ad7da8cdee1ce66c7cd37ab5ab05c463a86ffeb52b1a25fe7bd9293b36c/regex-2025.11.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", size = 912182, upload-time = "2025-11-03T21:31:42.002Z" }, + { url = "https://files.pythonhosted.org/packages/84/bd/9ce9f629fcb714ffc2c3faf62b6766ecb7a585e1e885eb699bcf130a5209/regex-2025.11.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", size = 803501, upload-time = "2025-11-03T21:31:43.815Z" }, + { url = "https://files.pythonhosted.org/packages/7c/0f/8dc2e4349d8e877283e6edd6c12bdcebc20f03744e86f197ab6e4492bf08/regex-2025.11.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", size = 787842, upload-time = "2025-11-03T21:31:45.353Z" }, + { url = "https://files.pythonhosted.org/packages/f9/73/cff02702960bc185164d5619c0c62a2f598a6abff6695d391b096237d4ab/regex-2025.11.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", size = 858519, upload-time = "2025-11-03T21:31:46.814Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/0e8d1ae71e15bc1dc36231c90b46ee35f9d52fab2e226b0e039e7ea9c10a/regex-2025.11.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", size = 850611, upload-time = "2025-11-03T21:31:48.289Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f5/70a5cdd781dcfaa12556f2955bf170cd603cb1c96a1827479f8faea2df97/regex-2025.11.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", size = 789759, upload-time = "2025-11-03T21:31:49.759Z" }, + { url = "https://files.pythonhosted.org/packages/59/9b/7c29be7903c318488983e7d97abcf8ebd3830e4c956c4c540005fcfb0462/regex-2025.11.3-cp312-cp312-win32.whl", hash = "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", size = 266194, upload-time = "2025-11-03T21:31:51.53Z" }, + { url = "https://files.pythonhosted.org/packages/1a/67/3b92df89f179d7c367be654ab5626ae311cb28f7d5c237b6bb976cd5fbbb/regex-2025.11.3-cp312-cp312-win_amd64.whl", hash = "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", size = 277069, upload-time = "2025-11-03T21:31:53.151Z" }, + { url = "https://files.pythonhosted.org/packages/d7/55/85ba4c066fe5094d35b249c3ce8df0ba623cfd35afb22d6764f23a52a1c5/regex-2025.11.3-cp312-cp312-win_arm64.whl", hash = "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", size = 270330, upload-time = "2025-11-03T21:31:54.514Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a7/dda24ebd49da46a197436ad96378f17df30ceb40e52e859fc42cac45b850/regex-2025.11.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4", size = 489081, upload-time = "2025-11-03T21:31:55.9Z" }, + { url = "https://files.pythonhosted.org/packages/19/22/af2dc751aacf88089836aa088a1a11c4f21a04707eb1b0478e8e8fb32847/regex-2025.11.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76", size = 291123, upload-time = "2025-11-03T21:31:57.758Z" }, + { url = "https://files.pythonhosted.org/packages/a3/88/1a3ea5672f4b0a84802ee9891b86743438e7c04eb0b8f8c4e16a42375327/regex-2025.11.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a", size = 288814, upload-time = "2025-11-03T21:32:01.12Z" }, + { url = "https://files.pythonhosted.org/packages/fb/8c/f5987895bf42b8ddeea1b315c9fedcfe07cadee28b9c98cf50d00adcb14d/regex-2025.11.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361", size = 798592, upload-time = "2025-11-03T21:32:03.006Z" }, + { url = "https://files.pythonhosted.org/packages/99/2a/6591ebeede78203fa77ee46a1c36649e02df9eaa77a033d1ccdf2fcd5d4e/regex-2025.11.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160", size = 864122, upload-time = "2025-11-03T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/94/d6/be32a87cf28cf8ed064ff281cfbd49aefd90242a83e4b08b5a86b38e8eb4/regex-2025.11.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe", size = 912272, upload-time = "2025-11-03T21:32:06.148Z" }, + { url = "https://files.pythonhosted.org/packages/62/11/9bcef2d1445665b180ac7f230406ad80671f0fc2a6ffb93493b5dd8cd64c/regex-2025.11.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850", size = 803497, upload-time = "2025-11-03T21:32:08.162Z" }, + { url = "https://files.pythonhosted.org/packages/e5/a7/da0dc273d57f560399aa16d8a68ae7f9b57679476fc7ace46501d455fe84/regex-2025.11.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc", size = 787892, upload-time = "2025-11-03T21:32:09.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/4b/732a0c5a9736a0b8d6d720d4945a2f1e6f38f87f48f3173559f53e8d5d82/regex-2025.11.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9", size = 858462, upload-time = "2025-11-03T21:32:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f5/a2a03df27dc4c2d0c769220f5110ba8c4084b0bfa9ab0f9b4fcfa3d2b0fc/regex-2025.11.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b", size = 850528, upload-time = "2025-11-03T21:32:13.906Z" }, + { url = "https://files.pythonhosted.org/packages/d6/09/e1cd5bee3841c7f6eb37d95ca91cdee7100b8f88b81e41c2ef426910891a/regex-2025.11.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7", size = 789866, upload-time = "2025-11-03T21:32:15.748Z" }, + { url = "https://files.pythonhosted.org/packages/eb/51/702f5ea74e2a9c13d855a6a85b7f80c30f9e72a95493260193c07f3f8d74/regex-2025.11.3-cp313-cp313-win32.whl", hash = "sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c", size = 266189, upload-time = "2025-11-03T21:32:17.493Z" }, + { url = "https://files.pythonhosted.org/packages/8b/00/6e29bb314e271a743170e53649db0fdb8e8ff0b64b4f425f5602f4eb9014/regex-2025.11.3-cp313-cp313-win_amd64.whl", hash = "sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5", size = 277054, upload-time = "2025-11-03T21:32:19.042Z" }, + { url = "https://files.pythonhosted.org/packages/25/f1/b156ff9f2ec9ac441710764dda95e4edaf5f36aca48246d1eea3f1fd96ec/regex-2025.11.3-cp313-cp313-win_arm64.whl", hash = "sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467", size = 270325, upload-time = "2025-11-03T21:32:21.338Z" }, + { url = "https://files.pythonhosted.org/packages/20/28/fd0c63357caefe5680b8ea052131acbd7f456893b69cc2a90cc3e0dc90d4/regex-2025.11.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281", size = 491984, upload-time = "2025-11-03T21:32:23.466Z" }, + { url = "https://files.pythonhosted.org/packages/df/ec/7014c15626ab46b902b3bcc4b28a7bae46d8f281fc7ea9c95e22fcaaa917/regex-2025.11.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39", size = 292673, upload-time = "2025-11-03T21:32:25.034Z" }, + { url = "https://files.pythonhosted.org/packages/23/ab/3b952ff7239f20d05f1f99e9e20188513905f218c81d52fb5e78d2bf7634/regex-2025.11.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7", size = 291029, upload-time = "2025-11-03T21:32:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/21/7e/3dc2749fc684f455f162dcafb8a187b559e2614f3826877d3844a131f37b/regex-2025.11.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed", size = 807437, upload-time = "2025-11-03T21:32:28.363Z" }, + { url = "https://files.pythonhosted.org/packages/1b/0b/d529a85ab349c6a25d1ca783235b6e3eedf187247eab536797021f7126c6/regex-2025.11.3-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19", size = 873368, upload-time = "2025-11-03T21:32:30.4Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/2d868155f8c9e3e9d8f9e10c64e9a9f496bb8f7e037a88a8bed26b435af6/regex-2025.11.3-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b", size = 914921, upload-time = "2025-11-03T21:32:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/2d/71/9d72ff0f354fa783fe2ba913c8734c3b433b86406117a8db4ea2bf1c7a2f/regex-2025.11.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a", size = 812708, upload-time = "2025-11-03T21:32:34.305Z" }, + { url = "https://files.pythonhosted.org/packages/e7/19/ce4bf7f5575c97f82b6e804ffb5c4e940c62609ab2a0d9538d47a7fdf7d4/regex-2025.11.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6", size = 795472, upload-time = "2025-11-03T21:32:36.364Z" }, + { url = "https://files.pythonhosted.org/packages/03/86/fd1063a176ffb7b2315f9a1b08d17b18118b28d9df163132615b835a26ee/regex-2025.11.3-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce", size = 868341, upload-time = "2025-11-03T21:32:38.042Z" }, + { url = "https://files.pythonhosted.org/packages/12/43/103fb2e9811205e7386366501bc866a164a0430c79dd59eac886a2822950/regex-2025.11.3-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd", size = 854666, upload-time = "2025-11-03T21:32:40.079Z" }, + { url = "https://files.pythonhosted.org/packages/7d/22/e392e53f3869b75804762c7c848bd2dd2abf2b70fb0e526f58724638bd35/regex-2025.11.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2", size = 799473, upload-time = "2025-11-03T21:32:42.148Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f9/8bd6b656592f925b6845fcbb4d57603a3ac2fb2373344ffa1ed70aa6820a/regex-2025.11.3-cp313-cp313t-win32.whl", hash = "sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a", size = 268792, upload-time = "2025-11-03T21:32:44.13Z" }, + { url = "https://files.pythonhosted.org/packages/e5/87/0e7d603467775ff65cd2aeabf1b5b50cc1c3708556a8b849a2fa4dd1542b/regex-2025.11.3-cp313-cp313t-win_amd64.whl", hash = "sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c", size = 280214, upload-time = "2025-11-03T21:32:45.853Z" }, + { url = "https://files.pythonhosted.org/packages/8d/d0/2afc6f8e94e2b64bfb738a7c2b6387ac1699f09f032d363ed9447fd2bb57/regex-2025.11.3-cp313-cp313t-win_arm64.whl", hash = "sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e", size = 271469, upload-time = "2025-11-03T21:32:48.026Z" }, + { url = "https://files.pythonhosted.org/packages/31/e9/f6e13de7e0983837f7b6d238ad9458800a874bf37c264f7923e63409944c/regex-2025.11.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6", size = 489089, upload-time = "2025-11-03T21:32:50.027Z" }, + { url = "https://files.pythonhosted.org/packages/a3/5c/261f4a262f1fa65141c1b74b255988bd2fa020cc599e53b080667d591cfc/regex-2025.11.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4", size = 291059, upload-time = "2025-11-03T21:32:51.682Z" }, + { url = "https://files.pythonhosted.org/packages/8e/57/f14eeb7f072b0e9a5a090d1712741fd8f214ec193dba773cf5410108bb7d/regex-2025.11.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73", size = 288900, upload-time = "2025-11-03T21:32:53.569Z" }, + { url = "https://files.pythonhosted.org/packages/3c/6b/1d650c45e99a9b327586739d926a1cd4e94666b1bd4af90428b36af66dc7/regex-2025.11.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f", size = 799010, upload-time = "2025-11-03T21:32:55.222Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/d66dcbc6b628ce4e3f7f0cbbb84603aa2fc0ffc878babc857726b8aab2e9/regex-2025.11.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d", size = 864893, upload-time = "2025-11-03T21:32:57.239Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2d/f238229f1caba7ac87a6c4153d79947fb0261415827ae0f77c304260c7d3/regex-2025.11.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be", size = 911522, upload-time = "2025-11-03T21:32:59.274Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3d/22a4eaba214a917c80e04f6025d26143690f0419511e0116508e24b11c9b/regex-2025.11.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db", size = 803272, upload-time = "2025-11-03T21:33:01.393Z" }, + { url = "https://files.pythonhosted.org/packages/84/b1/03188f634a409353a84b5ef49754b97dbcc0c0f6fd6c8ede505a8960a0a4/regex-2025.11.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62", size = 787958, upload-time = "2025-11-03T21:33:03.379Z" }, + { url = "https://files.pythonhosted.org/packages/99/6a/27d072f7fbf6fadd59c64d210305e1ff865cc3b78b526fd147db768c553b/regex-2025.11.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f", size = 859289, upload-time = "2025-11-03T21:33:05.374Z" }, + { url = "https://files.pythonhosted.org/packages/9a/70/1b3878f648e0b6abe023172dacb02157e685564853cc363d9961bcccde4e/regex-2025.11.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02", size = 850026, upload-time = "2025-11-03T21:33:07.131Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d5/68e25559b526b8baab8e66839304ede68ff6727237a47727d240006bd0ff/regex-2025.11.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed", size = 789499, upload-time = "2025-11-03T21:33:09.141Z" }, + { url = "https://files.pythonhosted.org/packages/fc/df/43971264857140a350910d4e33df725e8c94dd9dee8d2e4729fa0d63d49e/regex-2025.11.3-cp314-cp314-win32.whl", hash = "sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4", size = 271604, upload-time = "2025-11-03T21:33:10.9Z" }, + { url = "https://files.pythonhosted.org/packages/01/6f/9711b57dc6894a55faf80a4c1b5aa4f8649805cb9c7aef46f7d27e2b9206/regex-2025.11.3-cp314-cp314-win_amd64.whl", hash = "sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad", size = 280320, upload-time = "2025-11-03T21:33:12.572Z" }, + { url = "https://files.pythonhosted.org/packages/f1/7e/f6eaa207d4377481f5e1775cdeb5a443b5a59b392d0065f3417d31d80f87/regex-2025.11.3-cp314-cp314-win_arm64.whl", hash = "sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f", size = 273372, upload-time = "2025-11-03T21:33:14.219Z" }, + { url = "https://files.pythonhosted.org/packages/c3/06/49b198550ee0f5e4184271cee87ba4dfd9692c91ec55289e6282f0f86ccf/regex-2025.11.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc", size = 491985, upload-time = "2025-11-03T21:33:16.555Z" }, + { url = "https://files.pythonhosted.org/packages/ce/bf/abdafade008f0b1c9da10d934034cb670432d6cf6cbe38bbb53a1cfd6cf8/regex-2025.11.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49", size = 292669, upload-time = "2025-11-03T21:33:18.32Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ef/0c357bb8edbd2ad8e273fcb9e1761bc37b8acbc6e1be050bebd6475f19c1/regex-2025.11.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536", size = 291030, upload-time = "2025-11-03T21:33:20.048Z" }, + { url = "https://files.pythonhosted.org/packages/79/06/edbb67257596649b8fb088d6aeacbcb248ac195714b18a65e018bf4c0b50/regex-2025.11.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95", size = 807674, upload-time = "2025-11-03T21:33:21.797Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d9/ad4deccfce0ea336296bd087f1a191543bb99ee1c53093dcd4c64d951d00/regex-2025.11.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009", size = 873451, upload-time = "2025-11-03T21:33:23.741Z" }, + { url = "https://files.pythonhosted.org/packages/13/75/a55a4724c56ef13e3e04acaab29df26582f6978c000ac9cd6810ad1f341f/regex-2025.11.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9", size = 914980, upload-time = "2025-11-03T21:33:25.999Z" }, + { url = "https://files.pythonhosted.org/packages/67/1e/a1657ee15bd9116f70d4a530c736983eed997b361e20ecd8f5ca3759d5c5/regex-2025.11.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d", size = 812852, upload-time = "2025-11-03T21:33:27.852Z" }, + { url = "https://files.pythonhosted.org/packages/b8/6f/f7516dde5506a588a561d296b2d0044839de06035bb486b326065b4c101e/regex-2025.11.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6", size = 795566, upload-time = "2025-11-03T21:33:32.364Z" }, + { url = "https://files.pythonhosted.org/packages/d9/dd/3d10b9e170cc16fb34cb2cef91513cf3df65f440b3366030631b2984a264/regex-2025.11.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154", size = 868463, upload-time = "2025-11-03T21:33:34.459Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8e/935e6beff1695aa9085ff83195daccd72acc82c81793df480f34569330de/regex-2025.11.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267", size = 854694, upload-time = "2025-11-03T21:33:36.793Z" }, + { url = "https://files.pythonhosted.org/packages/92/12/10650181a040978b2f5720a6a74d44f841371a3d984c2083fc1752e4acf6/regex-2025.11.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379", size = 799691, upload-time = "2025-11-03T21:33:39.079Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/8f37138181c9a7690e7e4cb388debbd389342db3c7381d636d2875940752/regex-2025.11.3-cp314-cp314t-win32.whl", hash = "sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38", size = 274583, upload-time = "2025-11-03T21:33:41.302Z" }, + { url = "https://files.pythonhosted.org/packages/8f/cd/867f5ec442d56beb56f5f854f40abcfc75e11d10b11fdb1869dd39c63aaf/regex-2025.11.3-cp314-cp314t-win_amd64.whl", hash = "sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de", size = 284286, upload-time = "2025-11-03T21:33:43.324Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/32c0c4610cbc070362bf1d2e4ea86d1ea29014d400a6d6c2486fcfd57766/regex-2025.11.3-cp314-cp314t-win_arm64.whl", hash = "sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801", size = 274741, upload-time = "2025-11-03T21:33:45.557Z" }, ] [[package]] @@ -4471,16 +5109,16 @@ wheels = [ [[package]] name = "s3fs" -version = "2025.9.0" +version = "2025.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiobotocore" }, { name = "aiohttp" }, { name = "fsspec" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/f3/8e6371436666aedfd16e63ff68a51b8a8fcf5f33a0eee33c35e0b2476b27/s3fs-2025.9.0.tar.gz", hash = "sha256:6d44257ef19ea64968d0720744c4af7a063a05f5c1be0e17ce943bef7302bc30", size = 77823, upload-time = "2025-09-02T19:18:21.781Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/ee/7cf7de3b17ef6db10b027cc9f8a1108ceb6333e267943e666a35882b1474/s3fs-2025.10.0.tar.gz", hash = "sha256:e8be6cddc77aceea1681ece0f472c3a7f8ef71a0d2acddb1cc92bb6afa3e9e4f", size = 80383, upload-time = "2025-10-30T15:06:04.647Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/37/b3/ca7d58ca25b1bb6df57e6cbd0ca8d6437a4b9ce1cd35adc8a6b2949c113b/s3fs-2025.9.0-py3-none-any.whl", hash = "sha256:c33c93d48f66ed440dbaf6600be149cdf8beae4b6f8f0201a209c5801aeb7e30", size = 30319, upload-time = "2025-09-02T19:18:20.563Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fc/56cba14af8ad8fd020c85b6e44328520ac55939bb1f9d01444ad470504cb/s3fs-2025.10.0-py3-none-any.whl", hash = "sha256:da7ef25efc1541f5fca8e1116361e49ea1081f83f4e8001fbd77347c625da28a", size = 30357, upload-time = "2025-10-30T15:06:03.48Z" }, ] [[package]] @@ -4572,11 +5210,13 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.12.*' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", @@ -4864,33 +5504,39 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", @@ -4947,33 +5593,39 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", @@ -5074,15 +5726,15 @@ wheels = [ [[package]] name = "starlette" -version = "0.49.1" +version = "0.49.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1b/3f/507c21db33b66fb027a332f2cb3abbbe924cc3a79ced12f01ed8645955c9/starlette-0.49.1.tar.gz", hash = "sha256:481a43b71e24ed8c43b11ea02f5353d77840e01480881b8cb5a26b8cae64a8cb", size = 2654703, upload-time = "2025-10-28T17:34:10.928Z" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1a/608df0b10b53b0beb96a37854ee05864d182ddd4b1156a22f1ad3860425a/starlette-0.49.3.tar.gz", hash = "sha256:1c14546f299b5901a1ea0e34410575bc33bbd741377a10484a54445588d00284", size = 2655031, upload-time = "2025-11-01T15:12:26.13Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/da/545b75d420bb23b5d494b0517757b351963e974e79933f01e05c929f20a6/starlette-0.49.1-py3-none-any.whl", hash = "sha256:d92ce9f07e4a3caa3ac13a79523bd18e3bc0042bb8ff2d759a8e7dd0e1859875", size = 74175, upload-time = "2025-10-28T17:34:09.13Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e0/021c772d6a662f43b63044ab481dc6ac7592447605b5b35a957785363122/starlette-0.49.3-py3-none-any.whl", hash = "sha256:b579b99715fdc2980cf88c8ec96d3bf1ce16f5a8051a7c2b84ef9b1cdecaea2f", size = 74340, upload-time = "2025-11-01T15:12:24.387Z" }, ] [[package]] @@ -5090,7 +5742,7 @@ name = "sympy" version = "1.14.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mpmath", marker = "sys_platform != 'linux'" }, + { name = "mpmath" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } wheels = [ @@ -5143,10 +5795,12 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", ] dependencies = [ { name = "ml-dtypes", version = "0.4.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -5374,15 +6028,30 @@ name = "torch" version = "2.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "sys_platform != 'linux'" }, - { name = "fsspec", marker = "sys_platform != 'linux'" }, - { name = "jinja2", marker = "sys_platform != 'linux'" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and sys_platform != 'linux') or (python_full_version >= '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and sys_platform != 'linux') or (python_full_version < '3.11' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform == 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "setuptools", marker = "python_full_version >= '3.12' and sys_platform != 'linux'" }, - { name = "sympy", marker = "sys_platform != 'linux'" }, - { name = "triton", marker = "sys_platform == 'never'" }, - { name = "typing-extensions", marker = "sys_platform != 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cuda-cupti-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cuda-runtime-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cudnn-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cufft-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cufile-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-curand-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusolver-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-cusparselt-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nccl-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvshmem-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "nvidia-nvtx-cu12", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'x86_64' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (sys_platform != 'linux' and extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "setuptools", marker = "python_full_version >= '3.12' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'never' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/bb/86/245c240d2138c17ed572c943c289056c2721abab70810d772c6bf5495b28/torch-2.9.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:030bbfe367379ae6a4ae4042b6c44da25383343b8b3c68abaa9c7231efbaf2dd", size = 104213554, upload-time = "2025-10-15T15:45:59.798Z" }, @@ -5434,8 +6103,8 @@ name = "torchvision" version = "0.24.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "sys_platform != 'linux'" }, - { name = "pillow", marker = "sys_platform != 'linux'" }, + { name = "numpy" }, + { name = "pillow" }, { name = "torch", marker = "sys_platform == 'never'" }, ] wheels = [ @@ -5478,8 +6147,8 @@ dependencies = [ { name = "docstring-parser" }, { name = "filelock" }, { name = "fsspec" }, - { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev' or extra == 'extra-13-megatron-core-lts'" }, - { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "(extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts') or (extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts')" }, + { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-dev'" }, + { name = "importlib-metadata", version = "8.7.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'extra-13-megatron-core-lts' or extra != 'extra-13-megatron-core-dev'" }, { name = "pyre-extensions" }, { name = "pyyaml" }, { name = "tabulate" }, @@ -5495,7 +6164,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" } wheels = [ @@ -5504,14 +6173,14 @@ wheels = [ [[package]] name = "transformer-engine" -version = "2.9.0+c4c185db" -source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.9#c4c185dbec1aab3627ab2ecffbc4c429d31f23c0" } +version = "2.9.0+70f53666" +source = { git = "https://github.com/NVIDIA/TransformerEngine.git?rev=release_v2.9#70f536662ae10a62a54f4ed1ba92e3314c5cfd69" } dependencies = [ { name = "einops" }, { name = "importlib-metadata", version = "8.6.1", source = { registry = "https://pypi.org/simple" } }, { name = "onnx" }, { name = "onnxscript", version = "0.5.0", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, - { name = "onnxscript", version = "0.5.4", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "onnxscript", version = "0.5.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.13' and extra == 'extra-13-megatron-core-dev') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, { name = "packaging" }, { name = "pydantic" }, { name = "torch", marker = "sys_platform == 'never'" }, @@ -5909,11 +6578,13 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", "python_full_version == '3.12.*' and sys_platform == 'linux'", "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", "python_full_version == '3.12.*' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'linux'", "python_full_version == '3.11.*' and sys_platform != 'linux'", @@ -5990,20 +6661,36 @@ name = "wrapt" version = "2.0.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", - "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", - "python_full_version >= '3.13' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform != 'linux'", - "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version < '3.11' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.12.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version == '3.11.*' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform == 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", + "python_full_version < '3.11' and sys_platform != 'linux' and extra != 'extra-13-megatron-core-dev' and extra != 'extra-13-megatron-core-lts'", ] sdist = { url = "https://files.pythonhosted.org/packages/49/19/5e5bcd855d808892fe02d49219f97a50f64cd6d8313d75df3494ee97b1a3/wrapt-2.0.0.tar.gz", hash = "sha256:35a542cc7a962331d0279735c30995b024e852cf40481e384fd63caaa391cbb9", size = 81722, upload-time = "2025-10-19T23:47:54.07Z" } wheels = [ @@ -6147,6 +6834,124 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/41/d6298c95513eabe091a6851bff5e7928fab49ffd9143808feaaf7721cf33/xattr-1.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5eeaa944516b7507ec51456751334b4880e421de169bbd067c4f32242670d606", size = 42864, upload-time = "2025-10-13T22:16:38.811Z" }, ] +[[package]] +name = "xxhash" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845, upload-time = "2025-10-02T14:33:51.573Z" }, + { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807, upload-time = "2025-10-02T14:33:52.964Z" }, + { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786, upload-time = "2025-10-02T14:33:54.272Z" }, + { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830, upload-time = "2025-10-02T14:33:55.706Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606, upload-time = "2025-10-02T14:33:57.133Z" }, + { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872, upload-time = "2025-10-02T14:33:58.446Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217, upload-time = "2025-10-02T14:33:59.724Z" }, + { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139, upload-time = "2025-10-02T14:34:02.041Z" }, + { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669, upload-time = "2025-10-02T14:34:03.664Z" }, + { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018, upload-time = "2025-10-02T14:34:05.325Z" }, + { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058, upload-time = "2025-10-02T14:34:06.925Z" }, + { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628, upload-time = "2025-10-02T14:34:08.669Z" }, + { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577, upload-time = "2025-10-02T14:34:10.234Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487, upload-time = "2025-10-02T14:34:11.618Z" }, + { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863, upload-time = "2025-10-02T14:34:12.619Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844, upload-time = "2025-10-02T14:34:14.037Z" }, + { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809, upload-time = "2025-10-02T14:34:15.484Z" }, + { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665, upload-time = "2025-10-02T14:34:16.541Z" }, + { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550, upload-time = "2025-10-02T14:34:17.878Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384, upload-time = "2025-10-02T14:34:19.182Z" }, + { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749, upload-time = "2025-10-02T14:34:20.659Z" }, + { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880, upload-time = "2025-10-02T14:34:22.431Z" }, + { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912, upload-time = "2025-10-02T14:34:23.937Z" }, + { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654, upload-time = "2025-10-02T14:34:25.644Z" }, + { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867, upload-time = "2025-10-02T14:34:27.203Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012, upload-time = "2025-10-02T14:34:28.409Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409, upload-time = "2025-10-02T14:34:29.696Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574, upload-time = "2025-10-02T14:34:31.028Z" }, + { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481, upload-time = "2025-10-02T14:34:32.062Z" }, + { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861, upload-time = "2025-10-02T14:34:33.555Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" }, + { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" }, + { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" }, + { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" }, + { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" }, + { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" }, + { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" }, + { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" }, + { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" }, + { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" }, + { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" }, + { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" }, + { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" }, + { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" }, + { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" }, + { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" }, + { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" }, + { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" }, + { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" }, + { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" }, + { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" }, + { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" }, + { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" }, + { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" }, + { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" }, + { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" }, + { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" }, + { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" }, + { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" }, + { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" }, + { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" }, + { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" }, + { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" }, + { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" }, + { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" }, + { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" }, + { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" }, + { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" }, + { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" }, + { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" }, + { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" }, + { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" }, + { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" }, + { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" }, + { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" }, + { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" }, + { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" }, + { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" }, + { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" }, + { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" }, + { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481, upload-time = "2025-10-02T14:37:05.869Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565, upload-time = "2025-10-02T14:37:06.966Z" }, +] + [[package]] name = "yarl" version = "1.22.0" @@ -6273,6 +7078,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, ] +[[package]] +name = "zarr" +version = "2.18.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'linux'", +] +dependencies = [ + { name = "asciitree", marker = "python_full_version < '3.11'" }, + { name = "fasteners", marker = "python_full_version < '3.11' and sys_platform != 'emscripten'" }, + { name = "numcodecs", version = "0.13.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/23/c4/187a21ce7cf7c8f00c060dd0e04c2a81139bb7b1ab178bba83f2e1134ce2/zarr-2.18.3.tar.gz", hash = "sha256:2580d8cb6dd84621771a10d31c4d777dca8a27706a1a89b29f42d2d37e2df5ce", size = 3603224, upload-time = "2024-09-04T23:20:16.595Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/c9/142095e654c2b97133ff71df60979422717b29738b08bc8a1709a5d5e0d0/zarr-2.18.3-py3-none-any.whl", hash = "sha256:b1f7dfd2496f436745cdd4c7bcf8d3b4bc1dceef5fdd0d589c87130d842496dd", size = 210723, upload-time = "2024-09-04T23:20:14.491Z" }, +] + +[[package]] +name = "zarr" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation != 'PyPy' and sys_platform != 'linux'", + "python_full_version >= '3.14' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.13.*' and platform_python_implementation == 'PyPy' and sys_platform != 'linux'", + "python_full_version == '3.12.*' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'linux'", +] +dependencies = [ + { name = "donfig", marker = "python_full_version >= '3.11'" }, + { name = "numcodecs", version = "0.16.3", source = { registry = "https://pypi.org/simple" }, extra = ["crc32c"], marker = "(python_full_version >= '3.11' and extra == 'extra-13-megatron-core-lts') or (extra == 'extra-13-megatron-core-dev' and extra == 'extra-13-megatron-core-lts')" }, + { name = "numpy", marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d6/67/14be68a7bad15eecda09b1e81fca2420f7533645fe187bf4d6104c1aad52/zarr-3.1.3.tar.gz", hash = "sha256:01342f3e26a02ed5670db608a5576fbdb8d76acb5c280bd2d0082454b1ba6f79", size = 349125, upload-time = "2025-09-18T19:32:41.688Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/71/9de7229515a53d1cc5705ca9c411530f711a2242f962214d9dbfe2741aa4/zarr-3.1.3-py3-none-any.whl", hash = "sha256:45f67f87f65f14fa453f99dd8110a5936b7ac69f3a21981d33e90407c80c302a", size = 276427, upload-time = "2025-09-18T19:32:40.042Z" }, +] + [[package]] name = "zipp" version = "3.23.0"