Repository navigation
fix(profiler): make parallel label injective across (tp, pp, dp, moe_tp, moe_ep) #10086
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
tedzhouhk
merged 6 commits into
ai-dynamo:main
from
jooe0824:jooe0824/profiler-parallel-label
Jun 17, 2026
Merged
Changes from all commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
ec8b5d3
fix(profiler): make parallel label injective across (tp, pp, dp, moe_…
e50f477
test:add module pytestmark and docstrings per CodeRabbit review
e5231d0
Merge branch 'main' into jooe0824/profiler-parallel-label
jooe0824 cdda6d9
Merge branch 'main' into jooe0824/profiler-parallel-label
jooe0824 e8fb214
Merge branch 'main' into jooe0824/profiler-parallel-label
jooe0824 69e3775
Merge branch 'main' into jooe0824/profiler-parallel-label
jooe0824 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
components/src/dynamo/planner/tests/unit/test_parallelization_label.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| """Regression tests for ``PickedParallelConfig.label`` collisions. | ||
|
|
||
| The legacy 3-bucket label (``dep{moe_ep}`` / ``tep{moe_tp}`` / ``tp{tp}``) | ||
| collapsed distinct 5-tuples to the same string — notably | ||
| ``(tp=2, moe_ep=2)`` and ``(tp=1, dp=2, moe_ep=2)`` both mapped to | ||
| ``"dep2"``. That broke ``thorough.py`` work_dir naming and | ||
| ``aiconfigurator.sdk.picking`` ``groupby("parallel")`` dedup. These tests | ||
| pin the post-fix unique encoding. | ||
| """ | ||
| from __future__ import annotations | ||
|
|
||
| import pytest | ||
|
|
||
| pytestmark = [ | ||
| pytest.mark.gpu_0, | ||
| # TODO: revert to pytest.mark.post_merge after pre_merge validation on this PR | ||
| pytest.mark.pre_merge, | ||
| pytest.mark.unit, | ||
| pytest.mark.planner, | ||
| ] | ||
|
|
||
| try: | ||
| from dynamo.planner.config.parallelization import PickedParallelConfig | ||
| from dynamo.profiler.utils.aic_dataframe import make_parallel_label | ||
| except ImportError as e: | ||
| pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) | ||
|
|
||
|
|
||
| # Realistic enumeration: pure TEP / pure DEP / pure TP / dense at | ||
| # num_gpus in {1, 2, 4, 8, 16}. | ||
| _NUM_GPUS = [1, 2, 4, 8, 16] | ||
|
|
||
|
|
||
| def _enumerate() -> list[tuple[int, int, int, int, int]]: | ||
| """Enumerate realistic ``(tp, pp, dp, moe_tp, moe_ep)`` tuples used as | ||
| test inputs — mirrors ``filter_real_silicon_configs`` across four | ||
| archetypes (pure TP-of-experts / pure TEP-of-attention / pure DEP / | ||
| dense) at ``num_gpus ∈ {1, 2, 4, 8, 16}``.""" | ||
| out: list[tuple[int, int, int, int, int]] = [] | ||
| for n in _NUM_GPUS: | ||
| if n == 1: | ||
| out.append((1, 1, 1, 1, 1)) | ||
| continue | ||
| out.append((n, 1, 1, n, 1)) # pure TP-of-experts (moe_tp = tp = n) | ||
| out.append((n, 1, 1, 1, n)) # pure TEP-of-attention (tp = n, moe_ep = n) | ||
| out.append((1, 1, n, 1, n)) # pure DEP (dp = n, moe_ep = n) | ||
| out.append((n, 1, 1, 1, 1)) # dense (no MoE sharding) | ||
| return out | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("tup", _enumerate()) | ||
| def test_label_unique_per_tuple(tup: tuple[int, int, int, int, int]) -> None: | ||
| """Both label producers (``PickedParallelConfig.label`` and | ||
| ``make_parallel_label``) must agree on every enumerated tuple.""" | ||
| tp, pp, dp, moe_tp, moe_ep = tup | ||
| cfg = PickedParallelConfig(tp=tp, pp=pp, dp=dp, moe_tp=moe_tp, moe_ep=moe_ep) | ||
| df = make_parallel_label(tp, pp, dp, moe_tp, moe_ep) | ||
| assert cfg.label() == df | ||
|
|
||
|
|
||
| def test_label_distinct_across_all_enumerated_tuples() -> None: | ||
| """Every distinct 5-tuple must yield a distinct label string.""" | ||
| seen: dict[str, tuple[int, int, int, int, int]] = {} | ||
| for tup in _enumerate(): | ||
| cfg = PickedParallelConfig( | ||
| tp=tup[0], pp=tup[1], dp=tup[2], moe_tp=tup[3], moe_ep=tup[4] | ||
| ) | ||
| label = cfg.label() | ||
| assert ( | ||
| label not in seen | ||
| ), f"label collision: {label!r} from {tup} also from {seen[label]}" | ||
| seen[label] = tup | ||
|
|
||
|
|
||
| def test_regression_dep2_collision() -> None: | ||
| """Explicit regression for the historical ``dep2`` collision.""" | ||
| a = PickedParallelConfig(tp=2, dp=1, moe_ep=2) | ||
| b = PickedParallelConfig(tp=1, dp=2, moe_ep=2) | ||
| assert a != b | ||
| assert a.label() != b.label() | ||
|
|
||
|
|
||
| # (tp, pp, dp, moe_tp, moe_ep, expected_label) | ||
| _FORMAT_PINS: list[tuple[int, int, int, int, int, str]] = [ | ||
| # All-default baseline. | ||
| (1, 1, 1, 1, 1, "tp1"), | ||
| # Default-1 omission: only tp varies. | ||
| (4, 1, 1, 1, 1, "tp4"), | ||
| # Docstring examples. | ||
| (2, 1, 1, 2, 1, "tp2-moetp2"), | ||
| (2, 1, 1, 1, 2, "tp2-moeep2"), | ||
| (1, 1, 2, 1, 2, "tp1-dp2-moeep2"), | ||
| (4, 1, 1, 4, 1, "tp4-moetp4"), | ||
| # PP dimension (otherwise unexercised by _enumerate()). | ||
| (2, 2, 1, 1, 1, "tp2-pp2"), | ||
| # Dense-with-attention-DP (no MoE). | ||
| (2, 1, 2, 1, 1, "tp2-dp2"), | ||
| ] | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("tp,pp,dp,moe_tp,moe_ep,expected", _FORMAT_PINS) | ||
| def test_label_format_pin( | ||
| tp: int, pp: int, dp: int, moe_tp: int, moe_ep: int, expected: str | ||
| ) -> None: | ||
| """Pin the exact label string — downstream consumers (``thorough.py`` | ||
| work_dir, ``aiconfigurator.sdk.picking``) depend on the format, so a | ||
| separator / prefix change must be a deliberate, test-breaking edit.""" | ||
| cfg = PickedParallelConfig(tp=tp, pp=pp, dp=dp, moe_tp=moe_tp, moe_ep=moe_ep) | ||
| assert cfg.label() == expected | ||
| assert make_parallel_label(tp, pp, dp, moe_tp, moe_ep) == expected | ||
100 changes: 100 additions & 0 deletions
100
components/src/dynamo/profiler/tests/unit/test_aic_dataframe.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| """Regression tests for ``make_parallel_label`` collisions. | ||
|
|
||
| The legacy 3-bucket label collapsed distinct 5-tuples to the same string, | ||
| which corrupted ``thorough.py`` sweep work_dir naming and the | ||
| ``aiconfigurator.sdk.picking`` ``groupby("parallel")`` dedup. These tests | ||
| pin the post-fix unique encoding. | ||
| """ | ||
| from __future__ import annotations | ||
|
|
||
| import pandas as pd | ||
| import pytest | ||
|
|
||
| pytestmark = [ | ||
| pytest.mark.gpu_0, | ||
| # TODO: revert to pytest.mark.post_merge after pre_merge validation on this PR | ||
| pytest.mark.pre_merge, | ||
| pytest.mark.unit, | ||
| pytest.mark.planner, | ||
| ] | ||
|
|
||
| try: | ||
| from dynamo.profiler.utils.aic_dataframe import make_parallel_label | ||
| except ImportError as e: | ||
| pytest.skip(f"Skip (missing dependency): {e}", allow_module_level=True) | ||
|
|
||
|
|
||
| _NUM_GPUS = [1, 2, 4, 8, 16] | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
|
|
||
| def _enumerate() -> list[tuple[int, int, int, int, int]]: | ||
| """Enumerate realistic ``(tp, pp, dp, moe_tp, moe_ep)`` tuples used as | ||
| test inputs across four archetypes (pure TP-of-experts / | ||
| pure TEP-of-attention / pure DEP / dense) at | ||
| ``num_gpus ∈ {1, 2, 4, 8, 16}``.""" | ||
| out: list[tuple[int, int, int, int, int]] = [] | ||
| for n in _NUM_GPUS: | ||
| if n == 1: | ||
| out.append((1, 1, 1, 1, 1)) | ||
| continue | ||
| out.append((n, 1, 1, n, 1)) # pure TP-of-experts | ||
| out.append((n, 1, 1, 1, n)) # pure TEP-of-attention | ||
| out.append((1, 1, n, 1, n)) # pure DEP | ||
| out.append((n, 1, 1, 1, 1)) # dense | ||
| return out | ||
|
|
||
|
|
||
| def test_label_distinct_across_all_enumerated_tuples() -> None: | ||
| """Every distinct 5-tuple must yield a distinct label string.""" | ||
| seen: dict[str, tuple[int, int, int, int, int]] = {} | ||
| for tup in _enumerate(): | ||
| label = make_parallel_label(*tup) | ||
| assert ( | ||
| label not in seen | ||
| ), f"label collision: {label!r} from {tup} also from {seen[label]}" | ||
| seen[label] = tup | ||
|
|
||
|
|
||
| def test_groupby_does_not_merge_distinct_topologies() -> None: | ||
| """``aiconfigurator.sdk.picking`` does ``df.groupby("parallel")``. Two | ||
| distinct topologies must produce two groups, not one.""" | ||
| df = pd.DataFrame( | ||
| [ | ||
| {"parallel": make_parallel_label(2, 1, 1, 1, 2), "seq/s/gpu": 100.0}, | ||
| {"parallel": make_parallel_label(1, 1, 2, 1, 2), "seq/s/gpu": 90.0}, | ||
| ] | ||
| ) | ||
| groups = list(df.groupby("parallel")) | ||
| assert ( | ||
| len(groups) == 2 | ||
| ), "groupby merged distinct topologies — label is not injective" | ||
|
|
||
|
|
||
| # (tp, pp, dp, moe_tp, moe_ep, expected_label) | ||
| _FORMAT_PINS: list[tuple[int, int, int, int, int, str]] = [ | ||
| # All-default baseline. | ||
| (1, 1, 1, 1, 1, "tp1"), | ||
| # Default-1 omission: only tp varies. | ||
| (4, 1, 1, 1, 1, "tp4"), | ||
| # Docstring examples (mirrored from PickedParallelConfig.label). | ||
| (2, 1, 1, 2, 1, "tp2-moetp2"), | ||
| (2, 1, 1, 1, 2, "tp2-moeep2"), | ||
| (1, 1, 2, 1, 2, "tp1-dp2-moeep2"), | ||
| (4, 1, 1, 4, 1, "tp4-moetp4"), | ||
| # PP dimension (otherwise unexercised by _enumerate()). | ||
| (2, 2, 1, 1, 1, "tp2-pp2"), | ||
| # Dense-with-attention-DP (no MoE). | ||
| (2, 1, 2, 1, 1, "tp2-dp2"), | ||
| ] | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("tp,pp,dp,moe_tp,moe_ep,expected", _FORMAT_PINS) | ||
| def test_make_parallel_label_format_pin( | ||
| tp: int, pp: int, dp: int, moe_tp: int, moe_ep: int, expected: str | ||
| ) -> None: | ||
| """Pin the exact label string — ``aiconfigurator.sdk.picking`` and | ||
| ``thorough.py`` consume this string verbatim, so format drift must be | ||
| a deliberate, test-breaking edit.""" | ||
| assert make_parallel_label(tp, pp, dp, moe_tp, moe_ep) == expected | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.