Repository navigation
[Fix] nccl: Disable graph buffer registration whenever a captured graph can hold a collective #43216
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Fix] nccl: Disable graph buffer registration whenever a captured graph can hold a collective #43216
Changes from 2 commits
c348490
857be0d
8c615a8
7967f9c
112b009
44b2910
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -345,26 +345,35 @@ def handle_data_parallelism(server_args: Any): | |
|
|
||
| run_post_process_pass(server_args, _tp_lm_head_all_to_all_default) | ||
| run_post_process_pass(server_args, _dp_lm_head_validation) | ||
| if resolving_view(server_args).enable_tp_lm_head_all_to_all: | ||
| if _graph_may_capture_collectives(server_args): | ||
| _disable_nccl_graph_buffer_registration( | ||
| "the graph-captured TP LM-head all-to-all can deadlock with " | ||
| "registered buffers" | ||
| ) | ||
| if _graph_pool_is_pausable(server_args): | ||
| _disable_nccl_graph_buffer_registration( | ||
| "graph replay can hang once torch_memory_saver resume remaps the " | ||
| "graph pool, leaving the capture-time registrations on released pages" | ||
| "graph-captured collectives run on graph-pool temporaries whose " | ||
| "addresses the pool also hands to other tensors" | ||
| ) | ||
|
|
||
|
|
||
| def _graph_pool_is_pausable(server_args: Any) -> bool: | ||
| """Whether CUDA graphs are captured into torch_memory_saver memory, which | ||
| release/resume of the `cuda_graph` tag remaps to new physical pages at the | ||
| same virtual addresses.""" | ||
| return bool( | ||
| resolving_view(server_args).enable_memory_saver | ||
| and envs.SGLANG_MEMORY_SAVER_CUDA_GRAPH.get() | ||
| ) | ||
| def _graph_may_capture_collectives(server_args: Any) -> bool: | ||
| """Whether any captured CUDA graph can hold a cross-rank collective. | ||
|
|
||
| Capture binds a collective's send/recv buffers to graph-pool temporaries, | ||
| and the pool hands those same addresses to other tensors. Which collective | ||
| got captured, which phase captured it, and whether the pool is later | ||
| remapped by torch_memory_saver only change how that aliasing is reached - | ||
| not whether it exists - so the guard tracks the hazard rather than the | ||
| feature flags that expose it. A decode-only check would miss a prefill | ||
| capture into a pausable pool, which the pool guard used to cover. | ||
|
|
||
| Errs toward True. Later passes can still narrow cuda_graph_config, and | ||
| turning the registration off has no measured throughput cost, so a false | ||
| positive is cheaper than a missed deployment shape. | ||
| """ | ||
| view = resolving_view(server_args) | ||
| if max(view.tp_size, view.ep_size, view.dp_size) <= 1: | ||
| return False | ||
| config = getattr(view, "cuda_graph_config", None) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Codex]: Could this use direct access to view.cuda_graph_config, retaining the None check if needed? The resolution pipeline initializes this field before handle_data_parallelism, and that handler already accesses it directly in the attention-DP branch. The getattr fallback appears unnecessary and can hide a missing-field contract violation. |
||
| if config is None: | ||
| return True | ||
| return any(config[phase].backend != Backend.DISABLED for phase in Phase.ALL) | ||
|
|
||
|
|
||
| def _disable_nccl_graph_buffer_registration(reason: str) -> None: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3995,23 +3995,24 @@ def test_an_explicit_value_equal_to_the_generic_one_still_reads_as_set(self): | |
|
|
||
|
|
||
| class TestTpLmHeadAllToAllNcclGraphRegister(unittest.TestCase): | ||
| """The graph-captured TP LM-head all-to-all must not run with NCCL's | ||
| graph buffer registration: registered graph-pool temporaries deadlock the | ||
| exchange under DP-rank ramps.""" | ||
| """A decode graph that can capture a collective must not run with NCCL's | ||
| graph buffer registration: the registered graph-pool temporaries alias | ||
| tensors the pool hands out later, and replay then deadlocks.""" | ||
|
|
||
| def _resolve(self, **kwargs): | ||
| # The handler reads the DP-adjusted prefill knobs the pipeline would | ||
| # have settled by then; the dummy-model pipeline itself returns early. | ||
| server_args = ServerArgs( | ||
| fields = dict( | ||
| model_path="dummy", | ||
| attn_dp_size=2, | ||
| tp_size=2, | ||
| chunked_prefill_size=8192, | ||
| cuda_graph_config=CudaGraphConfig( | ||
| prefill=PhaseConfig(backend=Backend.DISABLED) | ||
| ), | ||
| **kwargs, | ||
| ) | ||
| fields.update(kwargs) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Codex]: Could we add a regression case with attn_dp_size=1, tp_size=2, enable_tp_lm_head_all_to_all=False, and enable_memory_saver=False, asserting that an initially unset NCCL_GRAPH_REGISTER becomes 0? The helper defaults to attention DP, so the new cases without LM-head all-to-all still do not directly cover the plain TP deployment that motivates this change. |
||
| server_args = ServerArgs(**fields) | ||
| parallel_hook.handle_data_parallelism(server_args) | ||
| return server_args | ||
|
|
||
|
|
@@ -4031,9 +4032,9 @@ def test_explicit_nccl_graph_register_is_kept(self): | |
| self.assertEqual(os.environ["NCCL_GRAPH_REGISTER"], "1") | ||
| self.assertIn("NCCL_GRAPH_REGISTER=1", "\n".join(logs.output)) | ||
|
|
||
| def test_without_all_to_all_env_is_untouched(self): | ||
| # Unified serving keeps the all-to-all off by default, and a decode | ||
| # node with the DP LM head never takes the all-to-all. | ||
| def test_without_all_to_all_registration_still_defaults_off(self): | ||
| # The hazard is the graph-captured collective, not the feature that | ||
| # happens to issue it, so the default holds with the all-to-all off. | ||
| for kwargs in ( | ||
| {}, | ||
| {"disaggregation_mode": "decode", "enable_dp_lm_head": True}, | ||
|
|
@@ -4044,7 +4045,52 @@ def test_without_all_to_all_env_is_untouched(self): | |
| self.assertFalse( | ||
| resolution_result(server_args, "enable_tp_lm_head_all_to_all") | ||
| ) | ||
| self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) | ||
| self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") | ||
|
|
||
| def test_pausable_graph_pool_defaults_off(self): | ||
| # Isolates the torch_memory_saver trigger: the all-to-all stays off, so | ||
| # only the remapped graph pool can account for the default. | ||
| with patch.dict(os.environ, {}, clear=False): | ||
| os.environ.pop("NCCL_GRAPH_REGISTER", None) | ||
| server_args = self._resolve(enable_memory_saver=True) | ||
| self.assertFalse( | ||
| resolution_result(server_args, "enable_tp_lm_head_all_to_all") | ||
| ) | ||
| self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") | ||
|
|
||
| def test_prefill_only_capture_defaults_off(self): | ||
| # Decode capture off, prefill still capturing into the same pausable | ||
| # pool: the collective is in a graph either way, so the registration has | ||
| # to go regardless of which phase holds it. | ||
| with patch.dict(os.environ, {}, clear=False): | ||
| os.environ.pop("NCCL_GRAPH_REGISTER", None) | ||
| self._resolve( | ||
| enable_memory_saver=True, | ||
| cuda_graph_config=CudaGraphConfig( | ||
| decode=PhaseConfig(backend=Backend.DISABLED), | ||
| prefill=PhaseConfig(backend=Backend.FULL), | ||
| ), | ||
| ) | ||
| self.assertEqual(os.environ.get("NCCL_GRAPH_REGISTER"), "0") | ||
|
|
||
| def test_single_rank_env_is_untouched(self): | ||
| # One rank issues no collective, so a capture registers nothing. | ||
| with patch.dict(os.environ, {}, clear=False): | ||
| os.environ.pop("NCCL_GRAPH_REGISTER", None) | ||
| self._resolve(attn_dp_size=1, tp_size=1) | ||
| self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) | ||
|
|
||
| def test_disabled_decode_graph_env_is_untouched(self): | ||
| # Without a decode graph no registration outlives the collective. | ||
| with patch.dict(os.environ, {}, clear=False): | ||
| os.environ.pop("NCCL_GRAPH_REGISTER", None) | ||
| self._resolve( | ||
| cuda_graph_config=CudaGraphConfig( | ||
| decode=PhaseConfig(backend=Backend.DISABLED), | ||
| prefill=PhaseConfig(backend=Backend.DISABLED), | ||
| ) | ||
| ) | ||
| self.assertNotIn("NCCL_GRAPH_REGISTER", os.environ) | ||
|
|
||
|
|
||
| class TestDcpCommBackendDefault(CustomTestCase): | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Codex]: Could we qualify this docstring as a conservative workaround and limit the throughput statement to the measured deployment? The PR description says the internal NCCL root cause remains open and the performance comparison has one run per configuration. The aliasing explanation above reads as an established mechanism, and that measurement does not establish the cost of disabling registration across all deployment shapes covered by this guard.