Repository navigation
Change the cudagraph distribution from linearly to exponentially-decreasing + grid for mixed prefill #3509
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Change the cudagraph distribution from linearly to exponentially-decreasing + grid for mixed prefill #3509
Changes from 24 commits
a09988a
a1555ba
20798af
24bc8d6
bbf2dd1
3c718e9
ffae8ef
cafe6af
618d016
1f3654d
96f5278
d62f2fc
116a785
ad6753e
9036827
6ea8abc
c06c327
b508d08
b5ec39d
48cda16
9488712
661179a
b347294
33e65c4
8561960
60e7929
579c69d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -213,7 +213,7 @@ class CUDAGraphBatchDimensionBuilder: | |
| """ | ||
|
|
||
| # Constant for rounding token counts when generating CUDA graph batch dimensions | ||
| CUDA_GRAPH_ROUNDER = 8 | ||
| CUDA_GRAPH_ROUNDER = 2 | ||
|
|
||
| @staticmethod | ||
| def _calculate_cuda_graph_token_counts( | ||
|
|
@@ -222,8 +222,9 @@ def _calculate_cuda_graph_token_counts( | |
| """ | ||
| Calculate CUDA graph token counts for a given configuration. | ||
|
|
||
| This method computes evenly-spaced token counts from step_size up to | ||
| cuda_graph_max_tokens, ensuring proper rounding and TP alignment. | ||
| This method computes exponentially-decreasing token counts (powers of 2) | ||
| from cuda_graph_max_tokens down to CUDA_GRAPH_ROUNDER, ensuring proper | ||
| rounding and TP alignment. | ||
|
|
||
| Args: | ||
| tp_size: Tensor parallel size (for alignment) | ||
|
|
@@ -235,35 +236,24 @@ def _calculate_cuda_graph_token_counts( | |
|
|
||
| Example: | ||
| >>> _calculate_cuda_graph_token_counts | ||
| (tp_size=2, num_cuda_graphs=4, cuda_graph_max_tokens=1000) | ||
| [1000, 752, 504, 256] | ||
| (tp_size=1, num_cuda_graphs=8, cuda_graph_max_tokens=128) | ||
| [128, 64, 32, 16, 8, 4, 2, 1] | ||
| """ | ||
| if num_cuda_graphs == -1: | ||
| # automatically determine the number of CUDA graphs to | ||
| # capture based on the `max_requests` value | ||
| cuda_graph_token_counts = ( | ||
| [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, cuda_graph_max_tokens + 1, 16)) | ||
| # Each step in the exponential-decay loop below halves the cudagraph size, so we need | ||
| # ~log2(max_tokens) steps with an extra +2 to leave headroom for dedup/trim. | ||
| auto_n = max(4, int(math.log2(max(2, cuda_graph_max_tokens))) + 2) | ||
|
mathemakitten marked this conversation as resolved.
Resolved
|
||
| return CUDAGraphBatchDimensionBuilder._calculate_cuda_graph_token_counts( | ||
| tp_size=tp_size, num_cuda_graphs=auto_n, cuda_graph_max_tokens=cuda_graph_max_tokens | ||
| ) | ||
| # Align each entry to TP size | ||
| cuda_graph_token_counts = list( | ||
| dict.fromkeys( | ||
| round_up_to_nearest_multiple(s, tp_size) for s in cuda_graph_token_counts | ||
| ) | ||
| ) | ||
| # Clamp to max tokens | ||
| cuda_graph_token_counts = [ | ||
| s for s in cuda_graph_token_counts if s <= cuda_graph_max_tokens | ||
| ] | ||
| if not cuda_graph_token_counts or cuda_graph_token_counts[-1] != cuda_graph_max_tokens: | ||
| cuda_graph_token_counts.append(cuda_graph_max_tokens) | ||
| cuda_graph_token_counts.reverse() | ||
| return cuda_graph_token_counts | ||
|
|
||
| assert num_cuda_graphs >= 1, f"num_cuda_graphs must be >= 1, got {num_cuda_graphs}" | ||
| assert ( | ||
| cuda_graph_max_tokens > 0 | ||
| ), f"cuda_graph_max_tokens must be > 0, got {cuda_graph_max_tokens}" | ||
|
|
||
| rounder = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER | ||
|
|
||
| # Cuda graph step size. | ||
| cuda_graph_step_size = cuda_graph_max_tokens / num_cuda_graphs | ||
| cuda_graph_step_size = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER * int( | ||
|
|
@@ -274,22 +264,41 @@ def _calculate_cuda_graph_token_counts( | |
| # Ensure non-zero step size (can happen when max_tokens < num_cuda_graphs). | ||
| cuda_graph_step_size = max(cuda_graph_step_size, tp_size) | ||
|
|
||
| # round down cuda graph max tokens to be multiple of TP size | ||
| # Round down cuda graph max tokens to be multiple of TP size | ||
| cuda_graph_max_tokens = (cuda_graph_max_tokens // tp_size) * tp_size | ||
|
|
||
| # Cuda graph token counts. | ||
| if num_cuda_graphs == 1: | ||
| cuda_graph_token_counts = [cuda_graph_max_tokens] | ||
| else: | ||
| cuda_graph_token_counts = list( | ||
| range(cuda_graph_step_size, cuda_graph_max_tokens, cuda_graph_step_size) | ||
| ) | ||
| if ( | ||
| len(cuda_graph_token_counts) == 0 | ||
| or cuda_graph_token_counts[-1] != cuda_graph_max_tokens | ||
| ): | ||
| cuda_graph_token_counts.append(cuda_graph_max_tokens) | ||
| cuda_graph_token_counts.reverse() | ||
| return [cuda_graph_max_tokens] | ||
|
|
||
| # Exponentially decreasing, stops after num_cuda_graphs entries | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I also vote to leave the linear-spaced CGs as an option, there's no harm in doing so since the code is already setup, we can just default to exponential in the arguments. One reason for keeping this setting & code is that vLLM uses linear spacing, they just create a ton more graphs than we do because they can create them so quickly and efficiently, and I think that just speaks to how unoptimized our CG system is. So I personally would keep the old option, and just plan to use it in the future.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would be against adding a new flag to toggle the distribution of inference cudagraphs. Inference already has a lot of flags and users don't know how to combine them effectively, and there is no empirical case that makes the argument to keep linear distribution around at the moment. I will leave a TODO to re-enable when someone wants to take it on.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also, #3527 already implements the vllm strategy orthogonal to this.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @mathemakitten on second thought, I am also in favor of slowly phasing out the older strategy - mostly because it's the most stress tested one we have right now. We could make yours default, while having the option to fallback onto that.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. re:3527 - it does use a linear function, but builds a lot more cudagraphs compared to our default strategy. |
||
| # or when below the minimum size. | ||
| cuda_graph_token_counts = [] | ||
|
yobibyte marked this conversation as resolved.
Resolved
|
||
| val = cuda_graph_max_tokens | ||
| for _ in range(num_cuda_graphs): | ||
| # Round down to multiple of rounder, then up to multiple of TP size | ||
| rounded = max(rounder, (val // rounder) * rounder) | ||
|
mathemakitten marked this conversation as resolved.
Resolved
|
||
| rounded = math.ceil(rounded / tp_size) * tp_size | ||
| if rounded not in cuda_graph_token_counts: | ||
| cuda_graph_token_counts.append(rounded) | ||
|
mathemakitten marked this conversation as resolved.
Resolved
|
||
| val //= 2 | ||
| if val < 1: | ||
| break | ||
|
|
||
| # Ensure cuda_graph_max_tokens is always included | ||
| if cuda_graph_token_counts[0] != cuda_graph_max_tokens: | ||
| cuda_graph_token_counts.insert(0, cuda_graph_max_tokens) | ||
|
|
||
| # Include a (possibly extra) size-1 graph | ||
| if cuda_graph_token_counts[-1] != tp_size: | ||
| cuda_graph_token_counts.append(tp_size) | ||
|
|
||
| # Trim from the middle if we exceed num_cuda_graphs requested by the user | ||
| # Since num_cuda_graphs >= 1, this only runs when we have at least 2 elements. | ||
| while len(cuda_graph_token_counts) > num_cuda_graphs: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Does this line need to be Otherwise
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We have a guarantee that |
||
| cuda_graph_token_counts.pop(-2) | ||
|
mathemakitten marked this conversation as resolved.
Resolved
|
||
|
|
||
| assert len(cuda_graph_token_counts) <= num_cuda_graphs | ||
| assert cuda_graph_max_tokens in cuda_graph_token_counts | ||
|
|
||
| return cuda_graph_token_counts | ||
|
|
||
|
|
@@ -402,6 +411,23 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int | |
| ) | ||
| ) | ||
|
|
||
| # Include the smallest decode-only graphs when auto-sizing (num_cuda_graphs == -1). | ||
| # Without this, TP alignment (size 1 -> tp_size) and the num_speculative_tokens floor | ||
| # division may drop the size 1 and size 2 graph sizes. | ||
|
mathemakitten marked this conversation as resolved.
Resolved
|
||
| if num_cuda_graphs == -1: | ||
| spec_unit = num_speculative_tokens + 1 | ||
| min_decode_tokens = math.lcm(spec_unit, tp_size) | ||
| for req_count_multiple in (1, 2): | ||
| floor_tokens = min_decode_tokens * req_count_multiple | ||
| if ( | ||
| floor_tokens <= cuda_graph_max_tokens_decode | ||
| and floor_tokens not in cuda_graph_decode_token_counts | ||
| ): | ||
| cuda_graph_decode_token_counts.append(floor_tokens) | ||
| cuda_graph_decode_token_counts = sorted( | ||
| set(cuda_graph_decode_token_counts), reverse=True | ||
| ) | ||
|
|
||
| cuda_graph_batch_dimensions_list = [] | ||
| if num_cuda_graphs is None: | ||
| cuda_graph_batch_dimensions_list = [] | ||
|
|
@@ -419,34 +445,55 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int | |
| token_count=token_count, prefill_req_count=0, decode_req_count=decode_req_count | ||
| ) | ||
| else: | ||
| # Mixed prefill and decode mode | ||
| # Mixed prefill and decode mode. | ||
| # | ||
| # Generate mixed CGs across a geometric P-grid rather than a single fixed P | ||
| # value. A captured graph at P=k bakes in a token layout for k prefill slots; | ||
| # a real batch with P != k pays for unused slot padding, which can overshoot | ||
| # the captured token_count and make the graph unusable. Geometric spacing | ||
| # bounds the relative overhead per real batch (worst case ~2x P slack) while | ||
| # keeping the total CG count log-bounded. | ||
| # | ||
| # `cuda_graph_mixed_prefill_request_count` is now used only as an on/off | ||
| # toggle for mixed CGs (>0 enables, <=0 routes to decode-only above). The | ||
| # P value it used to specify is superseded by the grid. | ||
| p_values = [] | ||
| p = 1 | ||
| while p < max_requests: | ||
| p_values.append(p) | ||
| p *= 2 | ||
| if not p_values or p_values[-1] != max_requests: | ||
| p_values.append(max_requests) | ||
|
|
||
| # Create prefill and mixed dimensions with full token counts | ||
| for size in cuda_graph_prefill_token_counts: | ||
| assert size % tp_size == 0 | ||
| prefill_req_count = min(cuda_graph_mixed_prefill_request_count, max_requests) | ||
| decode_req_count = max( | ||
| 0, | ||
| min( | ||
| (size - prefill_req_count) // (num_speculative_tokens + 1), | ||
| max_requests - prefill_req_count, | ||
| ), | ||
| ) | ||
| add_if_valid( | ||
| token_count=size, | ||
| prefill_req_count=prefill_req_count, | ||
| decode_req_count=decode_req_count, | ||
| ) | ||
| for prefill_req_count in p_values: | ||
| decode_req_count = max( | ||
| 0, | ||
| min( | ||
| (size - prefill_req_count) // (num_speculative_tokens + 1), | ||
| max_requests - prefill_req_count, | ||
| ), | ||
| ) | ||
| # Skip token_count == 1 with prefill_req == 1: the gather kernel asserts | ||
| # on index >= 1 against a 1-element tensor at capture time. Larger | ||
| # `(size, size, 0)` shapes (each prefill = 1 token, total batch >= 2) are | ||
| # fine because the gather has multiple indices to read. | ||
| if size < 2: | ||
| continue | ||
| add_if_valid( | ||
| token_count=size, | ||
| prefill_req_count=prefill_req_count, | ||
| decode_req_count=decode_req_count, | ||
| ) | ||
| # We need to ensure the prefill requests are shorter than the max sequence length, | ||
| # considering the one decode token is used for prefill request construction | ||
| prefill_only_minimal_num = max( | ||
| cuda_graph_mixed_prefill_request_count, | ||
| math.ceil(size / max(1, max_sequence_length - 1)), | ||
| ) | ||
| if prefill_only_minimal_num < max_requests: | ||
| prefill_only_minimal_num = max(1, math.ceil(size / max(1, max_sequence_length - 1))) | ||
| if prefill_only_minimal_num < max_requests and size >= 2: | ||
| prefill_req_count = max(prefill_only_minimal_num, min(max_requests, size)) | ||
| add_if_valid( | ||
| token_count=size, | ||
| prefill_req_count=max(prefill_only_minimal_num, min(max_requests, size)), | ||
| decode_req_count=0, | ||
| token_count=size, prefill_req_count=prefill_req_count, decode_req_count=0 | ||
| ) | ||
|
|
||
| # Create decode-only dimensions with optimized token counts | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -44,7 +44,7 @@ | |
| ) | ||
| from megatron.core.inference.utils import Counter, InferenceMode, await_process_call | ||
| from megatron.core.process_groups_config import ProcessGroupCollection | ||
| from megatron.core.transformer.cuda_graphs import delete_cuda_graphs | ||
| from megatron.core.transformer.cuda_graphs import CudaGraphManager, delete_cuda_graphs | ||
| from megatron.core.transformer.enums import InferenceCudaGraphScope | ||
| from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction | ||
| from megatron.core.utils import ( | ||
|
|
@@ -133,13 +133,42 @@ class EngineSuspendedError(Exception): | |
|
|
||
| def format_mem_bytes(mem_bytes): | ||
| """Convert a byte count to a human-readable string in tb, gb, mb, kb, or bytes.""" | ||
| if mem_bytes < 0: | ||
| return "-" + format_mem_bytes(-mem_bytes) | ||
| for power, suffix in [(4, "tb"), (3, "gb"), (2, "mb"), (1, "kb"), (0, "bytes")]: | ||
| suffix_bytes = 1024**power | ||
| if mem_bytes >= suffix_bytes: | ||
| return "%.1f %s" % (mem_bytes / suffix_bytes, suffix) | ||
| return "%d bytes" % mem_bytes | ||
|
|
||
|
|
||
| def _cuda_graph_mempool_bytes(): | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: can you make return type |
||
| """Return (reserved, allocated) bytes belonging to the global CUDA graph mempool. | ||
|
|
||
| PyTorch's `torch.cuda.memory_stats()` reports process-wide totals that mix in | ||
| every other allocation (KV cache, NCCL workspaces, layer scratch). To isolate | ||
| growth caused by graph capture, we walk `torch.cuda.memory_snapshot()` and | ||
| filter segments by their `segment_pool_id` against the graph pool handle. | ||
| Returns (0, 0) if the pool hasn't been created yet. | ||
| """ | ||
| pool_id = CudaGraphManager.global_mempool | ||
| if pool_id is None: | ||
| return 0, 0 | ||
| reserved = 0 | ||
| allocated = 0 | ||
| for seg in torch.cuda.memory_snapshot(): | ||
| seg_pool_id = ( | ||
| seg.get("segment_pool_id") | ||
| or seg.get("private_pool_id") | ||
| or seg.get("pool_id") | ||
| or seg.get("pool") | ||
| ) | ||
| if seg_pool_id == pool_id: | ||
| reserved += seg.get("total_size", 0) | ||
| allocated += seg.get("allocated_size", 0) | ||
| return reserved, allocated | ||
|
|
||
|
|
||
| @dataclass(kw_only=True) | ||
| class RequestEntry: | ||
| """Entry in the engine's `self.requests` dict.""" | ||
|
|
@@ -345,8 +374,16 @@ def create_cuda_graphs(self, reset_context: bool = True): | |
| controller = self.controller | ||
|
|
||
| time_start = time.time() | ||
| torch.cuda.reset_peak_memory_stats() | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it safe to reset the peak memory stats here for every request entry? Would this disrupt any existing memory recording? |
||
| mem_stats_start = torch.cuda.memory_stats() | ||
|
|
||
| # Snapshot of process-wide stats for the "total memory used by capture" summary. | ||
| start_proc_reserved = mem_stats_start["reserved_bytes.all.current"] | ||
| start_proc_alloc = mem_stats_start["allocated_bytes.all.current"] | ||
|
|
||
| # Pool-scoped baselines for the per-iteration deltas. | ||
| prev_pool_reserved, prev_pool_alloc = _cuda_graph_mempool_bytes() | ||
|
|
||
| logging.info("> dynamic_engine.py: building cuda graphs for ") | ||
| for graph in context.cuda_graph_batch_dimensions_list: | ||
| logging.info(graph) | ||
|
|
@@ -427,27 +464,44 @@ def create_cuda_graphs(self, reset_context: bool = True): | |
|
|
||
| context.reset() | ||
|
|
||
| # Per-iteration memory accounting, scoped to the CUDA-graph mempool. | ||
| # This isolates pool growth from process-wide scratch churn (KV cache, | ||
| # NCCL workspaces, etc.) that pollutes `torch.cuda.memory_stats()`. | ||
| pool_reserved, pool_alloc = _cuda_graph_mempool_bytes() | ||
| logging.info( | ||
| " [graph %d/%d] %s | pool reserved=%s (Δiter=%s) " "pool allocated=%s (Δiter=%s)", | ||
| tbar_idx + 1, | ||
| len(context.cuda_graph_batch_dimensions_list), | ||
| cuda_graph_batch_dimension, | ||
| format_mem_bytes(pool_reserved), | ||
| format_mem_bytes(pool_reserved - prev_pool_reserved), | ||
| format_mem_bytes(pool_alloc), | ||
| format_mem_bytes(pool_alloc - prev_pool_alloc), | ||
| ) | ||
| prev_pool_reserved, prev_pool_alloc = pool_reserved, pool_alloc | ||
| torch.cuda.reset_peak_memory_stats() | ||
|
|
||
| if mtp_warmup_enabled and mtp_seen_batch_sizes: | ||
| logging.info("> MTP CUDA graph warmup: %d batch size(s)", len(mtp_seen_batch_sizes)) | ||
|
|
||
| # Memory usage. | ||
| time_end = time.time() | ||
| mem_stats_end = torch.cuda.memory_stats() | ||
| final_pool_reserved, final_pool_alloc = _cuda_graph_mempool_bytes() | ||
| capture_stats = { | ||
| "time": time_end - time_start, | ||
| "allocated_bytes": ( | ||
| mem_stats_end["allocated_bytes.all.current"] | ||
| - mem_stats_start["allocated_bytes.all.current"] | ||
| ), | ||
| "reserved_bytes": ( | ||
| mem_stats_end["reserved_bytes.all.current"] | ||
| - mem_stats_start["reserved_bytes.all.current"] | ||
| ), | ||
| "allocated_bytes": (mem_stats_end["allocated_bytes.all.current"] - start_proc_alloc), | ||
| "reserved_bytes": (mem_stats_end["reserved_bytes.all.current"] - start_proc_reserved), | ||
| "pool_reserved_bytes": final_pool_reserved, | ||
| "pool_allocated_bytes": final_pool_alloc, | ||
| } | ||
| logging.info( | ||
| "> built cuda graph(s) in %.2f sec, with total memory usage: " | ||
| "allocated %s, reserved %s.", | ||
| "> built cuda graph(s) in %.2f sec. " | ||
| "Mempool: reserved %s, allocated %s. " | ||
| "Process-wide delta: allocated %s, reserved %s.", | ||
| capture_stats["time"], | ||
| format_mem_bytes(capture_stats["pool_reserved_bytes"]), | ||
| format_mem_bytes(capture_stats["pool_allocated_bytes"]), | ||
| format_mem_bytes(capture_stats["allocated_bytes"]), | ||
| format_mem_bytes(capture_stats["reserved_bytes"]), | ||
| ) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.