Skip to content
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
a09988a
Change from linear to exponentially decay cudagraph sizes
mathemakitten Feb 20, 2026
a1555ba
Merge branch 'main' into helenn-exponential-decay-cudagraph-sizes
mathemakitten Feb 20, 2026
20798af
Maybe include a size-1 graph
mathemakitten Feb 20, 2026
24bc8d6
Merge branch 'helenn-exponential-decay-cudagraph-sizes' of https://gi…
mathemakitten Feb 20, 2026
bbf2dd1
Merge branch 'main' of https://gitlab-master.nvidia.com/ADLR/megatron…
mathemakitten Feb 20, 2026
3c718e9
Update test_cuda_graph_token_counts
mathemakitten Feb 20, 2026
ffae8ef
Merge branch 'main' into helenn-exponential-decay-cudagraph-sizes
mathemakitten Feb 20, 2026
cafe6af
address comments
mathemakitten Feb 23, 2026
618d016
Merge branch 'helenn-exponential-decay-cudagraph-sizes' of https://gi…
mathemakitten Feb 23, 2026
1f3654d
Merge branch 'main' into helenn-exponential-decay-cudagraph-sizes
mathemakitten Feb 23, 2026
96f5278
keshav comments
mathemakitten Feb 23, 2026
d62f2fc
Merge branch 'helenn-exponential-decay-cudagraph-sizes' of https://gi…
mathemakitten Feb 23, 2026
116a785
Merge branch 'main' into helenn-exponential-decay-cudagraph-sizes
mathemakitten Feb 23, 2026
ad6753e
address comments
mathemakitten Mar 10, 2026
9036827
merge
mathemakitten May 18, 2026
6ea8abc
update print in example script to differentiate reserved / allocated
mathemakitten May 18, 2026
c06c327
exponential decay of graph size
mathemakitten May 18, 2026
b508d08
better logging for checking pool reuse etc
mathemakitten May 18, 2026
b5ec39d
fix merge w main
mathemakitten May 18, 2026
48cda16
fix import
mathemakitten May 18, 2026
9488712
minor cleanup
mathemakitten May 18, 2026
661179a
fix failing test
mathemakitten May 18, 2026
b347294
fix test
mathemakitten May 18, 2026
33e65c4
fix tests and recover small graphs for perf
mathemakitten May 18, 2026
8561960
keshav fixes
mathemakitten May 19, 2026
60e7929
merge main to get new tests
mathemakitten May 20, 2026
579c69d
Merge branch 'main' into helenn-exponential-decay-cudagraph-sizes
mathemakitten Jun 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/inference/gpt/gpt_dynamic_inference.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ def escape_str(s):
print(
f"{setup_prefix} … " f"throughput: {throughput:.3f} tok/s … ",
f"total time: {total_time:.3f}s … "
f"mem {peak_alloc_gb:.1f}/{peak_resvd_gb:.1f} GB … "
f"mem {peak_alloc_gb:.1f} allocated/{peak_resvd_gb:.1f} reserved GB … "
f"steps: {engine.context.step_count:d} … "
f"capture {capture_str}",
)
Expand Down
163 changes: 105 additions & 58 deletions megatron/core/inference/batch_dimensions_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ class CUDAGraphBatchDimensionBuilder:
"""

# Constant for rounding token counts when generating CUDA graph batch dimensions
CUDA_GRAPH_ROUNDER = 8
CUDA_GRAPH_ROUNDER = 2

@staticmethod
def _calculate_cuda_graph_token_counts(
Expand All @@ -222,8 +222,9 @@ def _calculate_cuda_graph_token_counts(
"""
Calculate CUDA graph token counts for a given configuration.

This method computes evenly-spaced token counts from step_size up to
cuda_graph_max_tokens, ensuring proper rounding and TP alignment.
This method computes exponentially-decreasing token counts (powers of 2)
from cuda_graph_max_tokens down to CUDA_GRAPH_ROUNDER, ensuring proper
rounding and TP alignment.

Args:
tp_size: Tensor parallel size (for alignment)
Expand All @@ -235,35 +236,24 @@ def _calculate_cuda_graph_token_counts(

Example:
>>> _calculate_cuda_graph_token_counts
(tp_size=2, num_cuda_graphs=4, cuda_graph_max_tokens=1000)
[1000, 752, 504, 256]
(tp_size=1, num_cuda_graphs=8, cuda_graph_max_tokens=128)
[128, 64, 32, 16, 8, 4, 2, 1]
"""
if num_cuda_graphs == -1:
Comment thread
mathemakitten marked this conversation as resolved.
Resolved
# automatically determine the number of CUDA graphs to
# capture based on the `max_requests` value
cuda_graph_token_counts = (
[1, 2, 4] + list(range(8, 256, 8)) + list(range(256, cuda_graph_max_tokens + 1, 16))
# Each step in the exponential-decay loop below halves the cudagraph size, so we need
# ~log2(max_tokens) steps with an extra +2 to leave headroom for dedup/trim.
auto_n = max(4, int(math.log2(max(2, cuda_graph_max_tokens))) + 2)
Comment thread
mathemakitten marked this conversation as resolved.
Resolved
return CUDAGraphBatchDimensionBuilder._calculate_cuda_graph_token_counts(
tp_size=tp_size, num_cuda_graphs=auto_n, cuda_graph_max_tokens=cuda_graph_max_tokens
)
# Align each entry to TP size
cuda_graph_token_counts = list(
dict.fromkeys(
round_up_to_nearest_multiple(s, tp_size) for s in cuda_graph_token_counts
)
)
# Clamp to max tokens
cuda_graph_token_counts = [
s for s in cuda_graph_token_counts if s <= cuda_graph_max_tokens
]
if not cuda_graph_token_counts or cuda_graph_token_counts[-1] != cuda_graph_max_tokens:
cuda_graph_token_counts.append(cuda_graph_max_tokens)
cuda_graph_token_counts.reverse()
return cuda_graph_token_counts

assert num_cuda_graphs >= 1, f"num_cuda_graphs must be >= 1, got {num_cuda_graphs}"
assert (
cuda_graph_max_tokens > 0
), f"cuda_graph_max_tokens must be > 0, got {cuda_graph_max_tokens}"

rounder = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER

# Cuda graph step size.
cuda_graph_step_size = cuda_graph_max_tokens / num_cuda_graphs
cuda_graph_step_size = CUDAGraphBatchDimensionBuilder.CUDA_GRAPH_ROUNDER * int(
Expand All @@ -274,22 +264,41 @@ def _calculate_cuda_graph_token_counts(
# Ensure non-zero step size (can happen when max_tokens < num_cuda_graphs).
cuda_graph_step_size = max(cuda_graph_step_size, tp_size)

# round down cuda graph max tokens to be multiple of TP size
# Round down cuda graph max tokens to be multiple of TP size
cuda_graph_max_tokens = (cuda_graph_max_tokens // tp_size) * tp_size

# Cuda graph token counts.
if num_cuda_graphs == 1:
cuda_graph_token_counts = [cuda_graph_max_tokens]
else:
cuda_graph_token_counts = list(
range(cuda_graph_step_size, cuda_graph_max_tokens, cuda_graph_step_size)
)
if (
len(cuda_graph_token_counts) == 0
or cuda_graph_token_counts[-1] != cuda_graph_max_tokens
):
cuda_graph_token_counts.append(cuda_graph_max_tokens)
cuda_graph_token_counts.reverse()
return [cuda_graph_max_tokens]

# Exponentially decreasing, stops after num_cuda_graphs entries

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also vote to leave the linear-spaced CGs as an option, there's no harm in doing so since the code is already setup, we can just default to exponential in the arguments.

One reason for keeping this setting & code is that vLLM uses linear spacing, they just create a ton more graphs than we do because they can create them so quickly and efficiently, and I think that just speaks to how unoptimized our CG system is. So I personally would keep the old option, and just plan to use it in the future.

@mathemakitten mathemakitten Feb 23, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would be against adding a new flag to toggle the distribution of inference cudagraphs. Inference already has a lot of flags and users don't know how to combine them effectively, and there is no empirical case that makes the argument to keep linear distribution around at the moment. I will leave a TODO to re-enable when someone wants to take it on.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, #3527 already implements the vllm strategy orthogonal to this.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mathemakitten on second thought, I am also in favor of slowly phasing out the older strategy - mostly because it's the most stress tested one we have right now. We could make yours default, while having the option to fallback onto that.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

re:3527 - it does use a linear function, but builds a lot more cudagraphs compared to our default strategy.

# or when below the minimum size.
cuda_graph_token_counts = []
Comment thread
yobibyte marked this conversation as resolved.
Resolved
val = cuda_graph_max_tokens
for _ in range(num_cuda_graphs):
# Round down to multiple of rounder, then up to multiple of TP size
rounded = max(rounder, (val // rounder) * rounder)
Comment thread
mathemakitten marked this conversation as resolved.
Resolved
rounded = math.ceil(rounded / tp_size) * tp_size
if rounded not in cuda_graph_token_counts:
cuda_graph_token_counts.append(rounded)
Comment thread
mathemakitten marked this conversation as resolved.
Resolved
val //= 2
if val < 1:
break

# Ensure cuda_graph_max_tokens is always included
if cuda_graph_token_counts[0] != cuda_graph_max_tokens:
cuda_graph_token_counts.insert(0, cuda_graph_max_tokens)

# Include a (possibly extra) size-1 graph
if cuda_graph_token_counts[-1] != tp_size:
cuda_graph_token_counts.append(tp_size)

# Trim from the middle if we exceed num_cuda_graphs requested by the user
# Since num_cuda_graphs >= 1, this only runs when we have at least 2 elements.
while len(cuda_graph_token_counts) > num_cuda_graphs:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this line need to be

while len(cuda_graph_token_counts) > num_cuda_graphs and len(cuda_graph_token_counts) >= 2

Otherwise pop(-2) might give an index error or (even worse) silently wrap around?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have a guarantee that num_cuda_graphs >= 1 at the top of the block and we also check while len(cuda_graph_token_counts) > num_cuda_graphs, so we're actually already guaranteed that len(cuda_graph_token_counts) >= 2 when this block runs.

cuda_graph_token_counts.pop(-2)
Comment thread
mathemakitten marked this conversation as resolved.
Resolved

assert len(cuda_graph_token_counts) <= num_cuda_graphs
assert cuda_graph_max_tokens in cuda_graph_token_counts

return cuda_graph_token_counts

Expand Down Expand Up @@ -402,6 +411,23 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int
)
)

# Include the smallest decode-only graphs when auto-sizing (num_cuda_graphs == -1).
# Without this, TP alignment (size 1 -> tp_size) and the num_speculative_tokens floor
# division may drop the size 1 and size 2 graph sizes.
Comment thread
mathemakitten marked this conversation as resolved.
Resolved
if num_cuda_graphs == -1:
spec_unit = num_speculative_tokens + 1
min_decode_tokens = math.lcm(spec_unit, tp_size)
for req_count_multiple in (1, 2):
floor_tokens = min_decode_tokens * req_count_multiple
if (
floor_tokens <= cuda_graph_max_tokens_decode
and floor_tokens not in cuda_graph_decode_token_counts
):
cuda_graph_decode_token_counts.append(floor_tokens)
cuda_graph_decode_token_counts = sorted(
set(cuda_graph_decode_token_counts), reverse=True
)

cuda_graph_batch_dimensions_list = []
if num_cuda_graphs is None:
cuda_graph_batch_dimensions_list = []
Expand All @@ -419,34 +445,55 @@ def add_if_valid(token_count: int, prefill_req_count: int, decode_req_count: int
token_count=token_count, prefill_req_count=0, decode_req_count=decode_req_count
)
else:
# Mixed prefill and decode mode
# Mixed prefill and decode mode.
#
# Generate mixed CGs across a geometric P-grid rather than a single fixed P
# value. A captured graph at P=k bakes in a token layout for k prefill slots;
# a real batch with P != k pays for unused slot padding, which can overshoot
# the captured token_count and make the graph unusable. Geometric spacing
# bounds the relative overhead per real batch (worst case ~2x P slack) while
# keeping the total CG count log-bounded.
#
# `cuda_graph_mixed_prefill_request_count` is now used only as an on/off
# toggle for mixed CGs (>0 enables, <=0 routes to decode-only above). The
# P value it used to specify is superseded by the grid.
p_values = []
p = 1
while p < max_requests:
p_values.append(p)
p *= 2
if not p_values or p_values[-1] != max_requests:
p_values.append(max_requests)

# Create prefill and mixed dimensions with full token counts
for size in cuda_graph_prefill_token_counts:
assert size % tp_size == 0
prefill_req_count = min(cuda_graph_mixed_prefill_request_count, max_requests)
decode_req_count = max(
0,
min(
(size - prefill_req_count) // (num_speculative_tokens + 1),
max_requests - prefill_req_count,
),
)
add_if_valid(
token_count=size,
prefill_req_count=prefill_req_count,
decode_req_count=decode_req_count,
)
for prefill_req_count in p_values:
decode_req_count = max(
0,
min(
(size - prefill_req_count) // (num_speculative_tokens + 1),
max_requests - prefill_req_count,
),
)
# Skip token_count == 1 with prefill_req == 1: the gather kernel asserts
# on index >= 1 against a 1-element tensor at capture time. Larger
# `(size, size, 0)` shapes (each prefill = 1 token, total batch >= 2) are
# fine because the gather has multiple indices to read.
if size < 2:
continue
add_if_valid(
token_count=size,
prefill_req_count=prefill_req_count,
decode_req_count=decode_req_count,
)
# We need to ensure the prefill requests are shorter than the max sequence length,
# considering the one decode token is used for prefill request construction
prefill_only_minimal_num = max(
cuda_graph_mixed_prefill_request_count,
math.ceil(size / max(1, max_sequence_length - 1)),
)
if prefill_only_minimal_num < max_requests:
prefill_only_minimal_num = max(1, math.ceil(size / max(1, max_sequence_length - 1)))
if prefill_only_minimal_num < max_requests and size >= 2:
prefill_req_count = max(prefill_only_minimal_num, min(max_requests, size))
add_if_valid(
token_count=size,
prefill_req_count=max(prefill_only_minimal_num, min(max_requests, size)),
decode_req_count=0,
token_count=size, prefill_req_count=prefill_req_count, decode_req_count=0
)

# Create decode-only dimensions with optimized token counts
Expand Down
76 changes: 65 additions & 11 deletions megatron/core/inference/engines/dynamic_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
)
from megatron.core.inference.utils import Counter, InferenceMode, await_process_call
from megatron.core.process_groups_config import ProcessGroupCollection
from megatron.core.transformer.cuda_graphs import delete_cuda_graphs
from megatron.core.transformer.cuda_graphs import CudaGraphManager, delete_cuda_graphs
from megatron.core.transformer.enums import InferenceCudaGraphScope
from megatron.core.transformer.moe.router_replay import RouterReplay, RouterReplayAction
from megatron.core.utils import (
Expand Down Expand Up @@ -133,13 +133,42 @@ class EngineSuspendedError(Exception):

def format_mem_bytes(mem_bytes):
"""Convert a byte count to a human-readable string in tb, gb, mb, kb, or bytes."""
if mem_bytes < 0:
return "-" + format_mem_bytes(-mem_bytes)
for power, suffix in [(4, "tb"), (3, "gb"), (2, "mb"), (1, "kb"), (0, "bytes")]:
suffix_bytes = 1024**power
if mem_bytes >= suffix_bytes:
return "%.1f %s" % (mem_bytes / suffix_bytes, suffix)
return "%d bytes" % mem_bytes


def _cuda_graph_mempool_bytes():

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: can you make return type -> Tuple[int, int]?

"""Return (reserved, allocated) bytes belonging to the global CUDA graph mempool.

PyTorch's `torch.cuda.memory_stats()` reports process-wide totals that mix in
every other allocation (KV cache, NCCL workspaces, layer scratch). To isolate
growth caused by graph capture, we walk `torch.cuda.memory_snapshot()` and
filter segments by their `segment_pool_id` against the graph pool handle.
Returns (0, 0) if the pool hasn't been created yet.
"""
pool_id = CudaGraphManager.global_mempool
if pool_id is None:
return 0, 0
reserved = 0
allocated = 0
for seg in torch.cuda.memory_snapshot():
seg_pool_id = (
seg.get("segment_pool_id")
or seg.get("private_pool_id")
or seg.get("pool_id")
or seg.get("pool")
)
if seg_pool_id == pool_id:
reserved += seg.get("total_size", 0)
allocated += seg.get("allocated_size", 0)
return reserved, allocated


@dataclass(kw_only=True)
class RequestEntry:
"""Entry in the engine's `self.requests` dict."""
Expand Down Expand Up @@ -345,8 +374,16 @@ def create_cuda_graphs(self, reset_context: bool = True):
controller = self.controller

time_start = time.time()
torch.cuda.reset_peak_memory_stats()

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it safe to reset the peak memory stats here for every request entry? Would this disrupt any existing memory recording?

mem_stats_start = torch.cuda.memory_stats()

# Snapshot of process-wide stats for the "total memory used by capture" summary.
start_proc_reserved = mem_stats_start["reserved_bytes.all.current"]
start_proc_alloc = mem_stats_start["allocated_bytes.all.current"]

# Pool-scoped baselines for the per-iteration deltas.
prev_pool_reserved, prev_pool_alloc = _cuda_graph_mempool_bytes()

logging.info("> dynamic_engine.py: building cuda graphs for ")
for graph in context.cuda_graph_batch_dimensions_list:
logging.info(graph)
Expand Down Expand Up @@ -427,27 +464,44 @@ def create_cuda_graphs(self, reset_context: bool = True):

context.reset()

# Per-iteration memory accounting, scoped to the CUDA-graph mempool.
# This isolates pool growth from process-wide scratch churn (KV cache,
# NCCL workspaces, etc.) that pollutes `torch.cuda.memory_stats()`.
pool_reserved, pool_alloc = _cuda_graph_mempool_bytes()
logging.info(
" [graph %d/%d] %s | pool reserved=%s (Δiter=%s) " "pool allocated=%s (Δiter=%s)",
tbar_idx + 1,
len(context.cuda_graph_batch_dimensions_list),
cuda_graph_batch_dimension,
format_mem_bytes(pool_reserved),
format_mem_bytes(pool_reserved - prev_pool_reserved),
format_mem_bytes(pool_alloc),
format_mem_bytes(pool_alloc - prev_pool_alloc),
)
prev_pool_reserved, prev_pool_alloc = pool_reserved, pool_alloc
torch.cuda.reset_peak_memory_stats()

if mtp_warmup_enabled and mtp_seen_batch_sizes:
logging.info("> MTP CUDA graph warmup: %d batch size(s)", len(mtp_seen_batch_sizes))

# Memory usage.
time_end = time.time()
mem_stats_end = torch.cuda.memory_stats()
final_pool_reserved, final_pool_alloc = _cuda_graph_mempool_bytes()
capture_stats = {
"time": time_end - time_start,
"allocated_bytes": (
mem_stats_end["allocated_bytes.all.current"]
- mem_stats_start["allocated_bytes.all.current"]
),
"reserved_bytes": (
mem_stats_end["reserved_bytes.all.current"]
- mem_stats_start["reserved_bytes.all.current"]
),
"allocated_bytes": (mem_stats_end["allocated_bytes.all.current"] - start_proc_alloc),
"reserved_bytes": (mem_stats_end["reserved_bytes.all.current"] - start_proc_reserved),
"pool_reserved_bytes": final_pool_reserved,
"pool_allocated_bytes": final_pool_alloc,
}
logging.info(
"> built cuda graph(s) in %.2f sec, with total memory usage: "
"allocated %s, reserved %s.",
"> built cuda graph(s) in %.2f sec. "
"Mempool: reserved %s, allocated %s. "
"Process-wide delta: allocated %s, reserved %s.",
capture_stats["time"],
format_mem_bytes(capture_stats["pool_reserved_bytes"]),
format_mem_bytes(capture_stats["pool_allocated_bytes"]),
format_mem_bytes(capture_stats["allocated_bytes"]),
format_mem_bytes(capture_stats["reserved_bytes"]),
)
Expand Down
10 changes: 8 additions & 2 deletions tests/unit_tests/inference/contexts/test_dynamic_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -1602,10 +1602,16 @@ def test_add_dummy_requests_for_expert_parallel_step_matches_slow_path(
num_speculative_tokens=num_speculative_tokens,
)

smallest = min(ctx.cuda_graph_batch_dimensions_list)
# The fast path is decode-only by construction, so pick the smallest decode-only batch_dim.
# With the geometric grid for mixed cudagraphs, the global min may now be a P=1 mixed shape
# when num_speculative_tokens > 0 makes decode-only token_count > 1)
smallest = min(
batchdim
for batchdim in ctx.cuda_graph_batch_dimensions_list
if batchdim.prefill_req_count == 0
)
N = smallest.decode_req_count
T = smallest.token_count # N * (num_speculative_tokens + 1)
assert smallest.prefill_req_count == 0, "smallest graph must be decode-only"

# --- slow path (reference) ---
ctx.add_dummy_requests_for_cudagraph_capture(smallest)
Expand Down
Loading
Loading