Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
19 changes: 18 additions & 1 deletion cpp/tensorrt_llm/common/attentionOp.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,17 @@ bool AttentionOp::convertMMHAParamsToXQAParams(tensorrt_llm::kernels::XQAParams&
= mAttentionChunkSize && !tc::getEnvDisableChunkedAttentionInGenPhase() ? *mAttentionChunkSize : INT_MAX;
xqaParams.max_attention_window_size = generationsParams.max_attention_window_size;
xqaParams.cyclic_attention_window_size = generationsParams.cyclic_attention_window_size;
// Treat the layer as sliding-window-causal only for explicit SWA masks or
// in-attention positional encodings whose max position exceeds the layer window.
// Exclude sparse attention and chunked attention
bool const has_in_attention_pos_encoding = mPositionEmbeddingType != PositionEmbeddingType::kLEARNED_ABSOLUTE;
// chunked_attention_size is set to INT_MAX above as the "disabled" sentinel.
bool const chunked_attention_enabled
= xqaParams.chunked_attention_size > 0 && xqaParams.chunked_attention_size != INT_MAX;
xqaParams.is_sliding_window = !mUseSparseAttention && !chunked_attention_enabled
&& ((mMaskType == AttentionMaskType::SLIDING_WINDOW_CAUSAL)
|| (has_in_attention_pos_encoding && generationsParams.max_attention_window_size > 0
&& generationsParams.max_attention_window_size < mRotaryEmbeddingMaxPositions));
xqaParams.max_blocks_per_sequence = generationsParams.max_blocks_per_sequence;
xqaParams.sink_token_length = generationsParams.sink_token_length;
xqaParams.max_past_kv_length = generationsParams.max_past_kv_length;
Expand Down Expand Up @@ -1113,7 +1124,13 @@ int AttentionOp::mlaGeneration(
tllmRunnerParams.mMaxSeqLenCacheKv = generation_params.max_attention_window_size;
// This should be set to numDraftTokens + 1.
tllmRunnerParams.mMaxSeqLenQ = params.acc_q_len / batch_beam;
tllmRunnerParams.mMaxSeqLenKv = generation_params.max_past_kv_length;
// Override mMaxSeqLenKv with the max cache capacity so FMHA picks the same kernel as
// CUDA graph warmup and avoids the eager-mode JIT miss/recompile. This is safe for
// PagedKv on this path because the strides do not depend on mMaxSeqLenKv, and extra
// KV CTAs exit early through seqLensKvPtr.
// TODO: mirror the is_swa + W+1 logic from xqaDispatcher.cpp when MLA gains SWA
// support (also requires adding Sliding cubins to the MLA gen kernel set).
tllmRunnerParams.mMaxSeqLenKv = generation_params.max_attention_window_size;
tllmRunnerParams.mSumOfSeqLensQ = int(batch_beam * tllmRunnerParams.mMaxSeqLenQ);
// Not used in the generation kernels as contiguous_kv or paged_kv layouts are used.
tllmRunnerParams.mSumOfSeqLensKv = int(batch_beam * tllmRunnerParams.mMaxSeqLenKv);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ struct XQAParams
int32_t chunked_attention_size = INT_MAX;
int32_t max_attention_window_size = 0;
int32_t cyclic_attention_window_size = 0;
// Whether this is a sliding-window-causal layer. Computed once in AttentionOp from the
// declared mask type / RoPE config so kernel-selection code does not need to re-derive
// it from window/rope heuristics. See AttentionOp::convertMMHAParamsToXQAParams.
bool is_sliding_window = false;
int32_t sink_token_length = 0;
int max_past_kv_length = 0;
void const* qkv_bias;
Expand Down
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Git LFS file not shown
Loading
Loading