diff --git a/tensorrt_llm/_torch/speculative/interface.py b/tensorrt_llm/_torch/speculative/interface.py index 9aeab730c6ef..4fbe3d9f393a 100644 --- a/tensorrt_llm/_torch/speculative/interface.py +++ b/tensorrt_llm/_torch/speculative/interface.py @@ -104,6 +104,8 @@ def prepare_attn_metadata_for_draft_replay(attn_metadata, attn_metadata.kv_cache_block_offsets = attn_metadata.draft_kv_cache_block_offsets attn_metadata.host_kv_cache_block_offsets = ( draft_kv_cache_manager.host_kv_cache_block_offsets) + if attn_metadata.enable_flash_mla: + attn_metadata.prepare_flash_mla() from ..attention_backend.sparse.dsa import (DSAtrtllmAttentionMetadata, Indexer) @@ -157,6 +159,8 @@ def restore_attn_metadata_after_draft_replay(attn_metadata, saved_state): saved_state['target_kv_cache_block_offsets']) attn_metadata.host_kv_cache_block_offsets = ( saved_state['target_host_kv_cache_block_offsets']) + if attn_metadata.enable_flash_mla: + attn_metadata.prepare_flash_mla() saved_dsa = saved_state.get('saved_dsa_state') if saved_dsa is not None: m = attn_metadata @@ -1599,6 +1603,8 @@ def draft_kv_cache_context(self, attn_metadata, draft_kv_cache_manager): attn_metadata.kv_cache_manager = draft_kv_cache_manager attn_metadata.kv_cache_block_offsets = attn_metadata.draft_kv_cache_block_offsets attn_metadata.host_kv_cache_block_offsets = draft_kv_cache_manager.host_kv_cache_block_offsets + if attn_metadata.enable_flash_mla: + attn_metadata.prepare_flash_mla() try: yield @@ -1607,6 +1613,8 @@ def draft_kv_cache_context(self, attn_metadata, draft_kv_cache_manager): attn_metadata.kv_cache_manager = target_kv_cache_manager attn_metadata.kv_cache_block_offsets = target_kv_cache_block_offsets attn_metadata.host_kv_cache_block_offsets = target_host_kv_cache_block_offsets + if attn_metadata.enable_flash_mla: + attn_metadata.prepare_flash_mla() def _sample_tokens_for_batch( self, diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index a7e95821ccaf..36d2eae38551 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -25,8 +25,6 @@ accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0- accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=True-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6305365) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=0-attention_dp=True-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6305404) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=True-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6388129) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=False-overlap_scheduler=True-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6084720) -accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16[mtp_nextn=2-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False-enable_chunked_prefill=False-v2_kv_cache=True] SKIP (https://nvbugs/6095851) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[ep4-mtp_nextn=2-attention_dp=True-cuda_graph=True-overlap_scheduler=True-torch_compile=True] SKIP (https://nvbugs/6402058) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=False-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337) accuracy/test_llm_api_pytorch.py::TestDeepSeekV3Lite::test_bfloat16_4gpus[pp4-mtp_nextn=0-attention_dp=False-cuda_graph=True-overlap_scheduler=False-torch_compile=False] SKIP (https://nvbugs/6278337)