From 4a80e1ebfa57e5724929239aa21ea6c418bfa4d0 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Fri, 28 Aug 2026 18:00:58 -0500 Subject: [PATCH 1/4] [TRTLLM-10657][fix] Resolve MIXED_PRECISION quant config for DeepSeek W4A8 MoE experts DeepSeek-R1-W4AFP8 ships an hf_quant_config with quant_algo=MIXED_PRECISION. On the PyTorch backend this hit a hard assertion while constructing DeepseekV3DecoderLayer ("MIXED_PRECISION is ambiguous"), so the 8-GPU TP8/EP8 quickstart aborted during executor init (surfacing as "Executor worker returned error"). The global MIXED_PRECISION algo does not map to a single QuantMode; the per-module configs (e.g. W4A8_AWQ for the MoE experts, FP8_BLOCK_SCALES for attention) must be resolved individually. This change: - Deepseekv3MoE: resolve the per-expert quant config via _get_experts_quant_config when the override is MIXED_PRECISION, and pass it to create_moe (weight_loading_mode derived from the resolved config). - DeepseekV3DecoderLayer: drop the assertion and set is_nvfp4 defensively for MIXED_PRECISION. - ConfigurableMoE._get_quant_config_dict: prefer the resolved per-module override over the global config. - Un-waive and enable test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus (l0_dgx_h200, 8-GPU post-merge). Verified on 8xH100: the model loads across TP8/EP8 and generates for all prompts (previously asserted during construction). Signed-off-by: Brian Nguyen --- .../_torch/models/modeling_deepseekv3.py | 33 ++++++++++++------- .../_torch/moe/fused_moe/configurable_moe.py | 7 ++-- .../test_lists/test-db/l0_dgx_h200.yml | 1 + tests/integration/test_lists/waives.txt | 1 - 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index 9dc29b58d744..af17db7025ed 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -980,6 +980,16 @@ def __init__(self, apply_routing=False, moe_backend=model_config.moe_backend, use_cute_dsl_bf16_gemm=model_config.use_cute_dsl_bf16_gemm) + # For MIXED_PRECISION, resolve the per-expert quant config (e.g. W4A8_AWQ) + # instead of using the ambiguous global MIXED_PRECISION config. + # For other cases (e.g. nvfp4, unquantized MTP layers), use + # override_quant_config as-is — it already encodes exclusions like MTP. + if (override_quant_config is not None and + override_quant_config.quant_algo == QuantAlgo.MIXED_PRECISION): + expert_quant_config = self._get_experts_quant_config( + model_config, layer_idx) + else: + expert_quant_config = override_quant_config self.experts = create_moe( num_experts=num_experts, routing_method=self.gate.routing_method, @@ -989,17 +999,15 @@ def __init__(self, reduce_results= False, # In both low‑latency and attention‑DP modes, FusedMoE skips the in‑op all‑reduce. model_config=model_config, - override_quant_config=override_quant_config, + override_quant_config=expert_quant_config, aux_stream_dict=aux_stream_dict, layer_idx=layer_idx, # DS-R1 W4A8 is only supported through custom quantization script from # examples/quantization/quantize_mixed_precision_moe.py - weight_loading_mode=( - MoEWeightLoadingMode.W4A8_CUSTOM - if self._get_experts_quant_config( - model_config, - layer_idx).layer_quant_mode.is_int4_weight_only_per_group() - else MoEWeightLoadingMode.VANILLA), + weight_loading_mode=(MoEWeightLoadingMode.W4A8_CUSTOM + if expert_quant_config.layer_quant_mode. + is_int4_weight_only_per_group() else + MoEWeightLoadingMode.VANILLA), ) self.mapping = model_config.mapping @@ -1300,13 +1308,14 @@ def __init__(self, "TRTLLM_DEEPSEEK_EAGER_FUSION_DISABLED", "0") == "0" self.enable_fusion &= not self.enable_attention_dp - # FIXME: incompatible with mixed quantization mode quant_config = self._get_decoder_layer_quant_config( model_config, layer_idx) - self.is_nvfp4 = quant_config.layer_quant_mode.has_nvfp4() - assert ( - quant_config.quant_algo - is not QuantAlgo.MIXED_PRECISION), "MIXED_PRECISION is ambiguous" + # For MIXED_PRECISION, the global quant_algo doesn't map to a single + # QuantMode. Per-module configs (e.g. expert W4A8_AWQ vs attention + # FP8_BLOCK_SCALES) are resolved individually where needed, so we + # conservatively set layer-level flags here. + self.is_nvfp4 = (quant_config.quant_algo != QuantAlgo.MIXED_PRECISION + and quant_config.layer_quant_mode.has_nvfp4()) self.allreduce = None self.moe_allreduce = None diff --git a/tensorrt_llm/_torch/moe/fused_moe/configurable_moe.py b/tensorrt_llm/_torch/moe/fused_moe/configurable_moe.py index d02bbf22a9c3..b5a20606580a 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/configurable_moe.py +++ b/tensorrt_llm/_torch/moe/fused_moe/configurable_moe.py @@ -463,10 +463,13 @@ def _get_quant_config_dict(self, model_config: ModelConfig) -> Optional[Dict]: Extract quantization configuration from model_config """ - if model_config.quant_config is None: + # Prefer the resolved per-module override (e.g. W4A8_AWQ for experts) + # over the global config which may be MIXED_PRECISION. + quant_config = getattr(self, "_override_quant_config", None) or model_config.quant_config + if quant_config is None: return None - quant_mode = model_config.quant_config.layer_quant_mode + quant_mode = quant_config.layer_quant_mode return { "has_fp8_qdq": quant_mode.has_fp8_qdq() if hasattr(quant_mode, "has_fp8_qdq") diff --git a/tests/integration/test_lists/test-db/l0_dgx_h200.yml b/tests/integration/test_lists/test-db/l0_dgx_h200.yml index 35a749f4c01c..1c7fcf2704e2 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h200.yml @@ -45,6 +45,7 @@ l0_dgx_h200: - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ctxtp2ep2pp2_gentp4_one_mtp_block_reuse[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8] - disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req120-conc64-qwen3_32b_fp8_mixed_stress] + - test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec - condition: ranges: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index b811bb2aa241..120606c731c0 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -300,7 +300,6 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_ perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6661856) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6661856) test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] SKIP (https://nvbugs/6624972) -test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] SKIP (https://nvbugs/6605819) test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] SKIP (bug pending, tracked in PR 17414) unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py::test_on_update_kv_lens_rebuilds_stale_map SKIP (https://nvbugs/6574939) From 60a15bc70cc56de2a5bde42da36df77cbb558224 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Tue, 8 Sep 2026 16:28:00 -0700 Subject: [PATCH 2/4] [TRTLLM-10657][test] Cover MIXED_PRECISION expert quant resolution with a unit test Replace the 8-GPU E2E coverage with a CPU-only unit test. The test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus entry goes back to waives.txt and out of l0_dgx_h200.yml. The new test exercises the substance of the fix without weights or GPUs: - Deepseekv3MoE._get_experts_quant_config resolves the per-module W4A8_AWQ config from quant_config_dict for a MIXED_PRECISION checkpoint, and falls back to the global config for unlisted layers or a missing dict. The resolved mode reports is_int4_weight_only_per_group(), which is what selects MoEWeightLoadingMode.W4A8_CUSTOM. - ConfigurableMoE._get_quant_config_dict prefers _override_quant_config over a MIXED_PRECISION global config (has_w4afp8=True), falls back to the global config without an override, and returns None when unquantized. Signed-off-by: Brian Nguyen --- .../test_lists/test-db/l0_dgx_h200.yml | 1 - tests/integration/test_lists/waives.txt | 1 + .../test_deepseekv3_mixed_precision_quant.py | 127 ++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py diff --git a/tests/integration/test_lists/test-db/l0_dgx_h200.yml b/tests/integration/test_lists/test-db/l0_dgx_h200.yml index 1c7fcf2704e2..35a749f4c01c 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h200.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h200.yml @@ -45,7 +45,6 @@ l0_dgx_h200: - disaggregated/test_disaggregated.py::test_disaggregated_deepseek_v3_lite_fp8_ctxtp2ep2pp2_gentp4_one_mtp_block_reuse[DeepSeek-V3-Lite-fp8] - disaggregated/test_disaggregated.py::test_disaggregated_qwen3_32b_fp8[Qwen3/Qwen3-32B-FP8] - disaggregated/test_disaggregated.py::test_disaggregated_mixed_stress_test[req120-conc64-qwen3_32b_fp8_mixed_stress] - - test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] - accuracy/test_disaggregated_serving.py::TestDeepSeekV3Lite::test_gen_only_spec_dec - condition: ranges: diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 120606c731c0..b811bb2aa241 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -300,6 +300,7 @@ perf/test_perf_sanity.py::test_e2e[aggr_upload-deepseek_r1_fp4_v2_2_nodes_grace_ perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6661856) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb300_deepseek-v4-pro-fp4_8k1k_con8_ctx1_dep4_gen4_tep8_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6661856) test_e2e.py::test_ptp_quickstart_advanced[Nemotron-Nano-9B-v2-nvfp4-NVIDIA-Nemotron-Nano-9B-v2-NVFP4] SKIP (https://nvbugs/6624972) +test_e2e.py::test_ptp_quickstart_advanced_deepseek_r1_w4afp8_8gpus[DeepSeek-R1-W4AFP8-DeepSeek-R1/DeepSeek-R1-W4AFP8] SKIP (https://nvbugs/5836830) test_e2e.py::test_ptp_quickstart_bert[TRTLLM-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] SKIP (https://nvbugs/6605819) test_e2e.py::test_ptp_quickstart_bert[VANILLA-BertForSequenceClassification-bert/bert-base-uncased-yelp-polarity] SKIP (bug pending, tracked in PR 17414) unittest/_torch/attention/sparse/dsa/test_req_idx_per_token.py::test_on_update_kv_lens_rebuilds_stale_map SKIP (https://nvbugs/6574939) diff --git a/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py b/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py new file mode 100644 index 000000000000..acd4feb9923d --- /dev/null +++ b/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Per-module quant config resolution for DeepSeek MIXED_PRECISION checkpoints. + +DeepSeek-R1-W4AFP8 ships an hf_quant_config with the global +quant_algo=MIXED_PRECISION, which does not map to a single QuantMode. The MoE +experts must resolve their own per-module config (W4A8_AWQ) instead of using +the global one. These tests exercise that resolution on CPU without weights. +""" + +from types import SimpleNamespace + +import pytest + +from tensorrt_llm._torch.models.modeling_deepseekv3 import Deepseekv3MoE +from tensorrt_llm._torch.moe.fused_moe.configurable_moe import ConfigurableMoE +from tensorrt_llm.models.modeling_utils import QuantConfig +from tensorrt_llm.quantization.mode import QuantAlgo + +EXPERTS_KEY = "model.layers.{}.mlp.experts" + + +@pytest.fixture +def mixed_precision_config(): + return QuantConfig(quant_algo=QuantAlgo.MIXED_PRECISION) + + +@pytest.fixture +def w4a8_awq_config(): + return QuantConfig(quant_algo=QuantAlgo.W4A8_AWQ, group_size=128) + + +def test_w4a8_awq_config_is_int4_weight_only_per_group(w4a8_awq_config): + # This predicate is what selects MoEWeightLoadingMode.W4A8_CUSTOM for the + # experts in Deepseekv3MoE, so pin it down explicitly. + assert w4a8_awq_config.layer_quant_mode.is_int4_weight_only_per_group() + + +def test_experts_quant_config_resolved_from_per_module_dict( + mixed_precision_config, w4a8_awq_config +): + model_config = SimpleNamespace( + quant_config=mixed_precision_config, + quant_config_dict={EXPERTS_KEY.format(0): w4a8_awq_config}, + ) + + resolved = Deepseekv3MoE._get_experts_quant_config(model_config, 0) + + assert resolved is w4a8_awq_config + assert resolved.quant_algo == QuantAlgo.W4A8_AWQ + assert resolved.layer_quant_mode.is_int4_weight_only_per_group() + assert not mixed_precision_config.layer_quant_mode.is_int4_weight_only_per_group() + + +def test_experts_quant_config_falls_back_to_global_for_unlisted_layer( + mixed_precision_config, w4a8_awq_config +): + model_config = SimpleNamespace( + quant_config=mixed_precision_config, + quant_config_dict={EXPERTS_KEY.format(0): w4a8_awq_config}, + ) + + assert Deepseekv3MoE._get_experts_quant_config(model_config, 1) is mixed_precision_config + + +def test_experts_quant_config_falls_back_to_global_without_dict(mixed_precision_config): + model_config = SimpleNamespace(quant_config=mixed_precision_config, quant_config_dict=None) + + assert Deepseekv3MoE._get_experts_quant_config(model_config, 0) is mixed_precision_config + + +def _bare_configurable_moe(override_quant_config): + moe = object.__new__(ConfigurableMoE) + moe._override_quant_config = override_quant_config + return moe + + +def test_quant_config_dict_prefers_override_over_mixed_precision_global( + mixed_precision_config, w4a8_awq_config +): + moe = _bare_configurable_moe(w4a8_awq_config) + model_config = SimpleNamespace(quant_config=mixed_precision_config) + + result = ConfigurableMoE._get_quant_config_dict(moe, model_config) + + assert result == { + "has_fp8_qdq": False, + "has_nvfp4": False, + "has_w4afp8": True, + "has_fp8_block_scales": False, + } + # The global MIXED_PRECISION mode would not have flagged w4afp8. + assert not mixed_precision_config.layer_quant_mode.is_int4_weight_only_per_group() + + +def test_quant_config_dict_falls_back_to_global_without_override(): + fp8_config = QuantConfig(quant_algo=QuantAlgo.FP8) + moe = _bare_configurable_moe(None) + model_config = SimpleNamespace(quant_config=fp8_config) + + result = ConfigurableMoE._get_quant_config_dict(moe, model_config) + + assert result == { + "has_fp8_qdq": True, + "has_nvfp4": False, + "has_w4afp8": False, + "has_fp8_block_scales": False, + } + + +def test_quant_config_dict_is_none_when_unquantized(): + moe = _bare_configurable_moe(None) + model_config = SimpleNamespace(quant_config=None) + + assert ConfigurableMoE._get_quant_config_dict(moe, model_config) is None From 68a733d8010c738e0bc00eb795cbd9e380ed17ab Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Tue, 8 Sep 2026 17:23:55 -0700 Subject: [PATCH 3/4] [TRTLLM-10657][fix] Make expert weight-loading-mode selection None-safe Addresses review feedback on the MoE expert construction: - override_quant_config is Optional, so on an unquantized layer the resolved expert config is None; the inline weight_loading_mode ternary dereferenced it via `.layer_quant_mode`. Extract the decision into a static helper `_expert_weight_loading_mode` that guards None and returns VANILLA. - Add unit tests exercising the helper directly (W4A8_AWQ -> W4A8_CUSTOM, MIXED_PRECISION/FP8/None -> VANILLA), so a regression in the assignment that turns the resolved config into the loading mode is caught. Signed-off-by: Brian Nguyen --- .../_torch/models/modeling_deepseekv3.py | 19 +++++++++--- .../test_deepseekv3_mixed_precision_quant.py | 29 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index af17db7025ed..607cd01d0759 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -1004,10 +1004,8 @@ def __init__(self, layer_idx=layer_idx, # DS-R1 W4A8 is only supported through custom quantization script from # examples/quantization/quantize_mixed_precision_moe.py - weight_loading_mode=(MoEWeightLoadingMode.W4A8_CUSTOM - if expert_quant_config.layer_quant_mode. - is_int4_weight_only_per_group() else - MoEWeightLoadingMode.VANILLA), + weight_loading_mode=self._expert_weight_loading_mode( + expert_quant_config), ) self.mapping = model_config.mapping @@ -1102,6 +1100,19 @@ def _get_experts_quant_config(model_config, layer_idx: int) -> QuantConfig: return model_config.quant_config_dict.get( f"model.layers.{layer_idx}.mlp.experts", model_config.quant_config) + @staticmethod + def _expert_weight_loading_mode( + expert_quant_config: Optional[QuantConfig]) -> MoEWeightLoadingMode: + # W4A8_CUSTOM only for the int4-weight-per-group experts produced by + # examples/quantization/quantize_mixed_precision_moe.py; everything else + # (including the unquantized case, where there is no expert quant config) + # uses VANILLA. Guard None: override_quant_config is Optional, so the + # resolved expert config can be None on an unquantized layer. + if (expert_quant_config is not None and expert_quant_config. + layer_quant_mode.is_int4_weight_only_per_group()): + return MoEWeightLoadingMode.W4A8_CUSTOM + return MoEWeightLoadingMode.VANILLA + @staticmethod def _get_shared_experts_quant_config(model_config, layer_idx: int) -> QuantConfig: diff --git a/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py b/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py index acd4feb9923d..1add65de5cfd 100644 --- a/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py +++ b/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py @@ -25,6 +25,7 @@ import pytest from tensorrt_llm._torch.models.modeling_deepseekv3 import Deepseekv3MoE +from tensorrt_llm._torch.moe.fused_moe import MoEWeightLoadingMode from tensorrt_llm._torch.moe.fused_moe.configurable_moe import ConfigurableMoE from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo @@ -81,6 +82,34 @@ def test_experts_quant_config_falls_back_to_global_without_dict(mixed_precision_ assert Deepseekv3MoE._get_experts_quant_config(model_config, 0) is mixed_precision_config +def test_expert_weight_loading_mode_w4a8_custom_for_w4a8_awq(w4a8_awq_config): + # The resolved W4A8_AWQ expert config selects the custom loading mode; this + # is the assignment the MoE construction makes from the resolved config. + assert ( + Deepseekv3MoE._expert_weight_loading_mode(w4a8_awq_config) + is MoEWeightLoadingMode.W4A8_CUSTOM + ) + + +def test_expert_weight_loading_mode_vanilla_for_non_int4(mixed_precision_config): + # Neither the ambiguous MIXED_PRECISION global nor a plain FP8 config is + # int4-weight-per-group, so both fall to VANILLA. + assert ( + Deepseekv3MoE._expert_weight_loading_mode(mixed_precision_config) + is MoEWeightLoadingMode.VANILLA + ) + assert ( + Deepseekv3MoE._expert_weight_loading_mode(QuantConfig(quant_algo=QuantAlgo.FP8)) + is MoEWeightLoadingMode.VANILLA + ) + + +def test_expert_weight_loading_mode_none_is_vanilla(): + # override_quant_config is Optional, so the resolved expert config is None on + # an unquantized layer; the mode selection must not dereference it. + assert Deepseekv3MoE._expert_weight_loading_mode(None) is MoEWeightLoadingMode.VANILLA + + def _bare_configurable_moe(override_quant_config): moe = object.__new__(ConfigurableMoE) moe._override_quant_config = override_quant_config From 11c3b3ee5e53b8d489329c72b0ac359bea81fe3b Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Wed, 9 Sep 2026 15:20:28 -0500 Subject: [PATCH 4/4] [TRTLLM-10657][test] Relocate DeepSeek-V3 mixed-precision quant unit test into modeling/, mark cpu_only Signed-off-by: Brian Nguyen --- .../test_modeling_deepseekv3.py} | 2 ++ 1 file changed, 2 insertions(+) rename tests/unittest/_torch/{models/test_deepseekv3_mixed_precision_quant.py => modeling/test_modeling_deepseekv3.py} (99%) diff --git a/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py b/tests/unittest/_torch/modeling/test_modeling_deepseekv3.py similarity index 99% rename from tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py rename to tests/unittest/_torch/modeling/test_modeling_deepseekv3.py index 1add65de5cfd..201d22142c6a 100644 --- a/tests/unittest/_torch/models/test_deepseekv3_mixed_precision_quant.py +++ b/tests/unittest/_torch/modeling/test_modeling_deepseekv3.py @@ -30,6 +30,8 @@ from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization.mode import QuantAlgo +pytestmark = pytest.mark.cpu_only + EXPERTS_KEY = "model.layers.{}.mlp.experts"