From e086d374017d250718c96362371e0e255b8ca7dd Mon Sep 17 00:00:00 2001 From: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> Date: Thu, 15 May 2025 00:11:06 +0000 Subject: [PATCH 1/4] add mixtral7x8b fp8 test with fixed cutlass fp8 moe gemm Signed-off-by: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> --- tests/integration/defs/test_e2e.py | 3 +++ tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml | 1 + 2 files changed, 4 insertions(+) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index e88ec422b8d7..ee756b291645 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1272,6 +1272,9 @@ def test_ptp_quickstart(llm_root, llm_venv): pytest.param('Mixtral-8x7B-NVFP4', 'nvfp4-quantized/Mixtral-8x7B-Instruct-v0.1', marks=skip_pre_blackwell), + pytest.param('Mixtral-8x7B-FP8', + 'Mixtral-8x7B-Instruct-v0.1-fp8', + marks=skip_pre_blackwell), ]) def test_ptp_quickstart_advanced(llm_root, llm_venv, model_name, model_path): print(f"Testing {model_name}.") diff --git a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml index 49a5f8e8f537..226e4ec7da0b 100644 --- a/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml +++ b/tests/integration/test_lists/test-db/l0_rtx_pro_6000.yml @@ -26,3 +26,4 @@ l0_rtx_pro_6000: - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-70B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-70B] - test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-70B-FP8-llama-3.1-model/Llama-3.1-70B-Instruct-FP8] - test_e2e.py::test_ptp_quickstart_advanced[Mixtral-8x7B-NVFP4-nvfp4-quantized/Mixtral-8x7B-Instruct-v0.1] + - test_e2e.py::test_ptp_quickstart_advanced[Mixtral-8x7B-FP8-Mixtral-8x7B-Instruct-v0.1-fp8] From 436fcbd3a55880222a9f5c0846e270efb3684dd5 Mon Sep 17 00:00:00 2001 From: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> Date: Fri, 16 May 2025 03:29:51 +0000 Subject: [PATCH 2/4] update cutlass versions Signed-off-by: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> --- .../tensorrt_llm_internal_cutlass_kernels_static.tar.xz | 4 ++-- .../internal_cutlass_kernels/aarch64-linux-gnu/version.txt | 4 ++-- .../tensorrt_llm_internal_cutlass_kernels_static.tar.xz | 4 ++-- .../internal_cutlass_kernels/x86_64-linux-gnu/version.txt | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz index 4ce5dc1d4b1d..0ef19276f3d6 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02d0983056f925c59313e03964bc04e4dcddd9f9fc46c7917db14296af4abdb5 -size 63562780 +oid sha256:2beea045788780ee0c21f1b1fd0f4e3e44a1a3500243bf50445676ee3915c48b +size 52913316 diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt index f15ea3c403cc..f59b66c8a4d5 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt @@ -1,2 +1,2 @@ -557bea7fa3fcd0b9230edecf3a8dbb10eeb1b8d78928861b1c6da1043f973c35 libtensorrt_llm_internal_cutlass_kernels_static.a -commit 2e68c0113ad486c0026619711e5974f5ae0dc626 +62a4c8d1269a95be3ee9b037090dcec6afe1bf857f1160c96b9c45f2a6d68ca5 libtensorrt_llm_internal_cutlass_kernels_static.a +commit 85675189cc26c868e1ec2b9b63c251055a3b5abb diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz index deef5aafc398..f7576d94f4a9 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:04f4daeb5e4f0762be994b583c50573fda5f994e95a5e737b0352ce2f6376d0b -size 63062972 +oid sha256:2545f48bf4f621ad278d6f5561bec82768fb016a6d4297799e67a850ef0d799a +size 52540292 diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt index 021d4f2b6213..d92229287ac0 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt @@ -1,2 +1,2 @@ -a956018ada7a223b2f9e4aa3331d7bdc4be8f69bdef9e3828f5f1c65beb11699 libtensorrt_llm_internal_cutlass_kernels_static.a -commit 2e68c0113ad486c0026619711e5974f5ae0dc626 +998b078a3debc2937466f654fe508f3a40594f39b4126ddc177f49b8a2756dfc libtensorrt_llm_internal_cutlass_kernels_static.a +commit 85675189cc26c868e1ec2b9b63c251055a3b5abb From 4d246a19a81b35d74b64415defe9377036a00f73 Mon Sep 17 00:00:00 2001 From: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> Date: Fri, 16 May 2025 11:49:35 +0000 Subject: [PATCH 3/4] added internal cutlass with fix and docker update Signed-off-by: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> --- .../tensorrt_llm_internal_cutlass_kernels_static.tar.xz | 4 ++-- .../internal_cutlass_kernels/aarch64-linux-gnu/version.txt | 4 ++-- .../tensorrt_llm_internal_cutlass_kernels_static.tar.xz | 4 ++-- .../internal_cutlass_kernels/x86_64-linux-gnu/version.txt | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz index 0ef19276f3d6..aa53b594f4aa 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2beea045788780ee0c21f1b1fd0f4e3e44a1a3500243bf50445676ee3915c48b -size 52913316 +oid sha256:090be002758e4fb864e16ec25c0db3f8eb562a0033e60a156bbbfd6bce67a5a1 +size 63577888 diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt index f59b66c8a4d5..ed1277dce44a 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/aarch64-linux-gnu/version.txt @@ -1,2 +1,2 @@ -62a4c8d1269a95be3ee9b037090dcec6afe1bf857f1160c96b9c45f2a6d68ca5 libtensorrt_llm_internal_cutlass_kernels_static.a -commit 85675189cc26c868e1ec2b9b63c251055a3b5abb +aff0f8e617f6ca2f95d121ab9cf0ab17c4e8077cf9e8896bf153d3942a4a50df libtensorrt_llm_internal_cutlass_kernels_static.a +commit d61e7684bc095c8ff5ec540363949bd1f491c960 diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz index f7576d94f4a9..6143671a5c82 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/tensorrt_llm_internal_cutlass_kernels_static.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2545f48bf4f621ad278d6f5561bec82768fb016a6d4297799e67a850ef0d799a -size 52540292 +oid sha256:16608c6e8ca9f1c3f26408c0ec97049ac05b1ee8b647480d969d488d9cef91f0 +size 63066816 diff --git a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt index d92229287ac0..3e93949bcee8 100644 --- a/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt +++ b/cpp/tensorrt_llm/kernels/internal_cutlass_kernels/x86_64-linux-gnu/version.txt @@ -1,2 +1,2 @@ -998b078a3debc2937466f654fe508f3a40594f39b4126ddc177f49b8a2756dfc libtensorrt_llm_internal_cutlass_kernels_static.a -commit 85675189cc26c868e1ec2b9b63c251055a3b5abb +9f0a29070b95a7db62f70cc45ef151e27c2a58697a2d50cbb002ff339035fb8e libtensorrt_llm_internal_cutlass_kernels_static.a +commit d61e7684bc095c8ff5ec540363949bd1f491c960 From 4c8a3b2684066e4ca7b2c5c7a8334a37d729f3e2 Mon Sep 17 00:00:00 2001 From: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> Date: Fri, 16 May 2025 20:46:22 +0000 Subject: [PATCH 4/4] added mixtral to pro 6000 Signed-off-by: Faraz Khoubsirat <58580514+farazkh80@users.noreply.github.com> --- tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt b/tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt index f0377701fce1..038b950ae5c8 100644 --- a/tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt +++ b/tests/integration/test_lists/qa/llm_release_rtx_pro_6000.txt @@ -25,5 +25,6 @@ test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-8B-NVFP4-nvfp4-quantized/Meta test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-70B-NVFP4-nvfp4-quantized/Meta-Llama-3.1-70B] test_e2e.py::test_ptp_quickstart_advanced[Llama3.1-70B-FP8-llama-3.1-model/Llama-3.1-70B-Instruct-FP8] test_e2e.py::test_ptp_quickstart_advanced[Mixtral-8x7B-NVFP4-nvfp4-quantized/Mixtral-8x7B-Instruct-v0.1] +test_e2e.py::test_ptp_quickstart_advanced[Mixtral-8x7B-FP8-Mixtral-8x7B-Instruct-v0.1-fp8] test_e2e.py::test_ptp_quickstart_advanced_2gpus_sm120[Llama3.1-70B-BF16-llama-3.1-model/Meta-Llama-3.1-70B] test_e2e.py::test_ptp_quickstart_advanced_2gpus_sm120[Mixtral-8x7B-BF16-Mixtral-8x7B-Instruct-v0.1]