From a6769e502e49ce6514d6a113706d787ea4aa2891 Mon Sep 17 00:00:00 2001 From: peaceh <103117813+peaceh-nv@users.noreply.github.com> Date: Mon, 14 Apr 2025 06:25:40 +0000 Subject: [PATCH 1/4] chore : Split more tests out of gpt tests Signed-off-by: peaceh <103117813+peaceh-nv@users.noreply.github.com> --- tests/integration/defs/cpp_common.py | 18 ++++++++++++++++-- tests/integration/defs/test_cpp.py | 4 ++-- .../integration/test_lists/test-db/l0_a30.yml | 2 ++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/integration/defs/cpp_common.py b/tests/integration/defs/cpp_common.py index a9c480c07fb..def0bf721c3 100755 --- a/tests/integration/defs/cpp_common.py +++ b/tests/integration/defs/cpp_common.py @@ -21,6 +21,8 @@ include_test_map = { "gpt": ("Gpt[^j]", ), "gpt_executor": ("GptExecutor", ), + "gpt_session": ("GptSession", ), + "gpt_tests": ("GptTests", ), "gptj": ("Gptj", ), "llama": ("Llama", ), "chatglm": ("ChatGlm", ), @@ -621,8 +623,8 @@ def prepare_model_tests(model_name: str, beams_arg = ['--beams', '1,2'] model_name = 'enc_dec' - # share the same script for gpt and gpt_executor - if model_name == 'gpt_executor': + # share the same script for gpt related tests + if model_name == 'gpt_executor' or model_name == 'gpt_session' or model_name == 'gpt_tests': model_name = 'gpt' build_engines = [ @@ -719,6 +721,12 @@ def run_single_gpu_tests(build_dir: _pl.Path, if "gpt" in test_list and "gpt_executor" not in test_list: excluded_tests.append("GptExecutor") + if "gpt" in test_list and "gpt_session" not in test_list: + excluded_tests.append("GptSession") + + if "gpt" in test_list and "gpt_tests" not in test_list: + excluded_tests.append("GptTests") + ctest = ["ctest", "--output-on-failure", "--output-junit", resultFileName] if included_tests: @@ -727,6 +735,12 @@ def run_single_gpu_tests(build_dir: _pl.Path, ctest.extend(["-E", "|".join(excluded_tests)]) parallel = default_test_parallel + + # gpt* tests are not parallelized as it would cause OOM because kv cache memory allocations + # exist in multiple running tests + if "gpt" in test_list or "gpt_session" in test_list or "gpt_tests" in test_list: + parallel = 1 + if parallel_override := _os.environ.get("LLM_TEST_PARALLEL_OVERRIDE", None): parallel = int(parallel_override) diff --git a/tests/integration/defs/test_cpp.py b/tests/integration/defs/test_cpp.py index f2eddcdfb08..2b7e86b385e 100644 --- a/tests/integration/defs/test_cpp.py +++ b/tests/integration/defs/test_cpp.py @@ -335,8 +335,8 @@ def test_unit_tests(build_google_tests, build_dir, lora_setup): indirect=True) @pytest.mark.parametrize("model", [ "bart", "chatglm", "eagle", "encoder", "enc_dec_language_adapter", "gpt", - "gpt_executor", "llama", "mamba", "medusa", "recurrentgemma", "redrafter", - "t5" + "gpt_executor", "gpt_session", "gpt_tests", "llama", "mamba", "medusa", + "recurrentgemma", "redrafter", "t5" ]) @pytest.mark.parametrize("run_fp8", [False, True], ids=["", "fp8"]) def test_model(build_google_tests, model, prepare_model, run_model_tests, diff --git a/tests/integration/test_lists/test-db/l0_a30.yml b/tests/integration/test_lists/test-db/l0_a30.yml index e7aeed5388b..a08ba5adc58 100644 --- a/tests/integration/test_lists/test-db/l0_a30.yml +++ b/tests/integration/test_lists/test-db/l0_a30.yml @@ -42,6 +42,8 @@ l0_a30: - test_cpp.py::test_unit_tests[80] - test_cpp.py::test_model[gpt-80] - test_cpp.py::test_model[gpt_executor-80] + - test_cpp.py::test_model[gpt_session-80] + - test_cpp.py::test_model[gpt_tests-80] - test_cpp.py::test_benchmarks[gpt-80] - condition: ranges: From 65b728f0d4060e1c7363f083a64775782fa7ff80 Mon Sep 17 00:00:00 2001 From: peaceh <103117813+peaceh-nv@users.noreply.github.com> Date: Tue, 15 Apr 2025 11:17:26 +0000 Subject: [PATCH 2/4] Address review comments, use a generator function Signed-off-by: peaceh <103117813+peaceh-nv@users.noreply.github.com> --- tests/integration/defs/cpp_common.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/integration/defs/cpp_common.py b/tests/integration/defs/cpp_common.py index def0bf721c3..ccea236f99e 100755 --- a/tests/integration/defs/cpp_common.py +++ b/tests/integration/defs/cpp_common.py @@ -72,6 +72,16 @@ def generate_result_file_name(test_list: List[str], yield "fp8" +def generate_excluded_test_list(test_list): + if "gpt" in test_list: + if "gpt_session" not in test_list: + yield "gpt_session" + if "gpt_executor" not in test_list: + yield "gpt_executor" + if "gpt_tests" not in test_list: + yield "gpt_tests" + + def find_dir_containing(files: Sequence[str], start_dir: Optional[_pl.Path] = None) -> _pl.Path: if start_dir is None: @@ -718,14 +728,7 @@ def run_single_gpu_tests(build_dir: _pl.Path, excluded_tests = ["FP8"] if not run_fp8 else [] - if "gpt" in test_list and "gpt_executor" not in test_list: - excluded_tests.append("GptExecutor") - - if "gpt" in test_list and "gpt_session" not in test_list: - excluded_tests.append("GptSession") - - if "gpt" in test_list and "gpt_tests" not in test_list: - excluded_tests.append("GptTests") + excluded_tests.extend(list(generate_excluded_test_list(test_list))) ctest = ["ctest", "--output-on-failure", "--output-junit", resultFileName] @@ -734,11 +737,11 @@ def run_single_gpu_tests(build_dir: _pl.Path, if excluded_tests: ctest.extend(["-E", "|".join(excluded_tests)]) - parallel = default_test_parallel + gpt_tests = {"gpt", "gpt_session", "gpt_tests", "gpt_executor"} # gpt* tests are not parallelized as it would cause OOM because kv cache memory allocations # exist in multiple running tests - if "gpt" in test_list or "gpt_session" in test_list or "gpt_tests" in test_list: + if gpt_tests.intersection(test_list): parallel = 1 if parallel_override := _os.environ.get("LLM_TEST_PARALLEL_OVERRIDE", From ca0465ff41bb6d5b6615ac9f2910876b7da852be Mon Sep 17 00:00:00 2001 From: peaceh <103117813+peaceh-nv@users.noreply.github.com> Date: Wed, 16 Apr 2025 02:27:19 +0000 Subject: [PATCH 3/4] fix pipeline failure Signed-off-by: peaceh <103117813+peaceh-nv@users.noreply.github.com> --- tests/integration/defs/cpp_common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/defs/cpp_common.py b/tests/integration/defs/cpp_common.py index ccea236f99e..b68e26ca227 100755 --- a/tests/integration/defs/cpp_common.py +++ b/tests/integration/defs/cpp_common.py @@ -743,6 +743,8 @@ def run_single_gpu_tests(build_dir: _pl.Path, # exist in multiple running tests if gpt_tests.intersection(test_list): parallel = 1 + else: + parallel = default_test_parallel if parallel_override := _os.environ.get("LLM_TEST_PARALLEL_OVERRIDE", None): From 22f8d3d1b452c3ca54aceec05805f7cf874c8a80 Mon Sep 17 00:00:00 2001 From: peaceh <103117813+peaceh-nv@users.noreply.github.com> Date: Wed, 16 Apr 2025 15:43:14 +0000 Subject: [PATCH 4/4] Fix the wrong name Signed-off-by: peaceh <103117813+peaceh-nv@users.noreply.github.com> --- tests/integration/defs/cpp_common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/cpp_common.py b/tests/integration/defs/cpp_common.py index b68e26ca227..46a08c2bbc1 100755 --- a/tests/integration/defs/cpp_common.py +++ b/tests/integration/defs/cpp_common.py @@ -75,11 +75,11 @@ def generate_result_file_name(test_list: List[str], def generate_excluded_test_list(test_list): if "gpt" in test_list: if "gpt_session" not in test_list: - yield "gpt_session" + yield "GptSession" if "gpt_executor" not in test_list: - yield "gpt_executor" + yield "GptExecutor" if "gpt_tests" not in test_list: - yield "gpt_tests" + yield "GptTests" def find_dir_containing(files: Sequence[str],