From bd7436c4b6a8db9d7b683fd3ab9531f9a9435f8b Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:41:45 -0700 Subject: [PATCH 1/6] spawn parallel workers for testing Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/test_unittests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index 3f13ed10b7b..0f50f60b1e1 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -153,7 +153,7 @@ def run_command(cmd): output_dir, f'parallel-sub-results-unittests-{case_fn}.xml.intermediate') parallel_command = command + [ - "-n", f"{num_workers}", f"--junitxml={parallel_output_xml}" + "-n", f"{num_workers}", "--dist-start-method", "spawn", f"--junitxml={parallel_output_xml}" ] passed = run_command(parallel_command) From 4880e02b49b86e2c748cdde606bd6b92822bc8ab Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:59:01 -0700 Subject: [PATCH 2/6] format Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/test_unittests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index 0f50f60b1e1..79ca845abb7 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -153,7 +153,8 @@ def run_command(cmd): output_dir, f'parallel-sub-results-unittests-{case_fn}.xml.intermediate') parallel_command = command + [ - "-n", f"{num_workers}", "--dist-start-method", "spawn", f"--junitxml={parallel_output_xml}" + "-n", f"{num_workers}", "--dist-start-method", "spawn", + f"--junitxml={parallel_output_xml}" ] passed = run_command(parallel_command) From 44d57c3d05170bcdc633fe0d975248bf876be0e6 Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:33:11 -0700 Subject: [PATCH 3/6] fix Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/unittest/pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/pytest.ini b/tests/unittest/pytest.ini index f1d72c7dbe6..1e93eae77ab 100644 --- a/tests/unittest/pytest.ini +++ b/tests/unittest/pytest.ini @@ -1,4 +1,5 @@ [pytest] +xdist_start_method = spawn asyncio_default_fixture_loop_scope = module threadleak = True threadleak_exclude = asyncio_\d+ From 0e109b72acc385fde467272d330cb7f49ed0c513 Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Tue, 2 Sep 2025 16:53:01 -0700 Subject: [PATCH 4/6] revert Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/test_unittests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/defs/test_unittests.py b/tests/integration/defs/test_unittests.py index 79ca845abb7..3f13ed10b7b 100644 --- a/tests/integration/defs/test_unittests.py +++ b/tests/integration/defs/test_unittests.py @@ -153,8 +153,7 @@ def run_command(cmd): output_dir, f'parallel-sub-results-unittests-{case_fn}.xml.intermediate') parallel_command = command + [ - "-n", f"{num_workers}", "--dist-start-method", "spawn", - f"--junitxml={parallel_output_xml}" + "-n", f"{num_workers}", f"--junitxml={parallel_output_xml}" ] passed = run_command(parallel_command) From dd3e0f37a3fbef512f6c4e420d47556e25ef9d2e Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Wed, 3 Sep 2025 09:16:56 -0700 Subject: [PATCH 5/6] set TORCHINDUCTOR_COMPILE_THREADS to 1 Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- .../unittest/_torch/thop/parallel/conftest.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/unittest/_torch/thop/parallel/conftest.py diff --git a/tests/unittest/_torch/thop/parallel/conftest.py b/tests/unittest/_torch/thop/parallel/conftest.py new file mode 100644 index 00000000000..27911b32157 --- /dev/null +++ b/tests/unittest/_torch/thop/parallel/conftest.py @@ -0,0 +1,28 @@ +import os + +import pytest + + +@pytest.fixture(autouse=True, scope='function') +def set_torchinductor_compile_threads(): + """ + Fixture to set TORCHINDUCTOR_COMPILE_THREADS=1 for tests in this directory. + """ + # --- Setup Phase --- + # Save the original value if it exists + original_value = os.environ.get('TORCHINDUCTOR_COMPILE_THREADS') + + # Set the desired value for the test + os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = '1' + + # Let the test run with the new environment variable + yield + + # --- Teardown Phase --- + # Restore the original environment state after the test is done + if original_value is None: + # If the variable didn't exist before, remove it + del os.environ['TORCHINDUCTOR_COMPILE_THREADS'] + else: + # Otherwise, restore its original value + os.environ['TORCHINDUCTOR_COMPILE_THREADS'] = original_value From 9c8ee7d37023e27548197342a6ab3700fd734f41 Mon Sep 17 00:00:00 2001 From: junq <22017000+QiJune@users.noreply.github.com> Date: Wed, 3 Sep 2025 17:44:37 -0700 Subject: [PATCH 6/6] increase to 16 Signed-off-by: junq <22017000+QiJune@users.noreply.github.com> --- tests/integration/defs/agg_unit_mem_df.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/defs/agg_unit_mem_df.csv b/tests/integration/defs/agg_unit_mem_df.csv index 2629c91b541..a33d05450fd 100644 --- a/tests/integration/defs/agg_unit_mem_df.csv +++ b/tests/integration/defs/agg_unit_mem_df.csv @@ -104,14 +104,14 @@ unittest/trt/model/test_mamba.py,NVIDIA H100,10, unittest/_torch/attention,NVIDIA Graphics Device,4,B200 Bring Up Board unittest/_torch/misc,NVIDIA Graphics Device,4,B200 Bring Up Board unittest/_torch/speculative,NVIDIA Graphics Device,4,B200 Bring Up Board -unittest/_torch/thop/parallel,NVIDIA Graphics Device,4,B200 Bring Up Board +unittest/_torch/thop/parallel,NVIDIA Graphics Device,16,B200 Bring Up Board "unittest/_torch/auto_deploy/unit/singlegpu -k ""not test_trtllm_bench_backend_comparison""",NVIDIA Graphics Device,4,B200 Bring Up Board unittest/_torch/attention,NVIDIA B200,4, unittest/_torch/misc,NVIDIA B200,4, unittest/_torch/speculative,NVIDIA B200,4, -unittest/_torch/thop/parallel,NVIDIA B200,4, +unittest/_torch/thop/parallel,NVIDIA B200,16, "unittest/_torch/auto_deploy/unit/singlegpu -k ""not test_trtllm_bench_backend_comparison""",NVIDIA B200,4, unittest/_torch/attention,NVIDIA H100,4, unittest/_torch/misc,NVIDIA H100,4, unittest/_torch/speculative,NVIDIA H100,2, -unittest/_torch/thop/parallel,NVIDIA H100,4, +unittest/_torch/thop/parallel,NVIDIA H100,16,