diff --git a/external/llvm-project/mlir/test/Integration/lit.local.cfg b/external/llvm-project/mlir/test/Integration/lit.local.cfg index 1dc40f986ead..6a21dec02feb 100644 --- a/external/llvm-project/mlir/test/Integration/lit.local.cfg +++ b/external/llvm-project/mlir/test/Integration/lit.local.cfg @@ -1,5 +1,5 @@ from lit.llvm import llvm_config - +llvm_config.with_environment("AMD_LOG_LEVEL", "4") if not config.mlir_include_integration_tests: config.unsupported = True diff --git a/external/llvm-project/mlir/test/lit.cfg.py b/external/llvm-project/mlir/test/lit.cfg.py index 98d0ddd9a2be..b2ed8912746c 100644 --- a/external/llvm-project/mlir/test/lit.cfg.py +++ b/external/llvm-project/mlir/test/lit.cfg.py @@ -75,6 +75,7 @@ def add_runtime(name): llvm_config.with_system_environment(["HOME", "INCLUDE", "LIB", "TMP", "TEMP"]) +llvm_config.with_environment("AMD_LOG_LEVEL", "4") llvm_config.use_default_substitutions() diff --git a/mlir/utils/jenkins/Jenkinsfile b/mlir/utils/jenkins/Jenkinsfile index dbb121a24db1..72e0a6a18955 100644 --- a/mlir/utils/jenkins/Jenkinsfile +++ b/mlir/utils/jenkins/Jenkinsfile @@ -222,12 +222,15 @@ def rebootNode() { void build_fixedE2ETests(String codepath) { // Limit the number of lit workers to 8 for navi21/navi3x codepath on nightly CI as a workaround for issue#702 limit_lit_workers = false + llvm_lit_log_level = '-v' if ( (codepath == 'navi21') && (params.nightly == true) ) { limit_lit_workers = true } // Filter out dense_output_bf16.mlir for navi3x codepath as a workaround for issue#1023 filter_out_tests = false if ( codepath == 'navi3x') { + // print verbose logs for all the tests on Navi3x for the debugging purposes + llvm_lit_log_level = '-va' filter_out_tests = true } buildProject('check-mlir-build-only check-rocmlir-build-only', """ @@ -235,12 +238,13 @@ void build_fixedE2ETests(String codepath) { -DROCMLIR_DRIVER_E2E_TEST_ENABLED=${params.nightly ? '1' : '0'} -DROCK_E2E_TEST_ENABLED=${params.nightly ? '1' : '0'} -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=1 - -DLLVM_LIT_ARGS='-v --time-tests ${ limit_lit_workers ? '-j 8' : ' ' } ${ filter_out_tests ? '--filter-out=dense_output_bf16.mlir' : ' '}' + -DLLVM_LIT_ARGS='${llvm_lit_log_level} --time-tests ${ limit_lit_workers ? '-j 8' : ' ' } ${ filter_out_tests ? '--filter-out=dense_output_bf16.mlir' : ' '}' -DCMAKE_EXPORT_COMPILE_COMMANDS=1 """) } void checkRocmlirOnNavi3x(boolean fixed, String testSuite) { + // print verbose logs for all the tests on Navi3x for the debugging purposes sh '[ ! -d build ] || rm -rf build' buildProject('check-rocmlir', """ -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0 @@ -249,7 +253,7 @@ void checkRocmlirOnNavi3x(boolean fixed, String testSuite) { -DROCK_E2E_TEST_SUITES=${testSuite} -DROCMLIR_DRIVER_RANDOM_DATA_SEED=${fixed ? 'none' : '1'} -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=${fixed ? 1 : 0} - -DLLVM_LIT_ARGS='-v --time-tests --filter-out=dense_output_bf16.mlir -j 4' + -DLLVM_LIT_ARGS='-va --time-tests --filter-out=dense_output_bf16.mlir -j 4' -DCMAKE_EXPORT_COMPILE_COMMANDS=1 """) } @@ -271,6 +275,7 @@ void check_RockE2ETests_Navi3x(boolean fixed) { } // Run nightly E2E tests in multiple smaller batches to increase the chance of successful completion else { + // print verbose logs for all the tests on Navi3x for debugging purposes sh '[ ! -d build ] || rm -rf build' buildProject('check-mlir check-rocmlir', """ -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0 @@ -279,7 +284,7 @@ void check_RockE2ETests_Navi3x(boolean fixed) { -DROCK_E2E_TEST_SUITES='part1' -DROCMLIR_DRIVER_RANDOM_DATA_SEED=${fixed ? 'none' : '1'} -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=${fixed ? 1 : 0} - -DLLVM_LIT_ARGS='-v --time-tests --filter-out=dense_output_bf16.mlir -j 4' + -DLLVM_LIT_ARGS='-va --time-tests --filter-out=dense_output_bf16.mlir -j 4' -DCMAKE_EXPORT_COMPILE_COMMANDS=1 """) checkRocmlirOnNavi3x(fixed, 'part2') @@ -292,16 +297,22 @@ void check_RockE2ETests_Navi3x(boolean fixed) { void check_randomE2ETests(String codepath) { // Limit the number of lit workers to 8 for navi21/navi3x codepath on nightly CI as a workaround for issue#702 limit_lit_workers = false + llvm_lit_log_level='-v' if ( (codepath == 'navi21') && (params.nightly == true) ) { limit_lit_workers = true } + if ( (codepath == 'navi3x') && (params.nightly == true) ) { + limit_lit_workers = true + // print verbose logs for all the tests on Navi3x for debugging purposes + llvm_lit_log_level='-va' + } buildProject('check-rocmlir', """ -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0 -DROCMLIR_DRIVER_E2E_TEST_ENABLED=1 -DROCK_E2E_TEST_ENABLED=1 -DROCMLIR_DRIVER_RANDOM_DATA_SEED=1 -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=0 - -DLLVM_LIT_ARGS='-v --time-tests ${ limit_lit_workers ? '-j 8' : ' ' }' + -DLLVM_LIT_ARGS='${llvm_lit_log_level} --time-tests ${ limit_lit_workers ? '-j 8' : ' ' }' -DCMAKE_EXPORT_COMPILE_COMMANDS=1 """) }