Add HIP API logs for the Navi3x nightly builds

external/llvm-project/mlir/test/Integration/lit.local.cfg

@@ -1,5 +1,5 @@
 from lit.llvm import llvm_config
-
+llvm_config.with_environment("AMD_LOG_LEVEL", "4")
 if not config.mlir_include_integration_tests:
     config.unsupported = True
 

external/llvm-project/mlir/test/lit.cfg.py

@@ -75,6 +75,7 @@ def add_runtime(name):
 
 
 llvm_config.with_system_environment(["HOME", "INCLUDE", "LIB", "TMP", "TEMP"])
+llvm_config.with_environment("AMD_LOG_LEVEL", "4")
 
 llvm_config.use_default_substitutions()
 

mlir/utils/jenkins/Jenkinsfile

@@ -222,25 +222,29 @@ def rebootNode() {
 void build_fixedE2ETests(String codepath) {
     // Limit the number of lit workers to 8 for navi21/navi3x codepath on nightly CI as a workaround for issue#702
     limit_lit_workers = false
+    llvm_lit_log_level = '-v'
     if ( (codepath == 'navi21') && (params.nightly == true) ) {
         limit_lit_workers = true
     }
     // Filter out dense_output_bf16.mlir for navi3x codepath as a workaround for issue#1023
     filter_out_tests = false
     if ( codepath == 'navi3x') {
+        // print verbose logs for all the tests on Navi3x for the debugging purposes
+        llvm_lit_log_level = '-va'
         filter_out_tests = true
     }
     buildProject('check-mlir-build-only check-rocmlir-build-only', """
               -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=${params.nightly ? '0' : '1'}
               -DROCMLIR_DRIVER_E2E_TEST_ENABLED=${params.nightly ? '1' : '0'}
               -DROCK_E2E_TEST_ENABLED=${params.nightly ? '1' : '0'}
               -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=1
-              -DLLVM_LIT_ARGS='-v --time-tests ${ limit_lit_workers ? '-j 8' : ' ' } ${ filter_out_tests ? '--filter-out=dense_output_bf16.mlir' : ' '}'
+              -DLLVM_LIT_ARGS='${llvm_lit_log_level} --time-tests ${ limit_lit_workers ? '-j 8' : ' ' } ${ filter_out_tests ? '--filter-out=dense_output_bf16.mlir' : ' '}'
               -DCMAKE_EXPORT_COMPILE_COMMANDS=1
              """)
 }
 
 void checkRocmlirOnNavi3x(boolean fixed, String testSuite) {
+    // print verbose logs for all the tests on Navi3x for the debugging purposes
     sh '[ ! -d build ] || rm -rf build'
     buildProject('check-rocmlir', """
              -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0
@@ -249,7 +253,7 @@ void checkRocmlirOnNavi3x(boolean fixed, String testSuite) {
              -DROCK_E2E_TEST_SUITES=${testSuite}
              -DROCMLIR_DRIVER_RANDOM_DATA_SEED=${fixed ? 'none' : '1'}
              -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=${fixed ? 1 : 0}
-             -DLLVM_LIT_ARGS='-v --time-tests --filter-out=dense_output_bf16.mlir -j 4'
+             -DLLVM_LIT_ARGS='-va --time-tests --filter-out=dense_output_bf16.mlir -j 4'
              -DCMAKE_EXPORT_COMPILE_COMMANDS=1
      """)
 }
@@ -271,6 +275,7 @@ void check_RockE2ETests_Navi3x(boolean fixed) {
     }
     // Run nightly E2E tests in multiple smaller batches to increase the chance of successful completion
     else {
+        // print verbose logs for all the tests on Navi3x for debugging purposes 
         sh '[ ! -d build ] || rm -rf build'
         buildProject('check-mlir check-rocmlir', """
              -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0
@@ -279,7 +284,7 @@ void check_RockE2ETests_Navi3x(boolean fixed) {
              -DROCK_E2E_TEST_SUITES='part1'
              -DROCMLIR_DRIVER_RANDOM_DATA_SEED=${fixed ? 'none' : '1'}
              -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=${fixed ? 1 : 0}
-             -DLLVM_LIT_ARGS='-v --time-tests --filter-out=dense_output_bf16.mlir -j 4'
+             -DLLVM_LIT_ARGS='-va --time-tests --filter-out=dense_output_bf16.mlir -j 4'
              -DCMAKE_EXPORT_COMPILE_COMMANDS=1
         """)
         checkRocmlirOnNavi3x(fixed, 'part2')
@@ -292,16 +297,22 @@ void check_RockE2ETests_Navi3x(boolean fixed) {
 void check_randomE2ETests(String codepath) {
     // Limit the number of lit workers to 8 for navi21/navi3x codepath on nightly CI as a workaround for issue#702
     limit_lit_workers = false
+    llvm_lit_log_level='-v'
     if ( (codepath == 'navi21') && (params.nightly == true) ) {
         limit_lit_workers = true
     }
+    if ( (codepath == 'navi3x') && (params.nightly == true) ) {
+        limit_lit_workers = true
+        // print verbose logs for all the tests on Navi3x for debugging purposes
+        llvm_lit_log_level='-va'
+    }
     buildProject('check-rocmlir', """
               -DROCMLIR_DRIVER_PR_E2E_TEST_ENABLED=0
               -DROCMLIR_DRIVER_E2E_TEST_ENABLED=1
               -DROCK_E2E_TEST_ENABLED=1
               -DROCMLIR_DRIVER_RANDOM_DATA_SEED=1
               -DROCMLIR_DRIVER_TEST_GPU_VALIDATION=0
-              -DLLVM_LIT_ARGS='-v --time-tests ${ limit_lit_workers ? '-j 8' : ' ' }'
+              -DLLVM_LIT_ARGS='${llvm_lit_log_level} --time-tests ${ limit_lit_workers ? '-j 8' : ' ' }'
               -DCMAKE_EXPORT_COMPILE_COMMANDS=1
              """)
 }