diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 089b2bcdf90..af8195de504 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1944,19 +1944,41 @@ def generateRerunReport(stageName, llmSrc) { def rerunBaseDir = "${WORKSPACE}/${stageName}/rerun" def regularRerunDir = "${rerunBaseDir}/regular" - // Check if regular rerun directory exists - def hasRegularReruns = sh(script: "[ -d '${regularRerunDir}' ] && echo 'true' || echo 'false'", returnStdout: true).trim() == 'true' + // Check if regular rerun directory has rerun_results_*.xml files + def hasRegularReruns = sh(script: "[ -d '${regularRerunDir}' ] && find '${regularRerunDir}' -name 'rerun_results_*.xml' | head -1 | grep -q . && echo 'true' || echo 'false'", returnStdout: true).trim() == 'true' - // Find all isolated rerun directories (isolated_0, isolated_1, etc.) - def isolatedRerunDirs = [] - def isolatedDirsOutput = sh(script: "find ${rerunBaseDir} -type d -name 'isolated_*' 2>/dev/null || true", returnStdout: true).trim() - if (isolatedDirsOutput) { - isolatedRerunDirs = isolatedDirsOutput.split('\n').findAll { it.trim() } + // Check if any isolated rerun directories have rerun_results_*.xml files + def hasIsolatedReruns = sh(script: "find ${rerunBaseDir} -type d -name 'isolated_*' -exec find {} -name 'rerun_results_*.xml' \\; 2>/dev/null | head -1 | grep -q . && echo 'true' || echo 'false'", returnStdout: true).trim() == 'true' + + // Find isolated tests that have actual rerun results and build mapping + def isolatedTestsWithReruns = [] + if (hasIsolatedReruns) { + def isolatedDirsOutput = sh(script: "find ${rerunBaseDir} -type d -name 'isolated_*' 2>/dev/null || true", returnStdout: true).trim() + if (isolatedDirsOutput) { + def isolatedDirs = isolatedDirsOutput.split('\n').findAll { it.trim() } + isolatedDirs.each { isolatedDir -> + // Extract the isolated number from directory name (e.g., isolated_0 -> 0) + def isolatedNum = isolatedDir.split('/').last().replace('isolated_', '') + + // Check if this isolated directory has any rerun results + def hasRerunResults = sh(script: "find '${isolatedDir}' -name 'rerun_results_*.xml' | head -1 | grep -q . && echo 'true' || echo 'false'", returnStdout: true).trim() == 'true' + + if (hasRerunResults) { + isolatedTestsWithReruns.add([ + dir: isolatedDir, + num: isolatedNum, + originalResult: "${WORKSPACE}/${stageName}/results_isolated_${isolatedNum}.xml" + ]) + } + } + } } - def hasIsolatedReruns = isolatedRerunDirs.size() > 0 + + // Collect rerun result files and corresponding original result files + def rerunResultFiles = [] echo "Found regular reruns: ${hasRegularReruns}" - echo "Found isolated rerun directories: ${isolatedRerunDirs}" + echo "Found isolated tests with reruns: ${isolatedTestsWithReruns.collect { "isolated_${it.num}" }}" if (!hasRegularReruns && !hasIsolatedReruns) { echo "No rerun results found, skipping rerun report generation" @@ -1972,9 +1994,13 @@ def generateRerunReport(stageName, llmSrc) { // Add original results if (fileExists("${WORKSPACE}/${stageName}/results.xml")) { allInputFiles.add("${WORKSPACE}/${stageName}/results.xml") + // Add to rerunResultFiles only if it has reruns + if (hasRegularReruns) { + rerunResultFiles.add("${WORKSPACE}/${stageName}/results.xml") + } } - // Add isolated test results + // Add ALL isolated test results to allInputFiles def isolatedResults = sh(script: "find ${WORKSPACE}/${stageName} -name 'results_isolated_*.xml' 2>/dev/null || true", returnStdout: true).trim() if (isolatedResults) { isolatedResults.split('\n').each { file -> @@ -1982,6 +2008,20 @@ def generateRerunReport(stageName, llmSrc) { allInputFiles.add(file.trim()) } } + // Add isolated test results that have reruns to rerunResultFiles and add their rerun results to allInputFiles + isolatedTestsWithReruns.each { isolatedTest -> + if (fileExists(isolatedTest.originalResult)) { + rerunResultFiles.add(isolatedTest.originalResult) + echo "Added isolated result with reruns to rerunResultFiles: ${isolatedTest.originalResult}" + } + for (times in [1, 2]) { + def rerunFile = "${isolatedTest.dir}/rerun_results_${times}.xml" + if (fileExists(rerunFile)) { + allInputFiles.add(rerunFile) + rerunResultFiles.add(rerunFile) + } + } + } } // Add regular rerun results @@ -1990,19 +2030,7 @@ def generateRerunReport(stageName, llmSrc) { def rerunFile = "${regularRerunDir}/rerun_results_${times}.xml" if (fileExists(rerunFile)) { allInputFiles.add(rerunFile) - } - } - } - - // Add isolated rerun results from all isolated directories - if (hasIsolatedReruns) { - isolatedRerunDirs.each { isolatedDir -> - for (times in [1, 2]) { - def rerunFile = "${isolatedDir}/rerun_results_${times}.xml" - if (fileExists(rerunFile)) { - allInputFiles.add(rerunFile) - echo "Added isolated rerun result: ${rerunFile}" - } + rerunResultFiles.add(rerunFile) } } } @@ -2012,14 +2040,14 @@ def generateRerunReport(stageName, llmSrc) { return } - echo "Generating rerun report with input files: ${allInputFiles.join(',')}" + echo "Generating rerun report with input files: ${rerunResultFiles.join(',')}" // Generate comprehensive rerun report sh """ python3 ${llmSrc}/jenkins/scripts/test_rerun.py \ generate_rerun_report \ --output-file=${WORKSPACE}/${stageName}/rerun_results.xml \ - --input-files=${allInputFiles.join(",")} + --input-files=${rerunResultFiles.join(",")} """ // Update original results xml file with all rerun results for junit diff --git a/jenkins/scripts/test_rerun.py b/jenkins/scripts/test_rerun.py index e3af8928f13..36300e51c58 100644 --- a/jenkins/scripts/test_rerun.py +++ b/jenkins/scripts/test_rerun.py @@ -385,7 +385,7 @@ def filter_failed_tests(xml_filename, output_filename): def generate_rerun_report(output_filename, input_filenames): - # Merge the input xml files (filter failed tests for results.xml) + # Merge the input xml files (filter failed tests for results.xml and results_isolated_*.xml) # and generate the rerun report html file. new_filename_list = [] for input_filename in input_filenames: @@ -394,6 +394,12 @@ def generate_rerun_report(output_filename, input_filenames): new_filename = input_filename.replace("results.xml", "failed_results.xml") filter_failed_tests(input_filename, new_filename) + elif "/results_isolated_" in input_filename and input_filename.endswith( + ".xml"): + # Handle isolation test result files: results_isolated_*.xml -> failed_results_isolated_*.xml + new_filename = input_filename.replace("results_isolated_", + "failed_results_isolated_") + filter_failed_tests(input_filename, new_filename) new_filename_list.append(new_filename) print(new_filename_list)