From 441b9cb2aab84a4594bce50b21b032a1510e6985 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:23:16 +0800 Subject: [PATCH 1/8] [None][feat] Upload per-arch single-GPU coverage early in post-merge pipeline In official post-merge runs, single-GPU coverage data is ready well before multi-GPU stages finish, but the selector had to wait for the full pipeline to complete before any coverage artifact appeared on Artifactory. Add uploadArchCoverage(arch) called immediately after each arch's single-GPU job (before multi-GPU starts). It spins up a lightweight alpine K8s pod, downloads the already-uploaded single-GPU result tarballs, merges the .cbtscov.*.sqlite files via pystart_report.py (sqlite only, no HTML), and uploads cbts_pystart_report_{arch}.tar.gz to cbts-coverage/. Update artifact.py to prefer these early per-arch artifacts when the selector fetches the touch DB: if either cbts_pystart_report_x86_64.tar.gz or cbts_pystart_report_SBSA.tar.gz exists for the latest build, their sqlite files are merged and returned immediately without waiting for the complete cbts_pystart_report.tar.gz (which collectTestResults still uploads at the end with full single+multi-GPU data and HTML report). Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 44 ++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 5285a63ce45f..92308cf0289d 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1263,6 +1263,47 @@ def getOnlyOneGroupChanged(pipeline, testFilter, globalVars) { return "" } +// Upload sqlite-only early coverage after single-GPU finishes, before multi-GPU starts; non-fatal. +def uploadArchCoverage(String arch, pipeline, testFilter) { + if (!testFilter[(CBTS_COVERAGE)]) { + return + } + try { + timeout(time: 15, unit: 'MINUTES') { + def podSpec = createKubernetesPodConfig("", "agent") + trtllm_utils.launchKubernetesPod(pipeline, podSpec, "alpine", { + stage("Upload Single-GPU Coverage (${arch})") { + def testResultLink = "https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/${JOB_NAME}/${BUILD_NUMBER}/test-results" + sh "rm -rf cov && mkdir -p cov" + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apk add --no-cache curl python3 py3-pip") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 config set global.break-system-packages true") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget ${testResultLink}/", allowStepFailed: true) + sh "cat index.html | grep \"tar.gz\" | cut -d \"\\\"\" -f 2 > result_file_names.txt" + trtllm_utils.llmExecStepWithRetry(pipeline, script: "cat result_file_names.txt | xargs -n1 -I {} wget -c -nv ${testResultLink}/{}", allowStepFailed: true) + sh "find . -name 'results-*.tar.gz' -type f -exec tar -zxvf {} \\; || true" + sh "find . -type f -name '.cbtscov.*.sqlite' -exec mv -t cov/ {} + || true" + def fileCount = sh(returnStdout: true, script: 'find cov -name ".cbtscov.*.sqlite" | wc -l').replaceAll("\\s","").toInteger() + if (fileCount > 0) { + trtllm_utils.checkoutSource(LLM_REPO, env.gitlabCommit, LLM_ROOT, true, true) + sh """ + python3 ${LLM_ROOT}/jenkins/scripts/cbts/coverage_utils/pystart_report.py \ + --glob 'cov/.cbtscov.*.sqlite' \ + --out-sqlite cov/cbts_touchmap.sqlite + """ + sh "cd cov && tar czf cbts_pystart_report_${arch}.tar.gz cbts_touchmap.sqlite" + trtllm_utils.uploadArtifacts("cov/cbts_pystart_report_${arch}.tar.gz", "${UPLOAD_PATH}/cbts-coverage/") + echo "CBTS early coverage (${arch}): https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/cbts-coverage/cbts_pystart_report_${arch}.tar.gz" + } else { + echo "CBTS early coverage (${arch}): no data files found, skipping." + } + } + }) + } + } catch (Exception e) { + echo "CBTS early coverage upload (${arch}) failed (non-fatal): ${e.toString()}" + } +} + def collectTestResults(pipeline, testFilter, globalVars) { collectResultPodSpec = createKubernetesPodConfig("", "agent") @@ -1620,6 +1661,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } } + uploadArchCoverage("x86_64", pipeline, testFilter) def requireMultiGpuTesting = currentBuild.description?.contains("Require x86_64 Multi-GPU Testing") ?: false echo "requireMultiGpuTesting: ${requireMultiGpuTesting}" @@ -1750,6 +1792,8 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + uploadArchCoverage("SBSA", pipeline, testFilter) + def requireMultiGpuTesting = currentBuild.description?.contains("Require SBSA Multi-GPU Testing") ?: false echo "requireMultiGpuTesting: ${requireMultiGpuTesting}" if (!requireMultiGpuTesting) { From e4c79623fb2a4280d99b0a3c6b4222926d76a066 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 20:40:16 +0800 Subject: [PATCH 2/8] [None][test] Temporarily enable CBTS coverage on all pipelines for upload verification Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 92308cf0289d..971a4bdad7ad 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -358,8 +358,8 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. - // Coverage runs only on the official post-merge pipeline. - testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) + // TODO: restrict back to PostMerge after coverage upload is verified. + testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}") getContainerURIs().each { k, v -> globalVars[k] = v From 886c4df042739f53b70848f97ed86271dac21584 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 4 Aug 2026 22:51:22 +0800 Subject: [PATCH 3/8] [None][fix] Use UPLOAD_PATH for test-results links in coverage and result collection Co-Authored-By: Claude Sonnet 4.6 (1M context) Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 971a4bdad7ad..1c0c0271062d 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1273,7 +1273,7 @@ def uploadArchCoverage(String arch, pipeline, testFilter) { def podSpec = createKubernetesPodConfig("", "agent") trtllm_utils.launchKubernetesPod(pipeline, podSpec, "alpine", { stage("Upload Single-GPU Coverage (${arch})") { - def testResultLink = "https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/${JOB_NAME}/${BUILD_NUMBER}/test-results" + def testResultLink = "https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/test-results" sh "rm -rf cov && mkdir -p cov" trtllm_utils.llmExecStepWithRetry(pipeline, script: "apk add --no-cache curl python3 py3-pip") trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 config set global.break-system-packages true") @@ -1312,7 +1312,7 @@ def collectTestResults(pipeline, testFilter, globalVars) stage ("Collect Test Result") { sh "rm -rf **/*.xml *.tar.gz" - testResultLink = "https://urm.nvidia.com/artifactory/sw-tensorrt-generic/llm-artifacts/${JOB_NAME}/${BUILD_NUMBER}/test-results" + testResultLink = "https://urm.nvidia.com/artifactory/${UPLOAD_PATH}/test-results" trtllm_utils.llmExecStepWithRetry(pipeline, script: "apk add --no-cache curl") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apk add python3") From 526f12dc31aa2872e5476ecd3b6c78ed32713673 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:25:05 +0800 Subject: [PATCH 4/8] [None][fix] Filter test-results by arch in uploadArchCoverage to prevent cross-contamination x86_64 and SBSA jobs upload to the same test-results directory. Without filtering, each arch's early coverage sqlite ends up containing data from both architectures. Filter the download list by known SBSA stage name prefixes (GH200-, GB10-, GB200-, GB300-, CPU-Generic-arm-) so each report only contains its own arch's coverage data. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 1c0c0271062d..8a7d409813c1 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1264,10 +1264,17 @@ def getOnlyOneGroupChanged(pipeline, testFilter, globalVars) { } // Upload sqlite-only early coverage after single-GPU finishes, before multi-GPU starts; non-fatal. +// Both x86_64 and SBSA jobs upload to the same test-results directory, so filter by arch before +// downloading to avoid cross-contamination. SBSA stage names begin with GH200-, GB10-, GB200-, +// GB300-, or CPU-Generic-arm-; all other stage names belong to x86_64. def uploadArchCoverage(String arch, pipeline, testFilter) { if (!testFilter[(CBTS_COVERAGE)]) { return } + def sbsaPrefixPattern = "^results-(GH200|GB10|GB200|GB300|CPU-Generic-arm)-" + def archGrepCmd = (arch == "SBSA") + ? "grep -E '${sbsaPrefixPattern}' result_file_names.txt > arch_file_names.txt || true" + : "grep -v -E '${sbsaPrefixPattern}' result_file_names.txt > arch_file_names.txt || true" try { timeout(time: 15, unit: 'MINUTES') { def podSpec = createKubernetesPodConfig("", "agent") @@ -1279,7 +1286,8 @@ def uploadArchCoverage(String arch, pipeline, testFilter) { trtllm_utils.llmExecStepWithRetry(pipeline, script: "pip3 config set global.break-system-packages true") trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget ${testResultLink}/", allowStepFailed: true) sh "cat index.html | grep \"tar.gz\" | cut -d \"\\\"\" -f 2 > result_file_names.txt" - trtllm_utils.llmExecStepWithRetry(pipeline, script: "cat result_file_names.txt | xargs -n1 -I {} wget -c -nv ${testResultLink}/{}", allowStepFailed: true) + sh archGrepCmd + trtllm_utils.llmExecStepWithRetry(pipeline, script: "cat arch_file_names.txt | xargs -n1 -I {} wget -c -nv ${testResultLink}/{}", allowStepFailed: true) sh "find . -name 'results-*.tar.gz' -type f -exec tar -zxvf {} \\; || true" sh "find . -type f -name '.cbtscov.*.sqlite' -exec mv -t cov/ {} + || true" def fileCount = sh(returnStdout: true, script: 'find cov -name ".cbtscov.*.sqlite" | wc -l').replaceAll("\\s","").toInteger() From 06e4a067b7aeac6c4c20ac42bbe9f572c4fc9079 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:48:33 +0800 Subject: [PATCH 5/8] Revert "[None][test] Temporarily enable CBTS coverage on all pipelines for upload verification" This reverts commit 803080fc33e2e7fa9e3088ae7d6ec6569d221080. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 8a7d409813c1..cd7bfcb00340 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -358,8 +358,8 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. - // TODO: restrict back to PostMerge after coverage upload is verified. - testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE + // Coverage runs only on the official post-merge pipeline. + testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}") getContainerURIs().each { k, v -> globalVars[k] = v From a5cf3e5444593e4dbb26d1d109a65574e81a29f0 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:50:14 +0800 Subject: [PATCH 6/8] [None][fix] Filter test-results by arch in uploadArchCoverage to prevent cross-contamination x86_64 and SBSA jobs upload to the same test-results directory. Without filtering, each arch's early coverage sqlite ends up containing data from both architectures. Filter the download list by known SBSA stage name prefixes (GH200-, GB10-, GB200-, GB300-, CPU-Generic-arm-) so each report only contains its own arch's coverage data. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index cd7bfcb00340..544dc26e63d7 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1264,9 +1264,6 @@ def getOnlyOneGroupChanged(pipeline, testFilter, globalVars) { } // Upload sqlite-only early coverage after single-GPU finishes, before multi-GPU starts; non-fatal. -// Both x86_64 and SBSA jobs upload to the same test-results directory, so filter by arch before -// downloading to avoid cross-contamination. SBSA stage names begin with GH200-, GB10-, GB200-, -// GB300-, or CPU-Generic-arm-; all other stage names belong to x86_64. def uploadArchCoverage(String arch, pipeline, testFilter) { if (!testFilter[(CBTS_COVERAGE)]) { return From ef898941162411371e5351673d308d0225362a04 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Mon, 10 Aug 2026 21:05:23 +0800 Subject: [PATCH 7/8] [None][fix] Rethrow InterruptedException in uploadArchCoverage Broad Exception catch was swallowing Jenkins pipeline cancellations and user aborts (FlowInterruptedException / InterruptedException), causing the helper to log them as non-fatal and continue. Rethrow to match the pattern used by other best-effort blocks in this file. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 544dc26e63d7..8815183bae95 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1304,6 +1304,8 @@ def uploadArchCoverage(String arch, pipeline, testFilter) { } }) } + } catch (InterruptedException e) { + throw e } catch (Exception e) { echo "CBTS early coverage upload (${arch}) failed (non-fatal): ${e.toString()}" } From 709f83a9dbc9500365ae492361ac403e0d20fd8d Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 12 Aug 2026 11:22:58 +0800 Subject: [PATCH 8/8] [None][fix] Distinguish helper timeout from pipeline abort in uploadArchCoverage The helper-specific timeout(15, MINUTES) raises FlowInterruptedException, which the broad InterruptedException handler rethrew, making the uploader timeout fail the entire pipeline against the non-fatal contract of this helper. Catch FlowInterruptedException and treat only an ExceededTimeout cause as non-fatal, continuing to rethrow user aborts and upstream pipeline interruptions. Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 8815183bae95..de9da034765c 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -25,6 +25,7 @@ import com.nvidia.bloom.Constants import com.nvidia.bloom.Logger import com.nvidia.bloom.JobBuilder import org.jenkinsci.plugins.workflow.cps.CpsThread +import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException import org.jsoup.Jsoup import org.jenkinsci.plugins.pipeline.modeldefinition.Utils as jUtils @@ -1304,6 +1305,14 @@ def uploadArchCoverage(String arch, pipeline, testFilter) { } }) } + } catch (FlowInterruptedException e) { + // The 15-minute timeout above is non-fatal; user aborts and upstream + // pipeline interruptions must keep propagating. + if (e.causes.any { it.class.simpleName == 'ExceededTimeout' }) { + echo "CBTS early coverage upload (${arch}) timed out after 15 min (non-fatal): skipping." + } else { + throw e + } } catch (InterruptedException e) { throw e } catch (Exception e) {