diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index de9da034765c..b386b1aca564 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1595,6 +1595,19 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor def logger = new Logger(pipeline) def (jenkinsURL, buildStatus) = JobBuilder.build(pipeline, logger, jobName, parameters, 1, false) + // Infra-scoped fail-fast (parent half). A downstream sub-job returns UNSTABLE + // when it saw only infra aborts and no genuine test/build failure (see + // runBranchesWithInfraDefer in L0_Test.groovy). That is incomplete coverage, + // not a failure: throwing here is exactly what trips the per-arch failFast and + // cancels the healthy sibling architecture, so do NOT throw. Mark the build + // UNSTABLE (visible + re-runnable) and let the sibling finish. FAILURE and + // ABORTED still throw below, so real failures fail-fast exactly as before. + if (buildStatus == "UNSTABLE") { + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + error "Downstream job ${jobName} is infra-incomplete (UNSTABLE); sibling not cancelled" + } + return buildStatus + } if (buildStatus != "SUCCESS") { error "Downstream job did not succeed" } @@ -1643,6 +1656,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false + def singleGpuInfraIncomplete = false stage(testStageName) { if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "x86_64 test job is skipped due to Jenkins configuration" @@ -1657,7 +1671,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + // launchJob returns UNSTABLE (without throwing) when the single-GPU + // sub-job was infra-incomplete: only infra aborts, no real failure. + def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1699,6 +1716,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a + // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than + // spend scarce multi-GPU resource on a partially-unverified premise -- + // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE + // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps + // running multi-GPU for max signal, mirroring the single-GPU-failed policy. + if (singleGpuInfraIncomplete) { + if (env.JOB_NAME ==~ /.*PostMerge.*/) { + echo "In the official post-merge pipeline, x86_64 single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running." + } else { + stage("[Test-x86_64-Multi-GPU] Skipped - single-GPU infra-incomplete") { + echo "x86_64 single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE." + } + return + } + } + // Label gate: check before entering the Remote Run stage so a // missing/unauthorized label shows as "Blocked" (not a Remote Run // failure) and does not trigger fail-fast. @@ -1774,6 +1808,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false + def singleGpuInfraIncomplete = false stage(testStageName) { if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "SBSA test job is skipped due to Jenkins configuration" @@ -1787,7 +1822,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + // launchJob returns UNSTABLE (without throwing) when the single-GPU + // sub-job was infra-incomplete: only infra aborts, no real failure. + def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1830,6 +1868,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a + // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than + // spend scarce multi-GPU resource on a partially-unverified premise -- + // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE + // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps + // running multi-GPU for max signal, mirroring the single-GPU-failed policy. + if (singleGpuInfraIncomplete) { + if (env.JOB_NAME ==~ /.*PostMerge.*/) { + echo "In the official post-merge pipeline, SBSA single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running." + } else { + stage("[Test-SBSA-Multi-GPU] Skipped - single-GPU infra-incomplete") { + echo "SBSA single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE." + } + return + } + } + def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA") if (sbsaLabelBlock) { stage("[Test-SBSA-Multi-GPU] Blocked") { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index adcc4c847e97..1b311965dcd4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -158,6 +158,22 @@ SLURM_INFRA_RETRY_MAX = 1 // to avoid nesting with the inner SLURM retry. K8S_INFRA_RETRY_MAX = 1 +// Infra-scoped fail-fast master switch. When true, a branch whose post-retry +// failure classifies as a positive K8s infra abort (via +// FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its sibling +// branches keep running instead of being SIGTERMed by failFast -- and a sub-job +// that saw only infra aborts (no genuine failure) resolves to UNSTABLE. When +// false, every failure rethrows and the original bare-boolean fail-fast is fully +// restored. Kept separate from params.enableFailFast so the scoped behavior can +// be disabled pipeline-wide without turning fail-fast itself off. Only K8s-scoped +// aborts are deferred today; SLURM-scoped aborts fall back to today's fail-fast +// (see runBranchesWithInfraDefer). +// +// Overridable without a code change by setting the ENABLE_INFRA_SCOPED_FAILFAST +// env var on the job. Env values are strings ("false" is truthy in Groovy), so +// the override goes through toBoolean() rather than the bare elvis. +ENABLE_INFRA_SCOPED_FAILFAST = env.ENABLE_INFRA_SCOPED_FAILFAST ? env.ENABLE_INFRA_SCOPED_FAILFAST.toBoolean() : true + // Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the // 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap // or disable stage-level infra retries for resource-scarce hardware pools -- @@ -5359,6 +5375,61 @@ def buildStageConfigs(stageName, platform, testlist, testCount, gpuCount, nodeCo return configs } +// Infra-scoped fail-fast (inner/branch layer). Runs `jobs` under `parallel` so a +// branch whose post-retry failure is a positive K8s infra abort +// (FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its siblings +// keep running instead of being SIGTERMed by failFast. A genuine test/build +// failure (or an unclassified one) is rethrown unchanged, so failFast stays fully +// active for real failures; an interrupt (e.g. a sibling's own fail-fast SIGTERM) +// is also rethrown and never swallowed. After the join, a sub-job that saw ONLY +// infra aborts and no real failure resolves to UNSTABLE (coverage incomplete, not +// a failure) so the parent layer (L0_MergeRequest.launchJob) can spare the healthy +// sibling architecture; a mixed sub-job already threw on its real failure and is +// FAILURE (currentBuild.result worst-of semantics won't downgrade it). +// +// Scope: classify() is scope-filtered, so this passes K8S -- the motivating +// pod-scheduling abort (KubernetesClientTimeoutException) is K8S-scoped. SLURM-only +// aborts do NOT match here and keep today's fail-fast; deferring those too means +// threading each stage's scope (opts.slurmDispatcher) in -- a follow-up, not this +// change. Gated on ENABLE_INFRA_SCOPED_FAILFAST; off restores today's behavior +// exactly (plain failFast + parallel, no wrapping, no UNSTABLE). +def runBranchesWithInfraDefer(Map jobs, boolean failFast) { + if (!ENABLE_INFRA_SCOPED_FAILFAST) { + jobs.failFast = failFast + parallel jobs + return + } + // CPS serializes parallel-branch continuations onto a single VM thread, so a + // plain list append from the catch blocks below is safe -- there is no + // JVM-level concurrency to guard against here. + def deferred = [] + def wrapped = jobs.collectEntries { stageName, body -> + [(stageName), { + try { + body() + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + if (FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S)) { + deferred.add([stage: stageName]) + echo "[INFRA-DEFER] ${stageName}: K8s infra abort recorded; " + + "siblings continue instead of fail-fast. ${e.toString()}" + return + } + throw e + } + }] + } + wrapped.failFast = failFast + parallel wrapped + if (deferred) { + echo "[INFRA-DEFER] ${deferred.size()} stage(s) infra-incomplete " + + "(${deferred.collect { it.stage }.join(', ')}); marking result UNSTABLE " + + "(coverage incomplete, no genuine test failure)." + currentBuild.result = 'UNSTABLE' + } +} + def launchTestJobs(pipeline, testFilter, globalVars) { def versionOverride = globalVars[TRTLLM_VERSION_OVERRIDE] ?: "" @@ -6527,31 +6598,27 @@ pipeline { echo "Skip multi-GPU testing. No test to run." } if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs + runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast) } else { echo "Skip single-GPU testing. No test to run." } } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { echo "Only run multi-GPU tests." if (dgxJobs.size() > 0) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs + runBranchesWithInfraDefer(dgxJobs, params.enableFailFast) } else { error "Skip multi-GPU testing. No test to run." } } else { if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs + runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast) } else { echo "Skip single-GPU testing. No test to run." } if (dgxJobs.size() > 0) { stage(testPhase2StageName) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs + runBranchesWithInfraDefer(dgxJobs, params.enableFailFast) } } } diff --git a/jenkins/TensorRT_LLM_PLC.groovy b/jenkins/TensorRT_LLM_PLC.groovy index c4162ee846a0..a6714f3c5c0b 100644 --- a/jenkins/TensorRT_LLM_PLC.groovy +++ b/jenkins/TensorRT_LLM_PLC.groovy @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + @Library(['trtllm-jenkins-shared-lib@main']) _ import groovy.json.JsonSlurper diff --git a/jenkins/runPerfSanityTriage.groovy b/jenkins/runPerfSanityTriage.groovy index a9af3aeccc4a..4a0890ffe128 100644 --- a/jenkins/runPerfSanityTriage.groovy +++ b/jenkins/runPerfSanityTriage.groovy @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + @Library(['trtllm-jenkins-shared-lib@main']) _ DOCKER_IMAGE = "artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-25.10-py3-x86_64-ubuntu24.04-trt10.13.3.9-skip-tritondevel-202510291120-8621"