[TRTLLMINF-311][infra] Infra-scoped fail-fast: defer K8s infra aborts instead of cascading

jenkins/L0_MergeRequest.groovy

@@ -1595,6 +1595,19 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor
 
     def logger = new Logger(pipeline)
     def (jenkinsURL, buildStatus) = JobBuilder.build(pipeline, logger, jobName, parameters, 1, false)
+    // Infra-scoped fail-fast (parent half). A downstream sub-job returns UNSTABLE
+    // when it saw only infra aborts and no genuine test/build failure (see
+    // runBranchesWithInfraDefer in L0_Test.groovy). That is incomplete coverage,
+    // not a failure: throwing here is exactly what trips the per-arch failFast and
+    // cancels the healthy sibling architecture, so do NOT throw. Mark the build
+    // UNSTABLE (visible + re-runnable) and let the sibling finish. FAILURE and
+    // ABORTED still throw below, so real failures fail-fast exactly as before.
+    if (buildStatus == "UNSTABLE") {
+        catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') {
+            error "Downstream job ${jobName} is infra-incomplete (UNSTABLE); sibling not cancelled"
+        }
+        return buildStatus
+    }
     if (buildStatus != "SUCCESS") {
         error "Downstream job did not succeed"
     }
@@ -1643,6 +1656,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
 
                 testStageName = "[Test-x86_64-Single-GPU] Remote Run"
                 def singleGpuTestFailed = false
+                def singleGpuInfraIncomplete = false
                 stage(testStageName) {
                     if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) {
                         echo "x86_64 test job is skipped due to Jenkins configuration"
@@ -1657,7 +1671,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                             'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"],
                         ]
 
-                        launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters)
+                        // launchJob returns UNSTABLE (without throwing) when the single-GPU
+                        // sub-job was infra-incomplete: only infra aborts, no real failure.
+                        def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters)
+                        singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE")
                     } catch (InterruptedException e) {
                         throw e
                     } catch (Exception e) {
@@ -1699,6 +1716,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                     }
                 }
 
+                // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a
+                // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than
+                // spend scarce multi-GPU resource on a partially-unverified premise --
+                // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE
+                // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps
+                // running multi-GPU for max signal, mirroring the single-GPU-failed policy.
+                if (singleGpuInfraIncomplete) {
+                    if (env.JOB_NAME ==~ /.*PostMerge.*/) {
+                        echo "In the official post-merge pipeline, x86_64 single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running."
+                    } else {
+                        stage("[Test-x86_64-Multi-GPU] Skipped - single-GPU infra-incomplete") {
+                            echo "x86_64 single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE."
+                        }
+                        return
+                    }
+                }
+
                 // Label gate: check before entering the Remote Run stage so a
                 // missing/unauthorized label shows as "Blocked" (not a Remote Run
                 // failure) and does not trigger fail-fast.
@@ -1774,6 +1808,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
 
                 testStageName = "[Test-SBSA-Single-GPU] Remote Run"
                 def singleGpuTestFailed = false
+                def singleGpuInfraIncomplete = false
                 stage(testStageName) {
                     if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) {
                         echo "SBSA test job is skipped due to Jenkins configuration"
@@ -1787,7 +1822,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                             'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"],
                         ]
 
-                        launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters)
+                        // launchJob returns UNSTABLE (without throwing) when the single-GPU
+                        // sub-job was infra-incomplete: only infra aborts, no real failure.
+                        def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters)
+                        singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE")
                     } catch (InterruptedException e) {
                         throw e
                     } catch (Exception e) {
@@ -1830,6 +1868,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
                     }
                 }
 
+                // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a
+                // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than
+                // spend scarce multi-GPU resource on a partially-unverified premise --
+                // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE
+                // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps
+                // running multi-GPU for max signal, mirroring the single-GPU-failed policy.
+                if (singleGpuInfraIncomplete) {
+                    if (env.JOB_NAME ==~ /.*PostMerge.*/) {
+                        echo "In the official post-merge pipeline, SBSA single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running."
+                    } else {
+                        stage("[Test-SBSA-Multi-GPU] Skipped - single-GPU infra-incomplete") {
+                            echo "SBSA single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE."
+                        }
+                        return
+                    }
+                }
+
                 def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA")
                 if (sbsaLabelBlock) {
                     stage("[Test-SBSA-Multi-GPU] Blocked") {

jenkins/L0_Test.groovy

@@ -158,6 +158,22 @@ SLURM_INFRA_RETRY_MAX = 1
 // to avoid nesting with the inner SLURM retry.
 K8S_INFRA_RETRY_MAX = 1
 
+// Infra-scoped fail-fast master switch. When true, a branch whose post-retry
+// failure classifies as a positive K8s infra abort (via
+// FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its sibling
+// branches keep running instead of being SIGTERMed by failFast -- and a sub-job
+// that saw only infra aborts (no genuine failure) resolves to UNSTABLE. When
+// false, every failure rethrows and the original bare-boolean fail-fast is fully
+// restored. Kept separate from params.enableFailFast so the scoped behavior can
+// be disabled pipeline-wide without turning fail-fast itself off. Only K8s-scoped
+// aborts are deferred today; SLURM-scoped aborts fall back to today's fail-fast
+// (see runBranchesWithInfraDefer).
+//
+// Overridable without a code change by setting the ENABLE_INFRA_SCOPED_FAILFAST
+// env var on the job. Env values are strings ("false" is truthy in Groovy), so
+// the override goes through toBoolean() rather than the bare elvis.
+ENABLE_INFRA_SCOPED_FAILFAST = env.ENABLE_INFRA_SCOPED_FAILFAST ? env.ENABLE_INFRA_SCOPED_FAILFAST.toBoolean() : true
+
 // Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the
 // 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap
 // or disable stage-level infra retries for resource-scarce hardware pools --
@@ -5359,6 +5375,61 @@ def buildStageConfigs(stageName, platform, testlist, testCount, gpuCount, nodeCo
     return configs
 }
 
+// Infra-scoped fail-fast (inner/branch layer). Runs `jobs` under `parallel` so a
+// branch whose post-retry failure is a positive K8s infra abort
+// (FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its siblings
+// keep running instead of being SIGTERMed by failFast. A genuine test/build
+// failure (or an unclassified one) is rethrown unchanged, so failFast stays fully
+// active for real failures; an interrupt (e.g. a sibling's own fail-fast SIGTERM)
+// is also rethrown and never swallowed. After the join, a sub-job that saw ONLY
+// infra aborts and no real failure resolves to UNSTABLE (coverage incomplete, not
+// a failure) so the parent layer (L0_MergeRequest.launchJob) can spare the healthy
+// sibling architecture; a mixed sub-job already threw on its real failure and is
+// FAILURE (currentBuild.result worst-of semantics won't downgrade it).
+//
+// Scope: classify() is scope-filtered, so this passes K8S -- the motivating
+// pod-scheduling abort (KubernetesClientTimeoutException) is K8S-scoped. SLURM-only
+// aborts do NOT match here and keep today's fail-fast; deferring those too means
+// threading each stage's scope (opts.slurmDispatcher) in -- a follow-up, not this
+// change. Gated on ENABLE_INFRA_SCOPED_FAILFAST; off restores today's behavior
+// exactly (plain failFast + parallel, no wrapping, no UNSTABLE).
+def runBranchesWithInfraDefer(Map jobs, boolean failFast) {
+    if (!ENABLE_INFRA_SCOPED_FAILFAST) {
+        jobs.failFast = failFast
+        parallel jobs
+        return
+    }
+    // CPS serializes parallel-branch continuations onto a single VM thread, so a
+    // plain list append from the catch blocks below is safe -- there is no
+    // JVM-level concurrency to guard against here.
+    def deferred = []
+    def wrapped = jobs.collectEntries { stageName, body ->
+        [(stageName), {
+            try {
+                body()
+            } catch (InterruptedException e) {
+                throw e
+            } catch (Exception e) {
+                if (FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S)) {
+                    deferred.add([stage: stageName])
+                    echo "[INFRA-DEFER] ${stageName}: K8s infra abort recorded; " +
+                         "siblings continue instead of fail-fast. ${e.toString()}"
+                    return
+                }
+                throw e
+            }
+        }]
+    }
+    wrapped.failFast = failFast
+    parallel wrapped
+    if (deferred) {
+        echo "[INFRA-DEFER] ${deferred.size()} stage(s) infra-incomplete " +
+             "(${deferred.collect { it.stage }.join(', ')}); marking result UNSTABLE " +
+             "(coverage incomplete, no genuine test failure)."
+        currentBuild.result = 'UNSTABLE'
+    }
+}
+
 def launchTestJobs(pipeline, testFilter, globalVars)
 {
     def versionOverride = globalVars[TRTLLM_VERSION_OVERRIDE] ?: ""
@@ -6527,31 +6598,27 @@ pipeline {
                                 echo "Skip multi-GPU testing. No test to run."
                             }
                             if (singleGpuJobs.size() > 0) {
-                                singleGpuJobs.failFast = params.enableFailFast
-                                parallel singleGpuJobs
+                                runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast)
                             } else {
                                 echo "Skip single-GPU testing. No test to run."
                             }
                         } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) {
                             echo "Only run multi-GPU tests."
                             if (dgxJobs.size() > 0) {
-                                dgxJobs.failFast = params.enableFailFast
-                                parallel dgxJobs
+                                runBranchesWithInfraDefer(dgxJobs, params.enableFailFast)
                             } else {
                                 error "Skip multi-GPU testing. No test to run."
                             }
                         } else {
                             if (singleGpuJobs.size() > 0) {
-                                singleGpuJobs.failFast = params.enableFailFast
-                                parallel singleGpuJobs
+                                runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast)
                             } else {
                                 echo "Skip single-GPU testing. No test to run."
                             }
 
                             if (dgxJobs.size() > 0) {
                                 stage(testPhase2StageName) {
-                                    dgxJobs.failFast = params.enableFailFast
-                                    parallel dgxJobs
+                                    runBranchesWithInfraDefer(dgxJobs, params.enableFailFast)
                                 }
                             }
                         }

jenkins/TensorRT_LLM_PLC.groovy

@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 @Library(['trtllm-jenkins-shared-lib@main']) _
 import groovy.json.JsonSlurper
 

jenkins/runPerfSanityTriage.groovy

@@ -1,3 +1,19 @@
+/*
+ * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 @Library(['trtllm-jenkins-shared-lib@main']) _
 
 DOCKER_IMAGE = "artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-25.10-py3-x86_64-ubuntu24.04-trt10.13.3.9-skip-tritondevel-202510291120-8621"