From 6790dc0184cd3ca8f319f6c03a14761fde4e39e7 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 13 Aug 2026 00:34:55 -0500 Subject: [PATCH 1/3] [TRTLLMINF-311][infra] Infra-scoped fail-fast: defer K8s infra aborts instead of cascading Today a K8s infra abort (e.g. KubernetesClientTimeoutException pod-launch timeout) in one architecture's test sub-job trips fail-fast and SIGTERMs the healthy sibling architecture, wasting its GPU work. This scopes fail-fast to genuine failures. Inner layer (L0_Test.groovy): runBranchesWithInfraDefer wraps each parallel branch so a post-retry failure classified as a positive K8s infra abort (FailureClassifier.isDeferrableInfra) is recorded and swallowed -- siblings keep running -- while a real test/build failure (or interrupt) is rethrown unchanged, so fail-fast stays active for real failures. A sub-job that saw only infra aborts resolves to UNSTABLE (coverage incomplete, not a failure). Gated on ENABLE_INFRA_SCOPED_FAILFAST (kill switch); SLURM-scoped aborts keep today's fail-fast for now. Parent layer (L0_MergeRequest.groovy): launchJob treats a downstream UNSTABLE result as non-fatal (do not throw -> do not cancel the sibling arch), marking the build UNSTABLE. The per-arch multi-GPU gates skip (pre-merge) or keep running (post-merge) on an infra-incomplete single-GPU run without escalating to FAILURE. Requires trtllm-jenkins-shared-lib isDeferrableInfra (separate MR). Signed-off-by: Brian Nguyen --- jenkins/L0_MergeRequest.groovy | 59 ++++++++++++++++++++++++- jenkins/L0_Test.groovy | 79 ++++++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index de9da034765c..b386b1aca564 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -1595,6 +1595,19 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor def logger = new Logger(pipeline) def (jenkinsURL, buildStatus) = JobBuilder.build(pipeline, logger, jobName, parameters, 1, false) + // Infra-scoped fail-fast (parent half). A downstream sub-job returns UNSTABLE + // when it saw only infra aborts and no genuine test/build failure (see + // runBranchesWithInfraDefer in L0_Test.groovy). That is incomplete coverage, + // not a failure: throwing here is exactly what trips the per-arch failFast and + // cancels the healthy sibling architecture, so do NOT throw. Mark the build + // UNSTABLE (visible + re-runnable) and let the sibling finish. FAILURE and + // ABORTED still throw below, so real failures fail-fast exactly as before. + if (buildStatus == "UNSTABLE") { + catchError(buildResult: 'UNSTABLE', stageResult: 'UNSTABLE') { + error "Downstream job ${jobName} is infra-incomplete (UNSTABLE); sibling not cancelled" + } + return buildStatus + } if (buildStatus != "SUCCESS") { error "Downstream job did not succeed" } @@ -1643,6 +1656,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false + def singleGpuInfraIncomplete = false stage(testStageName) { if (X86_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "x86_64 test job is skipped due to Jenkins configuration" @@ -1657,7 +1671,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + // launchJob returns UNSTABLE (without throwing) when the single-GPU + // sub-job was infra-incomplete: only infra aborts, no real failure. + def singleGpuStatus = launchJob(pipeline, "L0_Test-x86_64-Single-GPU", false, enableFailFast, globalVars, "x86_64", additionalParameters) + singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1699,6 +1716,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a + // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than + // spend scarce multi-GPU resource on a partially-unverified premise -- + // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE + // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps + // running multi-GPU for max signal, mirroring the single-GPU-failed policy. + if (singleGpuInfraIncomplete) { + if (env.JOB_NAME ==~ /.*PostMerge.*/) { + echo "In the official post-merge pipeline, x86_64 single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running." + } else { + stage("[Test-x86_64-Multi-GPU] Skipped - single-GPU infra-incomplete") { + echo "x86_64 single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE." + } + return + } + } + // Label gate: check before entering the Remote Run stage so a // missing/unauthorized label shows as "Blocked" (not a Remote Run // failure) and does not trigger fail-fast. @@ -1774,6 +1808,7 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false + def singleGpuInfraIncomplete = false stage(testStageName) { if (SBSA_TEST_CHOICE == STAGE_CHOICE_SKIP) { echo "SBSA test job is skipped due to Jenkins configuration" @@ -1787,7 +1822,10 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], ] - launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + // launchJob returns UNSTABLE (without throwing) when the single-GPU + // sub-job was infra-incomplete: only infra aborts, no real failure. + def singleGpuStatus = launchJob(pipeline, "L0_Test-SBSA-Single-GPU", false, enableFailFast, globalVars, "SBSA", additionalParameters) + singleGpuInfraIncomplete = (singleGpuStatus == "UNSTABLE") } catch (InterruptedException e) { throw e } catch (Exception e) { @@ -1830,6 +1868,23 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } } + // Single-GPU was infra-incomplete (UNSTABLE): its coverage is a + // prerequisite for multi-GPU. In pre-merge, skip multi-GPU rather than + // spend scarce multi-GPU resource on a partially-unverified premise -- + // the single-GPU sub-job should be re-run first. Keep the build UNSTABLE + // (already set by launchJob); do NOT escalate to FAILURE. Post-merge keeps + // running multi-GPU for max signal, mirroring the single-GPU-failed policy. + if (singleGpuInfraIncomplete) { + if (env.JOB_NAME ==~ /.*PostMerge.*/) { + echo "In the official post-merge pipeline, SBSA single-GPU test was infra-incomplete (UNSTABLE); multi-GPU test is still kept running." + } else { + stage("[Test-SBSA-Multi-GPU] Skipped - single-GPU infra-incomplete") { + echo "SBSA single-GPU was infra-incomplete (UNSTABLE); skipping multi-GPU (premise not fully validated). Build stays UNSTABLE." + } + return + } + } + def sbsaLabelBlock = requireMultiGpuApprovalLabel(pipeline, globalVars, "SBSA") if (sbsaLabelBlock) { stage("[Test-SBSA-Multi-GPU] Blocked") { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index adcc4c847e97..a8c1ca8b16c0 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -158,6 +158,18 @@ SLURM_INFRA_RETRY_MAX = 1 // to avoid nesting with the inner SLURM retry. K8S_INFRA_RETRY_MAX = 1 +// Infra-scoped fail-fast master switch. When true, a branch whose post-retry +// failure classifies as a positive K8s infra abort (via +// FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its sibling +// branches keep running instead of being SIGTERMed by failFast -- and a sub-job +// that saw only infra aborts (no genuine failure) resolves to UNSTABLE. When +// false, every failure rethrows and the original bare-boolean fail-fast is fully +// restored. Kept separate from params.enableFailFast so the scoped behavior can +// be disabled pipeline-wide without turning fail-fast itself off. Only K8s-scoped +// aborts are deferred today; SLURM-scoped aborts fall back to today's fail-fast +// (see runBranchesWithInfraDefer). +ENABLE_INFRA_SCOPED_FAILFAST = true + // Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the // 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap // or disable stage-level infra retries for resource-scarce hardware pools -- @@ -5359,6 +5371,61 @@ def buildStageConfigs(stageName, platform, testlist, testCount, gpuCount, nodeCo return configs } +// Infra-scoped fail-fast (inner/branch layer). Runs `jobs` under `parallel` so a +// branch whose post-retry failure is a positive K8s infra abort +// (FailureClassifier.isDeferrableInfra) is recorded and swallowed -- its siblings +// keep running instead of being SIGTERMed by failFast. A genuine test/build +// failure (or an unclassified one) is rethrown unchanged, so failFast stays fully +// active for real failures; an interrupt (e.g. a sibling's own fail-fast SIGTERM) +// is also rethrown and never swallowed. After the join, a sub-job that saw ONLY +// infra aborts and no real failure resolves to UNSTABLE (coverage incomplete, not +// a failure) so the parent layer (L0_MergeRequest.launchJob) can spare the healthy +// sibling architecture; a mixed sub-job already threw on its real failure and is +// FAILURE (currentBuild.result worst-of semantics won't downgrade it). +// +// Scope: classify() is scope-filtered, so this passes K8S -- the motivating +// pod-scheduling abort (KubernetesClientTimeoutException) is K8S-scoped. SLURM-only +// aborts do NOT match here and keep today's fail-fast; deferring those too means +// threading each stage's scope (opts.slurmDispatcher) in -- a follow-up, not this +// change. Gated on ENABLE_INFRA_SCOPED_FAILFAST; off restores today's behavior +// exactly (plain failFast + parallel, no wrapping, no UNSTABLE). +def runBranchesWithInfraDefer(Map jobs, boolean failFast) { + if (!ENABLE_INFRA_SCOPED_FAILFAST) { + jobs.failFast = failFast + parallel jobs + return + } + // CPS serializes parallel-branch continuations onto a single VM thread, so a + // plain list append from the catch blocks below is safe -- there is no + // JVM-level concurrency to guard against here. + def deferred = [] + def wrapped = jobs.collectEntries { stageName, body -> + [(stageName), { + try { + body() + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + if (FailureClassifier.isDeferrableInfra(e, InfraFailure.K8S)) { + deferred.add([stage: stageName]) + echo "[INFRA-DEFER] ${stageName}: K8s infra abort recorded; " + + "siblings continue instead of fail-fast. ${e.toString()}" + return + } + throw e + } + }] + } + wrapped.failFast = failFast + parallel wrapped + if (deferred) { + echo "[INFRA-DEFER] ${deferred.size()} stage(s) infra-incomplete " + + "(${deferred.collect { it.stage }.join(', ')}); marking result UNSTABLE " + + "(coverage incomplete, no genuine test failure)." + currentBuild.result = 'UNSTABLE' + } +} + def launchTestJobs(pipeline, testFilter, globalVars) { def versionOverride = globalVars[TRTLLM_VERSION_OVERRIDE] ?: "" @@ -6527,31 +6594,27 @@ pipeline { echo "Skip multi-GPU testing. No test to run." } if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs + runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast) } else { echo "Skip single-GPU testing. No test to run." } } else if (env.JOB_NAME ==~ /.*Multi-GPU.*/) { echo "Only run multi-GPU tests." if (dgxJobs.size() > 0) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs + runBranchesWithInfraDefer(dgxJobs, params.enableFailFast) } else { error "Skip multi-GPU testing. No test to run." } } else { if (singleGpuJobs.size() > 0) { - singleGpuJobs.failFast = params.enableFailFast - parallel singleGpuJobs + runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast) } else { echo "Skip single-GPU testing. No test to run." } if (dgxJobs.size() > 0) { stage(testPhase2StageName) { - dgxJobs.failFast = params.enableFailFast - parallel dgxJobs + runBranchesWithInfraDefer(dgxJobs, params.enableFailFast) } } } From 8f74c492d81c66f9440eccda1a57104e97268bab Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 13 Aug 2026 05:57:58 -0700 Subject: [PATCH 2/3] Address trivial review comments Signed-off-by: Brian Nguyen --- jenkins/TensorRT_LLM_PLC.groovy | 16 ++++++++++++++++ jenkins/runPerfSanityTriage.groovy | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/jenkins/TensorRT_LLM_PLC.groovy b/jenkins/TensorRT_LLM_PLC.groovy index c4162ee846a0..a6714f3c5c0b 100644 --- a/jenkins/TensorRT_LLM_PLC.groovy +++ b/jenkins/TensorRT_LLM_PLC.groovy @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + @Library(['trtllm-jenkins-shared-lib@main']) _ import groovy.json.JsonSlurper diff --git a/jenkins/runPerfSanityTriage.groovy b/jenkins/runPerfSanityTriage.groovy index a9af3aeccc4a..4a0890ffe128 100644 --- a/jenkins/runPerfSanityTriage.groovy +++ b/jenkins/runPerfSanityTriage.groovy @@ -1,3 +1,19 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + @Library(['trtllm-jenkins-shared-lib@main']) _ DOCKER_IMAGE = "artifactory.nvidia.com/sw-tensorrt-llm-docker-local/tensorrt-llm:pytorch-25.10-py3-x86_64-ubuntu24.04-trt10.13.3.9-skip-tritondevel-202510291120-8621" From 634dc2935800faa5ad3344ad21fbd166966b68eb Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 13 Aug 2026 17:28:39 -0500 Subject: [PATCH 3/3] Allow ENABLE_INFRA_SCOPED_FAILFAST override via env var Signed-off-by: Brian Nguyen --- jenkins/L0_Test.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a8c1ca8b16c0..1b311965dcd4 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -168,7 +168,11 @@ K8S_INFRA_RETRY_MAX = 1 // be disabled pipeline-wide without turning fail-fast itself off. Only K8s-scoped // aborts are deferred today; SLURM-scoped aborts fall back to today's fail-fast // (see runBranchesWithInfraDefer). -ENABLE_INFRA_SCOPED_FAILFAST = true +// +// Overridable without a code change by setting the ENABLE_INFRA_SCOPED_FAILFAST +// env var on the job. Env values are strings ("false" is truthy in Groovy), so +// the override goes through toBoolean() rather than the bare elvis. +ENABLE_INFRA_SCOPED_FAILFAST = env.ENABLE_INFRA_SCOPED_FAILFAST ? env.ENABLE_INFRA_SCOPED_FAILFAST.toBoolean() : true // Per-stage override of the above: set `infraRetryMax` in a stage's opts map (the // 3rd element of its parallel-jobs config tuple, alongside singleAttempt) to cap