diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 42903e069668..7ffd9ed01bfc 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1188,49 +1188,70 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, throw new Exception("Unsupported container runtime: ${cluster.containerRuntime}") } long executeStartMs = System.currentTimeMillis() - try { - executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations, retryContext) - } catch (InterruptedException e) { - throw e - } catch (Exception e) { - // Decide whether this failure is a SLURM timeout (the test ran to - // its partition walltime) rather than a transient infra blip. - // Measure elapsed from when the job was first observed RUNNING. - // Fall back to executeStartMs if the RUNNING stamp was never set. + // Reclassify a raw test-execution failure against the SLURM job's terminal + // state: a walltime kill is a UserFailure (not retryable), while a job still + // alive when the monitor lost contact is transient infra (retry on a fresh + // node). Kept as a closure so executeLLMTestOnSlurm can apply it INSIDE the + // task runner -- that way cacheErrorAndUploadResult suppresses this attempt's + // junit for the SAME typed failure the retry loop acts on. Deciding it only + // here (after junit already ran) is what left retried-and-passed agent stages + // UNSTABLE from an intermediate attempt's results. + // Classify each SLURM test-execution failure exactly once. The closure sets + // this when it runs (inside the task runner), so the outer catch reclassifies + // only failures that never reached it (image pull, node/agent bring-up) -- + // avoiding a second querySlurmJobState round trip that could reach a different + // verdict as elapsed time and the job's state move on. + boolean slurmFailureClassified = false + def classifySlurmFailure = { Throwable err -> + slurmFailureClassified = true + // Measure elapsed from when the job was first observed RUNNING; fall back + // to executeStartMs if the RUNNING stamp was never set. long timeoutBaselineMs = (jobRunningStartMs ?: executeStartMs) as long long elapsedMin = Math.floorDiv(System.currentTimeMillis() - timeoutBaselineMs, 60000L) Integer walltimeMin = (partition?.time ?: SlurmConfig.DEFAULT_TIMEOUT_SHORT) as Integer def slurmState = querySlurmJobState(pipeline, cluster, partition.clusterName, slurmJobID) if (slurmState == "TIMEOUT") { - throw new UserFailure( + return new UserFailure( "SLURM job ${slurmJobID} for ${stageName} ended in state TIMEOUT " + "(hit partition walltime ${walltimeMin}min); treating as a test timeout, not retrying. " + - "Original failure: ${e.message}", - e) + "Original failure: ${err.message}", + err) } if (slurmState == null && walltimeMin != null && elapsedMin >= (long)(SLURM_TIMEOUT_RETRY_FRACTION * walltimeMin)) { - throw new UserFailure( + return new UserFailure( "SLURM job ${slurmJobID} for ${stageName} ran ${elapsedMin}min " + "(>= ${(int)(SLURM_TIMEOUT_RETRY_FRACTION * 100)}% of the ${walltimeMin}min walltime) and " + "the SLURM controller was unreachable for an authoritative state; treating as a likely " + - "timeout, not retrying. Original failure: ${e.message}", - e) + "timeout, not retrying. Original failure: ${err.message}", + err) } if (isNonTerminalSlurmState(slurmState)) { - throw new InfraFailure( + return new InfraFailure( "SLURM job ${slurmJobID} for ${stageName} is still in non-terminal state ${slurmState} " + "(${elapsedMin}min of ${walltimeMin}min walltime); the monitor lost contact while the job was " + - "alive (transient infra), so this is not a test failure. Original failure: ${e.message}", - e, InfraFailure.TRANSIENT, InfraFailure.SLURM, "") + "alive (transient infra), so this is not a test failure. Original failure: ${err.message}", + err, InfraFailure.TRANSIENT, InfraFailure.SLURM, "") } echo "[INFRA-RETRY] ${stageName}: SLURM job ${slurmJobID} terminal state=${slurmState ?: 'unknown'}, " + "ran ${elapsedMin}min of ${walltimeMin}min walltime; deferring to failure classifier." + return err + } + try { + executeLLMTestOnSlurm(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, slurmRunner, postTag, useClusterDurations, retryContext, classifySlurmFailure) + } catch (InterruptedException e) { throw e + } catch (Exception e) { + // A test-execution failure was already labeled inside the task runner, so + // this attempt's junit suppression and the retry loop see the same typed + // failure -- just propagate it. Failures raised outside the task runner + // (image pull, node/agent bring-up, etc.) were never labeled, so classify + // them here on the agent. + throw (slurmFailureClassified || e instanceof TrtllmCiException ? e : classifySlurmFailure(e)) } } finally { // Resource cleanup must run even if SLURM metadata capture is interrupted. @@ -1254,12 +1275,26 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } } -def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", runner, String postTag="", boolean useClusterDurations=false, Map retryContext=null) +def executeLLMTestOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312", runner, String postTag="", boolean useClusterDurations=false, Map retryContext=null, Closure classifySlurmFailure=null) { runner { // TODO: refactor the finallyRunner to reuse within slurm or nonslurm job. cacheErrorAndUploadResult(stageName, { - runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, false, postTag, useClusterDurations) + try { + runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config, perfMode, stageName, splitId, splits, skipInstallWheel, cpver, false, postTag, useClusterDurations) + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + // Label the failure against the SLURM job's terminal state before + // cacheErrorAndUploadResult decides junit suppression, so a monitor- + // detected infra failure (e.g. slurm-job-still-running) suppresses this + // attempt's results and a passing retry stays green. Pipeline aborts + // (FlowInterruptedException) must not be relabeled. + if (classifySlurmFailure == null || e.getClass().name.contains("FlowInterruptedException")) { + throw e + } + throw classifySlurmFailure(e) + } }, { // If the execution test list is null, remove the test result xml sh """ @@ -2833,6 +2868,19 @@ def cacheErrorAndUploadResult(stageName, taskRunner, finallyRunner, noResultIfSu sh "STAGE_NAME=${stageName}" sh "STAGE_NAME=${stageName} && env | sort > ${stageName}/debug_env.txt" echo "Upload test results." + if (suppressTestReporting) { + // This attempt is superseded by a planned retry. Keep the tar for + // forensics, but move its result XMLs aside so the top-level Collect + // Test Result stage's junit('**/results*.xml') does not re-ingest a + // superseded attempt's results (e.g. a results-timeout.xml left by a + // monitor-cut still-running job) and flip the build UNSTABLE even + // though the stage passed on retry. junit() here was already gated. + sh """ + cd ${stageName} && for f in results*.xml; do + [ -e "\$f" ] && mv "\$f" "superseded-\$f" + done || true + """ + } sh "tar -czvf results-${stageName}${postTag}.tar.gz ${stageName}/" trtllm_utils.uploadArtifacts( "results-${stageName}${postTag}.tar.gz",