From 50f3973ff3d986c8fdeac82e4e354c2758a0b07e Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Wed, 12 Aug 2026 15:14:12 -0700 Subject: [PATCH 1/2] [None][infra] Retry SLURM agent online timeouts Signed-off-by: Maxim Gluhovskoi --- jenkins/L0_Test.groovy | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 8b8538486f1b..eac0e4a42692 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -1224,18 +1224,21 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, if (waitRc == 3) { checkSlurmJobActive() } + if (waitRc != 0) { + error "SLURM job ${slurmJobID} did not reach RUNNING during the queue wait. Terminating the job." + } // Phase 2: job is RUNNING; wait for the Jenkins agent to come online. isNodeOnline() // and Thread.sleep() emit no flow-nodes, so poll every 30s without bloating Blue // Ocean, and probe job status every ~3 min (every 6th iter) to fail fast if the - // job dies during bring-up. 120 * 30s = 1h. + // job dies during bring-up. 20 * 30s = 10 min. if (waitRc == 0) { // Job is RUNNING: stamp the walltime-budget origin for the // timeout duration fallback (within Phase 1's ~3min poll // granularity of the true RUNNING transition). jobRunningStartMs = System.currentTimeMillis() def onlineCounter = 0 - while (!CloudManager.isNodeOnline(nodeName) && onlineCounter < 120) { + while (!CloudManager.isNodeOnline(nodeName) && onlineCounter < 20) { Thread.sleep(30L * 1000L) if (onlineCounter % 6 == 0) { checkSlurmJobActive() @@ -1317,7 +1320,21 @@ def runLLMTestlistWithAgent(pipeline, platform, testList, config=VANILLA_CONFIG, } } } else { - error "The Slurm node does not come online in the waiting period. Terminating the job." + def setupLogPath = "/home/svc_tensorrt/slurm-logs/slurm-${slurmJobID}-${nodeName}.out" + try { + CloudManager.withSlurmFrontendFailover(pipeline, remotes) { logRemote -> + echoRemoteLogTail(pipeline, logRemote, setupLogPath) + } + } catch (InterruptedException e) { + throw e + } catch (Exception logEx) { + echo "Ignorable warning: could not retrieve ${setupLogPath}: ${logEx.message}" + } + throw new InfraFailure( + "SLURM agent ${nodeName} for job ${slurmJobID} did not come online within 10 minutes " + + "after the job started. Check SLURM logs at ${setupLogPath} on ${cluster.host}.", + null, InfraFailure.TRANSIENT, InfraFailure.SLURM, "" + ) } } } From e1a271beafce46d675bda10ac729df0caddbd321 Mon Sep 17 00:00:00 2001 From: Maxim Gluhovskoi Date: Wed, 12 Aug 2026 17:26:47 -0700 Subject: [PATCH 2/2] Handle interruptions when tailing SLURM logs Signed-off-by: Maxim Gluhovskoi --- jenkins/L0_Test.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index eac0e4a42692..1823c7129f72 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -282,6 +282,8 @@ def echoRemoteLogTail(def pipeline, Map remote, String remotePath, int lines = 2 numRetries: 1, )?.trim() pipeline.echo(tailOut ?: "") + } catch (InterruptedException e) { + throw e } catch (Exception tailEx) { pipeline.echo("Ignorable warning: could not tail ${remotePath} on ${remote.host}: ${tailEx.message}") }