Skip to content

[TRTLLMINF-161][infra] Add simplified infrastructure dry-run pipeline #17813

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mzweilz merged 34 commits into from
Aug 25, 2026
Merged

[TRTLLMINF-161][infra] Add simplified infrastructure dry-run pipeline #17813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e4241e6
ci: add infrastructure dry-run benchmark
mzweilz Jul 29, 2026
886ac48
ci: integrate infrastructure dry run into L0 tests
mzweilz Jul 29, 2026
2eedc15
ci: orchestrate infrastructure dry-run helpers
mzweilz Jul 30, 2026
d64e247
[TRTLLMINF-161] ci: harden infrastructure dry run
mzweilz Aug 3, 2026
841393c
[TRTLLMINF-161] ci: collect dry-run test results
mzweilz Aug 3, 2026
f955283
[TRTLLMINF-161] ci: cover CPU and docs dry runs
mzweilz Aug 4, 2026
8c0ddea
TRTLLMINF-161 Simplify dry-run test orchestration
mzweilz Aug 7, 2026
a1a17d3
TRTLLMINF-161 Make artifact download idempotent
mzweilz Aug 10, 2026
bba20a1
ci: run dry-run benchmark through standard pytest flow
mzweilz Aug 10, 2026
96de918
ci: collect dry-run benchmark during test-list validation
mzweilz Aug 10, 2026
1434369
ci: make dry-run spawn workers importable
mzweilz Aug 11, 2026
145febc
ci: simplify infrastructure dry run
mzweilz Aug 11, 2026
70f0697
ci: fix dry-run docs pytest options
mzweilz Aug 11, 2026
2522e7e
ci: preserve capture for dry-run docs upload
mzweilz Aug 11, 2026
7465728
ci: keep agent-flow in infrastructure dry-run path
mzweilz Aug 13, 2026
1b8fd23
ci: prepare agent-flow dry-run test environment
mzweilz Aug 13, 2026
9ca2bf2
ci: isolate dry-run conftest dependencies
mzweilz Aug 14, 2026
9bda5f6
ci: install mako for AgentFlow dry run
mzweilz Aug 14, 2026
499d77a
ci: format dry-run pipeline test
mzweilz Aug 14, 2026
2db0641
ci: keep dry-run import guard during collection
mzweilz Aug 14, 2026
a6842a2
ci: install unused fixtures plugin for dry run
mzweilz Aug 14, 2026
1eba03e
ci: use CPU stage identity for dry docs
mzweilz Aug 14, 2026
54a778b
Limit dry-run pytest collection to synthetic targets
mzweilz Aug 14, 2026
ffba2ed
Add dry-run S3 upload dependency for AgentFlow
mzweilz Aug 15, 2026
97226d0
ci: preserve normal paths around infrastructure dry run
mzweilz Aug 17, 2026
65a6489
ci: validate dry-run pytest target
mzweilz Aug 18, 2026
99dde8c
ci: annotate dry-run pipeline test helpers
mzweilz Aug 18, 2026
25484b0
ci: address dry-run review feedback
mzweilz Aug 18, 2026
dcbcc04
tests: annotate infra dry-run pipeline cases
mzweilz Aug 19, 2026
e561753
ci: merge duplicate dry-run setup branches
mzweilz Aug 19, 2026
7026694
ci: preserve Slurm agent dry-run coverage
mzweilz Aug 19, 2026
963a54a
ci: simplify infrastructure dry-run stage coverage
mzweilz Aug 21, 2026
95a2fdc
ci: skip docs junit when no results are expected
mzweilz Aug 21, 2026
0c242d7
ci: skip MR diff lookups during dry run
mzweilz Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 81 additions & 22 deletions jenkins/L0_MergeRequest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,8 @@ def CBTS_RESULT = "cbts_result"
def CBTS_COVERAGE = "cbts_coverage"
@Field
def DISABLE_CBTS = "disable_cbts"
@Field
def INFRA_DRY_RUN = "infra_dry_run"
// Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1.
@Field
def ENABLE_CBTS_COVERAGE = true
Expand All @@ -178,6 +180,7 @@ def testFilter = [
(CBTS_RESULT): null,
(CBTS_COVERAGE): false,
(DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false),
(INFRA_DRY_RUN): (params.InfraDryRun?.toString()?.toBoolean() ?: false),
]

String reuseBuild = gitlabParamsFromBot.get('reuse_build', null)
Expand Down Expand Up @@ -213,6 +216,7 @@ if (runMode == "nightly_release") {
// GenPostMergeBuilds pipelines do not update GitLab status.
boolean enableUpdateGitlabStatus =
!GEN_POST_MERGE_BUILDS_ONLY &&
!testFilter[INFRA_DRY_RUN] &&
!testFilter[ENABLE_SKIP_TEST] &&
!testFilter[ONLY_MULTI_GPU_TEST] &&
!testFilter[DISABLE_MULTI_GPU_TEST] &&
Expand Down Expand Up @@ -353,16 +357,23 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars)
}
echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}."
echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}."
if (!GEN_POST_MERGE_BUILDS_ONLY) {
if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'running', GITLAB_PROJECT_ID, env.gitlabCommit)
}
testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars)
testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars)
testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars)
testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars)
// Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag.
// Coverage runs only on the official post-merge pipeline.
testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/)
if (testFilter[INFRA_DRY_RUN]) {
pipeline.echo("Changed-file analysis is skipped for the infrastructure dry run.")
testFilter[(MULTI_GPU_FILE_CHANGED)] = false
testFilter[(ONLY_ONE_GROUP_CHANGED)] = ""
testFilter[(AUTO_TRIGGER_TAG_LIST)] = []
pipeline.echo("CBTS is skipped for the infrastructure dry run.")
} else {
testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars)
testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars)
testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars)
testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars)
// Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag.
testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/)
}
Comment thread
mzweilz marked this conversation as resolved.
pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}")
testFilter[(OSS_COMPLIANCE_FILE_CHANGED)] = getOssComplianceFileChanged(pipeline, globalVars)
getContainerURIs().each { k, v ->
Expand Down Expand Up @@ -463,7 +474,11 @@ def preparation(pipeline, testFilter, globalVars)
setupPipelineEnvironment(pipeline, testFilter, globalVars)
}
stage("Merge Test Waive List") {
mergeWaiveList(pipeline, globalVars)
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Merge Test Waive List for the infrastructure dry run."
} else {
mergeWaiveList(pipeline, globalVars)
}
}
})
}
Expand Down Expand Up @@ -717,7 +732,8 @@ def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) {

def getMergeRequestChangedFileList(pipeline, globalVars) {
def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/)
if (env.alternativeTRT ||
if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) ||
env.alternativeTRT ||
isOfficialPostMergeJob ||
runMode == "nightly_release") {
pipeline.echo("Force set changed file list to empty list.")
Expand Down Expand Up @@ -753,7 +769,8 @@ def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) {
// Note: This function intentionally propagates exceptions to the caller.
// If there is an error to get the changed file diff, skip merging the waive list.
def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/)
if (env.alternativeTRT ||
if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) ||
env.alternativeTRT ||
isOfficialPostMergeJob ||
runMode == "nightly_release") {
pipeline.echo("Force set changed file diff to empty string.")
Expand Down Expand Up @@ -1643,7 +1660,8 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor
]
}

if (env.testPhase2StageName) {
// Preserve an explicit empty phase override from the dry-run helper.
if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) {
parameters += [
'testPhase2StageName': env.testPhase2StageName,
]
Expand Down Expand Up @@ -1688,12 +1706,28 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor
return buildStatus
}

def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, imageParameters)
{
String testFilterJson = writeJSON returnText: true, json: testFilter
def additionalParameters = [
'testFilter': testFilterJson,
// Keep all synthetic GPU-count stages in this helper.
'testPhase2StageName': '',
Comment thread
mzweilz marked this conversation as resolved.
] + imageParameters
stage("[Test-${arch}-Single-GPU] Remote Run") {
launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters)
}
}

def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
{
stages = [
"Release-Check": {
script {
if (GEN_POST_MERGE_BUILDS_ONLY) {
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Release-Check for the infrastructure dry run."
return
} else if (GEN_POST_MERGE_BUILDS_ONLY) {
echo "Skipping Release-Check (GenPostMergeBuilds mode: builds only)"
return
}
Expand Down Expand Up @@ -1781,6 +1815,15 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
echo "Skipping x86_64 tests (PLC container scanning)"
return
}
if (testFilter[INFRA_DRY_RUN]) {
def imageParameters = [
'dockerImage': globalVars["LLM_DOCKER_IMAGE"],
'wheelDockerImagePy310': globalVars["LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE"],
'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"],
]
launchInfraDryRunTestJob(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters)
return
}

testStageName = "[Test-x86_64-Single-GPU] Remote Run"
def singleGpuTestFailed = false
Expand Down Expand Up @@ -1933,6 +1976,14 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
echo "Skipping SBSA tests (PLC container scanning)"
return
}
if (testFilter[INFRA_DRY_RUN]) {
def imageParameters = [
"dockerImage": globalVars["LLM_SBSA_DOCKER_IMAGE"],
'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"],
]
launchInfraDryRunTestJob(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters)
return
}

testStageName = "[Test-SBSA-Single-GPU] Remote Run"
def singleGpuTestFailed = false
Expand Down Expand Up @@ -2239,7 +2290,9 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars)
}
}]}

parallelJobs.failFast = enableFailFast
// Preserve both architecture tracks during dry acceptance.
def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast
parallelJobs.failFast = effectiveFailFast
pipeline.parallel parallelJobs
}

Expand All @@ -2263,24 +2316,26 @@ pipeline {
post {
unsuccessful {
script {
if (!GEN_POST_MERGE_BUILDS_ONLY) {
if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "failed", GITLAB_PROJECT_ID, env.gitlabCommit)
}
}
}
success {
script {
if (enableUpdateGitlabStatus) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit)
} else if (!GEN_POST_MERGE_BUILDS_ONLY) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit)
trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit)
if (!testFilter[INFRA_DRY_RUN]) {
if (enableUpdateGitlabStatus) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit)
} else if (!GEN_POST_MERGE_BUILDS_ONLY) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit)
trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit)
}
}
}
}
aborted {
script {
if (!GEN_POST_MERGE_BUILDS_ONLY) {
if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) {
trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'canceled', GITLAB_PROJECT_ID, env.gitlabCommit)
}
}
Expand Down Expand Up @@ -2333,7 +2388,11 @@ pipeline {
if (isReleaseCheckMode) {
stage("Release-Check") {
script {
launchReleaseCheck(this, globalVars)
if (testFilter[INFRA_DRY_RUN]) {
echo "Skipping Release-Check for the infrastructure dry run."
} else {
launchReleaseCheck(this, globalVars)
}
}
}
} else {
Expand Down
Loading
Loading