diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 133931393234..59fe9f0f3000 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -152,6 +152,8 @@ def CBTS_RESULT = "cbts_result" def CBTS_COVERAGE = "cbts_coverage" @Field def DISABLE_CBTS = "disable_cbts" +@Field +def INFRA_DRY_RUN = "infra_dry_run" // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true @@ -178,6 +180,7 @@ def testFilter = [ (CBTS_RESULT): null, (CBTS_COVERAGE): false, (DISABLE_CBTS): gitlabParamsFromBot.get((DISABLE_CBTS), false), + (INFRA_DRY_RUN): (params.InfraDryRun?.toString()?.toBoolean() ?: false), ] String reuseBuild = gitlabParamsFromBot.get('reuse_build', null) @@ -213,6 +216,7 @@ if (runMode == "nightly_release") { // GenPostMergeBuilds pipelines do not update GitLab status. boolean enableUpdateGitlabStatus = !GEN_POST_MERGE_BUILDS_ONLY && + !testFilter[INFRA_DRY_RUN] && !testFilter[ENABLE_SKIP_TEST] && !testFilter[ONLY_MULTI_GPU_TEST] && !testFilter[DISABLE_MULTI_GPU_TEST] && @@ -353,16 +357,23 @@ def setupPipelineEnvironment(pipeline, testFilter, globalVars) } echo "Env.gitlabMergeRequestLastCommit: ${env.gitlabMergeRequestLastCommit}." echo "Freeze GitLab commit. Branch: ${env.gitlabBranch}. Commit: ${env.gitlabCommit}." - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'running', GITLAB_PROJECT_ID, env.gitlabCommit) } - testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) - testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) - testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) - testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) - // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. - // Coverage runs only on the official post-merge pipeline. - testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) + if (testFilter[INFRA_DRY_RUN]) { + pipeline.echo("Changed-file analysis is skipped for the infrastructure dry run.") + testFilter[(MULTI_GPU_FILE_CHANGED)] = false + testFilter[(ONLY_ONE_GROUP_CHANGED)] = "" + testFilter[(AUTO_TRIGGER_TAG_LIST)] = [] + pipeline.echo("CBTS is skipped for the infrastructure dry run.") + } else { + testFilter[(MULTI_GPU_FILE_CHANGED)] = getMultiGpuFileChanged(pipeline, testFilter, globalVars) + testFilter[(ONLY_ONE_GROUP_CHANGED)] = getOnlyOneGroupChanged(pipeline, testFilter, globalVars) + testFilter[(AUTO_TRIGGER_TAG_LIST)] = getAutoTriggerTagList(pipeline, testFilter, globalVars) + testFilter[(CBTS_RESULT)] = getCbtsResult(pipeline, testFilter, globalVars) + // Decide CBTS coverage eligibility here so L0_Test only consumes the propagated flag. + testFilter[(CBTS_COVERAGE)] = ENABLE_CBTS_COVERAGE && (env.JOB_NAME ==~ /.*PostMerge.*/) + } pipeline.echo("CBTS coverage eligible: ${testFilter[(CBTS_COVERAGE)]}") testFilter[(OSS_COMPLIANCE_FILE_CHANGED)] = getOssComplianceFileChanged(pipeline, globalVars) getContainerURIs().each { k, v -> @@ -463,7 +474,11 @@ def preparation(pipeline, testFilter, globalVars) setupPipelineEnvironment(pipeline, testFilter, globalVars) } stage("Merge Test Waive List") { - mergeWaiveList(pipeline, globalVars) + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Merge Test Waive List for the infrastructure dry run." + } else { + mergeWaiveList(pipeline, globalVars) + } } }) } @@ -717,7 +732,8 @@ def requireMultiGpuApprovalLabel(pipeline, globalVars, String arch) { def getMergeRequestChangedFileList(pipeline, globalVars) { def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || + if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || + env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file list to empty list.") @@ -753,7 +769,8 @@ def getMergeRequestOneFileChanges(pipeline, globalVars, filePath) { // Note: This function intentionally propagates exceptions to the caller. // If there is an error to get the changed file diff, skip merging the waive list. def isOfficialPostMergeJob = (env.JOB_NAME ==~ /.*PostMerge.*/) - if (env.alternativeTRT || + if ((params.InfraDryRun?.toString()?.toBoolean() ?: false) || + env.alternativeTRT || isOfficialPostMergeJob || runMode == "nightly_release") { pipeline.echo("Force set changed file diff to empty string.") @@ -1643,7 +1660,8 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor ] } - if (env.testPhase2StageName) { + // Preserve an explicit empty phase override from the dry-run helper. + if (!additionalParameters.containsKey('testPhase2StageName') && env.testPhase2StageName) { parameters += [ 'testPhase2StageName': env.testPhase2StageName, ] @@ -1688,12 +1706,28 @@ def launchJob(pipeline, jobName, reuseBuild, enableFailFast, globalVars, platfor return buildStatus } +def launchInfraDryRunTestJob(pipeline, arch, testFilter, globalVars, platform, imageParameters) +{ + String testFilterJson = writeJSON returnText: true, json: testFilter + def additionalParameters = [ + 'testFilter': testFilterJson, + // Keep all synthetic GPU-count stages in this helper. + 'testPhase2StageName': '', + ] + imageParameters + stage("[Test-${arch}-Single-GPU] Remote Run") { + launchJob(pipeline, "L0_Test-${arch}-Single-GPU", false, false, globalVars, platform, additionalParameters) + } +} + def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) { stages = [ "Release-Check": { script { - if (GEN_POST_MERGE_BUILDS_ONLY) { + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Release-Check for the infrastructure dry run." + return + } else if (GEN_POST_MERGE_BUILDS_ONLY) { echo "Skipping Release-Check (GenPostMergeBuilds mode: builds only)" return } @@ -1781,6 +1815,15 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping x86_64 tests (PLC container scanning)" return } + if (testFilter[INFRA_DRY_RUN]) { + def imageParameters = [ + 'dockerImage': globalVars["LLM_DOCKER_IMAGE"], + 'wheelDockerImagePy310': globalVars["LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE"], + 'wheelDockerImagePy312': globalVars["LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE"], + ] + launchInfraDryRunTestJob(pipeline, "x86_64", testFilter, globalVars, "x86_64", imageParameters) + return + } testStageName = "[Test-x86_64-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -1933,6 +1976,14 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) echo "Skipping SBSA tests (PLC container scanning)" return } + if (testFilter[INFRA_DRY_RUN]) { + def imageParameters = [ + "dockerImage": globalVars["LLM_SBSA_DOCKER_IMAGE"], + 'wheelDockerImage': globalVars["LLM_SBSA_WHEEL_DOCKER_IMAGE"], + ] + launchInfraDryRunTestJob(pipeline, "SBSA", testFilter, globalVars, "SBSA", imageParameters) + return + } testStageName = "[Test-SBSA-Single-GPU] Remote Run" def singleGpuTestFailed = false @@ -2239,7 +2290,9 @@ def launchStages(pipeline, reuseBuild, testFilter, enableFailFast, globalVars) } }]} - parallelJobs.failFast = enableFailFast + // Preserve both architecture tracks during dry acceptance. + def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast + parallelJobs.failFast = effectiveFailFast pipeline.parallel parallelJobs } @@ -2263,24 +2316,26 @@ pipeline { post { unsuccessful { script { - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "failed", GITLAB_PROJECT_ID, env.gitlabCommit) } } } success { script { - if (enableUpdateGitlabStatus) { - trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit) - } else if (!GEN_POST_MERGE_BUILDS_ONLY) { - trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit) - trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit) + if (!testFilter[INFRA_DRY_RUN]) { + if (enableUpdateGitlabStatus) { + trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "success", GITLAB_PROJECT_ID, env.gitlabCommit) + } else if (!GEN_POST_MERGE_BUILDS_ONLY) { + trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, "canceled", GITLAB_PROJECT_ID, env.gitlabCommit) + trtllm_utils.updateGitlabStatus("Custom Jenkins build", "success", GITLAB_PROJECT_ID, env.gitlabCommit) + } } } } aborted { script { - if (!GEN_POST_MERGE_BUILDS_ONLY) { + if (!GEN_POST_MERGE_BUILDS_ONLY && !testFilter[INFRA_DRY_RUN]) { trtllm_utils.updateGitlabStatus(BUILD_STATUS_NAME, 'canceled', GITLAB_PROJECT_ID, env.gitlabCommit) } } @@ -2333,7 +2388,11 @@ pipeline { if (isReleaseCheckMode) { stage("Release-Check") { script { - launchReleaseCheck(this, globalVars) + if (testFilter[INFRA_DRY_RUN]) { + echo "Skipping Release-Check for the infrastructure dry run." + } else { + launchReleaseCheck(this, globalVars) + } } } } else { diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 321cb817838a..64522b27e634 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -102,6 +102,9 @@ def LLVM_CONFIG = "LLVM" @Field def LINUX_AARCH64_CONFIG = "linux_aarch64" +@Field +def INFRA_DRY_RUN_TEST_CONTEXT = "infra_dry_run" + @Field def BUILD_CONFIGS = [ // Vanilla TARNAME is used for packaging in runLLMPackage @@ -228,6 +231,10 @@ COMMON_SSH_OPTIONS = Utils.DEFAULT_CUSTOM_SSH_OPTIONS // Per-stage CBTS coverage exclusions applied on top of the upstream eligibility decision. CBTS_EXCLUDE_STAGES = [] as Set +def isInfraDryRun() { + return testFilter[(INFRA_DRY_RUN)] ?: false +} + def isCbtsStage(String stageName) { // Pipeline-level eligibility (post-merge gate + kill switch) is decided in L0_MergeRequest.groovy and propagated via testFilter. if (!(testFilter[(CBTS_COVERAGE)] ?: false)) { @@ -562,6 +569,24 @@ def runIsolatedTests(preprocessedLists, testCmdLine, llmSrc, stageName) { return rerunFailed // Return the updated value } +def getInfraDryRunPytestTargets(testListPath) { + if (!isInfraDryRun()) { + return [] + } + + // --test-list filters after collection, so also pass the exact rendered + // nodeid positionally to avoid importing unrelated product tests. + def targets = readFile(file: testListPath).readLines() + .collect { it.trim().split(/\s+/, 2)[0] } + .findAll { it.contains("::") } + def expectedTarget = + "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" + if (targets != [expectedTarget]) { + error "Unexpected pytest targets in infrastructure dry-run list ${testListPath}: ${targets}" + } + return targets +} + def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, durationsPath="") { // Preprocess testDBList to extract ISOLATION markers echo "Preprocessing testDBList to extract ISOLATION markers..." @@ -634,6 +659,7 @@ def processShardTestList(llmSrc, testDBList, splitId, splits, perfMode=false, du if (durationsPath) { testListCmd += ["--durations-path ${durationsPath}"] } + testListCmd += getInfraDryRunPytestTargets(cleanedTestDBList) try { // First execute the pytest command and check if it succeeds @@ -1697,6 +1723,13 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}" def disaggMultiNodeMode = stageName.contains("Disagg-PerfSanity") def aggMultiNodeMode = !disaggMultiNodeMode && nodeCount > 1 && stageName.contains("PerfSanity") + def infraDryRun = isInfraDryRun() + if (infraDryRun) { + testList = INFRA_DRY_RUN_TEST_CONTEXT + splitId = 1 + splits = 1 + perfMode = false + } Utils.exec(pipeline, script: "env | sort && pwd && ls -alh") @@ -1727,6 +1760,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG def scriptBashUtilsPathNode = "${jobWorkspace}/${jobUID}-bash_utils.sh" def testListPathNode = "${jobWorkspace}/${testList}.txt" def waivesListPathNode = "${jobWorkspace}/waives.txt" + def waivesListPathLocal = infraDryRun + ? "${llmPath}/infra_dry_run_waives.txt" + : "${llmSrcLocal}/tests/integration/test_lists/waives.txt" def slurmJobLogPath = "${jobWorkspace}/job-output.log" def scriptLaunchPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch.sh") def scriptLaunchPathNode = "${jobWorkspace}/${jobUID}-slurm_launch.sh" @@ -1801,18 +1837,22 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG script: Utils.sshUserCmd(remote, "\"mv -f ${testListPathNode}.tmp ${testListPathNode}\"") ) - // Download and Merge waives.txt - mergeWaivesTxt(pipeline, llmSrcLocal, stageName) + if (infraDryRun) { + sh "mkdir -p ${llmPath} && : > ${waivesListPathLocal}" + } else { + // Download and Merge waives.txt + mergeWaivesTxt(pipeline, llmSrcLocal, stageName) - // Add passed test list from previous pipeline run to the waives.txt - if (testFilter[(REUSE_TEST)] != false) { - reusePassedTestResults(llmSrcLocal, stageName, "${llmSrcLocal}/tests/integration/test_lists/waives.txt", postTag) + // Add passed test list from previous pipeline run to the waives.txt + if (testFilter[(REUSE_TEST)] != false) { + reusePassedTestResults(llmSrcLocal, stageName, waivesListPathLocal, postTag) + } } Utils.copyFileToRemoteHost( pipeline, remote, - "${llmSrcLocal}/tests/integration/test_lists/waives.txt", + waivesListPathLocal, waivesListPathNode ) @@ -1908,7 +1948,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "--s3-upload-mode=deferred", ] } - def pytestCommand = getPytestBaseCommandLine( + def pytestCommandParts = getPytestBaseCommandLine( llmSrcNode, stageName, waivesListPathNode, @@ -1917,7 +1957,9 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG "$jobWorkspace/.coveragerc", pytestUtil, extraArgs, - ).join(" ") + ) + pytestCommandParts += getInfraDryRunPytestTargets(testListPathLocal) + def pytestCommand = pytestCommandParts.join(" ") // Generate Job Launch Script def container = LLM_DOCKER_IMAGE @@ -2091,6 +2133,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG export llmSrcNode=$llmSrcNode export stageName=$stageName export perfMode=$perfMode + ${infraDryRun ? "export infraDryRun=true" : ""} export resourcePathNode=$resourcePathNode export pytestCommand="$pytestCommand" export coverageConfigFile="$coverageConfigFile" @@ -2110,7 +2153,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG ${srunPrologue} """.replaceAll("(?m)^\\s*", "") - if (disaggMultiNodeMode || aggMultiNodeMode) { + if (!isInfraDryRun() && (disaggMultiNodeMode || aggMultiNodeMode)) { def scriptLaunchPrefixPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch_prefix.sh") def scriptLaunchSrunArgsPathLocal = Utils.createTempLocation(pipeline, "./slurm_srun_args.txt") // The unified submit.py handles both agg and disagg; only the @@ -2831,6 +2874,8 @@ def CBTS_RESULT = "cbts_result" // Pipeline-level CBTS coverage eligibility, decided in L0_MergeRequest.groovy. @Field def CBTS_COVERAGE = "cbts_coverage" +@Field +def INFRA_DRY_RUN = "infra_dry_run" // Suffix for CBTS-narrowed stages so their results aren't reused by non-CBTS runs. // A suffix (not prefix) keeps the GPU type as the first '-' token for positional parsers. @Field @@ -2858,6 +2903,7 @@ def testFilter = [ (DETAILED_LOG): false, (CBTS_RESULT): null, (CBTS_COVERAGE): false, + (INFRA_DRY_RUN): false, ] @Field @@ -3782,6 +3828,18 @@ def runLLMAgentFlowTest(pipeline, stageName) // These resolve from the container's default PyPI mirror. trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${agentFlowRoot} && pip3 install -e \".[test]\"") + if (isInfraDryRun()) { + // Keep the normal environment and reporting path, but replace product tests. + sh """ + rm -rf "${agentFlowRoot}/tests" && \ + mkdir -p "${agentFlowRoot}/tests" && \ + printf '%s\\n' \ + 'def test_infra_dry_run_placeholder():' \ + ' pass' \ + > "${agentFlowRoot}/tests/test_infra_dry_run_placeholder.py" + """ + } + sh "mkdir -p ${WORKSPACE}/${stageName}" // test_workflow_entrypoint_modules_run_without_import_warnings is deselected @@ -4819,6 +4877,13 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def noRegularTests = false def noIsolateTests = false def rerunFailed = false + def infraDryRun = isInfraDryRun() + if (infraDryRun) { + testList = INFRA_DRY_RUN_TEST_CONTEXT + splitId = 1 + splits = 1 + perfMode = false + } // When useClusterDurations is set, use a per-cluster durations file keyed on // partition.clusterName (e.g. "oci-hsg", "dlcluster"). This lets each cluster @@ -4836,13 +4901,20 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO } def testDBList = renderTestDB(pipeline, testList, llmSrc, stageName, null, clusterNameForDurations) + def waivesFilePath = infraDryRun + ? "${llmSrc}/infra_dry_run_waives.txt" + : "${llmSrc}/tests/integration/test_lists/waives.txt" - // Download and Merge waives.txt - mergeWaivesTxt(pipeline, llmSrc, stageName) + if (infraDryRun) { + sh ": > ${waivesFilePath}" + } else { + // Download and Merge waives.txt + mergeWaivesTxt(pipeline, llmSrc, stageName) - // Add passed test list from previous pipeline run to the waives.txt - if (testFilter[(REUSE_TEST)] != false) { - reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt", postTag) + // Add passed test list from previous pipeline run to the waives.txt + if (testFilter[(REUSE_TEST)] != false) { + reusePassedTestResults(llmSrc, stageName, waivesFilePath, postTag) + } } // Process shard test list and create separate files for regular and isolate tests @@ -4890,7 +4962,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO def pytestCommand = getPytestBaseCommandLine( llmSrc, stageName, - "${llmSrc}/tests/integration/test_lists/waives.txt", + waivesFilePath, perfMode, "${WORKSPACE}/${stageName}", coverageConfigFile, @@ -4903,6 +4975,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO // Only add --test-list if there are regular tests to run if (preprocessedLists.regularCount > 0) { pytestCommand += ["--test-list=${preprocessedLists.regular}"] + pytestCommand += getInfraDryRunPytestTargets(preprocessedLists.regular) } def containerPIP_LLM_LIB_PATH = sh(script: "pip3 show tensorrt_llm | grep \"Location\" | awk -F\":\" '{ gsub(/ /, \"\", \$2); print \$2\"/tensorrt_llm/libs\"}'", returnStdout: true).replaceAll("\\s","") @@ -4912,7 +4985,11 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO containerLD_LIBRARY_PATH = "${containerPIP_LLM_LIB_PATH}:${containerLD_LIBRARY_PATH}" } containerLD_LIBRARY_PATH = containerLD_LIBRARY_PATH.replaceAll(':+$', '') - withEnv(["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"]) { + def testEnvironment = ["LD_LIBRARY_PATH=${containerLD_LIBRARY_PATH}"] + if (infraDryRun) { + testEnvironment += ["stageName=${stageName}"] + } + withEnv(testEnvironment) { withCredentials([ string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'), string(credentialsId: 'svc_tensorrt-swift-stack-key', variable: 'S3_SECRET_KEY'), @@ -6935,6 +7012,8 @@ pipeline { } if (singleGpuJobs.size() > 0) { runBranchesWithInfraDefer(singleGpuJobs, params.enableFailFast, stageInfraScope) + } else if (isInfraDryRun()) { + error "Skip single-GPU testing. No test to run for infrastructure dry run." } else { echo "Skip single-GPU testing. No test to run." } diff --git a/jenkins/scripts/slurm_run.sh b/jenkins/scripts/slurm_run.sh index c587825fa167..6097eca724ed 100755 --- a/jenkins/scripts/slurm_run.sh +++ b/jenkins/scripts/slurm_run.sh @@ -1,4 +1,18 @@ #!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # Set up error handling set -xEeuo pipefail @@ -15,9 +29,9 @@ if [ $SLURM_PROCID -eq 0 ]; then fi fi -# Aggregated mode will run install together with pytest in slurm_run.sh +# Aggregated mode and infrastructure dry runs install in slurm_run.sh. # Disaggregated mode will run install separately in slurm_install.sh -if [[ "$stageName" != *Disagg* ]]; then +if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]; then installScriptPath="$(dirname "${BASH_SOURCE[0]}")/$(basename "${BASH_SOURCE[0]}" | sed 's/slurm_run\.sh/slurm_install.sh/')" source "$installScriptPath" slurm_install_setup diff --git a/tests/integration/defs/test_infra_dry_run_benchmark.py b/tests/integration/defs/test_infra_dry_run_benchmark.py new file mode 100644 index 000000000000..e9698841fad7 --- /dev/null +++ b/tests/integration/defs/test_infra_dry_run_benchmark.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small, model-free benchmark for the infrastructure dry-run test context.""" + +from __future__ import annotations + +import os + +import torch + +_MATRIX_SIZE = 32 + + +def _validate_matmul(device: torch.device, dtype: torch.dtype) -> None: + left = torch.full((_MATRIX_SIZE, _MATRIX_SIZE), 0.25, dtype=dtype, device=device) + right = torch.full((_MATRIX_SIZE, _MATRIX_SIZE), 0.5, dtype=dtype, device=device) + output = torch.matmul(left, right) + expected = torch.full_like(output, _MATRIX_SIZE * 0.25 * 0.5) + assert output.device.type == device.type + assert output.dtype == dtype + assert torch.isfinite(output).all().item() + assert torch.equal(output, expected) + + +def _run_cpu() -> None: + _validate_matmul(torch.device("cpu"), torch.float32) + + +def _run_cuda() -> None: + assert torch.cuda.is_available(), "CUDA is required for this infrastructure dry-run stage" + device_count = torch.cuda.device_count() + assert device_count > 0, "no CUDA devices are visible to the infrastructure dry run" + for device_index in range(device_count): + device = torch.device("cuda", device_index) + torch.cuda.set_device(device) + _validate_matmul(device, torch.float16) + torch.cuda.synchronize(device) + + +def test_infra_dry_run_benchmark() -> None: + """Exercise the CPU or every CUDA device visible to the pytest runner.""" + if os.environ.get("stageName", "").startswith("CPU-"): + _run_cpu() + else: + _run_cuda() diff --git a/tests/integration/test_lists/test-db/infra_dry_run.yml b/tests/integration/test_lists/test-db/infra_dry_run.yml new file mode 100644 index 000000000000..ab73d456bb79 --- /dev/null +++ b/tests/integration/test_lists/test-db/infra_dry_run.yml @@ -0,0 +1,9 @@ +version: 0.0.1 +infra_dry_run: +- condition: + ranges: + system_gpu_count: + gte: 0 + lte: 1024 + tests: + - test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark diff --git a/tests/unittest/tools/test_infra_dry_run_pipeline.py b/tests/unittest/tools/test_infra_dry_run_pipeline.py new file mode 100644 index 000000000000..5687293e6520 --- /dev/null +++ b/tests/unittest/tools/test_infra_dry_run_pipeline.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +L0_TEST = (REPO_ROOT / "jenkins" / "L0_Test.groovy").read_text() +L0_PARENT = (REPO_ROOT / "jenkins" / "L0_MergeRequest.groovy").read_text() +SLURM_RUN = (REPO_ROOT / "jenkins" / "scripts" / "slurm_run.sh").read_text() +DRY_RUN_DB = ( + REPO_ROOT / "tests" / "integration" / "test_lists" / "test-db" / "infra_dry_run.yml" +).read_text() + + +def _function_body(source: str, name: str, next_name: str) -> str: + start = source.index(f"def {name}") + return source[start : source.index(f"def {next_name}", start + len(name))] + + +class InfraDryRunPipelineTest(unittest.TestCase): + def test_dry_run_allows_only_the_synthetic_pytest_target(self) -> None: + expected = "test_infra_dry_run_benchmark.py::test_infra_dry_run_benchmark" + targets = _function_body(L0_TEST, "getInfraDryRunPytestTargets", "processShardTestList") + + self.assertEqual(DRY_RUN_DB.count("::test_"), 1) + self.assertIn(expected, DRY_RUN_DB) + self.assertIn(f'expectedTarget =\n "{expected}"', targets) + self.assertIn("if (targets != [expectedTarget])", targets) + self.assertIn("return targets", targets) + + def test_slurm_dry_run_preserves_agent_and_sbatch_dispatch(self) -> None: + dispatch = _function_body(L0_TEST, "runLLMTestlistOnSlurm", "INFRA_DRY_RUN") + agent = _function_body(L0_TEST, "runLLMTestlistWithAgent", "executeLLMTestOnSlurm") + sbatch = _function_body(L0_TEST, "runLLMTestlistWithSbatch", "runLLMTestlistOnSlurm") + + self.assertIn("if (nodeCount > 1 || runWithSbatch)", dispatch) + self.assertNotIn("isInfraDryRun() || nodeCount", dispatch) + self.assertIn("runInDockerOnNodeMultiStage", agent) + self.assertIn("runInEnrootOnNode", agent) + self.assertIn("testList = INFRA_DRY_RUN_TEST_CONTEXT", sbatch) + self.assertIn("pytestCommandParts += getInfraDryRunPytestTargets", sbatch) + self.assertIn( + 'if [[ "${infraDryRun:-false}" == "true" || "$stageName" != *Disagg* ]]', + SLURM_RUN, + ) + + def test_parent_dry_run_is_opt_in_non_fail_fast_and_collects_results(self) -> None: + helper = _function_body(L0_PARENT, "launchInfraDryRunTestJob", "launchStages") + + self.assertIn( + "(INFRA_DRY_RUN): (params.InfraDryRun?.toString()?.toBoolean() ?: false)", + L0_PARENT, + ) + self.assertIn('"L0_Test-${arch}-Single-GPU"', helper) + self.assertIn("'testPhase2StageName': ''", helper) + self.assertIn( + "def effectiveFailFast = testFilter[INFRA_DRY_RUN] ? false : enableFailFast", + L0_PARENT, + ) + self.assertIn("parallelJobs.failFast = effectiveFailFast", L0_PARENT) + self.assertIn("collectTestResults(this, testFilter, globalVars)", L0_PARENT) + + def test_dry_run_skips_merge_request_diff_lookups(self) -> None: + infra_dry_run_check = "(params.InfraDryRun?.toString()?.toBoolean() ?: false)" + changed_files = _function_body( + L0_PARENT, "getMergeRequestChangedFileList", "getMergeRequestOneFileChanges" + ) + one_file_diff = _function_body( + L0_PARENT, "getMergeRequestOneFileChanges", "getAutoTriggerTagList" + ) + + for body, empty_result in ((changed_files, "return []"), (one_file_diff, 'return ""')): + with self.subTest(empty_result=empty_result): + self.assertIn(f"if ({infra_dry_run_check} ||", body) + self.assertLess(body.index(infra_dry_run_check), body.index("def githubPrApiUrl")) + self.assertLess(body.index(empty_result), body.index("def githubPrApiUrl")) + + def test_docs_skip_junit_after_a_successful_build(self) -> None: + self.assertIn( + 'cacheErrorAndUploadResult("${key}", values[1], {}, true, attemptTag, ' + "isFinalAttempt, retryContext)", + L0_TEST, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unittest/tools/test_infra_dry_run_pytest.py b/tests/unittest/tools/test_infra_dry_run_pytest.py new file mode 100644 index 000000000000..e8dd886c4db7 --- /dev/null +++ b/tests/unittest/tools/test_infra_dry_run_pytest.py @@ -0,0 +1,238 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import os +import subprocess +import sys +import tempfile +import textwrap +import types +import unittest +from pathlib import Path +from unittest import mock + +REPO_ROOT = Path(__file__).resolve().parents[3] +BENCHMARK_PATH = REPO_ROOT / "tests" / "integration" / "defs" / "test_infra_dry_run_benchmark.py" + +_TORCH_IMPORT_STUB = types.ModuleType("torch") +with mock.patch.dict(sys.modules, {"torch": _TORCH_IMPORT_STUB}): + _SPEC = importlib.util.spec_from_file_location("test_infra_dry_run_benchmark", BENCHMARK_PATH) + assert _SPEC is not None and _SPEC.loader is not None + BENCHMARK = importlib.util.module_from_spec(_SPEC) + _SPEC.loader.exec_module(BENCHMARK) + +_FAKE_TORCH_SOURCE = textwrap.dedent( + """ + float32 = "float32" + class Device: + def __init__(self, kind, index=None): + self.type, self.index = kind, index + class Scalar: + def item(self): return True + class Tensor: + def __init__(self, value, dtype, device): + self.value, self.dtype, self.device = value, dtype, device + def all(self): return Scalar() + def device(kind, index=None): return Device(kind, index) + def full(_shape, value, *, dtype, device): return Tensor(value, dtype, device) + def matmul(left, _right): return Tensor(4.0, left.dtype, left.device) + def full_like(tensor, value): return Tensor(value, tensor.dtype, tensor.device) + def isfinite(tensor): return tensor + def equal(left, right): return left.value == right.value + """ +) + + +def _write_benchmark_sandbox(root: Path) -> None: + (root / BENCHMARK_PATH.name).write_text(BENCHMARK_PATH.read_text()) + (root / "torch.py").write_text(_FAKE_TORCH_SOURCE) + + +class _Scalar: + def __init__(self, value): + self._value = value + + def item(self): + return self._value + + +class _Tensor: + def __init__(self, value, *, dtype, device): + self.value = value + self.dtype = dtype + self.device = device + + def all(self): + return _Scalar(True) + + +class _Cuda: + def __init__(self, available=True, count=2): + self.available = available + self.count = count + self.selected = [] + self.synchronized = [] + + def is_available(self): + return self.available + + def device_count(self): + return self.count + + def set_device(self, device): + self.selected.append(device.index) + + def synchronize(self, device): + self.synchronized.append(device.index) + + +class _Torch: + float16 = "float16" + float32 = "float32" + + def __init__(self, *, cuda_available=True, cuda_count=2): + self.cuda = _Cuda(cuda_available, cuda_count) + self.devices = [] + + def device(self, device_type, index=None): + device = types.SimpleNamespace(type=device_type, index=index) + self.devices.append(device) + return device + + def full(self, _shape, value, *, dtype, device): + return _Tensor(value, dtype=dtype, device=device) + + def matmul(self, _left, _right): + device = self.devices[-1] + dtype = self.float32 if device.type == "cpu" else self.float16 + return _Tensor(4.0, dtype=dtype, device=device) + + def full_like(self, tensor, value): + return _Tensor(value, dtype=tensor.dtype, device=tensor.device) + + def isfinite(self, tensor): + return tensor + + def equal(self, left, right): + return left.value == right.value + + +class InfraDryRunBenchmarkTest(unittest.TestCase): + def test_cpu_path_uses_fp32_cpu_matmul(self): + torch_stub = _Torch() + with mock.patch.object(BENCHMARK, "torch", torch_stub): + BENCHMARK._run_cpu() + self.assertEqual( + [(device.type, device.index) for device in torch_stub.devices], [("cpu", None)] + ) + + def test_cuda_path_exercises_every_visible_device(self): + torch_stub = _Torch(cuda_count=3) + with mock.patch.object(BENCHMARK, "torch", torch_stub): + BENCHMARK._run_cuda() + self.assertEqual(torch_stub.cuda.selected, [0, 1, 2]) + self.assertEqual(torch_stub.cuda.synchronized, [0, 1, 2]) + + def test_cuda_path_does_not_fall_back_to_cpu(self): + with mock.patch.object(BENCHMARK, "torch", _Torch(cuda_available=False)): + with self.assertRaisesRegex(AssertionError, "CUDA is required"): + BENCHMARK._run_cuda() + + def test_standard_pytest_collection_selects_only_the_requested_context(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + _write_benchmark_sandbox(root) + (root / "test_product.py").write_text("def test_product(): pass\n") + (root / "conftest.py").write_text( + textwrap.dedent( + """ + def pytest_addoption(parser): + parser.addoption("--test-list") + def pytest_collection_modifyitems(config, items): + wanted = { + line.strip() for line in open(config.getoption("--test-list")) + if line.strip() + } + selected = [item for item in items if item.nodeid in wanted] + config.hook.pytest_deselected( + items=[item for item in items if item not in selected] + ) + items[:] = selected + """ + ) + ) + + dry_list = root / "dry.txt" + dry_list.write_text(f"{BENCHMARK_PATH.name}::test_infra_dry_run_benchmark\n") + normal_list = root / "normal.txt" + normal_list.write_text("test_product.py::test_product\n") + env = {**os.environ, "stageName": "CPU-Generic-x86-1"} + for test_list, expected in ( + (dry_list, BENCHMARK_PATH.name), + (normal_list, "test_product.py"), + ): + result = subprocess.run( + [sys.executable, "-m", "pytest", f"--test-list={test_list}", "-vv"], + cwd=root, + env=env, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(expected, result.stdout) + self.assertIn("1 passed, 1 deselected", result.stdout) + + def test_positional_nodeid_does_not_import_unrelated_product_test(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + _write_benchmark_sandbox(root) + (root / "test_unrelated_product.py").write_text( + 'raise RuntimeError("unrelated product test imported")\n' + ) + (root / "conftest.py").write_text( + textwrap.dedent( + """ + def pytest_addoption(parser): + parser.addoption("--test-list") + """ + ) + ) + + target = f"{BENCHMARK_PATH.name}::test_infra_dry_run_benchmark" + dry_list = root / "dry.txt" + dry_list.write_text(f"{target}\n") + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + f"--test-list={dry_list}", + target, + "-q", + ], + cwd=root, + env={**os.environ, "stageName": "CPU-Generic-x86-1"}, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn(target, result.stdout) + self.assertNotIn("test_unrelated_product.py", result.stdout) + + +if __name__ == "__main__": + unittest.main()