[TRTLLM-7101][infra] Reuse passed tests

jenkins/L0_MergeRequest.groovy

@@ -80,6 +80,8 @@ def trimForStageList(stageNameList)
     return trimedList
 }
 
+@Field
+def REUSE_TEST = "reuse_test"   // Only used to determine if the pipeline should reuse the test results from the previous pipeline.
 @Field
 def REUSE_STAGE_LIST = "reuse_stage_list"
 @Field
@@ -114,6 +116,7 @@ def DEBUG_MODE = "debug"
 def DETAILED_LOG = "detailed_log"
 
 def testFilter = [
+    (REUSE_TEST): gitlabParamsFromBot.get(REUSE_TEST, null),
     (REUSE_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get(REUSE_STAGE_LIST, null)?.tokenize(',')),
     (ENABLE_SKIP_TEST): gitlabParamsFromBot.get((ENABLE_SKIP_TEST), false),
     (TEST_STAGE_LIST): trimForStageList(gitlabParamsFromBot.get((TEST_STAGE_LIST), null)?.tokenize(',')),

jenkins/L0_Test.groovy

@@ -736,6 +736,7 @@ def getNodeArgs(int nodeCount, int gpuCount) {
 def getPytestBaseCommandLine(
     String llmSrc,
     String stageName,
+    String waivesFilePath,
     Boolean perfMode,
     String outputPath,
     String trtllmWheelPath,
@@ -756,6 +757,7 @@ def getPytestBaseCommandLine(
         "LLM_BACKEND_ROOT=${llmSrc}/triton_backend",
         "LLM_MODELS_ROOT=${MODEL_CACHE_DIR}",
         "MODEL_CACHE_DIR=${MODEL_CACHE_DIR}",
+        "COLUMNS=200",
         extraInternalEnv,
         pytestUtil,
         "pytest",
@@ -766,7 +768,7 @@ def getPytestBaseCommandLine(
         "--timeout=${pytestTestTimeout}",
         "--rootdir ${llmSrc}/tests/integration/defs",
         "--test-prefix=${stageName}",
-        "--waives-file=${llmSrc}/tests/integration/test_lists/waives.txt",
+        "--waives-file=${waivesFilePath}",
         "--output-dir=${outputPath}/",
         "--csv=${outputPath}/report.csv",
         "--junit-xml ${outputPath}/results.xml",
@@ -826,7 +828,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             Utils.exec(pipeline, script: "apt-get update && apt-get install -y sshpass openssh-client")
             def tarName = BUILD_CONFIGS[config][TARNAME]
             def llmTarfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/${tarName}"
-            def llmWaivesTxtfile = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt"
             def llmPath = sh (script: "realpath .", returnStdout: true).trim()
             def jobWorkspace = "/home/svc_tensorrt/bloom/scripts/${jobUID}"
             def resourcePathNode = "/tmp"
@@ -835,6 +836,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             def scriptRunLocalPath = "${llmSrcLocal}/jenkins/scripts/slurm_run.sh"
             def scriptRunPathNode = "${jobWorkspace}/${jobUID}-slurm_run.sh"
             def testListPathNode = "${jobWorkspace}/${testList}.txt"
+            def waivesListPathNode = "${jobWorkspace}/waives.txt"
             def outputPath = "${jobWorkspace}/job-output.log"
             def scriptLaunchPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch.sh")
             def scriptLaunchPathNode = "${jobWorkspace}/${jobUID}-slurm_launch.sh"
@@ -873,6 +875,21 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     testListPathNode
                 )
 
+                // Download and Merge waives.txt
+                mergeWaivesTxt(pipeline, llmSrcLocal, stageName)
+
+                // Add passed test list from previous pipeline run to the waives.txt
+                if (testFilter[(REUSE_TEST)] != false) {
+                    reusePassedTestResults(llmSrcLocal, stageName, "${llmSrcLocal}/tests/integration/test_lists/waives.txt")
+                }
+
+                Utils.copyFileToRemoteHost(
+                    pipeline,
+                    remote,
+                    "${llmSrcLocal}/tests/integration/test_lists/waives.txt",
+                    waivesListPathNode
+                )
+
                 // generate .coveragerc in workspace and add file path to pytest command
                 sh """
                     touch ./.coveragerc
@@ -900,6 +917,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 def pytestCommand = getPytestBaseCommandLine(
                     llmSrcNode,
                     stageName,
+                    waivesListPathNode,
                     perfMode,
                     jobWorkspace,
                     "__PLACEHOLDER_TRTLLM_WHL_PATH__",
@@ -989,7 +1007,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     export jobWorkspace=$jobWorkspace
                     export tarName=$tarName
                     export llmTarfile=$llmTarfile
-                    export llmWaivesTxtfile=$llmWaivesTxtfile
                     export llmSrcNode=$llmSrcNode
                     export stageName=$stageName
                     export perfMode=$perfMode
@@ -1101,6 +1118,8 @@ def trimForStageList(stageNameList)
 
 // Test filter flags
 @Field
+def REUSE_TEST = "reuse_test"
+@Field
 def REUSE_STAGE_LIST = "reuse_stage_list"
 @Field
 def ENABLE_SKIP_TEST = "skip_test"
@@ -1132,6 +1151,7 @@ def DEBUG_MODE = "debug"
 def DETAILED_LOG = "detailed_log"
 @Field
 def testFilter = [
+    (REUSE_TEST): null,
     (REUSE_STAGE_LIST): null,
     (ENABLE_SKIP_TEST): false,
     (TEST_STAGE_LIST): null,
@@ -2081,6 +2101,60 @@ def generateRerunReport(stageName, llmSrc) {
     echo "Rerun report generation completed for stage: ${stageName}"
 }
 
+def mergeWaivesTxt(pipeline, llmSrc, stageName) {
+    def waivesTxt = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt"
+    try {
+        trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${waivesTxt}")
+        if (!fileExists("waives.txt")) {
+            error "There is no merged waives.txt file, use the default waives.txt."
+        }
+        sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt"
+        sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt"
+        echo "Download merged waives.txt successfully"
+    } catch (InterruptedException e) {
+        throw e
+    } catch (Exception e) {
+        echo "Failed to download merged waives.txt, use the default waives.txt. Error: ${e.message}"
+    }
+}
+
+def reusePassedTestResults(llmSrc, stageName, waivesTxt) {
+    try {
+        // Get passed test list from open search
+        def passedTestListFile = "${WORKSPACE}/${stageName}/passed_test_list.txt"
+        sh """
+            python3 ${llmSrc}/jenkins/scripts/open_search_query.py \
+            --commit-id ${env.gitlabCommit} \
+            --stage-name ${stageName} \
+            --output-file ${passedTestListFile}
+        """
+
+        def passedTestList = readFile(file: passedTestListFile).readLines()
+        def reusedTests = passedTestList.collect { test -> test.trim() }
+
+        // Append reused tests to waives.txt
+        if (reusedTests.size() > 0) {
+            sh(label: "Reused Tests", script: "echo \"Reused tests:\n${reusedTests.join('\n')}\"")
+
+            // Build the content to append
+            def reusedTestsContent = reusedTests.collect { test ->
+                "${test} SKIP (Reused from previous pipeline)"
+            }.join('\n')
+
+            sh """
+                cat >> ${waivesTxt} << 'REUSED_TESTS_EOF'
+    ${reusedTestsContent}
+    REUSED_TESTS_EOF
+            """
+            sh(label: "Updated Waives File", script: "echo \"Appended ${reusedTests.size()} reused tests to ${waivesTxt}\"")
+        }
+    } catch (InterruptedException e) {
+        throw e
+    } catch (Exception e) {
+        echo "Failed to add passed test list from previous pipeline run to the waives.txt. Error: ${e.message}"
+    }
+}
+
 def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, skipInstallWheel=false, cpver="cp312")
 {
     // Step 1: create LLM_ROOT dir and clean up the workspace
@@ -2135,22 +2209,6 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         trtllm_utils.llmExecStepWithRetry(pipeline, script: "cd ${llmPath} && wget -nv ${llmTarfile}")
         sh "cd ${llmPath} && tar -zxf ${tarName}"
 
-        // Download the new merged waives.txt
-        def waivesTxt = "https://urm.nvidia.com/artifactory/${ARTIFACT_PATH}/waive_list/waives.txt"
-        try {
-            trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget -nv ${waivesTxt}")
-            if (!fileExists("waives.txt")) {
-                error "There is no merged waives.txt file, use the default waives.txt."
-            }
-            sh "rm ${llmSrc}/tests/integration/test_lists/waives.txt"
-            sh "mv waives.txt ${llmSrc}/tests/integration/test_lists/waives.txt"
-            echo "Download merged waives.txt successfully"
-        } catch (InterruptedException e) {
-            throw e
-        } catch (Exception e) {
-            echo "Failed to download merged waives.txt, use the default waives.txt. Error: ${e.message}"
-        }
-
         // install python package
         if (env.alternativeTRT) {
             sh "cd ${llmSrc} && sed -i 's#tensorrt~=.*\$#tensorrt#g' requirements.txt && cat requirements.txt"
@@ -2264,6 +2322,14 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
 
         def testDBList = renderTestDB(testList, llmSrc, stageName)
 
+        // Download and Merge waives.txt
+        mergeWaivesTxt(pipeline, llmSrc, stageName)
+
+        // Add passed test list from previous pipeline run to the waives.txt
+        if (testFilter[(REUSE_TEST)] != false) {
+            reusePassedTestResults(llmSrc, stageName, "${llmSrc}/tests/integration/test_lists/waives.txt")
+        }
+
         // Process shard test list and create separate files for regular and isolate tests
         def preprocessedLists = processShardTestList(llmSrc, testDBList, splitId, splits, perfMode)
 
@@ -2286,6 +2352,7 @@ def runLLMTestlistOnPlatformImpl(pipeline, platform, testList, config=VANILLA_CO
         def pytestCommand = getPytestBaseCommandLine(
             llmSrc,
             stageName,
+            "${llmSrc}/tests/integration/test_lists/waives.txt",
             perfMode,
             "${WORKSPACE}/${stageName}",
             TRTLLM_WHL_PATH,

jenkins/scripts/open_search_query.py

@@ -0,0 +1,125 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# =============================================================================
+# open_search_query.py
+#
+# This module provides functions to query the OpenSearch database for passed
+# test results from previous pipeline runs. It retrieves test names that have
+# passed for a given commit ID and stage name, which can be reused to skip
+# redundant test execution in subsequent runs.
+#
+# Main functionality:
+# - queryJobEvents: Queries OpenSearch for job events with pagination support
+# - getPassedTestList: Retrieves and deduplicates passed test names
+# - writeTestListToFile: Writes test list to a file for further processing
+#
+# =============================================================================
+
+import argparse
+import json
+import os
+import sys
+
+from open_search_db import OpenSearchDB
+
+
+def queryJobEvents(commitID="", stageName="", onlySuccess=True):
+    """
+    Query OpenSearch database for job events with pagination.
+
+    Args:
+        commitID: Git commit SHA to filter by (optional)
+        stageName: Stage name to filter by (optional)
+        onlySuccess: If True, only return PASSED tests (default: True)
+
+    Returns:
+        List of all matching test result records
+    """
+    mustConditions = []
+    if commitID:
+        mustConditions.append({"term": {"s_trigger_mr_commit": commitID}})
+    if stageName:
+        mustConditions.append({"term": {"s_stage_name": stageName}})
+    if onlySuccess:
+        mustConditions.append({"term": {"s_status": "PASSED"}})
+
+    all_results = []
+    page_size = 1000
+    from_index = 0
+
+    while True:
+        requestBody = {
+            "query": {"bool": {"must": mustConditions}},
+            "_source": [
+                "s_job_name",
+                "s_status",
+                "s_build_id",
+                "s_turtle_name",
+                "s_test_name",
+                "s_gpu_type",
+            ],
+            "size": page_size,
+            "from": from_index,
+        }
+
+        formattedRequestBody = json.dumps(requestBody)
+        response = OpenSearchDB.queryFromOpenSearchDB(
+            formattedRequestBody, "swdl-trtllm-infra-ci-prod-test_info"
+        )
+        if response is None:
+            print("Failed to query from OpenSearchDB")
+            break
+        data = response.json()
+
+        hits = data["hits"]["hits"]
+        if not hits:
+            break
+
+        all_results.extend(hits)
+        from_index += page_size
+
+        print(f"Fetched {len(all_results)} records...")
+
+    return all_results
+
+
+def writeTestListToFile(testList, fileName):
+    os.makedirs(os.path.dirname(fileName), exist_ok=True)
+
+    with open(fileName, "w") as f:
+        for test in testList:
+            f.write(test + "\n")
+
+
+def getPassedTestList(commitID, stageName, outputFile):
+    hits = queryJobEvents(commitID=commitID, stageName=stageName, onlySuccess=True)
+    # Use set to automatically remove duplicates
+    testSet = set()
+    for hit in hits:
+        testSet.add(hit["_source"]["s_turtle_name"])
+    testList = list(testSet)
+    writeTestListToFile(testList, outputFile)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--commit-id", required=True, help="Commit ID")
+    parser.add_argument("--stage-name", required=True, help="Stage Name")
+    parser.add_argument("--output-file", required=True, help="Output File")
+    args = parser.parse_args(sys.argv[1:])
+    getPassedTestList(
+        commitID=args.commit_id, stageName=args.stage_name, outputFile=args.output_file
+    )

jenkins/scripts/slurm_run.sh

@@ -38,18 +38,6 @@ fi
 if [ $SLURM_LOCALID -eq 0 ]; then
     wget -nv $llmTarfile
     tar -zxf $tarName
-
-    # Download the new merged waives.txt
-    if ! wget -nv "$llmWaivesTxtfile"; then
-        echo "Failed to download merged waives.txt, use the default waives.txt."
-    elif [ ! -f "waives.txt" ]; then
-        echo "Downloaded but file does not exist, use the default waives.txt."
-    else
-        rm -f "$llmSrcNode/tests/integration/test_lists/waives.txt"
-        mv waives.txt "$llmSrcNode/tests/integration/test_lists/waives.txt"
-        echo "Download merged waives.txt successfully"
-    fi
-
     which python3
     python3 --version
     apt-get install -y libffi-dev