[None][infra] Cherry-pick Perf Test Scripts

jenkins/L0_Test.groovy

@@ -895,19 +895,22 @@ def getMountListForSlurmTest(SlurmCluster cluster, boolean useSbatch = false)
 
 def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG, perfMode=false, stageName="Undefined", splitId=1, splits=1, gpuCount=1, nodeCount=1, skipInstallWheel=false, cpver="cp312")
 {
-    SlurmPartition partition = SlurmConfig.partitionConfig[platform] as SlurmPartition
+    SlurmPartition partition = SlurmConfig.resolvePlatform(platform)
     SlurmCluster cluster = SlurmConfig.clusterConfig[partition.clusterName]
 
     // Create a unique suffix for the job name
     String customSuffix = "${env.BUILD_TAG}-${UUID.randomUUID().toString().replaceAll("-", "").substring(0, 6)}".toLowerCase()
     def jobUID = "${cluster.host}-multi_node_test-${customSuffix}"
-    def disaggMode = stageName.contains("PerfSanity-Disagg")
+    def disaggMode = stageName.contains("Disagg-PerfSanity")
 
     Utils.exec(pipeline, script: "env | sort && pwd && ls -alh")
 
+    def stageIsInterrupted = false
+
     try {
         // Run ssh command to start node in desired cluster via SLURM
         withCredentials([
+            string(credentialsId: 'TRTLLM_HF_TOKEN', variable: 'HF_TOKEN'),
             usernamePassword(
                 credentialsId: 'svc_tensorrt',
                 usernameVariable: 'USERNAME',
@@ -1164,6 +1167,7 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                     export resourcePathNode=$resourcePathNode
                     export pytestCommand="$pytestCommand"
                     export coverageConfigFile="$coverageConfigFile"
+                    export HF_TOKEN=$HF_TOKEN
                     export NVIDIA_IMEX_CHANNELS=\${NVIDIA_IMEX_CHANNELS:-0}
                     export NVIDIA_VISIBLE_DEVICES=\${NVIDIA_VISIBLE_DEVICES:-\$(seq -s, 0 \$((\$(nvidia-smi --query-gpu=count -i 0 --format=csv,noheader)-1)))}
                     ${envExportStatements}
@@ -1175,10 +1179,6 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                 """.replaceAll("(?m)^\\s*", "")
 
                 if (disaggMode) {
-                    if(nodeCount > 1) {
-                        srunArgs.add("--mpi=pmix")
-                    }
-
                     def scriptLaunchPrefixPathLocal = Utils.createTempLocation(pipeline, "./slurm_launch_prefix.sh")
                     def scriptLaunchSrunArgsPathLocal = Utils.createTempLocation(pipeline, "./slurm_srun_args.txt")
                     def scriptLaunchDraftPathLocal = "${llmSrcLocal}/jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh"
@@ -1198,7 +1198,8 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
                         --run-sh ${scriptRunPathNode} \\
                         --install-sh ${scriptInstallPathNode} \\
                         --script-prefix ${scriptLaunchPrefixPathLocal} \\
-                        --srun-args ${scriptLaunchSrunArgsPathLocal}
+                        --srun-args ${scriptLaunchSrunArgsPathLocal} \\
+                        --split-group ${splitId}
                     """
                 } else {
                     if(nodeCount > 1) {
@@ -1376,8 +1377,11 @@ def runLLMTestlistWithSbatch(pipeline, platform, testList, config=VANILLA_CONFIG
             }
             echo "Finished test stage execution."
         }
+    } catch (InterruptedException e) {
+        stageIsInterrupted = true
+        throw e
     } finally {
-        uploadResults(pipeline, cluster, jobUID, stageName)
+        uploadResults(pipeline, cluster, jobUID, stageName, stageIsInterrupted)
         stage("Clean Up Slurm Resource") {
             // Workaround to handle the interruption during clean up SLURM resources
             retry(3) {

jenkins/scripts/open_search_db.py

@@ -283,6 +283,78 @@ def queryFromOpenSearchDB(json_data, project) -> dict:
         )
         return None
 
+    @staticmethod
+    def queryPerfDataFromOpenSearchDB(project_name,
+                                      must_clauses,
+                                      size=DEFAULT_QUERY_SIZE,
+                                      must_not_clauses=None):
+        """
+        Query perf data from OpenSearchDB using must and must_not clauses.
+
+        :param project_name: Name of the project.
+        :param must_clauses: List of must clauses for query.
+        :param size: Query size.
+        :param must_not_clauses: List of must_not clauses for query.
+        :return: list of data dicts, empty list if no data, None on error.
+        """
+        if DISABLE_OPEN_SEARCH_DB_FOR_LOCAL_TEST:
+            return []
+        if must_clauses is None:
+            must_clauses = []
+        if must_not_clauses is None:
+            must_not_clauses = []
+        if not isinstance(must_clauses, list):
+            OpenSearchDB.logger.info(
+                f"Invalid must_clauses type: {type(must_clauses).__name__}")
+            return None
+        if not isinstance(must_not_clauses, list):
+            OpenSearchDB.logger.info(
+                f"Invalid must_not_clauses type: {type(must_not_clauses).__name__}"
+            )
+            return None
+
+        bool_query = {"must": must_clauses}
+        if must_not_clauses:
+            bool_query["must_not"] = must_not_clauses
+
+        json_data = {
+            "query": {
+                "bool": bool_query
+            },
+            "size": size,
+        }
+
+        data_list = []
+        try:
+            res = OpenSearchDB.queryFromOpenSearchDB(json_data, project_name)
+            if res is None:
+                OpenSearchDB.logger.info(
+                    f"Failed to query from {project_name}, returned no response"
+                )
+                return None
+            payload = res.json().get("hits", {}).get("hits", [])
+            if len(payload) == 0:
+                OpenSearchDB.logger.info(
+                    f"No data found in {project_name}, returned empty list")
+                return []
+            for hit in payload:
+                data_dict = hit.get("_source", {})
+                data_dict["_id"] = hit.get("_id", "")
+                if data_dict["_id"] == "":
+                    OpenSearchDB.logger.info(
+                        f"Failed to query from {project_name}, returned data with no _id"
+                    )
+                    return None
+                data_list.append(data_dict)
+            OpenSearchDB.logger.info(
+                f"Successfully queried from {project_name}, queried {len(data_list)} entries"
+            )
+            return data_list
+        except Exception as e:
+            OpenSearchDB.logger.warning(
+                f"Failed to query from {project_name}, returned error: {e}")
+            return None
+
     @staticmethod
     def queryBuildIdFromOpenSearchDB(job_name, last_days=DEFAULT_LOOKBACK_DAYS):
         if DISABLE_OPEN_SEARCH_DB_FOR_LOCAL_TEST:

jenkins/scripts/perf/README.md

@@ -0,0 +1,182 @@
+# Perf Sanity Scripts
+
+This directory contains scripts for running perf sanity tests and managing perf sanity data.
+
+## Directory Structure
+
+```
+jenkins/scripts/perf/
+  aggregated/
+    slurm_launch_draft.sh    # Draft template for aggregated SLURM launch scripts
+  disaggregated/
+    submit.py                # CI pipeline submit script (disaggregated only)
+    slurm_launch_draft.sh    # Draft template for disaggregated SLURM launch scripts
+  local/
+    submit.py                # Local submit script (aggregated and disaggregated)
+    slurm_install.sh         # Build wheel + pip install inside container
+    slurm_run.sh             # Run pytest inside container
+  perf_utils.py              # Shared utilities (regression detection, baseline, charts, OpenSearch queries)
+  get_pre_merge_html.py      # Pre-merge HTML report with history, baseline, and threshold
+  perf_sanity_triage.py      # Query/update OpenSearch data and send Slack notifications
+```
+
+## Submit Scripts
+
+Both `local/submit.py` and `disaggregated/submit.py` share a similar workflow. They read
+a test config YAML and use the appropriate draft template
+(`aggregated/slurm_launch_draft.sh` or `disaggregated/slurm_launch_draft.sh`) to generate
+a complete `slurm_launch.sh`. Then the user or CI pipeline can run `sbatch slurm_launch.sh`
+to submit the job. Inside the SLURM job, `slurm_install.sh` builds the wheel and runs
+installation, then `slurm_run.sh` runs pytest.
+
+```
+submit.py
+  |
+  v
+slurm_launch.sh  (generated)
+  |
+  |-- srun --> slurm_install.sh   (build wheel + pip install)
+  |-- srun --> slurm_run.sh       (run pytest)
+```
+
+Both submit scripts read `AGG_CONFIG_FOLDER` and `DISAGG_CONFIG_FOLDER` environment
+variables (with defaults of `tests/scripts/perf-sanity/aggregated` and
+`tests/scripts/perf-sanity/disaggregated`) and propagate them via `PYTEST_COMMON_VARS`
+into the pytest execution environment where `test_perf_sanity.py` uses them to locate
+config files.
+
+### `local/submit.py`
+
+Used for **local runs**. Supports both **aggregated** and **disaggregated** modes. It
+detects the mode from the test config YAML (aggregated configs have `server_configs`,
+disaggregated configs have `worker_config`) and selects the correct draft template
+automatically.
+
+See [`local/README.md`](local/README.md) for full argument reference and examples.
+
+### `disaggregated/submit.py`
+
+Used by the **CI pipeline** (called from `jenkins/L0_Test.groovy`'s
+`runLLMTestlistWithSbatch`). Only supports **disaggregated** mode. It receives a
+script prefix and srun args from the CI pipeline and combines them with disagg-specific
+environment variables and hardware configuration to generate `slurm_launch.sh`.
+
+## Shared Utilities
+
+### `perf_utils.py`
+
+Shared module imported by `get_post_merge_html.py`, `get_pre_merge_html.py`, and
+`perf_sanity_triage.py`. Contains:
+
+- **Constants**: `CHART_METRICS` (4 key throughput metrics), `METRIC_LABELS`,
+  algorithm parameters, curve type colors/labels.
+- **Baseline computation**: Rolling smooth (window=3) + P95 percentile algorithm.
+  Replaces the previous `max(daily_values)` approach which was vulnerable to
+  occasional spikes inflating the baseline.
+- **Regression detection**: Two-step classification (regression check + subtype
+  pattern matching). Supports per-metric thresholds from baseline data
+  (`d_threshold_pre_merge_*` fields, defaulting to 5%).
+- **OpenSearch query + grouping**: `get_history_data()` queries both baseline and
+  non-baseline data, groups by `(s_test_case_name, s_gpu_type)`.
+- **SVG chart generation**: Unified chart function supporting history lines,
+  new data points, baseline line, threshold line, curve type badges, and jump
+  interval shading.
+- **HTML dashboard**: `generate_post_merge_html()` produces a full interactive
+  report with three-way cascading filters and click-to-inspect data-point popups.
+
+## MPI/PMI Handling in Disaggregated Tests
+
+### Background
+
+Disaggregated tests run four srun steps within a single SLURM job. Only CTX/GEN workers
+need MPI (they use `trtllm-llmapi-launch`). The disagg server (`trtllm-serve
+disaggregated`) and benchmark client are single-process, non-MPI tasks.
+
+When srun launches a process with `--mpi=pmix`, it sets PMI/PMIx environment variables.
+If the launched process imports libraries with MPI support (e.g., PyTorch links Open MPI),
+`MPI_Init` may be triggered automatically. If the container's MPI build lacks SLURM PMI
+support, this causes:
+```
+PMI2_Init failed to initialize.  Return code: 14
+```
+
+### Solution
+
+The `--mpi=pmix` flag is added **only** to the CTX/GEN worker srun commands in
+`slurm_launch_draft.sh`, not to the shared `srunArgs` array. This way, the disagg server
+and benchmark srun steps never see MPI flags.
+
+**Where MPI is configured:**
+- `jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh` — `--mpi=pmix` on
+  ctx/gen srun commands only
+- `jenkins/scripts/perf/local/submit.py` — `--mpi=pmi2` for aggregated mode only,
+  no MPI flag for disaggregated mode (handled by the draft template)
+- `jenkins/L0_Test.groovy` — `--mpi=pmi2` for non-disagg multi-node only
+
+### Key Rules
+
+When modifying disaggregated SLURM scripts, keep these invariants:
+
+1. **srunArgs are shared**: All srun steps in `slurm_launch_draft.sh` use the same
+   `"${srunArgs[@]}"`. Never add MPI flags to srunArgs for disaggregated mode.
+2. **Only CTX/GEN workers need MPI**: Add `--mpi=pmix` directly on their srun command
+   lines in `slurm_launch_draft.sh`, not in the shared srunArgs.
+3. **Non-MPI roles must stay MPI-free**: The disagg server and benchmark steps must
+   not receive `--mpi` flags. If adding a new srun step, consider whether it needs MPI.
+
+## Post-Processing and Triage
+
+### `get_pre_merge_html.py`
+
+Triggered at the end of the CI pipeline in `jenkins/L0_MergeRequest.groovy`. It has
+3 main functions:
+
+1. **`load_perf_data`**: Reads perf_data.yaml files produced by test stages and
+   gathers all new perf data together.
+2. **`get_pre_merge_history_data`**: Queries OpenSearch for post-merge history data
+   (both baseline and non-baseline), grouped by `(s_test_case_name, s_gpu_type)`.
+3. **`generate_pre_merge_html`**: Generates an HTML report visualizing each test
+   case's key metrics (`d_seq_throughput`, `d_token_throughput`,
+   `d_total_token_throughput`, `d_user_throughput`) with history curve, new data
+   points, baseline line, and threshold line for regression comparison.
+
+### `perf_sanity_triage.py`
+
+Triggered by `jenkins/runPerfSanityTriage.groovy`. It supports two operations:
+
+1. **`SLACK BOT SENDS MESSAGE`**: Runs the perf-regression-detector pipeline
+   (`get_history_data` -> `get_baseline` -> `classify_test_case` ->
+   `generate_post_merge_html`), then sends the generated HTML dashboard to a
+   Slack channel.
+
+2. **`UPDATE SET ... (WHERE ...)`**: Updates fields on existing perf records that match
+   a query scope and posts the updated documents back to OpenSearch.
+
+**Examples**
+
+```
+SLACK BOT SENDS MESSAGE
+```
+
+```
+UPDATE SET b_is_valid=false WHERE s_test_case_name='test1'
+UPDATE SET b_is_valid=false WHERE ts_created <= 'Feb 18, 2026 @ 22:32:02.960' AND s_test_case_name='test1'
+```
+
+See the `UPDATE` operation section below for supported operators and date formats.
+
+#### UPDATE Operators
+
+- SET clause: Only `=` is supported.
+- WHERE clause: Supports `=`, `!=`, `>`, `<`, `>=`, `<=` operators.
+- `=` and `!=` operators are allowed for all fields.
+- `>`, `<`, `>=`, `<=` operators are only allowed for `ts_created` field (timestamp) or fields starting with `d_` (double type) or `l_` (integer type).
+
+#### `ts_created` Date Formats
+
+The `ts_created` field accepts date strings in the following formats:
+- `'Feb 18, 2026 @ 22:32:02.960'` (with milliseconds)
+- `'Feb 18, 2026 @ 22:32:02'` (without milliseconds)
+- `'2026/02/18'` (date only)
+
+All date strings are interpreted as UTC for consistent timestamp conversion.

jenkins/scripts/perf/aggregated/slurm_launch_draft.sh

@@ -0,0 +1,23 @@
+
+cleanup_on_failure() {
+    echo "Error: $1"
+    scancel ${SLURM_JOB_ID}
+    exit 1
+}
+
+mkdir -p $jobWorkspace
+chmod +x $runScript
+
+# Run aggregated test
+echo "Starting aggregated test..."
+world_size=$((totalNodes * gpusPerNodePerServer))
+if ! srun "${srunArgs[@]}" --kill-on-bad-exit=1 \
+    -N $totalNodes \
+    --ntasks=$world_size \
+    --ntasks-per-node=$gpusPerNodePerServer \
+    $runScript; then
+    cleanup_on_failure "Aggregated test failed. Check logs in ${jobWorkspace} for details"
+fi
+
+echo "Aggregated test completed successfully"
+echo "Total runtime: $SECONDS seconds"

jenkins/scripts/perf/disaggregated/slurm_launch_draft.sh

@@ -19,33 +19,33 @@ echo "Installation completed on all nodes"
 # Start gen servers
 echo "Starting gen servers..."
 for i in $(seq 0 $((numGenServers - 1))); do
-    gen_world_size=$((nodesPerGenServer * gpusPerNode))
+    gen_world_size=$((nodesPerGenServer * gpusPerNodePerGenServer))
     export DISAGG_SERVING_TYPE="GEN_$i"
-    export pytestCommand="$pytestCommandWorker"
-    srun "${srunArgs[@]}" --kill-on-bad-exit=1 \
+    export pytestCommand="$pytestCommandGENWorker"
+    srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \
         -N $nodesPerGenServer \
         --ntasks=$gen_world_size \
-        --ntasks-per-node=$gpusPerNode \
+        --ntasks-per-node=$gpusPerNodePerGenServer \
         $runScript &> $jobWorkspace/gen_server_$i.log &
     echo "Started gen server $i"
 done
 
-# Start ctx servers (skip if gen_only mode)
+# Start ctx servers (skip if gen_only_no_context mode)
 if [ "${TRTLLM_DISAGG_BENCHMARK_GEN_ONLY:-0}" != "1" ]; then
     echo "Starting ctx servers..."
     for i in $(seq 0 $((numCtxServers - 1))); do
-        ctx_world_size=$((nodesPerCtxServer * gpusPerNode))
+        ctx_world_size=$((nodesPerCtxServer * gpusPerNodePerCtxServer))
         export DISAGG_SERVING_TYPE="CTX_$i"
-        export pytestCommand="$pytestCommandWorker"
-        srun "${srunArgs[@]}" --kill-on-bad-exit=1 \
+        export pytestCommand="$pytestCommandCTXWorker"
+        srun "${srunArgs[@]}" --mpi=pmix --kill-on-bad-exit=1 \
             -N $nodesPerCtxServer \
-            --ntasks=$ctx_world_size \
-            --ntasks-per-node=$gpusPerNode \
+        --ntasks=$ctx_world_size \
+        --ntasks-per-node=$gpusPerNodePerCtxServer \
             $runScript &> $jobWorkspace/ctx_server_$i.log &
         echo "Started ctx server $i"
     done
 else
-    echo "Skipping ctx servers (gen_only mode)"
+    echo "Skipping ctx servers (gen_only_no_context mode)"
 fi