[None][feat] perf-sanity: upload per-request disagg lifecycle spans to OpenSearch

jenkins/L0_Test.groovy

@@ -6513,7 +6513,7 @@ def launchTestJobs(pipeline, testFilter, globalVars)
         "GB300-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX6-NODE1-GPU4-GEN1-NODE4-GPU16-Post-Merge",
         "auto:gb300-flex",
         "l0_gb300_multi_nodes_perf_sanity_ctx6_node1_gpu4_gen1_node4_gpu16",
-        2,
+        3,
         40,
         10
     )

jenkins/scripts/perf/README.md

@@ -147,16 +147,26 @@ wins when set.
 Test-ID format:
 
 ```
-perf/test_perf_sanity.py::test_e2e[<runtime>-<mode>-<yaml-stem>[-<server-cfg>]]
+perf/test_perf_sanity.py::test_e2e[<runtime>-<mode>[-<modifier>]-<yaml-stem>[-<server-cfg>]]
 ```
 
 - `<runtime>` = `disagg` | `aggr`
-- `<mode>` (disagg) = `e2e` | `gen_only` | `ctx_only`
+- `<mode>` = `e2e` | `gen_only` with `disagg`, or `ctx_only` with `aggr`.
+  `ctx_only` reads a disaggregated YAML but runs its ctx worker as a single
+  aggregated server, so it is spelled `aggr-ctx_only-<yaml-stem>`
+- `<modifier>` — optional instrumentation flag, orthogonal to `<mode>`; the only
+  one today is `time_breakdown`, which additionally uploads the per-request
+  lifecycle spans as `d_tb_<span>_<stat>`. It changes what the run *records*,
+  never the workload or the launch topology, so `--benchmark-mode` is still
+  handed the bare `<mode>`. Supported for `disagg-e2e` and `aggr-ctx_only`
 - `<yaml-stem>` matches a YAML file in `tests/scripts/perf-sanity/disaggregated/`
   (or `aggregated/`)
 - `<server-cfg>` — only for normal aggregated tests — the `name:` field of one of
   the YAML's `server_configs` entries
 
+A disagg `<yaml-stem>` may itself contain `-` (`..._ccb-NIXL`), so the stem is
+everything after the mode and the optional modifier, not a fixed segment count.
+
 `run_disagg.sh` errors out if any entry still contains the literal placeholder
 `CHANGE_ME`.
 

jenkins/scripts/perf/local/README.md

@@ -29,6 +29,8 @@ slurm_launch.sh  (generated)
 - `--test-list`: Test string, e.g., `perf/test_perf_sanity.py::test_e2e[aggr-config-test_name]`. If both `--test-list` and `--config-file` are provided, `--test-list` takes precedence.
 - `--config-file`: Path to config YAML file.
 - `--test-name`: Test name (only used for aggregated mode when `--config-file` is provided).
+- `--benchmark-mode`: `e2e` | `gen_only` | `ctx_only` (only used for a disagg `--config-file`; with `--test-list` the mode is read off the test id).
+- `--time-breakdown`: Also record the per-request lifecycle breakdown. This adds the `time_breakdown` modifier segment to the generated test id (`disagg-e2e-time_breakdown-<yaml-stem>`); the modifier is orthogonal to `--benchmark-mode` and does not change the workload.
 - `--time`: SLURM time limit (default: `02:00:00`).
 - `--mounts`: Container mounts.
 - `--work-dir`: Work directory (used for both workdir and container-workdir).

jenkins/scripts/perf/local/configs/example.conf

@@ -61,9 +61,14 @@ llm_models_path="${YOUR_LLM_MODELS_PATH:-/path/to/llm_models}"
 mounts="$trtllm:$trtllm,$llm_models_path:$llm_models_path"
 
 # Test ID(s). Format:
-#   perf/test_perf_sanity.py::test_e2e[disagg-<mode>-<yaml-stem>]
+#   perf/test_perf_sanity.py::test_e2e[<runtime>-<mode>[-<modifier>]-<yaml-stem>]
 # The <yaml-stem> matches a file in tests/scripts/perf-sanity/disaggregated/.
-# <mode> is e2e | gen_only | ctx_only.
+# <runtime>-<mode> is disagg-e2e | disagg-gen_only | aggr-ctx_only: ctx_only reads
+# the same disagg yaml but runs a single server, so it takes the aggr prefix.
+# <modifier> is an optional instrumentation flag, orthogonal to the mode; the only
+# one today is time_breakdown, e.g.
+#   perf/test_perf_sanity.py::test_e2e[disagg-e2e-time_breakdown-<yaml-stem>]
+#   perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-time_breakdown-<yaml-stem>]
 #
 # Two ways to declare tests — use ONE of these:
 #

jenkins/scripts/perf/local/submit.py

@@ -49,6 +49,30 @@ def _import_precheck_config(llm_src):
     "DISAGG_CONFIG_FOLDER", "tests/scripts/perf-sanity/disaggregated"
 )
 
+# Optional instrumentation segments that may follow the benchmark mode in a test
+# id. Keep in sync with test_perf_sanity.py:TEST_ID_MODIFIERS -- the grammar is
+# only decidable because no config file stem starts with one of these.
+TIME_BREAKDOWN_MODIFIER = "time_breakdown"
+TEST_ID_MODIFIERS = (TIME_BREAKDOWN_MODIFIER,)
+
+# Benchmark modes test_perf_sanity.py actually mints a time_breakdown test id
+# for. gen_only is deliberately absent: its regression metric is the gen-worker
+# device step time, so the lifecycle spans add nothing there, and the collector
+# generates no such id. Keep in sync with the two *_TIME_BREAKDOWN_CONFIGS loops
+# in test_perf_sanity.py:get_disagg_test_cases.
+TIME_BREAKDOWN_BENCHMARK_MODES = ("e2e", "ctx_only")
+
+
+def format_test_label(benchmark_mode: str, time_breakdown: bool = False) -> str:
+    """Compose the mode segment(s) of a test id.
+
+    Mirrors test_perf_sanity.py:format_test_label so the regenerated id matches
+    the collected one.
+    """
+    if time_breakdown:
+        return f"{benchmark_mode}-{TIME_BREAKDOWN_MODIFIER}"
+    return benchmark_mode
+
 
 def get_llm_src_default():
     """Get default llm_src path by going up 4 directories from this script."""
@@ -90,41 +114,64 @@ def parse_test_string(test_case_name: str):
 
     Test name formats:
     - Disagg e2e: disagg_upload-e2e-{config_base}
+    - Disagg e2e + lifecycle breakdown: disagg_upload-e2e-time_breakdown-{config_base}
     - Disagg gen_only: disagg_upload-gen_only-{config_base}
     - ctx_only: aggr_upload-ctx_only-{config_base} (runs aggr mode but reads disagg config)
+    - ctx_only + lifecycle breakdown: aggr_upload-ctx_only-time_breakdown-{config_base}
     - Regular aggr: aggr_upload-{config}-{server_name}
 
+    The optional modifier segment (TEST_ID_MODIFIERS) sits between the benchmark
+    mode and the config stem and is orthogonal to the mode.
+
     Returns:
-        tuple: (config_base_name, select_pattern, runtime_mode, benchmark_mode)
+        tuple: (config_base_name, select_pattern, runtime_mode, benchmark_mode,
+                time_breakdown)
             - runtime_mode: "aggregated" or "disaggregated"
-            - benchmark_mode: "e2e", "gen_only", "ctx_only", or None (for normal aggr)
+            - benchmark_mode: "e2e", "gen_only", "ctx_only", or None (normal aggr)
+            - time_breakdown: True when the "time_breakdown" modifier is present
     """
     labels = test_case_name.split("-")
 
-    assert len(labels) > 1, "perf_sanity test must have a config file!"
+    # ValueError rather than assert throughout: these are test-id grammar
+    # violations, and `python -O` removes assert statements, which would turn a
+    # malformed id into a silent IndexError or a submission against the wrong
+    # config instead of a clear rejection. Matches the sibling parser in
+    # jenkins/scripts/perf/submit.py, which already raises.
+    if len(labels) <= 1:
+        raise ValueError(f"perf_sanity test must have a config file: {test_case_name}")
+
+    def split_modifiers(rest):
+        """Peel the optional modifier segment off the front of the stem."""
+        time_breakdown = bool(rest) and rest[0] == TIME_BREAKDOWN_MODIFIER
+        if time_breakdown:
+            rest = rest[1:]
+        if not rest:
+            raise ValueError(f"Test name has a modifier but no config: {test_case_name}")
+        return time_breakdown, "-".join(rest)
 
     prefix = labels[0]
     is_disagg_prefix = "disagg" in prefix
     is_aggr_prefix = "aggr" in prefix
+    time_breakdown = False
 
     if is_disagg_prefix:
-        # Disagg format: disagg_upload-{e2e|gen_only}-{config_base}
-        assert len(labels) > 2, "Disagg test must have benchmark_mode and config!"
+        # Disagg format: disagg_upload-{e2e|gen_only}[-{modifier}]-{config_base}
+        if len(labels) <= 2:
+            raise ValueError(f"Disagg test must have benchmark_mode and config: {test_case_name}")
         benchmark_mode = labels[1]  # e2e or gen_only
-        assert benchmark_mode in ("e2e", "gen_only"), (
-            f"Invalid benchmark_mode for disagg: {benchmark_mode}"
-        )
+        if benchmark_mode not in ("e2e", "gen_only"):
+            raise ValueError(f"Invalid benchmark_mode for disagg: {benchmark_mode}")
         runtime_mode = "disaggregated"
-        config_base_name = "-".join(labels[2:])
+        time_breakdown, config_base_name = split_modifiers(labels[2:])
         select_pattern = None
     elif is_aggr_prefix:
         # Check if this is ctx_only (aggr_upload-ctx_only-{config_base})
         if len(labels) > 2 and labels[1] == "ctx_only":
-            # ctx_only: aggr_upload-ctx_only-{config_base}
+            # ctx_only: aggr_upload-ctx_only[-{modifier}]-{config_base}
             # Runs in aggregated mode but reads disagg config
             benchmark_mode = "ctx_only"
             runtime_mode = "aggregated"
-            config_base_name = "-".join(labels[2:])
+            time_breakdown, config_base_name = split_modifiers(labels[2:])
             select_pattern = None
         else:
             # Regular aggr: aggr_upload-config_yml or aggr_upload-config_yml-server_config_name
@@ -136,7 +183,7 @@ def parse_test_string(test_case_name: str):
     else:
         raise ValueError(f"Invalid test name prefix: {prefix}")
 
-    return config_base_name, select_pattern, runtime_mode, benchmark_mode
+    return config_base_name, select_pattern, runtime_mode, benchmark_mode, time_breakdown
 
 
 def get_config_yaml_path(llm_src, config_base_name, benchmark_mode):
@@ -541,18 +588,21 @@ def generate_pytest_command(
     runtime_mode,
     benchmark_mode,
     waives_file="",
+    time_breakdown=False,
 ):
     """Generate pytest command and test list."""
     # Generate test list content based on runtime_mode and benchmark_mode
     if runtime_mode == "disaggregated":
-        # disagg_upload-{e2e|gen_only}-{config_base}
+        # disagg_upload-{e2e|gen_only}[-{modifier}]-{config_base}
+        label = format_test_label(benchmark_mode, time_breakdown)
         test_list_content = (
-            f"perf/test_perf_sanity.py::test_e2e[disagg-{benchmark_mode}-{config_file_base_name}]"
+            f"perf/test_perf_sanity.py::test_e2e[disagg-{label}-{config_file_base_name}]"
         )
     elif benchmark_mode == "ctx_only":
-        # aggr_upload-ctx_only-{config_base}
+        # aggr_upload-ctx_only[-{modifier}]-{config_base}
+        label = format_test_label("ctx_only", time_breakdown)
         test_list_content = (
-            f"perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-{config_file_base_name}]"
+            f"perf/test_perf_sanity.py::test_e2e[aggr-{label}-{config_file_base_name}]"
         )
     else:
         # Normal aggr: aggr-{config}-{select_pattern}
@@ -656,6 +706,13 @@ def main():
         choices=["", "e2e", "gen_only", "ctx_only"],
         help="Benchmark mode for disagg config (when --config-file is provided)",
     )
+    parser.add_argument(
+        "--time-breakdown",
+        action="store_true",
+        help="Record the per-request lifecycle breakdown; adds the "
+        f"'{TIME_BREAKDOWN_MODIFIER}' modifier segment to the generated test id "
+        "(when --config-file is provided)",
+    )
     parser.add_argument(
         "--partition",
         required=True,
@@ -745,9 +802,13 @@ def main():
     # --test-list takes precedence over --config-file
     if args.test_list:
         test_case_name = extract_test_case_name(args.test_list)
-        config_file_base_name, select_pattern, runtime_mode, benchmark_mode = parse_test_string(
-            test_case_name
-        )
+        (
+            config_file_base_name,
+            select_pattern,
+            runtime_mode,
+            benchmark_mode,
+            time_breakdown,
+        ) = parse_test_string(test_case_name)
         config_yaml = get_config_yaml_path(llm_src, config_file_base_name, benchmark_mode)
     elif args.config_file:
         config_yaml = os.path.abspath(args.config_file)
@@ -767,11 +828,24 @@ def main():
             else:
                 runtime_mode = "disaggregated"
             select_pattern = None
+            time_breakdown = args.time_breakdown
+            # Refuse here rather than at collection: the id this would compose
+            # (e.g. `disagg-gen_only-time_breakdown-<stem>`) is well-formed and
+            # parses fine, but test_perf_sanity.py never generates it, so pytest
+            # would exit "no tests ran" after the whole job has been queued,
+            # built and allocated.
+            if time_breakdown and benchmark_mode not in TIME_BREAKDOWN_BENCHMARK_MODES:
+                raise ValueError(
+                    f"--time-breakdown is not supported for --benchmark_mode "
+                    f"{benchmark_mode!r}; supported modes are "
+                    f"{', '.join(TIME_BREAKDOWN_BENCHMARK_MODES)}"
+                )
         else:
             # Aggr config
             runtime_mode = "aggregated"
             benchmark_mode = None
             select_pattern = args.test_name
+            time_breakdown = False
             if not select_pattern:
                 raise ValueError("--test-name is required for aggregated config")
     else:
@@ -784,9 +858,11 @@ def main():
     # would carry `_upload` while test_perf_sanity.py creates its working dir
     # under the stripped form — producing two divergent folders.
     if runtime_mode == "disaggregated":
-        test_case_name = f"disagg-{benchmark_mode}-{config_file_base_name}"
+        label = format_test_label(benchmark_mode, time_breakdown)
+        test_case_name = f"disagg-{label}-{config_file_base_name}"
     elif benchmark_mode == "ctx_only":
-        test_case_name = f"aggr-ctx_only-{config_file_base_name}"
+        label = format_test_label("ctx_only", time_breakdown)
+        test_case_name = f"aggr-{label}-{config_file_base_name}"
     else:
         test_case_name = f"aggr-{config_file_base_name}-{select_pattern}"
 
@@ -867,6 +943,7 @@ def main():
         runtime_mode,
         benchmark_mode,
         waives_file=args.waives_file,
+        time_breakdown=time_breakdown,
     )
 
     # Write test list file