ci(atomesh): Add aggregated-service CI, AgentX runtime fixes and Kimi-K3 2P1D/agg cells

.github/benchmark/models_atomesh.yaml

@@ -640,6 +640,8 @@ models:
         # move the published numbers.
         ATOM_USE_FLYDSL_GATHER_KV_B_PROJ: "1"
         ATOM_STATE_CHECKPOINT_DEMAND: "0"
+        # AgentX runs FlyDSL FP8 prefill attention on every band.
+        ATOM_USE_FLYDSL_FP8_PREFILL_ATTN: "1"
         # AgentX band default: ReplaySSM off on CONC 1/2/4, on for the mid band
         # (CONC 8-16), off again from CONC 32. Cases that declare their own env
         # block must restate the key, because a YAML merge key loses to an
@@ -687,7 +689,7 @@ models:
               workers: 1
               tp: 8
               # AgentX pins the compile path and captures the dense query-token
-              # range 2..2*CONC*(1+spec) that `auto` derives per concurrency.
+              # range 1..2*CONC*(1+spec) that `auto` derives per concurrency.
               cudagraph: auto
               cudagraph_mode: FULL
               compilation_level: 3
@@ -772,14 +774,6 @@ models:
             # paged KV instead of a state pool nothing replays from.
             prefill: *kimi_k3_agentic_lmcache128_env
 
-        - <<: *kimi_k3_agentic_no_dspark
-          name: kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c40
-          concurrency: [40]
-          server:
-            common_args:
-              <<: *kimi_k3_agentic_no_dspark_args
-              max_num_seqs: 80
-
         - <<: *kimi_k3_agentic_no_dspark
           name: kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c48
           concurrency: [48]
@@ -823,6 +817,121 @@ models:
               max_num_seqs: 144
           env: *kimi_k3_agentic_lmcache192_env
 
+        - <<: *kimi_k3_agentic_no_dspark
+          name: kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c88
+          concurrency: [88]
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 176
+          env: *kimi_k3_agentic_lmcache192_env
+
+        # Non-disaggregated counterpart to the C72 cell above: one server does
+        # prefill and decode on a single node, same workload and same server
+        # knobs, so the pair isolates what the P/D split itself is worth.
+        - <<: *kimi_k3_agentic_no_dspark
+          name: kimi-k3-mxfp4-agg-tp8-dcp8-agentic-lmcache-1m-c72
+          topology: agg
+          display_topology: AGG-TP8-DCP8
+          # Placement only; `topology: agg` above is what says there is no
+          # prefill/decode split to place. The other single_node cells halve the
+          # node between two services, so this one keeps all 8 GPUs at TP8.
+          pd_worker_layout: single_node
+          concurrency: [72]
+          # Unused here; the aggregated server is configured by the decode block.
+          prefill: {}
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 144
+          env:
+            common:
+              ATOM_ENABLE_REPLAYSSM: "0"
+              AITER_REUSE_IDENTICAL_COMM_GROUPS: "1"
+            # The aggregated server reads env.decode. These are the C72 prefill's
+            # LMCache settings, spelled out rather than merged from its anchor:
+            # that anchor also carries the mooncake half of the `multi`
+            # connector, which has no peer to stream KV to here. What is left is
+            # the same 192 GiB CPU tier behind a standalone offload connector.
+            decode:
+              LMCACHE_LOCAL_CPU: "True"
+              LMCACHE_MAX_LOCAL_CPU_SIZE: "192"
+              LMCACHE_CHUNK_SIZE: "1024"
+              LMCACHE_NUMA_MODE: auto
+              ATOM_NUMA_BIND: "1"
+              ATOM_NUMA_NODE: "0,0,0,0,1,1,1,1"
+              ATOM_AUTO_NUMA_BIND: "0"
+              OFFLOAD_PROFILE: "1"
+              OFFLOAD_GPU_STAGING_CHUNKS: "32"
+              # AgentX throughput band (CONC 16 up): with prefill and decode on
+              # one server, a ready prefill waits four decode passes, and the
+              # PrefillDelayer coalesces queued prefill for up to 5 s.
+              ATOM_PREFILL_DECODE_INTERVAL: "4"
+              ATOM_PREFILL_DELAYER_MAX_QUEUE_MS: "5000"
+              DECODE_KV_TRANSFER_CONFIG: >-
+                {"kv_connector":"lmcache_offload","kv_role":"offload"}
+
+        - &kimi_k3_agentic_2p1d
+          <<: *kimi_k3_agentic_no_dspark
+          name: kimi-k3-mxfp4-2p1d-tp8-dcp8-agentic-lmcache-1m-c72
+          topology: 2p1d
+          display_topology: 2P1D-TP8-DCP8
+          nodes: "${ATOMESH_2P1D_NODES}"
+          concurrency: [72]
+          prefill:
+            workers: 2
+            tp: 8
+            dcp: 8
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 144
+          env: *kimi_k3_agentic_lmcache192_env
+
+        - <<: *kimi_k3_agentic_2p1d
+          name: kimi-k3-mxfp4-2p1d-tp8-dcp8-agentic-lmcache-1m-c128
+          concurrency: [128]
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 256
+
+        - <<: *kimi_k3_agentic_2p1d
+          name: kimi-k3-mxfp4-2p1d-tp8-dcp8-agentic-lmcache-1m-c256
+          concurrency: [256]
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 512
+
+      # Dispatch-only: schedule and push run `nightly`, so nothing here reaches
+      # the published dashboard. The stop_profile export stalls both engines,
+      # which makes this cell's aiperf numbers incomparable with its C72 twin.
+      profile:
+        - <<: *kimi_k3_agentic_no_dspark
+          name: kimi-k3-mxfp4-1p1d-tp8-dcp8-agentic-lmcache-1m-c72-torch-profile
+          concurrency: [72]
+          server:
+            common_args:
+              <<: *kimi_k3_agentic_no_dspark_args
+              max_num_seqs: 144
+          env:
+            common:
+              ATOM_ENABLE_REPLAYSSM: "0"
+              AITER_REUSE_IDENTICAL_COMM_GROUPS: "1"
+              # 30 s starting 900 s into the aiperf Profiling phase, on every
+              # prefill and decode server; traces land under ${RUN_DIR}/torch_traces/<worker>/rank_<tp>/.
+              ATOMESH_TORCH_PROFILE_START: "900"
+              ATOMESH_TORCH_PROFILE_DURATION: "30"
+              ATOMESH_TORCH_PROFILE_ROLES: prefill,decode
+              # Kernel names and durations only; shapes/stacks on a TP8 window
+              # risk OOMing the ranks during export.
+              ATOM_PROFILER_MORE: "0"
+              ATOM_PROFILER_TIMEOUT: "1800"
+            prefill:
+              <<: *kimi_k3_agentic_lmcache128_env
+              LMCACHE_MAX_LOCAL_CPU_SIZE: "192"
+
     accuracy:
       task: gsm8k
       fewshot: 5

.github/dashboard/index.html

@@ -559,12 +559,15 @@ <h1>Benchmark Dashboard</h1>
 }
 
 // process_result.py names the interactivity column after the definition that
-// produced it ("Interactivity (P90 E2E Normalized)"), so the key is not a fixed
-// string. Everything else in the table has a stable header.
+// produced it ("P90 E2E Normalized Interactivity", formerly "Interactivity (P90
+// E2E Normalized)"), so the key is not a fixed string. "P90 Interactivity" is a
+// separate fixed column (1/p90 ITL) and must not be mistaken for it.
 const ATOMESH_INTERACTIVITY_KEY = 'interactivity';
+const ATOMESH_P90_INTERACTIVITY_KEY = 'p90interactivity';
 
 function atomeshSummaryInteractivityKey(headers) {
-  return headers.find(header => header.startsWith(ATOMESH_INTERACTIVITY_KEY))
+  return headers.find(header => header.includes(ATOMESH_INTERACTIVITY_KEY)
+    && header !== ATOMESH_P90_INTERACTIVITY_KEY)
     || ATOMESH_INTERACTIVITY_KEY;
 }
 
@@ -620,6 +623,7 @@ <h1>Benchmark Dashboard</h1>
         concurrency: parseAtomeshNumber(raw.concurrency),
         interactivity,
         interactivityMethod: atomeshInteractivityMethod(raw.intvtydef || headerMethod),
+        interactivityP90Itl: parseAtomeshNumber(raw[ATOMESH_P90_INTERACTIVITY_KEY]),
         totalTput: parseAtomeshNumber(raw.totaltoks),
         inputTput: parseAtomeshNumber(raw.inputtoks),
         outputTput: parseAtomeshNumber(raw.outputtoks),
@@ -1292,17 +1296,17 @@ <h1>Benchmark Dashboard</h1>
 const ATOMESH_INTERACTIVITY_METRICS = [
   {
     key: 'e2e_normalized',
-    label: 'E2E Normalized Interactivity',
+    label: 'P90 E2E Normalized Interactivity',
     formula: '1 / p90(ITL + TTFT / output tokens) — amortizes prefill over the turn',
     emptyHint: 'It is derived from every run, so an empty chart here means the selected rows have no throughput/latency data at all.',
     value: row => row.interactivity,
     frontierEligible: row => row.interactivityMethod === 'p90_e2e_normalized',
   },
   {
     key: 'p90_itl',
-    label: 'Interactivity',
+    label: 'P90 Interactivity',
     formula: '1 / p90(ITL) — decode speed only, matching InferenceX p90_intvty',
-    emptyHint: 'It is recorded only for agentic runs published after this metric was added; older points carry E2E Normalized Interactivity only.',
+    emptyHint: 'It is recorded only for agentic runs published after this metric was added; older points carry P90 E2E Normalized Interactivity only.',
     value: row => row.interactivityP90Itl,
     frontierEligible: row => Number.isFinite(row.interactivityP90Itl),
   },
@@ -3234,13 +3238,13 @@ <h3>${escHTML(model)} Accuracy</h3>
   const isP90 = row.interactivityMethod === 'p90_e2e_normalized';
   // Same wording as the markdown summary's column header, so the two read as
   // the same metric rather than two that happen to share a name.
-  const definition = isP90 ? 'P90 E2E Normalized' : '1 / median_tpot_s';
-  let html = `<div class="pop-row"><span class="pop-key">Interactivity (${definition}) (tok/s/user)</span><span class="pop-val">${atomeshDetailValue(row.interactivity, v => fmtNum(v, 3))}</span></div>`;
+  const definition = isP90 ? 'P90 E2E Normalized Interactivity' : 'Interactivity (1 / median_tpot_s)';
+  let html = `<div class="pop-row"><span class="pop-key">${definition} (tok/s/user)</span><span class="pop-val">${atomeshDetailValue(row.interactivity, v => fmtNum(v, 3))}</span></div>`;
   // Second row only when the plain metric was actually measured: rendering an
   // em dash for every legacy point would suggest a value went missing rather
   // than that the run predates the metric.
   if (Number.isFinite(row.interactivityP90Itl)) {
-    html += `<div class="pop-row"><span class="pop-key">Interactivity p90 (tok/s/user)</span><span class="pop-val">${atomeshDetailValue(row.interactivityP90Itl, v => fmtNum(v, 3))}</span></div>`;
+    html += `<div class="pop-row"><span class="pop-key">P90 Interactivity (tok/s/user)</span><span class="pop-val">${atomeshDetailValue(row.interactivityP90Itl, v => fmtNum(v, 3))}</span></div>`;
   }
   return html;
 }
@@ -4012,7 +4016,7 @@ <h3>${escHTML(spec.model)} · ${spec.seriesCount} selected series</h3>
   html += `<div class="section-container"><div class="section-header"><span class="section-title">Detail Table</span></div>
   <div class="table-watermark"><div class="table-watermark-scroll"><table class="perf-table"><thead><tr>
     <th>Backend</th><th>Model</th><th>Topology</th><th>ISL/OSL</th><th class="num">Conc</th>
-    <th class="num">E2E Norm. Intvty</th><th class="num">Intvty</th><th class="num">Total tok/s</th><th class="num">Input tok/s</th><th class="num">Output tok/s</th>
+    <th class="num">P90 E2E Normalized Interactivity</th><th class="num">P90 Interactivity</th><th class="num">Total tok/s</th><th class="num">Input tok/s</th><th class="num">Output tok/s</th>
     <th class="num">Total tok/s/GPU</th><th class="num">Input tok/s/GPU</th><th class="num">Output tok/s/GPU</th>
     <th class="num">TTFT ms</th><th class="num">TPOT ms</th><th class="num">E2E ms</th><th class="num">Accuracy</th><th>Run</th>
   </tr></thead><tbody>`;

.github/scripts/aiperf_dashboard.py

@@ -5,8 +5,23 @@
 import json
 import os
 import shlex
+import sys
 from pathlib import Path
 
+sys.path.insert(0, str(Path(__file__).resolve().parent / "atomesh"))
+from interactivity import RECORDS_FILENAME, agentic_throughput
+
+
+def records_throughput(src):
+    """InferenceX's records-over-span throughput, or None without usable records."""
+    records = src.parent / RECORDS_FILENAME
+    if not records.is_file():
+        return None
+    try:
+        return agentic_throughput(records)
+    except (OSError, ValueError):
+        return None
+
 
 def dashboard_summary(data, src, conc, env, *, single_node=False):
     def avg(name):
@@ -30,6 +45,9 @@ def total_tokens(name):
     # its internal warmup and requests cancelled during grace-period draining.
     cache_hit_tokens = total_tokens("total_usage_prompt_cache_read_tokens")
     cache_total_tokens = total_tokens("total_usage_prompt_tokens")
+    # Prefer the per-request records so the throughput matches InferenceX;
+    # AIPerf's time-sliced aggregates remain the fallback.
+    throughput = records_throughput(src) or {}
 
     payload = {
         "benchmark_backend": "atom",
@@ -46,6 +64,14 @@ def total_tokens(name):
         "public_dataset": env.get("AIPERF_PUBLIC_DATASET"),
         "topology": env.get("TOPOLOGY") or data.get("topology"),
         "display_topology": env.get("DISPLAY_TOPOLOGY") or data.get("display_topology"),
+        # Whether a prefill/decode split produced these numbers. The launcher knows
+        # it for certain, and the dashboard needs it to size the cell: an
+        # aggregated run puts both phases on one server's GPUs, so its GPU count is
+        # that server's, not a prefill plus decode sum. Callers that are not the
+        # P/D launcher leave AGGREGATED unset, and the field stays absent.
+        "disaggregated": (
+            env.get("AGGREGATED") != "1" if "AGGREGATED" in env else None
+        ),
         "precision": env.get("PRECISION") or data.get("precision"),
         "random_input_len": int(
             data.get("max_context_length") or env.get("AIPERF_MAX_CONTEXT_LENGTH") or 0
@@ -66,9 +92,13 @@ def total_tokens(name):
         "median_e2el_ms": pct("request_latency", "p50"),
         "p90_e2el_ms": pct("request_latency", "p90"),
         "p99_e2el_ms": pct("request_latency", "p99"),
-        "input_throughput": avg("input_token_throughput"),
-        "output_throughput": avg("output_token_throughput"),
-        "total_token_throughput": avg("total_token_throughput"),
+        "input_throughput": throughput.get("input_tput_tps")
+        or avg("input_token_throughput"),
+        "output_throughput": throughput.get("output_tput_tps")
+        or avg("output_token_throughput"),
+        "total_token_throughput": throughput.get("total_tput_tps")
+        or avg("total_token_throughput"),
+        "throughput_window_s": throughput.get("duration_s"),
         "successful_requests": avg("request_count"),
         "completed": avg("request_count"),
         "benchmark_duration_s": avg("benchmark_duration")

.github/scripts/atomesh/interactivity.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Per-request interactivity metrics for ATOMesh agentic (AIPerf) runs.
+"""Per-request interactivity and throughput metrics for ATOMesh agentic (AIPerf) runs.
 
 Mirrors ``compute_p90_e2e_normalized_interactivity.py`` from
 seungrokj/agentx_skills, extended to also produce the plain definition, so the
@@ -50,6 +50,7 @@
 TPOT_KEY = "inter_token_latency"
 TTFT_KEY = "time_to_first_token"
 OSL_KEY = "output_sequence_length"
+ISL_KEY = "input_sequence_length"
 
 PROFILING_PHASE = "profiling"
 
@@ -179,6 +180,58 @@ def agentic_interactivity(
     }
 
 
+def agentic_throughput(jsonl_path: Path | str) -> dict[str, Any]:
+    """Token throughput as InferenceX computes it for one ``profile_export.jsonl``.
+
+    Mirrors ``compute_throughput_stats`` in InferenceX's
+    ``infx/results/agentic/request_metrics.py``: the summed ISL and OSL of the
+    successful profiling records (a record without a phase counts as profiling)
+    over the span from the first request start to the last request end. AIPerf's
+    own ``*_token_throughput`` attributes tokens to time slices inside the
+    benchmark window instead, which reads several percent lower on long
+    agentic turns, so the two cannot be compared with each other.
+
+    Raises ValueError when no record carries a usable start/end span.
+    """
+    path = Path(jsonl_path)
+    total_input = 0
+    total_output = 0
+    first_start_ns: int | None = None
+    last_end_ns: int | None = None
+    for record, _raw in iter_records(path):
+        if record is None or record.get("error"):
+            continue
+        metadata = record.get("metadata")
+        if not isinstance(metadata, dict):
+            metadata = {}
+        phase = metadata.get("benchmark_phase")
+        if phase is not None and phase != PROFILING_PHASE:
+            continue
+        isl = _metric(record, ISL_KEY)
+        osl = _metric(record, OSL_KEY)
+        if isl is not None:
+            total_input += int(isl)
+        if osl is not None:
+            total_output += int(osl)
+        if metadata.get("request_start_ns"):
+            start_ns = int(metadata["request_start_ns"])
+            first_start_ns = (
+                start_ns if first_start_ns is None else min(first_start_ns, start_ns)
+            )
+        if metadata.get("request_end_ns"):
+            end_ns = int(metadata["request_end_ns"])
+            last_end_ns = end_ns if last_end_ns is None else max(last_end_ns, end_ns)
+    if first_start_ns is None or last_end_ns is None or last_end_ns <= first_start_ns:
+        raise ValueError(f"no profiling request span in {path}")
+    duration_s = (last_end_ns - first_start_ns) / 1e9
+    return {
+        "input_tput_tps": total_input / duration_s,
+        "output_tput_tps": total_output / duration_s,
+        "total_tput_tps": (total_input + total_output) / duration_s,
+        "duration_s": duration_s,
+    }
+
+
 def locate_records(
     result_path: Path,
     model: str | None = None,

.github/scripts/atomesh/observability/README.md

@@ -110,6 +110,10 @@ For multi-node DP, include every node's `--server-port` as a target (repeat
 `--prefill` / `--decode`); native histogram storage is local to each node.
 The coordinator's endpoint does not include remote histogram observations.
 
+An aggregated topology runs one server for both phases and no router, so it
+passes `--standalone <address>` instead of the `--prefill` / `--decode` pair and
+omits `--mesh`. Its report uses the `standalone` deployment panels.
+
 Prometheus 3.5.0 is downloaded and verified against its release checksums when
 no executable is already available. `ATOMESH_PROMETHEUS_BIN` can point to an
 installed executable. `ATOMESH_MESH_TARGET_DIR` selects the host's persistent