adibarra · Oct 17, 2026 · Oct 7, 2026 · Oct 7, 2026 · Oct 8, 2026 · Oct 9, 2026
@@ -447,6 +447,15 @@ jobs:
             ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log
           if-no-files-found: ignore
 
+      - name: Upload job event
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: job_event_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
+          path: ${{ env.INFERENCEX_E2E_ROOT }}/job_event.json
+          if-no-files-found: ignore
+
       - name: Upload agentic aggregated result
         id: upload-agentic-aggregated
         if: ${{ always() && !inputs.eval-only && inputs.scenario-type == 'agentic-coding' }}

@@ -454,6 +454,15 @@ jobs:
             ${{ env.INFERENCEX_E2E_ROOT }}/srt-setup.log
           if-no-files-found: ignore
 
+      - name: Upload job event
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: job_event_${{ env.RESULT_FILENAME }}${{ inputs.eval-only && format('_{0}_{1}', env.EVAL_FRAMEWORK, env.EVAL_SUITE) || '' }}
+          path: ${{ env.INFERENCEX_E2E_ROOT }}/job_event.json
+          if-no-files-found: ignore
+
       - name: Upload eval results (if any)
         id: upload-eval
         if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}

@@ -0,0 +1,45 @@
+name: Template - Collect Job Events
+
+on:
+  workflow_call:
+    inputs:
+      tooling-ref:
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  collect-job-events:
+    name: collect-job-events
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ inputs.tooling-ref }}
+          persist-credentials: false
+
+      # Job records are diagnostics: they never fail the run.
+      - name: Download job events
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          path: job_events/
+          pattern: job_event_*
+
+      - name: Collect job events
+        continue-on-error: true
+        env:
+          PYTHONPATH: ${{ github.workspace }}/inferencex-e2e
+        run: python3 -m infx.results.collect_events job_events/ events.jsonl
+
+      - name: Upload job events
+        continue-on-error: true
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: job_events
+          path: events.jsonl
+          if-no-files-found: ignore
@@ -556,6 +556,14 @@ jobs:
         with:
             tooling-ref: ${{ needs.get-jobs.outputs.tooling-ref }}
 
+    collect-job-events:
+        name: collect-job-events
+        needs: [get-jobs, test-sweep-multi-node, test-sweep-multi-node-evals, test-sweep-agentic, test-sweep-agentic-evals, test-sweep-multi-node-agentic, test-sweep-multi-node-agentic-evals, test-sweep-single-node, test-sweep-evals]
+        if: ${{ !cancelled() && needs.get-jobs.result == 'success' }}
+        uses: $/.github/workflows/collect-job-events.yml
+        with:
+            tooling-ref: ${{ needs.get-jobs.outputs.tooling-ref }}
+
     calc-success-rate:
         name: calc-success-rate
         needs: [get-jobs, collect-results, collect-evals]

@@ -767,6 +767,29 @@ jobs:
         with:
             tooling-ref: ${{ needs.setup.outputs.tooling-ref }}
 
+    collect-job-events:
+        name: collect-job-events
+        needs:
+            [
+                setup,
+                canary-sweep,
+                canary-multi-node-sweep,
+                sweep-single-node-1k1k,
+                sweep-single-node-8k1k,
+                sweep-agentic,
+                sweep-multi-node-1k1k,
+                sweep-multi-node-8k1k,
+                sweep-multi-node-agentic,
+                sweep-evals,
+                sweep-agentic-evals,
+                sweep-multi-node-evals,
+                sweep-multi-node-agentic-evals,
+            ]
+        if: ${{ !cancelled() && needs.setup.result == 'success' }}
+        uses: $/.github/workflows/collect-job-events.yml
+        with:
+            tooling-ref: ${{ needs.setup.outputs.tooling-ref }}
+
     upload-changelog-metadata:
         name: upload-changelog-metadata
         needs: [setup, collect-results]

diff --git a/inferencex-e2e/docs/troubleshooting.md b/inferencex-e2e/docs/troubleshooting.md
@@ -6,6 +6,7 @@ Classify a failure by the first layer that did not establish its contract. Prese
 
 - [Sources of truth](#sources-of-truth)
 - [Evidence before remediation](#evidence-before-remediation)
+- [Job event records](#job-event-records)
 - [Failure-layer matrix](#failure-layer-matrix)
 - [Changelog and matrix](#changelog-and-matrix)
 - [Runner and AMD root files](#runner-and-amd-root-files)
@@ -35,6 +36,31 @@ Before rerunning or editing, record:
 
 Do not rerun first: reruns can replace logs, change runner/node placement, or make a deterministic failure look transient.
 
+## Job event records
+
+Every `python -m infx.launch run` leaves one `job_event.json` in the job workspace when it exits, on success or failure. The benchmark templates upload it as `job_event_<RESULT_FILENAME>` (eval-only jobs append `_<framework>_<suite>`, as their server logs do). After every benchmark and eval job in the run has finished, [`collect-job-events.yml`](../../.github/workflows/collect-job-events.yml) joins the records into `events.jsonl` in the `job_events` artifact; a cancelled run has only the per-job artifacts. On GitHub Actions, a failed launch also adds one `::error` annotation titled with the failing stage. Read a job's record before its logs; [`infx/launch/event.py`](../infx/launch/event.py) is the schema.
+
+| Fields | Meaning |
+| --- | --- |
+| `run_id`, `run_attempt`, `runner`, `result_filename`, `exp_name` | Workflow attempt, physical runner, and the job identity used in artifact names |
+| `model`, `model_prefix`, `framework`, `precision`, `spec_decoding`, `scenario`, `multinode`, `conc`, `conc_list`, `isl`, `osl` | Point identity |
+| `image`, `recipe`, `recipe_fingerprint`, `cluster`, `slurm_job_id`, `batch_job_id`, `nodes` | What ran where: the selected recipe variant, the benchmark job, the batch allocation around it (batch lanes only), and its hosts |
+| `launch_path`, `power`, `golden_acceptance_length`, `eval_only`, `run_eval`, `kv_offloading`, `kv_offload_backend` | Launch decisions |
+| `started_at`, `duration_s`, `stages` | Seconds per stage: `prepare`, `submit`, `queue_wait` (until the job starts), `run`, `collect`; a batch lane's allocation adds `batch_*` stages |
+| `outcome`, `error`, `artifacts` | `success` or `failure`; for a failure, the first error's `stage`, `type`, `message`, launcher `exit_code`, `retriable`, and `evidence` (the log to read first); the workspace files the launch wrote |
+
+`retriable` marks infrastructure failures (Slurm or image errors, lost nodes, interruptions) that a rerun may clear. It is a hint, not a diagnosis.
+
+```bash
+gh run download <run-id> -n job_events
+# Failed jobs: stage, error type, job, and the log to open first
+jq -r 'select(.outcome == "failure") | [.error.stage, .error.type, .result_filename, .error.evidence] | @tsv' events.jsonl
+# Failures per cluster, and how many look like infrastructure
+jq -s 'map(select(.outcome == "failure")) | group_by(.cluster) | map({cluster: .[0].cluster, failures: length, retriable: map(select(.error.retriable)) | length})' events.jsonl
+# Ten longest queue waits
+jq -s 'sort_by(-(.stages.queue_wait // 0)) | .[:10] | map({result_filename, cluster, queue_wait: .stages.queue_wait})' events.jsonl
+```
+
 ## Failure-layer matrix
 
 | First failing layer | Typical symptom | Safe evidence | First safe action |

@@ -4,6 +4,7 @@
 
 import io
 import os
+import re
 import shlex
 import signal
 import subprocess
@@ -17,17 +18,27 @@
     from typing_extensions import Self
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
+_SECRET_NAME = re.compile(r"TOKEN|SECRET", re.IGNORECASE)
 
 
 def status(returncode: int) -> int:
     """Shell-style status: death by signal N is 128 + N."""
     return 128 - returncode if returncode < 0 else returncode
 
 
-def echo(argv: Sequence[str]) -> None:
-    """Print ``argv`` like a ``set -x`` shell line, after any pending output."""
+def echo(argv: Sequence[str | os.PathLike[str]], env: Mapping[str, str] | None = None) -> None:
+    """Print ``+ <argv>`` to stderr, masking the values of ``*TOKEN*`` and ``*SECRET*`` variables."""
+    text = shlex.join(map(os.fspath, argv))
+    secrets = {
+        value
+        for source in (os.environ, env or {})
+        for name, value in source.items()
+        if _SECRET_NAME.search(name) and len(value) >= 4
+    }
+    for value in sorted(secrets, key=len, reverse=True):
+        text = text.replace(value, "***")
     sys.stdout.flush()
-    print(f"+ {shlex.join(argv)}", file=sys.stderr, flush=True)
+    print(f"+ {text}", file=sys.stderr, flush=True)
 
 
 def call(
@@ -38,7 +49,7 @@ def call(
     timeout: float | None = None,
 ) -> int:
     """Run ``argv`` echoed like ``set -x``; 127 if it cannot start, 124 past ``timeout``."""
-    echo(argv)
+    echo(argv, env)
     try:
         return status(
             subprocess.run(argv, env=env, cwd=cwd, timeout=timeout, check=False).returncode
@@ -53,7 +64,7 @@ def call(
 
 def tee(argv: Sequence[str], log: Path, env: Mapping[str, str] | None = None) -> int:
     """``argv 2>&1 | tee log``; return argv's status."""
-    echo(argv)
+    echo(argv, env)
     with (
         log.open("wb") as sink,
         subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) as child,

@@ -1,7 +1,8 @@
 """Run the benchmark point the workflow environment describes, or clean up after earlier ones.
 
-``run`` exits with the point's return code (128 + N after signal N); ``cleanup`` has the
-runner's cluster backend remove what earlier launches on the runner left behind.
+``run`` exits with the point's return code (128 + N after signal N) and leaves its record,
+``job_event.json``, in the workspace; ``cleanup`` has the runner's cluster backend remove what
+earlier launches on the runner left behind.
 """
 
 from __future__ import annotations
@@ -18,28 +19,43 @@
 from infx.launch.backends import BACKENDS, backend_class
 from infx.launch.backends.base import BackendError
 from infx.launch.context import LaunchError
+from infx.launch.event import JobEventBuilder
 from infx.launch.lifecycle import Lifecycle
 from infx.launch.request import LaunchRequest, RequestError
 
 
-def launch(cluster: Cluster, request: LaunchRequest) -> int:
+def launch(cluster: Cluster, request: LaunchRequest, event: JobEventBuilder | None = None) -> int:
     """Run ``request`` on ``cluster``; a launch error is one ``ERROR:`` line, not a traceback."""
-    with Lifecycle() as life:
+    with Lifecycle(event) as life:
         try:
-            life.record(drivers.run(cluster, request, life))
+            rc = drivers.run(cluster, request, life)
         except (LaunchError, BackendError, RequestError) as error:
-            print(f"ERROR: {error}", file=sys.stderr)
-            life.record(1)
+            life.event.error(error)
+            rc = 1
+        life.event.exited(rc)
+        life.record(rc)
     return life.returncode
 
 
 def _run(runner_config: Path | None) -> int:
+    event = JobEventBuilder.begin(os.environ)
+    rc = 1
     try:
         request = LaunchRequest.from_env()
+        cluster = resolve_cluster(request.runner_name, runner_config)
+        event.set(cluster=cluster.id)
+        rc = launch(cluster, request, event)
     except RequestError as error:
-        print(f"ERROR: {error}", file=sys.stderr)
-        return 1
-    return launch(resolve_cluster(request.runner_name, runner_config), request)
+        event.error(error)
+    except SystemExit as stop:
+        rc = 0 if stop.code is None else stop.code if isinstance(stop.code, int) else 1
+        raise
+    except BaseException as error:
+        event.error(error, report=False)
+        raise
+    finally:
+        event.emit(rc)
+    return rc
 
 
 def _cleanup(runner_config: Path | None) -> int:

@@ -80,7 +80,7 @@ def copy_fixed_sequence_results(logs_dir: Path, workspace: Path, result_filename
     """Copy srt-slurm's ``results_concurrency_*.json`` points under bounded workspace names."""
     subdirs = _result_subdirs(logs_dir)
     if not subdirs:
-        _say(f"Warning: No result subdirectories found in {logs_dir}")
+        _warn(f"WARNING: no result subdirectories found in {logs_dir}")
     for subdir in subdirs:
         _say(f"Processing result subdirectory: {subdir}")
         config_name = subdir.name

@@ -12,7 +12,7 @@
 from dataclasses import dataclass, field
 from enum import StrEnum
 from pathlib import Path, PurePosixPath
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
 
 if TYPE_CHECKING:
     from infx.clusters import Cluster
@@ -24,6 +24,8 @@
 class BackendError(RuntimeError):
     """A scheduler, image, or storage operation failed; reported without a traceback."""
 
+    retriable: ClassVar[bool] = True
+
 
 class JobState(StrEnum):
     PENDING = "pending"
@@ -40,11 +42,13 @@ class JobStatus:
 
     ``exit_code`` is the shell-style status (128 + N after signal N) when the scheduler
     reports one. Only ``state`` decides success: Slurm reports a cancelled job as ``0:0``.
+    ``nodes`` names the hosts the job ran on, in the scheduler's notation, when it says.
     """
 
     state: JobState
     raw: str
     exit_code: int | None = None
+    nodes: str | None = None
 
     @property
     def succeeded(self) -> bool: