feat(ci): add nightly onboard trace timing summaries

.github/actions/run-e2e-script/action.yaml

@@ -5,7 +5,7 @@ name: Run E2E script
 
 description: >-
   Runs a repository E2E script from an already-checked-out workspace and uploads
-  the configured artifacts when the script fails.
+  configured failure artifacts plus optional sanitized trace summary artifacts.
 
 inputs:
   script:
@@ -21,6 +21,18 @@ inputs:
   artifact-path:
     description: Newline-capable artifact path glob list.
     required: true
+  always-artifact-name:
+    description: Optional sanitized trace artifact name to upload after every script run.
+    required: false
+    default: ""
+  always-artifact-path:
+    description: Optional legacy trace summary path; trusted code writes uploads to RUNNER_TEMP.
+    required: false
+    default: ""
+  always-artifact-trace-source-path:
+    description: Optional target-controlled trace path to reduce to a trusted timing-only summary before upload.
+    required: false
+    default: ""
 
 runs:
   using: composite
@@ -62,3 +74,44 @@ runs:
         name: ${{ inputs.artifact-name }}
         path: ${{ inputs.artifact-path }}
         if-no-files-found: ignore
+
+    - name: Sanitize E2E trace artifacts
+      id: sanitize-trace-artifacts
+      if: always() && inputs.always-artifact-name != '' && inputs.always-artifact-path != '' && inputs.always-artifact-trace-source-path != ''
+      shell: bash
+      working-directory: ${{ inputs.working-directory }}
+      env:
+        E2E_TRACE_SOURCE_PATH: ${{ inputs.always-artifact-trace-source-path }}
+      run: |
+        set -euo pipefail
+        trusted_summary_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/nemoclaw-trace-summary.XXXXXX")"
+        python3 "$GITHUB_ACTION_PATH/sanitize-trace-artifacts.py" \
+          "$E2E_TRACE_SOURCE_PATH" \
+          "$trusted_summary_dir"
+
+        summary_file="$trusted_summary_dir/cloud-onboard-trace-timing-summary.json"
+        if [ -e "$summary_file" ]; then
+          if [ -L "$summary_file" ] || [ ! -f "$summary_file" ]; then
+            echo "::error::Sanitized trace summary must be a regular file: $summary_file" >&2
+            exit 1
+          fi
+
+          unexpected="$(
+            find "$trusted_summary_dir" -mindepth 1 -maxdepth 1 \
+              ! -name cloud-onboard-trace-timing-summary.json -print -quit
+          )"
+          if [ -n "$unexpected" ]; then
+            echo "::error::Unexpected file in trusted trace summary directory: $unexpected" >&2
+            exit 1
+          fi
+        fi
+
+        printf 'summary-file=%s\n' "$summary_file" >> "$GITHUB_OUTPUT"
+
+    - name: Upload E2E artifacts
+      if: always() && inputs.always-artifact-name != '' && inputs.always-artifact-path != '' && inputs.always-artifact-trace-source-path != '' && steps.sanitize-trace-artifacts.outcome == 'success'
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+      with:
+        name: ${{ inputs.always-artifact-name }}
+        path: ${{ steps.sanitize-trace-artifacts.outputs.summary-file }}
+        if-no-files-found: warn

.github/actions/run-e2e-script/sanitize-trace-artifacts.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Build a trusted timing-only trace artifact from target-ref trace output.
+
+The E2E script under test controls NEMOCLAW_TRACE_DIR, so the reusable trusted
+workflow must never upload that raw directory from a secret-bearing job. This
+helper reads candidate NemoClaw trace JSON files, validates the minimal timing
+shape needed by the scorecard, and writes a single allowlisted summary that
+contains no attributes, events, file names, prompts, environment dumps, or raw
+error messages.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+from typing import Any
+
+SCHEMA_VERSION = "nemoclaw.trace_timing.v1"
+OUTPUT_FILE = "cloud-onboard-trace-timing-summary.json"
+ONBOARD_ROOT_SPAN = "nemoclaw.onboard"
+ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase."
+MAX_JSON_FILES = 100
+MAX_JSON_BYTES = 2 * 1024 * 1024
+MAX_SLOWEST_SPANS = 10
+TRACE_ID_RE = re.compile(r"^[0-9a-f]{32}$")
+STATUS_VALUES = {"OK", "ERROR", "UNSET"}
+
+
+def finite_number(value: Any) -> float | None:
+    if isinstance(value, bool):
+        return None
+    try:
+        number = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(number) or number < 0:
+        return None
+    return number
+
+
+def safe_status(value: Any) -> str:
+    return value if isinstance(value, str) and value in STATUS_VALUES else "UNSET"
+
+
+def safe_span_name(value: Any) -> str | None:
+    if not isinstance(value, str):
+        return None
+    if value == ONBOARD_ROOT_SPAN or value.startswith(ONBOARD_PHASE_PREFIX):
+        return value
+    return None
+
+
+def iter_json_files(source: Path) -> list[Path]:
+    if not source.exists():
+        return []
+    if source.is_file():
+        return [source] if source.suffix == ".json" and not source.is_symlink() else []
+    if not source.is_dir() or source.is_symlink():
+        return []
+    files: list[Path] = []
+    for path in sorted(source.rglob("*.json")):
+        if path.is_file() and not path.is_symlink():
+            files.append(path)
+            if len(files) >= MAX_JSON_FILES:
+                break
+    return files
+
+
+def load_json(path: Path) -> Any | None:
+    try:
+        if path.stat().st_size > MAX_JSON_BYTES:
+            return None
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeDecodeError, json.JSONDecodeError):
+        return None
+
+
+def first_dict(values: Any) -> dict[str, Any]:
+    if isinstance(values, list) and values and isinstance(values[0], dict):
+        return values[0]
+    return {}
+
+
+def extract_spans(artifact: Any) -> list[dict[str, Any]]:
+    if not isinstance(artifact, dict):
+        return []
+    resource = first_dict(artifact.get("resource_spans"))
+    scope = first_dict(resource.get("scope_spans"))
+    spans = scope.get("spans", [])
+    return [span for span in spans if isinstance(span, dict)] if isinstance(spans, list) else []
+
+
+def extract_candidate(artifact: Any) -> dict[str, Any] | None:
+    if not isinstance(artifact, dict):
+        return None
+    spans = extract_spans(artifact)
+    if not any(span.get("name") == ONBOARD_ROOT_SPAN for span in spans):
+        return None
+
+    summary = artifact.get("summary") if isinstance(artifact.get("summary"), dict) else {}
+    total_ms = finite_number(summary.get("total_duration_ms"))
+    if total_ms is None:
+        return None
+
+    phases: dict[str, float] = {}
+    for span in spans:
+        name = span.get("name")
+        duration_ms = finite_number(span.get("duration_ms"))
+        if isinstance(name, str) and name.startswith(ONBOARD_PHASE_PREFIX) and duration_ms is not None:
+            phases[name] = phases.get(name, 0.0) + duration_ms
+
+    if not phases:
+        return None
+
+    slowest_spans = []
+    for span in summary.get("slowest_spans", []) if isinstance(summary.get("slowest_spans"), list) else []:
+        if not isinstance(span, dict):
+            continue
+        name = safe_span_name(span.get("name"))
+        duration_ms = finite_number(span.get("duration_ms"))
+        if name is None or duration_ms is None:
+            continue
+        slowest_spans.append(
+            {
+                "name": name,
+                "duration_ms": round(duration_ms, 3),
+                "status": safe_status(span.get("status")),
+            }
+        )
+        if len(slowest_spans) >= MAX_SLOWEST_SPANS:
+            break
+
+    trace_id = summary.get("trace_id")
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "trace_id": trace_id if isinstance(trace_id, str) and TRACE_ID_RE.fullmatch(trace_id) else None,
+        "total_duration_ms": round(total_ms, 3),
+        "phases": {name: round(phases[name], 3) for name in sorted(phases)},
+        "slowest_spans": slowest_spans,
+    }
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 3:
+        print("usage: sanitize-trace-artifacts.py <source-file-or-dir> <output-dir>", file=sys.stderr)
+        return 2
+
+    source_input = Path(argv[1]).absolute()
+    if source_input.is_symlink():
+        print("trace source must not be a symlink", file=sys.stderr)
+        return 2
+
+    source = source_input.resolve(strict=False)
+    output_dir = Path(argv[2]).absolute()
+    if source == output_dir.resolve(strict=False):
+        print("trace source and trusted output directory must be distinct", file=sys.stderr)
+        return 2
+
+    if output_dir.exists() or output_dir.is_symlink():
+        if output_dir.is_symlink():
+            output_dir.unlink()
+        elif output_dir.is_dir():
+            shutil.rmtree(output_dir)
+        else:
+            output_dir.unlink()
+    output_dir.mkdir(parents=True, mode=0o700)
+
+    candidates = []
+    for json_file in iter_json_files(source):
+        artifact = load_json(json_file)
+        candidate = extract_candidate(artifact)
+        if candidate is not None:
+            candidates.append(candidate)
+
+    if not candidates:
+        print("No valid NemoClaw onboard trace found; no timing summary emitted.")
+        return 0
+
+    selected = max(candidates, key=lambda item: item["total_duration_ms"])
+    output = output_dir / OUTPUT_FILE
+    output.write_text(json.dumps(selected, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    os.chmod(output, 0o600)
+    print(f"Wrote trusted trace timing summary: {output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))

.github/workflows/e2e-script.yaml

@@ -32,6 +32,21 @@ on:
         description: Newline-capable failure artifact path glob list.
         required: true
         type: string
+      always_artifact_name:
+        description: Optional sanitized trace artifact name to upload after every script run.
+        required: false
+        type: string
+        default: ""
+      always_artifact_path:
+        description: Optional legacy trace summary path; trusted code writes uploads to RUNNER_TEMP.
+        required: false
+        type: string
+        default: ""
+      always_artifact_trace_source_path:
+        description: Optional target-controlled trace path to reduce to a trusted timing-only summary before upload.
+        required: false
+        type: string
+        default: ""
       env_json:
         description: JSON object of non-secret environment variables for the script.
         required: false
@@ -127,7 +142,8 @@ jobs:
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           ref: ${{ github.ref }}
-          sparse-checkout: .github/actions/run-e2e-script
+          sparse-checkout: |
+            .github/actions/run-e2e-script
           path: workflow-actions
           persist-credentials: false
 
@@ -203,6 +219,9 @@ jobs:
           script: ${{ inputs.script }}
           artifact-name: ${{ inputs.artifact_name }}
           artifact-path: ${{ inputs.artifact_path }}
+          always-artifact-name: ${{ inputs.always_artifact_name }}
+          always-artifact-path: ${{ inputs.always_artifact_path }}
+          always-artifact-trace-source-path: ${{ inputs.always_artifact_trace_source_path }}
         env:
           BRAVE_API_KEY: ${{ inputs.brave_api_key && secrets.BRAVE_API_KEY || '' }}
           GITHUB_TOKEN: ${{ inputs.github_token && github.token || '' }}