From 1a13cc9e7c9a14ff6214719205c01acedc7eed26 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sun, 26 Apr 2026 01:09:47 -0500 Subject: [PATCH] Add weekly parse attribution and keepalive NDJSON repair --- .github/workflows/agents-keepalive-loop.yml | 2 +- .github/workflows/agents-weekly-metrics.yml | 2 + scripts/aggregate_agent_metrics.py | 208 +++++++++++++++++- .../workflows/agents-81-gate-followups.yml | 2 +- .../workflows/agents-weekly-metrics.yml | 2 + .../scripts/aggregate_agent_metrics.py | 208 +++++++++++++++++- tests/scripts/test_aggregate_agent_metrics.py | 77 ++++++- .../test_workflow_agents_consolidation.py | 14 ++ 8 files changed, 490 insertions(+), 25 deletions(-) diff --git a/.github/workflows/agents-keepalive-loop.yml b/.github/workflows/agents-keepalive-loop.yml index c0a60afaf..6cb71c106 100644 --- a/.github/workflows/agents-keepalive-loop.yml +++ b/.github/workflows/agents-keepalive-loop.yml @@ -914,7 +914,7 @@ jobs: tasks_completed=$(( tasks_total - tasks_unchecked )) if [ "$tasks_completed" -lt 0 ]; then tasks_completed=0; fi - metrics_json=$(jq -n \ + metrics_json=$(jq -cn \ --arg pr "${PR_NUMBER:-0}" \ --arg iteration "${ITERATION:-0}" \ --arg action "${ACTION:-}" \ diff --git a/.github/workflows/agents-weekly-metrics.yml b/.github/workflows/agents-weekly-metrics.yml index 5740a92ba..7efa3181e 100644 --- a/.github/workflows/agents-weekly-metrics.yml +++ b/.github/workflows/agents-weekly-metrics.yml @@ -124,6 +124,7 @@ jobs: env: METRICS_DIR: artifacts OUTPUT_PATH: agent-weekly-metrics.md + OUTPUT_JSON_PATH: agent-weekly-metrics.json run: | python scripts/aggregate_agent_metrics.py @@ -236,6 +237,7 @@ jobs: name: agent-weekly-metrics path: | agent-weekly-metrics.md + agent-weekly-metrics.json artifacts/metric-artifacts-selection.json artifacts/metric-artifacts-selection.md terminal-disposition-coverage.json diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index fa8936e2e..a4b43f88c 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -6,16 +6,61 @@ import datetime as _dt import json import os +import re import sys from collections import Counter from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path from typing import Any _DEFAULT_METRICS_DIR = "agent-metrics" _DEFAULT_OUTPUT = "agent-metrics-summary.md" +_DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "2026-04-26T04:25:00Z" +_EXACT_ARTIFACT_FAMILIES = { + "keepalive-metrics", + "agents-autofix-metrics", + "agents-verifier-metrics", + "agents-verifier-disposition-metrics", +} +_PREFIXED_ARTIFACT_FAMILIES = ( + "autopilot-metrics-", + "issue-optimizer-metrics-", + "issue-intake-format-metrics-", + "verifier-terminal-disposition-", + "review-thread-terminal-disposition-", +) +_PATTERNED_ARTIFACT_FAMILIES = ( + ( + "bot-comment-auth-coverage-wrapper", + re.compile(r"^bot-comment-auth-coverage-wrapper(?:-[A-Za-z0-9][A-Za-z0-9._-]*)?$"), + ), + ( + "bot-comment-auth-coverage-reusable", + re.compile(r"^bot-comment-auth-coverage-reusable(?:-[A-Za-z0-9][A-Za-z0-9._-]*)?$"), + ), +) +_MAX_PARSE_ERROR_ROWS = 25 + + +@dataclass(frozen=True) +class ParseErrorDetail: + path: str + artifact: str + artifact_family: str + line: int | None + reason: str + + def as_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "artifact": self.artifact, + "artifact_family": self.artifact_family, + "line": self.line, + "reason": self.reason, + } def _parse_timestamp(value: Any) -> _dt.datetime | None: @@ -52,29 +97,82 @@ def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Pa return sorted(path for path in root.rglob("*.ndjson") if path.is_file()) -def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], int]: +def _artifact_family(artifact: str) -> str: + if artifact in _EXACT_ARTIFACT_FAMILIES: + return artifact + for family, pattern in _PATTERNED_ARTIFACT_FAMILIES: + if pattern.match(artifact): + return family + for prefix in _PREFIXED_ARTIFACT_FAMILIES: + if artifact.startswith(prefix): + return prefix.rstrip("-") + return "unknown" + + +def _infer_artifact_name(path: Path) -> str: + parts = path.parts + for index, part in enumerate(parts): + if part == "agent-metrics" and index > 0: + return parts[index - 1] + if path.parent.name: + return path.parent.name + return "unknown" + + +def _parse_error_detail(path: Path, line: int | None, reason: str) -> ParseErrorDetail: + artifact = _infer_artifact_name(path) + return ParseErrorDetail( + path=path.as_posix(), + artifact=artifact, + artifact_family=_artifact_family(artifact), + line=line, + reason=reason, + ) + + +def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[ParseErrorDetail]]: entries: list[dict[str, Any]] = [] - errors = 0 + errors: list[ParseErrorDetail] = [] for path in files: try: handle = path.open("r", encoding="utf-8") except OSError: - errors += 1 + errors.append(_parse_error_detail(path, None, "unreadable-file")) continue + file_entries: list[dict[str, Any]] = [] + file_errors: list[ParseErrorDetail] = [] + raw_lines: list[str] = [] with handle: - for line in handle: + for line_number, line in enumerate(handle, start=1): raw = line.strip() if not raw: continue + raw_lines.append(raw) try: parsed = json.loads(raw) except json.JSONDecodeError: - errors += 1 + file_errors.append(_parse_error_detail(path, line_number, "invalid-json")) continue if isinstance(parsed, dict): - entries.append(parsed) + file_entries.append(parsed) else: - errors += 1 + file_errors.append(_parse_error_detail(path, line_number, "non-object-json")) + if file_errors and not file_entries and raw_lines: + try: + parsed_file = json.loads("\n".join(raw_lines)) + except json.JSONDecodeError: + pass + else: + if isinstance(parsed_file, dict): + file_entries.append(parsed_file) + file_errors = [] + elif isinstance(parsed_file, list) and all( + isinstance(item, dict) for item in parsed_file + ): + file_entries.extend(parsed_file) + file_errors = [] + entries.extend(file_entries) + errors.extend(file_errors) return entries, errors @@ -433,7 +531,52 @@ def _format_rate(numerator: int, denominator: int) -> str: return f"{rate:.1f}% ({numerator}/{denominator})" -def build_summary(entries: list[dict[str, Any]], errors: int) -> str: +def _format_parse_error_details(parse_error_details: list[ParseErrorDetail]) -> list[str]: + if not parse_error_details: + return [] + family_counts = Counter(detail.artifact_family for detail in parse_error_details) + artifact_counts = Counter(detail.artifact for detail in parse_error_details) + lines = [ + "", + "## Parse Error Details", + f"- By artifact family: {_format_counter(family_counts)}", + f"- By artifact: {_format_counter(artifact_counts)}", + "", + "| Artifact family | Artifact | File | Line | Reason |", + "|-----------------|----------|------|------|--------|", + ] + for detail in parse_error_details[:_MAX_PARSE_ERROR_ROWS]: + line = str(detail.line) if detail.line is not None else "n/a" + lines.append( + "| " + f"{detail.artifact_family} | {detail.artifact} | {detail.path} | {line} | " + f"{detail.reason} |" + ) + remaining = len(parse_error_details) - _MAX_PARSE_ERROR_ROWS + if remaining > 0: + lines.append("") + lines.append(f"- Additional parse errors omitted from table: {remaining}") + return lines + + +def _parse_error_contract(parse_error_details: list[ParseErrorDetail]) -> dict[str, Any]: + family_counts = Counter(detail.artifact_family for detail in parse_error_details) + artifact_counts = Counter(detail.artifact for detail in parse_error_details) + reason_counts = Counter(detail.reason for detail in parse_error_details) + return { + "count": len(parse_error_details), + "by_artifact_family": dict(sorted(family_counts.items())), + "by_artifact": dict(sorted(artifact_counts.items())), + "by_reason": dict(sorted(reason_counts.items())), + "details": [detail.as_dict() for detail in parse_error_details], + } + + +def build_summary( + entries: list[dict[str, Any]], + errors: int, + parse_error_details: list[ParseErrorDetail] | None = None, +) -> str: buckets: dict[str, list[dict[str, Any]]] = { "keepalive": [], "autofix": [], @@ -479,6 +622,9 @@ def build_summary(entries: list[dict[str, Any]], errors: int) -> str: latest = max(timestamps).isoformat().replace("+00:00", "Z") lines.append(f"Range: {earliest} to {latest}") + if parse_error_details: + lines.extend(_format_parse_error_details(parse_error_details)) + lines.extend( [ "", @@ -563,25 +709,67 @@ def build_summary(entries: list[dict[str, Any]], errors: int) -> str: return "\n".join(lines) + "\n" +def build_summary_contract( + entries: list[dict[str, Any]], parse_error_details: list[ParseErrorDetail] +) -> dict[str, Any]: + buckets: dict[str, int] = Counter(_classify_entry(entry) for entry in entries) + timestamps: list[_dt.datetime] = [] + for entry in entries: + for key in ("timestamp", "created_at", "time", "run_started_at"): + ts = _parse_timestamp(entry.get(key)) + if ts is not None: + timestamps.append(ts) + break + generated_at = ( + _dt.datetime.now(_dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + ) + contract: dict[str, Any] = { + "schema": "workflows-agent-metrics-summary/v1", + "generated_at": generated_at, + "record_count": len(entries), + "record_buckets": dict(sorted(buckets.items())), + "parse_errors": _parse_error_contract(parse_error_details), + } + if timestamps: + contract["range"] = { + "earliest": min(timestamps).isoformat().replace("+00:00", "Z"), + "latest": max(timestamps).isoformat().replace("+00:00", "Z"), + } + return contract + + def main() -> int: metrics_paths_raw = os.environ.get("METRICS_PATHS", "") metrics_paths = [item.strip() for item in metrics_paths_raw.split(",") if item.strip()] metrics_dir = os.environ.get("METRICS_DIR", _DEFAULT_METRICS_DIR) output_path = Path(os.environ.get("OUTPUT_PATH", _DEFAULT_OUTPUT)) + output_json_path = Path(os.environ.get("OUTPUT_JSON_PATH", _DEFAULT_JSON_OUTPUT)) files = _gather_metrics_files(metrics_paths, metrics_dir) if not files: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text("No metrics files found to aggregate.\n", encoding="utf-8") + output_json_path.parent.mkdir(parents=True, exist_ok=True) + output_json_path.write_text( + json.dumps(build_summary_contract([], []), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) print("No metrics files found to aggregate.", file=sys.stderr) return 0 - entries, errors = _read_ndjson(files) - summary = build_summary(entries, errors) + entries, parse_error_details = _read_ndjson(files) + summary = build_summary(entries, len(parse_error_details), parse_error_details) + summary_contract = build_summary_contract(entries, parse_error_details) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(summary, encoding="utf-8") + output_json_path.parent.mkdir(parents=True, exist_ok=True) + output_json_path.write_text( + json.dumps(summary_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) print(f"Wrote metrics summary to {output_path}") + print(f"Wrote metrics summary JSON to {output_json_path}") return 0 diff --git a/templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml b/templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml index 67848dc4d..8072c18c7 100644 --- a/templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml +++ b/templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml @@ -402,7 +402,7 @@ jobs: tasks_completed=$(( tasks_total - tasks_unchecked )) if [ "$tasks_completed" -lt 0 ]; then tasks_completed=0; fi - metrics_json=$(jq -n \ + metrics_json=$(jq -cn \ --arg pr "${PR_NUMBER:-0}" \ --arg iteration "${ITERATION:-0}" \ --arg action "${ACTION:-}" \ diff --git a/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml b/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml index ad4ea6256..9b0531ae4 100644 --- a/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml +++ b/templates/consumer-repo/.github/workflows/agents-weekly-metrics.yml @@ -135,6 +135,7 @@ jobs: env: METRICS_DIR: artifacts OUTPUT_PATH: agent-weekly-metrics.md + OUTPUT_JSON_PATH: agent-weekly-metrics.json run: | python scripts/aggregate_agent_metrics.py @@ -228,6 +229,7 @@ jobs: name: agent-weekly-metrics path: | agent-weekly-metrics.md + agent-weekly-metrics.json artifacts/metric-artifacts-selection.json artifacts/metric-artifacts-selection.md terminal-disposition-coverage.json diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index fa8936e2e..a4b43f88c 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -6,16 +6,61 @@ import datetime as _dt import json import os +import re import sys from collections import Counter from collections.abc import Iterable +from dataclasses import dataclass from pathlib import Path from typing import Any _DEFAULT_METRICS_DIR = "agent-metrics" _DEFAULT_OUTPUT = "agent-metrics-summary.md" +_DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json" _DEFAULT_UNSUPPORTED_VERIFIER_MODELS = {"gpt-5.2-codex"} _DEFAULT_VERIFIER_MODEL_METADATA_REQUIRED_AFTER = "2026-04-26T04:25:00Z" +_EXACT_ARTIFACT_FAMILIES = { + "keepalive-metrics", + "agents-autofix-metrics", + "agents-verifier-metrics", + "agents-verifier-disposition-metrics", +} +_PREFIXED_ARTIFACT_FAMILIES = ( + "autopilot-metrics-", + "issue-optimizer-metrics-", + "issue-intake-format-metrics-", + "verifier-terminal-disposition-", + "review-thread-terminal-disposition-", +) +_PATTERNED_ARTIFACT_FAMILIES = ( + ( + "bot-comment-auth-coverage-wrapper", + re.compile(r"^bot-comment-auth-coverage-wrapper(?:-[A-Za-z0-9][A-Za-z0-9._-]*)?$"), + ), + ( + "bot-comment-auth-coverage-reusable", + re.compile(r"^bot-comment-auth-coverage-reusable(?:-[A-Za-z0-9][A-Za-z0-9._-]*)?$"), + ), +) +_MAX_PARSE_ERROR_ROWS = 25 + + +@dataclass(frozen=True) +class ParseErrorDetail: + path: str + artifact: str + artifact_family: str + line: int | None + reason: str + + def as_dict(self) -> dict[str, Any]: + return { + "path": self.path, + "artifact": self.artifact, + "artifact_family": self.artifact_family, + "line": self.line, + "reason": self.reason, + } def _parse_timestamp(value: Any) -> _dt.datetime | None: @@ -52,29 +97,82 @@ def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Pa return sorted(path for path in root.rglob("*.ndjson") if path.is_file()) -def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], int]: +def _artifact_family(artifact: str) -> str: + if artifact in _EXACT_ARTIFACT_FAMILIES: + return artifact + for family, pattern in _PATTERNED_ARTIFACT_FAMILIES: + if pattern.match(artifact): + return family + for prefix in _PREFIXED_ARTIFACT_FAMILIES: + if artifact.startswith(prefix): + return prefix.rstrip("-") + return "unknown" + + +def _infer_artifact_name(path: Path) -> str: + parts = path.parts + for index, part in enumerate(parts): + if part == "agent-metrics" and index > 0: + return parts[index - 1] + if path.parent.name: + return path.parent.name + return "unknown" + + +def _parse_error_detail(path: Path, line: int | None, reason: str) -> ParseErrorDetail: + artifact = _infer_artifact_name(path) + return ParseErrorDetail( + path=path.as_posix(), + artifact=artifact, + artifact_family=_artifact_family(artifact), + line=line, + reason=reason, + ) + + +def _read_ndjson(files: Iterable[Path]) -> tuple[list[dict[str, Any]], list[ParseErrorDetail]]: entries: list[dict[str, Any]] = [] - errors = 0 + errors: list[ParseErrorDetail] = [] for path in files: try: handle = path.open("r", encoding="utf-8") except OSError: - errors += 1 + errors.append(_parse_error_detail(path, None, "unreadable-file")) continue + file_entries: list[dict[str, Any]] = [] + file_errors: list[ParseErrorDetail] = [] + raw_lines: list[str] = [] with handle: - for line in handle: + for line_number, line in enumerate(handle, start=1): raw = line.strip() if not raw: continue + raw_lines.append(raw) try: parsed = json.loads(raw) except json.JSONDecodeError: - errors += 1 + file_errors.append(_parse_error_detail(path, line_number, "invalid-json")) continue if isinstance(parsed, dict): - entries.append(parsed) + file_entries.append(parsed) else: - errors += 1 + file_errors.append(_parse_error_detail(path, line_number, "non-object-json")) + if file_errors and not file_entries and raw_lines: + try: + parsed_file = json.loads("\n".join(raw_lines)) + except json.JSONDecodeError: + pass + else: + if isinstance(parsed_file, dict): + file_entries.append(parsed_file) + file_errors = [] + elif isinstance(parsed_file, list) and all( + isinstance(item, dict) for item in parsed_file + ): + file_entries.extend(parsed_file) + file_errors = [] + entries.extend(file_entries) + errors.extend(file_errors) return entries, errors @@ -433,7 +531,52 @@ def _format_rate(numerator: int, denominator: int) -> str: return f"{rate:.1f}% ({numerator}/{denominator})" -def build_summary(entries: list[dict[str, Any]], errors: int) -> str: +def _format_parse_error_details(parse_error_details: list[ParseErrorDetail]) -> list[str]: + if not parse_error_details: + return [] + family_counts = Counter(detail.artifact_family for detail in parse_error_details) + artifact_counts = Counter(detail.artifact for detail in parse_error_details) + lines = [ + "", + "## Parse Error Details", + f"- By artifact family: {_format_counter(family_counts)}", + f"- By artifact: {_format_counter(artifact_counts)}", + "", + "| Artifact family | Artifact | File | Line | Reason |", + "|-----------------|----------|------|------|--------|", + ] + for detail in parse_error_details[:_MAX_PARSE_ERROR_ROWS]: + line = str(detail.line) if detail.line is not None else "n/a" + lines.append( + "| " + f"{detail.artifact_family} | {detail.artifact} | {detail.path} | {line} | " + f"{detail.reason} |" + ) + remaining = len(parse_error_details) - _MAX_PARSE_ERROR_ROWS + if remaining > 0: + lines.append("") + lines.append(f"- Additional parse errors omitted from table: {remaining}") + return lines + + +def _parse_error_contract(parse_error_details: list[ParseErrorDetail]) -> dict[str, Any]: + family_counts = Counter(detail.artifact_family for detail in parse_error_details) + artifact_counts = Counter(detail.artifact for detail in parse_error_details) + reason_counts = Counter(detail.reason for detail in parse_error_details) + return { + "count": len(parse_error_details), + "by_artifact_family": dict(sorted(family_counts.items())), + "by_artifact": dict(sorted(artifact_counts.items())), + "by_reason": dict(sorted(reason_counts.items())), + "details": [detail.as_dict() for detail in parse_error_details], + } + + +def build_summary( + entries: list[dict[str, Any]], + errors: int, + parse_error_details: list[ParseErrorDetail] | None = None, +) -> str: buckets: dict[str, list[dict[str, Any]]] = { "keepalive": [], "autofix": [], @@ -479,6 +622,9 @@ def build_summary(entries: list[dict[str, Any]], errors: int) -> str: latest = max(timestamps).isoformat().replace("+00:00", "Z") lines.append(f"Range: {earliest} to {latest}") + if parse_error_details: + lines.extend(_format_parse_error_details(parse_error_details)) + lines.extend( [ "", @@ -563,25 +709,67 @@ def build_summary(entries: list[dict[str, Any]], errors: int) -> str: return "\n".join(lines) + "\n" +def build_summary_contract( + entries: list[dict[str, Any]], parse_error_details: list[ParseErrorDetail] +) -> dict[str, Any]: + buckets: dict[str, int] = Counter(_classify_entry(entry) for entry in entries) + timestamps: list[_dt.datetime] = [] + for entry in entries: + for key in ("timestamp", "created_at", "time", "run_started_at"): + ts = _parse_timestamp(entry.get(key)) + if ts is not None: + timestamps.append(ts) + break + generated_at = ( + _dt.datetime.now(_dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + ) + contract: dict[str, Any] = { + "schema": "workflows-agent-metrics-summary/v1", + "generated_at": generated_at, + "record_count": len(entries), + "record_buckets": dict(sorted(buckets.items())), + "parse_errors": _parse_error_contract(parse_error_details), + } + if timestamps: + contract["range"] = { + "earliest": min(timestamps).isoformat().replace("+00:00", "Z"), + "latest": max(timestamps).isoformat().replace("+00:00", "Z"), + } + return contract + + def main() -> int: metrics_paths_raw = os.environ.get("METRICS_PATHS", "") metrics_paths = [item.strip() for item in metrics_paths_raw.split(",") if item.strip()] metrics_dir = os.environ.get("METRICS_DIR", _DEFAULT_METRICS_DIR) output_path = Path(os.environ.get("OUTPUT_PATH", _DEFAULT_OUTPUT)) + output_json_path = Path(os.environ.get("OUTPUT_JSON_PATH", _DEFAULT_JSON_OUTPUT)) files = _gather_metrics_files(metrics_paths, metrics_dir) if not files: output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text("No metrics files found to aggregate.\n", encoding="utf-8") + output_json_path.parent.mkdir(parents=True, exist_ok=True) + output_json_path.write_text( + json.dumps(build_summary_contract([], []), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) print("No metrics files found to aggregate.", file=sys.stderr) return 0 - entries, errors = _read_ndjson(files) - summary = build_summary(entries, errors) + entries, parse_error_details = _read_ndjson(files) + summary = build_summary(entries, len(parse_error_details), parse_error_details) + summary_contract = build_summary_contract(entries, parse_error_details) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(summary, encoding="utf-8") + output_json_path.parent.mkdir(parents=True, exist_ok=True) + output_json_path.write_text( + json.dumps(summary_contract, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) print(f"Wrote metrics summary to {output_path}") + print(f"Wrote metrics summary JSON to {output_json_path}") return 0 diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 8dd8d1b76..d7660293b 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -102,6 +102,7 @@ def test_main_writes_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> keepalive_path = tmp_path / "keepalive.ndjson" autofix_path = tmp_path / "autofix.ndjson" output_path = tmp_path / "summary.md" + output_json_path = tmp_path / "summary.json" _write_ndjson( keepalive_path, @@ -129,6 +130,7 @@ def test_main_writes_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> monkeypatch.setenv("METRICS_PATHS", f"{keepalive_path},{autofix_path}") monkeypatch.setenv("OUTPUT_PATH", str(output_path)) + monkeypatch.setenv("OUTPUT_JSON_PATH", str(output_json_path)) exit_code = aggregate_agent_metrics.main() @@ -137,9 +139,13 @@ def test_main_writes_summary(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> summary = output_path.read_text(encoding="utf-8") assert "Keepalive" in summary assert "Autofix" in summary + summary_json = json.loads(output_json_path.read_text(encoding="utf-8")) + assert summary_json["schema"] == "workflows-agent-metrics-summary/v1" + assert summary_json["parse_errors"]["count"] == 0 monkeypatch.delenv("METRICS_PATHS", raising=False) monkeypatch.delenv("OUTPUT_PATH", raising=False) + monkeypatch.delenv("OUTPUT_JSON_PATH", raising=False) def test_parse_timestamp_variants() -> None: @@ -206,7 +212,9 @@ def test_read_ndjson_counts_parse_errors(tmp_path: Path) -> None: entries, errors = aggregate_agent_metrics._read_ndjson([path]) assert entries == [{"key": "value"}] - assert errors == 2 + assert len(errors) == 2 + assert [error.reason for error in errors] == ["invalid-json", "non-object-json"] + assert errors[0].line == 2 def test_read_ndjson_streams_file_lines(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: @@ -221,14 +229,73 @@ def fail_read_text(*_args: object, **_kwargs: object) -> str: entries, errors = aggregate_agent_metrics._read_ndjson([path]) assert entries == [{"key": "value"}] - assert errors == 0 + assert errors == [] def test_read_ndjson_counts_unreadable_file(tmp_path: Path) -> None: missing = tmp_path / "missing.ndjson" entries, errors = aggregate_agent_metrics._read_ndjson([missing]) assert entries == [] - assert errors == 1 + assert len(errors) == 1 + assert errors[0].reason == "unreadable-file" + + +def test_read_ndjson_attributes_parse_errors_to_artifact_family(tmp_path: Path) -> None: + metrics_dir = ( + tmp_path / "artifacts" / "review-thread-terminal-disposition-123" / "agent-metrics" + ) + metrics_dir.mkdir(parents=True) + path = metrics_dir / "terminal.ndjson" + path.write_text('{"ok": true}\n{"broken": true\n', encoding="utf-8") + + entries, errors = aggregate_agent_metrics._read_ndjson([path]) + + assert entries == [{"ok": True}] + assert len(errors) == 1 + assert errors[0].artifact == "review-thread-terminal-disposition-123" + assert errors[0].artifact_family == "review-thread-terminal-disposition" + assert errors[0].line == 2 + + summary = aggregate_agent_metrics.build_summary(entries, len(errors), errors) + assert "## Parse Error Details" in summary + assert "By artifact family: review-thread-terminal-disposition (1)" in summary + assert "review-thread-terminal-disposition-123" in summary + + contract = aggregate_agent_metrics.build_summary_contract(entries, errors) + assert contract["parse_errors"]["count"] == 1 + assert contract["parse_errors"]["by_artifact_family"] == { + "review-thread-terminal-disposition": 1 + } + assert contract["parse_errors"]["details"][0]["reason"] == "invalid-json" + + +def test_read_ndjson_accepts_legacy_pretty_json_object(tmp_path: Path) -> None: + metrics_dir = tmp_path / "artifacts" / "keepalive-metrics" + metrics_dir.mkdir(parents=True) + path = metrics_dir / "keepalive-metrics.ndjson" + path.write_text( + json.dumps( + { + "schema": "workflows-keepalive-metrics/v1", + "pr_number": 1872, + "iteration_count": 0, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + entries, errors = aggregate_agent_metrics._read_ndjson([path]) + + assert errors == [] + assert entries == [ + { + "schema": "workflows-keepalive-metrics/v1", + "pr_number": 1872, + "iteration_count": 0, + } + ] def test_classify_entry_prefers_explicit_type() -> None: @@ -490,12 +557,16 @@ def test_main_writes_placeholder_when_no_files( monkeypatch.setenv("METRICS_PATHS", "") monkeypatch.setenv("METRICS_DIR", str(tmp_path / "missing")) output_path = tmp_path / "summary.md" + output_json_path = tmp_path / "summary.json" monkeypatch.setenv("OUTPUT_PATH", str(output_path)) + monkeypatch.setenv("OUTPUT_JSON_PATH", str(output_json_path)) exit_code = aggregate_agent_metrics.main() assert exit_code == 0 assert output_path.read_text(encoding="utf-8") == "No metrics files found to aggregate.\n" + summary_json = json.loads(output_json_path.read_text(encoding="utf-8")) + assert summary_json["parse_errors"]["count"] == 0 def test_autopilot_metrics_summarised() -> None: diff --git a/tests/workflows/test_workflow_agents_consolidation.py b/tests/workflows/test_workflow_agents_consolidation.py index 48b942487..a2d9edb65 100644 --- a/tests/workflows/test_workflow_agents_consolidation.py +++ b/tests/workflows/test_workflow_agents_consolidation.py @@ -275,6 +275,10 @@ def test_weekly_metrics_uploads_selector_report_on_failure(): assert ( "artifacts/metric-artifacts-selection.json" in text ), "Weekly metrics must include selector JSON in uploaded artifacts" + assert ( + "OUTPUT_JSON_PATH: agent-weekly-metrics.json" in text + and "agent-weekly-metrics.json" in text + ), "Weekly metrics must upload a machine-readable aggregate summary" assert ( "uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6" in text ), "Weekly metrics must pin the Node runtime setup action to the v6 commit SHA" @@ -378,6 +382,16 @@ def test_weekly_metrics_aggregate_script_is_synced_to_consumers(): assert Path("templates/consumer-repo/scripts/aggregate_agent_metrics.py").is_file() +def test_keepalive_metrics_emit_compact_ndjson(): + workflow_paths = [ + WORKFLOWS_DIR / "agents-keepalive-loop.yml", + Path("templates/consumer-repo/.github/workflows/agents-81-gate-followups.yml"), + ] + for path in workflow_paths: + text = path.read_text(encoding="utf-8") + assert "metrics_json=$(jq -cn \\" in text, f"{path} must emit one JSON object per line" + + def test_terminal_disposition_records_include_artifact_identity(): workflow_paths = [ WORKFLOWS_DIR / "agents-verify-to-issue-v2.yml",