Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion WORKFLOW_USER_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,6 @@ Issue: "Add user authentication"
| `agents:optimize` | Issue | Analyze and suggest improvements |
| `agents:apply-suggestions` | Issue | Apply optimization suggestions |
| `agents:auto-pilot` | Issue | Full end-to-end automation |
| `agents:allow-change` | PR | Permission signal for `agents-guard`; automatically applied to dependency-bot PRs by `maint-auto-label-dep-prs.yml` |
| `agents:capability-check` | Issue | Check if agent can complete |
| `agents:decompose` | Issue | Break into smaller issues |
| `agents:dedup` | Issue | Check for duplicates |
Expand All @@ -166,6 +165,7 @@ Issue: "Add user authentication"
| `agent:needs-attention` | Human intervention required |
| `agents:auto-pilot-pause` | Auto-pilot paused |
| `agents:auto-pilot-failed` | Auto-pilot stopped due to errors |
| `agents:allow-change` | Permission signal for `agents-guard`; bypasses CODEOWNER approval only for automated dependency PRs from Dependabot/Renovate. Auto-applied by `maint-auto-label-dep-prs.yml`; manual application does not bypass guard enforcement. |
| `needs-human` | Escalated to human |
| `follow-up` | Created as follow-up to another issue/PR |
| `duplicate` | Potential duplicate detected |
Expand Down
121 changes: 116 additions & 5 deletions scripts/aggregate_agent_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,11 @@
import re
import sys
from collections import Counter
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any

from src.ndjson_parser import read_ndjson_file

_DEFAULT_METRICS_DIR = "agent-metrics"
_DEFAULT_OUTPUT = "agent-metrics-summary.md"
_DEFAULT_JSON_OUTPUT = "agent-metrics-summary.json"
Expand Down Expand Up @@ -305,16 +303,129 @@ def _canonical_parse_error_detail(path: Path, error: str) -> ParseErrorDetail:
return _parse_error_detail(path, line, reason)


def _format_parse_error(path: Path, line: int | None, reason: str, detail: str = "") -> str:
if reason == "invalid-json":
suffix = f" ({detail})" if detail else ""
return f"{path}:{line}: invalid JSON{suffix}"
if reason == "non-object-json":
return f"{path}:{line}: expected object, got {detail}"
if reason == "legacy-json-fallback-buffer-limit":
return f"{path}: legacy-json-fallback-buffer-limit"
return f"{path}: {detail}" if detail else f"{path}: unreadable file"


def _read_ndjson_file_streaming(
path: Path,
record_error: Callable[[int | None, str, str], None],
) -> tuple[list[dict[str, Any]], bool]:
try:
handle = path.open("r", encoding="utf-8")
except OSError as exc:
record_error(None, "unreadable-file", str(exc))
return [], False

entries: list[dict[str, Any]] = []
raw_lines_for_fallback: list[str] = []
raw_fallback_bytes = 0
raw_fallback_truncated = False
saw_parse_error = False
saw_read_error = False
with handle:
try:
for line_number, line in enumerate(handle, start=1):
raw = line.strip()
if not raw:
continue
if not raw_fallback_truncated and (not entries or saw_parse_error):
raw_bytes = len(raw.encode("utf-8")) + 1
fallback_within_limit = (
len(raw_lines_for_fallback) < _MAX_LEGACY_JSON_FALLBACK_LINES
and raw_fallback_bytes + raw_bytes <= _MAX_LEGACY_JSON_FALLBACK_BYTES
)
if fallback_within_limit:
raw_fallback_bytes += raw_bytes
raw_lines_for_fallback.append(raw)
else:
raw_fallback_truncated = True
raw_lines_for_fallback = []
try:
parsed = json.loads(raw)
except json.JSONDecodeError as exc:
saw_parse_error = True
record_error(line_number, "invalid-json", exc.msg)
continue
if isinstance(parsed, dict):
entries.append(parsed)
if not saw_parse_error:
raw_lines_for_fallback = []
raw_fallback_bytes = 0
else:
saw_parse_error = True
record_error(line_number, "non-object-json", type(parsed).__name__)
except (OSError, UnicodeDecodeError) as exc:
saw_read_error = True
record_error(None, "unreadable-file", str(exc))

if not saw_parse_error or saw_read_error:
return entries, False

raw_text = "\n".join(raw_lines_for_fallback)
if raw_fallback_truncated:
record_error(None, "legacy-json-fallback-buffer-limit", "")
return entries, False

try:
parsed_file = json.loads(raw_text)
except json.JSONDecodeError:
return entries, False

if isinstance(parsed_file, dict):
return [parsed_file], True
if isinstance(parsed_file, list) and all(isinstance(item, dict) for item in parsed_file):
return list(parsed_file), True
return entries, False


def read_ndjson_file(path: Path) -> tuple[list[dict[str, Any]], list[str]]:
"""Read an NDJSON file without depending on Workflows-only modules."""
errors: list[str] = []

def record_error(line: int | None, reason: str, detail: str) -> None:
errors.append(_format_parse_error(path, line, reason, detail))

entries, legacy_fallback_used = _read_ndjson_file_streaming(path, record_error)
if legacy_fallback_used:
return entries, []
return entries, errors


def read_metric_ndjson_files(
files: Iterable[Path],
) -> tuple[list[dict[str, Any]], list[ParseErrorDetail]]:
entries: list[dict[str, Any]] = []
errors: list[ParseErrorDetail] = []
for path in files:
file_entries, file_errors = read_ndjson_file(path)
file_errors: list[ParseErrorDetail] = []

def record_error(
line: int | None,
reason: str,
_detail: str,
*,
target_errors: list[ParseErrorDetail] = file_errors,
current_path: Path = path,
) -> None:
_append_parse_error_detail(
target_errors,
_parse_error_detail(current_path, line, reason),
)

file_entries, legacy_fallback_used = _read_ndjson_file_streaming(path, record_error)
if legacy_fallback_used:
file_errors = []
entries.extend(_attach_metric_source(entry, path) for entry in file_entries)
for error in file_errors:
_append_parse_error_detail(errors, _canonical_parse_error_detail(path, error))
_append_parse_error_detail(errors, error)
return entries, errors


Expand Down
Loading