Skip to content

chore: sync workflow templates #304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
stranske wants to merge 1 commit into from
Closed

chore: sync workflow templates #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/scripts/terminal_disposition.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ function normalizeTerminalDisposition(input = {}) {
dispatch_outcome: input.dispatch_outcome ?? input.dispatchOutcome,
llm_model: input.llm_model ?? input.llmModel ?? input.model,
model_selection_reason: input.model_selection_reason ?? input.modelSelectionReason,
llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version,

Copilot AI Apr 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

normalizeTerminalDisposition maps llm_cli_version from llm_cli_version, llmCliVersion, or cli_version, but it doesn't accept a camelCase cliVersion (even though other fields support both snake_case and camelCase variants). If callers provide cliVersion, the value will be dropped. Consider including input.cliVersion in the fallback chain.

Suggested change
llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version,
llm_cli_version:
input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version ?? input.cliVersion,

Copilot uses AI. Check for mistakes.
verifier_mode: input.verifier_mode ?? input.verifierMode,
};

Expand Down
3 changes: 3 additions & 0 deletions .github/scripts/weekly_metrics_artifacts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ const EXACT_METRICS_ARTIFACTS = new Set([
'agents-autofix-metrics',
'agents-verifier-metrics',
'agents-verifier-disposition-metrics',
'codex-cli-freshness',
]);

const PREFIXED_METRICS_ARTIFACTS = [
'autopilot-metrics-',
'issue-optimizer-metrics-',
'issue-intake-format-metrics-',
'codex-cli-freshness-',
'verifier-terminal-disposition-',
'review-thread-terminal-disposition-',
];
Expand All @@ -34,6 +36,7 @@ const PATTERNED_METRICS_ARTIFACTS = [
];

const PRIORITY_METRICS_FAMILIES = [
'codex-cli-freshness',
'verifier-terminal-disposition',
'review-thread-terminal-disposition',
'bot-comment-auth-coverage-wrapper',
Expand Down
29 changes: 23 additions & 6 deletions .github/scripts/weekly_metrics_download_manifest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,27 @@ function selectedArtifactsFromSelection(selection = {}) {
return Array.isArray(selection.selected_artifacts) ? selection.selected_artifacts : [];
}

function compactSelectionDetails(selection = {}, selectionPath = '') {
return {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selectedArtifactsFromSelection(selection).length,
candidate_count: Number.isFinite(Number(selection.candidate_count))
? Number(selection.candidate_count)
: 0,
candidate_family_counts: selection.candidate_family_counts || {},
selected_family_counts: selection.selected_family_counts || {},
missing_priority_families: Array.isArray(selection.missing_priority_families)
? selection.missing_priority_families
: [],
priority_family_statuses: Array.isArray(selection.priority_family_statuses)
? selection.priority_family_statuses
: [],
latest_candidate_by_family: selection.latest_candidate_by_family || {},
};
}

function defaultArtifactDir(root, artifact) {
return path.posix.join(root, safeArtifactPathSegment(artifact.name), String(artifact.id || ''));
}
Expand All @@ -56,12 +77,7 @@ function buildInitialManifest(selection = {}, options = {}) {
schema: DOWNLOAD_MANIFEST_SCHEMA,
status: selection.status === 'error' ? 'error' : 'pending',
generated_at: generatedAt,
selection: {
path: selectionPath,
schema: cleanString(selection.schema),
status: cleanString(selection.status || 'pass'),
selected_count: selected.length,
},
selection: compactSelectionDetails(selection, selectionPath),
stats: {
selected_count: selected.length,
download_pass_count: 0,
Expand Down Expand Up @@ -313,6 +329,7 @@ if (require.main === module) {
module.exports = {
DOWNLOAD_MANIFEST_SCHEMA,
buildInitialManifest,
compactSelectionDetails,
finalizeManifest,
formatMarkdown,
safeArtifactPathSegment,
Expand Down
86 changes: 86 additions & 0 deletions scripts/aggregate_agent_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@
"agents-autofix-metrics",
"agents-verifier-metrics",
"agents-verifier-disposition-metrics",
"codex-cli-freshness",
}
_PREFIXED_ARTIFACT_FAMILIES = (
"autopilot-metrics-",
"issue-optimizer-metrics-",
"issue-intake-format-metrics-",
"codex-cli-freshness-",
"verifier-terminal-disposition-",
"review-thread-terminal-disposition-",
)
Expand Down Expand Up @@ -323,6 +325,8 @@ def _classify_entry(entry: dict[str, Any]) -> str:
return "terminal_disposition"
if schema == "workflows-verifier-followup-ledger/v1":
return "verifier_followup_ledger"
if schema == "workflows-codex-cli-freshness/v1":
return "codex_cli_freshness"
explicit = entry.get("metric_type") or entry.get("type") or entry.get("workflow")
if isinstance(explicit, str):
lowered = explicit.lower()
Expand Down Expand Up @@ -506,6 +510,7 @@ def _summarise_verifier(
terminal_sources = Counter()
verifier_models = Counter()
model_selection_reasons = Counter()
verifier_cli_versions = Counter()
unsupported_verifier_models = Counter()
unsupported_model_dispositions = Counter()
missing_verifier_model_metadata = Counter()
Expand Down Expand Up @@ -574,6 +579,14 @@ def _summarise_verifier(
)
if model_selection_reason:
model_selection_reasons[str(model_selection_reason)] += 1
cli_version = (
entry.get("codex_cli_version")
or entry.get("llm_cli_version")
or entry.get("cli_version")
)
cli_version_text = str(cli_version).strip() if cli_version is not None else ""
if cli_version_text:
verifier_cli_versions[cli_version_text.lower()] += 1
verifier_mode = str(entry.get("verifier_mode") or "").strip().lower()
if verifier_mode:
verifier_modes[verifier_mode] += 1
Expand Down Expand Up @@ -623,6 +636,7 @@ def _summarise_verifier(
"terminal_dispositions": terminal_dispositions,
"terminal_sources": terminal_sources,
"verifier_models": verifier_models,
"verifier_cli_versions": verifier_cli_versions,
"unsupported_verifier_models": unsupported_verifier_models,
"unsupported_model_dispositions": unsupported_model_dispositions,
"missing_verifier_model_metadata": missing_verifier_model_metadata,
Expand Down Expand Up @@ -728,6 +742,54 @@ def _summarise_autopilot(entries: list[dict[str, Any]]) -> dict[str, Any]:
}


def _summarise_codex_cli_freshness(entries: list[dict[str, Any]]) -> dict[str, Any]:
statuses = Counter()
packages = Counter()
pinned_versions = Counter()
latest_versions = Counter()
max_major_delta = 0
max_minor_delta = 0
max_patch_delta = 0
update_targets = Counter()
for entry in entries:
status = str(entry.get("status") or "unknown")
statuses[status] += 1
package = str(entry.get("package") or "unknown")
packages[package] += 1
pinned = str(entry.get("pinned_version") or "unknown")
latest = str(entry.get("latest_version") or "unknown")
pinned_versions[pinned] += 1
latest_versions[latest] += 1
delta = entry.get("version_delta")
if isinstance(delta, dict):
max_major_delta = max(max_major_delta, _safe_int(delta.get("major")) or 0)
max_minor_delta = max(max_minor_delta, _safe_int(delta.get("minor")) or 0)
max_patch_delta = max(max_patch_delta, _safe_int(delta.get("patch")) or 0)
targets = entry.get("update_targets")
if isinstance(targets, list):
for target in targets:
if not isinstance(target, dict):
continue
path = str(target.get("path") or "").strip()
if path:
update_targets[path] += 1
return {
"records": len(entries),
"statuses": statuses,
"packages": packages,
"pinned_versions": pinned_versions,
"latest_versions": latest_versions,
"outdated_records": statuses.get("outdated", 0),
"latest_unavailable_records": statuses.get("latest-unavailable", 0),
"max_version_delta": {
Comment on lines +754 to +784

Copilot AI Apr 26, 2026

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In _summarise_codex_cli_freshness, status is counted using the raw entry.get('status') string, but derived fields (outdated_records, latest_unavailable_records) are computed using exact lowercase keys ('outdated', 'latest-unavailable'). If upstream emits status with different casing/whitespace, these derived counts will be incorrect and the statuses counter will fragment. Consider normalizing status (trim + lowercase) before incrementing and before looking up derived counts, and apply the same normalization to package / versions if you want stable aggregates.

Copilot uses AI. Check for mistakes.
"major": max_major_delta,
"minor": max_minor_delta,
"patch": max_patch_delta,
},
"update_targets": update_targets,
}


def _format_counter(counter: Counter[str]) -> str:
if not counter:
return "n/a"
Expand Down Expand Up @@ -765,6 +827,7 @@ def _bucket_entries(entries: list[dict[str, Any]]) -> dict[str, list[dict[str, A
"verifier": [],
"terminal_disposition": [],
"verifier_followup_ledger": [],
"codex_cli_freshness": [],
"autopilot": [],
"unknown": [],
}
Expand All @@ -784,6 +847,7 @@ def _summary_metrics_contract(buckets: dict[str, list[dict[str, Any]]]) -> dict[
buckets["verifier_followup_ledger"],
),
"autopilot": _summarise_autopilot(buckets["autopilot"]),
"codex_cli_freshness": _summarise_codex_cli_freshness(buckets["codex_cli_freshness"]),
"unknown": {"records": len(buckets["unknown"])},
}
)
Expand Down Expand Up @@ -947,6 +1011,7 @@ def build_summary(
buckets["verifier_followup_ledger"],
)
autopilot = _summarise_autopilot(buckets["autopilot"])
codex_cli_freshness = _summarise_codex_cli_freshness(buckets["codex_cli_freshness"])

now = _dt.datetime.now(_dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
lines = [
Expand All @@ -959,6 +1024,7 @@ def build_summary(
f"verifier {verifier['runs']}, "
f"terminal dispositions {verifier['terminal_records']}, "
f"verifier follow-up ledgers {verifier['ledger_records']}, "
f"codex CLI freshness {codex_cli_freshness['records']}, "
f"autopilot {autopilot['records']}, "
f"unknown {len(buckets['unknown'])})"
),
Expand Down Expand Up @@ -1019,6 +1085,7 @@ def build_summary(
f"{verifier['ledger_policy_depth_limit_exceeded']}"
),
f"- Verifier models: {_format_counter(verifier['verifier_models'])}",
f"- Verifier CLI versions: {_format_counter(verifier['verifier_cli_versions'])}",
f"- Unsupported verifier models: {_format_counter(verifier['unsupported_verifier_models'])}",
(
"- Unsupported model dispositions: "
Expand All @@ -1034,6 +1101,25 @@ def build_summary(
),
f"- Model selection reasons: {_format_counter(verifier['model_selection_reasons'])}",
f"- Verifier modes: {_format_counter(verifier['verifier_modes'])}",
"",
"## Codex CLI Freshness",
f"- Records: {codex_cli_freshness['records']}",
f"- Statuses: {_format_counter(codex_cli_freshness['statuses'])}",
f"- Packages: {_format_counter(codex_cli_freshness['packages'])}",
f"- Pinned versions: {_format_counter(codex_cli_freshness['pinned_versions'])}",
f"- Latest versions: {_format_counter(codex_cli_freshness['latest_versions'])}",
f"- Outdated records: {codex_cli_freshness['outdated_records']}",
(
"- Latest unavailable records: "
f"{codex_cli_freshness['latest_unavailable_records']}"
),
(
"- Max version delta: "
f"major {codex_cli_freshness['max_version_delta']['major']}, "
f"minor {codex_cli_freshness['max_version_delta']['minor']}, "
f"patch {codex_cli_freshness['max_version_delta']['patch']}"
),
f"- Update targets: {_format_counter(codex_cli_freshness['update_targets'])}",
]
)

Expand Down
Loading