From 83b09b90d69d5f001ced5144db796c4090fa6cb5 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sun, 26 Apr 2026 16:39:03 -0500 Subject: [PATCH] Normalize verifier telemetry contracts --- .../terminal-disposition-coverage.test.js | 3 ++ .../__tests__/terminal-disposition.test.js | 9 ++++- .github/scripts/terminal_disposition.js | 17 +++++++++ .../scripts/terminal_disposition_coverage.js | 4 +++ scripts/aggregate_agent_metrics.py | 36 ++++++++++++++++--- scripts/check_codex_cli_freshness.py | 2 ++ .../.github/scripts/terminal_disposition.js | 17 +++++++++ .../scripts/terminal_disposition_coverage.js | 4 +++ .../scripts/aggregate_agent_metrics.py | 36 ++++++++++++++++--- tests/scripts/test_aggregate_agent_metrics.py | 8 ++--- .../scripts/test_check_codex_cli_freshness.py | 13 +++++++ 11 files changed, 134 insertions(+), 15 deletions(-) diff --git a/.github/scripts/__tests__/terminal-disposition-coverage.test.js b/.github/scripts/__tests__/terminal-disposition-coverage.test.js index 232447fe7..bcfe046cd 100644 --- a/.github/scripts/__tests__/terminal-disposition-coverage.test.js +++ b/.github/scripts/__tests__/terminal-disposition-coverage.test.js @@ -392,11 +392,14 @@ test('requires verifier model metadata for post-contract records with unknown mo model_metadata_required_after: '2026-04-26T04:25:00Z', } ); + const markdown = formatTerminalDispositionCoverageMarkdown(report); assert.equal(report.status, 'warning'); assert.equal(report.verifier_model_compatibility.missing_model_record_count, 1); + assert.equal(report.verifier_model_compatibility.missing_model_unknown_mode_record_count, 1); assert.deepEqual(report.enforcement.blockers, ['missing-verifier-model-metadata']); assert.equal(report.verifier_model_compatibility.missing_model_records[0].verifier_mode, 'unknown'); + assert.match(markdown, /Missing verifier model metadata records with unknown mode: 1/); }); test('suppresses pre-contract verifier terminal records missing model metadata', () => { diff --git a/.github/scripts/__tests__/terminal-disposition.test.js b/.github/scripts/__tests__/terminal-disposition.test.js index 6d129065a..07228e643 100644 --- a/.github/scripts/__tests__/terminal-disposition.test.js +++ b/.github/scripts/__tests__/terminal-disposition.test.js @@ -6,6 +6,7 @@ const { normalizeVerifierFollowupLedger, normalizeVerifierFollowupPolicy, normalizeLedgerDisposition, + normalizeCliVersion, summarizeTerminalDispositionSources, formatTerminalDispositionMarkdown, sourceKey, @@ -25,7 +26,7 @@ test('normalizes terminal disposition records with stable source keys', () => { artifactFamily: 'verifier-terminal-disposition', llmModel: 'gpt-5.3-codex', modelSelectionReason: 'default', - llmCliVersion: 'codex-cli 0.125.0', + llmCliVersion: 'Codex CLI v0.125.0', verifierMode: 'checkbox', needsHuman: false, timestamp: '2026-04-25T00:00:00Z', @@ -50,6 +51,12 @@ test('normalizes terminal disposition records with stable source keys', () => { assert.equal(record.needs_human, false); }); +test('normalizes Codex CLI version strings', () => { + assert.equal(normalizeCliVersion('Codex CLI v0.125.0\n'), 'codex-cli 0.125.0'); + assert.equal(normalizeCliVersion('@openai/codex@0.126.1'), 'codex-cli 0.126.1'); + assert.equal(normalizeCliVersion('custom tool 1.0.0'), 'custom tool 1.0.0'); +}); + test('normalizes boolean-like optional fields without stringifying them', () => { const terminal = normalizeTerminalDisposition({ sourceId: 42, diff --git a/.github/scripts/terminal_disposition.js b/.github/scripts/terminal_disposition.js index 9be4ae256..c15031753 100644 --- a/.github/scripts/terminal_disposition.js +++ b/.github/scripts/terminal_disposition.js @@ -24,6 +24,18 @@ function normalizeToken(value, fallback = 'unknown') { return normalized || fallback; } +function normalizeCliVersion(value) { + const text = cleanString(value); + if (!text) return ''; + const versionMatch = text.match(/(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)/); + const version = versionMatch ? versionMatch[1] : ''; + const lower = text.toLowerCase().replace(/_/g, '-'); + if (version && /\bcodex(?:-|\s+)cli\b|\bopenai\/codex\b|\bcodex\b/.test(lower)) { + return `codex-cli ${version}`; + } + return lower; +} + function normalizeSourceType(value) { const text = cleanString(value).toLowerCase(); if (!text) return 'unknown'; @@ -47,6 +59,10 @@ function normalizeOptionalValue(key, value) { const parsed = cleanBool(value); return parsed === null ? undefined : parsed; } + if (key === 'llm_cli_version' || key === 'codex_cli_version' || key === 'cli_version') { + const normalized = normalizeCliVersion(value); + return normalized || undefined; + } const cleaned = typeof value === 'boolean' ? value : cleanString(value); if (cleaned === '') return undefined; return typeof value === 'string' ? cleaned : value; @@ -369,6 +385,7 @@ module.exports = { normalizeVerifierFollowupPolicy, normalizeLedgerDisposition, normalizeOptionalValue, + normalizeCliVersion, summarizeTerminalDispositionSources, formatTerminalDispositionMarkdown, sourceKey, diff --git a/.github/scripts/terminal_disposition_coverage.js b/.github/scripts/terminal_disposition_coverage.js index 290b7f5c3..643fc3a6c 100644 --- a/.github/scripts/terminal_disposition_coverage.js +++ b/.github/scripts/terminal_disposition_coverage.js @@ -161,6 +161,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { const unsupportedRecords = []; const missingModelRecords = []; const legacyMissingModelRecords = []; + let missingUnknownModeRecordCount = 0; let verifierRecordCount = 0; for (const raw of records) { @@ -192,6 +193,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { if (isPreContractVerifierModelRecord(record, metadata, modelMetadataContract)) { legacyMissingModelRecords.push(missingRecord); } else { + if (!verifierMode) missingUnknownModeRecordCount += 1; missingModelRecords.push(missingRecord); } } @@ -215,6 +217,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { unsupported_models: unsupportedModels, unsupported_record_count: unsupportedRecords.length, missing_model_record_count: missingModelRecords.length, + missing_model_unknown_mode_record_count: missingUnknownModeRecordCount, legacy_missing_model_record_count: legacyMissingModelRecords.length, selected_models: Object.fromEntries( Object.entries(selectedModels).sort((a, b) => a[0].localeCompare(b[0])) @@ -615,6 +618,7 @@ function formatTerminalDispositionCoverageMarkdown(report) { `- Verifier model compatibility: ${modelCompatibility.status}`, `- Unsupported verifier model records: ${modelCompatibility.unsupported_record_count}`, `- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`, + `- Missing verifier model metadata records with unknown mode: ${modelCompatibility.missing_model_unknown_mode_record_count || 0}`, `- Legacy missing verifier model metadata records: ${modelCompatibility.legacy_missing_model_record_count || 0}` ); if (modelCompatibility.model_metadata_contract?.required_after) { diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index e4a8bb931..6bbd78d88 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -102,6 +102,32 @@ def _parse_timestamp(value: Any) -> _dt.datetime | None: return None +def _normalize_version_text(value: Any) -> str: + if value is None: + return "" + text = str(value).strip() + if not text: + return "" + match = re.search(r"(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)", text) + return match.group(1) if match else text + + +def _normalize_cli_version(value: Any) -> str: + text = str(value).strip() if value is not None else "" + if not text: + return "" + version = _normalize_version_text(text) + lower = text.lower().replace("_", "-") + if version and re.search(r"\bcodex(?:-|\s+)cli\b|\bopenai/codex\b|\bcodex\b", lower): + return f"codex-cli {version}".lower() + return lower + + +def _normalize_counter_token(value: Any, fallback: str = "unknown") -> str: + text = str(value).strip().lower().replace("_", "-") if value is not None else "" + return text or fallback + + def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Path]: if metrics_paths: return [Path(path) for path in metrics_paths if path] @@ -586,7 +612,7 @@ def _summarise_verifier( ) cli_version_text = str(cli_version).strip() if cli_version is not None else "" if cli_version_text: - verifier_cli_versions[cli_version_text.lower()] += 1 + verifier_cli_versions[_normalize_cli_version(cli_version_text)] += 1 verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() if verifier_mode: verifier_modes[verifier_mode] += 1 @@ -752,12 +778,12 @@ def _summarise_codex_cli_freshness(entries: list[dict[str, Any]]) -> dict[str, A max_patch_delta = 0 update_targets = Counter() for entry in entries: - status = str(entry.get("status") or "unknown") + status = _normalize_counter_token(entry.get("status")) statuses[status] += 1 - package = str(entry.get("package") or "unknown") + package = str(entry.get("package") or "unknown").strip() or "unknown" packages[package] += 1 - pinned = str(entry.get("pinned_version") or "unknown") - latest = str(entry.get("latest_version") or "unknown") + pinned = _normalize_version_text(entry.get("pinned_version")) or "unknown" + latest = _normalize_version_text(entry.get("latest_version")) or "unknown" pinned_versions[pinned] += 1 latest_versions[latest] += 1 delta = entry.get("version_delta") diff --git a/scripts/check_codex_cli_freshness.py b/scripts/check_codex_cli_freshness.py index cda9f46dc..fb1c4488b 100644 --- a/scripts/check_codex_cli_freshness.py +++ b/scripts/check_codex_cli_freshness.py @@ -90,6 +90,8 @@ def build_contract( query_error: str = "", generated_at: str | None = None, ) -> dict[str, Any]: + pinned_version = _version_text(pinned_version) + latest_version = _version_text(latest_version) if latest_version else "" pinned_tuple = parse_semver(pinned_version) latest_tuple = parse_semver(latest_version) if latest_version else None status = "unknown" diff --git a/templates/consumer-repo/.github/scripts/terminal_disposition.js b/templates/consumer-repo/.github/scripts/terminal_disposition.js index 9be4ae256..c15031753 100644 --- a/templates/consumer-repo/.github/scripts/terminal_disposition.js +++ b/templates/consumer-repo/.github/scripts/terminal_disposition.js @@ -24,6 +24,18 @@ function normalizeToken(value, fallback = 'unknown') { return normalized || fallback; } +function normalizeCliVersion(value) { + const text = cleanString(value); + if (!text) return ''; + const versionMatch = text.match(/(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)/); + const version = versionMatch ? versionMatch[1] : ''; + const lower = text.toLowerCase().replace(/_/g, '-'); + if (version && /\bcodex(?:-|\s+)cli\b|\bopenai\/codex\b|\bcodex\b/.test(lower)) { + return `codex-cli ${version}`; + } + return lower; +} + function normalizeSourceType(value) { const text = cleanString(value).toLowerCase(); if (!text) return 'unknown'; @@ -47,6 +59,10 @@ function normalizeOptionalValue(key, value) { const parsed = cleanBool(value); return parsed === null ? undefined : parsed; } + if (key === 'llm_cli_version' || key === 'codex_cli_version' || key === 'cli_version') { + const normalized = normalizeCliVersion(value); + return normalized || undefined; + } const cleaned = typeof value === 'boolean' ? value : cleanString(value); if (cleaned === '') return undefined; return typeof value === 'string' ? cleaned : value; @@ -369,6 +385,7 @@ module.exports = { normalizeVerifierFollowupPolicy, normalizeLedgerDisposition, normalizeOptionalValue, + normalizeCliVersion, summarizeTerminalDispositionSources, formatTerminalDispositionMarkdown, sourceKey, diff --git a/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js b/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js index 290b7f5c3..643fc3a6c 100644 --- a/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js +++ b/templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js @@ -161,6 +161,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { const unsupportedRecords = []; const missingModelRecords = []; const legacyMissingModelRecords = []; + let missingUnknownModeRecordCount = 0; let verifierRecordCount = 0; for (const raw of records) { @@ -192,6 +193,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { if (isPreContractVerifierModelRecord(record, metadata, modelMetadataContract)) { legacyMissingModelRecords.push(missingRecord); } else { + if (!verifierMode) missingUnknownModeRecordCount += 1; missingModelRecords.push(missingRecord); } } @@ -215,6 +217,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) { unsupported_models: unsupportedModels, unsupported_record_count: unsupportedRecords.length, missing_model_record_count: missingModelRecords.length, + missing_model_unknown_mode_record_count: missingUnknownModeRecordCount, legacy_missing_model_record_count: legacyMissingModelRecords.length, selected_models: Object.fromEntries( Object.entries(selectedModels).sort((a, b) => a[0].localeCompare(b[0])) @@ -615,6 +618,7 @@ function formatTerminalDispositionCoverageMarkdown(report) { `- Verifier model compatibility: ${modelCompatibility.status}`, `- Unsupported verifier model records: ${modelCompatibility.unsupported_record_count}`, `- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`, + `- Missing verifier model metadata records with unknown mode: ${modelCompatibility.missing_model_unknown_mode_record_count || 0}`, `- Legacy missing verifier model metadata records: ${modelCompatibility.legacy_missing_model_record_count || 0}` ); if (modelCompatibility.model_metadata_contract?.required_after) { diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index e4a8bb931..6bbd78d88 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -102,6 +102,32 @@ def _parse_timestamp(value: Any) -> _dt.datetime | None: return None +def _normalize_version_text(value: Any) -> str: + if value is None: + return "" + text = str(value).strip() + if not text: + return "" + match = re.search(r"(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)", text) + return match.group(1) if match else text + + +def _normalize_cli_version(value: Any) -> str: + text = str(value).strip() if value is not None else "" + if not text: + return "" + version = _normalize_version_text(text) + lower = text.lower().replace("_", "-") + if version and re.search(r"\bcodex(?:-|\s+)cli\b|\bopenai/codex\b|\bcodex\b", lower): + return f"codex-cli {version}".lower() + return lower + + +def _normalize_counter_token(value: Any, fallback: str = "unknown") -> str: + text = str(value).strip().lower().replace("_", "-") if value is not None else "" + return text or fallback + + def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Path]: if metrics_paths: return [Path(path) for path in metrics_paths if path] @@ -586,7 +612,7 @@ def _summarise_verifier( ) cli_version_text = str(cli_version).strip() if cli_version is not None else "" if cli_version_text: - verifier_cli_versions[cli_version_text.lower()] += 1 + verifier_cli_versions[_normalize_cli_version(cli_version_text)] += 1 verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() if verifier_mode: verifier_modes[verifier_mode] += 1 @@ -752,12 +778,12 @@ def _summarise_codex_cli_freshness(entries: list[dict[str, Any]]) -> dict[str, A max_patch_delta = 0 update_targets = Counter() for entry in entries: - status = str(entry.get("status") or "unknown") + status = _normalize_counter_token(entry.get("status")) statuses[status] += 1 - package = str(entry.get("package") or "unknown") + package = str(entry.get("package") or "unknown").strip() or "unknown" packages[package] += 1 - pinned = str(entry.get("pinned_version") or "unknown") - latest = str(entry.get("latest_version") or "unknown") + pinned = _normalize_version_text(entry.get("pinned_version")) or "unknown" + latest = _normalize_version_text(entry.get("latest_version")) or "unknown" pinned_versions[pinned] += 1 latest_versions[latest] += 1 delta = entry.get("version_delta") diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index fa5a3f8a0..eedce9aff 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -76,9 +76,9 @@ def test_build_summary_formats_sections() -> None: "schema": "workflows-codex-cli-freshness/v1", "package": "@openai/codex", "component": "agents-verifier", - "status": "outdated", - "pinned_version": "0.125.0", - "latest_version": "0.127.3", + "status": " OUTDATED ", + "pinned_version": "@openai/codex@0.125.0", + "latest_version": "v0.127.3\n", "version_delta": {"major": 0, "minor": 2, "patch": 3}, "update_targets": [ {"path": ".github/workflows/reusable-agents-verifier.yml"}, @@ -723,7 +723,7 @@ def test_summary_helpers_cover_branches() -> None: "pr_number": 304, "disposition": "verified-pass", "llm_model": "gpt-5.4", - "codex_cli_version": "Codex-CLI 0.125.0", + "codex_cli_version": "Codex CLI v0.125.0", "verifier_mode": "checkbox", }, ] diff --git a/tests/scripts/test_check_codex_cli_freshness.py b/tests/scripts/test_check_codex_cli_freshness.py index 5d0e3f1cb..d743448cd 100644 --- a/tests/scripts/test_check_codex_cli_freshness.py +++ b/tests/scripts/test_check_codex_cli_freshness.py @@ -41,6 +41,19 @@ def test_build_contract_reports_outdated_pin() -> None: assert "@openai/codex@0.127.3" in markdown +def test_build_contract_normalizes_version_strings() -> None: + report = check_codex_cli_freshness.build_contract( + pinned_version="@openai/codex@0.125.0", + latest_version="v0.127.3\n", + generated_at="2026-04-26T19:00:00Z", + ) + + assert report["status"] == "outdated" + assert report["pinned_version"] == "0.125.0" + assert report["latest_version"] == "0.127.3" + assert report["version_delta"] == {"major": 0, "minor": 2, "patch": 3} + + def test_query_latest_npm_version_uses_isolated_cache(monkeypatch) -> None: captured_env = {}