Skip to content

[codex] Normalize verifier telemetry contracts #1923

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stranske merged 1 commit into from
Apr 26, 2026
Merged

[codex] Normalize verifier telemetry contracts #1923

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/scripts/__tests__/terminal-disposition-coverage.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,11 +392,14 @@ test('requires verifier model metadata for post-contract records with unknown mo
model_metadata_required_after: '2026-04-26T04:25:00Z',
}
);
const markdown = formatTerminalDispositionCoverageMarkdown(report);

assert.equal(report.status, 'warning');
assert.equal(report.verifier_model_compatibility.missing_model_record_count, 1);
assert.equal(report.verifier_model_compatibility.missing_model_unknown_mode_record_count, 1);
assert.deepEqual(report.enforcement.blockers, ['missing-verifier-model-metadata']);
assert.equal(report.verifier_model_compatibility.missing_model_records[0].verifier_mode, 'unknown');
assert.match(markdown, /Missing verifier model metadata records with unknown mode: 1/);
});

test('suppresses pre-contract verifier terminal records missing model metadata', () => {
Expand Down
9 changes: 8 additions & 1 deletion .github/scripts/__tests__/terminal-disposition.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const {
normalizeVerifierFollowupLedger,
normalizeVerifierFollowupPolicy,
normalizeLedgerDisposition,
normalizeCliVersion,
summarizeTerminalDispositionSources,
formatTerminalDispositionMarkdown,
sourceKey,
Expand All @@ -25,7 +26,7 @@ test('normalizes terminal disposition records with stable source keys', () => {
artifactFamily: 'verifier-terminal-disposition',
llmModel: 'gpt-5.3-codex',
modelSelectionReason: 'default',
llmCliVersion: 'codex-cli 0.125.0',
llmCliVersion: 'Codex CLI v0.125.0',
verifierMode: 'checkbox',
needsHuman: false,
timestamp: '2026-04-25T00:00:00Z',
Expand All @@ -50,6 +51,12 @@ test('normalizes terminal disposition records with stable source keys', () => {
assert.equal(record.needs_human, false);
});

test('normalizes Codex CLI version strings', () => {
assert.equal(normalizeCliVersion('Codex CLI v0.125.0\n'), 'codex-cli 0.125.0');
assert.equal(normalizeCliVersion('@openai/codex@0.126.1'), 'codex-cli 0.126.1');
assert.equal(normalizeCliVersion('custom tool 1.0.0'), 'custom tool 1.0.0');
});

test('normalizes boolean-like optional fields without stringifying them', () => {
const terminal = normalizeTerminalDisposition({
sourceId: 42,
Expand Down
17 changes: 17 additions & 0 deletions .github/scripts/terminal_disposition.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@ function normalizeToken(value, fallback = 'unknown') {
return normalized || fallback;
}

function normalizeCliVersion(value) {
const text = cleanString(value);
if (!text) return '';
const versionMatch = text.match(/(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)/);
const version = versionMatch ? versionMatch[1] : '';
const lower = text.toLowerCase().replace(/_/g, '-');
if (version && /\bcodex(?:-|\s+)cli\b|\bopenai\/codex\b|\bcodex\b/.test(lower)) {
return `codex-cli ${version}`;
}
return lower;
}

function normalizeSourceType(value) {
const text = cleanString(value).toLowerCase();
if (!text) return 'unknown';
Expand All @@ -47,6 +59,10 @@ function normalizeOptionalValue(key, value) {
const parsed = cleanBool(value);
return parsed === null ? undefined : parsed;
}
if (key === 'llm_cli_version' || key === 'codex_cli_version' || key === 'cli_version') {
const normalized = normalizeCliVersion(value);
return normalized || undefined;
}
const cleaned = typeof value === 'boolean' ? value : cleanString(value);
if (cleaned === '') return undefined;
return typeof value === 'string' ? cleaned : value;
Expand Down Expand Up @@ -369,6 +385,7 @@ module.exports = {
normalizeVerifierFollowupPolicy,
normalizeLedgerDisposition,
normalizeOptionalValue,
normalizeCliVersion,
summarizeTerminalDispositionSources,
formatTerminalDispositionMarkdown,
sourceKey,
Expand Down
4 changes: 4 additions & 0 deletions .github/scripts/terminal_disposition_coverage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
const unsupportedRecords = [];
const missingModelRecords = [];
const legacyMissingModelRecords = [];
let missingUnknownModeRecordCount = 0;
let verifierRecordCount = 0;

for (const raw of records) {
Expand Down Expand Up @@ -192,6 +193,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
if (isPreContractVerifierModelRecord(record, metadata, modelMetadataContract)) {
legacyMissingModelRecords.push(missingRecord);
} else {
if (!verifierMode) missingUnknownModeRecordCount += 1;

@chatgpt-codex-connector chatgpt-codex-connector Bot Apr 26, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Count explicit unknown modes in unknown-mode metric

The new unknown-mode counter only increments when verifier_mode is blank, so records that explicitly set verifier_mode to 'unknown' are added to missing_model_records but not included in missing_model_unknown_mode_record_count. That creates an inconsistent machine-readable contract (and markdown summary) for the exact category this field is meant to report, which can undercount unknown-mode gaps in post-contract enforcement windows.

Useful? React with 👍 / 👎.

missingModelRecords.push(missingRecord);
}
}
Expand All @@ -215,6 +217,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
unsupported_models: unsupportedModels,
unsupported_record_count: unsupportedRecords.length,
missing_model_record_count: missingModelRecords.length,
missing_model_unknown_mode_record_count: missingUnknownModeRecordCount,
legacy_missing_model_record_count: legacyMissingModelRecords.length,
selected_models: Object.fromEntries(
Object.entries(selectedModels).sort((a, b) => a[0].localeCompare(b[0]))
Expand Down Expand Up @@ -615,6 +618,7 @@ function formatTerminalDispositionCoverageMarkdown(report) {
`- Verifier model compatibility: ${modelCompatibility.status}`,
`- Unsupported verifier model records: ${modelCompatibility.unsupported_record_count}`,
`- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`,
`- Missing verifier model metadata records with unknown mode: ${modelCompatibility.missing_model_unknown_mode_record_count || 0}`,
`- Legacy missing verifier model metadata records: ${modelCompatibility.legacy_missing_model_record_count || 0}`
);
if (modelCompatibility.model_metadata_contract?.required_after) {
Expand Down
36 changes: 31 additions & 5 deletions scripts/aggregate_agent_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,32 @@ def _parse_timestamp(value: Any) -> _dt.datetime | None:
return None


def _normalize_version_text(value: Any) -> str:
if value is None:
return ""
text = str(value).strip()
if not text:
return ""
match = re.search(r"(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)", text)
return match.group(1) if match else text


def _normalize_cli_version(value: Any) -> str:
text = str(value).strip() if value is not None else ""
if not text:
return ""
version = _normalize_version_text(text)
lower = text.lower().replace("_", "-")
if version and re.search(r"\bcodex(?:-|\s+)cli\b|\bopenai/codex\b|\bcodex\b", lower):
return f"codex-cli {version}".lower()
return lower


def _normalize_counter_token(value: Any, fallback: str = "unknown") -> str:
text = str(value).strip().lower().replace("_", "-") if value is not None else ""
return text or fallback


def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Path]:
if metrics_paths:
return [Path(path) for path in metrics_paths if path]
Expand Down Expand Up @@ -586,7 +612,7 @@ def _summarise_verifier(
)
cli_version_text = str(cli_version).strip() if cli_version is not None else ""
if cli_version_text:
verifier_cli_versions[cli_version_text.lower()] += 1
verifier_cli_versions[_normalize_cli_version(cli_version_text)] += 1
verifier_mode = str(entry.get("verifier_mode") or "").strip().lower()
if verifier_mode:
verifier_modes[verifier_mode] += 1
Expand Down Expand Up @@ -752,12 +778,12 @@ def _summarise_codex_cli_freshness(entries: list[dict[str, Any]]) -> dict[str, A
max_patch_delta = 0
update_targets = Counter()
for entry in entries:
status = str(entry.get("status") or "unknown")
status = _normalize_counter_token(entry.get("status"))
statuses[status] += 1
package = str(entry.get("package") or "unknown")
package = str(entry.get("package") or "unknown").strip() or "unknown"
packages[package] += 1
pinned = str(entry.get("pinned_version") or "unknown")
latest = str(entry.get("latest_version") or "unknown")
pinned = _normalize_version_text(entry.get("pinned_version")) or "unknown"
latest = _normalize_version_text(entry.get("latest_version")) or "unknown"
pinned_versions[pinned] += 1
latest_versions[latest] += 1
delta = entry.get("version_delta")
Expand Down
2 changes: 2 additions & 0 deletions scripts/check_codex_cli_freshness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ def build_contract(
query_error: str = "",
generated_at: str | None = None,
) -> dict[str, Any]:
pinned_version = _version_text(pinned_version)
latest_version = _version_text(latest_version) if latest_version else ""
pinned_tuple = parse_semver(pinned_version)
latest_tuple = parse_semver(latest_version) if latest_version else None
status = "unknown"
Expand Down
17 changes: 17 additions & 0 deletions templates/consumer-repo/.github/scripts/terminal_disposition.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,18 @@ function normalizeToken(value, fallback = 'unknown') {
return normalized || fallback;
}

function normalizeCliVersion(value) {
const text = cleanString(value);
if (!text) return '';
const versionMatch = text.match(/(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)/);
const version = versionMatch ? versionMatch[1] : '';
const lower = text.toLowerCase().replace(/_/g, '-');
if (version && /\bcodex(?:-|\s+)cli\b|\bopenai\/codex\b|\bcodex\b/.test(lower)) {
return `codex-cli ${version}`;
}
return lower;
}

function normalizeSourceType(value) {
const text = cleanString(value).toLowerCase();
if (!text) return 'unknown';
Expand All @@ -47,6 +59,10 @@ function normalizeOptionalValue(key, value) {
const parsed = cleanBool(value);
return parsed === null ? undefined : parsed;
}
if (key === 'llm_cli_version' || key === 'codex_cli_version' || key === 'cli_version') {
const normalized = normalizeCliVersion(value);
return normalized || undefined;
}
const cleaned = typeof value === 'boolean' ? value : cleanString(value);
if (cleaned === '') return undefined;
return typeof value === 'string' ? cleaned : value;
Expand Down Expand Up @@ -369,6 +385,7 @@ module.exports = {
normalizeVerifierFollowupPolicy,
normalizeLedgerDisposition,
normalizeOptionalValue,
normalizeCliVersion,
summarizeTerminalDispositionSources,
formatTerminalDispositionMarkdown,
sourceKey,
Expand Down
4 changes: 4 additions & 0 deletions templates/consumer-repo/.github/scripts/terminal_disposition_coverage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
const unsupportedRecords = [];
const missingModelRecords = [];
const legacyMissingModelRecords = [];
let missingUnknownModeRecordCount = 0;
let verifierRecordCount = 0;

for (const raw of records) {
Expand Down Expand Up @@ -192,6 +193,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
if (isPreContractVerifierModelRecord(record, metadata, modelMetadataContract)) {
legacyMissingModelRecords.push(missingRecord);
} else {
if (!verifierMode) missingUnknownModeRecordCount += 1;
missingModelRecords.push(missingRecord);
}
}
Expand All @@ -215,6 +217,7 @@ function summarizeVerifierModelCompatibility(records = [], options = {}) {
unsupported_models: unsupportedModels,
unsupported_record_count: unsupportedRecords.length,
missing_model_record_count: missingModelRecords.length,
missing_model_unknown_mode_record_count: missingUnknownModeRecordCount,
legacy_missing_model_record_count: legacyMissingModelRecords.length,
selected_models: Object.fromEntries(
Object.entries(selectedModels).sort((a, b) => a[0].localeCompare(b[0]))
Expand Down Expand Up @@ -615,6 +618,7 @@ function formatTerminalDispositionCoverageMarkdown(report) {
`- Verifier model compatibility: ${modelCompatibility.status}`,
`- Unsupported verifier model records: ${modelCompatibility.unsupported_record_count}`,
`- Missing verifier model metadata records: ${modelCompatibility.missing_model_record_count}`,
`- Missing verifier model metadata records with unknown mode: ${modelCompatibility.missing_model_unknown_mode_record_count || 0}`,
`- Legacy missing verifier model metadata records: ${modelCompatibility.legacy_missing_model_record_count || 0}`
);
if (modelCompatibility.model_metadata_contract?.required_after) {
Expand Down
36 changes: 31 additions & 5 deletions templates/consumer-repo/scripts/aggregate_agent_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,32 @@ def _parse_timestamp(value: Any) -> _dt.datetime | None:
return None


def _normalize_version_text(value: Any) -> str:
if value is None:
return ""
text = str(value).strip()
if not text:
return ""
match = re.search(r"(\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?)", text)
return match.group(1) if match else text


def _normalize_cli_version(value: Any) -> str:
text = str(value).strip() if value is not None else ""
if not text:
return ""
version = _normalize_version_text(text)
lower = text.lower().replace("_", "-")
if version and re.search(r"\bcodex(?:-|\s+)cli\b|\bopenai/codex\b|\bcodex\b", lower):
return f"codex-cli {version}".lower()
return lower


def _normalize_counter_token(value: Any, fallback: str = "unknown") -> str:
text = str(value).strip().lower().replace("_", "-") if value is not None else ""
return text or fallback


def _gather_metrics_files(metrics_paths: list[str], metrics_dir: str) -> list[Path]:
if metrics_paths:
return [Path(path) for path in metrics_paths if path]
Expand Down Expand Up @@ -586,7 +612,7 @@ def _summarise_verifier(
)
cli_version_text = str(cli_version).strip() if cli_version is not None else ""
if cli_version_text:
verifier_cli_versions[cli_version_text.lower()] += 1
verifier_cli_versions[_normalize_cli_version(cli_version_text)] += 1
verifier_mode = str(entry.get("verifier_mode") or "").strip().lower()
if verifier_mode:
verifier_modes[verifier_mode] += 1
Expand Down Expand Up @@ -752,12 +778,12 @@ def _summarise_codex_cli_freshness(entries: list[dict[str, Any]]) -> dict[str, A
max_patch_delta = 0
update_targets = Counter()
for entry in entries:
status = str(entry.get("status") or "unknown")
status = _normalize_counter_token(entry.get("status"))
statuses[status] += 1
package = str(entry.get("package") or "unknown")
package = str(entry.get("package") or "unknown").strip() or "unknown"
packages[package] += 1
pinned = str(entry.get("pinned_version") or "unknown")
latest = str(entry.get("latest_version") or "unknown")
pinned = _normalize_version_text(entry.get("pinned_version")) or "unknown"
latest = _normalize_version_text(entry.get("latest_version")) or "unknown"
pinned_versions[pinned] += 1
latest_versions[latest] += 1
delta = entry.get("version_delta")
Expand Down
8 changes: 4 additions & 4 deletions tests/scripts/test_aggregate_agent_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ def test_build_summary_formats_sections() -> None:
"schema": "workflows-codex-cli-freshness/v1",
"package": "@openai/codex",
"component": "agents-verifier",
"status": "outdated",
"pinned_version": "0.125.0",
"latest_version": "0.127.3",
"status": " OUTDATED ",
"pinned_version": "@openai/codex@0.125.0",
"latest_version": "v0.127.3\n",
"version_delta": {"major": 0, "minor": 2, "patch": 3},
"update_targets": [
{"path": ".github/workflows/reusable-agents-verifier.yml"},
Expand Down Expand Up @@ -723,7 +723,7 @@ def test_summary_helpers_cover_branches() -> None:
"pr_number": 304,
"disposition": "verified-pass",
"llm_model": "gpt-5.4",
"codex_cli_version": "Codex-CLI 0.125.0",
"codex_cli_version": "Codex CLI v0.125.0",
"verifier_mode": "checkbox",
},
]
Expand Down
13 changes: 13 additions & 0 deletions tests/scripts/test_check_codex_cli_freshness.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,19 @@ def test_build_contract_reports_outdated_pin() -> None:
assert "@openai/codex@0.127.3" in markdown


def test_build_contract_normalizes_version_strings() -> None:
report = check_codex_cli_freshness.build_contract(
pinned_version="@openai/codex@0.125.0",
latest_version="v0.127.3\n",
generated_at="2026-04-26T19:00:00Z",
)

assert report["status"] == "outdated"
assert report["pinned_version"] == "0.125.0"
assert report["latest_version"] == "0.127.3"
assert report["version_delta"] == {"major": 0, "minor": 2, "patch": 3}


def test_query_latest_npm_version_uses_isolated_cache(monkeypatch) -> None:
captured_env = {}

Expand Down
Loading