diff --git a/.github/scripts/__tests__/terminal-disposition.test.js b/.github/scripts/__tests__/terminal-disposition.test.js index aa5b63cdd..663615370 100644 --- a/.github/scripts/__tests__/terminal-disposition.test.js +++ b/.github/scripts/__tests__/terminal-disposition.test.js @@ -25,6 +25,7 @@ test('normalizes terminal disposition records with stable source keys', () => { artifactFamily: 'verifier-terminal-disposition', llmModel: 'gpt-5.3-codex', modelSelectionReason: 'default', + llmCliVersion: 'codex-cli 0.125.0', verifierMode: 'checkbox', needsHuman: false, timestamp: '2026-04-25T00:00:00Z', @@ -44,6 +45,7 @@ test('normalizes terminal disposition records with stable source keys', () => { assert.equal(record.artifact_family, 'verifier-terminal-disposition'); assert.equal(record.llm_model, 'gpt-5.3-codex'); assert.equal(record.model_selection_reason, 'default'); + assert.equal(record.llm_cli_version, 'codex-cli 0.125.0'); assert.equal(record.verifier_mode, 'checkbox'); assert.equal(record.needs_human, false); }); diff --git a/.github/scripts/terminal_disposition.js b/.github/scripts/terminal_disposition.js index 846624fab..4e2df164b 100644 --- a/.github/scripts/terminal_disposition.js +++ b/.github/scripts/terminal_disposition.js @@ -211,6 +211,7 @@ function normalizeTerminalDisposition(input = {}) { dispatch_outcome: input.dispatch_outcome ?? input.dispatchOutcome, llm_model: input.llm_model ?? input.llmModel ?? input.model, model_selection_reason: input.model_selection_reason ?? input.modelSelectionReason, + llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version, verifier_mode: input.verifier_mode ?? input.verifierMode, }; diff --git a/.github/workflows/reusable-agents-verifier.yml b/.github/workflows/reusable-agents-verifier.yml index 7efe5d1f6..e12fb17ca 100644 --- a/.github/workflows/reusable-agents-verifier.yml +++ b/.github/workflows/reusable-agents-verifier.yml @@ -484,11 +484,14 @@ jobs: echo "Codex auth configured" - name: Install Codex CLI + id: codex_cli if: steps.context.outputs.should_run == 'true' && inputs.mode != 'evaluate' run: | set -euo pipefail npm install -g "@openai/codex@0.125.0" - echo "Installed Codex CLI version: $(codex --version)" >> "$GITHUB_STEP_SUMMARY" + codex_cli_version="$(codex --version | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')" + echo "version=$codex_cli_version" >> "$GITHUB_OUTPUT" + echo "Installed Codex CLI version: $codex_cli_version" >> "$GITHUB_STEP_SUMMARY" - name: Resolve Codex verifier model id: codex_model @@ -1321,6 +1324,7 @@ jobs: SKIP_REASON: ${{ steps.context.outputs.skip_reason }} CODEX_MODEL: ${{ steps.codex.outputs.model || steps.codex_model.outputs.model }} CODEX_MODEL_SELECTION_REASON: ${{ steps.codex.outputs.selection_reason || steps.codex_model.outputs.selection_reason }} + CODEX_CLI_VERSION: ${{ steps.codex_cli.outputs.version }} VERIFIER_MODE: ${{ inputs.mode }} CHAIN_DEPTH: ${{ steps.context.outputs.chain_depth || '0' }} run: | @@ -1341,6 +1345,7 @@ jobs: issue_number = os.environ.get("ISSUE_NUMBER") or "" codex_model = os.environ.get("CODEX_MODEL") or "" codex_model_selection_reason = os.environ.get("CODEX_MODEL_SELECTION_REASON") or "" + codex_cli_version = os.environ.get("CODEX_CLI_VERSION") or "" verifier_mode = os.environ.get("VERIFIER_MODE") or "" chain_depth = int(os.environ.get("CHAIN_DEPTH") or 0) @@ -1373,6 +1378,7 @@ jobs: "skip_reason": skip_reason, "codex_model": codex_model, "codex_model_selection_reason": codex_model_selection_reason, + "codex_cli_version": codex_cli_version, "verifier_mode": verifier_mode, "chain_depth": chain_depth, "recorded_at": datetime.now(timezone.utc).isoformat(), @@ -1395,6 +1401,7 @@ jobs: SKIP_REASON: ${{ steps.context.outputs.skip_reason }} CODEX_MODEL: ${{ steps.codex.outputs.model || steps.codex_model.outputs.model }} CODEX_MODEL_SELECTION_REASON: ${{ steps.codex.outputs.selection_reason || steps.codex_model.outputs.selection_reason }} + CODEX_CLI_VERSION: ${{ steps.codex_cli.outputs.version }} VERIFIER_MODE: ${{ inputs.mode }} CHAIN_DEPTH: ${{ steps.context.outputs.chain_depth || '0' }} FINAL_VERDICT: >- @@ -1449,6 +1456,7 @@ jobs: const codexModel = process.env.CODEX_MODEL || metrics.codex_model || ''; const codexModelSelectionReason = process.env.CODEX_MODEL_SELECTION_REASON || metrics.codex_model_selection_reason || ''; + const codexCliVersion = process.env.CODEX_CLI_VERSION || metrics.codex_cli_version || ''; const verifierMode = process.env.VERIFIER_MODE || metrics.verifier_mode || ''; const chainDepth = Number.parseInt(process.env.CHAIN_DEPTH || metrics.chain_depth || '0', 10) || 0; @@ -1513,6 +1521,7 @@ jobs: dispatch_outcome: shouldRun ? 'verifier-ran' : 'verifier-skipped', llm_model: codexModel || undefined, model_selection_reason: codexModelSelectionReason || undefined, + llm_cli_version: codexCliVersion || undefined, verifier_mode: verifierMode || undefined, })); diff --git a/scripts/aggregate_agent_metrics.py b/scripts/aggregate_agent_metrics.py index cfbea5907..3036e9524 100755 --- a/scripts/aggregate_agent_metrics.py +++ b/scripts/aggregate_agent_metrics.py @@ -506,6 +506,7 @@ def _summarise_verifier( terminal_sources = Counter() verifier_models = Counter() model_selection_reasons = Counter() + verifier_cli_versions = Counter() unsupported_verifier_models = Counter() unsupported_model_dispositions = Counter() missing_verifier_model_metadata = Counter() @@ -574,6 +575,14 @@ def _summarise_verifier( ) if model_selection_reason: model_selection_reasons[str(model_selection_reason)] += 1 + cli_version = ( + entry.get("codex_cli_version") + or entry.get("llm_cli_version") + or entry.get("cli_version") + ) + cli_version_text = str(cli_version).strip() if cli_version is not None else "" + if cli_version_text: + verifier_cli_versions[cli_version_text.lower()] += 1 verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() if verifier_mode: verifier_modes[verifier_mode] += 1 @@ -623,6 +632,7 @@ def _summarise_verifier( "terminal_dispositions": terminal_dispositions, "terminal_sources": terminal_sources, "verifier_models": verifier_models, + "verifier_cli_versions": verifier_cli_versions, "unsupported_verifier_models": unsupported_verifier_models, "unsupported_model_dispositions": unsupported_model_dispositions, "missing_verifier_model_metadata": missing_verifier_model_metadata, @@ -1019,6 +1029,7 @@ def build_summary( f"{verifier['ledger_policy_depth_limit_exceeded']}" ), f"- Verifier models: {_format_counter(verifier['verifier_models'])}", + f"- Verifier CLI versions: {_format_counter(verifier['verifier_cli_versions'])}", f"- Unsupported verifier models: {_format_counter(verifier['unsupported_verifier_models'])}", ( "- Unsupported model dispositions: " diff --git a/templates/consumer-repo/.github/scripts/terminal_disposition.js b/templates/consumer-repo/.github/scripts/terminal_disposition.js index 846624fab..4e2df164b 100644 --- a/templates/consumer-repo/.github/scripts/terminal_disposition.js +++ b/templates/consumer-repo/.github/scripts/terminal_disposition.js @@ -211,6 +211,7 @@ function normalizeTerminalDisposition(input = {}) { dispatch_outcome: input.dispatch_outcome ?? input.dispatchOutcome, llm_model: input.llm_model ?? input.llmModel ?? input.model, model_selection_reason: input.model_selection_reason ?? input.modelSelectionReason, + llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version, verifier_mode: input.verifier_mode ?? input.verifierMode, }; diff --git a/templates/consumer-repo/scripts/aggregate_agent_metrics.py b/templates/consumer-repo/scripts/aggregate_agent_metrics.py index cfbea5907..3036e9524 100755 --- a/templates/consumer-repo/scripts/aggregate_agent_metrics.py +++ b/templates/consumer-repo/scripts/aggregate_agent_metrics.py @@ -506,6 +506,7 @@ def _summarise_verifier( terminal_sources = Counter() verifier_models = Counter() model_selection_reasons = Counter() + verifier_cli_versions = Counter() unsupported_verifier_models = Counter() unsupported_model_dispositions = Counter() missing_verifier_model_metadata = Counter() @@ -574,6 +575,14 @@ def _summarise_verifier( ) if model_selection_reason: model_selection_reasons[str(model_selection_reason)] += 1 + cli_version = ( + entry.get("codex_cli_version") + or entry.get("llm_cli_version") + or entry.get("cli_version") + ) + cli_version_text = str(cli_version).strip() if cli_version is not None else "" + if cli_version_text: + verifier_cli_versions[cli_version_text.lower()] += 1 verifier_mode = str(entry.get("verifier_mode") or "").strip().lower() if verifier_mode: verifier_modes[verifier_mode] += 1 @@ -623,6 +632,7 @@ def _summarise_verifier( "terminal_dispositions": terminal_dispositions, "terminal_sources": terminal_sources, "verifier_models": verifier_models, + "verifier_cli_versions": verifier_cli_versions, "unsupported_verifier_models": unsupported_verifier_models, "unsupported_model_dispositions": unsupported_model_dispositions, "missing_verifier_model_metadata": missing_verifier_model_metadata, @@ -1019,6 +1029,7 @@ def build_summary( f"{verifier['ledger_policy_depth_limit_exceeded']}" ), f"- Verifier models: {_format_counter(verifier['verifier_models'])}", + f"- Verifier CLI versions: {_format_counter(verifier['verifier_cli_versions'])}", f"- Unsupported verifier models: {_format_counter(verifier['unsupported_verifier_models'])}", ( "- Unsupported model dispositions: " diff --git a/tests/scripts/test_aggregate_agent_metrics.py b/tests/scripts/test_aggregate_agent_metrics.py index 62fb82b83..1ca7c9ace 100644 --- a/tests/scripts/test_aggregate_agent_metrics.py +++ b/tests/scripts/test_aggregate_agent_metrics.py @@ -49,6 +49,7 @@ def test_build_summary_formats_sections() -> None: "disposition": "follow-up-created", "llm_model": "gpt-5.3-codex", "model_selection_reason": "default", + "llm_cli_version": "codex-cli 0.125.0", "verifier_mode": "checkbox", }, { @@ -102,6 +103,7 @@ def test_build_summary_formats_sections() -> None: assert "Verifier follow-up policy triggers: verifier-concerns (1)" in summary assert "Verifier follow-up depth-limit records: 0" in summary assert "Verifier models: gpt-5.3-codex (1)" in summary + assert "Verifier CLI versions: codex-cli 0.125.0 (1)" in summary assert "Unsupported verifier models: n/a" in summary assert "Unsupported model dispositions: n/a" in summary assert "Missing verifier model metadata: n/a" in summary @@ -112,6 +114,7 @@ def test_build_summary_formats_sections() -> None: contract = aggregate_agent_metrics.build_summary_contract(entries, []) verifier_contract = contract["summaries"]["verifier"] assert verifier_contract["verifier_models"] == {"gpt-5.3-codex": 1} + assert verifier_contract["verifier_cli_versions"] == {"codex-cli 0.125.0": 1} assert verifier_contract["model_selection_reasons"] == {"default": 1} assert verifier_contract["ledger_policy_actions"] == {"create-follow-up": 1} assert verifier_contract["ledger_policy_triggers"] == {"verifier-concerns": 1} @@ -642,6 +645,7 @@ def test_summary_helpers_cover_branches() -> None: "disposition": "follow-up-created", "llm_model": "gpt-5.3-codex", "model_selection_reason": "fallback-unsupported-chatgpt-codex-model", + "llm_cli_version": "codex-cli 0.125.0", "verifier_mode": " Checkbox ", }, { @@ -650,6 +654,7 @@ def test_summary_helpers_cover_branches() -> None: "pr_number": 304, "disposition": "verified-pass", "llm_model": "gpt-5.4", + "codex_cli_version": "Codex-CLI 0.125.0", "verifier_mode": "checkbox", }, ] @@ -658,6 +663,7 @@ def test_summary_helpers_cover_branches() -> None: assert verifier_with_terminal["terminal_records"] == 2 assert verifier_with_terminal["verifier_models"]["gpt-5.3-codex"] == 1 assert verifier_with_terminal["verifier_models"]["gpt-5.4"] == 1 + assert verifier_with_terminal["verifier_cli_versions"]["codex-cli 0.125.0"] == 2 assert verifier_with_terminal["unsupported_verifier_models"] == Counter() assert verifier_with_terminal["missing_verifier_model_metadata"] == Counter() assert ( @@ -679,6 +685,7 @@ def test_summary_contract_exposes_runtime_verifier_model_fallback() -> None: "disposition": "verified-pass", "llm_model": "gpt-5.4", "model_selection_reason": "runtime-fallback-model-unavailable", + "llm_cli_version": "codex-cli 0.125.0", "verifier_mode": "checkbox", "timestamp": "2026-04-26T14:58:00Z", } @@ -690,6 +697,7 @@ def test_summary_contract_exposes_runtime_verifier_model_fallback() -> None: assert contract["record_buckets"] == {"terminal_disposition": 1} assert verifier["terminal_records"] == 1 assert verifier["verifier_models"] == {"gpt-5.4": 1} + assert verifier["verifier_cli_versions"] == {"codex-cli 0.125.0": 1} assert verifier["model_selection_reasons"] == {"runtime-fallback-model-unavailable": 1} assert verifier["verifier_modes"] == {"checkbox": 1} diff --git a/tests/workflows/test_verifier_terminal_disposition.py b/tests/workflows/test_verifier_terminal_disposition.py index d14c3b579..26221b1d8 100644 --- a/tests/workflows/test_verifier_terminal_disposition.py +++ b/tests/workflows/test_verifier_terminal_disposition.py @@ -1,14 +1,39 @@ +import re from pathlib import Path import yaml ROOT = Path(__file__).resolve().parents[2] +MIN_CODEX_CLI_BY_MODEL = { + "gpt-5.5": (0, 125, 0), + "gpt-5.4": (0, 125, 0), + "gpt-5.3-codex": (0, 101, 0), +} + def _load_yaml(path: Path) -> dict: return yaml.safe_load(path.read_text(encoding="utf-8")) or {} +def _parse_version_tuple(value: str) -> tuple[int, int, int]: + match = re.search(r"(\d+)\.(\d+)\.(\d+)", value) + assert match, f"Could not parse semantic version from: {value!r}" + return tuple(int(part) for part in match.groups()) + + +def _extract_codex_cli_pin(run_script: str) -> tuple[int, int, int]: + match = re.search(r'@openai/codex@([0-9]+\.[0-9]+\.[0-9]+)"', run_script) + assert match, "Install Codex CLI step must pin @openai/codex to an explicit version" + return _parse_version_tuple(match.group(1)) + + +def _model_candidates(resolve_step: dict) -> list[str]: + default_model = resolve_step["env"]["DEFAULT_CODEX_MODEL"] + fallback_models = resolve_step["env"]["FALLBACK_CODEX_MODELS"].split() + return [default_model, *fallback_models] + + def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: workflow = _load_yaml(ROOT / ".github/workflows/reusable-agents-verifier.yml") steps = workflow["jobs"]["verifier"]["steps"] @@ -27,6 +52,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: ) assert resolve_step.get("id") == "codex_model" + assert install_step.get("id") == "codex_cli" assert ( resolve_step.get("if") == "steps.context.outputs.should_run == 'true' && inputs.mode != 'evaluate'" @@ -52,6 +78,8 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: "${{ steps.codex.outputs.selection_reason || " "steps.codex_model.outputs.selection_reason }}" ) + assert collect_step["env"]["CODEX_CLI_VERSION"] == "${{ steps.codex_cli.outputs.version }}" + assert '"codex_cli_version": codex_cli_version' in collect_step["run"] assert write_step.get("if") == "always()" assert write_step["env"]["CODEX_MODEL"] == ( "${{ steps.codex.outputs.model || steps.codex_model.outputs.model }}" @@ -60,6 +88,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: "${{ steps.codex.outputs.selection_reason || " "steps.codex_model.outputs.selection_reason }}" ) + assert write_step["env"]["CODEX_CLI_VERSION"] == "${{ steps.codex_cli.outputs.version }}" assert write_step["env"]["SOURCE_ISSUE_NUMBERS_JSON"] == ( "${{ steps.context.outputs.issue_numbers || '[]' }}" ) @@ -73,6 +102,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: assert "CHAIN_DEPTH" in write_step["env"] assert "llm_model" in write_step["run"] assert "model_selection_reason" in write_step["run"] + assert "llm_cli_version" in write_step["run"] assert "source-issue" in write_step["run"] assert "pull-request" in write_step["run"] assert "verified-pass" in write_step["run"] @@ -84,3 +114,27 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None: assert "agent-metrics/verifier-followup-ledger.ndjson" in upload_step["with"]["path"] assert upload_step["with"]["if-no-files-found"] == "error" assert upload_step["with"]["retention-days"] == 14 + + +def test_reusable_verifier_codex_model_cli_compatibility_contract() -> None: + workflow = _load_yaml(ROOT / ".github/workflows/reusable-agents-verifier.yml") + steps = workflow["jobs"]["verifier"]["steps"] + resolve_step = next( + step for step in steps if step.get("name") == "Resolve Codex verifier model" + ) + install_step = next(step for step in steps if step.get("name") == "Install Codex CLI") + + installed_cli = _extract_codex_cli_pin(install_step["run"]) + candidates = _model_candidates(resolve_step) + unreviewed_models = [model for model in candidates if model not in MIN_CODEX_CLI_BY_MODEL] + assert not unreviewed_models, ( + "Verifier Codex model candidates need an explicit reviewed minimum CLI mapping: " + + ", ".join(unreviewed_models) + ) + + for model in candidates: + minimum_cli = MIN_CODEX_CLI_BY_MODEL[model] + assert installed_cli >= minimum_cli, ( + f"Verifier model {model} requires @openai/codex >= {minimum_cli}, " + f"but reusable-agents-verifier.yml installs {installed_cli}." + )