[codex] Add verifier CLI compatibility telemetry

.github/scripts/__tests__/terminal-disposition.test.js

@@ -25,6 +25,7 @@ test('normalizes terminal disposition records with stable source keys', () => {
     artifactFamily: 'verifier-terminal-disposition',
     llmModel: 'gpt-5.3-codex',
     modelSelectionReason: 'default',
+    llmCliVersion: 'codex-cli 0.125.0',
     verifierMode: 'checkbox',
     needsHuman: false,
     timestamp: '2026-04-25T00:00:00Z',
@@ -44,6 +45,7 @@ test('normalizes terminal disposition records with stable source keys', () => {
   assert.equal(record.artifact_family, 'verifier-terminal-disposition');
   assert.equal(record.llm_model, 'gpt-5.3-codex');
   assert.equal(record.model_selection_reason, 'default');
+  assert.equal(record.llm_cli_version, 'codex-cli 0.125.0');
   assert.equal(record.verifier_mode, 'checkbox');
   assert.equal(record.needs_human, false);
 });

.github/scripts/terminal_disposition.js

@@ -211,6 +211,7 @@ function normalizeTerminalDisposition(input = {}) {
     dispatch_outcome: input.dispatch_outcome ?? input.dispatchOutcome,
     llm_model: input.llm_model ?? input.llmModel ?? input.model,
     model_selection_reason: input.model_selection_reason ?? input.modelSelectionReason,
+    llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version,
     verifier_mode: input.verifier_mode ?? input.verifierMode,
   };
 

.github/workflows/reusable-agents-verifier.yml

@@ -484,11 +484,14 @@ jobs:
           echo "Codex auth configured"
 
       - name: Install Codex CLI
+        id: codex_cli
         if: steps.context.outputs.should_run == 'true' && inputs.mode != 'evaluate'
         run: |
           set -euo pipefail
           npm install -g "@openai/codex@0.125.0"
-          echo "Installed Codex CLI version: $(codex --version)" >> "$GITHUB_STEP_SUMMARY"
+          codex_cli_version="$(codex --version | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')"
+          echo "version=$codex_cli_version" >> "$GITHUB_OUTPUT"
+          echo "Installed Codex CLI version: $codex_cli_version" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Resolve Codex verifier model
         id: codex_model
@@ -1321,6 +1324,7 @@ jobs:
           SKIP_REASON: ${{ steps.context.outputs.skip_reason }}
           CODEX_MODEL: ${{ steps.codex.outputs.model || steps.codex_model.outputs.model }}
           CODEX_MODEL_SELECTION_REASON: ${{ steps.codex.outputs.selection_reason || steps.codex_model.outputs.selection_reason }}
+          CODEX_CLI_VERSION: ${{ steps.codex_cli.outputs.version }}
           VERIFIER_MODE: ${{ inputs.mode }}
           CHAIN_DEPTH: ${{ steps.context.outputs.chain_depth || '0' }}
         run: |
@@ -1341,6 +1345,7 @@ jobs:
           issue_number = os.environ.get("ISSUE_NUMBER") or ""
           codex_model = os.environ.get("CODEX_MODEL") or ""
           codex_model_selection_reason = os.environ.get("CODEX_MODEL_SELECTION_REASON") or ""
+          codex_cli_version = os.environ.get("CODEX_CLI_VERSION") or ""
           verifier_mode = os.environ.get("VERIFIER_MODE") or ""
           chain_depth = int(os.environ.get("CHAIN_DEPTH") or 0)
 
@@ -1373,6 +1378,7 @@ jobs:
               "skip_reason": skip_reason,
               "codex_model": codex_model,
               "codex_model_selection_reason": codex_model_selection_reason,
+              "codex_cli_version": codex_cli_version,
               "verifier_mode": verifier_mode,
               "chain_depth": chain_depth,
               "recorded_at": datetime.now(timezone.utc).isoformat(),
@@ -1395,6 +1401,7 @@ jobs:
           SKIP_REASON: ${{ steps.context.outputs.skip_reason }}
           CODEX_MODEL: ${{ steps.codex.outputs.model || steps.codex_model.outputs.model }}
           CODEX_MODEL_SELECTION_REASON: ${{ steps.codex.outputs.selection_reason || steps.codex_model.outputs.selection_reason }}
+          CODEX_CLI_VERSION: ${{ steps.codex_cli.outputs.version }}
           VERIFIER_MODE: ${{ inputs.mode }}
           CHAIN_DEPTH: ${{ steps.context.outputs.chain_depth || '0' }}
           FINAL_VERDICT: >-
@@ -1449,6 +1456,7 @@ jobs:
           const codexModel = process.env.CODEX_MODEL || metrics.codex_model || '';
           const codexModelSelectionReason =
             process.env.CODEX_MODEL_SELECTION_REASON || metrics.codex_model_selection_reason || '';
+          const codexCliVersion = process.env.CODEX_CLI_VERSION || metrics.codex_cli_version || '';
           const verifierMode = process.env.VERIFIER_MODE || metrics.verifier_mode || '';
           const chainDepth =
             Number.parseInt(process.env.CHAIN_DEPTH || metrics.chain_depth || '0', 10) || 0;
@@ -1513,6 +1521,7 @@ jobs:
             dispatch_outcome: shouldRun ? 'verifier-ran' : 'verifier-skipped',
             llm_model: codexModel || undefined,
             model_selection_reason: codexModelSelectionReason || undefined,
+            llm_cli_version: codexCliVersion || undefined,
             verifier_mode: verifierMode || undefined,
           }));
 

scripts/aggregate_agent_metrics.py

@@ -506,6 +506,7 @@ def _summarise_verifier(
     terminal_sources = Counter()
     verifier_models = Counter()
     model_selection_reasons = Counter()
+    verifier_cli_versions = Counter()
     unsupported_verifier_models = Counter()
     unsupported_model_dispositions = Counter()
     missing_verifier_model_metadata = Counter()
@@ -574,6 +575,14 @@ def _summarise_verifier(
         )
         if model_selection_reason:
             model_selection_reasons[str(model_selection_reason)] += 1
+        cli_version = (
+            entry.get("codex_cli_version")
+            or entry.get("llm_cli_version")
+            or entry.get("cli_version")
+        )
+        cli_version_text = str(cli_version).strip() if cli_version is not None else ""
+        if cli_version_text:
+            verifier_cli_versions[cli_version_text.lower()] += 1
         verifier_mode = str(entry.get("verifier_mode") or "").strip().lower()
         if verifier_mode:
             verifier_modes[verifier_mode] += 1
@@ -623,6 +632,7 @@ def _summarise_verifier(
         "terminal_dispositions": terminal_dispositions,
         "terminal_sources": terminal_sources,
         "verifier_models": verifier_models,
+        "verifier_cli_versions": verifier_cli_versions,
         "unsupported_verifier_models": unsupported_verifier_models,
         "unsupported_model_dispositions": unsupported_model_dispositions,
         "missing_verifier_model_metadata": missing_verifier_model_metadata,
@@ -1019,6 +1029,7 @@ def build_summary(
                 f"{verifier['ledger_policy_depth_limit_exceeded']}"
             ),
             f"- Verifier models: {_format_counter(verifier['verifier_models'])}",
+            f"- Verifier CLI versions: {_format_counter(verifier['verifier_cli_versions'])}",
             f"- Unsupported verifier models: {_format_counter(verifier['unsupported_verifier_models'])}",
             (
                 "- Unsupported model dispositions: "

templates/consumer-repo/.github/scripts/terminal_disposition.js

@@ -211,6 +211,7 @@ function normalizeTerminalDisposition(input = {}) {
     dispatch_outcome: input.dispatch_outcome ?? input.dispatchOutcome,
     llm_model: input.llm_model ?? input.llmModel ?? input.model,
     model_selection_reason: input.model_selection_reason ?? input.modelSelectionReason,
+    llm_cli_version: input.llm_cli_version ?? input.llmCliVersion ?? input.cli_version,
     verifier_mode: input.verifier_mode ?? input.verifierMode,
   };
 

templates/consumer-repo/scripts/aggregate_agent_metrics.py

@@ -506,6 +506,7 @@ def _summarise_verifier(
     terminal_sources = Counter()
     verifier_models = Counter()
     model_selection_reasons = Counter()
+    verifier_cli_versions = Counter()
     unsupported_verifier_models = Counter()
     unsupported_model_dispositions = Counter()
     missing_verifier_model_metadata = Counter()
@@ -574,6 +575,14 @@ def _summarise_verifier(
         )
         if model_selection_reason:
             model_selection_reasons[str(model_selection_reason)] += 1
+        cli_version = (
+            entry.get("codex_cli_version")
+            or entry.get("llm_cli_version")
+            or entry.get("cli_version")
+        )
+        cli_version_text = str(cli_version).strip() if cli_version is not None else ""
+        if cli_version_text:
+            verifier_cli_versions[cli_version_text.lower()] += 1
         verifier_mode = str(entry.get("verifier_mode") or "").strip().lower()
         if verifier_mode:
             verifier_modes[verifier_mode] += 1
@@ -623,6 +632,7 @@ def _summarise_verifier(
         "terminal_dispositions": terminal_dispositions,
         "terminal_sources": terminal_sources,
         "verifier_models": verifier_models,
+        "verifier_cli_versions": verifier_cli_versions,
         "unsupported_verifier_models": unsupported_verifier_models,
         "unsupported_model_dispositions": unsupported_model_dispositions,
         "missing_verifier_model_metadata": missing_verifier_model_metadata,
@@ -1019,6 +1029,7 @@ def build_summary(
                 f"{verifier['ledger_policy_depth_limit_exceeded']}"
             ),
             f"- Verifier models: {_format_counter(verifier['verifier_models'])}",
+            f"- Verifier CLI versions: {_format_counter(verifier['verifier_cli_versions'])}",
             f"- Unsupported verifier models: {_format_counter(verifier['unsupported_verifier_models'])}",
             (
                 "- Unsupported model dispositions: "

tests/scripts/test_aggregate_agent_metrics.py

@@ -49,6 +49,7 @@ def test_build_summary_formats_sections() -> None:
             "disposition": "follow-up-created",
             "llm_model": "gpt-5.3-codex",
             "model_selection_reason": "default",
+            "llm_cli_version": "codex-cli 0.125.0",
             "verifier_mode": "checkbox",
         },
         {
@@ -102,6 +103,7 @@ def test_build_summary_formats_sections() -> None:
     assert "Verifier follow-up policy triggers: verifier-concerns (1)" in summary
     assert "Verifier follow-up depth-limit records: 0" in summary
     assert "Verifier models: gpt-5.3-codex (1)" in summary
+    assert "Verifier CLI versions: codex-cli 0.125.0 (1)" in summary
     assert "Unsupported verifier models: n/a" in summary
     assert "Unsupported model dispositions: n/a" in summary
     assert "Missing verifier model metadata: n/a" in summary
@@ -112,6 +114,7 @@ def test_build_summary_formats_sections() -> None:
     contract = aggregate_agent_metrics.build_summary_contract(entries, [])
     verifier_contract = contract["summaries"]["verifier"]
     assert verifier_contract["verifier_models"] == {"gpt-5.3-codex": 1}
+    assert verifier_contract["verifier_cli_versions"] == {"codex-cli 0.125.0": 1}
     assert verifier_contract["model_selection_reasons"] == {"default": 1}
     assert verifier_contract["ledger_policy_actions"] == {"create-follow-up": 1}
     assert verifier_contract["ledger_policy_triggers"] == {"verifier-concerns": 1}
@@ -642,6 +645,7 @@ def test_summary_helpers_cover_branches() -> None:
                 "disposition": "follow-up-created",
                 "llm_model": "gpt-5.3-codex",
                 "model_selection_reason": "fallback-unsupported-chatgpt-codex-model",
+                "llm_cli_version": "codex-cli 0.125.0",
                 "verifier_mode": " Checkbox ",
             },
             {
@@ -650,6 +654,7 @@ def test_summary_helpers_cover_branches() -> None:
                 "pr_number": 304,
                 "disposition": "verified-pass",
                 "llm_model": "gpt-5.4",
+                "codex_cli_version": "Codex-CLI 0.125.0",
                 "verifier_mode": "checkbox",
             },
         ]
@@ -658,6 +663,7 @@ def test_summary_helpers_cover_branches() -> None:
     assert verifier_with_terminal["terminal_records"] == 2
     assert verifier_with_terminal["verifier_models"]["gpt-5.3-codex"] == 1
     assert verifier_with_terminal["verifier_models"]["gpt-5.4"] == 1
+    assert verifier_with_terminal["verifier_cli_versions"]["codex-cli 0.125.0"] == 2
     assert verifier_with_terminal["unsupported_verifier_models"] == Counter()
     assert verifier_with_terminal["missing_verifier_model_metadata"] == Counter()
     assert (
@@ -679,6 +685,7 @@ def test_summary_contract_exposes_runtime_verifier_model_fallback() -> None:
             "disposition": "verified-pass",
             "llm_model": "gpt-5.4",
             "model_selection_reason": "runtime-fallback-model-unavailable",
+            "llm_cli_version": "codex-cli 0.125.0",
             "verifier_mode": "checkbox",
             "timestamp": "2026-04-26T14:58:00Z",
         }
@@ -690,6 +697,7 @@ def test_summary_contract_exposes_runtime_verifier_model_fallback() -> None:
     assert contract["record_buckets"] == {"terminal_disposition": 1}
     assert verifier["terminal_records"] == 1
     assert verifier["verifier_models"] == {"gpt-5.4": 1}
+    assert verifier["verifier_cli_versions"] == {"codex-cli 0.125.0": 1}
     assert verifier["model_selection_reasons"] == {"runtime-fallback-model-unavailable": 1}
     assert verifier["verifier_modes"] == {"checkbox": 1}
 

tests/workflows/test_verifier_terminal_disposition.py

@@ -1,14 +1,39 @@
+import re
 from pathlib import Path
 
 import yaml
 
 ROOT = Path(__file__).resolve().parents[2]
 
+MIN_CODEX_CLI_BY_MODEL = {
+    "gpt-5.5": (0, 125, 0),
+    "gpt-5.4": (0, 125, 0),
+    "gpt-5.3-codex": (0, 101, 0),
+}
+
 
 def _load_yaml(path: Path) -> dict:
     return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
 
 
+def _parse_version_tuple(value: str) -> tuple[int, int, int]:
+    match = re.search(r"(\d+)\.(\d+)\.(\d+)", value)
+    assert match, f"Could not parse semantic version from: {value!r}"
+    return tuple(int(part) for part in match.groups())
+
+
+def _extract_codex_cli_pin(run_script: str) -> tuple[int, int, int]:
+    match = re.search(r'@openai/codex@([0-9]+\.[0-9]+\.[0-9]+)"', run_script)
+    assert match, "Install Codex CLI step must pin @openai/codex to an explicit version"
+    return _parse_version_tuple(match.group(1))
+
+
+def _model_candidates(resolve_step: dict) -> list[str]:
+    default_model = resolve_step["env"]["DEFAULT_CODEX_MODEL"]
+    fallback_models = resolve_step["env"]["FALLBACK_CODEX_MODELS"].split()
+    return [default_model, *fallback_models]
+
+
 def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
     workflow = _load_yaml(ROOT / ".github/workflows/reusable-agents-verifier.yml")
     steps = workflow["jobs"]["verifier"]["steps"]
@@ -27,6 +52,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
     )
 
     assert resolve_step.get("id") == "codex_model"
+    assert install_step.get("id") == "codex_cli"
     assert (
         resolve_step.get("if")
         == "steps.context.outputs.should_run == 'true' && inputs.mode != 'evaluate'"
@@ -52,6 +78,8 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
         "${{ steps.codex.outputs.selection_reason || "
         "steps.codex_model.outputs.selection_reason }}"
     )
+    assert collect_step["env"]["CODEX_CLI_VERSION"] == "${{ steps.codex_cli.outputs.version }}"
+    assert '"codex_cli_version": codex_cli_version' in collect_step["run"]
     assert write_step.get("if") == "always()"
     assert write_step["env"]["CODEX_MODEL"] == (
         "${{ steps.codex.outputs.model || steps.codex_model.outputs.model }}"
@@ -60,6 +88,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
         "${{ steps.codex.outputs.selection_reason || "
         "steps.codex_model.outputs.selection_reason }}"
     )
+    assert write_step["env"]["CODEX_CLI_VERSION"] == "${{ steps.codex_cli.outputs.version }}"
     assert write_step["env"]["SOURCE_ISSUE_NUMBERS_JSON"] == (
         "${{ steps.context.outputs.issue_numbers || '[]' }}"
     )
@@ -73,6 +102,7 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
     assert "CHAIN_DEPTH" in write_step["env"]
     assert "llm_model" in write_step["run"]
     assert "model_selection_reason" in write_step["run"]
+    assert "llm_cli_version" in write_step["run"]
     assert "source-issue" in write_step["run"]
     assert "pull-request" in write_step["run"]
     assert "verified-pass" in write_step["run"]
@@ -84,3 +114,27 @@ def test_reusable_verifier_uploads_terminal_disposition_artifact() -> None:
     assert "agent-metrics/verifier-followup-ledger.ndjson" in upload_step["with"]["path"]
     assert upload_step["with"]["if-no-files-found"] == "error"
     assert upload_step["with"]["retention-days"] == 14
+
+
+def test_reusable_verifier_codex_model_cli_compatibility_contract() -> None:
+    workflow = _load_yaml(ROOT / ".github/workflows/reusable-agents-verifier.yml")
+    steps = workflow["jobs"]["verifier"]["steps"]
+    resolve_step = next(
+        step for step in steps if step.get("name") == "Resolve Codex verifier model"
+    )
+    install_step = next(step for step in steps if step.get("name") == "Install Codex CLI")
+
+    installed_cli = _extract_codex_cli_pin(install_step["run"])
+    candidates = _model_candidates(resolve_step)
+    unreviewed_models = [model for model in candidates if model not in MIN_CODEX_CLI_BY_MODEL]
+    assert not unreviewed_models, (
+        "Verifier Codex model candidates need an explicit reviewed minimum CLI mapping: "
+        + ", ".join(unreviewed_models)
+    )
+
+    for model in candidates:
+        minimum_cli = MIN_CODEX_CLI_BY_MODEL[model]
+        assert installed_cli >= minimum_cli, (
+            f"Verifier model {model} requires @openai/codex >= {minimum_cli}, "
+            f"but reusable-agents-verifier.yml installs {installed_cli}."
+        )