From 19b0a978308d0836524e900b6fd506dbaa31add5 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:38:29 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#553=20?= =?UTF-8?q?=E2=80=94=20Enable=20new=20Claude=20models=20(Fable=205=20/=20O?= =?UTF-8?q?pus=204.8)=20in=20dev-lead=20+=20pr-review=20engines=20with=20c?= =?UTF-8?q?apability-aware=20selection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/engine.sh | 22 +- scripts/lib/model-pricing.tsv | 1 + tests/dev-lead/unit/test_engine_chain.bats | 73 +++--- .../dev-lead/unit/test_engine_new_models.bats | 233 ++++++++++++++++++ 4 files changed, 286 insertions(+), 43 deletions(-) create mode 100644 tests/dev-lead/unit/test_engine_new_models.bats diff --git a/scripts/engine.sh b/scripts/engine.sh index e9962682f..1ea2fe35f 100644 --- a/scripts/engine.sh +++ b/scripts/engine.sh @@ -50,12 +50,12 @@ set_engine_config() { case "$REVIEW_ENGINE" in claude) ENGINE_TRIAGE_MODEL="claude-haiku-4-5-20251001" - ENGINE_DEEP_MODEL="claude-sonnet-4-6" - ENGINE_AUDIT_MODEL="claude-opus-4-7" + ENGINE_DEEP_MODEL="claude-opus-4-8" + ENGINE_AUDIT_MODEL="claude-fable-5" ENGINE_ACTION_MODEL="claude-sonnet-4-6" - ENGINE_SINGLE_MODEL="claude-opus-4-7" - ENGINE_LABEL="triage: haiku 4.5 → deep: sonnet 4.6 + duck: o4-mini → audit: opus 4.7" - ENGINE_SINGLE_LABEL="single-reviewer mode: opus 4.7" + ENGINE_SINGLE_MODEL="claude-fable-5" + ENGINE_LABEL="triage: haiku 4.5 → deep: opus 4.8 + duck: o4-mini → audit: fable 5" + ENGINE_SINGLE_LABEL="single-reviewer mode: fable 5" # Cross-engine rubber duck: use Copilot when Claude is primary DUCK_ENGINE="copilot" DUCK_MODEL="o4-mini" @@ -65,11 +65,15 @@ set_engine_config() { # are independent, so swapping models within Claude often recovers without # leaving the provider. (Daily subscription cap is shared — see issue #206.) # Override per workflow via env to tune cost/capability trade-offs. + # Fable 5 notes (honored by the claude CLI automatically): + # - adaptive thinking only; budget_tokens/temperature/top_p/top_k removed + # - omit thinking param entirely (disabled returns 400 on fable-5) + # - min cacheable prefix: fable-5 = 2048 tok, opus-4-8 = 4096 tok CLAUDE_TRIAGE_MODEL_CHAIN="${CLAUDE_TRIAGE_MODEL_CHAIN:-claude-haiku-4-5-20251001,claude-sonnet-4-6}" - CLAUDE_DEEP_MODEL_CHAIN="${CLAUDE_DEEP_MODEL_CHAIN:-claude-sonnet-4-6,claude-opus-4-7}" - CLAUDE_AUDIT_MODEL_CHAIN="${CLAUDE_AUDIT_MODEL_CHAIN:-claude-opus-4-7,claude-sonnet-4-6}" - CLAUDE_ACTION_MODEL_CHAIN="${CLAUDE_ACTION_MODEL_CHAIN:-claude-sonnet-4-6,claude-opus-4-7}" - CLAUDE_SINGLE_MODEL_CHAIN="${CLAUDE_SINGLE_MODEL_CHAIN:-claude-opus-4-7,claude-sonnet-4-6}" + CLAUDE_DEEP_MODEL_CHAIN="${CLAUDE_DEEP_MODEL_CHAIN:-claude-opus-4-8,claude-sonnet-4-6}" + CLAUDE_AUDIT_MODEL_CHAIN="${CLAUDE_AUDIT_MODEL_CHAIN:-claude-fable-5,claude-opus-4-8,claude-opus-4-7}" + CLAUDE_ACTION_MODEL_CHAIN="${CLAUDE_ACTION_MODEL_CHAIN:-claude-sonnet-4-6,claude-opus-4-8}" + CLAUDE_SINGLE_MODEL_CHAIN="${CLAUDE_SINGLE_MODEL_CHAIN:-claude-fable-5,claude-opus-4-8,claude-opus-4-7}" ;; gemini) ENGINE_TRIAGE_MODEL="gemini-2.0-flash" diff --git a/scripts/lib/model-pricing.tsv b/scripts/lib/model-pricing.tsv index b7f76ec17..20c6961ac 100644 --- a/scripts/lib/model-pricing.tsv +++ b/scripts/lib/model-pricing.tsv @@ -30,6 +30,7 @@ # model_glob effective_from input cache_read cache_write output # ── Anthropic Claude (authoritative) ────────────────────────────────────────── +claude-fable-5* 2026-06-11 10.00 1.00 12.50 50.00 claude-opus-4-1* 2025-01-01 15.00 1.50 18.75 75.00 claude-opus-4-0* 2025-01-01 15.00 1.50 18.75 75.00 claude-opus-4-* 2025-11-01 5.00 0.50 6.25 25.00 diff --git a/tests/dev-lead/unit/test_engine_chain.bats b/tests/dev-lead/unit/test_engine_chain.bats index 578daa1bb..c1c066e09 100644 --- a/tests/dev-lead/unit/test_engine_chain.bats +++ b/tests/dev-lead/unit/test_engine_chain.bats @@ -13,6 +13,11 @@ setup() { export GITHUB_ENV="$(mktemp)" export GITHUB_OUTPUT="$(mktemp)" + # Clear any pre-set chain vars from the runner environment so engine.sh + # defaults are evaluated fresh for every test. + unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN + unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN + STUB_BIN_DIR="$(mktemp -d)" cp "$STUB_ENGINES_DIR/stub-claude" "$STUB_BIN_DIR/claude" cp "$STUB_ENGINES_DIR/stub-gemini" "$STUB_BIN_DIR/gemini" @@ -120,23 +125,23 @@ _source_engine() { # ── End-to-end: writer uses the chain via CLAUDE_ACTION_MODEL_CHAIN ──────────── -@test "writer: sonnet rate-limited → opus tried via CLAUDE_ACTION_MODEL_CHAIN" { +@test "writer: sonnet rate-limited → opus-4-8 tried via CLAUDE_ACTION_MODEL_CHAIN" { _source_engine "claude" - # Defaults from set_engine_config: CLAUDE_ACTION_MODEL_CHAIN=sonnet,opus - export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-7=0" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=too many requests (429)|claude-opus-4-7=opus did the work" + # Defaults from set_engine_config: CLAUDE_ACTION_MODEL_CHAIN=sonnet,opus-4-8 + export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-8=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=too many requests (429)|claude-opus-4-8=opus-4-8 did the work" run run_writer "$TEST_PROMPT" [ "$status" -eq 0 ] grep -q "claude-sonnet-4-6" "$MODEL_RECORD" - grep -q "claude-opus-4-7" "$MODEL_RECORD" + grep -q "claude-opus-4-8" "$MODEL_RECORD" } -@test "writer: sonnet rate-limited and opus rate-limited → exit 2 (cross-provider fallback signal)" { +@test "writer: sonnet rate-limited and opus-4-8 rate-limited → exit 2 (cross-provider fallback signal)" { _source_engine "claude" - export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-7=1" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=quota exceeded|claude-opus-4-7=quota exceeded" + export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-8=1" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=quota exceeded|claude-opus-4-8=quota exceeded" run run_writer "$TEST_PROMPT" @@ -157,27 +162,27 @@ _source_engine() { # ── End-to-end: agentic respects per-tier chain selection ──────────────────── -@test "agentic: deep tier sonnet rate-limited → opus tried (CLAUDE_DEEP_MODEL_CHAIN)" { +@test "agentic: deep tier opus-4-8 rate-limited → sonnet tried (CLAUDE_DEEP_MODEL_CHAIN)" { _source_engine "claude" - export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-7=0" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=service overload|claude-opus-4-7=deep result" + export STUB_ENGINE_EXIT_BY_MODEL="claude-opus-4-8=1|claude-sonnet-4-6=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-opus-4-8=service overload|claude-sonnet-4-6=deep result" - run run_agentic "$TEST_PROMPT" "claude-sonnet-4-6" "deep" + run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep" [ "$status" -eq 0 ] - grep -q "claude-opus-4-7" "$MODEL_RECORD" + grep -q "claude-sonnet-4-6" "$MODEL_RECORD" } -@test "agentic: audit tier opus rate-limited → sonnet tried (CLAUDE_AUDIT_MODEL_CHAIN)" { +@test "agentic: audit tier fable-5 rate-limited → opus-4-8 tried (CLAUDE_AUDIT_MODEL_CHAIN)" { _source_engine "claude" - # Default CLAUDE_AUDIT_MODEL_CHAIN = opus,sonnet — opus is first - export STUB_ENGINE_EXIT_BY_MODEL="claude-opus-4-7=1|claude-sonnet-4-6=0" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-opus-4-7=usage limit reached|claude-sonnet-4-6=audit ok" + # Default CLAUDE_AUDIT_MODEL_CHAIN = fable-5,opus-4-8,opus-4-7 — fable-5 is first + export STUB_ENGINE_EXIT_BY_MODEL="claude-fable-5=1|claude-opus-4-8=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-fable-5=usage limit reached|claude-opus-4-8=audit ok" - run run_agentic "$TEST_PROMPT" "claude-opus-4-7" "audit" + run run_agentic "$TEST_PROMPT" "claude-fable-5" "audit" [ "$status" -eq 0 ] - grep -q "claude-sonnet-4-6" "$MODEL_RECORD" + grep -q "claude-opus-4-8" "$MODEL_RECORD" } # ── Gemini/Copilot unchanged: no chain applied ──────────────────────────────── @@ -261,15 +266,15 @@ _source_engine() { @test "chain: writer non-RL failure after RL attempt propagates correctly (not remapped to 2)" { # Model A (sonnet) rate-limited → warning emitted → 2>&1 merges into _tmp. - # Model B (opus) fails with a non-rate-limit error. run_writer must return - # opus's exit code, not 2, even though _tmp contains the throttled warning. + # Model B (opus-4-8) fails with a non-rate-limit error. run_writer must return + # opus-4-8's exit code, not 2, even though _tmp contains the throttled warning. _source_engine "claude" - export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-7=1" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=rate limit exceeded|claude-opus-4-7=segfault in agent runtime" + export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-8=1" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=rate limit exceeded|claude-opus-4-8=segfault in agent runtime" run run_writer "$TEST_PROMPT" - # opus's non-RL failure → exit 1 (NOT 2). If the throttled-warning text + # opus-4-8's non-RL failure → exit 1 (NOT 2). If the throttled-warning text # matched is_rate_limited, this would incorrectly return 2. [ "$status" -eq 1 ] } @@ -283,18 +288,18 @@ _source_engine() { @test "agentic: explicit model arg differing from tier default is honored (no chain expansion)" { _source_engine "claude" - # Caller pins haiku for deep tier (overriding default sonnet → opus chain). - # Chain expansion would record sonnet+opus; pinning must record ONLY haiku. + # Caller pins haiku for deep tier (overriding default opus-4-8 → sonnet chain). + # Chain expansion would record opus-4-8+sonnet; pinning must record ONLY haiku. export STUB_ENGINE_EXIT=0 run run_agentic "$TEST_PROMPT" "claude-haiku-4-5-20251001" "deep" [ "$status" -eq 0 ] - # Only one invocation, and it's the pinned model — not sonnet or opus. + # Only one invocation, and it's the pinned model — not opus-4-8 or sonnet. [ "$(wc -l < "$MODEL_RECORD")" -eq 1 ] grep -q "claude-haiku-4-5-20251001" "$MODEL_RECORD" ! grep -q "claude-sonnet-4-6" "$MODEL_RECORD" - ! grep -q "claude-opus-4-7" "$MODEL_RECORD" + ! grep -q "claude-opus-4-8" "$MODEL_RECORD" } @test "writer: explicit model arg differing from action default is honored (no chain expansion)" { @@ -307,19 +312,19 @@ _source_engine() { [ "$(wc -l < "$MODEL_RECORD")" -eq 1 ] grep -q "claude-haiku-4-5-20251001" "$MODEL_RECORD" ! grep -q "claude-sonnet-4-6" "$MODEL_RECORD" - ! grep -q "claude-opus-4-7" "$MODEL_RECORD" + ! grep -q "claude-opus-4-8" "$MODEL_RECORD" } @test "agentic: passing tier default model still expands to full chain on rate-limit" { # Regression guard for the pin-check above: when caller passes the tier - # default (sonnet), chain expansion still works (sonnet → opus fallback). + # default (opus-4-8), chain expansion still works (opus-4-8 → sonnet fallback). _source_engine "claude" - export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-7=0" - export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=429 too many|claude-opus-4-7=ok" + export STUB_ENGINE_EXIT_BY_MODEL="claude-opus-4-8=1|claude-sonnet-4-6=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-opus-4-8=429 too many|claude-sonnet-4-6=ok" - run run_agentic "$TEST_PROMPT" "claude-sonnet-4-6" "deep" + run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep" [ "$status" -eq 0 ] + grep -q "claude-opus-4-8" "$MODEL_RECORD" grep -q "claude-sonnet-4-6" "$MODEL_RECORD" - grep -q "claude-opus-4-7" "$MODEL_RECORD" } diff --git a/tests/dev-lead/unit/test_engine_new_models.bats b/tests/dev-lead/unit/test_engine_new_models.bats new file mode 100644 index 000000000..c19f9a1c9 --- /dev/null +++ b/tests/dev-lead/unit/test_engine_new_models.bats @@ -0,0 +1,233 @@ +#!/usr/bin/env bats +# Unit tests for engine.sh — new Claude model tier assignments and chains. +# +# Verifies that Fable 5 (claude-fable-5) and Opus 4.8 (claude-opus-4-8) are +# wired into the correct tiers and fallback chains for the claude engine: +# triage → haiku-4-5 (unchanged) +# action → sonnet-4-6 (unchanged) +# deep → opus-4-8 (was sonnet-4-6) +# audit → fable-5 (was opus-4-7) +# single → fable-5 (was opus-4-7) + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" +ENGINE_SCRIPT="$SCRIPT_DIR/scripts/engine.sh" +STUB_ENGINES_DIR="$SCRIPT_DIR/tests/dev-lead/fixtures/engines" + +setup() { + export GITHUB_ENV="$(mktemp)" + export GITHUB_OUTPUT="$(mktemp)" + + # Clear any pre-set chain vars from the runner environment so engine.sh + # defaults are evaluated fresh for every test. + unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN + unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN + + STUB_BIN_DIR="$(mktemp -d)" + cp "$STUB_ENGINES_DIR/stub-claude" "$STUB_BIN_DIR/claude" + cp "$STUB_ENGINES_DIR/stub-gemini" "$STUB_BIN_DIR/gemini" + chmod +x "$STUB_BIN_DIR/claude" "$STUB_BIN_DIR/gemini" + export PATH="$STUB_BIN_DIR:$PATH" + export STUB_BIN_DIR + + TEST_PROMPT="$(mktemp)" + echo "test prompt content" > "$TEST_PROMPT" + export TEST_PROMPT + + MODEL_RECORD="$(mktemp)" + export STUB_ENGINE_RECORD_MODELS="$MODEL_RECORD" + + export DEV_LEAD_DRY_RUN="false" +} + +teardown() { + rm -f "$GITHUB_ENV" "$GITHUB_OUTPUT" "$TEST_PROMPT" "$MODEL_RECORD" + rm -rf "$STUB_BIN_DIR" + unset STUB_ENGINE_EXIT_BY_MODEL STUB_ENGINE_RESPONSE_BY_MODEL + unset STUB_ENGINE_RECORD_MODELS STUB_ENGINE_EXIT STUB_ENGINE_RESPONSE + unset CLAUDE_ACTION_MODEL_CHAIN CLAUDE_DEEP_MODEL_CHAIN + unset CLAUDE_TRIAGE_MODEL_CHAIN CLAUDE_AUDIT_MODEL_CHAIN CLAUDE_SINGLE_MODEL_CHAIN +} + +_source_engine() { + local engine="${1:-claude}" + export REVIEW_ENGINE="$engine" + source "$ENGINE_SCRIPT" 2>/dev/null || true +} + +# ── Tier default model assignments ─────────────────────────────────────────── + +@test "new-models: ENGINE_AUDIT_MODEL is claude-fable-5 for claude engine" { + _source_engine "claude" + [ "$ENGINE_AUDIT_MODEL" = "claude-fable-5" ] +} + +@test "new-models: ENGINE_SINGLE_MODEL is claude-fable-5 for claude engine" { + _source_engine "claude" + [ "$ENGINE_SINGLE_MODEL" = "claude-fable-5" ] +} + +@test "new-models: ENGINE_DEEP_MODEL is claude-opus-4-8 for claude engine" { + _source_engine "claude" + [ "$ENGINE_DEEP_MODEL" = "claude-opus-4-8" ] +} + +@test "new-models: ENGINE_ACTION_MODEL still claude-sonnet-4-6 (unchanged)" { + _source_engine "claude" + [ "$ENGINE_ACTION_MODEL" = "claude-sonnet-4-6" ] +} + +@test "new-models: ENGINE_TRIAGE_MODEL still claude-haiku-4-5-20251001 (unchanged)" { + _source_engine "claude" + [ "$ENGINE_TRIAGE_MODEL" = "claude-haiku-4-5-20251001" ] +} + +# ── Fallback chain configuration ────────────────────────────────────────────── + +@test "new-models: CLAUDE_AUDIT_MODEL_CHAIN starts with claude-fable-5" { + _source_engine "claude" + local first + first="$(printf '%s' "$CLAUDE_AUDIT_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + [ "$first" = "claude-fable-5" ] +} + +@test "new-models: CLAUDE_AUDIT_MODEL_CHAIN includes claude-opus-4-8 as second element" { + _source_engine "claude" + local second + second="$(printf '%s' "$CLAUDE_AUDIT_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + [ "$second" = "claude-opus-4-8" ] +} + +@test "new-models: CLAUDE_AUDIT_MODEL_CHAIN includes claude-opus-4-7 as final fallback" { + _source_engine "claude" + [[ "$CLAUDE_AUDIT_MODEL_CHAIN" == *"claude-opus-4-7"* ]] +} + +@test "new-models: CLAUDE_SINGLE_MODEL_CHAIN starts with claude-fable-5" { + _source_engine "claude" + local first + first="$(printf '%s' "$CLAUDE_SINGLE_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + [ "$first" = "claude-fable-5" ] +} + +@test "new-models: CLAUDE_SINGLE_MODEL_CHAIN includes claude-opus-4-8 as second element" { + _source_engine "claude" + local second + second="$(printf '%s' "$CLAUDE_SINGLE_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + [ "$second" = "claude-opus-4-8" ] +} + +@test "new-models: CLAUDE_DEEP_MODEL_CHAIN starts with claude-opus-4-8" { + _source_engine "claude" + local first + first="$(printf '%s' "$CLAUDE_DEEP_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + [ "$first" = "claude-opus-4-8" ] +} + +@test "new-models: CLAUDE_ACTION_MODEL_CHAIN second element is claude-opus-4-8 (not opus-4-7)" { + _source_engine "claude" + local second + second="$(printf '%s' "$CLAUDE_ACTION_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + [ "$second" = "claude-opus-4-8" ] + [[ "$second" != *"opus-4-7"* ]] +} + +# ── Intent dispatch returns new tier defaults ───────────────────────────────── + +@test "new-models: model_for_intent(fix-issue) returns claude-opus-4-8" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-issue")" + [ "$result" = "claude-opus-4-8" ] +} + +@test "new-models: model_for_intent(human) returns claude-opus-4-8" { + _source_engine "claude" + local result + result="$(model_for_intent "human")" + [ "$result" = "claude-opus-4-8" ] +} + +@test "new-models: model_for_intent(fix-reviews) still returns claude-sonnet-4-6" { + _source_engine "claude" + local result + result="$(model_for_intent "fix-reviews")" + [ "$result" = "claude-sonnet-4-6" ] +} + +# ── Chain fallback behaviour with new models ────────────────────────────────── + +@test "new-models: audit fable-5 rate-limited → opus-4-8 tried" { + _source_engine "claude" + export STUB_ENGINE_EXIT_BY_MODEL="claude-fable-5=1|claude-opus-4-8=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-fable-5=hit your limit|claude-opus-4-8=audit ok" + + run run_agentic "$TEST_PROMPT" "claude-fable-5" "audit" + + [ "$status" -eq 0 ] + grep -q "claude-fable-5" "$MODEL_RECORD" + grep -q "claude-opus-4-8" "$MODEL_RECORD" + [[ "$output" == *"audit ok"* ]] + [[ "$output" != *"hit your limit"* ]] +} + +@test "new-models: audit chain fully exhausted → exit 2" { + _source_engine "claude" + export STUB_ENGINE_EXIT_BY_MODEL="claude-fable-5=1|claude-opus-4-8=1|claude-opus-4-7=1" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-fable-5=429 too many|claude-opus-4-8=429 too many|claude-opus-4-7=429 too many" + + run run_agentic "$TEST_PROMPT" "claude-fable-5" "audit" + + [ "$status" -eq 2 ] +} + +@test "new-models: deep opus-4-8 rate-limited → sonnet-4-6 tried" { + _source_engine "claude" + export STUB_ENGINE_EXIT_BY_MODEL="claude-opus-4-8=1|claude-sonnet-4-6=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-opus-4-8=quota exceeded|claude-sonnet-4-6=deep result ok" + + run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep" + + [ "$status" -eq 0 ] + grep -q "claude-opus-4-8" "$MODEL_RECORD" + grep -q "claude-sonnet-4-6" "$MODEL_RECORD" +} + +@test "new-models: action chain sonnet rate-limited → opus-4-8 tried (not opus-4-7)" { + _source_engine "claude" + export STUB_ENGINE_EXIT_BY_MODEL="claude-sonnet-4-6=1|claude-opus-4-8=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-sonnet-4-6=too many requests|claude-opus-4-8=action done" + + run run_writer "$TEST_PROMPT" + + [ "$status" -eq 0 ] + grep -q "claude-opus-4-8" "$MODEL_RECORD" + ! grep -q "claude-opus-4-7" "$MODEL_RECORD" +} + +@test "new-models: passing tier default opus-4-8 expands to full deep chain on rate-limit" { + _source_engine "claude" + export STUB_ENGINE_EXIT_BY_MODEL="claude-opus-4-8=1|claude-sonnet-4-6=0" + export STUB_ENGINE_RESPONSE_BY_MODEL="claude-opus-4-8=429 too many|claude-sonnet-4-6=ok" + + run run_agentic "$TEST_PROMPT" "claude-opus-4-8" "deep" + + [ "$status" -eq 0 ] + grep -q "claude-opus-4-8" "$MODEL_RECORD" + grep -q "claude-sonnet-4-6" "$MODEL_RECORD" +} + +# ── Non-claude engines unaffected ───────────────────────────────────────────── + +@test "new-models: gemini engine unchanged — no claude model in any tier" { + _source_engine "gemini" + [[ "$ENGINE_AUDIT_MODEL" != *"claude"* ]] + [[ "$ENGINE_DEEP_MODEL" != *"claude"* ]] + [[ "$ENGINE_SINGLE_MODEL" != *"claude"* ]] +} + +@test "new-models: copilot engine unchanged — all tiers still o4-mini" { + _source_engine "copilot" + [ "$ENGINE_AUDIT_MODEL" = "o4-mini" ] + [ "$ENGINE_SINGLE_MODEL" = "o4-mini" ] + [ "$ENGINE_DEEP_MODEL" = "o4-mini" ] +} From 688bb7f36b5718675c2bed989ab5bef31bb802b7 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Thu, 11 Jun 2026 12:44:17 +0000 Subject: [PATCH 2/3] chore: apply manual instructions [skip ci-relay] --- .../dev-lead/unit/test_engine_new_models.bats | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/dev-lead/unit/test_engine_new_models.bats b/tests/dev-lead/unit/test_engine_new_models.bats index c19f9a1c9..7a88fdf96 100644 --- a/tests/dev-lead/unit/test_engine_new_models.bats +++ b/tests/dev-lead/unit/test_engine_new_models.bats @@ -86,14 +86,17 @@ _source_engine() { @test "new-models: CLAUDE_AUDIT_MODEL_CHAIN starts with claude-fable-5" { _source_engine "claude" local first - first="$(printf '%s' "$CLAUDE_AUDIT_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + first="${CLAUDE_AUDIT_MODEL_CHAIN%%,*}" + first="${first// /}" [ "$first" = "claude-fable-5" ] } @test "new-models: CLAUDE_AUDIT_MODEL_CHAIN includes claude-opus-4-8 as second element" { _source_engine "claude" local second - second="$(printf '%s' "$CLAUDE_AUDIT_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + second="${CLAUDE_AUDIT_MODEL_CHAIN#*,}" + second="${second%%,*}" + second="${second// /}" [ "$second" = "claude-opus-4-8" ] } @@ -105,28 +108,34 @@ _source_engine() { @test "new-models: CLAUDE_SINGLE_MODEL_CHAIN starts with claude-fable-5" { _source_engine "claude" local first - first="$(printf '%s' "$CLAUDE_SINGLE_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + first="${CLAUDE_SINGLE_MODEL_CHAIN%%,*}" + first="${first// /}" [ "$first" = "claude-fable-5" ] } @test "new-models: CLAUDE_SINGLE_MODEL_CHAIN includes claude-opus-4-8 as second element" { _source_engine "claude" local second - second="$(printf '%s' "$CLAUDE_SINGLE_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + second="${CLAUDE_SINGLE_MODEL_CHAIN#*,}" + second="${second%%,*}" + second="${second// /}" [ "$second" = "claude-opus-4-8" ] } @test "new-models: CLAUDE_DEEP_MODEL_CHAIN starts with claude-opus-4-8" { _source_engine "claude" local first - first="$(printf '%s' "$CLAUDE_DEEP_MODEL_CHAIN" | cut -d',' -f1 | tr -d ' ')" + first="${CLAUDE_DEEP_MODEL_CHAIN%%,*}" + first="${first// /}" [ "$first" = "claude-opus-4-8" ] } @test "new-models: CLAUDE_ACTION_MODEL_CHAIN second element is claude-opus-4-8 (not opus-4-7)" { _source_engine "claude" local second - second="$(printf '%s' "$CLAUDE_ACTION_MODEL_CHAIN" | cut -d',' -f2 | tr -d ' ')" + second="${CLAUDE_ACTION_MODEL_CHAIN#*,}" + second="${second%%,*}" + second="${second// /}" [ "$second" = "claude-opus-4-8" ] [[ "$second" != *"opus-4-7"* ]] } From 7d608fee1f21dd721f0175b84ecccb05107fba8e Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:30:12 +0000 Subject: [PATCH 3/3] chore: apply manual instructions [skip ci-relay] --- .gitleaks.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitleaks.toml b/.gitleaks.toml index 7862d375f..412a441e6 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -51,3 +51,10 @@ regexes = [ '''test-key''', '''03-VERIFICATION\.md''', ] + +[[allowlists]] +# commit bda766be added bmad skill knowledge files and config manifests containing +# example API key patterns used in tutorial/documentation content — not real credentials. +# All generic-api-key findings in this commit are false positives. +description = "Suppress generic-api-key false positives in historical bmad skill/config additions" +commits = ["bda766be1f1156327d54bc7497da74225ee853bf"]