feat: implement issue #773 — pr-review: rubber-duck tier broken under default claude engine — COPILOT_API_MODEL: unbound variable

evals/validate-cases.py

@@ -147,6 +147,79 @@ def validate_directory(eval_root: Path) -> None:
           f"no cross-split id overlap.")
 
 
+# ── Directory mode: dev/holdout split hygiene ─────────────────────────────────
+
+def load_split_ids(cases_path: Path) -> list[str]:
+    try:
+        raw = cases_path.read_text(encoding="utf-8")
+    except OSError as exc:
+        fail(f"could not read {cases_path}: {exc}")
+
+    ids: list[str] = []
+    seen: set[str] = set()
+    for lineno, line in enumerate(raw.splitlines(), start=1):
+        if not line.strip():
+            continue
+        try:
+            case = json.loads(line)
+        except json.JSONDecodeError as exc:
+            fail(f"{cases_path}:{lineno}: invalid JSON: {exc}")
+        if not isinstance(case, dict):
+            fail(f"{cases_path}:{lineno}: each case line must be a JSON object")
+        if "id" not in case:
+            fail(f"{cases_path}:{lineno}: case is missing required field 'id'")
+        cid = case["id"]
+        if not isinstance(cid, str) or not cid.strip():
+            fail(f"{cases_path}:{lineno}: case 'id' must be a non-empty string")
+        if cid in seen:
+            fail(f"{cases_path}:{lineno}: duplicate id '{cid}' within this split")
+        seen.add(cid)
+        ids.append(cid)
+    return ids
+
+
+def validate_skill(skill_dir: Path) -> int:
+    split_ids: dict[str, list[str]] = {}
+    for split in SPLITS:
+        cases_path = skill_dir / split / CASES_FILENAME
+        if not cases_path.is_file():
+            fail(f"skill '{skill_dir.name}' is missing its {split} split "
+                 f"(expected {cases_path})")
+        split_ids[split] = load_split_ids(cases_path)
+
+    overlap = set(split_ids["dev"]) & set(split_ids["holdout"])
+    if overlap:
+        joined = ", ".join(sorted(overlap))
+        fail(f"skill '{skill_dir.name}': id(s) appear in both dev and holdout "
+             f"splits (a case must not be in both): {joined}")
+
+    return sum(len(ids) for ids in split_ids.values())
+
+
+def discover_skills(eval_root: Path) -> list[Path]:
+    skills = []
+    for child in sorted(eval_root.iterdir()):
+        if not child.is_dir():
+            continue
+        if any((child / split).is_dir() for split in SPLITS):
+            skills.append(child)
+    return skills
+
+
+def validate_directory(eval_root: Path) -> None:
+    skills = discover_skills(eval_root)
+    if not skills:
+        fail(f"no skills found under {eval_root} "
+             f"(expected <skill>/dev/ and <skill>/holdout/ subdirectories)")
+
+    total = 0
+    for skill_dir in skills:
+        total += validate_skill(skill_dir)
+
+    print(f"OK: {len(skills)} skill(s), {total} case(s) across dev+holdout, "
+          f"no cross-split id overlap.")
+
+
 # ── Entry point ───────────────────────────────────────────────────────────────
 
 def main() -> None:

prompts/cascade-action.md

@@ -50,6 +50,9 @@ cat > /tmp/cascade/review-body.txt << 'BODYEOF'
 ### Summary
 PLACEHOLDER_SUMMARY
 
+### Cross-engine agreement (if deep+duck)
+<If tier is deep+duck and agreement field exists, include this section>
+
 ### Findings
 PLACEHOLDER_FINDINGS_LIST
 

scripts/dev-lead-fix-ci.sh

@@ -187,6 +187,23 @@ ${details}"
   fi
 }
 
+post_exhaustion() {
+  local reason="$1"
+  local body="${EXHAUSTION_MARKER}
+## Dev-Lead Fix CI — exhausted
+
+This PR has had **${MAX_FAIL_ATTEMPTS}** consecutive engine failures (timeouts or errors). Automated CI fixing has been paused to avoid consuming further tokens.
+
+**Reason for last failure:** ${reason}
+
+To re-enable, delete this comment or push a new commit with a substantially different change."
+  if [ "${DEV_LEAD_DRY_RUN:-false}" = "true" ]; then
+    echo "[dry-run] would post exhaustion comment"
+  else
+    gh pr comment "$PR_NUMBER" --repo "$REPO" --body "$body"
+  fi
+}
+
 main() {
   if [ -z "$PR_NUMBER" ] || [ -z "$HEAD_SHA" ]; then
     echo "::error::PR_NUMBER and HEAD_SHA are required"