Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/agents-verify-to-issue-v2.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ jobs:
.github/scripts/github-api-with-retry.js
.github/scripts/terminal_disposition.js
.github/scripts/token_load_balancer.js
config
scripts/langchain
tools
sparse-checkout-cone-mode: false
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/reusable-agents-verifier.yml
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ jobs:
.github/agents/registry.yml
.github/scripts
.github/codex/prompts
config
scripts
tools
sparse-checkout-cone-mode: false
Expand Down
53 changes: 53 additions & 0 deletions tests/workflows/test_workflow_llm_installs.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
AUTO_PILOT = WORKFLOWS_DIR / "agents-auto-pilot.yml"
ISSUE_OPTIMIZER = WORKFLOWS_DIR / "agents-issue-optimizer.yml"
VERIFIER = WORKFLOWS_DIR / "reusable-agents-verifier.yml"
VERIFY_TO_ISSUE = WORKFLOWS_DIR / "agents-verify-to-issue-v2.yml"
REUSABLE_CODEX_RUN = WORKFLOWS_DIR / "reusable-codex-run.yml"
REUSABLE_CLAUDE_RUN = WORKFLOWS_DIR / "reusable-claude-run.yml"
NEEDS_HUMAN_COMMENT = Path("agents/codex-1447.md")
Expand All @@ -26,6 +27,8 @@
"gpt-5.4": (0, 125, 0),
}
ACTIONS_CACHE_V6_REF = "actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9"
LLM_REGISTRY_MODULE = "tools/llm_registry.py"
LLM_CONFIG_PATHS = (Path("config/llm_slots.json"), Path("config/model_registry.json"))


def _load_text(path: Path) -> str:
Expand Down Expand Up @@ -90,6 +93,20 @@ def _iter_steps(workflow: dict) -> list[dict]:
return steps


def _sparse_checkout_paths(step: dict) -> list[str]:
raw = str((step.get("with") or {}).get("sparse-checkout", ""))
return [line.strip() for line in raw.splitlines() if line.strip()]


def _workflows_library_checkout_steps(workflow: dict) -> list[dict]:
steps = []
for step in _iter_steps(workflow):
with_block = step.get("with") or {}
if with_block.get("repository") == "stranske/Workflows" and "sparse-checkout" in with_block:
steps.append(step)
return steps


def _find_step_by_name(workflow: dict, step_name: str) -> dict:
for step in _iter_steps(workflow):
if step.get("name") == step_name:
Expand Down Expand Up @@ -241,6 +258,42 @@ def test_reusable_agents_verifier_pip_cache_is_configured() -> None:
_assert_pip_cache(workflow, ".workflows-lib/tools/requirements-llm.txt", VERIFIER.name)


@pytest.mark.parametrize("workflow_path", [VERIFIER, VERIFY_TO_ISSUE])
Comment thread
stranske marked this conversation as resolved.
Resolved
def test_llm_workflows_vendor_the_model_registry_config(workflow_path: Path) -> None:
workflow = _load_workflow(workflow_path)
checkouts = _workflows_library_checkout_steps(workflow)
assert checkouts, f"{workflow_path.name} must sparse-checkout stranske/Workflows"

vendors_tools = False
for step in checkouts:
paths = _sparse_checkout_paths(step)
if "tools" not in paths:
continue
vendors_tools = True
missing = [entry for entry in ("config",) if entry not in paths]
assert not missing, (
f"{workflow_path.name} vendors `tools` but not {missing}; "
f"{LLM_REGISTRY_MODULE} resolves {', '.join(str(p) for p in LLM_CONFIG_PATHS)} "
"relative to the vendored tree, so every judge slot resolves to no model and "
'compare mode reports "available families: none".'
)

assert vendors_tools, f"{workflow_path.name} must vendor `tools` for the LLM client"


def test_bundled_llm_config_resolves_two_cross_family_judges() -> None:
for config_path in LLM_CONFIG_PATHS:
assert config_path.is_file(), f"Vendored LLM config {config_path} must exist"

from tools.llm_registry import load_slot_config

families = {slot.provider for slot in load_slot_config() if slot.model}
assert len(families) >= 2, (
"Compare mode needs two cross-family judges; the bundled config resolved "
f"models for {sorted(families) or 'no provider'}."
)


Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
def test_reusable_codex_run_persists_refreshed_auth_bundle_with_app_token() -> None:
workflow = _load_workflow(REUSABLE_CODEX_RUN)
step = _find_step_by_name(workflow, "Persist refreshed Codex auth secret")
Expand Down
Loading