From 08d9b459a8876040eb37580595de2a9482a98f12 Mon Sep 17 00:00:00 2001 From: James Wiesebron Date: Thu, 7 May 2026 10:50:07 -0700 Subject: [PATCH 1/4] =?UTF-8?q?Fix=20#2527:=20validate=20task=20role?= =?UTF-8?q?=E2=86=94file=20alignment=20at=20plan=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `validate_task_role_alignment` in `shared/egg_contracts/plan_parser.py` that mirrors the gateway's push-time blocked-pattern check for each task's `role` against its `files_affected`. The plan reviewer's prompt now runs the validator on the parsed plan draft and injects a "Structural Role-Alignment Check" section listing every offending task with the eligible-role hint (or the `.github-staging/` remediation when no producer role can push the file set). The plan-review criteria gain a deterministic blocking item that points at this section, so a mis-assignment surfaces as a NACK before any producer cycle is wasted. Per-task logic lives in `_check_role_files` so the #2530 follow-up (`includes_tests: true` opt-in for coders coupling tests with their own production code) has a clear hook point. Section is omitted when the validator reports no violations; the prompt is unchanged for clean plans. Push-time enforcement remains in place as defense in depth. --- orchestrator/routes/pipelines.py | 95 ++++++- orchestrator/tests/test_pipeline_prompts.py | 150 +++++++++++ shared/egg_contracts/__init__.py | 2 + shared/egg_contracts/plan_parser.py | 136 ++++++++++ .../test_validate_task_role_alignment.py | 239 ++++++++++++++++++ 5 files changed, 621 insertions(+), 1 deletion(-) create mode 100644 shared/egg_contracts/tests/test_validate_task_role_alignment.py diff --git a/orchestrator/routes/pipelines.py b/orchestrator/routes/pipelines.py index d5be4613a4..1ef6bdd450 100644 --- a/orchestrator/routes/pipelines.py +++ b/orchestrator/routes/pipelines.py @@ -4577,7 +4577,19 @@ def _get_plan_review_criteria() -> str: "### 7. Completeness\n" "- Does the plan cover all aspects of the original request?\n" "- Are documentation updates included where needed?\n" - "- Are there any obvious gaps or missing tasks?\n" + "- Are there any obvious gaps or missing tasks?\n\n" + "### 8. Task Role ↔ Files Alignment (deterministic, see #2527)\n" + "- If a **Structural Role-Alignment Check** section appears below, " + "every entry there is a deterministic finding from the same " + "blocked-pattern logic the gateway enforces at push time.\n" + "- Each listed task would fail at push with " + "`403 restricted_path_modified` if implemented as assigned.\n" + "- **Treat any entry as blocking**: flag the plan as " + "`needs_revision` (or NACK in concurrent mode) and quote the errors " + "verbatim in your feedback so the planner re-emits the plan with " + "corrected `role:` fields.\n" + "- The absence of that section means the automated check found no " + "violations — task-role assignments are not a blocking issue.\n" ) @@ -5751,6 +5763,78 @@ def _render_contract_tasks( return "\n".join(lines) if len(lines) > 1 else None +def _build_role_alignment_check_section( + repo_path: str | None, + draft_rel: str | None, +) -> str: + """Build the plan reviewer's "Structural Role-Alignment Check" section. + + Runs ``validate_task_role_alignment`` against the parsed plan draft and + formats any violations for inclusion in the reviewer prompt. Returns + an empty string when the draft is unavailable, the plan parses to no + slices, or the validator reports no violations — in which case the + section is omitted from the prompt entirely. + + The check mirrors the gateway's push-time blocked-pattern enforcement + (see ``shared/egg_contracts/plan_parser.py::validate_task_role_alignment`` + and ``gateway/phase_filter.py::FileRestriction.is_file_blocked``), so + a violation surfaced here predicts a push-time + ``403 restricted_path_modified`` before any producer cycle is wasted + (issue #2527). + """ + if not repo_path or not draft_rel: + return "" + try: + from egg_contracts.plan_parser import ( + parse_plan, + validate_task_role_alignment, + ) + except ImportError: + return "" + plan_file = Path(repo_path) / draft_rel + try: + if not plan_file.exists(): + return "" + result = parse_plan(plan_file.read_text()) + except Exception as parse_err: # noqa: BLE001 + logger.debug( + "plan reviewer role-alignment check: parse failed", + draft=str(plan_file), + error=str(parse_err), + ) + return "" + if not result.success: + return "" + try: + slices = result.to_contract_slices() + errors = validate_task_role_alignment(slices) + except Exception as validator_err: # noqa: BLE001 + logger.debug( + "plan reviewer role-alignment check: validator failed", + draft=str(plan_file), + error=str(validator_err), + ) + return "" + if not errors: + return "" + + lines = [ + "## Structural Role-Alignment Check", + "", + "An automated check ran against the plan's task→role assignments using " + "the same blocked-pattern logic the gateway enforces at push time " + "(`shared/egg_restrictions/patterns.py`). The following tasks would " + "fail at push with `403 restricted_path_modified` if implemented as " + "assigned. **NACK the plan and quote these errors verbatim in your " + "feedback so the planner re-emits the plan with corrected roles.**", + "", + ] + for err in errors: + lines.append(f"- {err}") + lines.append("") + return "\n".join(lines) + + def _build_review_prompt( phase: str, pipeline_id: str, @@ -5916,6 +6000,15 @@ def _build_review_prompt( lines.append(_get_review_criteria_for_type(reviewer_type, phase, repo_path=repo_path)) lines.append("") + # Structural role-alignment check for plan reviewers (issue #2527). + # Surfaces deterministic role-vs-files violations the LLM reviewer + # otherwise would not catch; section is emitted only when violations + # exist, so a clean plan keeps the prompt unchanged. + if reviewer_type == "plan": + role_alignment_section = _build_role_alignment_check_section(repo_path, draft_path) + if role_alignment_section: + lines.append(role_alignment_section) + # Review conventions — quality standards aligned with PR reviewer thoroughness lines.append("## Review Conventions\n") if reviewer_type in ("code", "code-holistic"): diff --git a/orchestrator/tests/test_pipeline_prompts.py b/orchestrator/tests/test_pipeline_prompts.py index 53e1f9056a..5bc2e3ca10 100644 --- a/orchestrator/tests/test_pipeline_prompts.py +++ b/orchestrator/tests/test_pipeline_prompts.py @@ -25,12 +25,14 @@ _build_producer_orientation, _build_review_prompt, _build_reviewer_preparation, + _build_role_alignment_check_section, _build_role_context, _build_role_restrictions_section, _extract_plan_overview, _get_agent_design_criteria, _get_code_review_criteria, _get_contract_review_criteria, + _get_plan_review_criteria, _get_reviewer_scope_preamble, _read_shared_criteria, _read_tester_gaps, @@ -4300,3 +4302,151 @@ def test_revert_of_coordination_detected(self): assert "HANDOFF" in preamble # Must clarify relationship to consensus assert "supplementary" in preamble or "consensus" in preamble.lower() + + +# --------------------------------------------------------------------------- +# #2527 — plan reviewer's structural role-alignment check +# --------------------------------------------------------------------------- + + +_PLAN_WITH_MISASSIGNED_TASK = """\ +# Plan + +```yaml +# yaml-tasks +slices: + - id: 1 + name: Setup + goal: scaffolding + tasks: + - id: TASK-1-1 + description: Add pytest fixtures + acceptance: fixtures load + role: coder + files: + - integration_tests/conftest.py +``` +""" + + +_PLAN_WITH_CLEAN_ASSIGNMENTS = """\ +# Plan + +```yaml +# yaml-tasks +slices: + - id: 1 + name: Setup + goal: scaffolding + tasks: + - id: TASK-1-1 + description: Add pytest fixtures + acceptance: fixtures load + role: tester + files: + - integration_tests/conftest.py +``` +""" + + +def _write_plan_draft(repo_path: Path, issue_number: int, body: str) -> None: + drafts_dir = repo_path / ".egg-state" / "drafts" + drafts_dir.mkdir(parents=True, exist_ok=True) + (drafts_dir / f"{issue_number}-plan.md").write_text(body) + + +class TestRoleAlignmentCheckSection: + """The pure helper that runs the validator and formats the section.""" + + def test_returns_empty_when_repo_path_missing(self): + assert _build_role_alignment_check_section(None, ".egg-state/drafts/1-plan.md") == "" + + def test_returns_empty_when_draft_path_missing(self): + assert _build_role_alignment_check_section("/tmp", None) == "" + + def test_returns_empty_when_draft_file_does_not_exist(self): + with tempfile.TemporaryDirectory() as tmpdir: + section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/1-plan.md") + assert section == "" + + def test_returns_empty_when_plan_is_clean(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_CLEAN_ASSIGNMENTS) + section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/2527-plan.md") + assert section == "" + + def test_emits_section_when_violations_exist(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) + section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/2527-plan.md") + assert section != "" + assert "Structural Role-Alignment Check" in section + assert "task-1-1" in section + assert "'coder'" in section + assert "integration_tests/conftest.py" in section + # Hint must surface the eligible role. + assert "tester" in section + # Reviewer must be told to NACK and quote verbatim. + assert "NACK" in section + + +class TestPlanReviewPromptIncludesAlignmentCheck: + """End-to-end check on _build_review_prompt: when reviewer_type=='plan' + the section is appended whenever the validator reports violations, + and is omitted otherwise.""" + + def test_plan_reviewer_prompt_includes_section_on_violations(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) + prompt = _build_review_prompt( + phase="plan", + pipeline_id="issue-2527", + pipeline_mode="issue", + reviewer_type="plan", + issue_number=2527, + repo_path=tmpdir, + ) + assert "## Structural Role-Alignment Check" in prompt + assert "task-1-1" in prompt + + def test_plan_reviewer_prompt_omits_section_when_clean(self): + with tempfile.TemporaryDirectory() as tmpdir: + _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_CLEAN_ASSIGNMENTS) + prompt = _build_review_prompt( + phase="plan", + pipeline_id="issue-2527", + pipeline_mode="issue", + reviewer_type="plan", + issue_number=2527, + repo_path=tmpdir, + ) + assert "## Structural Role-Alignment Check" not in prompt + + def test_non_plan_reviewer_does_not_run_check(self): + # Even if a draft exists with violations, only the plan reviewer + # should surface this section. A code reviewer prompt must not + # include it (different phase, different review surface). + with tempfile.TemporaryDirectory() as tmpdir: + _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) + prompt = _build_review_prompt( + phase="implement", + pipeline_id="issue-2527", + pipeline_mode="issue", + reviewer_type="code", + issue_number=2527, + repo_path=tmpdir, + ) + assert "## Structural Role-Alignment Check" not in prompt + + +class TestPlanReviewCriteriaReferencesCheck: + """The plan-review criteria string must point the reviewer at the + structural check section so an LLM following the criteria knows to + treat any listed entry as blocking.""" + + def test_criteria_references_role_alignment_section(self): + criteria = _get_plan_review_criteria() + assert "Role" in criteria and "Alignment" in criteria + # Must reference the issue and instruct treating entries as blocking. + assert "#2527" in criteria + assert "needs_revision" in criteria or "NACK" in criteria diff --git a/shared/egg_contracts/__init__.py b/shared/egg_contracts/__init__.py index 40fbb8f787..82341d739a 100644 --- a/shared/egg_contracts/__init__.py +++ b/shared/egg_contracts/__init__.py @@ -198,6 +198,7 @@ parse_plan, parse_plan_file, validate_forest, + validate_task_role_alignment, ) from .resilience import ( CheckpointState, @@ -299,6 +300,7 @@ "parse_plan", "parse_plan_file", "validate_forest", + "validate_task_role_alignment", # Phase Defaults "get_default_phase_config", "get_effective_phase_config", diff --git a/shared/egg_contracts/plan_parser.py b/shared/egg_contracts/plan_parser.py index 8d8eefd653..c52861731f 100644 --- a/shared/egg_contracts/plan_parser.py +++ b/shared/egg_contracts/plan_parser.py @@ -1272,6 +1272,141 @@ def dfs(node: str, path: list[str]) -> None: return cycles +# --------------------------------------------------------------------------- +# #2527 — task role ↔ files_affected alignment +# --------------------------------------------------------------------------- + + +def _is_file_blocked_for_role(role: str, file_path: str) -> bool: + """Return True if ``file_path`` is blocked for ``role`` per the role's + ``AGENT_PATTERNS`` blocklist (with block-exempt carve-outs). + + Mirrors ``gateway/phase_filter.py::FileRestriction.is_file_blocked`` + so plan-time validation matches push-time enforcement 1:1. The + gateway's check intentionally consults only blocked + block-exempt + patterns (not allowed_patterns), and so does this function. + """ + import posixpath + + from egg_restrictions.matchers import match_pattern + from egg_restrictions.patterns import AGENT_PATTERNS + + pattern = AGENT_PATTERNS.get(role) + if pattern is None: + return False + + normalized = posixpath.normpath(file_path) + if normalized.startswith("./"): + normalized = normalized[2:] + if normalized.startswith("../") or normalized.startswith("/"): + return True + + if not any(match_pattern(normalized, p) for p in pattern.blocked_patterns): + return False + if any(match_pattern(normalized, p) for p in pattern.block_exempt_patterns): + return False + return True + + +def _eligible_producer_roles(files: list[str]) -> list[str]: + """Return the producer roles (coder/tester/documenter) for which + every file in ``files`` passes the gateway's blocked-pattern check. + + The result preserves the canonical coder→tester→documenter ordering + so suggestions are deterministic across runs. + """ + from .agent_roles import AgentRole + + ordered_roles = (AgentRole.CODER, AgentRole.TESTER, AgentRole.DOCUMENTER) + eligible: list[str] = [] + for role in ordered_roles: + if all(not _is_file_blocked_for_role(role, f) for f in files): + eligible.append(role.value) + return eligible + + +def _check_role_files(task: Task, slice_id: str) -> str | None: + """Return a structured error string for a misaligned task, or + ``None`` if the task's ``role`` can push every file in + ``files_affected``. + + Per-task hook so the #2530 follow-up can thread a future + ``includes_tests: true`` opt-in through here without restructuring + the outer walk: a coder task that legitimately couples tests to + its own production code is the most common false-positive case + (24 of 25 misassignments in the #2530 audit), and that flag is the + proposed exception. Until the flag exists this function reports + every coder-with-test-files mismatch. + + Tasks without a ``role`` or with empty ``files_affected`` return + ``None`` — the parser already treats ``role`` as optional, and an + empty file list leaves nothing to check (prose/research tasks). + """ + role = task.role + files = list(task.files_affected or []) + if not role or not files: + return None + blocked = [f for f in files if _is_file_blocked_for_role(role, f)] + if not blocked: + return None + eligible = _eligible_producer_roles(files) + if len(eligible) == 1: + hint = f"Reassign to role '{eligible[0]}' — it can push every file in this task." + elif len(eligible) > 1: + hint = ( + f"Eligible roles for this file set: {eligible}. " + "Pick one and update the task's 'role' field." + ) + else: + hint = ( + "No producer role can push every file in this task. Either " + "split the task so each subtask falls within a single " + "role's scope, or — for `.github/` files — stage them " + "under top-level `.github-staging/` and let the PR " + "builder emit a manual reviewer step (issue #2508)." + ) + return ( + f"Task '{task.id}' (slice '{slice_id}') is assigned role " + f"'{role}' but files {blocked} are blocked for that role per " + f"shared/egg_restrictions/patterns.py. {hint}" + ) + + +def validate_task_role_alignment(slices: list[Slice]) -> list[str]: + """Walk the slice/task tree and reject tasks whose ``role`` cannot + push their ``files_affected``. + + Added in #2527. The plan-phase ``task_planner`` can assign tasks to + producer roles whose ``shared/egg_restrictions/patterns.py`` + blocklist forbids the listed files; the mismatch is otherwise only + caught at push time by the gateway's + ``check_file_restrictions``, which means the producer agent gets + spawned, explores, sometimes builds workarounds, and only then + hits ``403 restricted_path_modified``. Running the same check + at plan time lets the plan reviewer NACK the planner before any + producer cycle is wasted. + + Per-task logic lives in ``_check_role_files`` so the #2530 + ``includes_tests`` follow-up has a clear hook point. + + Args: + slices: The slice list extracted from the contract / plan. + + Returns: + A list of structured-error strings — one entry per offending + task. Each entry names the task ID, the assigned role, the + blocked files, and the eligible-role hint so the plan reviewer + can surface an actionable NACK reason. + """ + errors: list[str] = [] + for slice_ in slices: + for task in slice_.tasks: + err = _check_role_files(task, slice_.id) + if err is not None: + errors.append(err) + return errors + + __all__ = ( "ParsedPhase", "ParsedTask", @@ -1281,4 +1416,5 @@ def dfs(node: str, path: list[str]) -> None: "parse_plan", "parse_plan_file", "validate_forest", + "validate_task_role_alignment", ) diff --git a/shared/egg_contracts/tests/test_validate_task_role_alignment.py b/shared/egg_contracts/tests/test_validate_task_role_alignment.py new file mode 100644 index 0000000000..4e404ce049 --- /dev/null +++ b/shared/egg_contracts/tests/test_validate_task_role_alignment.py @@ -0,0 +1,239 @@ +"""Tests for ``plan_parser.validate_task_role_alignment`` (#2527). + +The plan-phase ``task_planner`` can assign tasks to producer roles +(``coder`` / ``tester`` / ``documenter``) whose +``shared/egg_restrictions/patterns.py`` blocklist forbids the listed +files. The mismatch is otherwise only caught at push time by the +gateway, after a producer has been spawned and burned tokens. The +validator runs the same blocked-pattern check at plan time so the plan +reviewer can NACK before any producer cycle starts. + +These tests cover: + +* Clean assignments (each producer role pushing files within its scope) + return no errors. +* Coder assigned to test files — the dominant misassignment in the + #2530 audit (24 of 25 cases) — is flagged with ``tester`` as the + single eligible role. +* Coder assigned to ``**/conftest.py`` is flagged (separate fixture + pattern from ``test_*.py``). +* Coder assigned to ``.github/`` files — no producer role can push + these — surfaces the ``.github-staging/`` (#2508) remediation hint. +* Coder assigned to a markdown file is flagged with ``documenter`` as + the single eligible role. +* Tasks without an explicit ``role`` or without + ``files_affected`` are skipped (no error, no false positive). +* The actual ``files`` from issue #2527's evidence table parse to the + expected structured errors. +""" + +from __future__ import annotations + +from egg_contracts.models import Slice, Task +from egg_contracts.plan_parser import validate_task_role_alignment + + +def _slice(slice_id: str, tasks: list[Task]) -> Slice: + return Slice(id=slice_id, name=f"slice {slice_id}", tasks=tasks) + + +def _task( + task_id: str, + files: list[str], + role: str | None, + description: str = "task", +) -> Task: + return Task( + id=task_id, + description=description, + acceptance_criteria="acc", + files_affected=files, + role=role, + ) + + +class TestCleanAssignments: + """Properly assigned tasks should produce zero errors.""" + + def test_empty_input(self) -> None: + assert validate_task_role_alignment([]) == [] + + def test_coder_with_source_file(self) -> None: + slices = [_slice("slice-1", [_task("task-1-1", ["src/foo.py"], "coder")])] + assert validate_task_role_alignment(slices) == [] + + def test_tester_with_test_file(self) -> None: + slices = [_slice("slice-1", [_task("task-1-1", ["tests/test_foo.py"], "tester")])] + assert validate_task_role_alignment(slices) == [] + + def test_tester_with_conftest(self) -> None: + slices = [ + _slice( + "slice-1", + [_task("task-1-1", ["integration_tests/conftest.py"], "tester")], + ) + ] + assert validate_task_role_alignment(slices) == [] + + def test_documenter_with_markdown(self) -> None: + slices = [_slice("slice-1", [_task("task-1-1", ["docs/guide.md"], "documenter")])] + assert validate_task_role_alignment(slices) == [] + + +class TestSkippedTasks: + """Tasks that the validator must intentionally pass over.""" + + def test_no_role_is_skipped(self) -> None: + # Tasks without an explicit role default downstream; the + # validator's job is catching mis-assignments, not enforcing + # role declaration (#2527 scope). + slices = [ + _slice( + "slice-1", + [_task("task-1-1", ["integration_tests/conftest.py"], None)], + ) + ] + assert validate_task_role_alignment(slices) == [] + + def test_empty_files_is_skipped(self) -> None: + # Prose/research tasks legitimately omit files_affected — + # nothing to check. + slices = [_slice("slice-1", [_task("task-1-1", [], "coder")])] + assert validate_task_role_alignment(slices) == [] + + def test_role_without_files_is_skipped(self) -> None: + slices = [_slice("slice-1", [_task("task-1-1", [], "documenter")])] + assert validate_task_role_alignment(slices) == [] + + +class TestSingleEligibleRoleHint: + """When exactly one producer role can push every file in the task, + the validator must name that role in its hint.""" + + def test_coder_with_conftest_suggests_tester(self) -> None: + slices = [ + _slice( + "slice-1", + [_task("task-1-1", ["integration_tests/conftest.py"], "coder")], + ) + ] + errors = validate_task_role_alignment(slices) + assert len(errors) == 1 + msg = errors[0] + assert "task-1-1" in msg + assert "slice-1" in msg + assert "'coder'" in msg + assert "integration_tests/conftest.py" in msg + assert "Reassign to role 'tester'" in msg + + def test_coder_with_test_py_suggests_tester(self) -> None: + # 24 of 25 misassignments in the #2530 audit were coder + # tasks containing test_*.py files. + slices = [_slice("slice-1", [_task("task-1-1", ["tests/test_foo.py"], "coder")])] + errors = validate_task_role_alignment(slices) + assert len(errors) == 1 + assert "Reassign to role 'tester'" in errors[0] + + def test_coder_with_markdown_suggests_documenter(self) -> None: + slices = [_slice("slice-1", [_task("task-1-1", ["docs/guide.md"], "coder")])] + errors = validate_task_role_alignment(slices) + assert len(errors) == 1 + assert "Reassign to role 'documenter'" in errors[0] + + +class TestNoEligibleRoleHint: + """Files no producer role can push — `.github/` is the canonical case + per #2508 — must surface the `.github-staging/` remediation.""" + + def test_coder_with_github_workflow_no_eligible_role(self) -> None: + slices = [ + _slice( + "slice-1", + [_task("task-1-1", [".github/workflows/ci.yml"], "coder")], + ) + ] + errors = validate_task_role_alignment(slices) + assert len(errors) == 1 + msg = errors[0] + assert ".github/workflows/ci.yml" in msg + assert "No producer role can push" in msg + assert ".github-staging/" in msg + + +class TestMixedRoleFiles: + """When files cross role boundaries no single role is eligible — the + validator must say so without listing a specific role to switch to.""" + + def test_task_mixing_test_and_doc_files_has_no_eligible_role(self) -> None: + # tests/test_foo.py is blocked for coder + documenter (the + # latter via the ``tests/`` directory rule). + # docs/guide.md is blocked for coder + tester (both via + # ``**/*.md``). No producer role can push both. + slices = [ + _slice( + "slice-1", + [ + _task( + "task-1-1", + ["tests/test_foo.py", "docs/guide.md"], + "coder", + ) + ], + ) + ] + errors = validate_task_role_alignment(slices) + assert len(errors) == 1 + assert "No producer role can push" in errors[0] + + +class TestMultipleSlicesAndTasks: + """The walk must cover every (slice, task) pair and emit one error + per offender.""" + + def test_evidence_from_issue_2527(self) -> None: + # Reproduces the misassignments listed in the issue's evidence + # table for pipeline issue-2474-v2 slice-1: TASK-1-1 (conftest), + # TASK-1-3 (test_*.py + .github/), TASK-1-4 (conftest) all + # assigned to coder. + slices = [ + _slice( + "slice-1", + [ + _task( + "task-1-1", + [ + "integration_tests/conftest.py", + "integration_tests/local_pipeline/conftest.py", + ], + "coder", + ), + # task-1-2 is correctly assigned -> no error + _task("task-1-2", ["src/foo.py"], "coder"), + _task( + "task-1-3", + [ + ".github/workflows/test-e2e.yml", + "integration_tests/test_e2e.py", + ], + "coder", + ), + _task("task-1-4", ["integration_tests/conftest.py"], "coder"), + ], + ) + ] + errors = validate_task_role_alignment(slices) + assert len(errors) == 3 + offending_ids = { + tid for err in errors for tid in ("task-1-1", "task-1-3", "task-1-4") if tid in err + } + assert offending_ids == {"task-1-1", "task-1-3", "task-1-4"} + + def test_errors_walk_every_slice(self) -> None: + slices = [ + _slice("slice-1", [_task("task-1-1", ["tests/test_a.py"], "coder")]), + _slice("slice-2", [_task("task-2-1", ["docs/x.md"], "coder")]), + ] + errors = validate_task_role_alignment(slices) + assert len(errors) == 2 + assert any("slice-1" in e and "task-1-1" in e for e in errors) + assert any("slice-2" in e and "task-2-1" in e for e in errors) From bacac5ecbf393ed0115bedfc59a3530f81c0e53c Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 18:28:37 +0000 Subject: [PATCH 2/4] Move #2527 validator to orchestrator-side propose-time enforcement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR-1 review flagged a cross-module silent no-op: in concurrent BRC mode (the default for plan phase) the original prompt-time helper ``_build_role_alignment_check_section`` always returned ``""`` because ``_run_concurrent_phase`` builds every reviewer prompt up-front before the planner has produced the plan. The criteria text then told the reviewer that "absence of that section means the automated check found no violations" — the opposite of the truth. Replace it with deterministic enforcement at the right seam: ``_validate_planner_role_alignment`` runs in ``handle_consensus_propose_signal`` for ``agent_role=="task_planner"``, mirroring the existing ``_validate_tester_check_coverage`` pattern. It reads the plan content as committed at the proposed SHA via ``git show :`` (so a stale local checkout can't mask a real misassignment) and raises ``ValueError`` on violations, which the caller turns into HTTP 400 — the proposal is rejected BEFORE the tracker is mutated and BEFORE any reviewer sees it. Also addresses the non-blocking comments: * Lazy ``posixpath`` / ``match_pattern`` / ``AGENT_PATTERNS`` imports in ``_is_file_blocked_for_role`` are moved to module scope (no circular-import risk; per-call overhead removed). * Tests now exercise the production sequence end-to-end: ``test_rejected_proposal_does_not_mutate_tracker`` builds the exact propose signal a planner emits in concurrent BRC mode, mocks ``git show`` to return a misassigned plan, and asserts the tracker is left untouched. The PR-1 prompt-emission tests are removed (the helper they pinned is gone) and replaced with criteria-text regression guards that lock out the "absence = no violations" wording. --- orchestrator/routes/pipelines.py | 108 +---- orchestrator/routes/signals.py | 122 ++++++ orchestrator/tests/test_pipeline_prompts.py | 413 +++++++++++++------- shared/egg_contracts/plan_parser.py | 8 +- 4 files changed, 412 insertions(+), 239 deletions(-) diff --git a/orchestrator/routes/pipelines.py b/orchestrator/routes/pipelines.py index 1ef6bdd450..05913796d2 100644 --- a/orchestrator/routes/pipelines.py +++ b/orchestrator/routes/pipelines.py @@ -4579,17 +4579,22 @@ def _get_plan_review_criteria() -> str: "- Are documentation updates included where needed?\n" "- Are there any obvious gaps or missing tasks?\n\n" "### 8. Task Role ↔ Files Alignment (deterministic, see #2527)\n" - "- If a **Structural Role-Alignment Check** section appears below, " - "every entry there is a deterministic finding from the same " - "blocked-pattern logic the gateway enforces at push time.\n" - "- Each listed task would fail at push with " - "`403 restricted_path_modified` if implemented as assigned.\n" - "- **Treat any entry as blocking**: flag the plan as " - "`needs_revision` (or NACK in concurrent mode) and quote the errors " - "verbatim in your feedback so the planner re-emits the plan with " - "corrected `role:` fields.\n" - "- The absence of that section means the automated check found no " - "violations — task-role assignments are not a blocking issue.\n" + "- Task role↔files alignment is enforced **orchestrator-side** at " + "`CONSENSUS_PROPOSE`: a planner proposal whose task `role:` " + "assignments cannot push their `files:` (per " + "`shared/egg_restrictions/patterns.py`, the same blocklist the " + "gateway uses) is rejected with HTTP 400 before this prompt is " + "ever rendered. By the time you see a proposal, structural " + "role↔files alignment is therefore already validated — no manual " + "check is required for this dimension.\n" + "- If you want belt-and-suspenders verification, you can run the " + "validator yourself against the proposed plan: " + '`python3 -c "from egg_contracts.plan_parser import parse_plan_file, ' + "validate_task_role_alignment as v; r = parse_plan_file(''); " + "print('\\n'.join(v(r.to_contract_slices())))\"`. " + "Errors here would predict a push-time `403 " + "restricted_path_modified` — NACK the planner and quote the " + "structured errors verbatim if any surface.\n" ) @@ -5763,78 +5768,6 @@ def _render_contract_tasks( return "\n".join(lines) if len(lines) > 1 else None -def _build_role_alignment_check_section( - repo_path: str | None, - draft_rel: str | None, -) -> str: - """Build the plan reviewer's "Structural Role-Alignment Check" section. - - Runs ``validate_task_role_alignment`` against the parsed plan draft and - formats any violations for inclusion in the reviewer prompt. Returns - an empty string when the draft is unavailable, the plan parses to no - slices, or the validator reports no violations — in which case the - section is omitted from the prompt entirely. - - The check mirrors the gateway's push-time blocked-pattern enforcement - (see ``shared/egg_contracts/plan_parser.py::validate_task_role_alignment`` - and ``gateway/phase_filter.py::FileRestriction.is_file_blocked``), so - a violation surfaced here predicts a push-time - ``403 restricted_path_modified`` before any producer cycle is wasted - (issue #2527). - """ - if not repo_path or not draft_rel: - return "" - try: - from egg_contracts.plan_parser import ( - parse_plan, - validate_task_role_alignment, - ) - except ImportError: - return "" - plan_file = Path(repo_path) / draft_rel - try: - if not plan_file.exists(): - return "" - result = parse_plan(plan_file.read_text()) - except Exception as parse_err: # noqa: BLE001 - logger.debug( - "plan reviewer role-alignment check: parse failed", - draft=str(plan_file), - error=str(parse_err), - ) - return "" - if not result.success: - return "" - try: - slices = result.to_contract_slices() - errors = validate_task_role_alignment(slices) - except Exception as validator_err: # noqa: BLE001 - logger.debug( - "plan reviewer role-alignment check: validator failed", - draft=str(plan_file), - error=str(validator_err), - ) - return "" - if not errors: - return "" - - lines = [ - "## Structural Role-Alignment Check", - "", - "An automated check ran against the plan's task→role assignments using " - "the same blocked-pattern logic the gateway enforces at push time " - "(`shared/egg_restrictions/patterns.py`). The following tasks would " - "fail at push with `403 restricted_path_modified` if implemented as " - "assigned. **NACK the plan and quote these errors verbatim in your " - "feedback so the planner re-emits the plan with corrected roles.**", - "", - ] - for err in errors: - lines.append(f"- {err}") - lines.append("") - return "\n".join(lines) - - def _build_review_prompt( phase: str, pipeline_id: str, @@ -6000,15 +5933,6 @@ def _build_review_prompt( lines.append(_get_review_criteria_for_type(reviewer_type, phase, repo_path=repo_path)) lines.append("") - # Structural role-alignment check for plan reviewers (issue #2527). - # Surfaces deterministic role-vs-files violations the LLM reviewer - # otherwise would not catch; section is emitted only when violations - # exist, so a clean plan keeps the prompt unchanged. - if reviewer_type == "plan": - role_alignment_section = _build_role_alignment_check_section(repo_path, draft_path) - if role_alignment_section: - lines.append(role_alignment_section) - # Review conventions — quality standards aligned with PR reviewer thoroughness lines.append("## Review Conventions\n") if reviewer_type in ("code", "code-holistic"): diff --git a/orchestrator/routes/signals.py b/orchestrator/routes/signals.py index 13ef92213f..9a9d693c32 100644 --- a/orchestrator/routes/signals.py +++ b/orchestrator/routes/signals.py @@ -917,6 +917,123 @@ def _validate_tester_check_coverage( ) +def _validate_planner_role_alignment( + pipeline_id: str, payload: dict[str, Any], repo_path: Path +) -> None: + """Validate task role↔files alignment for a planner proposal (#2527). + + Reads the plan draft at the proposed commit via ``git show`` against + the orchestrator's pipeline worktree and runs + ``validate_task_role_alignment``. Raises ``ValueError`` if any task + is assigned to a role that cannot push its files — the caller's + ``handle_consensus_propose_signal`` ``except`` block then returns 400 + to the planner before the proposal is recorded on the tracker, so no + reviewer cycle is wasted on a structurally-broken plan. + + The check mirrors the gateway's push-time blocked-pattern logic + (``gateway/phase_filter.py::FileRestriction.is_file_blocked``), so a + rejection here predicts a push-time ``403 restricted_path_modified`` + in the implement phase. Caught here it costs the planner one + re-propose; caught at push time it costs a full producer cycle. + + Graceful degradation: when the plan file doesn't exist at the + proposed commit, ``git show`` fails, parsing fails, or the validator + raises, this function returns silently. The push-time gateway check + remains the backstop, and the existing forest-validation pass at + plan-ingestion time still runs. + """ + commit_sha = (payload.get("commit_sha") or "").strip() + if not commit_sha: + return + + try: + pipeline = get_state_store(repo_path).load_pipeline(pipeline_id) + except StateStoreError: + return + + if not pipeline.branch: + return + + # Build the plan draft path. Imported lazily to avoid pulling the + # 16k-line ``routes.pipelines`` module into ``signals`` import time. + try: + from routes.pipelines import _get_draft_path + except ImportError: + try: + from .pipelines import _get_draft_path # type: ignore[no-redef] + except ImportError: + return + plan_rel = _get_draft_path( + "plan", + issue_number=pipeline.issue_number, + pipeline_id=pipeline_id, + mode=getattr(pipeline, "mode", None), + ) + if not plan_rel: + return + + worktree_path = resolve_worktree_path(pipeline_id, repo_path) + + # Read plan content as committed at the proposed SHA so a stale + # local checkout cannot mask a real misassignment. The preceding + # ``_verify_commit_on_branch`` call has already done a ``git fetch``, + # making the commit reachable in the worktree. + try: + result = subprocess.run( + ["git", "-C", str(worktree_path), "show", f"{commit_sha}:{plan_rel}"], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + if result.returncode != 0: + return # Plan not present at this commit — nothing to validate. + plan_text = result.stdout + except Exception as exc: + logger.warning( + "plan role-alignment validation: git show failed (non-blocking)", + pipeline_id=pipeline_id, + commit_sha=commit_sha, + error=str(exc), + ) + return + + try: + from egg_contracts.plan_parser import ( + parse_plan, + validate_task_role_alignment, + ) + except ImportError: + return + + try: + parsed = parse_plan(plan_text) + if not parsed.success: + return + slices = parsed.to_contract_slices() + errors = validate_task_role_alignment(slices) + except Exception as exc: + logger.warning( + "plan role-alignment validation: validator raised (non-blocking)", + pipeline_id=pipeline_id, + commit_sha=commit_sha, + error=str(exc), + ) + return + + if not errors: + return + + bullet_list = "\n".join(f" - {e}" for e in errors) + raise ValueError( + "Plan proposal rejected: task role↔files alignment violations.\n" + "The following tasks are assigned to roles whose blocklist " + "forbids their files (would 403 at push time per " + "shared/egg_restrictions/patterns.py). Update the affected " + "tasks' 'role' field and re-propose:\n" + bullet_list + ) + + def _resolve_pipeline_phase(pipeline_id: str, repo_path: Path) -> str: """Resolve the current phase for a pipeline, with graceful fallback. @@ -1101,6 +1218,11 @@ def handle_consensus_propose_signal( # rejected proposals. if agent_role == "tester": _validate_tester_check_coverage(pipeline_id, payload, repo_path) + # Validate task_planner proposals don't misassign tasks to roles + # whose blocklist forbids their files (#2527). Same placement + # rule: BEFORE handle_propose so the tracker isn't mutated. + elif agent_role == "task_planner": + _validate_planner_role_alignment(pipeline_id, payload, repo_path) # Check if this is a re-proposal changed_artifacts = data.get("changed_artifacts") diff --git a/orchestrator/tests/test_pipeline_prompts.py b/orchestrator/tests/test_pipeline_prompts.py index 5bc2e3ca10..c110ad75d2 100644 --- a/orchestrator/tests/test_pipeline_prompts.py +++ b/orchestrator/tests/test_pipeline_prompts.py @@ -3,6 +3,7 @@ """ import json +import subprocess import sys import tempfile from pathlib import Path @@ -25,7 +26,6 @@ _build_producer_orientation, _build_review_prompt, _build_reviewer_preparation, - _build_role_alignment_check_section, _build_role_context, _build_role_restrictions_section, _extract_plan_overview, @@ -2134,6 +2134,228 @@ def test_rejected_proposal_does_not_mutate_tracker(self): mock_tracker.handle_propose.assert_not_called() +class TestPlannerRoleAlignmentValidation: + """Tests for ``_validate_planner_role_alignment`` in ``signals.py`` (#2527). + + Exercises the production code path the original PR-1 implementation + couldn't reach: in concurrent BRC mode, ``_run_concurrent_phase`` + builds every reviewer prompt up-front before the planner has produced + the plan, so a prompt-time validator can never fire on the first + cycle. This validator runs at ``CONSENSUS_PROPOSE`` instead — by + that point the planner has pushed the plan to origin, and the + orchestrator reads it via ``git show :``. + """ + + _PLAN_WITH_MISASSIGNED_TASK = ( + "# Plan\n" + "\n" + "```yaml\n" + "# yaml-tasks\n" + "slices:\n" + " - id: 1\n" + " name: Setup\n" + " goal: scaffolding\n" + " tasks:\n" + " - id: TASK-1-1\n" + " description: Add pytest fixtures\n" + " acceptance: fixtures load\n" + " role: coder\n" + " files:\n" + " - integration_tests/conftest.py\n" + "```\n" + ) + + _PLAN_WITH_CLEAN_ASSIGNMENTS = ( + "# Plan\n" + "\n" + "```yaml\n" + "# yaml-tasks\n" + "slices:\n" + " - id: 1\n" + " name: Setup\n" + " goal: scaffolding\n" + " tasks:\n" + " - id: TASK-1-1\n" + " description: Add pytest fixtures\n" + " acceptance: fixtures load\n" + " role: tester\n" + " files:\n" + " - integration_tests/conftest.py\n" + "```\n" + ) + + @staticmethod + def _patched_store(issue_number: int | None = 2527, branch: str = "egg/issue-2527"): + mock_pipeline = MagicMock() + mock_pipeline.issue_number = issue_number + mock_pipeline.branch = branch + mock_pipeline.mode = None + mock_store = MagicMock() + mock_store.load_pipeline.return_value = mock_pipeline + return patch("routes.signals.get_state_store", return_value=mock_store) + + @staticmethod + def _patched_subprocess(plan_text: str, returncode: int = 0): + result = subprocess.CompletedProcess( + args=[], returncode=returncode, stdout=plan_text, stderr="" + ) + return patch("routes.signals.subprocess.run", return_value=result) + + @staticmethod + def _patched_worktree(): + return patch("routes.signals.resolve_worktree_path", return_value=Path("/tmp/wt")) + + def test_skips_when_commit_sha_missing(self): + """No commit SHA on payload → nothing to validate against.""" + from routes.signals import _validate_planner_role_alignment + + # Should not raise even with no other patches in place — the + # bail-out happens before any state-store / git access. + _validate_planner_role_alignment("issue-2527", {"payload": {}}, Path("/tmp")) + _validate_planner_role_alignment("issue-2527", {"commit_sha": ""}, Path("/tmp")) + + def test_rejects_misassigned_plan_at_propose_time(self): + """Planner pushed a plan with coder→test-files: validator raises.""" + from routes.signals import _validate_planner_role_alignment + + with ( + self._patched_store(), + self._patched_worktree(), + self._patched_subprocess(self._PLAN_WITH_MISASSIGNED_TASK), + ): + payload = {"commit_sha": "abc1234"} + with pytest.raises(ValueError, match="role↔files alignment violations"): + _validate_planner_role_alignment("issue-2527", payload, Path("/tmp/repo")) + + def test_accepts_clean_plan(self): + """Planner pushed a plan with correctly-assigned roles: no raise.""" + from routes.signals import _validate_planner_role_alignment + + with ( + self._patched_store(), + self._patched_worktree(), + self._patched_subprocess(self._PLAN_WITH_CLEAN_ASSIGNMENTS), + ): + payload = {"commit_sha": "abc1234"} + # Should not raise. + _validate_planner_role_alignment("issue-2527", payload, Path("/tmp/repo")) + + def test_skips_when_git_show_fails(self): + """``git show`` non-zero exit (plan absent at commit) → graceful skip.""" + from routes.signals import _validate_planner_role_alignment + + with ( + self._patched_store(), + self._patched_worktree(), + self._patched_subprocess("", returncode=128), + ): + payload = {"commit_sha": "abc1234"} + # Should not raise. + _validate_planner_role_alignment("issue-2527", payload, Path("/tmp/repo")) + + def test_skips_when_pipeline_lookup_fails(self): + """State store load failure → graceful skip.""" + from routes.signals import _validate_planner_role_alignment + from state_store import StateValidationError + + mock_store = MagicMock() + mock_store.load_pipeline.side_effect = StateValidationError("corrupt state") + + with ( + patch("routes.signals.get_state_store", return_value=mock_store), + self._patched_worktree(), + ): + payload = {"commit_sha": "abc1234"} + # Should not raise — graceful degradation + _validate_planner_role_alignment("issue-2527", payload, Path("/tmp/repo")) + + def test_skips_when_pipeline_has_no_branch(self): + """A pipeline with ``branch=None`` → graceful skip (no git context).""" + from routes.signals import _validate_planner_role_alignment + + with ( + self._patched_store(branch=None), + self._patched_worktree(), + ): + payload = {"commit_sha": "abc1234"} + # Should not raise — branch is required to resolve the worktree commit. + _validate_planner_role_alignment("issue-2527", payload, Path("/tmp/repo")) + + def test_rejected_proposal_does_not_mutate_tracker(self): + """Integration: a planner proposal carrying a misassigned plan + is rejected at ``handle_consensus_propose_signal`` BEFORE the + tracker is mutated — same guarantee as + ``test_rejected_proposal_does_not_mutate_tracker`` for + testers (#1459). + + This is the production-sequence end-to-end test the PR-1 review + flagged as missing: it builds the propose signal exactly the + way the planner agent does in concurrent BRC mode, mocks + ``git show`` to return the misassigned plan (the file the + orchestrator's worktree would read at the proposed commit), + and asserts the tracker is left untouched. + """ + from flask import Flask + from routes.signals import handle_consensus_propose_signal + + mock_tracker = MagicMock() + mock_tracker.handle_propose = MagicMock(return_value={"version": 1}) + + # Two subprocess calls happen in this path: + # 1. _verify_commit_on_branch's git fetch + # 2. _verify_commit_on_branch's git branch --contains + # 3. _validate_planner_role_alignment's git show + # The first two return success; the third returns the misassigned plan. + side_effect = [ + subprocess.CompletedProcess(args=[], returncode=0, stdout="", stderr=""), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=" origin/egg/issue-2527\n", + stderr="", + ), + subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=self._PLAN_WITH_MISASSIGNED_TASK, + stderr="", + ), + ] + + app = Flask(__name__) + with ( + app.app_context(), + self._patched_store(), + self._patched_worktree(), + patch("routes.signals.subprocess.run", side_effect=side_effect), + patch("peer_consensus.get_peer_consensus_tracker", return_value=mock_tracker), + ): + data = { + "agent_role": "task_planner", + "payload": { + "summary": ( + "Plan v1: 1 slice / 1 task with task-1-1 assigned " + "(coder) for the integration_tests fixture work" + ), + "artifacts": [".egg-state/drafts/2527-plan.md"], + "commit_sha": "abc1234", + }, + } + response, status_code = handle_consensus_propose_signal( + "issue-2527", data, Path("/tmp/repo") + ) + # Rejected with 400 (ValueError → make_error_response 400). + assert status_code == 400 + data_out = response.get_json() + assert "role↔files alignment violations" in data_out.get("message", "") + # Tracker.handle_propose must NOT have been called — the + # validator runs BEFORE the tracker, so a rejected proposal + # never mutates tracker state. This is the regression + # guarantee the PR-1 review flagged as the missing + # production-sequence test. + mock_tracker.handle_propose.assert_not_called() + + class TestReadTesterGapsNamespacedEdgeCases: """Additional edge-case tests for _read_tester_gaps with identifier.""" @@ -4305,148 +4527,55 @@ def test_revert_of_coordination_detected(self): # --------------------------------------------------------------------------- -# #2527 — plan reviewer's structural role-alignment check +# #2527 — plan reviewer's task role↔files alignment check # --------------------------------------------------------------------------- - - -_PLAN_WITH_MISASSIGNED_TASK = """\ -# Plan - -```yaml -# yaml-tasks -slices: - - id: 1 - name: Setup - goal: scaffolding - tasks: - - id: TASK-1-1 - description: Add pytest fixtures - acceptance: fixtures load - role: coder - files: - - integration_tests/conftest.py -``` -""" - - -_PLAN_WITH_CLEAN_ASSIGNMENTS = """\ -# Plan - -```yaml -# yaml-tasks -slices: - - id: 1 - name: Setup - goal: scaffolding - tasks: - - id: TASK-1-1 - description: Add pytest fixtures - acceptance: fixtures load - role: tester - files: - - integration_tests/conftest.py -``` -""" - - -def _write_plan_draft(repo_path: Path, issue_number: int, body: str) -> None: - drafts_dir = repo_path / ".egg-state" / "drafts" - drafts_dir.mkdir(parents=True, exist_ok=True) - (drafts_dir / f"{issue_number}-plan.md").write_text(body) - - -class TestRoleAlignmentCheckSection: - """The pure helper that runs the validator and formats the section.""" - - def test_returns_empty_when_repo_path_missing(self): - assert _build_role_alignment_check_section(None, ".egg-state/drafts/1-plan.md") == "" - - def test_returns_empty_when_draft_path_missing(self): - assert _build_role_alignment_check_section("/tmp", None) == "" - - def test_returns_empty_when_draft_file_does_not_exist(self): - with tempfile.TemporaryDirectory() as tmpdir: - section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/1-plan.md") - assert section == "" - - def test_returns_empty_when_plan_is_clean(self): - with tempfile.TemporaryDirectory() as tmpdir: - _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_CLEAN_ASSIGNMENTS) - section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/2527-plan.md") - assert section == "" - - def test_emits_section_when_violations_exist(self): - with tempfile.TemporaryDirectory() as tmpdir: - _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) - section = _build_role_alignment_check_section(tmpdir, ".egg-state/drafts/2527-plan.md") - assert section != "" - assert "Structural Role-Alignment Check" in section - assert "task-1-1" in section - assert "'coder'" in section - assert "integration_tests/conftest.py" in section - # Hint must surface the eligible role. - assert "tester" in section - # Reviewer must be told to NACK and quote verbatim. - assert "NACK" in section - - -class TestPlanReviewPromptIncludesAlignmentCheck: - """End-to-end check on _build_review_prompt: when reviewer_type=='plan' - the section is appended whenever the validator reports violations, - and is omitted otherwise.""" - - def test_plan_reviewer_prompt_includes_section_on_violations(self): - with tempfile.TemporaryDirectory() as tmpdir: - _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) - prompt = _build_review_prompt( - phase="plan", - pipeline_id="issue-2527", - pipeline_mode="issue", - reviewer_type="plan", - issue_number=2527, - repo_path=tmpdir, - ) - assert "## Structural Role-Alignment Check" in prompt - assert "task-1-1" in prompt - - def test_plan_reviewer_prompt_omits_section_when_clean(self): - with tempfile.TemporaryDirectory() as tmpdir: - _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_CLEAN_ASSIGNMENTS) - prompt = _build_review_prompt( - phase="plan", - pipeline_id="issue-2527", - pipeline_mode="issue", - reviewer_type="plan", - issue_number=2527, - repo_path=tmpdir, - ) - assert "## Structural Role-Alignment Check" not in prompt - - def test_non_plan_reviewer_does_not_run_check(self): - # Even if a draft exists with violations, only the plan reviewer - # should surface this section. A code reviewer prompt must not - # include it (different phase, different review surface). - with tempfile.TemporaryDirectory() as tmpdir: - _write_plan_draft(Path(tmpdir), 2527, _PLAN_WITH_MISASSIGNED_TASK) - prompt = _build_review_prompt( - phase="implement", - pipeline_id="issue-2527", - pipeline_mode="issue", - reviewer_type="code", - issue_number=2527, - repo_path=tmpdir, - ) - assert "## Structural Role-Alignment Check" not in prompt - - -class TestPlanReviewCriteriaReferencesCheck: - """The plan-review criteria string must point the reviewer at the - structural check section so an LLM following the criteria knows to - treat any listed entry as blocking.""" - - def test_criteria_references_role_alignment_section(self): +# +# Original PR-1 design built a "Structural Role-Alignment Check" section +# into the reviewer prompt at _build_review_prompt time. PR-1 review +# flagged that as a cross-module silent no-op in concurrent BRC mode: +# in production, all agent prompts are built up-front by +# _run_concurrent_phase BEFORE the planner has produced the plan +# (concurrent_executor.spawn_all). The plan draft does not exist on +# the orchestrator's worktree at that moment, so the section was always +# empty and the reviewer was told "absence = no violations" — the +# opposite of the truth. +# +# Resolution (this PR-2): orchestrator-side validation runs at +# CONSENSUS_PROPOSE in routes/signals.py:_validate_planner_role_alignment, +# rejecting the planner's proposal with HTTP 400 before the tracker +# state is mutated. The reviewer prompt no longer carries a per-prompt +# section; the validator-runs-here tests live in test_signals.py +# (test_propose_validates_planner_role_alignment). + + +class TestPlanReviewCriteriaReflectsOrchestratorSideValidation: + """The plan-review criteria string documents that role↔files + alignment is enforced orchestrator-side at CONSENSUS_PROPOSE so a + reviewer reading the criteria does not expect a per-prompt section + (which the broken PR-1 wiring promised but never delivered in + concurrent mode).""" + + def test_criteria_references_orchestrator_side_enforcement(self): criteria = _get_plan_review_criteria() + # Must still call out the dimension by name. assert "Role" in criteria and "Alignment" in criteria - # Must reference the issue and instruct treating entries as blocking. assert "#2527" in criteria - assert "needs_revision" in criteria or "NACK" in criteria + # Must explicitly tell the reviewer the check runs at + # CONSENSUS_PROPOSE rather than at prompt-build time. + assert "CONSENSUS_PROPOSE" in criteria + assert "orchestrator-side" in criteria + # Must describe rejection of the proposal (not "absence = no + # violations" — that was the PR-1 false-clean wording). + assert "rejected" in criteria + # The push-time backstop (`403 restricted_path_modified`) is + # the link to the gateway's existing enforcement. + assert "403" in criteria + + def test_criteria_does_not_reuse_pr1_false_clean_wording(self): + # Specific regression guard: PR-1 included the line + # "The absence of that section means the automated check found + # no violations". That sentence was structurally a false-clean + # in concurrent BRC mode (the section was always absent because + # the plan didn't exist yet). Make sure it doesn't reappear. + criteria = _get_plan_review_criteria() + assert "absence of that section" not in criteria.lower() diff --git a/shared/egg_contracts/plan_parser.py b/shared/egg_contracts/plan_parser.py index c52861731f..57d65a17fc 100644 --- a/shared/egg_contracts/plan_parser.py +++ b/shared/egg_contracts/plan_parser.py @@ -56,12 +56,15 @@ from __future__ import annotations +import posixpath import re from dataclasses import dataclass, field from pathlib import Path from typing import Any import yaml +from egg_restrictions.matchers import match_pattern +from egg_restrictions.patterns import AGENT_PATTERNS from .agent_roles import EXECUTION_ROLE_VALUES from .models import Slice, SliceStatus, Task, TaskStatus @@ -1286,11 +1289,6 @@ def _is_file_blocked_for_role(role: str, file_path: str) -> bool: gateway's check intentionally consults only blocked + block-exempt patterns (not allowed_patterns), and so does this function. """ - import posixpath - - from egg_restrictions.matchers import match_pattern - from egg_restrictions.patterns import AGENT_PATTERNS - pattern = AGENT_PATTERNS.get(role) if pattern is None: return False From 51030fad838c09b39c157cf864f748391d54a18c Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 18:57:18 +0000 Subject: [PATCH 3/4] Address PR review feedback (round 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blocking: - Revert egg_restrictions.patterns import in plan_parser.py to lazy function-local. The module-scope hoist in PR-1 round 1 re-introduced the egg_restrictions ↔ egg_contracts import cycle that shared/egg_restrictions/matchers.py was deliberately split out to avoid (see its docstring), breaking the gateway production boot path (python3 gateway/gateway.py). Conftest pre-load order hid the cycle in pytest. egg_restrictions.matchers.match_pattern stays at module scope — only AGENT_PATTERNS needs to be lazy. - Add TestImportOrderingRegression that subprocess-runs 'import egg_restrictions.patterns' under PYTHONPATH=shared so the cycle surfaces in a clean interpreter (mirrors gateway boot). Non-blocking: - Reword the role-alignment criterion in _get_plan_review_criteria to say 'before the proposal reaches you' instead of 'before this prompt is ever rendered' — concurrent BRC mode builds reviewer prompts up-front, so prompt-render time isn't the right reference point. - Update stale comment in test_pipeline_prompts.py that pointed at test_signals.py::test_propose_validates_planner_role_alignment (no such file/test) — the validator-runs-here tests live in this same file under TestPlannerRoleAlignmentValidation. - Thread already-loaded pipeline_state and worktree_path from handle_consensus_propose_signal into _validate_planner_role_alignment via keyword args (with backward-compat fallback to in-function loads) so the validator's dependency on the prior _verify_commit_on_branch block is explicit and the state-store + worktree lookups aren't duplicated. --- orchestrator/routes/pipelines.py | 8 +-- orchestrator/routes/signals.py | 59 ++++++++++++++----- orchestrator/tests/test_pipeline_prompts.py | 4 +- shared/egg_contracts/plan_parser.py | 10 +++- .../test_validate_task_role_alignment.py | 44 ++++++++++++++ 5 files changed, 102 insertions(+), 23 deletions(-) diff --git a/orchestrator/routes/pipelines.py b/orchestrator/routes/pipelines.py index 05913796d2..299567be18 100644 --- a/orchestrator/routes/pipelines.py +++ b/orchestrator/routes/pipelines.py @@ -4583,10 +4583,10 @@ def _get_plan_review_criteria() -> str: "`CONSENSUS_PROPOSE`: a planner proposal whose task `role:` " "assignments cannot push their `files:` (per " "`shared/egg_restrictions/patterns.py`, the same blocklist the " - "gateway uses) is rejected with HTTP 400 before this prompt is " - "ever rendered. By the time you see a proposal, structural " - "role↔files alignment is therefore already validated — no manual " - "check is required for this dimension.\n" + "gateway uses) is rejected with HTTP 400 before the proposal " + "reaches you. By the time you act on a `CONSENSUS_PROPOSE`, " + "structural role↔files alignment is therefore already validated — " + "no manual check is required for this dimension.\n" "- If you want belt-and-suspenders verification, you can run the " "validator yourself against the proposed plan: " '`python3 -c "from egg_contracts.plan_parser import parse_plan_file, ' diff --git a/orchestrator/routes/signals.py b/orchestrator/routes/signals.py index 9a9d693c32..70693bf39c 100644 --- a/orchestrator/routes/signals.py +++ b/orchestrator/routes/signals.py @@ -918,7 +918,12 @@ def _validate_tester_check_coverage( def _validate_planner_role_alignment( - pipeline_id: str, payload: dict[str, Any], repo_path: Path + pipeline_id: str, + payload: dict[str, Any], + repo_path: Path, + *, + pipeline_state: Any | None = None, + worktree_path: Path | None = None, ) -> None: """Validate task role↔files alignment for a planner proposal (#2527). @@ -941,17 +946,25 @@ def _validate_planner_role_alignment( raises, this function returns silently. The push-time gateway check remains the backstop, and the existing forest-validation pass at plan-ingestion time still runs. + + ``pipeline_state`` and ``worktree_path`` should be passed in by + ``handle_consensus_propose_signal`` so the state-store + worktree + lookups it has already performed for ``_verify_commit_on_branch`` + aren't duplicated. The function falls back to loading them itself + when called directly (e.g. from unit tests), preserving backward + compatibility with patches on ``get_state_store`` / ``resolve_worktree_path``. """ commit_sha = (payload.get("commit_sha") or "").strip() if not commit_sha: return - try: - pipeline = get_state_store(repo_path).load_pipeline(pipeline_id) - except StateStoreError: - return + if pipeline_state is None: + try: + pipeline_state = get_state_store(repo_path).load_pipeline(pipeline_id) + except StateStoreError: + return - if not pipeline.branch: + if not pipeline_state.branch: return # Build the plan draft path. Imported lazily to avoid pulling the @@ -965,14 +978,15 @@ def _validate_planner_role_alignment( return plan_rel = _get_draft_path( "plan", - issue_number=pipeline.issue_number, + issue_number=pipeline_state.issue_number, pipeline_id=pipeline_id, - mode=getattr(pipeline, "mode", None), + mode=getattr(pipeline_state, "mode", None), ) if not plan_rel: return - worktree_path = resolve_worktree_path(pipeline_id, repo_path) + if worktree_path is None: + worktree_path = resolve_worktree_path(pipeline_id, repo_path) # Read plan content as committed at the proposed SHA so a stale # local checkout cannot mask a real misassignment. The preceding @@ -1179,28 +1193,35 @@ def handle_consensus_propose_signal( # Verify commit SHA exists on the expected branch before accepting # the proposal (#1473). Reuses _verify_commit_on_branch() from the # completion handler — graceful degradation on network errors (None). + # + # ``pipeline_state`` and ``worktree_path`` are loaded once here and + # threaded into ``_validate_planner_role_alignment`` below so the + # validator's dependency on this block is explicit and the + # state-store + worktree lookups aren't duplicated. commit_sha = payload.get("commit_sha", "") + pipeline_state = None + worktree_path = None if commit_sha: try: store_mod = get_state_store(repo_path) - pipeline = store_mod.load_pipeline(pipeline_id) - if pipeline.branch: + pipeline_state = store_mod.load_pipeline(pipeline_id) + if pipeline_state.branch: worktree_path = resolve_worktree_path(pipeline_id, repo_path) branch_verified = _verify_commit_on_branch( commit_sha, - pipeline.branch, + pipeline_state.branch, worktree_path, pipeline_id, ) if branch_verified is False: return make_error_response( f"Proposal rejected: commit {commit_sha} not found on " - f"expected branch {pipeline.branch}. Push your work before " - f"proposing consensus.", + f"expected branch {pipeline_state.branch}. Push your " + f"work before proposing consensus.", status_code=409, details={ "commit_sha": commit_sha, - "expected_branch": pipeline.branch, + "expected_branch": pipeline_state.branch, "pipeline_id": pipeline_id, }, ) @@ -1222,7 +1243,13 @@ def handle_consensus_propose_signal( # whose blocklist forbids their files (#2527). Same placement # rule: BEFORE handle_propose so the tracker isn't mutated. elif agent_role == "task_planner": - _validate_planner_role_alignment(pipeline_id, payload, repo_path) + _validate_planner_role_alignment( + pipeline_id, + payload, + repo_path, + pipeline_state=pipeline_state, + worktree_path=worktree_path, + ) # Check if this is a re-proposal changed_artifacts = data.get("changed_artifacts") diff --git a/orchestrator/tests/test_pipeline_prompts.py b/orchestrator/tests/test_pipeline_prompts.py index c110ad75d2..021554e01b 100644 --- a/orchestrator/tests/test_pipeline_prompts.py +++ b/orchestrator/tests/test_pipeline_prompts.py @@ -4544,8 +4544,8 @@ def test_revert_of_coordination_detected(self): # CONSENSUS_PROPOSE in routes/signals.py:_validate_planner_role_alignment, # rejecting the planner's proposal with HTTP 400 before the tracker # state is mutated. The reviewer prompt no longer carries a per-prompt -# section; the validator-runs-here tests live in test_signals.py -# (test_propose_validates_planner_role_alignment). +# section; the validator-runs-here tests live in this same file under +# class TestPlannerRoleAlignmentValidation (above). class TestPlanReviewCriteriaReflectsOrchestratorSideValidation: diff --git a/shared/egg_contracts/plan_parser.py b/shared/egg_contracts/plan_parser.py index 57d65a17fc..0f38e8913b 100644 --- a/shared/egg_contracts/plan_parser.py +++ b/shared/egg_contracts/plan_parser.py @@ -64,7 +64,6 @@ import yaml from egg_restrictions.matchers import match_pattern -from egg_restrictions.patterns import AGENT_PATTERNS from .agent_roles import EXECUTION_ROLE_VALUES from .models import Slice, SliceStatus, Task, TaskStatus @@ -1289,6 +1288,15 @@ def _is_file_blocked_for_role(role: str, file_path: str) -> bool: gateway's check intentionally consults only blocked + block-exempt patterns (not allowed_patterns), and so does this function. """ + # AGENT_PATTERNS is imported lazily here to avoid a circular import: + # egg_restrictions.patterns imports egg_contracts.agent_roles, which + # triggers egg_contracts/__init__.py, which imports this module. A + # module-scope import would deadlock that cycle and break the gateway + # production boot path. egg_restrictions.matchers.match_pattern is + # deliberately split out of patterns.py for safe module-scope use + # (see matchers.py docstring); only AGENT_PATTERNS needs to be lazy. + from egg_restrictions.patterns import AGENT_PATTERNS + pattern = AGENT_PATTERNS.get(role) if pattern is None: return False diff --git a/shared/egg_contracts/tests/test_validate_task_role_alignment.py b/shared/egg_contracts/tests/test_validate_task_role_alignment.py index 4e404ce049..ecce1d5c22 100644 --- a/shared/egg_contracts/tests/test_validate_task_role_alignment.py +++ b/shared/egg_contracts/tests/test_validate_task_role_alignment.py @@ -237,3 +237,47 @@ def test_errors_walk_every_slice(self) -> None: assert len(errors) == 2 assert any("slice-1" in e and "task-1-1" in e for e in errors) assert any("slice-2" in e and "task-2-1" in e for e in errors) + + +class TestImportOrderingRegression: + """Guard against the egg_restrictions ↔ egg_contracts import cycle. + + A clean-interpreter ``import egg_restrictions.patterns`` must succeed + even when nothing has pre-loaded ``egg_contracts``. Hoisting + ``AGENT_PATTERNS`` to module scope in ``plan_parser.py`` triggers an + ``ImportError: cannot import name 'AGENT_PATTERNS' from partially + initialized module`` because patterns.py imports + ``egg_contracts.agent_roles``, which runs ``egg_contracts/__init__.py``, + which loads ``plan_parser`` mid-cycle. The gateway production boot + path runs ``python3 gateway.py`` (no pytest pre-loader), so this test + runs the import in a subprocess to mirror that condition. See + ``shared/egg_restrictions/matchers.py`` docstring for context. + """ + + def test_egg_restrictions_patterns_imports_cleanly(self) -> None: + import os + import subprocess + import sys + from pathlib import Path + + # Resolve the repo's ``shared/`` directory from this file's location: + # tests/ → egg_contracts/ → shared/. The gateway boots with + # ``PYTHONPATH=shared`` (see scripts/start-gateway.sh), so this + # mirrors that surface — no orchestrator/, no gateway/. + shared_dir = Path(__file__).resolve().parents[2] + env = {**os.environ, "PYTHONPATH": str(shared_dir)} + + # Fresh interpreter — no pytest pre-loader has run, so the cycle + # surfaces if AGENT_PATTERNS is hoisted to module scope in + # plan_parser.py. + result = subprocess.run( + [sys.executable, "-c", "import egg_restrictions.patterns"], + capture_output=True, + text=True, + check=False, + env=env, + ) + assert result.returncode == 0, ( + f"import egg_restrictions.patterns failed in clean interpreter:\n" + f"stderr:\n{result.stderr}" + ) From 38010075abcfcdd90e8f707c6601a027ac31113a Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 19:15:37 +0000 Subject: [PATCH 4/4] Fix stale gateway boot path comment in import-ordering regression test The PYTHONPATH=shared mirror comment cited scripts/start-gateway.sh which doesn't exist. Replace with the actual production references: gateway/Dockerfile:99 (PYTHONPATH=/app), gateway/entrypoint.sh:286 (exec python3 gateway.py), gateway/Dockerfile:70-75 (shared/ COPY). --- .../tests/test_validate_task_role_alignment.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shared/egg_contracts/tests/test_validate_task_role_alignment.py b/shared/egg_contracts/tests/test_validate_task_role_alignment.py index ecce1d5c22..47cf47e08c 100644 --- a/shared/egg_contracts/tests/test_validate_task_role_alignment.py +++ b/shared/egg_contracts/tests/test_validate_task_role_alignment.py @@ -261,9 +261,12 @@ def test_egg_restrictions_patterns_imports_cleanly(self) -> None: from pathlib import Path # Resolve the repo's ``shared/`` directory from this file's location: - # tests/ → egg_contracts/ → shared/. The gateway boots with - # ``PYTHONPATH=shared`` (see scripts/start-gateway.sh), so this - # mirrors that surface — no orchestrator/, no gateway/. + # tests/ → egg_contracts/ → shared/. The gateway runs + # ``python3 gateway.py`` with ``PYTHONPATH=/app`` (see + # gateway/Dockerfile:99 and gateway/entrypoint.sh:286) where + # shared/ modules are copied to /app/ (see + # gateway/Dockerfile:70-75). ``PYTHONPATH=shared`` mirrors that + # import surface — no orchestrator/, no gateway/. shared_dir = Path(__file__).resolve().parents[2] env = {**os.environ, "PYTHONPATH": str(shared_dir)}