Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 110 additions & 3 deletions orchestrator/routes/pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -15647,7 +15647,83 @@ def _contract_loader() -> Any:
except ImportError:
from state_store import get_pipeline_state_lock # type: ignore[no-redef]

def _persist_slice_status_complete(slice_id: str) -> None:
def _commit_and_push_slice_statefiles(message: str) -> None:
"""Commit + push pipeline-scoped ``.egg-state/`` writes to the work branch.

Contract mutations — agent task-record updates via
``mutate_contract`` and the ``slice.status`` flips below — land
on the shared pipeline worktree's disk copy only. Without a
slice-boundary commit, the work branch's contract file stays
frozen at the init-time "Initialize SDLC contract" commit for
the entire implement phase, and a mid-phase orchestrator crash
or worktree prune loses every accumulated task record (#3117).
The phase-boundary commit at the end of the run loop is too
coarse for multi-slice phases.

Scope (per #3117): this closes durability for the post-prune
audit record, operator/PR-side review of mid-phase contract
state, and orchestrator-restart resume at slice granularity.
It is deliberately NOT the read path for live agents — agents
read the contract via ``mcp__sdlc__show_contract`` against the
orchestrator's in-memory state, never from their checkout's
``.egg-state/contracts/`` file (#3077).

Best-effort: slice completion must not block on statefile
durability; failures are logged and the next boundary (later
slice close or phase completion) carries the writes. The commit
runs under the per-pipeline state lock to serialise concurrent
slice-close threads against the shared worktree's git index;
the push runs outside the lock. The expected case is a linear
fast-forward (lock-serialised commits stack), and a no-op FF
of the same SHA from two threads is harmless. The residual
hazard is ``_reconcile_and_retry_push`` on a non-FF rejection
(``gateway_client.py:1361``): two threads both fetching+rebasing
in the shared worktree can interleave ``.git/index.lock``.
Within the implement phase no other writer pushes to
``pipeline.branch`` so non-FF shouldn't fire in normal
operation; an external push (operator hand-fix, stale
concurrent orchestrator) is the only known trigger.
"""
try:
with get_pipeline_state_lock(pipeline_id):
committed = _commit_statefiles_to_worktree(
worktree_repo_path,
message,
_pipeline_identifier(issue_number, pipeline_id),
pipeline_id=pipeline_id,
)
except Exception as commit_err: # noqa: BLE001
# The helper raises CalledProcessError / TimeoutExpired
# from subprocess.run and OSError from glob (#2219 family).
logger.warning(
"Failed to commit slice statefiles to work branch (continuing) (#3117)",
pipeline_id=pipeline_id,
commit_message=message,
error=str(commit_err),
)
return
if not committed or not pipeline.branch or worktree_repo_path == store.repo_path:
return
try:
spawner.gateway.push_worktree_branch(
pipeline_id=pipeline_id,
repo_path=str(worktree_repo_path),
branch=pipeline.branch,
mode=gateway_mode, # type: ignore[arg-type]
base_branch=pipeline.base_branch,
)
except Exception as push_err: # noqa: BLE001
# Gateway HTTP push (GatewayError / OSError). The commit is
# already on the local work branch; the next successful
# push carries it.
logger.warning(
"Failed to push slice statefiles to work branch (continuing) (#3117)",
pipeline_id=pipeline_id,
commit_message=message,
error=str(push_err),
)

def _persist_slice_status_complete(slice_id: str, *, commit_to_branch: bool = True) -> None:
"""Mark ``slice_id`` as ``SliceStatus.COMPLETE`` on the contract.

Durable signal so the bootstrap reconciliation pass below and
Expand All @@ -15656,6 +15732,23 @@ def _persist_slice_status_complete(slice_id: str) -> None:
on save failure the in-memory scheduler state still reflects
completion for this pass and the next ``start_pipeline``
re-detects via the merged-detection helper.

With *commit_to_branch* (the default), the saved contract —
along with any other uncommitted pipeline statefiles, e.g.
agent task-record mutations made during the slice — is
committed and pushed to the pipeline work branch so the durable
copy tracks the live one (#3117). The bootstrap reconciliation
passes set it to ``False`` and batch a single commit after the
loop instead of one per reconciled slice.

Called only after a slice successfully closes (BRC consensus
reached + PR opened, or merged-skip / bootstrap-COMPLETE
reconciliation). Failed slices — ``exit_code_inner != 0``
(#16410) or ``pr_created == False`` (#16588) — return early
without calling this helper, so their accumulated task-record
mutations remain uncommitted in the worktree until the next
successful slice's commit (the pipeline-scoped glob picks them
up) or the phase-boundary commit, whichever fires first.
"""
try:
with get_pipeline_state_lock(pipeline_id):
Expand All @@ -15677,6 +15770,11 @@ def _persist_slice_status_complete(slice_id: str) -> None:
slice_id=slice_id,
error=str(save_err),
)
return
if commit_to_branch:
_commit_and_push_slice_statefiles(
f"Persist contract after slice {slice_id} completion (#3117)"
)

# Bootstrap reconciliation pass (#2549). Before the run loop ticks,
# fold in two sources of "this slice is already done" state that
Expand Down Expand Up @@ -15769,7 +15867,7 @@ def _bootstrap_check_one(slice_obj: Any) -> tuple[str, bool]:
for slice_id, already_merged in results:
if already_merged:
scheduler.record_complete(slice_id)
_persist_slice_status_complete(slice_id)
_persist_slice_status_complete(slice_id, commit_to_branch=False)
bootstrap_merged.append(slice_id)

# Layer (C): non-COMPLETE slice classification (slice-4 TASK-4-4).
Expand Down Expand Up @@ -15833,7 +15931,7 @@ def _bootstrap_check_one(slice_obj: Any) -> tuple[str, bool]:
slice_id=s.id,
)
scheduler.record_complete(s.id)
_persist_slice_status_complete(s.id)
_persist_slice_status_complete(s.id, commit_to_branch=False)
bootstrap_consensus_complete.append(s.id)
continue
if classification == "resume":
Expand Down Expand Up @@ -15861,6 +15959,15 @@ def _bootstrap_check_one(slice_obj: Any) -> tuple[str, bool]:
continue
# "fresh" → no Layer-C action, scheduler re-yields READY.

# The bootstrap passes above persist with ``commit_to_branch=False``
# — one batched commit+push here covers every reconciled slice
# (Layer B merged-detection + Layer-C case 3) instead of a commit
# per slice (#3117).
if bootstrap_merged or bootstrap_consensus_complete:
_commit_and_push_slice_statefiles(
"Persist slice completion statuses after bootstrap reconciliation (#3117)"
)

if bootstrap_complete or bootstrap_merged:
logger.info(
"Slice bootstrap reconciliation marked slices complete",
Expand Down
234 changes: 234 additions & 0 deletions orchestrator/tests/test_slice_run_loop_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -1292,6 +1292,240 @@ def test_bootstrap_detection_failure_falls_through(self) -> None:
spawner.gateway.create_slice_pr.assert_called_once()


class TestSliceBoundaryStatefileCommit:
"""#3117 — slice completion must commit+push statefiles to the work branch.

Contract task-record mutations and ``slice.status`` flips land on
the shared pipeline worktree's disk copy only; without a
slice-boundary commit the work branch's contract file stays frozen
at the init-time "Initialize SDLC contract" commit for the whole
implement phase (live repro: ``pipeline-2d9cc50d``, every task row
pending forever on the branch copy).
"""

WORK_BRANCH_KW = "branch"

def _make_spawner(self) -> MagicMock:
spawner = MagicMock()
spawner.gateway = MagicMock()
spawner.gateway.create_slice_pr.return_value = "https://example/pr/1"
spawner.gateway.is_slice_branch_merged_into_parent.return_value = False
return spawner

def _run(
self,
pipeline: Pipeline,
contract: Contract,
spawner: MagicMock,
commit_mock: MagicMock,
store: MagicMock | None = None,
) -> tuple[int, str]:
with (
patch("egg_contracts.loader.load_contract", return_value=contract),
patch("egg_contracts.loader.save_contract"),
patch("routes.pipelines._start_stacked_pr_reconciler") as mock_start_recon,
patch("routes.pipelines._run_concurrent_phase", return_value=(0, "ok")),
patch("orchestrator.peer_consensus.remove_peer_consensus_tracker"),
patch("routes.pipelines._commit_statefiles_to_worktree", commit_mock),
):
mock_start_recon.return_value = (MagicMock(), threading.Event())
return _run_implement_phase_slices(
pipeline_id=pipeline.id,
pipeline=pipeline,
spawner=spawner,
repo_volumes={},
gateway_mode="public",
repos=["owner/repo"],
sandbox_env={},
store=store if store is not None else MagicMock(),
certs_volume=None,
worktree_repo_path=Path("/tmp/x"),
)

@staticmethod
def _slice_commit_calls(commit_mock: MagicMock) -> list[Any]:
"""Slice-boundary commit calls (positional arg 1 is the message)."""
return [
c for c in commit_mock.call_args_list if "Persist contract after slice" in c.args[1]
]

@staticmethod
def _bootstrap_commit_calls(commit_mock: MagicMock) -> list[Any]:
return [c for c in commit_mock.call_args_list if "bootstrap reconciliation" in c.args[1]]

def _work_branch_pushes(self, spawner: MagicMock, pipeline: Pipeline) -> list[Any]:
"""Pushes targeting the work branch (the per-slice BRC hook
pushes integration branches through the same gateway method)."""
return [
c
for c in spawner.gateway.push_worktree_branch.call_args_list
if c.kwargs.get(self.WORK_BRANCH_KW) == pipeline.branch
]

def test_slice_close_commits_and_pushes_contract(self) -> None:
"""Successful slice close → contract committed to the work
branch (scoped to this pipeline) and pushed via launcher-auth."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
contract = _make_contract(slices=[slice1])
commit_mock = MagicMock(return_value=True)
spawner = self._make_spawner()

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
slice_commits = self._slice_commit_calls(commit_mock)
assert len(slice_commits) == 1, (
"slice close must commit statefiles to the work branch exactly once"
)
call = slice_commits[0]
assert call.args[0] == Path("/tmp/x")
assert "slice-1" in call.args[1]
assert call.kwargs["pipeline_id"] == pipeline.id, (
"commit must be scoped with pipeline_id so the contract file "
"(keyed by pipeline id, not issue number) matches the glob (#1829)"
)
pushes = self._work_branch_pushes(spawner, pipeline)
assert len(pushes) == 1, "committed statefiles must be pushed to origin"
assert pushes[0].kwargs["repo_path"] == "/tmp/x"

def test_no_push_when_commit_is_noop(self) -> None:
"""``_commit_statefiles_to_worktree`` returning False (nothing
staged) must skip the push — no no-op fast-forward round-trip."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
contract = _make_contract(slices=[slice1])
commit_mock = MagicMock(return_value=False)
spawner = self._make_spawner()

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
assert len(self._slice_commit_calls(commit_mock)) == 1
assert self._work_branch_pushes(spawner, pipeline) == []

def test_commit_failure_does_not_block_slice_completion(self) -> None:
"""Statefile durability is best-effort: a git failure in the
commit helper must not fail the slice or skip its COMPLETE
status."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
contract = _make_contract(slices=[slice1])
commit_mock = MagicMock(side_effect=RuntimeError("git transient"))
spawner = self._make_spawner()

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
assert slice1.status == SliceStatus.COMPLETE
assert self._work_branch_pushes(spawner, pipeline) == []

def test_push_failure_does_not_block_slice_completion(self) -> None:
"""A gateway push failure is logged and swallowed — the commit
is already on the local work branch and the next boundary's
push carries it."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
contract = _make_contract(slices=[slice1])
commit_mock = MagicMock(return_value=True)
spawner = self._make_spawner()
spawner.gateway.push_worktree_branch.side_effect = RuntimeError("gateway transient")

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
assert slice1.status == SliceStatus.COMPLETE

def test_no_push_when_worktree_is_main_repo(self) -> None:
"""Mirror of the phase-boundary guard: never push when the
'worktree' is actually the main repo checkout."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
contract = _make_contract(slices=[slice1])
commit_mock = MagicMock(return_value=True)
spawner = self._make_spawner()
store = MagicMock()
# ``StateStore.repo_path`` is a Path in production
# (``get_state_store(repo_path: Path | str)`` accepts both but
# the orchestrator constructs with Path); the slice-loop guard
# compares Paths directly (matching the phase-boundary guard
# style, pipelines.py:22926).
store.repo_path = Path("/tmp/x") # same path passed as worktree_repo_path

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock, store=store)

assert exit_code == 0
assert self._work_branch_pushes(spawner, pipeline) == []

def test_bootstrap_reconciliation_batches_single_commit(self) -> None:
"""Bootstrap merged-detection marking N slices complete must
produce ONE batched commit, not one per slice — and no
per-slice-close commits (nothing ran)."""
pipeline = _make_pipeline()
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
slice2 = _make_slice("slice-2", tasks=[_make_task("task-2-1")])
contract = _make_contract(slices=[slice1, slice2])
commit_mock = MagicMock(return_value=True)
spawner = self._make_spawner()
spawner.gateway.is_slice_branch_merged_into_parent.return_value = True

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
assert self._slice_commit_calls(commit_mock) == []
bootstrap_commits = self._bootstrap_commit_calls(commit_mock)
assert len(bootstrap_commits) == 1, (
"bootstrap passes must batch one commit for all reconciled slices"
)
assert len(self._work_branch_pushes(spawner, pipeline)) == 1

def test_concurrent_wave_no_op_collapse_elides_second_push(self) -> None:
"""Two independent slices in the same ready batch run through
``_run_one_slice`` concurrently and both hit
``_persist_slice_status_complete``. When the first thread's
commit sweeps the pipeline-scoped glob and the second thread's
commit sees no staged changes, ``_commit_statefiles_to_worktree``
returns False and the second push is correctly elided.

Scope: this verifies the no-op-collapse semantics — both slices
reach the helper, only one push hits the gateway. It does NOT
directly verify lock serialisation: ``side_effect=[True, False]``
is consumed in mock-call order, not in slice-completion order,
so the assertions hold under any interleaving rather than
proving the per-pipeline state lock enforces one. Production
correctness for the serialisation property comes from the lock
itself; this test pins the externally observable behavior
(single push per wave when subsequent commits are no-ops) that
single-slice tests can't reach."""
pipeline = _make_pipeline()
# No deps between slices → both READY in the first wave →
# ThreadPoolExecutor dispatches both into ``_run_one_slice``
# concurrently.
slice1 = _make_slice("slice-1", tasks=[_make_task("task-1-1")])
slice2 = _make_slice("slice-2", tasks=[_make_task("task-2-1")])
contract = _make_contract(slices=[slice1, slice2])
# Whichever thread's commit lands first sweeps the contract +
# any task-record mutations; the other thread's commit sees an
# empty staged set and returns False (the no-op collapse).
commit_mock = MagicMock(side_effect=[True, False])
spawner = self._make_spawner()

exit_code, _ = self._run(pipeline, contract, spawner, commit_mock)

assert exit_code == 0
slice_commits = self._slice_commit_calls(commit_mock)
assert len(slice_commits) == 2, (
"both successful slices must call _commit_and_push_slice_statefiles"
)
# The no-op commit (second call → returned False) must NOT
# trigger a push; only the first slice's push hits the gateway.
pushes = self._work_branch_pushes(spawner, pipeline)
assert len(pushes) == 1, (
"second slice's no-op commit must collapse to zero pushes "
"(no FF round-trip of the same SHA)"
)


# ---------------------------------------------------------------------------
# #2914 — restart_phase resume guard: bootstrap must verify pods are alive
# before mark_spawned, or the pipeline wedges with no agents running.
Expand Down
Loading