Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions orchestrator/event_loop/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -372,10 +372,13 @@ class EventDecision:
``timing`` is a structured mapping for the slice-4 latency budget on a
fresh spawn, ``None`` otherwise. ``blocked`` (#3496) names why a
spawn-action decision did not spawn when the block is operator-relevant:
``"exhausted"`` (terminal retry-budget exhaustion) or ``"parked"``
(#3548 — a no-op-park that only the retry heartbeat will release), so
the all-arms wedge detections can distinguish them from the benign
not-spawned shapes (dedupe, backoff).
``"exhausted"`` (terminal retry-budget exhaustion), ``"parked"``
(#3548 — a no-op-park that only the retry heartbeat will release), or
``"stopped"`` (#3633 — the loop was stopped mid-tick, typically by a
cancel), so the all-arms wedge detections can distinguish them from the
benign not-spawned shapes (dedupe, backoff). Only ``"exhausted"`` and
``"parked"`` count toward a wedge; ``"stopped"`` is a teardown, not a
condition an operator can resolve.
"""

role: str
Expand Down
16 changes: 16 additions & 0 deletions orchestrator/event_loop/_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -616,6 +616,22 @@ def _handle_role(self, role: str) -> EventDecision:
# wait / unknown — nothing to spawn.
return EventDecision(role=role, action=action)

# #3633: never spawn once the loop has been stopped. ``run()`` checks the
# stop event between ticks, but ``stop()`` is also called from outside the
# loop's own thread — the cancel route stops every live loop for a
# pipeline synchronously — so a stop that lands mid-tick would otherwise
# still get one final cohort of one-shot Jobs out the door. Re-checking
# immediately before the spawn decision closes that window.
if self._stop.is_set():
logger.info(
"event-loop: spawn blocked, loop is stopping",
pipeline_id=self.pipeline_id,
slice_id=self.slice_id,
role=role,
action=action,
)
return EventDecision(role=role, action=action, spawned=False, blocked="stopped")

identity = _pkg.event_identity(action, payload)
key = _pkg.compute_dedupe_key(
self.pipeline_id, self.slice_id, self.phase, role, action, identity
Expand Down
4 changes: 3 additions & 1 deletion orchestrator/routes/pipelines/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,7 @@ def stream_pipeline(pipeline_id: str) -> Response:
_compute_gateway_mode,
_mark_pipeline_records_terminated,
_normalize_submission_repos,
_stop_pipeline_event_loops,
_sync_contract_phase_to_pipeline,
)
from ._overseer import ( # noqa: E402,F401
Expand Down Expand Up @@ -1382,11 +1383,11 @@ def stream_pipeline(pipeline_id: str) -> Response:
)
from ._run_concurrent_support import ( # noqa: E402,F401
_latest_proposal_ts_impl,
_phase_bail_reason_impl,
_record_container_exit_impl,
_record_spawned_agents_impl,
_retry_transient_spawn_failures_impl,
_stop_running_containers_impl,
_superseded_by_restart_impl,
_update_agents_complete_impl,
)
from ._run_hitl_gate import ( # noqa: E402,F401
Expand Down Expand Up @@ -1432,6 +1433,7 @@ def stream_pipeline(pipeline_id: str) -> Response:
from ._run_support import ( # noqa: E402,F401
_clear_stale_impasses_for_producers,
_parse_resolution,
_pipeline_cancelled,
_pipeline_superseded_by_restart,
_spawn_and_wait,
)
Expand Down
60 changes: 60 additions & 0 deletions orchestrator/routes/pipelines/_lifecycle_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,66 @@ def _assert_repo_set_uniform(repos: list[str]) -> str | None:
return None


def _stop_pipeline_event_loops(pipeline_id: str, *, reason: str) -> int:
"""Stop every live orchestrator-owned BRC event loop for ``pipeline_id`` (#3633).

``cancel_task`` used to set the status to CANCELLED, tear down the
pipeline's containers, and clear its runtime state — but nothing stopped
the thing that *creates* containers. Each slice's ``OrchestratorEventLoop``
kept polling on its daemon thread, so the next tick re-derived every
role's arm and requested fresh one-shot Jobs. Killing the pods removed
the symptom while the spawner ran on: ``issue-3596-v2`` was cancelled at
20:48Z and spawned slice-3 agents at 22:55Z, against a pipeline the
operator believed was stopped.

The loops are reachable through the ``event_loop`` live-loop registry
(#3496), which is keyed by ``(pipeline_id, slice_id)`` and populated by
``start()`` — so this covers every concurrent slice of the run.

Call this BEFORE container cleanup: cleanup that races a live loop is
removing pods the loop is still entitled to replace.

``stop()`` is called with ``join_timeout=0.0`` because this runs in the
PATCH request thread. Both effects that matter — setting the loop's stop
event and evicting it from the registry — are synchronous, so the daemon
thread winds down on its own without making the operator wait on it.

Returns the number of loops signalled (0 when none were live, which is
the normal case for a pipeline cancelled between phases).
"""
try:
try:
from event_loop import get_live_event_loops
except ImportError:
from ..event_loop import ( # type: ignore[no-redef]
get_live_event_loops,
)
except ImportError:
return 0

stopped = 0
for loop in get_live_event_loops(pipeline_id):
try:
loop.stop(join_timeout=0.0)
stopped += 1
except Exception as e: # noqa: BLE001 — best-effort teardown
_pkg.logger.warning(
"Failed to stop BRC event loop",
pipeline_id=pipeline_id,
slice_id=getattr(loop, "slice_id", None),
reason=reason,
error=str(e),
)
if stopped:
_pkg.logger.info(
"Stopped live BRC event loops",
pipeline_id=pipeline_id,
reason=reason,
loops_stopped=stopped,
)
return stopped


def _clear_pipeline_runtime_state(pipeline_id: str, *, reason: str) -> None:
"""Evict per-pipeline runtime state that is keyed by pipeline_id alone.

Expand Down
21 changes: 21 additions & 0 deletions orchestrator/routes/pipelines/_routes_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,27 @@ def _update_pipeline_body(pipeline_id: str) -> tuple[_pkg.Response, int]:
):
_pkg._emit_pipeline_event(pipeline, "pipeline.cancelled")

# Stop the machinery that spawns agents BEFORE tearing down the
# agents it spawned (#3633). Cancel used to do only the teardown, so
# the live BRC event loops kept deriving arms and requesting one-shot
# Jobs against a pipeline the operator believed was stopped — and the
# container cleanup below raced loops that were still entitled to
# spawn replacements. Ordering it first closes that race.
#
# Gated on the CANCELLED *transition*: an idempotent re-cancel has
# nothing live left to stop, and FAILED is excluded deliberately —
# ``container_monitor`` reconciliation can mark a live pipeline FAILED
# mid-phase, and the poll loop recovers it to RUNNING once consensus
# completes (#1273). Tearing the loop down there would convert that
# recoverable transient into a permanently idle pipeline. The driver
# loops' own CANCELLED re-reads are the backstop for a loop that
# started after this ran.
if (
pipeline.status == _pkg.PipelineStatus.CANCELLED
and prev_status != _pkg.PipelineStatus.CANCELLED
):
_pkg._stop_pipeline_event_loops(pipeline_id, reason="pipeline_cancelled")

# If pipeline is being cancelled or failed, clean up containers
# and cancel any pending decisions so wait_for_decision() unblocks.
if pipeline.status in (_pkg.PipelineStatus.CANCELLED, _pkg.PipelineStatus.FAILED):
Expand Down
54 changes: 25 additions & 29 deletions orchestrator/routes/pipelines/_run_concurrent.py
Original file line number Diff line number Diff line change
Expand Up @@ -397,21 +397,19 @@ def _live_pipeline_phase() -> str:
start_time = _pkg.time.monotonic()
objection_decision_created = False

# ``run_epoch`` is the authoritative epoch the owning ``_run_pipeline``
# thread captured at start (#1638). The poll loop uses it to detect a
# ``restart_phase`` (or any restart that bumps ``run_epoch``) that
# superseded this thread (#3315). ``start_time`` is a fresh monotonic
# clock per call, but a parked-then-restarted phase leaves the *old*
# ``_run_concurrent_phase`` thread alive in its poll loop with a
# ``start_time`` from the original phase start; once its ``elapsed``
# crosses ``consensus_timeout`` it would fire a spurious consensus-timeout
# OVERSEER_ALERT + HITL decision against the freshly-restarted phase. The
# new ``_run_pipeline`` thread owns the pipeline now, so this stale thread
# must bail before escalating. When ``run_epoch`` is not supplied (legacy
# / direct-call callers) the guard is dormant — behaviour is unchanged.

_superseded_by_restart = _pkg.functools.partial(
_pkg._superseded_by_restart_impl,
# Per-tick "should this thread still be here?" check. Two conditions,
# one pipeline load — see ``_phase_bail_reason_impl`` for the full
# rationale of each and for why FAILED is excluded:
#
# * a restart bumped ``run_epoch``, so a new ``_run_pipeline`` thread
# owns the pipeline and this stale one must stop before it fires a
# spurious consensus-timeout escalation against the fresh phase
# (#3315; dormant when ``run_epoch`` is not supplied);
# * the operator cancelled the run, so this thread must stop driving
# the slice DAG rather than keep spawning into it (#3633).

_phase_bail_reason = _pkg.functools.partial(
_pkg._phase_bail_reason_impl,
store=store,
pipeline_id=pipeline_id,
run_epoch=run_epoch,
Expand Down Expand Up @@ -465,26 +463,24 @@ def _live_pipeline_phase() -> str:
driver_heartbeat.record_tick(pipeline_id)
elapsed = _pkg.time.monotonic() - start_time

# 0. Bail if a restart superseded this thread (#3315). A parked phase
# that is restarted after the consensus-timeout budget elapsed
# leaves this old thread polling with a stale ``start_time``; the
# new ``_run_pipeline`` thread already owns the pipeline. Exit
# cleanly — stop this executor's event loop so it stops requesting
# one-shot spawns — WITHOUT firing the timeout escalation. Return a
# NON-zero exit so the caller never mistakes this for success and
# advances the phase; the post-return epoch check (#1638) at the
# call site re-confirms the restart and exits the old thread without
# marking the phase FAILED.
if _superseded_by_restart():
# 0. Bail if this thread no longer owns the phase (restart, #3315) or
# the run was cancelled (#3633). Exit cleanly — stop this executor's
# event loop so it stops requesting one-shot spawns — WITHOUT firing
# the timeout escalation. Return a NON-zero exit so the caller never
# mistakes this for success and advances the phase; the post-return
# checks at the call site re-confirm the reason and exit the thread
# without marking the phase FAILED.
_bail_reason = _phase_bail_reason()
if _bail_reason is not None:
_pkg.logger.info(
"Phase superseded by restart (run_epoch changed) — exiting stale "
"_run_concurrent_phase thread without escalation",
"Exiting _run_concurrent_phase poll loop without escalation",
pipeline_id=pipeline_id,
phase=phase,
slice_id=slice_id,
reason=_bail_reason,
)
executor.stop_event_loop()
return 1, "Phase superseded by restart; stale monitor thread exited."
return 1, f"Phase monitor thread exited: {_bail_reason}."

# 1. Check consensus
try:
Expand Down
24 changes: 15 additions & 9 deletions orchestrator/routes/pipelines/_run_concurrent_retry.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,18 +132,24 @@ def _run_concurrent_phase_with_impasse_retry(
# Defense-in-depth (#3315 facet a, slice path): if a restart bumped
# ``run_epoch`` while this thread was running, a stale producer-written
# impasse file could otherwise drive ``route_impasses`` into a HITL
# against the freshly-restarted phase. The poll loop in
# ``_run_concurrent_phase`` already bails on supersession before any
# escalation; mirror that here so the "no escalation when superseded"
# property holds on the slice path too — return the (superseded) result
# without routing.
if _pkg._pipeline_superseded_by_restart(store, pipeline_id, run_epoch):
# against the freshly-restarted phase. The same applies to a cancel
# (#3633): the phase call above returns immediately once the pipeline
# is CANCELLED, and without this guard a stale impasse would escalate
# a HITL — and, on the all-delegated branch, drive another retry
# iteration — against a run the operator already stopped. The poll loop
# in ``_run_concurrent_phase`` bails on both conditions before any
# escalation; mirror that here so the "no escalation once this thread
# no longer owns the phase" property holds on the slice path too.
_bail_reason = _pkg._phase_bail_reason_impl(
store=store, pipeline_id=pipeline_id, run_epoch=run_epoch
)
if _bail_reason is not None:
_pkg.logger.info(
"Restart superseded this thread before impasse routing; "
"skipping route_impasses to avoid escalating against a "
"freshly-restarted phase",
"Thread no longer owns this phase before impasse routing; "
"skipping route_impasses to avoid escalating against it",
pipeline_id=pipeline_id,
slice_id=slice_id,
reason=_bail_reason,
)
return last_exit, last_logs

Expand Down
52 changes: 42 additions & 10 deletions orchestrator/routes/pipelines/_run_concurrent_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,49 @@
import routes.pipelines as _pkg # noqa: E402,F401


def _superseded_by_restart_impl(*, store, pipeline_id, run_epoch) -> bool:
"""True if a newer run_epoch means another thread owns this pipeline.

Reloads pipeline state and compares its ``run_epoch`` against the
epoch this thread runs under. Mirrors the post-return epoch check
(#1638) but runs *inside* the poll loop so a superseded thread stops
polling before it can fire stale escalations. Best-effort: a load
failure returns ``False`` so a transient store hiccup never tears
down a legitimately-running phase.
def _phase_bail_reason_impl(*, store, pipeline_id, run_epoch) -> str | None:
"""Why this poll loop must exit without escalating, or ``None`` to keep polling.

Two independent conditions, resolved from a single pipeline load:

``superseded_by_restart``
A newer ``run_epoch`` means another ``_run_pipeline`` thread owns
this pipeline (#3315). Mirrors the post-return epoch check (#1638)
but runs *inside* the poll loop so a superseded thread stops
polling before it can fire stale escalations.

``pipeline_cancelled``
The operator cancelled the run (#3633). The cancel route stops this
phase's event loop directly, but nothing stops *this* thread, and
before this check it kept polling — admitting the next slice,
creating its integration branch, and spawning agents against a
pipeline the operator believes is stopped. Re-reading the persisted
status bounds a missed stop signal to one poll interval.

FAILED is deliberately NOT a bail condition: ``container_monitor``
reconciliation can mark a live pipeline FAILED while this loop polls,
and the consensus-complete branch recovers it to RUNNING (#1273).
Bailing here would turn that recoverable transient into a real failure.

Best-effort: a missing store or a load failure returns ``None`` so a
transient store hiccup never tears down a legitimately-running phase.
"""
return _pkg._pipeline_superseded_by_restart(store, pipeline_id, run_epoch)
if store is None:
return None
try:
_pip = store.load_pipeline(pipeline_id)
except Exception as _err: # noqa: BLE001 — never wedge the caller
_pkg.logger.debug(
"Phase bail check failed; continuing",
pipeline_id=pipeline_id,
error=str(_err),
)
return None
if _pip.status == _pkg.PipelineStatus.CANCELLED:
return "pipeline_cancelled"
if run_epoch is not None and (_pip.run_epoch or _pip.created_at) != run_epoch:
return "superseded_by_restart"
return None


def _record_container_exit_impl(
Expand Down
32 changes: 30 additions & 2 deletions orchestrator/routes/pipelines/_run_implement.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,12 @@ def _run_implement_phase_slices(
slice reached CONFIRMED; non-zero means at least one slice failed.
"""
try:
from orchestrator.slice_scheduler import SliceScheduler
from orchestrator.slice_scheduler import SchedulerSliceState, SliceScheduler
except ImportError:
from slice_scheduler import SliceScheduler
from slice_scheduler import ( # type: ignore[no-redef]
SchedulerSliceState,
SliceScheduler,
)

try:
from egg_contracts.loader import load_contract, save_contract
Expand Down Expand Up @@ -528,6 +531,31 @@ def _bootstrap_check_one(slice_obj: _pkg.Any) -> tuple[str, bool]:
try:
while not scheduler.all_done():
driver_heartbeat.record_tick(pipeline_id) # #3540 liveness tick

# 0. Stop walking the DAG once the operator cancels (#3633).
# ``cancel_task`` tears down the pipeline's containers and
# stops its live BRC event loops, but nothing used to stop
# THIS loop: the next tick admitted the next ready slice,
# created its integration branch, and spawned a fresh agent
# cohort — observed opening slice-3 two hours after a cancel.
# Re-read the persisted status here so an in-flight wave is
# the last one, and bail before admitting another.
# ``_run_phase_execution`` maps the non-zero exit onto a clean
# thread return rather than a phase FAILURE.
if _pkg._pipeline_cancelled(store, pipeline_id):
_pkg.logger.info(
"Pipeline cancelled — stopping the slice loop without admitting further slices",
pipeline_id=pipeline_id,
unfinished=[
rt.slice_id
for rt in scheduler.list_slices()
if rt.state != SchedulerSliceState.COMPLETE
],
)
aggregate_logs.append("--- slice loop stopped: pipeline cancelled ---")
overall_exit = 1
break

# 1. Snapshot ready slices for this tick.
ready_batch = list(scheduler.iter_ready())
if not ready_batch:
Expand Down
Loading
Loading