Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions cron/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2751,8 +2751,17 @@ def run_one_job(
# (#123401). Without this the outage is silent — no cron_incidents
# row, no ping — while executions.db keeps piling up failed rows.
if not post_handoff:
delivery_error, delivery_outcome = _deliver_crash_failure(
job, error, adapters=adapters, loop=loop)
# The notice reads the home channel and bot credentials through
# get_secret, which fails closed under multiplex with no scope.
# _run_one_job_body installs the firing profile's scope, and
# record_unknown_worker_outcome does the same for the post-handoff
# notice; this branch returns before either runs.
scope_tokens = _install_fire_secret_scope()
try:
delivery_error, delivery_outcome = _deliver_crash_failure(
job, error, adapters=adapters, loop=loop)
finally:
_reset_fire_secret_scope(scope_tokens)
mark_job_run(
job["id"],
False,
Expand Down
47 changes: 47 additions & 0 deletions tests/cron/test_restart_safe_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -911,6 +911,53 @@ def _handoff_boom(_job):
"suppressed_acked"


def test_dispatch_failure_notice_resolves_the_owning_profiles_home_channel(
execution_ledger, monkeypatch, tmp_path
):
"""Under multiplex the failure notice for a failed handoff resolves the job's home channel
through ``get_secret``. The in-process run path installs the owning profile's secret scope
before delivery; the dispatch-failure branch must too, or the read fails closed with
UnscopedSecretError and the notice never leaves."""
import cron.incidents as incidents
import cron.scheduler as scheduler
import cron.scheduler_delivery as delivery
from agent import secret_scope

home = tmp_path / "profiles" / "worker"
home.mkdir(parents=True)
(home / ".env").write_text('TELEGRAM_HOME_CHANNEL="111111111"\n', encoding="utf-8")
monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: home)
monkeypatch.delenv("TELEGRAM_HOME_CHANNEL", raising=False)

def _handoff_boom(_job):
raise RuntimeError("worker exited before ownership acknowledgement")

monkeypatch.setattr(scheduler, "_launch_external_cron_worker", _handoff_boom)
monkeypatch.setattr(scheduler, "mark_job_run", lambda *_a, **_k: True)
resolved = []

def _deliver(job, content, **_kw):
resolved.append(delivery._env_home_target_chat_id("telegram"))
return None

monkeypatch.setattr(scheduler, "_deliver_result", _deliver)

record = execution_ledger.create_execution("job-scoped", source="builtin")
job = {"id": "job-scoped", "execution_id": record["id"], "deliver": "telegram"}
secret_scope.set_multiplex_active(True)
try:
assert secret_scope.current_secret_scope() is None
assert scheduler.run_one_job(job, adapters=None) is True
assert secret_scope.current_secret_scope() is None # scope does not leak past the fire
finally:
secret_scope.set_multiplex_active(False)

assert resolved == ["111111111"]
assert len(incidents.list_incidents()) == 1
finished = execution_ledger.get_execution(record["id"])
assert finished["delivery_outcome"] == "delivered"


def test_shutdown_does_not_interrupt_restart_safe_waiter():
import cron.scheduler as scheduler

Expand Down